# Copyright (c) 2016-2017 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
import os
import yaml
import re
import time
import glob
import uuid
import random
import logging
import itertools
import xml.etree.ElementTree as ET
from yardstick import ssh
from yardstick.network_services.utils import get_nsb_option
from yardstick.network_services.utils import provision_tool
from yardstick.benchmark.contexts.standalone import StandaloneContext
log = logging.getLogger(__name__)
VM_TEMPLATE = """
vm1
{random_uuid}
102400
102400
20
hvm
SandyBridge
destroy
restart
restart
/usr/bin/kvm-spice
"""
class Sriov(StandaloneContext):
def __init__(self):
self.name = None
self.file_path = None
self.nodes = []
self.vm_deploy = False
self.sriov = []
self.first_run = True
self.dpdk_nic_bind = ""
self.user = ""
self.ssh_ip = ""
self.passwd = ""
self.ssh_port = ""
self.auth_type = ""
def init(self):
log.debug("In init")
self.parse_pod_and_get_data(self.file_path)
def parse_pod_and_get_data(self, file_path):
self.file_path = file_path
log.debug("parsing pod file: {0}".format(self.file_path))
try:
with open(self.file_path) as stream:
cfg = yaml.load(stream)
except IOError:
log.error("File {0} does not exist".format(self.file_path))
raise
self.sriov.extend([node for node in cfg["nodes"]
if node["role"] == "Sriov"])
self.user = self.sriov[0]['user']
self.ssh_ip = self.sriov[0]['ip']
if self.sriov[0]['auth_type'] == "password":
self.passwd = self.sriov[0]['password']
else:
self.ssh_port = self.sriov[0]['ssh_port']
self.key_filename = self.sriov[0]['key_filename']
def ssh_remote_machine(self):
if self.sriov[0]['auth_type'] == "password":
self.connection = ssh.SSH(
self.user,
self.ssh_ip,
password=self.passwd)
self.connection.wait()
else:
if self.ssh_port is not None:
ssh_port = self.ssh_port
else:
ssh_port = ssh.DEFAULT_PORT
self.connection = ssh.SSH(
self.user,
self.ssh_ip,
port=ssh_port,
key_filename=self.key_filename)
self.connection.wait()
self.dpdk_nic_bind = provision_tool(
self.connection,
os.path.join(get_nsb_option("bin_path"), "dpdk_nic_bind.py"))
def get_nic_details(self):
nic_details = {}
nic_details = {
'interface': {},
'pci': self.sriov[0]['phy_ports'],
'phy_driver': self.sriov[0]['phy_driver'],
'vf_macs': self.sriov[0]['vf_macs']
}
# Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
for i, _ in enumerate(nic_details['pci']):
err, out, _ = self.connection.execute(
"{dpdk_nic_bind} --force -b {driver} {port}".format(
dpdk_nic_bind=self.dpdk_nic_bind,
driver=self.sriov[0]['phy_driver'],
port=self.sriov[0]['phy_ports'][i]))
err, out, _ = self.connection.execute(
"lshw -c network -businfo | grep '{port}'".format(
port=self.sriov[0]['phy_ports'][i]))
a = out.split()[1]
err, out, _ = self.connection.execute(
"ip -s link show {interface}".format(
interface=out.split()[1]))
nic_details['interface'][i] = str(a)
log.info("{0}".format(nic_details))
return nic_details
def install_req_libs(self):
if self.first_run:
log.info("Installing required libraries...")
err, out, _ = self.connection.execute("apt-get update")
log.debug("{0}".format(out))
err, out, _ = self.connection.execute(
"apt-get -y install qemu-kvm libvirt-bin")
log.debug("{0}".format(out))
err, out, _ = self.connection.execute(
"apt-get -y install libvirt-dev bridge-utils numactl")
log.debug("{0}".format(out))
self.first_run = False
def configure_nics_for_sriov(self, host_driver, nic_details):
vf_pci = [[], []]
self.connection.execute(
"rmmod {0}".format(host_driver))[1].splitlines()
self.connection.execute(
"modprobe {0} num_vfs=1".format(host_driver))[1].splitlines()
nic_details['vf_pci'] = {}
for i in range(len(nic_details['pci'])):
self.connection.execute(
"echo 1 > /sys/bus/pci/devices/{0}/sriov_numvfs".format(
nic_details['pci'][i]))
err, out, _ = self.connection.execute(
"ip link set {interface} vf 0 mac {mac}".format(
interface=nic_details['interface'][i],
mac=nic_details['vf_macs'][i]))
time.sleep(3)
vf_pci[i] = self.get_vf_datas(
'vf_pci',
nic_details['pci'][i],
nic_details['vf_macs'][i])
nic_details['vf_pci'][i] = vf_pci[i]
log.debug("NIC DETAILS : {0}".format(nic_details))
return nic_details
def setup_sriov_context(self, pcis, nic_details, host_driver):
blacklist = "/etc/modprobe.d/blacklist.conf"
# 1 : Blacklist the vf driver in /etc/modprobe.d/blacklist.conf
vfnic = "{0}vf".format(host_driver)
lines = self.read_from_file(blacklist)
if vfnic not in lines:
vfblacklist = "blacklist {vfnic}".format(vfnic=vfnic)
self.connection.execute(
"echo {vfblacklist} >> {blacklist}".format(
vfblacklist=vfblacklist,
blacklist=blacklist))
# 2 : modprobe host_driver with num_vfs
nic_details = self.configure_nics_for_sriov(host_driver, nic_details)
# 3: Setup vm_sriov.xml to launch VM
cfg_sriov = '/tmp/vm_sriov.xml'
mac = [0x00, 0x24, 0x81,
random.randint(0x00, 0x7f),
random.randint(0x00, 0xff),
random.randint(0x00, 0xff)]
mac_address = ':'.join(map(lambda x: "%02x" % x, mac))
vm_sriov_xml = VM_TEMPLATE.format(
random_uuid=uuid.uuid4(),
mac_addr=mac_address,
vm_image=self.sriov[0]["images"])
with open(cfg_sriov, 'w') as f:
f.write(vm_sriov_xml)
vf = nic_details['vf_pci']
for index in range(len(nic_details['vf_pci'])):
self.add_sriov_interface(
index,
vf[index]['vf_pci'],
mac_address,
"/tmp/vm_sriov.xml")
self.connection.execute(
"ifconfig {interface} up".format(
interface=nic_details['interface'][index]))
# 4: Create and start the VM
self.connection.put(cfg_sriov, cfg_sriov)
time.sleep(10)
err, out = self.check_output("virsh list --name | grep -i vm1")
try:
if out == "vm1":
log.info("VM is already present")
else:
# FIXME: launch through libvirt
log.info("virsh create ...")
err, out, _ = self.connection.execute(
"virsh create /tmp/vm_sriov.xml")
time.sleep(10)
log.error("err : {0}".format(err))
log.error("{0}".format(_))
log.debug("out : {0}".format(out))
except ValueError:
raise
# 5: Tunning for better performace
self.pin_vcpu(pcis)
self.connection.execute(
"echo 1 > /sys/module/kvm/parameters/"
"allow_unsafe_assigned_interrupts")
self.connection.execute(
"echo never > /sys/kernel/mm/transparent_hugepage/enabled")
def add_sriov_interface(self, index, vf_pci, vfmac, xml):
root = ET.parse(xml)
pattern = "0000:(\d+):(\d+).(\d+)"
m = re.search(pattern, vf_pci, re.MULTILINE)
device = root.find('devices')
interface = ET.SubElement(device, 'interface')
interface.set('managed', 'yes')
interface.set('type', 'hostdev')
mac = ET.SubElement(interface, 'mac')
mac.set('address', vfmac)
source = ET.SubElement(interface, 'source')
addr = ET.SubElement(source, "address")
addr.set('domain', "0x0")
addr.set('bus', "{0}".format(m.group(1)))
addr.set('function', "{0}".format(m.group(3)))
addr.set('slot', "{0}".format(m.group(2)))
addr.set('type', "pci")
vf_pci = ET.SubElement(interface, 'address')
vf_pci.set('type', 'pci')
vf_pci.set('domain', '0x0000')
vf_pci.set('bus', '0x00')
vf_pci.set('slot', '0x0{0}'.format(index + 7))
vf_pci.set('function', '0x00')
root.write(xml)
# This is roughly compatible with check_output function in subprocess
# module which is only available in python 2.7
def check_output(self, cmd, stderr=None):
# Run a command and capture its output
err, out, _ = self.connection.execute(cmd)
return err, out
def get_virtual_devices(self, pci):
pf_vfs = {}
err, extra_info = self.check_output(
"cat /sys/bus/pci/devices/{0}/virtfn0/uevent".format(pci))
pattern = "PCI_SLOT_NAME=(?P[0-9:.\s.]+)"
m = re.search(pattern, extra_info, re.MULTILINE)
if m:
pf_vfs.update({pci: str(m.group(1).rstrip())})
log.info("pf_vfs : {0}".format(pf_vfs))
return pf_vfs
def get_vf_datas(self, key, value, vfmac):
vfret = {}
pattern = "0000:(\d+):(\d+).(\d+)"
vfret["mac"] = vfmac
vfs = self.get_virtual_devices(value)
log.info("vfs: {0}".format(vfs))
for k, v in vfs.items():
m = re.search(pattern, k, re.MULTILINE)
m1 = re.search(pattern, value, re.MULTILINE)
if m.group(1) == m1.group(1):
vfret["vf_pci"] = str(v)
break
return vfret
def read_from_file(self, filename):
data = ""
with open(filename, 'r') as the_file:
data = the_file.read()
return data
def write_to_file(self, filename, content):
with open(filename, 'w') as the_file:
the_file.write(content)
def pin_vcpu(self, pcis):
nodes = self.get_numa_nodes()
log.info("{0}".format(nodes))
num_nodes = len(nodes)
for i in range(0, 10):
self.connection.execute(
"virsh vcpupin vm1 {0} {1}".format(
i, nodes[str(num_nodes - 1)][i]))
def get_numa_nodes(self):
nodes_sysfs = glob.iglob("/sys/devices/system/node/node*")
nodes = {}
for node_sysfs in nodes_sysfs:
num = os.path.basename(node_sysfs).replace("node", "")
with open(os.path.join(node_sysfs, "cpulist")) as cpulist_file:
cpulist = cpulist_file.read().strip()
nodes[num] = self.split_cpu_list(cpulist)
log.info("nodes: {0}".format(nodes))
return nodes
def split_cpu_list(self, cpu_list):
if cpu_list:
ranges = cpu_list.split(',')
bounds = ([int(b) for b in r.split('-')] for r in ranges)
range_objects =\
(range(bound[0], bound[1] + 1 if len(bound) == 2
else bound[0] + 1) for bound in bounds)
return sorted(itertools.chain.from_iterable(range_objects))
else:
return []
def destroy_vm(self):
host_driver = self.sriov[0]["phy_driver"]
err, out = self.check_output("virsh list --name | grep -i vm1")
log.info("{0}".format(out))
if err == 0:
self.connection.execute("virsh shutdown vm1")
self.connection.execute("virsh destroy vm1")
self.check_output("rmmod {0}".format(host_driver))[1].splitlines()
self.check_output("modprobe {0}".format(host_driver))[
1].splitlines()
else:
log.error("error : {0}".format(err))