diff options
Diffstat (limited to 'yardstick/benchmark/contexts/standalone')
| -rw-r--r-- | yardstick/benchmark/contexts/standalone/model.py | 276 | ||||
| -rw-r--r-- | yardstick/benchmark/contexts/standalone/ovs_dpdk.py | 324 | ||||
| -rw-r--r-- | yardstick/benchmark/contexts/standalone/sriov.py | 119 |
3 files changed, 498 insertions, 221 deletions
diff --git a/yardstick/benchmark/contexts/standalone/model.py b/yardstick/benchmark/contexts/standalone/model.py index 85ae14b1d..a15426872 100644 --- a/yardstick/benchmark/contexts/standalone/model.py +++ b/yardstick/benchmark/contexts/standalone/model.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import import os import re import time @@ -25,17 +24,19 @@ from netaddr import IPNetwork import xml.etree.ElementTree as ET from yardstick import ssh -from yardstick.common.constants import YARDSTICK_ROOT_PATH -from yardstick.common.yaml_loader import yaml_load +from yardstick.common import constants +from yardstick.common import exceptions +from yardstick.common import utils as common_utils +from yardstick.common import yaml_loader from yardstick.network_services.utils import PciAddress from yardstick.network_services.helpers.cpu import CpuSysCores -from yardstick.common.utils import write_file + LOG = logging.getLogger(__name__) VM_TEMPLATE = """ <domain type="kvm"> - <name>{vm_name}</name> + <name>{vm_name}</name> <uuid>{random_uuid}</uuid> <memory unit="MB">{memory}</memory> <currentMemory unit="MB">{memory}</currentMemory> @@ -45,7 +46,7 @@ VM_TEMPLATE = """ <vcpu cpuset='{cpuset}'>{vcpu}</vcpu> {cputune} <os> - <type arch="x86_64" machine="pc-i440fx-utopic">hvm</type> + <type arch="x86_64" machine="{machine}">hvm</type> <boot dev="hd" /> </os> <features> @@ -80,9 +81,39 @@ VM_TEMPLATE = """ <source bridge="br-int" /> <model type='virtio'/> </interface> - </devices> + <serial type='pty'> + <target port='0'/> + </serial> + <console type='pty'> + <target type='serial' port='0'/> + </console> + </devices> </domain> """ + +USER_DATA_TEMPLATE = """ +cat > {user_file} <<EOF +#cloud-config +preserve_hostname: false +hostname: {host} +users: +{user_config} +EOF +""" + +NETWORK_DATA_TEMPLATE = """ +cat > {network_file} <<EOF +#cloud-config +version: 2 +ethernets: + ens3: + match: + macaddress: {mac_address} + addresses: + - {ip_address} +EOF +""" + WAIT_FOR_BOOT = 30 @@ -100,12 +131,17 @@ class Libvirt(object): @staticmethod def virsh_create_vm(connection, cfg): - err = connection.execute("virsh create %s" % cfg)[0] - LOG.info("VM create status: %s", err) + LOG.info('VM create, XML config: %s', cfg) + status, _, error = connection.execute('virsh create %s' % cfg) + if status: + raise exceptions.LibvirtCreateError(error=error) @staticmethod def virsh_destroy_vm(vm_name, connection): - connection.execute("virsh destroy %s" % vm_name) + LOG.info('VM destroy, VM name: %s', vm_name) + status, _, error = connection.execute('virsh destroy %s' % vm_name) + if status: + LOG.warning('Error destroying VM %s. Error: %s', vm_name, error) @staticmethod def _add_interface_address(interface, pci_address): @@ -126,7 +162,8 @@ class Libvirt(object): return vm_pci @classmethod - def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml): + def add_ovs_interface(cls, vpath, port_num, vpci, vports_mac, xml_str, + queues): """Add a DPDK OVS 'interface' XML node in 'devices' node <devices> @@ -150,7 +187,7 @@ class Libvirt(object): vhost_path = ('{0}/var/run/openvswitch/dpdkvhostuser{1}'. format(vpath, port_num)) - root = ET.parse(xml) + root = ET.fromstring(xml_str) pci_address = PciAddress(vpci.strip()) device = root.find('devices') @@ -168,17 +205,17 @@ class Libvirt(object): model.set('type', 'virtio') driver = ET.SubElement(interface, 'driver') - driver.set('queues', '4') + driver.set('queues', str(queues)) host = ET.SubElement(driver, 'host') host.set('mrg_rxbuf', 'off') cls._add_interface_address(interface, pci_address) - root.write(xml) + return ET.tostring(root) @classmethod - def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml): + def add_sriov_interfaces(cls, vm_pci, vf_pci, vf_mac, xml_str): """Add a SR-IOV 'interface' XML node in 'devices' node <devices> @@ -201,7 +238,7 @@ class Libvirt(object): -sr_iov-how_sr_iov_libvirt_works """ - root = ET.parse(xml) + root = ET.fromstring(xml_str) device = root.find('devices') interface = ET.SubElement(device, 'interface') @@ -212,27 +249,53 @@ class Libvirt(object): mac.set('address', vf_mac) source = ET.SubElement(interface, 'source') - addr = ET.SubElement(source, 'address') pci_address = PciAddress(vf_pci.strip()) - cls._add_interface_address(addr, pci_address) + cls._add_interface_address(source, pci_address) pci_vm_address = PciAddress(vm_pci.strip()) cls._add_interface_address(interface, pci_vm_address) - root.write(xml) + return ET.tostring(root) @staticmethod - def create_snapshot_qemu(connection, index, vm_image): - # build snapshot image - image = "/var/lib/libvirt/images/%s.qcow2" % index - connection.execute("rm %s" % image) - qemu_template = "qemu-img create -f qcow2 -o backing_file=%s %s" - connection.execute(qemu_template % (vm_image, image)) + def create_snapshot_qemu(connection, index, base_image): + """Create the snapshot image for a VM using a base image - return image + :param connection: SSH connection to the remote host + :param index: index of the VM to be spawn + :param base_image: path of the VM base image in the remote host + :return: snapshot image path + """ + vm_image = '/var/lib/libvirt/images/%s.qcow2' % index + connection.execute('rm -- "%s"' % vm_image) + status, _, _ = connection.execute('test -r %s' % base_image) + if status: + if not os.access(base_image, os.R_OK): + raise exceptions.LibvirtQemuImageBaseImageNotPresent( + vm_image=vm_image, base_image=base_image) + # NOTE(ralonsoh): done in two steps to avoid root permission + # issues. + LOG.info('Copy %s from execution host to remote host', base_image) + file_name = os.path.basename(os.path.normpath(base_image)) + connection.put_file(base_image, '/tmp/%s' % file_name) + status, _, error = connection.execute( + 'mv -- "/tmp/%s" "%s"' % (file_name, base_image)) + if status: + raise exceptions.LibvirtQemuImageCreateError( + vm_image=vm_image, base_image=base_image, error=error) + + LOG.info('Convert image %s to %s', base_image, vm_image) + qemu_cmd = ('qemu-img create -f qcow2 -o backing_file=%s %s' % + (base_image, vm_image)) + status, _, error = connection.execute(qemu_cmd) + if status: + raise exceptions.LibvirtQemuImageCreateError( + vm_image=vm_image, base_image=base_image, error=error) + return vm_image @classmethod - def build_vm_xml(cls, connection, flavor, cfg, vm_name, index): + def build_vm_xml(cls, connection, flavor, vm_name, index, cdrom_img): + """Build the XML from the configuration parameters""" memory = flavor.get('ram', '4096') extra_spec = flavor.get('extra_specs', {}) cpu = extra_spec.get('hw:cpu_cores', '2') @@ -244,6 +307,7 @@ class Libvirt(object): cpuset = Libvirt.pin_vcpu_for_perf(connection, hw_socket) cputune = extra_spec.get('cputune', '') + machine = extra_spec.get('machine_type', 'pc-i440fx-xenial') mac = StandaloneContextHelper.get_mac_address(0x00) image = cls.create_snapshot_qemu(connection, index, flavor.get("images", None)) @@ -254,11 +318,13 @@ class Libvirt(object): memory=memory, vcpu=vcpu, cpu=cpu, numa_cpus=numa_cpus, socket=socket, threads=threads, - vm_image=image, cpuset=cpuset, cputune=cputune) + vm_image=image, cpuset=cpuset, + machine=machine, cputune=cputune) - write_file(cfg, vm_xml) + # Add CD-ROM device + vm_xml = Libvirt.add_cdrom(cdrom_img, vm_xml) - return [vcpu, mac] + return vm_xml, mac @staticmethod def update_interrupts_hugepages_perf(connection): @@ -278,6 +344,82 @@ class Libvirt(object): cpuset = "%s,%s" % (cores, threads) return cpuset + @classmethod + def write_file(cls, file_name, xml_str): + """Dump a XML string to a file""" + root = ET.fromstring(xml_str) + et = ET.ElementTree(element=root) + et.write(file_name, encoding='utf-8', method='xml') + + @classmethod + def add_cdrom(cls, file_path, xml_str): + """Add a CD-ROM disk XML node in 'devices' node + + <devices> + <disk type='file' device='cdrom'> + <driver name='qemu' type='raw'/> + <source file='/var/lib/libvirt/images/data.img'/> + <target dev='hdb'/> + <readonly/> + </disk> + ... + </devices> + """ + + root = ET.fromstring(xml_str) + device = root.find('devices') + + disk = ET.SubElement(device, 'disk') + disk.set('type', 'file') + disk.set('device', 'cdrom') + + driver = ET.SubElement(disk, 'driver') + driver.set('name', 'qemu') + driver.set('type', 'raw') + + source = ET.SubElement(disk, 'source') + source.set('file', file_path) + + target = ET.SubElement(disk, 'target') + target.set('dev', 'hdb') + + ET.SubElement(disk, 'readonly') + return ET.tostring(root) + + @staticmethod + def gen_cdrom_image(connection, file_path, vm_name, vm_user, key_filename, mac, ip): + """Generate ISO image for CD-ROM """ + + user_config = [" - name: {user_name}", + " ssh_authorized_keys:", + " - {pub_key_str}"] + if vm_user != "root": + user_config.append(" sudo: ALL=(ALL) NOPASSWD:ALL") + + meta_data = "/tmp/meta-data" + user_data = "/tmp/user-data" + network_data = "/tmp/network-config" + with open(".".join([key_filename, "pub"]), "r") as pub_key_file: + pub_key_str = pub_key_file.read().rstrip() + user_conf = os.linesep.join(user_config).format(pub_key_str=pub_key_str, user_name=vm_user) + + cmd_lst = [ + "touch %s" % meta_data, + USER_DATA_TEMPLATE.format(user_file=user_data, host=vm_name, user_config=user_conf), + NETWORK_DATA_TEMPLATE.format(network_file=network_data, mac_address=mac, + ip_address=ip), + "genisoimage -output {0} -volid cidata -joliet -r {1} {2} {3}".format(file_path, + meta_data, + user_data, + network_data), + "rm {0} {1} {2}".format(meta_data, user_data, network_data), + ] + for cmd in cmd_lst: + LOG.info(cmd) + status, _, error = connection.execute(cmd) + if status: + raise exceptions.LibvirtQemuImageCreateError(error=error) + class StandaloneContextHelper(object): """ This class handles all the common code for standalone @@ -287,8 +429,9 @@ class StandaloneContextHelper(object): super(StandaloneContextHelper, self).__init__() @staticmethod - def install_req_libs(connection, extra_pkgs=[]): - pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping"] + def install_req_libs(connection, extra_pkgs=None): + extra_pkgs = extra_pkgs or [] + pkgs = ["qemu-kvm", "libvirt-bin", "bridge-utils", "numactl", "fping", "genisoimage"] pkgs.extend(extra_pkgs) cmd_template = "dpkg-query -W --showformat='${Status}\\n' \"%s\"|grep 'ok installed'" for pkg in pkgs: @@ -304,7 +447,7 @@ class StandaloneContextHelper(object): return driver @classmethod - def get_nic_details(cls, connection, networks, dpdk_nic_bind): + def get_nic_details(cls, connection, networks, dpdk_devbind): for key, ports in networks.items(): if key == "mgmt": continue @@ -314,11 +457,11 @@ class StandaloneContextHelper(object): driver = cls.get_kernel_module(connection, phy_ports, phy_driver) # Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe - bind_cmd = "{dpdk_nic_bind} --force -b {driver} {port}" + bind_cmd = "{dpdk_devbind} --force -b {driver} {port}" lshw_cmd = "lshw -c network -businfo | grep '{port}'" link_show_cmd = "ip -s link show {interface}" - cmd = bind_cmd.format(dpdk_nic_bind=dpdk_nic_bind, + cmd = bind_cmd.format(dpdk_devbind=dpdk_devbind, driver=driver, port=ports['phy_port']) connection.execute(cmd) @@ -351,25 +494,18 @@ class StandaloneContextHelper(object): return pf_vfs - def read_config_file(self): - """Read from config file""" - - with open(self.file_path) as stream: - LOG.info("Parsing pod file: %s", self.file_path) - cfg = yaml_load(stream) - return cfg - def parse_pod_file(self, file_path, nfvi_role='Sriov'): self.file_path = file_path nodes = [] nfvi_host = [] try: - cfg = self.read_config_file() + cfg = yaml_loader.read_yaml_file(self.file_path) except IOError as io_error: if io_error.errno != errno.ENOENT: raise - self.file_path = os.path.join(YARDSTICK_ROOT_PATH, file_path) - cfg = self.read_config_file() + self.file_path = os.path.join(constants.YARDSTICK_ROOT_PATH, + file_path) + cfg = yaml_loader.read_yaml_file(self.file_path) nodes.extend([node for node in cfg["nodes"] if str(node["role"]) != nfvi_role]) nfvi_host.extend([node for node in cfg["nodes"] if str(node["role"]) == nfvi_role]) @@ -419,8 +555,41 @@ class StandaloneContextHelper(object): ip = cls.get_mgmt_ip(connection, node["mac"], mgmtip, node) if ip: node["ip"] = ip + client = ssh.SSH.from_node(node) + LOG.debug("OS version: %s", + common_utils.get_os_version(client)) + LOG.debug("Kernel version: %s", + common_utils.get_kernel_version(client)) + vnfs_data = common_utils.get_sample_vnf_info(client) + for vnf_name, vnf_data in vnfs_data.items(): + LOG.debug("VNF name: '%s', commit ID/branch: '%s'", + vnf_name, vnf_data["branch_commit"]) + LOG.debug("%s", vnf_data["md5_result"]) return nodes + @classmethod + def check_update_key(cls, connection, node, vm_name, id_name, cdrom_img, mac): + # Generate public/private keys if private key file is not provided + user_name = node.get('user') + if not user_name: + node['user'] = 'root' + user_name = node.get('user') + if not node.get('key_filename'): + key_filename = ''.join( + [constants.YARDSTICK_ROOT_PATH, + 'yardstick/resources/files/yardstick_key-', + id_name, '-', vm_name]) + ssh.SSH.gen_keys(key_filename) + node['key_filename'] = key_filename + # Update image with public key + key_filename = node.get('key_filename') + ip_netmask = "{0}/{1}".format(node.get('ip'), node.get('netmask')) + ip_netmask = "{0}/{1}".format(node.get('ip'), + IPNetwork(ip_netmask).prefixlen) + Libvirt.gen_cdrom_image(connection, cdrom_img, vm_name, user_name, key_filename, mac, + ip_netmask) + return node + class Server(object): """ This class handles geting vnf nodes @@ -433,7 +602,7 @@ class Server(object): for key, vfs in vnf["network_ports"].items(): if key == "mgmt": - mgmtip = str(IPNetwork(vfs['cidr']).ip) + mgmt_cidr = IPNetwork(vfs['cidr']) continue vf = ports[vfs[0]] @@ -450,14 +619,15 @@ class Server(object): }) index = index + 1 - return mgmtip, interfaces + return mgmt_cidr, interfaces @classmethod def generate_vnf_instance(cls, flavor, ports, ip, key, vnf, mac): - mgmtip, interfaces = cls.build_vnf_interfaces(vnf, ports) + mgmt_cidr, interfaces = cls.build_vnf_interfaces(vnf, ports) result = { - "ip": mgmtip, + "ip": str(mgmt_cidr.ip), + "netmask": str(mgmt_cidr.netmask), "mac": mac, "host": ip, "user": flavor.get('user', 'root'), @@ -500,7 +670,7 @@ class OvsDeploy(object): StandaloneContextHelper.install_req_libs(self.connection, pkgs) def ovs_deploy(self): - ovs_deploy = os.path.join(YARDSTICK_ROOT_PATH, + ovs_deploy = os.path.join(constants.YARDSTICK_ROOT_PATH, "yardstick/resources/scripts/install/", self.OVS_DEPLOY_SCRIPT) if os.path.isfile(ovs_deploy): @@ -516,4 +686,6 @@ class OvsDeploy(object): cmd = "sudo -E %s --ovs='%s' --dpdk='%s' -p='%s'" % (remote_ovs_deploy, ovs, dpdk, http_proxy) - self.connection.execute(cmd) + exit_status, _, stderr = self.connection.execute(cmd) + if exit_status: + raise exceptions.OVSDeployError(stderr=stderr) diff --git a/yardstick/benchmark/contexts/standalone/ovs_dpdk.py b/yardstick/benchmark/contexts/standalone/ovs_dpdk.py index 3755b84e9..c6e19f614 100644 --- a/yardstick/benchmark/contexts/standalone/ovs_dpdk.py +++ b/yardstick/benchmark/contexts/standalone/ovs_dpdk.py @@ -12,33 +12,34 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import absolute_import -import os -import logging +import io import collections +import logging +import os +import re import time -from collections import OrderedDict - from yardstick import ssh +from yardstick.benchmark import contexts +from yardstick.benchmark.contexts import base +from yardstick.benchmark.contexts.standalone import model +from yardstick.common import exceptions +from yardstick.common import utils as common_utils +from yardstick.network_services import utils from yardstick.network_services.utils import get_nsb_option -from yardstick.network_services.utils import provision_tool -from yardstick.benchmark.contexts.base import Context -from yardstick.benchmark.contexts.standalone.model import Libvirt -from yardstick.benchmark.contexts.standalone.model import StandaloneContextHelper -from yardstick.benchmark.contexts.standalone.model import Server -from yardstick.benchmark.contexts.standalone.model import OvsDeploy -from yardstick.network_services.utils import PciAddress + LOG = logging.getLogger(__name__) +MAIN_BRIDGE = 'br0' -class OvsDpdkContext(Context): + +class OvsDpdkContext(base.Context): """ This class handles OVS standalone nodes - VM running on Non-Managed NFVi Configuration: ovs_dpdk """ - __context_type__ = "StandaloneOvsDpdk" + __context_type__ = contexts.CONTEXT_STANDALONEOVSDPDK SUPPORTED_OVS_TO_DPDK_MAP = { '2.6.0': '16.07.1', @@ -46,36 +47,42 @@ class OvsDpdkContext(Context): '2.7.0': '16.11.1', '2.7.1': '16.11.2', '2.7.2': '16.11.3', - '2.8.0': '17.05.2' + '2.8.0': '17.05.2', + '2.8.1': '17.05.2' } DEFAULT_OVS = '2.6.0' - - PKILL_TEMPLATE = "pkill %s %s" + CMD_TIMEOUT = 30 + DEFAULT_USER_PATH = '/usr/local' def __init__(self): self.file_path = None self.sriov = [] self.first_run = True - self.dpdk_nic_bind = "" + self.dpdk_devbind = os.path.join(get_nsb_option('bin_path'), + 'dpdk-devbind.py') self.vm_names = [] - self.name = None self.nfvi_host = [] self.nodes = [] self.networks = {} self.attrs = {} self.vm_flavor = None self.servers = None - self.helper = StandaloneContextHelper() - self.vnf_node = Server() + self.helper = model.StandaloneContextHelper() + self.vnf_node = model.Server() self.ovs_properties = {} self.wait_for_vswitchd = 10 super(OvsDpdkContext, self).__init__() + def get_dpdk_socket_mem_size(self, socket_id): + """Get the size of OvS DPDK socket memory (Mb)""" + ram = self.ovs_properties.get("ram", {}) + return ram.get('socket_%d' % (socket_id), 2048) + def init(self, attrs): """initializes itself from the supplied arguments""" + super(OvsDpdkContext, self).init(attrs) - self.name = attrs["name"] self.file_path = attrs.get("file", "pod.yaml") self.nodes, self.nfvi_host, self.host_mgmt = \ @@ -94,34 +101,32 @@ class OvsDpdkContext(Context): LOG.debug("Networks: %r", self.networks) def setup_ovs(self): - vpath = self.ovs_properties.get("vpath", "/usr/local") - xargs_kill_cmd = self.PKILL_TEMPLATE % ('-9', 'ovs') - + """Initialize OVS-DPDK""" + vpath = self.ovs_properties.get('vpath', self.DEFAULT_USER_PATH) create_from = os.path.join(vpath, 'etc/openvswitch/conf.db') create_to = os.path.join(vpath, 'share/openvswitch/vswitch.ovsschema') cmd_list = [ - "chmod 0666 /dev/vfio/*", - "chmod a+x /dev/vfio", - "pkill -9 ovs", - xargs_kill_cmd, - "killall -r 'ovs*'", - "mkdir -p {0}/etc/openvswitch".format(vpath), - "mkdir -p {0}/var/run/openvswitch".format(vpath), - "rm {0}/etc/openvswitch/conf.db".format(vpath), - "ovsdb-tool create {0} {1}".format(create_from, create_to), - "modprobe vfio-pci", - "chmod a+x /dev/vfio", - "chmod 0666 /dev/vfio/*", + 'killall -r "ovs.*" -q | true', + 'mkdir -p {0}/etc/openvswitch'.format(vpath), + 'mkdir -p {0}/var/run/openvswitch'.format(vpath), + 'rm {0}/etc/openvswitch/conf.db | true'.format(vpath), + 'ovsdb-tool create {0} {1}'.format(create_from, create_to), + 'modprobe vfio-pci', + 'chmod a+x /dev/vfio', + 'chmod 0666 /dev/vfio/*', ] + + bind_cmd = '%s --force -b vfio-pci {port}' % self.dpdk_devbind + for port in self.networks.values(): + cmd_list.append(bind_cmd.format(port=port.get('phy_port'))) + for cmd in cmd_list: - self.connection.execute(cmd) - bind_cmd = "{dpdk_nic_bind} --force -b {driver} {port}" - phy_driver = "vfio-pci" - for _, port in self.networks.items(): - vpci = port.get("phy_port") - self.connection.execute(bind_cmd.format(dpdk_nic_bind=self.dpdk_nic_bind, - driver=phy_driver, port=vpci)) + LOG.info(cmd) + exit_status, _, stderr = self.connection.execute( + cmd, timeout=self.CMD_TIMEOUT) + if exit_status: + raise exceptions.OVSSetupError(command=cmd, error=stderr) def start_ovs_serverswitch(self): vpath = self.ovs_properties.get("vpath") @@ -134,9 +139,6 @@ class OvsDpdkContext(Context): if pmd_cpu_mask: pmd_mask = pmd_cpu_mask - socket0 = self.ovs_properties.get("ram", {}).get("socket_0", "2048") - socket1 = self.ovs_properties.get("ram", {}).get("socket_1", "2048") - ovs_other_config = "ovs-vsctl {0}set Open_vSwitch . other_config:{1}" detach_cmd = "ovs-vswitchd unix:{0}{1} --pidfile --detach --log-file={2}" @@ -144,16 +146,23 @@ class OvsDpdkContext(Context): if lcore_mask: lcore_mask = ovs_other_config.format("--no-wait ", "dpdk-lcore-mask='%s'" % lcore_mask) + max_idle = self.ovs_properties.get("max_idle", '') + if max_idle: + max_idle = ovs_other_config.format("", "max-idle=%s" % max_idle) + cmd_list = [ "mkdir -p /usr/local/var/run/openvswitch", "mkdir -p {}".format(os.path.dirname(log_path)), - "ovsdb-server --remote=punix:/{0}/{1} --pidfile --detach".format(vpath, - ovs_sock_path), + ("ovsdb-server --remote=punix:/{0}/{1} --remote=ptcp:6640" + " --pidfile --detach").format(vpath, ovs_sock_path), ovs_other_config.format("--no-wait ", "dpdk-init=true"), - ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%s,%s'" % (socket0, socket1)), + ovs_other_config.format("--no-wait ", "dpdk-socket-mem='%d,%d'" % ( + self.get_dpdk_socket_mem_size(0), + self.get_dpdk_socket_mem_size(1))), lcore_mask, detach_cmd.format(vpath, ovs_sock_path, log_path), ovs_other_config.format("", "pmd-cpu-mask=%s" % pmd_mask), + max_idle, ] for cmd in cmd_list: @@ -163,56 +172,92 @@ class OvsDpdkContext(Context): def setup_ovs_bridge_add_flows(self): dpdk_args = "" - dpdk_list = [] vpath = self.ovs_properties.get("vpath", "/usr/local") version = self.ovs_properties.get('version', {}) ovs_ver = [int(x) for x in version.get('ovs', self.DEFAULT_OVS).split('.')] - ovs_add_port = \ - "ovs-vsctl add-port {br} {port} -- set Interface {port} type={type_}{dpdk_args}" - ovs_add_queue = "ovs-vsctl set Interface {port} options:n_rxq={queue}" - chmod_vpath = "chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*" - - cmd_dpdk_list = [ - "ovs-vsctl del-br br0", - "rm -rf {0}/var/run/openvswitch/dpdkvhostuser*".format(vpath), - "ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev", + ovs_add_port = ('ovs-vsctl add-port {br} {port} -- ' + 'set Interface {port} type={type_}{dpdk_args}' + '{dpdk_rxq}{pmd_rx_aff}') + chmod_vpath = 'chmod 0777 {0}/var/run/openvswitch/dpdkvhostuser*' + + cmd_list = [ + 'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE), + 'rm -rf {0}/var/run/openvswitch/dpdkvhostuser*'.format(vpath), + 'ovs-vsctl add-br {0} -- set bridge {0} datapath_type=netdev'. + format(MAIN_BRIDGE) ] + dpdk_rxq = "" + queues = self.ovs_properties.get("queues") + if queues: + dpdk_rxq = " options:n_rxq={queue}".format(queue=queues) - ordered_network = OrderedDict(self.networks) + # Sorting the array to make sure we execute dpdk0... in the order + ordered_network = collections.OrderedDict( + sorted(self.networks.items(), key=lambda t: t[1].get('port_num', 0))) + pmd_rx_aff_ports = self.ovs_properties.get("dpdk_pmd-rxq-affinity", {}) for index, vnf in enumerate(ordered_network.values()): if ovs_ver >= [2, 7, 0]: dpdk_args = " options:dpdk-devargs=%s" % vnf.get("phy_port") - dpdk_list.append(ovs_add_port.format(br='br0', port='dpdk%s' % vnf.get("port_num", 0), - type_='dpdk', dpdk_args=dpdk_args)) - dpdk_list.append(ovs_add_queue.format(port='dpdk%s' % vnf.get("port_num", 0), - queue=self.ovs_properties.get("queues", 1))) - - # Sorting the array to make sure we execute dpdk0... in the order - list.sort(dpdk_list) - cmd_dpdk_list.extend(dpdk_list) + affinity = pmd_rx_aff_ports.get(vnf.get("port_num", -1), "") + if affinity: + pmd_rx_aff = ' other_config:pmd-rxq-affinity=' \ + '"{affinity}"'.format(affinity=affinity) + else: + pmd_rx_aff = "" + cmd_list.append(ovs_add_port.format( + br=MAIN_BRIDGE, port='dpdk%s' % vnf.get("port_num", 0), + type_='dpdk', dpdk_args=dpdk_args, dpdk_rxq=dpdk_rxq, + pmd_rx_aff=pmd_rx_aff)) # Need to do two for loop to maintain the dpdk/vhost ports. + pmd_rx_aff_ports = self.ovs_properties.get("vhost_pmd-rxq-affinity", + {}) for index, _ in enumerate(ordered_network): - cmd_dpdk_list.append(ovs_add_port.format(br='br0', port='dpdkvhostuser%s' % index, - type_='dpdkvhostuser', dpdk_args="")) - - for cmd in cmd_dpdk_list: - LOG.info(cmd) - self.connection.execute(cmd) - - # Fixme: add flows code - ovs_flow = "ovs-ofctl add-flow br0 in_port=%s,action=output:%s" - + affinity = pmd_rx_aff_ports.get(index) + if affinity: + pmd_rx_aff = ' other_config:pmd-rxq-affinity=' \ + '"{affinity}"'.format(affinity=affinity) + else: + pmd_rx_aff = "" + cmd_list.append(ovs_add_port.format( + br=MAIN_BRIDGE, port='dpdkvhostuser%s' % index, + type_='dpdkvhostuser', dpdk_args="", dpdk_rxq=dpdk_rxq, + pmd_rx_aff=pmd_rx_aff)) + + ovs_flow = ("ovs-ofctl add-flow {0} in_port=%s,action=output:%s". + format(MAIN_BRIDGE)) network_count = len(ordered_network) + 1 for in_port, out_port in zip(range(1, network_count), range(network_count, network_count * 2)): - self.connection.execute(ovs_flow % (in_port, out_port)) - self.connection.execute(ovs_flow % (out_port, in_port)) + cmd_list.append(ovs_flow % (in_port, out_port)) + cmd_list.append(ovs_flow % (out_port, in_port)) + + cmd_list.append(chmod_vpath.format(vpath)) - self.connection.execute(chmod_vpath.format(vpath)) + for cmd in cmd_list: + LOG.info(cmd) + exit_status, _, stderr = self.connection.execute( + cmd, timeout=self.CMD_TIMEOUT) + if exit_status: + raise exceptions.OVSSetupError(command=cmd, error=stderr) + + def _check_hugepages(self): + meminfo = io.BytesIO() + self.connection.get_file_obj('/proc/meminfo', meminfo) + regex = re.compile(r"HugePages_Total:\s+(?P<hp_total>\d+)[\n\r]" + r"HugePages_Free:\s+(?P<hp_free>\d+)") + match = regex.search(meminfo.getvalue().decode('utf-8')) + if not match: + raise exceptions.OVSHugepagesInfoError() + if int(match.group('hp_total')) == 0: + raise exceptions.OVSHugepagesNotConfigured() + if int(match.group('hp_free')) == 0: + raise exceptions.OVSHugepagesZeroFree( + total_hugepages=int(match.group('hp_total'))) def cleanup_ovs_dpdk_env(self): - self.connection.execute("ovs-vsctl del-br br0") + self.connection.execute( + 'ovs-vsctl --if-exists del-br {0}'.format(MAIN_BRIDGE)) self.connection.execute("pkill -9 ovs") def check_ovs_dpdk_env(self): @@ -224,13 +269,15 @@ class OvsDpdkContext(Context): supported_version = self.SUPPORTED_OVS_TO_DPDK_MAP.get(ovs_ver, None) if supported_version is None or supported_version.split('.')[:2] != dpdk_ver[:2]: - raise Exception("Unsupported ovs '{}'. Please check the config...".format(ovs_ver)) + raise exceptions.OVSUnsupportedVersion( + ovs_version=ovs_ver, + ovs_to_dpdk_map=self.SUPPORTED_OVS_TO_DPDK_MAP) status = self.connection.execute("ovs-vsctl -V | grep -i '%s'" % ovs_ver)[0] if status: - deploy = OvsDeploy(self.connection, - get_nsb_option("bin_path"), - self.ovs_properties) + deploy = model.OvsDeploy(self.connection, + utils.get_nsb_option("bin_path"), + self.ovs_properties) deploy.ovs_deploy() def deploy(self): @@ -241,26 +288,21 @@ class OvsDpdkContext(Context): return self.connection = ssh.SSH.from_node(self.host_mgmt) - self.dpdk_nic_bind = provision_tool( - self.connection, - os.path.join(get_nsb_option("bin_path"), "dpdk-devbind.py")) # Check dpdk/ovs version, if not present install self.check_ovs_dpdk_env() # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config. - StandaloneContextHelper.install_req_libs(self.connection) - self.networks = StandaloneContextHelper.get_nic_details(self.connection, - self.networks, - self.dpdk_nic_bind) + model.StandaloneContextHelper.install_req_libs(self.connection) + self.networks = model.StandaloneContextHelper.get_nic_details( + self.connection, self.networks, self.dpdk_devbind) self.setup_ovs() self.start_ovs_serverswitch() self.setup_ovs_bridge_add_flows() self.nodes = self.setup_ovs_dpdk_context() LOG.debug("Waiting for VM to come up...") - self.nodes = StandaloneContextHelper.wait_for_vnfs_to_start(self.connection, - self.servers, - self.nodes) + self.nodes = model.StandaloneContextHelper.wait_for_vnfs_to_start( + self.connection, self.servers, self.nodes) def undeploy(self): @@ -271,16 +313,31 @@ class OvsDpdkContext(Context): self.cleanup_ovs_dpdk_env() # Bind nics back to kernel - bind_cmd = "{dpdk_nic_bind} --force -b {driver} {port}" + bind_cmd = "{dpdk_devbind} --force -b {driver} {port}" for port in self.networks.values(): vpci = port.get("phy_port") phy_driver = port.get("driver") - self.connection.execute(bind_cmd.format(dpdk_nic_bind=self.dpdk_nic_bind, - driver=phy_driver, port=vpci)) + self.connection.execute(bind_cmd.format( + dpdk_devbind=self.dpdk_devbind, driver=phy_driver, port=vpci)) # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config. for vm in self.vm_names: - Libvirt.check_if_vm_exists_and_delete(vm, self.connection) + model.Libvirt.check_if_vm_exists_and_delete(vm, self.connection) + + def _get_physical_nodes(self): + return self.nfvi_host + + def _get_physical_node_for_server(self, server_name): + node_name, ctx_name = self.split_host_name(server_name) + if ctx_name is None or self.name != ctx_name: + return None + + matching_nodes = [s for s in self.servers if s == node_name] + if len(matching_nodes) == 0: + return None + + # self.nfvi_host always contain only one host + return "{}.{}".format(self.nfvi_host[0]["name"], self._name) def _get_server(self, attr_name): """lookup server info by name from context @@ -288,7 +345,7 @@ class OvsDpdkContext(Context): Keyword arguments: attr_name -- A name for a server listed in nodes config file """ - node_name, name = self.split_name(attr_name) + node_name, name = self.split_host_name(attr_name) if name is None or self.name != name: return None @@ -335,57 +392,78 @@ class OvsDpdkContext(Context): return result def configure_nics_for_ovs_dpdk(self): - portlist = OrderedDict(self.networks) + portlist = collections.OrderedDict(self.networks) for key in portlist: - mac = StandaloneContextHelper.get_mac_address() + mac = model.StandaloneContextHelper.get_mac_address() portlist[key].update({'mac': mac}) self.networks = portlist LOG.info("Ports %s", self.networks) - def _enable_interfaces(self, index, vfs, cfg): + def _enable_interfaces(self, index, vfs, xml_str): vpath = self.ovs_properties.get("vpath", "/usr/local") + queue = self.ovs_properties.get("queues", 1) vf = self.networks[vfs[0]] port_num = vf.get('port_num', 0) - vpci = PciAddress(vf['vpci'].strip()) + vpci = utils.PciAddress(vf['vpci'].strip()) # Generate the vpci for the interfaces slot = index + port_num + 10 vf['vpci'] = \ "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function) - Libvirt.add_ovs_interface(vpath, port_num, vf['vpci'], vf['mac'], str(cfg)) + return model.Libvirt.add_ovs_interface( + vpath, port_num, vf['vpci'], vf['mac'], xml_str, queue) def setup_ovs_dpdk_context(self): nodes = [] self.configure_nics_for_ovs_dpdk() - for index, (key, vnf) in enumerate(OrderedDict(self.servers).items()): + hp_total_mb = int(self.vm_flavor.get('ram', '4096')) * len(self.servers) + common_utils.setup_hugepages(self.connection, (hp_total_mb + \ + self.get_dpdk_socket_mem_size(0) + \ + self.get_dpdk_socket_mem_size(1)) * 1024) + + self._check_hugepages() + + for index, (key, vnf) in enumerate(collections.OrderedDict( + self.servers).items()): cfg = '/tmp/vm_ovs_%d.xml' % index - vm_name = "vm_%d" % index + vm_name = "vm-%d" % index + cdrom_img = "/var/lib/libvirt/images/cdrom-%d.img" % index # 1. Check and delete VM if already exists - Libvirt.check_if_vm_exists_and_delete(vm_name, self.connection) + model.Libvirt.check_if_vm_exists_and_delete(vm_name, + self.connection) + xml_str, mac = model.Libvirt.build_vm_xml( + self.connection, self.vm_flavor, vm_name, index, cdrom_img) - _, mac = Libvirt.build_vm_xml(self.connection, self.vm_flavor, - cfg, vm_name, index) # 2: Cleanup already available VMs - for vkey, vfs in OrderedDict(vnf["network_ports"]).items(): - if vkey == "mgmt": - continue - self._enable_interfaces(index, vfs, cfg) + for vfs in [vfs for vfs_name, vfs in vnf["network_ports"].items() + if vfs_name != 'mgmt']: + xml_str = self._enable_interfaces(index, vfs, xml_str) # copy xml to target... + model.Libvirt.write_file(cfg, xml_str) self.connection.put(cfg, cfg) + node = self.vnf_node.generate_vnf_instance(self.vm_flavor, + self.networks, + self.host_mgmt.get('ip'), + key, vnf, mac) + # Generate public/private keys if password or private key file is not provided + node = model.StandaloneContextHelper.check_update_key(self.connection, + node, + vm_name, + self.name, + cdrom_img, + mac) + + # store vnf node details + nodes.append(node) + # NOTE: launch through libvirt LOG.info("virsh create ...") - Libvirt.virsh_create_vm(self.connection, cfg) + model.Libvirt.virsh_create_vm(self.connection, cfg) self.vm_names.append(vm_name) - # build vnf node details - nodes.append(self.vnf_node.generate_vnf_instance(self.vm_flavor, - self.networks, - self.host_mgmt.get('ip'), - key, vnf, mac)) - return nodes diff --git a/yardstick/benchmark/contexts/standalone/sriov.py b/yardstick/benchmark/contexts/standalone/sriov.py index 9d8423b5f..e037dd85a 100644 --- a/yardstick/benchmark/contexts/standalone/sriov.py +++ b/yardstick/benchmark/contexts/standalone/sriov.py @@ -16,49 +16,47 @@ from __future__ import absolute_import import os import logging import collections -from collections import OrderedDict from yardstick import ssh +from yardstick.benchmark import contexts +from yardstick.benchmark.contexts import base +from yardstick.benchmark.contexts.standalone import model +from yardstick.common import utils from yardstick.network_services.utils import get_nsb_option -from yardstick.network_services.utils import provision_tool -from yardstick.benchmark.contexts.base import Context -from yardstick.benchmark.contexts.standalone.model import Libvirt -from yardstick.benchmark.contexts.standalone.model import StandaloneContextHelper -from yardstick.benchmark.contexts.standalone.model import Server from yardstick.network_services.utils import PciAddress LOG = logging.getLogger(__name__) -class SriovContext(Context): +class SriovContext(base.Context): """ This class handles SRIOV standalone nodes - VM running on Non-Managed NFVi Configuration: sr-iov """ - __context_type__ = "StandaloneSriov" + __context_type__ = contexts.CONTEXT_STANDALONESRIOV def __init__(self): self.file_path = None self.sriov = [] self.first_run = True - self.dpdk_nic_bind = "" + self.dpdk_devbind = os.path.join(get_nsb_option('bin_path'), + 'dpdk-devbind.py') self.vm_names = [] - self.name = None self.nfvi_host = [] self.nodes = [] self.networks = {} self.attrs = {} self.vm_flavor = None self.servers = None - self.helper = StandaloneContextHelper() - self.vnf_node = Server() + self.helper = model.StandaloneContextHelper() + self.vnf_node = model.Server() self.drivers = [] super(SriovContext, self).__init__() def init(self, attrs): """initializes itself from the supplied arguments""" + super(SriovContext, self).init(attrs) - self.name = attrs["name"] self.file_path = attrs.get("file", "pod.yaml") self.nodes, self.nfvi_host, self.host_mgmt = \ @@ -83,21 +81,16 @@ class SriovContext(Context): return self.connection = ssh.SSH.from_node(self.host_mgmt) - self.dpdk_nic_bind = provision_tool( - self.connection, - os.path.join(get_nsb_option("bin_path"), "dpdk_nic_bind.py")) # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config. - StandaloneContextHelper.install_req_libs(self.connection) - self.networks = StandaloneContextHelper.get_nic_details(self.connection, - self.networks, - self.dpdk_nic_bind) + model.StandaloneContextHelper.install_req_libs(self.connection) + self.networks = model.StandaloneContextHelper.get_nic_details( + self.connection, self.networks, self.dpdk_devbind) self.nodes = self.setup_sriov_context() LOG.debug("Waiting for VM to come up...") - self.nodes = StandaloneContextHelper.wait_for_vnfs_to_start(self.connection, - self.servers, - self.nodes) + self.nodes = model.StandaloneContextHelper.wait_for_vnfs_to_start( + self.connection, self.servers, self.nodes) def undeploy(self): """don't need to undeploy""" @@ -107,7 +100,7 @@ class SriovContext(Context): # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config. for vm in self.vm_names: - Libvirt.check_if_vm_exists_and_delete(vm, self.connection) + model.Libvirt.check_if_vm_exists_and_delete(vm, self.connection) # Bind nics back to kernel for ports in self.networks.values(): @@ -115,13 +108,29 @@ class SriovContext(Context): build_vfs = "echo 0 > /sys/bus/pci/devices/{0}/sriov_numvfs" self.connection.execute(build_vfs.format(ports.get('phy_port'))) + def _get_physical_nodes(self): + return self.nfvi_host + + def _get_physical_node_for_server(self, server_name): + + # self.nfvi_host always contain only one host. + node_name, ctx_name = self.split_host_name(server_name) + if ctx_name is None or self.name != ctx_name: + return None + + matching_nodes = [s for s in self.servers if s == node_name] + if len(matching_nodes) == 0: + return None + + return "{}.{}".format(self.nfvi_host[0]["name"], self._name) + def _get_server(self, attr_name): """lookup server info by name from context Keyword arguments: attr_name -- A name for a server listed in nodes config file """ - node_name, name = self.split_name(attr_name) + node_name, name = self.split_host_name(attr_name) if name is None or self.name != name: return None @@ -138,8 +147,8 @@ class SriovContext(Context): except StopIteration: pass else: - raise ValueError("Duplicate nodes!!! Nodes: %s %s", - (node, duplicate)) + raise ValueError("Duplicate nodes!!! Nodes: %s %s" + % (node, duplicate)) node["name"] = attr_name return node @@ -181,7 +190,7 @@ class SriovContext(Context): self.connection.execute(build_vfs.format(ports.get('phy_port'))) # configure VFs... - mac = StandaloneContextHelper.get_mac_address() + mac = model.StandaloneContextHelper.get_mac_address() interface = ports.get('interface') if interface is not None: self.connection.execute(vf_cmd.format(interface, mac)) @@ -203,10 +212,10 @@ class SriovContext(Context): slot = index + idx + 10 vf['vpci'] = \ "{}:{}:{:02x}.{}".format(vpci.domain, vpci.bus, slot, vpci.function) - Libvirt.add_sriov_interfaces( - vf['vpci'], vf['vf_pci']['vf_pci'], vf['mac'], str(cfg)) self.connection.execute("ifconfig %s up" % vf['interface']) self.connection.execute(vf_spoofchk.format(vf['interface'])) + return model.Libvirt.add_sriov_interfaces( + vf['vpci'], vf['vf_pci']['vf_pci'], vf['mac'], str(cfg)) def setup_sriov_context(self): nodes = [] @@ -214,35 +223,52 @@ class SriovContext(Context): # 1 : modprobe host_driver with num_vfs self.configure_nics_for_sriov() - for index, (key, vnf) in enumerate(OrderedDict(self.servers).items()): + hp_total_mb = int(self.vm_flavor.get('ram', '4096')) * len(self.servers) + utils.setup_hugepages(self.connection, hp_total_mb * 1024) + + for index, (key, vnf) in enumerate(collections.OrderedDict( + self.servers).items()): cfg = '/tmp/vm_sriov_%s.xml' % str(index) - vm_name = "vm_%s" % str(index) + vm_name = "vm-%s" % str(index) + cdrom_img = "/var/lib/libvirt/images/cdrom-%d.img" % index # 1. Check and delete VM if already exists - Libvirt.check_if_vm_exists_and_delete(vm_name, self.connection) + model.Libvirt.check_if_vm_exists_and_delete(vm_name, + self.connection) + xml_str, mac = model.Libvirt.build_vm_xml( + self.connection, self.vm_flavor, vm_name, index, cdrom_img) - _, mac = Libvirt.build_vm_xml(self.connection, self.vm_flavor, cfg, vm_name, index) # 2: Cleanup already available VMs - for idx, (vkey, vfs) in enumerate(OrderedDict(vnf["network_ports"]).items()): - if vkey == "mgmt": - continue - self._enable_interfaces(index, idx, vfs, cfg) + network_ports = collections.OrderedDict( + {k: v for k, v in vnf["network_ports"].items() if k != 'mgmt'}) + for idx, vfs in enumerate(network_ports.values()): + xml_str = self._enable_interfaces(index, idx, vfs, xml_str) # copy xml to target... + model.Libvirt.write_file(cfg, xml_str) self.connection.put(cfg, cfg) + node = self.vnf_node.generate_vnf_instance(self.vm_flavor, + self.networks, + self.host_mgmt.get('ip'), + key, vnf, mac) + # Generate public/private keys if password or private key file is not provided + node = model.StandaloneContextHelper.check_update_key(self.connection, + node, + vm_name, + self.name, + cdrom_img, + mac) + + # store vnf node details + nodes.append(node) + # NOTE: launch through libvirt LOG.info("virsh create ...") - Libvirt.virsh_create_vm(self.connection, cfg) + model.Libvirt.virsh_create_vm(self.connection, cfg) self.vm_names.append(vm_name) - # build vnf node details - nodes.append(self.vnf_node.generate_vnf_instance(self.vm_flavor, - self.networks, - self.host_mgmt.get('ip'), - key, vnf, mac)) - return nodes def _get_vf_data(self, value, vfmac, pfif): @@ -250,7 +276,8 @@ class SriovContext(Context): "mac": vfmac, "pf_if": pfif } - vfs = StandaloneContextHelper.get_virtual_devices(self.connection, value) + vfs = model.StandaloneContextHelper.get_virtual_devices( + self.connection, value) for k, v in vfs.items(): m = PciAddress(k.strip()) m1 = PciAddress(value.strip()) |