diff options
Diffstat (limited to 'yardstick/benchmark/scenarios/networking')
17 files changed, 922 insertions, 507 deletions
diff --git a/yardstick/benchmark/scenarios/networking/iperf3.py b/yardstick/benchmark/scenarios/networking/iperf3.py index 98c45990e..51e044e7b 100644 --- a/yardstick/benchmark/scenarios/networking/iperf3.py +++ b/yardstick/benchmark/scenarios/networking/iperf3.py @@ -92,7 +92,7 @@ For more info see http://software.es.net/iperf def teardown(self): LOG.debug("teardown") self.host.close() - status, stdout, stderr = self.target.execute("pkill iperf3") + status, _, stderr = self.target.execute("pkill iperf3") if status: LOG.warning(stderr) self.target.close() @@ -145,7 +145,7 @@ For more info see http://software.es.net/iperf LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.host.execute(cmd) + status, stdout, _ = self.host.execute(cmd) if status: # error cause in json dict on stdout raise RuntimeError(stdout) @@ -165,16 +165,17 @@ For more info see http://software.es.net/iperf bit_per_second = \ int(iperf_result["end"]["sum_received"]["bits_per_second"]) bytes_per_second = bit_per_second / 8 - assert bytes_per_second >= sla_bytes_per_second, \ - "bytes_per_second %d < sla:bytes_per_second (%d); " % \ - (bytes_per_second, sla_bytes_per_second) + self.verify_SLA( + bytes_per_second >= sla_bytes_per_second, + "bytes_per_second %d < sla:bytes_per_second (%d); " + % (bytes_per_second, sla_bytes_per_second)) else: sla_jitter = float(sla_iperf["jitter"]) jitter_ms = float(iperf_result["end"]["sum"]["jitter_ms"]) - assert jitter_ms <= sla_jitter, \ - "jitter_ms %f > sla:jitter %f; " % \ - (jitter_ms, sla_jitter) + self.verify_SLA(jitter_ms <= sla_jitter, + "jitter_ms %f > sla:jitter %f; " + % (jitter_ms, sla_jitter)) def _test(): diff --git a/yardstick/benchmark/scenarios/networking/moongen_testpmd.bash b/yardstick/benchmark/scenarios/networking/moongen_testpmd.bash new file mode 100644 index 000000000..3e92cc900 --- /dev/null +++ b/yardstick/benchmark/scenarios/networking/moongen_testpmd.bash @@ -0,0 +1,62 @@ +############################################################################## +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +#!/bin/bash + +set -e + +# Commandline arguments +MOONGEN_PORT1_MAC=$1 # MAC address of the peer port +MOONGEN_PORT2_MAC=$2 # MAC address of the peer port +TESTPMD_QUEUE=$3 + +BIND_ROOT='/opt/nsb_bin' +DRIVER_ROOT='/opt/tempT/dpdk-17.02/' + +load_modules() +{ + if ! lsmod | grep "uio" &> /dev/null; then + modprobe uio + fi + + if ! lsmod | grep "igb_uio" &> /dev/null; then + insmod ${DRIVER_ROOT}/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko + fi + + if ! lsmod | grep "rte_kni" &> /dev/null; then + insmod ${DRIVER_ROOT}/x86_64-native-linuxapp-gcc/kmod/rte_kni.ko + fi +} + +change_permissions() +{ + chmod 777 /sys/bus/pci/drivers/virtio-pci/* + chmod 777 /sys/bus/pci/drivers/igb_uio/* +} + +add_interface_to_dpdk(){ + interfaces=$(lspci |grep Eth |tail -n +2 |awk '{print $1}') + ${BIND_ROOT}/dpdk_nic_bind.py --bind=igb_uio $interfaces &> /dev/null +} + +run_testpmd() +{ + blacklist=$(lspci |grep Eth |awk '{print $1}'|head -1) + cd ${DRIVER_ROOT} + sudo ./x86_64-native-linuxapp-gcc/app/testpmd -c 0x3f -n 4 -b $blacklist -- -a --nb-cores=4 --coremask=0x3c --burst=64 --txd=4096 --rxd=4096 --rxq=$TESTPMD_QUEUE --txq=$TESTPMD_QUEUE --rss-udp --eth-peer=0,$MOONGEN_PORT1_MAC --eth-peer=1,$MOONGEN_PORT2_MAC --forward-mode=mac +} + +main() +{ + load_modules + change_permissions + add_interface_to_dpdk + run_testpmd +} + +main diff --git a/yardstick/benchmark/scenarios/networking/moongen_testpmd.py b/yardstick/benchmark/scenarios/networking/moongen_testpmd.py new file mode 100644 index 000000000..e3bd7af46 --- /dev/null +++ b/yardstick/benchmark/scenarios/networking/moongen_testpmd.py @@ -0,0 +1,379 @@ +# Copyright (c) 2018 Huawei Technologies Co.,Ltd and others. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" VsperfDPDK specific scenario definition """ + +from __future__ import absolute_import +import pkg_resources +import logging +import subprocess +import time +import re +from oslo_serialization import jsonutils + +import yardstick.ssh as ssh +import yardstick.common.utils as utils +from yardstick.benchmark.scenarios import base + +LOG = logging.getLogger(__name__) + + +class MoongenTestPMD(base.Scenario): + """Execute vsperf with defined parameters + + Parameters: + frame_size - a frame size for which test should be executed; + Multiple frame sizes can be tested by modification of sequence runner + section inside TC YAML definition. + type: string + default: "64" + multistream - the number of simulated streams + type: string + default: 0 (disabled) + testpmd_queue - specifies how many queues you will use the VM + only useful when forward_type is true. + type: int + default: 1(one queue) + trafficgen_port1 - specifies device name of 1st interface connected to + the trafficgen + type: string + default: NA + trafficgen_port2 - specifies device name of 2nd interface connected to + the trafficgen + type: string + default: NA + moongen_host_user - specifies moongen host ssh user name + type: string + default: root + moongen_host_passwd - specifies moongen host ssh user password + type: string + default: root + moongen_host_ip - specifies moongen host ssh ip address + type: string + default NA + moongen_dir - specifies where is the moongen installtion dir + type: string + default NA + moongen_runBidirec - specifies moongen will run in one traffic + or two traffic. + type: string + default true + Package_Loss - specifies the package_Loss number in moongen server. + type: int + default 0(0%) + SearchRuntime - specifies the SearchRuntime and validation time + on moongen server. + type: int + default 60(s) + moongen_port1_mac - moongen server port1 mac address. + type: string + default NA + moongen_port2_mac - moongen server port2 mac address. + type: string + default NA + forward_type - VM forward type is l2fwd or testpmd. + type: string + default: testpmd + """ + __scenario_type__ = "MoongenTestPMD" + + TESTPMD_SCRIPT = 'moongen_testpmd.bash' + VSPERF_CONFIG = '/tmp/opnfv-vsperf-cfg.lua' + + def __init__(self, scenario_cfg, context_cfg): + self.scenario_cfg = scenario_cfg + self.context_cfg = context_cfg + self.forward_setup_done = False + self.options = scenario_cfg.get('options', {}) + self.moongen_host_user = \ + self.options.get('moongen_host_user', "root") + self.moongen_host_passwd = \ + self.options.get('moongen_host_passwd', "r00t") + self.moongen_dir = \ + self.options.get('moongen_dir', '~/moongen.py') + self.testpmd_queue = \ + self.options.get('testpmd_queue', 1) + self.moongen_host_ip = \ + self.options.get('moongen_host_ip', "127.0.0.1") + self.moongen_port1_mac = \ + self.options.get('moongen_port1_mac', None) + self.moongen_port2_mac = \ + self.options.get('moongen_port2_mac', None) + self.tg_port1 = \ + self.options.get('trafficgen_port1', "enp2s0f0") + self.tg_port2 = \ + self.options.get('trafficgen_port2', "enp2s0f1") + self.forward_type = \ + self.options.get('forward_type', 'testpmd') + self.tgen_port1_mac = None + self.tgen_port2_mac = None + + def setup(self): + """scenario setup""" + host = self.context_cfg['host'] + + task_id = self.scenario_cfg['task_id'] + context_number = task_id.split('-')[0] + self.tg_port1_nw = 'demo' + \ + "-" + context_number + "-" + \ + self.options.get('trafficgen_port1_nw', 'test2') + self.tg_port2_nw = 'demo' + \ + "-" + context_number + "-" + \ + self.options.get('trafficgen_port2_nw', 'test3') + + # copy vsperf conf to VM + self.client = ssh.SSH.from_node(host, defaults={"user": "ubuntu"}) + # traffic generation could last long + self.client.wait(timeout=1800) + + self.server = ssh.SSH( + self.moongen_host_user, + self.moongen_host_ip, + password=self.moongen_host_passwd + ) + # traffic generation could last long + self.server.wait(timeout=1800) + + self.setup_done = True + + def forward_setup(self): + """forward tool setup""" + + # setup forward loopback in VM + self.testpmd_script = pkg_resources.resource_filename( + 'yardstick.benchmark.scenarios.networking', + self.TESTPMD_SCRIPT) + + self.client._put_file_shell(self.testpmd_script, + '~/testpmd_vsperf.sh') + + # disable Address Space Layout Randomization (ASLR) + cmd = "echo 0 | sudo tee /proc/sys/kernel/randomize_va_space" + self.client.send_command(cmd) + + if not self._is_forward_setup(): + self.tgen_port1_ip = \ + utils.get_port_ip(self.client, self.tg_port1) + self.tgen_port1_mac = \ + utils.get_port_mac(self.client, self.tg_port1) + self.client.run("tee ~/.testpmd.ipaddr.port1 > /dev/null", + stdin=self.tgen_port1_ip) + self.client.run("tee ~/.testpmd.macaddr.port1 > /dev/null", + stdin=self.tgen_port1_mac) + self.tgen_port2_ip = \ + utils.get_port_ip(self.client, self.tg_port2) + self.tgen_port2_mac = \ + utils.get_port_mac(self.client, self.tg_port2) + self.client.run("tee ~/.testpmd.ipaddr.port2 > /dev/null", + stdin=self.tgen_port2_ip) + self.client.run("tee ~/.testpmd.macaddr.port2 > /dev/null", + stdin=self.tgen_port2_mac) + else: + cmd = "cat ~/.testpmd.macaddr.port1" + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + self.tgen_port1_mac = stdout + cmd = "cat ~/.testpmd.ipaddr.port1" + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + self.tgen_port1_ip = stdout + cmd = "cat ~/.testpmd.macaddr.port2" + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + self.tgen_port2_mac = stdout + cmd = "cat ~/.testpmd.ipaddr.port2" + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + self.tgen_port2_ip = stdout + + LOG.info("forward type is %s", self.forward_type) + if self.forward_type == 'testpmd': + cmd = "sudo ip link set %s down" % (self.tg_port1) + LOG.debug("Executing command: %s", cmd) + self.client.execute(cmd) + cmd = "sudo ip link set %s down" % (self.tg_port2) + LOG.debug("Executing command: %s", cmd) + self.client.execute(cmd) + cmd = "screen -d -m sudo -E bash ~/testpmd_vsperf.sh %s %s %d" % \ + (self.moongen_port1_mac, self.moongen_port2_mac, + self.testpmd_queue) + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + elif self.forward_type == 'l2fwd': + cmd = ('sed -i "s/static char *net1 = \\\"eth1\\\";' + '/static char *net1 = \\\"%s %s %s\\\";/g" /home/l2fwd/l2fwd.c' + % (self.tg_port1, self.tgen_port1_ip, self.moongen_port1_mac)) + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.client.execute(cmd) + + cmd = ('sed -i "s/static char *net2 = \\\"eth2\\\";' + '/static char *net2 = \\\"%s %s %s\\\";/g" /home/l2fwd/l2fwd.c' + % (self.tg_port2, self.tgen_port2_ip, self.moongen_port2_mac)) + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.client.execute(cmd) + + cmd = ('cd /home/l2fwd/;make;./gen_debian_package.sh;' + 'sudo dpkg -i *.deb;' + 'sudo modprobe l2fwd') + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.client.execute(cmd) + + time.sleep(1) + + self.forward_setup_done = True + + def _is_forward_setup(self): + """Is forward already setup in the host?""" + if self.forward_type is 'testpmd': + is_run = True + cmd = "ip a | grep %s 2>/dev/null" % (self.tg_port1) + LOG.debug("Executing command: %s", cmd) + _, stdout, _ = self.client.execute(cmd) + if stdout: + is_run = False + return is_run + elif self.forward_type is 'l2fwd': + cmd = ('sudo lsmod |grep l2fwd') + LOG.debug("Executing command: %s", cmd) + _, stdout, _ = self.client.execute(cmd) + if stdout: + return True + else: + return False + + def generate_config_file(self, frame_size, multistream, + runBidirec, tg_port1_vlan, tg_port2_vlan, + SearchRuntime, Package_Loss): + out_text = """\ +VSPERF { +testType = 'throughput', +nrFlows = %d, +runBidirec = %s, +frameSize = %d, +srcMacs = {\'%s\', \'%s\'}, +dstMacs = {\'%s\', \'%s\'}, +vlanIds = {%d, %d}, +searchRunTime = %d, +validationRunTime = %d, +acceptableLossPct = %d, +ports = {0,1}, +} +""" % (multistream, runBidirec, frame_size, self.moongen_port1_mac, + self.moongen_port2_mac, self.tgen_port1_mac, self.tgen_port2_mac, + tg_port1_vlan, tg_port2_vlan, SearchRuntime, SearchRuntime, Package_Loss) + with open(self.VSPERF_CONFIG, "wt") as out_file: + out_file.write(out_text) + self.CONFIG_FILE = True + + def result_to_data(self, result): + search_pattern = re.compile( + r'\[REPORT\]\s+total\:\s+' + r'Tx\s+frames\:\s+(\d+)\s+' + r'Rx\s+Frames\:\s+(\d+)\s+' + r'frame\s+loss\:\s+(\d+)\,' + r'\s+(\d+\.\d+|\d+)%\s+' + r'Tx\s+Mpps\:\s+(\d+.\d+|\d+)\s+' + r'Rx\s+Mpps\:\s+(\d+\.\d+|\d+)', + re.IGNORECASE) + results_match = search_pattern.search(result) + if results_match: + rx_mpps = float(results_match.group(6)) + tx_mpps = float(results_match.group(5)) + else: + rx_mpps = 0 + tx_mpps = 0 + test_result = {"rx_mpps": rx_mpps, "tx_mpps": tx_mpps} + self.TO_DATA = True + return test_result + + def run(self, result): + """ execute the vsperf benchmark and return test results + within result dictionary + """ + + if not self.setup_done: + self.setup() + + # get vsperf options + multistream = self.options.get("multistream", 1) + + if not self.forward_setup_done: + self.forward_setup() + + if 'frame_size' in self.options: + frame_size = self.options.get("frame_size", 64) + Package_Loss = self.options.get("Package_Loss", 0) + runBidirec = self.options.get("moongen_runBidirec", + "true") + SearchRuntime = self.options.get("SearchRuntime", 10) + + cmd = "openstack network show %s --format json -c " \ + "provider:segmentation_id" % (self.tg_port1_nw) + LOG.debug("Executing command: %s", cmd) + output = subprocess.check_output(cmd, shell=True) + try: + tg_port1_vlan = jsonutils.loads(output).get("provider:segmentation_id", 1) + except TypeError: + tg_port1_vlan = 1 + + cmd = "openstack network show %s --format json -c " \ + "provider:segmentation_id" % (self.tg_port2_nw) + LOG.debug("Executing command: %s", cmd) + output = subprocess.check_output(cmd, shell=True) + try: + tg_port2_vlan = jsonutils.loads(output).get("provider:segmentation_id", 2) + except TypeError: + tg_port2_vlan = 2 + + self.generate_config_file(frame_size, multistream, + runBidirec, tg_port1_vlan, + tg_port2_vlan, SearchRuntime, Package_Loss) + + self.server.execute("rm -f -- %s/opnfv-vsperf-cfg.lua" % + (self.moongen_dir)) + self.server._put_file_shell(self.VSPERF_CONFIG, + "%s/opnfv-vsperf-cfg.lua" + % (self.moongen_dir)) + + # execute moongen + cmd = ("cd %s;./MoonGen/build/MoonGen ./trafficgen.lua" + % (self.moongen_dir)) + status, stdout, stderr = self.server.execute(cmd) + if status: + raise RuntimeError(stderr) + + moongen_result = self.result_to_data(stdout) + LOG.info(moongen_result) + result.update(moongen_result) + + if "sla" in self.scenario_cfg: + throughput_rx_mpps = int( + self.scenario_cfg["sla"]["throughput_rx_mpps"]) + + self.verify_SLA( + throughput_rx_mpps <= moongen_result["tx_mpps"], + "sla_throughput_rx_mpps %f > throughput_rx_mpps(%f); " + % (throughput_rx_mpps, moongen_result["tx_mpps"])) + + def teardown(self): + """cleanup after the test execution""" + + # execute external setup script + self.setup_done = False diff --git a/yardstick/benchmark/scenarios/networking/netperf.py b/yardstick/benchmark/scenarios/networking/netperf.py index a8d9010ed..9f1a81413 100755 --- a/yardstick/benchmark/scenarios/networking/netperf.py +++ b/yardstick/benchmark/scenarios/networking/netperf.py @@ -104,7 +104,9 @@ class Netperf(base.Scenario): cmd_args = "-H %s -l %s -t %s" % (ipaddr, testlen, testname) # get test specific options - default_args = "-O 'THROUGHPUT,THROUGHPUT_UNITS,MEAN_LATENCY'" + output_opt = options.get( + "output_opt", "THROUGHPUT,THROUGHPUT_UNITS,MEAN_LATENCY") + default_args = "-O %s" % output_opt cmd_args += " -- %s" % default_args option_pair_list = [("send_msg_size", "-m"), ("recv_msg_size", "-M"), @@ -136,9 +138,9 @@ class Netperf(base.Scenario): sla_max_mean_latency = int( self.scenario_cfg["sla"]["mean_latency"]) - assert mean_latency <= sla_max_mean_latency, \ - "mean_latency %f > sla_max_mean_latency(%f); " % \ - (mean_latency, sla_max_mean_latency) + self.verify_SLA(mean_latency <= sla_max_mean_latency, + "mean_latency %f > sla_max_mean_latency(%f); " + % (mean_latency, sla_max_mean_latency)) def _test(): diff --git a/yardstick/benchmark/scenarios/networking/netperf_node.py b/yardstick/benchmark/scenarios/networking/netperf_node.py index d52e6b9e1..0ad2ecff5 100755 --- a/yardstick/benchmark/scenarios/networking/netperf_node.py +++ b/yardstick/benchmark/scenarios/networking/netperf_node.py @@ -156,9 +156,10 @@ class NetperfNode(base.Scenario): sla_max_mean_latency = int( self.scenario_cfg["sla"]["mean_latency"]) - assert mean_latency <= sla_max_mean_latency, \ - "mean_latency %f > sla_max_mean_latency(%f); " % \ - (mean_latency, sla_max_mean_latency) + self.verify_SLA( + mean_latency <= sla_max_mean_latency, + "mean_latency %f > sla_max_mean_latency(%f); " + % (mean_latency, sla_max_mean_latency)) def teardown(self): """remove netperf from nodes after test""" diff --git a/yardstick/benchmark/scenarios/networking/nstat.py b/yardstick/benchmark/scenarios/networking/nstat.py index 10c560769..ea067f8ab 100644 --- a/yardstick/benchmark/scenarios/networking/nstat.py +++ b/yardstick/benchmark/scenarios/networking/nstat.py @@ -121,4 +121,4 @@ class Nstat(base.Scenario): if rate > sla_rate: sla_error += "%s rate %f > sla:%s_rate(%f); " % \ (i, rate, i, sla_rate) - assert sla_error == "", sla_error + self.verify_SLA(sla_error == "", sla_error) diff --git a/yardstick/benchmark/scenarios/networking/ping.py b/yardstick/benchmark/scenarios/networking/ping.py index e7d9beea8..1c9510220 100644 --- a/yardstick/benchmark/scenarios/networking/ping.py +++ b/yardstick/benchmark/scenarios/networking/ping.py @@ -91,9 +91,10 @@ class Ping(base.Scenario): result.update(utils.flatten_dict_key(ping_result)) if sla_max_rtt is not None: sla_max_rtt = float(sla_max_rtt) - assert rtt_result[target_vm_name] <= sla_max_rtt,\ - "rtt %f > sla: max_rtt(%f); " % \ - (rtt_result[target_vm_name], sla_max_rtt) + self.verify_SLA( + rtt_result[target_vm_name] <= sla_max_rtt, + "rtt %f > sla: max_rtt(%f); " + % (rtt_result[target_vm_name], sla_max_rtt)) else: LOG.error("ping '%s' '%s' timeout", options, target_vm) # we need to specify a result to satisfy influxdb schema @@ -103,12 +104,13 @@ class Ping(base.Scenario): # store result before potential AssertionError result.update(utils.flatten_dict_key(ping_result)) if sla_max_rtt is not None: - raise AssertionError("packet dropped rtt {:f} > sla: max_rtt({:f})".format( - rtt_result[target_vm_name], sla_max_rtt)) - + self.verify_SLA(rtt_result[target_vm_name] <= sla_max_rtt, + "packet dropped rtt %f > sla: max_rtt(%f)" + % (rtt_result[target_vm_name], sla_max_rtt)) else: - raise AssertionError( - "packet dropped rtt {:f}".format(rtt_result[target_vm_name])) + self.verify_SLA(False, + "packet dropped rtt %f" + % (rtt_result[target_vm_name])) def _test(): # pragma: no cover diff --git a/yardstick/benchmark/scenarios/networking/ping6.py b/yardstick/benchmark/scenarios/networking/ping6.py index 74855a10f..377278004 100644 --- a/yardstick/benchmark/scenarios/networking/ping6.py +++ b/yardstick/benchmark/scenarios/networking/ping6.py @@ -59,8 +59,7 @@ class Ping6(base.Scenario): # pragma: no cover self._ssh_host(node_name) self.client._put_file_shell( self.pre_setup_script, '~/pre_setup.sh') - status, stdout, stderr = self.client.execute( - "sudo bash pre_setup.sh") + self.client.execute("sudo bash pre_setup.sh") def _get_controller_node(self, host_list): for host_name in host_list: @@ -122,7 +121,7 @@ class Ping6(base.Scenario): # pragma: no cover cmd = "sudo bash %s %s %s" % \ (setup_bash_file, self.openrc, self.external_network) LOG.debug("Executing setup command: %s", cmd) - status, stdout, stderr = self.client.execute(cmd) + self.client.execute(cmd) self.setup_done = True @@ -171,8 +170,9 @@ class Ping6(base.Scenario): # pragma: no cover result["rtt"] = float(stdout) if "sla" in self.scenario_cfg: sla_max_rtt = int(self.scenario_cfg["sla"]["max_rtt"]) - assert result["rtt"] <= sla_max_rtt, \ - "rtt %f > sla:max_rtt(%f); " % (result["rtt"], sla_max_rtt) + self.verify_SLA(result["rtt"] <= sla_max_rtt, + "rtt %f > sla:max_rtt(%f); " + % (result["rtt"], sla_max_rtt)) else: LOG.error("ping6 timeout!!!") self.run_done = True @@ -216,5 +216,4 @@ class Ping6(base.Scenario): # pragma: no cover self._ssh_host(node_name) self.client._put_file_shell( self.post_teardown_script, '~/post_teardown.sh') - status, stdout, stderr = self.client.execute( - "sudo bash post_teardown.sh") + self.client.execute("sudo bash post_teardown.sh") diff --git a/yardstick/benchmark/scenarios/networking/pktgen.py b/yardstick/benchmark/scenarios/networking/pktgen.py index b79b91539..c78108adb 100644 --- a/yardstick/benchmark/scenarios/networking/pktgen.py +++ b/yardstick/benchmark/scenarios/networking/pktgen.py @@ -87,7 +87,7 @@ class Pktgen(base.Scenario): self.server.send_command(cmd) self.client.send_command(cmd) - """multiqueue setup""" + # multiqueue setup if not self._is_irqbalance_disabled(): self._disable_irqbalance() @@ -112,18 +112,14 @@ class Pktgen(base.Scenario): def _get_vnic_driver_name(self): cmd = "readlink /sys/class/net/%s/device/driver" % self.vnic_name LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) return os.path.basename(stdout.strip()) def _is_irqbalance_disabled(self): """Did we disable irqbalance already in the guest?""" is_disabled = False cmd = "grep ENABLED /etc/default/irqbalance" - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) if "0" in stdout: is_disabled = True @@ -132,49 +128,35 @@ class Pktgen(base.Scenario): def _disable_irqbalance(self): cmd = "sudo sed -i -e 's/ENABLED=\"1\"/ENABLED=\"0\"/g' " \ "/etc/default/irqbalance" - status, stdout, stderr = self.server.execute(cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.server.run(cmd) + self.client.run(cmd) cmd = "sudo service irqbalance stop" - status, stdout, stderr = self.server.execute(cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.server.run(cmd) + self.client.run(cmd) cmd = "sudo service irqbalance disable" - status, stdout, stderr = self.server.execute(cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.server.run(cmd) + self.client.run(cmd) def _setup_irqmapping_ovs(self, queue_number): cmd = "grep 'virtio0-input.0' /proc/interrupts |" \ "awk '{match($0,/ +[0-9]+/)} " \ "{print substr($1,RSTART,RLENGTH-1)}'" - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout)) - status, stdout, stderr = self.server.execute(cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.server.run(cmd) + self.client.run(cmd) cmd = "grep 'virtio0-output.0' /proc/interrupts |" \ "awk '{match($0,/ +[0-9]+/)} " \ "{print substr($1,RSTART,RLENGTH-1)}'" - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout)) - status, stdout, stderr = self.server.execute(cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.server.run(cmd) + self.client.run(cmd) if queue_number == 1: return @@ -186,44 +168,32 @@ class Pktgen(base.Scenario): cmd = "grep 'virtio0-input.%s' /proc/interrupts |" \ "awk '{match($0,/ +[0-9]+/)} " \ "{print substr($1,RSTART,RLENGTH-1)}'" % (i) - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \ % (smp_affinity_mask, int(stdout)) - status, stdout, stderr = self.server.execute(cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.server.run(cmd) + self.client.run(cmd) cmd = "grep 'virtio0-output.%s' /proc/interrupts |" \ "awk '{match($0,/ +[0-9]+/)} " \ "{print substr($1,RSTART,RLENGTH-1)}'" % (i) - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \ % (smp_affinity_mask, int(stdout)) - status, stdout, stderr = self.server.execute(cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.server.run(cmd) + self.client.run(cmd) def _setup_irqmapping_sriov(self, queue_number): cmd = "grep '%s-TxRx-0' /proc/interrupts |" \ "awk '{match($0,/ +[0-9]+/)} " \ "{print substr($1,RSTART,RLENGTH-1)}'" % self.vnic_name - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) cmd = "echo 1 | sudo tee /proc/irq/%s/smp_affinity" % (int(stdout)) - status, stdout, stderr = self.server.execute(cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.server.run(cmd) + self.client.run(cmd) if queue_number == 1: return @@ -234,24 +204,18 @@ class Pktgen(base.Scenario): cmd = "grep '%s-TxRx-%s' /proc/interrupts |" \ "awk '{match($0,/ +[0-9]+/)} " \ "{print substr($1,RSTART,RLENGTH-1)}'" % (self.vnic_name, i) - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) cmd = "echo %s | sudo tee /proc/irq/%s/smp_affinity" \ % (smp_affinity_mask, int(stdout)) - status, stdout, stderr = self.server.execute(cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.server.run(cmd) + self.client.run(cmd) def _get_sriov_queue_number(self): """Get queue number from server as both VMs are the same""" cmd = "grep %s-TxRx- /proc/interrupts | wc -l" % self.vnic_name LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) return int(stdout) def _get_available_queue_number(self): @@ -259,9 +223,7 @@ class Pktgen(base.Scenario): cmd = "sudo ethtool -l %s | grep Combined | head -1 |" \ "awk '{printf $2}'" % self.vnic_name LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) return int(stdout) def _get_usable_queue_number(self): @@ -269,9 +231,7 @@ class Pktgen(base.Scenario): cmd = "sudo ethtool -l %s | grep Combined | tail -1 |" \ "awk '{printf $2}'" % self.vnic_name LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) return int(stdout) def _enable_ovs_multiqueue(self): @@ -282,10 +242,8 @@ class Pktgen(base.Scenario): cmd = "sudo ethtool -L %s combined %s" % \ (self.vnic_name, available_queue_number) LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.server.execute(cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.server.run(cmd) + self.client.run(cmd) return available_queue_number def _iptables_setup(self): @@ -294,9 +252,7 @@ class Pktgen(base.Scenario): "sudo iptables -A INPUT -p udp --dport 1000:%s -j DROP" \ % (1000 + self.number_of_ports) LOG.debug("Executing command: %s", cmd) - status, _, stderr = self.server.execute(cmd, timeout=SSH_TIMEOUT) - if status: - raise RuntimeError(stderr) + self.server.run(cmd, timeout=SSH_TIMEOUT) def _iptables_get_result(self): """Get packet statistics from server""" @@ -304,9 +260,7 @@ class Pktgen(base.Scenario): "awk '/dpts:1000:%s/ {{printf \"%%s\", $1}}'" \ % (1000 + self.number_of_ports) LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.server.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.server.execute(cmd, raise_on_error=True) return int(stdout) def run(self, result): @@ -356,10 +310,8 @@ class Pktgen(base.Scenario): duration, queue_number, pps) LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.client.execute(cmd, timeout=SSH_TIMEOUT) - - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.client.execute(cmd, raise_on_error=True, + timeout=SSH_TIMEOUT) result.update(jsonutils.loads(stdout)) @@ -374,8 +326,8 @@ class Pktgen(base.Scenario): if "sla" in self.scenario_cfg: LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm) sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"]) - assert ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " \ - % (ppm, sla_max_ppm) + self.verify_SLA(ppm <= sla_max_ppm, + "ppm %d > sla_max_ppm %d; " % (ppm, sla_max_ppm)) def _test(): # pragma: no cover diff --git a/yardstick/benchmark/scenarios/networking/pktgen_dpdk.py b/yardstick/benchmark/scenarios/networking/pktgen_dpdk.py index ce8a7f497..efb7d8b5d 100644 --- a/yardstick/benchmark/scenarios/networking/pktgen_dpdk.py +++ b/yardstick/benchmark/scenarios/networking/pktgen_dpdk.py @@ -70,39 +70,42 @@ class PktgenDPDKLatency(base.Scenario): def run(self, result): """execute the benchmark""" + options = self.scenario_cfg['options'] + eth1 = options.get("eth1", "ens4") + eth2 = options.get("eth2", "ens5") if not self.setup_done: self.setup() if not self.testpmd_args: - self.testpmd_args = utils.get_port_mac(self.client, 'eth2') + self.testpmd_args = utils.get_port_mac(self.client, eth2) if not self.pktgen_args: - server_rev_mac = utils.get_port_mac(self.server, 'eth1') - server_send_mac = utils.get_port_mac(self.server, 'eth2') - client_src_ip = utils.get_port_ip(self.client, 'eth1') - client_dst_ip = utils.get_port_ip(self.client, 'eth2') + server_rev_mac = utils.get_port_mac(self.server, eth1) + server_send_mac = utils.get_port_mac(self.server, eth2) + client_src_ip = utils.get_port_ip(self.client, eth1) + client_dst_ip = utils.get_port_ip(self.client, eth2) self.pktgen_args = [client_src_ip, client_dst_ip, server_rev_mac, server_send_mac] - options = self.scenario_cfg['options'] packetsize = options.get("packetsize", 64) rate = options.get("rate", 100) - cmd = "screen sudo -E bash ~/testpmd_fwd.sh %s " % (self.testpmd_args) + cmd = "screen sudo -E bash ~/testpmd_fwd.sh %s %s %s" % \ + (self.testpmd_args, eth1, eth2) LOG.debug("Executing command: %s", cmd) self.server.send_command(cmd) time.sleep(1) - cmd = "screen sudo -E bash ~/pktgen_dpdk.sh %s %s %s %s %s %s" % \ + cmd = "screen sudo -E bash ~/pktgen_dpdk.sh %s %s %s %s %s %s %s %s" % \ (self.pktgen_args[0], self.pktgen_args[1], self.pktgen_args[2], - self.pktgen_args[3], rate, packetsize) + self.pktgen_args[3], rate, packetsize, eth1, eth2) LOG.debug("Executing command: %s", cmd) self.client.send_command(cmd) # wait for finishing test - time.sleep(1) + time.sleep(60) cmd = r"""\ cat ~/result.log -vT \ @@ -110,10 +113,7 @@ cat ~/result.log -vT \ {print substr($0,RSTART,RLENGTH)}' \ |grep -v ^$ |awk '{if ($2 != 0) print $2}'\ """ - client_status, client_stdout, client_stderr = self.client.execute(cmd) - - if client_status: - raise RuntimeError(client_stderr) + _, client_stdout, _ = self.client.execute(cmd, raise_on_error=True) avg_latency = 0 if client_stdout: @@ -132,4 +132,4 @@ cat ~/result.log -vT \ LOG.info("sla_max_latency: %d", sla_max_latency) debug_info = "avg_latency %d > sla_max_latency %d" \ % (avg_latency, sla_max_latency) - assert avg_latency <= sla_max_latency, debug_info + self.verify_SLA(avg_latency <= sla_max_latency, debug_info) diff --git a/yardstick/benchmark/scenarios/networking/pktgen_dpdk_latency_benchmark.bash b/yardstick/benchmark/scenarios/networking/pktgen_dpdk_latency_benchmark.bash index b872aa3df..dcd5a9bfb 100644 --- a/yardstick/benchmark/scenarios/networking/pktgen_dpdk_latency_benchmark.bash +++ b/yardstick/benchmark/scenarios/networking/pktgen_dpdk_latency_benchmark.bash @@ -7,7 +7,7 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -!/bin/sh +#!/bin/sh set -e @@ -18,6 +18,11 @@ FWD_REV_MAC=$3 # MAC address of forwarding receiver in VM B FWD_SEND_MAC=$4 # MAC address of forwarding sender in VM B RATE=$5 # packet rate in percentage PKT_SIZE=$6 # packet size +ETH1=$7 +ETH2=$8 + +DPDK_VERSION="dpdk-17.02" +PKTGEN_VERSION="pktgen-3.2.12" load_modules() @@ -31,13 +36,13 @@ load_modules() if lsmod | grep "igb_uio" &> /dev/null ; then echo "igb_uio module is loaded" else - insmod /dpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko + insmod /opt/tempT/$DPDK_VERSION/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko fi if lsmod | grep "rte_kni" &> /dev/null ; then echo "rte_kni module is loaded" else - insmod /dpdk/x86_64-native-linuxapp-gcc/kmod/rte_kni.ko + insmod /opt/tempT/$DPDK_VERSION/x86_64-native-linuxapp-gcc/kmod/rte_kni.ko fi } @@ -48,8 +53,10 @@ change_permissions() } add_interface_to_dpdk(){ + ip link set $ETH1 down + ip link set $ETH2 down interfaces=$(lspci |grep Eth |tail -n +2 |awk '{print $1}') - /dpdk/tools/dpdk-devbind.py --bind=igb_uio $interfaces + /opt/tempT/$DPDK_VERSION/usertools/dpdk-devbind.py --bind=igb_uio $interfaces } @@ -106,20 +113,14 @@ spawn ./app/app/x86_64-native-linuxapp-gcc/pktgen -c 0x07 -n 4 -b $blacklist -- expect "Pktgen>" send "\n" expect "Pktgen>" -send "screen on\n" +send "on\n" expect "Pktgen>" set count 10 while { $count } { send "page latency\n" - expect { - timeout { send "\n" } - -regexp {..*} { - set result "${result}$expect_out(0,string)" - set timeout 1 - exp_continue - } - "Pktgen>" - } + expect -re "(..*)" + set result "${result}$expect_out(0,string)" + set timeout 1 set count [expr $count-1] } send "stop 0\n" @@ -136,7 +137,7 @@ EOF run_pktgen() { blacklist=$(lspci |grep Eth |awk '{print $1}'|head -1) - cd /pktgen-dpdk + cd /opt/tempT/$PKTGEN_VERSION touch /home/ubuntu/result.log result_log="/home/ubuntu/result.log" sudo expect /home/ubuntu/pktgen.exp $blacklist $result_log @@ -153,4 +154,3 @@ main() } main - diff --git a/yardstick/benchmark/scenarios/networking/pktgen_dpdk_throughput.py b/yardstick/benchmark/scenarios/networking/pktgen_dpdk_throughput.py index 497e59ee8..97b9cf73f 100644 --- a/yardstick/benchmark/scenarios/networking/pktgen_dpdk_throughput.py +++ b/yardstick/benchmark/scenarios/networking/pktgen_dpdk_throughput.py @@ -143,11 +143,11 @@ class PktgenDPDK(base.Scenario): cmd = "ip a | grep eth1 2>/dev/null" LOG.debug("Executing command: %s in %s", cmd, host) if "server" in host: - status, stdout, stderr = self.server.execute(cmd) + _, stdout, _ = self.server.execute(cmd) if stdout: is_run = False else: - status, stdout, stderr = self.client.execute(cmd) + _, stdout, _ = self.client.execute(cmd) if stdout: is_run = False @@ -222,5 +222,5 @@ class PktgenDPDK(base.Scenario): ppm += (sent - received) % sent > 0 LOG.debug("Lost packets %d - Lost ppm %d", (sent - received), ppm) sla_max_ppm = int(self.scenario_cfg["sla"]["max_ppm"]) - assert ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " \ - % (ppm, sla_max_ppm) + self.verify_SLA(ppm <= sla_max_ppm, "ppm %d > sla_max_ppm %d; " + % (ppm, sla_max_ppm)) diff --git a/yardstick/benchmark/scenarios/networking/sfc_openstack.py b/yardstick/benchmark/scenarios/networking/sfc_openstack.py index d5feabbbe..aaab2131a 100644 --- a/yardstick/benchmark/scenarios/networking/sfc_openstack.py +++ b/yardstick/benchmark/scenarios/networking/sfc_openstack.py @@ -34,11 +34,13 @@ def get_credentials(service): # pragma: no cover # The most common way to pass these info to the script is to do it through # environment variables. + # NOTE(ralonsoh): OS_TENANT_NAME is deprecated. + project_name = os.environ.get('OS_PROJECT_NAME', 'admin') creds.update({ "username": os.environ.get('OS_USERNAME', "admin"), password: os.environ.get("OS_PASSWORD", 'admin'), "auth_url": os.environ.get("OS_AUTH_URL"), - tenant: os.environ.get("OS_TENANT_NAME", "admin"), + tenant: os.environ.get("OS_TENANT_NAME", project_name), }) cacert = os.environ.get("OS_CACERT") if cacert is not None: @@ -59,7 +61,7 @@ def get_instances(nova_client): # pragma: no cover try: instances = nova_client.servers.list(search_opts={'all_tenants': 1}) return instances - except Exception as e: + except Exception as e: # pylint: disable=broad-except print("Error [get_instances(nova_client)]:", e) return None @@ -72,7 +74,7 @@ def get_SFs(nova_client): # pragma: no cover if "sfc_test" not in instance.name: SFs.append(instance) return SFs - except Exception as e: + except Exception as e: # pylint: disable=broad-except print("Error [get_SFs(nova_client)]:", e) return None @@ -93,7 +95,7 @@ def create_floating_ips(neutron_client): # pragma: no cover ip_json = neutron_client.create_floatingip({'floatingip': props}) fip_addr = ip_json['floatingip']['floating_ip_address'] ips.append(fip_addr) - except Exception as e: + except Exception as e: # pylint: disable=broad-except print("Error [create_floating_ip(neutron_client)]:", e) return None return ips @@ -106,7 +108,7 @@ def floatIPtoSFs(SFs, floatips): # pragma: no cover SF.add_floating_ip(floatips[i]) i = i + 1 return True - except Exception as e: + except Exception as e: # pylint: disable=broad-except print(("Error [add_floating_ip(nova_client, '%s', '%s')]:" % (SF, floatips[i]), e)) return False @@ -122,7 +124,3 @@ def get_an_IP(): # pragma: no cover floatips = create_floating_ips(neutron_client) floatIPtoSFs(SFs, floatips) return floatips - - -if __name__ == '__main__': # pragma: no cover - get_an_IP() diff --git a/yardstick/benchmark/scenarios/networking/testpmd_fwd.bash b/yardstick/benchmark/scenarios/networking/testpmd_fwd.bash index 247a8a833..30b63a734 100644 --- a/yardstick/benchmark/scenarios/networking/testpmd_fwd.bash +++ b/yardstick/benchmark/scenarios/networking/testpmd_fwd.bash @@ -13,6 +13,10 @@ set -e # Commandline arguments DST_MAC=$1 # MAC address of the peer port +ETH1=$2 +ETH2=$3 + +DPDK_VERSION="dpdk-17.02" load_modules() { @@ -25,13 +29,13 @@ load_modules() if lsmod | grep "igb_uio" &> /dev/null ; then echo "igb_uio module is loaded" else - insmod /dpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko + insmod /opt/tempT/$DPDK_VERSION/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko fi if lsmod | grep "rte_kni" &> /dev/null ; then echo "rte_kni module is loaded" else - insmod /dpdk/x86_64-native-linuxapp-gcc/kmod/rte_kni.ko + insmod /opt/tempT/$DPDK_VERSION/x86_64-native-linuxapp-gcc/kmod/rte_kni.ko fi } @@ -42,15 +46,17 @@ change_permissions() } add_interface_to_dpdk(){ + ip link set $ETH1 down + ip link set $ETH2 down interfaces=$(lspci |grep Eth |tail -n +2 |awk '{print $1}') - /dpdk/tools/dpdk-devbind.py --bind=igb_uio $interfaces + /opt/tempT/$DPDK_VERSION/usertools//dpdk-devbind.py --bind=igb_uio $interfaces } run_testpmd() { blacklist=$(lspci |grep Eth |awk '{print $1}'|head -1) - cd /dpdk - sudo ./destdir/bin/testpmd -c 0x07 -n 4 -b $blacklist -- -a --eth-peer=1,$DST_MAC --forward-mode=mac + cd /opt/tempT/$DPDK_VERSION/x86_64-native-linuxapp-gcc/app + sudo ./testpmd -c 0x07 -n 4 -b $blacklist -- -a --eth-peer=1,$DST_MAC --forward-mode=mac } main() diff --git a/yardstick/benchmark/scenarios/networking/vnf_generic.py b/yardstick/benchmark/scenarios/networking/vnf_generic.py index b94bfc9ab..c5e75d093 100644 --- a/yardstick/benchmark/scenarios/networking/vnf_generic.py +++ b/yardstick/benchmark/scenarios/networking/vnf_generic.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016-2017 Intel Corporation +# Copyright (c) 2016-2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,168 +11,116 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" NSPerf specific scenario definition """ - -from __future__ import absolute_import - -import logging -import errno - -import ipaddress import copy +import ipaddress +from itertools import chain +import logging import os import sys -import re -from itertools import chain +import time import six import yaml -from collections import defaultdict -from yardstick.benchmark.scenarios import base +from yardstick.benchmark.contexts import base as context_base +from yardstick.benchmark.scenarios import base as scenario_base from yardstick.common.constants import LOG_DIR +from yardstick.common import exceptions from yardstick.common.process import terminate_children -from yardstick.common.utils import import_modules_from_package, itersubclasses -from yardstick.common.yaml_loader import yaml_load +from yardstick.common import utils from yardstick.network_services.collector.subscriber import Collector from yardstick.network_services.vnf_generic import vnfdgen from yardstick.network_services.vnf_generic.vnf.base import GenericVNF -from yardstick.network_services.traffic_profile.base import TrafficProfile +from yardstick.network_services import traffic_profile +from yardstick.network_services.traffic_profile import base as tprofile_base from yardstick.network_services.utils import get_nsb_option from yardstick import ssh -LOG = logging.getLogger(__name__) - - -class SSHError(Exception): - """Class handles ssh connection error exception""" - pass - - -class SSHTimeout(SSHError): - """Class handles ssh connection timeout exception""" - pass +traffic_profile.register_modules() -class IncorrectConfig(Exception): - """Class handles incorrect configuration during setup""" - pass - - -class IncorrectSetup(Exception): - """Class handles incorrect setup during setup""" - pass - - -class SshManager(object): - def __init__(self, node, timeout=120): - super(SshManager, self).__init__() - self.node = node - self.conn = None - self.timeout = timeout - - def __enter__(self): - """ - args -> network device mappings - returns -> ssh connection ready to be used - """ - try: - self.conn = ssh.SSH.from_node(self.node) - self.conn.wait(timeout=self.timeout) - except SSHError as error: - LOG.info("connect failed to %s, due to %s", self.node["ip"], error) - # self.conn defaults to None - return self.conn - - def __exit__(self, exc_type, exc_val, exc_tb): - if self.conn: - self.conn.close() - - -def find_relative_file(path, task_path): - """ - Find file in one of places: in abs of path or - relative to TC scenario file. In this order. - - :param path: - :param task_path: - :return str: full path to file - """ - # fixme: create schema to validate all fields have been provided - for lookup in [os.path.abspath(path), os.path.join(task_path, path)]: - try: - with open(lookup): - return lookup - except IOError: - pass - raise IOError(errno.ENOENT, 'Unable to find {} file'.format(path)) - - -def open_relative_file(path, task_path): - try: - return open(path) - except IOError as e: - if e.errno == errno.ENOENT: - return open(os.path.join(task_path, path)) - raise +LOG = logging.getLogger(__name__) -class NetworkServiceTestCase(base.Scenario): - """Class handles Generic framework to do pre-deployment VNF & - Network service testing """ +class NetworkServiceBase(scenario_base.Scenario): + """Base class for Network service testing scenarios""" - __scenario_type__ = "NSPerf" + __scenario_type__ = "" - def __init__(self, scenario_cfg, context_cfg): # Yardstick API - super(NetworkServiceTestCase, self).__init__() + def __init__(self, scenario_cfg, context_cfg): # pragma: no cover + super(NetworkServiceBase, self).__init__() self.scenario_cfg = scenario_cfg self.context_cfg = context_cfg - # fixme: create schema to validate all fields have been provided - with open_relative_file(scenario_cfg["topology"], - scenario_cfg['task_path']) as stream: - topology_yaml = yaml_load(stream) - - self.topology = topology_yaml["nsd:nsd-catalog"]["nsd"][0] + self._render_topology() self.vnfs = [] self.collector = None self.traffic_profile = None self.node_netdevs = {} + self.bin_path = get_nsb_option('bin_path', '') + + def run(self, *args): + pass + + def teardown(self): + """ Stop the collector and terminate VNF & TG instance + + :return + """ + + try: + try: + self.collector.stop() + for vnf in self.vnfs: + LOG.info("Stopping %s", vnf.name) + vnf.terminate() + LOG.debug("all VNFs terminated: %s", ", ".join(vnf.name for vnf in self.vnfs)) + finally: + terminate_children() + except Exception: + # catch any exception in teardown and convert to simple exception + # never pass exceptions back to multiprocessing, because some exceptions can + # be unpicklable + # https://bugs.python.org/issue9400 + LOG.exception("") + raise RuntimeError("Error in teardown") + + def is_ended(self): + return self.traffic_profile is not None and self.traffic_profile.is_ended() def _get_ip_flow_range(self, ip_start_range): + """Retrieve a CIDR first and last viable IPs - # IP range is specified as 'x.x.x.x-y.y.y.y' + :param ip_start_range: could be the IP range itself or a dictionary + with the host name and the port. + :return: (str) IP range (min, max) with this format "x.x.x.x-y.y.y.y" + """ if isinstance(ip_start_range, six.string_types): return ip_start_range - node_name, range_or_interface = next(iter(ip_start_range.items()), (None, '0.0.0.0')) + node_name, range_or_interface = next(iter(ip_start_range.items()), + (None, '0.0.0.0')) if node_name is None: - # we are manually specifying the range - ip_addr_range = range_or_interface + return range_or_interface + + node = self.context_cfg['nodes'].get(node_name, {}) + interface = node.get('interfaces', {}).get(range_or_interface) + if interface: + ip = interface['local_ip'] + mask = interface['netmask'] else: - node = self.context_cfg["nodes"].get(node_name, {}) - try: - # the ip_range is the interface name - interface = node.get("interfaces", {})[range_or_interface] - except KeyError: - ip = "0.0.0.0" - mask = "255.255.255.0" - else: - ip = interface["local_ip"] - # we can't default these values, they must both exist to be valid - mask = interface["netmask"] - - ipaddr = ipaddress.ip_network(six.text_type('{}/{}'.format(ip, mask)), strict=False) - hosts = list(ipaddr.hosts()) - if len(hosts) > 2: - # skip the first host in case of gateway - ip_addr_range = "{}-{}".format(hosts[1], hosts[-1]) - else: - LOG.warning("Only single IP in range %s", ipaddr) - # fall back to single IP range - ip_addr_range = ip + ip = '0.0.0.0' + mask = '255.255.255.0' + + ipaddr = ipaddress.ip_network( + six.text_type('{}/{}'.format(ip, mask)), strict=False) + if ipaddr.prefixlen + 2 < ipaddr.max_prefixlen: + ip_addr_range = '{}-{}'.format(ipaddr[2], ipaddr[-2]) + else: + LOG.warning('Only single IP in range %s', ipaddr) + ip_addr_range = ip return ip_addr_range def _get_traffic_flow(self): @@ -196,7 +144,15 @@ class NetworkServiceTestCase(base.Scenario): for index, dst_port in enumerate(fflow.get("dst_port", [])): flow["dst_port_{}".format(index)] = dst_port - flow["count"] = fflow["count"] + if "count" in fflow: + flow["count"] = fflow["count"] + + if "srcseed" in fflow: + flow["srcseed"] = fflow["srcseed"] + + if "dstseed" in fflow: + flow["dstseed"] = fflow["dstseed"] + except KeyError: flow = {} return {"flow": flow} @@ -208,43 +164,95 @@ class NetworkServiceTestCase(base.Scenario): imix = {} return imix + def _get_ip_priority(self): + try: + priority = self.scenario_cfg['options']['priority'] + except KeyError: + priority = {} + return priority + def _get_traffic_profile(self): profile = self.scenario_cfg["traffic_profile"] path = self.scenario_cfg["task_path"] - with open_relative_file(profile, path) as infile: + with utils.open_relative_file(profile, path) as infile: return infile.read() + def _get_duration(self): + options = self.scenario_cfg.get('options', {}) + return options.get('duration', + tprofile_base.TrafficProfileConfig.DEFAULT_DURATION) + + def _key_list_to_dict(self, key, value_list): + value_dict = {} + try: + for index, count in enumerate(value_list[key]): + value_dict["{}_{}".format(key, index)] = count + except KeyError: + value_dict = {} + + return value_dict + + def _get_simulated_users(self): + users = self.scenario_cfg.get("options", {}).get("simulated_users", {}) + simulated_users = self._key_list_to_dict("uplink", users) + return {"simulated_users": simulated_users} + + def _get_page_object(self): + objects = self.scenario_cfg.get("options", {}).get("page_object", {}) + page_object = self._key_list_to_dict("uplink", objects) + return {"page_object": page_object} + def _fill_traffic_profile(self): - traffic_mapping = self._get_traffic_profile() - traffic_map_data = { + tprofile = self._get_traffic_profile() + extra_args = self.scenario_cfg.get('extra_args', {}) + tprofile_data = { 'flow': self._get_traffic_flow(), 'imix': self._get_traffic_imix(), - TrafficProfile.UPLINK: {}, - TrafficProfile.DOWNLINK: {}, - } + 'priority': self._get_ip_priority(), + tprofile_base.TrafficProfile.UPLINK: {}, + tprofile_base.TrafficProfile.DOWNLINK: {}, + 'extra_args': extra_args, + 'duration': self._get_duration(), + 'page_object': self._get_page_object(), + 'simulated_users': self._get_simulated_users()} + traffic_vnfd = vnfdgen.generate_vnfd(tprofile, tprofile_data) + + traffic_config = \ + self.scenario_cfg.get("options", {}).get("traffic_config", {}) + + traffic_vnfd.setdefault("traffic_profile", {}) + traffic_vnfd["traffic_profile"].update(traffic_config) + + self.traffic_profile = \ + tprofile_base.TrafficProfile.get(traffic_vnfd) + + def _get_topology(self): + topology = self.scenario_cfg["topology"] + path = self.scenario_cfg["task_path"] + with utils.open_relative_file(topology, path) as infile: + return infile.read() - traffic_vnfd = vnfdgen.generate_vnfd(traffic_mapping, traffic_map_data) - self.traffic_profile = TrafficProfile.get(traffic_vnfd) - return self.traffic_profile + def _render_topology(self): + topology = self._get_topology() + topology_args = self.scenario_cfg.get('extra_args', {}) + topolgy_data = { + 'extra_args': topology_args + } + topology_yaml = vnfdgen.generate_vnfd(topology, topolgy_data) + self.topology = topology_yaml["nsd:nsd-catalog"]["nsd"][0] - def _find_vnf_name_from_id(self, vnf_id): + def _find_vnf_name_from_id(self, vnf_id): # pragma: no cover return next((vnfd["vnfd-id-ref"] for vnfd in self.topology["constituent-vnfd"] if vnf_id == vnfd["member-vnf-index"]), None) - @staticmethod - def get_vld_networks(networks): - # network name is vld_id - vld_map = {} - for name, n in networks.items(): - try: - vld_map[n['vld_id']] = n - except KeyError: - vld_map[name] = n - return vld_map + def _find_vnfd_from_vnf_idx(self, vnf_id): # pragma: no cover + return next((vnfd + for vnfd in self.topology["constituent-vnfd"] + if vnf_id == vnfd["member-vnf-index"]), None) @staticmethod - def find_node_if(nodes, name, if_name, vld_id): + def find_node_if(nodes, name, if_name, vld_id): # pragma: no cover try: # check for xe0, xe1 intf = nodes[name]["interfaces"][if_name] @@ -260,8 +268,9 @@ class NetworkServiceTestCase(base.Scenario): try: node0_data, node1_data = vld["vnfd-connection-point-ref"] except (ValueError, TypeError): - raise IncorrectConfig("Topology file corrupted, " - "wrong endpoint count for connection") + raise exceptions.IncorrectConfig( + error_msg='Topology file corrupted, wrong endpoint count ' + 'for connection') node0_name = self._find_vnf_name_from_id(node0_data["member-vnf-index-ref"]) node1_name = self._find_vnf_name_from_id(node1_data["member-vnf-index-ref"]) @@ -293,7 +302,9 @@ class NetworkServiceTestCase(base.Scenario): node1_if["peer_ifname"] = node0_if_name # just load the network - vld_networks = self.get_vld_networks(self.context_cfg["networks"]) + vld_networks = {n.get('vld_id', name): n for name, n in + self.context_cfg["networks"].items()} + node0_if["network"] = vld_networks.get(vld["id"], {}) node1_if["network"] = vld_networks.get(vld["id"], {}) @@ -305,15 +316,17 @@ class NetworkServiceTestCase(base.Scenario): except KeyError: LOG.exception("") - raise IncorrectConfig("Required interface not found, " - "topology file corrupted") + raise exceptions.IncorrectConfig( + error_msg='Required interface not found, topology file ' + 'corrupted') for vld in self.topology['vld']: try: node0_data, node1_data = vld["vnfd-connection-point-ref"] except (ValueError, TypeError): - raise IncorrectConfig("Topology file corrupted, " - "wrong endpoint count for connection") + raise exceptions.IncorrectConfig( + error_msg='Topology file corrupted, wrong endpoint count ' + 'for connection') node0_name = self._find_vnf_name_from_id(node0_data["member-vnf-index-ref"]) node1_name = self._find_vnf_name_from_id(node1_data["member-vnf-index-ref"]) @@ -332,55 +345,14 @@ class NetworkServiceTestCase(base.Scenario): node0_if["peer_intf"] = node1_copy node1_if["peer_intf"] = node0_copy - def _find_vnfd_from_vnf_idx(self, vnf_idx): - return next((vnfd for vnfd in self.topology["constituent-vnfd"] - if vnf_idx == vnfd["member-vnf-index"]), None) - - def _update_context_with_topology(self): + def _update_context_with_topology(self): # pragma: no cover for vnfd in self.topology["constituent-vnfd"]: vnf_idx = vnfd["member-vnf-index"] vnf_name = self._find_vnf_name_from_id(vnf_idx) vnfd = self._find_vnfd_from_vnf_idx(vnf_idx) self.context_cfg["nodes"][vnf_name].update(vnfd) - def _probe_netdevs(self, node, node_dict, timeout=120): - try: - return self.node_netdevs[node] - except KeyError: - pass - - netdevs = {} - cmd = "PATH=$PATH:/sbin:/usr/sbin ip addr show" - - with SshManager(node_dict, timeout=timeout) as conn: - if conn: - exit_status = conn.execute(cmd)[0] - if exit_status != 0: - raise IncorrectSetup("Node's %s lacks ip tool." % node) - exit_status, stdout, _ = conn.execute( - self.FIND_NETDEVICE_STRING) - if exit_status != 0: - raise IncorrectSetup( - "Cannot find netdev info in sysfs" % node) - netdevs = node_dict['netdevs'] = self.parse_netdev_info(stdout) - - self.node_netdevs[node] = netdevs - return netdevs - - @classmethod - def _probe_missing_values(cls, netdevs, network): - - mac_lower = network['local_mac'].lower() - for netdev in netdevs.values(): - if netdev['address'].lower() != mac_lower: - continue - network.update({ - 'driver': netdev['driver'], - 'vpci': netdev['pci_bus_id'], - 'ifindex': netdev['ifindex'], - }) - - def _generate_pod_yaml(self): + def _generate_pod_yaml(self): # pragma: no cover context_yaml = os.path.join(LOG_DIR, "pod-{}.yaml".format(self.scenario_cfg['task_id'])) # convert OrderedDict to a list # pod.yaml nodes is a list @@ -394,7 +366,7 @@ class NetworkServiceTestCase(base.Scenario): explicit_start=True) @staticmethod - def _serialize_node(node): + def _serialize_node(node): # pragma: no cover new_node = copy.deepcopy(node) # name field is required # remove context suffix @@ -405,96 +377,31 @@ class NetworkServiceTestCase(base.Scenario): pass return new_node - TOPOLOGY_REQUIRED_KEYS = frozenset({ - "vpci", "local_ip", "netmask", "local_mac", "driver"}) - def map_topology_to_infrastructure(self): """ This method should verify if the available resources defined in pod.yaml match the topology.yaml file. :return: None. Side effect: context_cfg is updated """ - num_nodes = len(self.context_cfg["nodes"]) - # OpenStack instance creation time is probably proportional to the number - # of instances - timeout = 120 * num_nodes - for node, node_dict in self.context_cfg["nodes"].items(): - - for network in node_dict["interfaces"].values(): - missing = self.TOPOLOGY_REQUIRED_KEYS.difference(network) - if not missing: - continue - - # only ssh probe if there are missing values - # ssh probe won't work on Ixia, so we had better define all our values - try: - netdevs = self._probe_netdevs(node, node_dict, timeout=timeout) - except (SSHError, SSHTimeout): - raise IncorrectConfig( - "Unable to probe missing interface fields '%s', on node %s " - "SSH Error" % (', '.join(missing), node)) - try: - self._probe_missing_values(netdevs, network) - except KeyError: - pass - else: - missing = self.TOPOLOGY_REQUIRED_KEYS.difference( - network) - if missing: - raise IncorrectConfig( - "Require interface fields '%s' not found, topology file " - "corrupted" % ', '.join(missing)) - - # we have to generate pod.yaml here so we have vpci and driver - self._generate_pod_yaml() # 3. Use topology file to find connections & resolve dest address self._resolve_topology() self._update_context_with_topology() - FIND_NETDEVICE_STRING = r"""find /sys/devices/pci* -type d -name net -exec sh -c '{ grep -sH ^ \ -$1/ifindex $1/address $1/operstate $1/device/vendor $1/device/device \ -$1/device/subsystem_vendor $1/device/subsystem_device ; \ -printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ -' sh \{\}/* \; -""" - BASE_ADAPTER_RE = re.compile( - '^/sys/devices/(.*)/net/([^/]*)/([^:]*):(.*)$', re.M) - - @classmethod - def parse_netdev_info(cls, stdout): - network_devices = defaultdict(dict) - matches = cls.BASE_ADAPTER_RE.findall(stdout) - for bus_path, interface_name, name, value in matches: - dirname, bus_id = os.path.split(bus_path) - if 'virtio' in bus_id: - # for some stupid reason VMs include virtio1/ - # in PCI device path - bus_id = os.path.basename(dirname) - # remove extra 'device/' from 'device/vendor, - # device/subsystem_vendor', etc. - if 'device/' in name: - name = name.split('/')[1] - network_devices[interface_name][name] = value - network_devices[interface_name][ - 'interface_name'] = interface_name - network_devices[interface_name]['pci_bus_id'] = bus_id - # convert back to regular dict - return dict(network_devices) - @classmethod - def get_vnf_impl(cls, vnf_model_id): + def get_vnf_impl(cls, vnf_model_id): # pragma: no cover """ Find the implementing class from vnf_model["vnf"]["name"] field :param vnf_model_id: parsed vnfd model ID field :return: subclass of GenericVNF """ - import_modules_from_package( + utils.import_modules_from_package( "yardstick.network_services.vnf_generic.vnf") expected_name = vnf_model_id classes_found = [] def impl(): - for name, class_ in ((c.__name__, c) for c in itersubclasses(GenericVNF)): + for name, class_ in ((c.__name__, c) for c in + utils.itersubclasses(GenericVNF)): if name == expected_name: yield class_ classes_found.append(name) @@ -504,11 +411,12 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ except StopIteration: pass - raise IncorrectConfig("No implementation for %s found in %s" % - (expected_name, classes_found)) + message = ('No implementation for %s found in %s' + % (expected_name, classes_found)) + raise exceptions.IncorrectConfig(error_msg=message) @staticmethod - def create_interfaces_from_node(vnfd, node): + def create_interfaces_from_node(vnfd, node): # pragma: no cover ext_intfs = vnfd["vdu"][0]["external-interface"] = [] # have to sort so xe0 goes first for intf_name, intf in sorted(node['interfaces'].items()): @@ -547,7 +455,7 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ context_cfg = self.context_cfg vnfs = [] - # we assume OrderedDict for consistenct in instantiation + # we assume OrderedDict for consistency in instantiation for node_name, node in context_cfg["nodes"].items(): LOG.debug(node) try: @@ -556,7 +464,7 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ LOG.debug("no model for %s, skipping", node_name) continue file_path = scenario_cfg['task_path'] - with open_relative_file(file_name, file_path) as stream: + with utils.open_relative_file(file_name, file_path) as stream: vnf_model = stream.read() vnfd = vnfdgen.generate_vnfd(vnf_model, node) # TODO: here add extra context_cfg["nodes"] regardless of template @@ -576,11 +484,26 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ self.vnfs = vnfs return vnfs - def setup(self): - """ Setup infrastructure, provission VNFs & start traffic + def pre_run_wait_time(self, time_seconds): # pragma: no cover + """Time waited before executing the run method""" + time.sleep(time_seconds) - :return: - """ + def post_run_wait_time(self, time_seconds): # pragma: no cover + """Time waited after executing the run method""" + pass + + +class NetworkServiceTestCase(NetworkServiceBase): + """Class handles Generic framework to do pre-deployment VNF & + Network service testing """ + + __scenario_type__ = "NSPerf" + + def __init__(self, scenario_cfg, context_cfg): # pragma: no cover + super(NetworkServiceTestCase, self).__init__(scenario_cfg, context_cfg) + + def setup(self): + """Setup infrastructure, provission VNFs & start traffic""" # 1. Verify if infrastructure mapping can meet topology self.map_topology_to_infrastructure() # 1a. Load VNF models @@ -606,13 +529,16 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ vnf.terminate() raise + # we have to generate pod.yaml here after VNF has probed so we know vpci and driver + self._generate_pod_yaml() + # 3. Run experiment # Start listeners first to avoid losing packets for traffic_gen in traffic_runners: traffic_gen.listen_traffic(self.traffic_profile) # register collector with yardstick for KPI collection. - self.collector = Collector(self.vnfs, self.context_cfg["nodes"], self.traffic_profile) + self.collector = Collector(self.vnfs, context_base.Context.get_physical_nodes()) self.collector.start() # Start the actual traffic @@ -634,25 +560,125 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ result.update(self.collector.get_kpi()) - def teardown(self): - """ Stop the collector and terminate VNF & TG instance - :return +class NetworkServiceRFC2544(NetworkServiceBase): + """Class handles RFC2544 Network service testing""" + + __scenario_type__ = "NSPerf-RFC2544" + + def __init__(self, scenario_cfg, context_cfg): # pragma: no cover + super(NetworkServiceRFC2544, self).__init__(scenario_cfg, context_cfg) + + def setup(self): + """Setup infrastructure, provision VNFs""" + self.map_topology_to_infrastructure() + self.load_vnf_models() + + traffic_runners = [vnf for vnf in self.vnfs if vnf.runs_traffic] + non_traffic_runners = [vnf for vnf in self.vnfs if not vnf.runs_traffic] + try: + for vnf in chain(traffic_runners, non_traffic_runners): + LOG.info("Instantiating %s", vnf.name) + vnf.instantiate(self.scenario_cfg, self.context_cfg) + LOG.info("Waiting for %s to instantiate", vnf.name) + vnf.wait_for_instantiate() + except: + LOG.exception("") + for vnf in self.vnfs: + vnf.terminate() + raise + + self._generate_pod_yaml() + + def run(self, output): + """ Run experiment + + :param output: scenario output to push results + :return: None """ + self._fill_traffic_profile() + + traffic_runners = [vnf for vnf in self.vnfs if vnf.runs_traffic] + + for traffic_gen in traffic_runners: + traffic_gen.listen_traffic(self.traffic_profile) + + self.collector = Collector(self.vnfs, + context_base.Context.get_physical_nodes()) + self.collector.start() + + test_completed = False + while not test_completed: + for traffic_gen in traffic_runners: + LOG.info("Run traffic on %s", traffic_gen.name) + traffic_gen.run_traffic_once(self.traffic_profile) + + test_completed = True + for traffic_gen in traffic_runners: + # wait for all tg to complete running traffic + status = traffic_gen.wait_on_traffic() + LOG.info("Run traffic on %s complete status=%s", + traffic_gen.name, status) + if status == 'CONTINUE': + # continue running if at least one tg is running + test_completed = False + + output.push(self.collector.get_kpi()) + + self.collector.stop() + +class NetworkServiceRFC3511(NetworkServiceBase): + """Class handles RFC3511 Network service testing""" + + __scenario_type__ = "NSPerf-RFC3511" + + def __init__(self, scenario_cfg, context_cfg): # pragma: no cover + super(NetworkServiceRFC3511, self).__init__(scenario_cfg, context_cfg) + + def setup(self): + """Setup infrastructure, provision VNFs""" + self.map_topology_to_infrastructure() + self.load_vnf_models() + + traffic_runners = [vnf for vnf in self.vnfs if vnf.runs_traffic] + non_traffic_runners = [vnf for vnf in self.vnfs if not vnf.runs_traffic] try: - try: - self.collector.stop() - for vnf in self.vnfs: - LOG.info("Stopping %s", vnf.name) - vnf.terminate() - LOG.debug("all VNFs terminated: %s", ", ".join(vnf.name for vnf in self.vnfs)) - finally: - terminate_children() - except Exception: - # catch any exception in teardown and convert to simple exception - # never pass exceptions back to multiprocessing, because some exceptions can - # be unpicklable - # https://bugs.python.org/issue9400 + for vnf in chain(traffic_runners, non_traffic_runners): + LOG.info("Instantiating %s", vnf.name) + vnf.instantiate(self.scenario_cfg, self.context_cfg) + LOG.info("Waiting for %s to instantiate", vnf.name) + vnf.wait_for_instantiate() + except: LOG.exception("") - raise RuntimeError("Error in teardown") + for vnf in self.vnfs: + vnf.terminate() + raise + + self._generate_pod_yaml() + + def run(self, output): + """ Run experiment + + :param output: scenario output to push results + :return: None + """ + + self._fill_traffic_profile() + + traffic_runners = [vnf for vnf in self.vnfs if vnf.runs_traffic] + + for traffic_gen in traffic_runners: + traffic_gen.listen_traffic(self.traffic_profile) + + self.collector = Collector(self.vnfs, + context_base.Context.get_physical_nodes()) + self.collector.start() + + for traffic_gen in traffic_runners: + LOG.info("Run traffic on %s", traffic_gen.name) + traffic_gen.run_traffic(self.traffic_profile) + + output.push(self.collector.get_kpi()) + + self.collector.stop() diff --git a/yardstick/benchmark/scenarios/networking/vsperf.py b/yardstick/benchmark/scenarios/networking/vsperf.py index 705544c41..8344b1595 100644 --- a/yardstick/benchmark/scenarios/networking/vsperf.py +++ b/yardstick/benchmark/scenarios/networking/vsperf.py @@ -193,37 +193,34 @@ class Vsperf(base.Scenario): cmd += "--conf-file ~/vsperf.conf " cmd += "--test-params=\"%s\"" % (';'.join(test_params)) LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.client.execute(cmd) - - if status: - raise RuntimeError(stderr) + self.client.run(cmd) # get test results cmd = "cat /tmp/results*/result.csv" LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.client.execute(cmd) - - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.client.execute(cmd, raise_on_error=True) # convert result.csv to JSON format - reader = csv.DictReader(stdout.split('\r\n')) - result.update(next(reader)) + reader = csv.DictReader(stdout.split('\r\n'), strict=True) + try: + result.update(next(reader)) + except StopIteration: + pass # sla check; go through all defined SLAs and check if values measured # by VSPERF are higher then those defined by SLAs if 'sla' in self.scenario_cfg and \ 'metrics' in self.scenario_cfg['sla']: for metric in self.scenario_cfg['sla']['metrics'].split(','): - assert metric in result, \ - '%s is not collected by VSPERF' % (metric) - assert metric in self.scenario_cfg['sla'], \ - '%s is not defined in SLA' % (metric) + self.verify_SLA(metric in result, + '%s was not collected by VSPERF' % metric) + self.verify_SLA(metric in self.scenario_cfg['sla'], + '%s is not defined in SLA' % metric) vs_res = float(result[metric]) sla_res = float(self.scenario_cfg['sla'][metric]) - assert vs_res >= sla_res, \ - 'VSPERF_%s(%f) < SLA_%s(%f)' % \ - (metric, vs_res, metric, sla_res) + self.verify_SLA(vs_res >= sla_res, + 'VSPERF_%s(%f) < SLA_%s(%f)' + % (metric, vs_res, metric, sla_res)) def teardown(self): """cleanup after the test execution""" diff --git a/yardstick/benchmark/scenarios/networking/vsperf_dpdk.py b/yardstick/benchmark/scenarios/networking/vsperf_dpdk.py index 454587829..d5c8a3bfe 100644 --- a/yardstick/benchmark/scenarios/networking/vsperf_dpdk.py +++ b/yardstick/benchmark/scenarios/networking/vsperf_dpdk.py @@ -205,22 +205,17 @@ class VsperfDPDK(base.Scenario): self.client.send_command(cmd) else: cmd = "cat ~/.testpmd.macaddr.port1" - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.client.execute(cmd, raise_on_error=True) self.tgen_port1_mac = stdout + cmd = "cat ~/.testpmd.macaddr.port2" - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.client.execute(cmd, raise_on_error=True) self.tgen_port2_mac = stdout cmd = "screen -d -m sudo -E bash ~/testpmd_vsperf.sh %s %s" % \ (self.moongen_port1_mac, self.moongen_port2_mac) LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.client.run(cmd) time.sleep(1) @@ -231,7 +226,7 @@ class VsperfDPDK(base.Scenario): is_run = True cmd = "ip a | grep %s 2>/dev/null" % (self.tg_port1) LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.client.execute(cmd) + _, stdout, _ = self.client.execute(cmd) if stdout: is_run = False return is_run @@ -245,7 +240,7 @@ class VsperfDPDK(base.Scenario): self.setup() # remove results from previous tests - self.client.execute("rm -rf /tmp/results*") + self.client.run("rm -rf /tmp/results*", raise_on_error=False) # get vsperf options options = self.scenario_cfg['options'] @@ -291,9 +286,7 @@ class VsperfDPDK(base.Scenario): cmd = "sshpass -p yardstick ssh-copy-id -o StrictHostKeyChecking=no " \ "root@%s -p 22" % (self.moongen_host_ip) LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.client.execute(cmd) - if status: - raise RuntimeError(stderr) + self.client.run(cmd) # execute vsperf cmd = "source ~/vsperfenv/bin/activate ; cd vswitchperf ; " @@ -302,22 +295,19 @@ class VsperfDPDK(base.Scenario): cmd += "--conf-file ~/vsperf.conf " cmd += "--test-params=\"%s\"" % (';'.join(test_params)) LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.client.execute(cmd) - - if status: - raise RuntimeError(stderr) + self.client.run(cmd) # get test results cmd = "cat /tmp/results*/result.csv" LOG.debug("Executing command: %s", cmd) - status, stdout, stderr = self.client.execute(cmd) - - if status: - raise RuntimeError(stderr) + _, stdout, _ = self.client.execute(cmd, raise_on_error=True) # convert result.csv to JSON format reader = csv.DictReader(stdout.split('\r\n')) - result.update(next(reader)) + try: + result.update(next(reader)) + except StopIteration: + pass result['nrFlows'] = multistream # sla check; go through all defined SLAs and check if values measured @@ -325,15 +315,15 @@ class VsperfDPDK(base.Scenario): if 'sla' in self.scenario_cfg and \ 'metrics' in self.scenario_cfg['sla']: for metric in self.scenario_cfg['sla']['metrics'].split(','): - assert metric in result, \ - '%s is not collected by VSPERF' % (metric) - assert metric in self.scenario_cfg['sla'], \ - '%s is not defined in SLA' % (metric) + self.verify_SLA(metric in result, + '%s was not collected by VSPERF' % metric) + self.verify_SLA(metric in self.scenario_cfg['sla'], + '%s is not defined in SLA' % metric) vs_res = float(result[metric]) sla_res = float(self.scenario_cfg['sla'][metric]) - assert vs_res >= sla_res, \ - 'VSPERF_%s(%f) < SLA_%s(%f)' % \ - (metric, vs_res, metric, sla_res) + self.verify_SLA(vs_res >= sla_res, + 'VSPERF_%s(%f) < SLA_%s(%f)' + % (metric, vs_res, metric, sla_res)) def teardown(self): """cleanup after the test execution""" |