diff options
| -rw-r--r-- | api/resources/env_action.py | 2 | ||||
| -rw-r--r-- | api/utils/common.py | 2 | ||||
| -rw-r--r-- | api/utils/influx.py | 2 | ||||
| -rw-r--r-- | samples/nstat.yaml | 35 | ||||
| -rwxr-xr-x | tests/ci/load_images.sh | 2 | ||||
| -rw-r--r-- | tests/opnfv/test_cases/opnfv_yardstick_tc052.yaml | 2 | ||||
| -rw-r--r-- | tests/opnfv/test_cases/opnfv_yardstick_tc076.yaml | 56 | ||||
| -rw-r--r-- | tests/unit/benchmark/scenarios/networking/test_nstat.py | 118 | ||||
| -rwxr-xr-x | tools/ubuntu-server-cloudimg-dpdk-modify.sh | 3 | ||||
| -rwxr-xr-x | tools/ubuntu-server-cloudimg-modify.sh | 3 | ||||
| -rw-r--r-- | yardstick/__init__.py | 2 | ||||
| -rw-r--r-- | yardstick/benchmark/core/__init__.py | 4 | ||||
| -rwxr-xr-x | yardstick/benchmark/runners/arithmetic.py | 2 | ||||
| -rw-r--r-- | yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash | 2 | ||||
| -rw-r--r-- | yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash | 2 | ||||
| -rw-r--r-- | yardstick/benchmark/scenarios/networking/nstat.py | 130 | ||||
| -rw-r--r-- | yardstick/benchmark/scenarios/networking/sfc_openstack.py | 2 | ||||
| -rwxr-xr-x | yardstick/cmd/NSBperf.py | 2 | ||||
| -rwxr-xr-x | yardstick/common/task_template.py | 4 |
19 files changed, 361 insertions, 14 deletions
diff --git a/api/resources/env_action.py b/api/resources/env_action.py index 8955f3cb6..917681c37 100644 --- a/api/resources/env_action.py +++ b/api/resources/env_action.py @@ -248,7 +248,7 @@ def _get_remote_rc_file(rc_file, installer_ip, installer_type): cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type, '-a', installer_ip] p = subprocess.Popen(cmd, stdout=subprocess.PIPE) - p.communicate()[0] + p.communicate() if p.returncode != 0: logger.debug('Failed to fetch credentials from installer') diff --git a/api/utils/common.py b/api/utils/common.py index 1c800ce49..3e9bf8f8b 100644 --- a/api/utils/common.py +++ b/api/utils/common.py @@ -33,7 +33,7 @@ def get_command_list(command_list, opts, args): command_list.append(args) - command_list.extend(('--{}'.format(k) for k in opts if 'task-args' != k)) + command_list.extend(('--{}'.format(k) for k in opts if k != 'task-args')) task_args = opts.get('task-args', '') if task_args: diff --git a/api/utils/influx.py b/api/utils/influx.py index 275c63a24..08996b9c9 100644 --- a/api/utils/influx.py +++ b/api/utils/influx.py @@ -24,7 +24,7 @@ def get_data_db_client(): try: parser.read(conf.OUTPUT_CONFIG_FILE_PATH) - if 'influxdb' != parser.get('DEFAULT', 'dispatcher'): + if parser.get('DEFAULT', 'dispatcher') != 'influxdb': raise RuntimeError return _get_client(parser) diff --git a/samples/nstat.yaml b/samples/nstat.yaml new file mode 100644 index 000000000..0a5aa80c9 --- /dev/null +++ b/samples/nstat.yaml @@ -0,0 +1,35 @@ +--- + +schema: "yardstick:task:0.1" + +description: > + Sample benchmark task config file; + Monitor network metrics provided by the kernel in a host and calculate + IP datagram error rate, ICMP message error rate, TCP segment error rate and + UDP datagram error rate. + +scenarios: +- + type: Nstat + options: + duration: 60 + + host: poseidon.demo + + runner: + type: Iteration + iterations: 1 + +context: + name: demo + image: yardstick-image + flavor: yardstick-flavor + user: ubuntu + + servers: + poseidon: + floating_ip: true + + networks: + test: + cidr: '10.0.1.0/24' diff --git a/tests/ci/load_images.sh b/tests/ci/load_images.sh index e1d717749..6f950ec72 100755 --- a/tests/ci/load_images.sh +++ b/tests/ci/load_images.sh @@ -206,7 +206,7 @@ create_nova_flavor() # Create the nova flavor used by some sample test cases openstack flavor create --id 100 --ram 512 --disk 3 --vcpus 1 yardstick-flavor # DPDK-enabled OVS requires guest memory to be backed by large pages - if [[ "$DEPLOY_SCENARIO" == *"-ovs-"* ]]; then + if [[ $DEPLOY_SCENARIO == *[_-]ovs[_-]* ]]; then openstack flavor set --property hw:mem_page_size=large yardstick-flavor fi # VPP requires guest memory to be backed by large pages diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc052.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc052.yaml index 67619d3d0..4d3ae93f5 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc052.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc052.yaml @@ -20,7 +20,7 @@ scenarios: - monitor_type: "openstack-cmd" key: "nova-flavor-list" - command_name: "nova flavor-list" + command_name: "openstack flavor list" monitor_time: 10 sla: max_outage_time: 5 diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc076.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc076.yaml new file mode 100644 index 000000000..c23ee97c2 --- /dev/null +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc076.yaml @@ -0,0 +1,56 @@ +--- + +schema: "yardstick:task:0.1" + +description: > + Yardstick TC076 config file; + Monitor network metrics provided by the kernel in a host and calculate + IP datagram error rate, ICMP message error rate, TCP segment error rate and + UDP datagram error rate. + +scenarios: +- + type: Ping + run_in_background: true + options: + packetsize: 200 + + host: demeter.yardstick-TC076 + target: poseidon.yardstick-TC076 + +- + type: Nstat + options: + duration: 300 + + host: poseidon.yardstick-TC076 + + runner: + type: Iteration + iterations: 1 + + sla: + IP_datagram_error_rate: 0.01 + action: monitor + +context: + name: yardstick-TC076 + image: yardstick-image + flavor: yardstick-flavor + user: ubuntu + + placement_groups: + pgrp1: + policy: "availability" + + servers: + demeter: + floating_ip: true + placement: "pgrp1" + poseidon: + floating_ip: true + placement: "pgrp1" + + networks: + test: + cidr: '10.0.1.0/24' diff --git a/tests/unit/benchmark/scenarios/networking/test_nstat.py b/tests/unit/benchmark/scenarios/networking/test_nstat.py new file mode 100644 index 000000000..87a766302 --- /dev/null +++ b/tests/unit/benchmark/scenarios/networking/test_nstat.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python + +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Unittest for yardstick.benchmark.scenarios.networking.nstat.Nstat + +from __future__ import absolute_import + +import unittest + +import mock + +from yardstick.benchmark.scenarios.networking import nstat + +@mock.patch('yardstick.benchmark.scenarios.networking.nstat.ssh') +class NstatTestCase(unittest.TestCase): + + def setUp(self): + self.ctx = { + "host": { + "ip": "192.168.50.28", + "user": "root", + "key_filename": "mykey.key" + } + } + + def test_nstat_successful_setup(self, mock_ssh): + + n = nstat.Nstat({}, self.ctx) + n.setup() + + mock_ssh.SSH().execute.return_value = (0, '', '') + self.assertIsNotNone(n.client) + self.assertEqual(n.setup_done, True) + + def test_nstat_successful_no_sla(self, mock_ssh): + + options = { + "duration": 60 + } + args = { + "options": options, + } + n = nstat.Nstat(args, self.ctx) + result = {} + + sample_output = '#kernel\nIpInReceives 1837 0.0\nIpInHdrErrors 0 0.0\nIpInAddrErrors 2 0.0\nIcmpInMsgs 319 0.0\nIcmpInErrors 0 0.0\nTcpInSegs 36 0.0\nTcpInErrs 0 0.0\nUdpInDatagrams 1318 0.0\nUdpInErrors 0 0.0\n' + + mock_ssh.SSH().execute.return_value = (0, sample_output, '') + + n.run(result) + expected_result = {"TcpInErrs": 0, "UdpInDatagrams": 1318, + "Tcp_segment_error_rate": 0.0, "IpInAddrErrors": 2, + "IpInHdrErrors": 0, "IcmpInErrors": 0, "IpErrors": 2, + "TcpInSegs": 36, "IpInReceives": 1837, "IcmpInMsgs": 319, + "IP_datagram_error_rate": 0.001, "Udp_datagram_error_rate": 0.0, + "Icmp_message_error_rate": 0.0, "UdpInErrors": 0} + self.assertEqual(result, expected_result) + + def test_nstat_successful_sla(self, mock_ssh): + + options = { + "duration": 60 + } + sla = { + "IP_datagram_error_rate": 0.1 + } + args = { + "options": options, + "sla": sla + } + n = nstat.Nstat(args, self.ctx) + result = {} + + sample_output = '#kernel\nIpInReceives 1837 0.0\nIpInHdrErrors 0 0.0\nIpInAddrErrors 2 0.0\nIcmpInMsgs 319 0.0\nIcmpInErrors 0 0.0\nTcpInSegs 36 0.0\nTcpInErrs 0 0.0\nUdpInDatagrams 1318 0.0\nUdpInErrors 0 0.0\n' + + mock_ssh.SSH().execute.return_value = (0, sample_output, '') + + n.run(result) + expected_result = {"TcpInErrs": 0, "UdpInDatagrams": 1318, + "Tcp_segment_error_rate": 0.0, "IpInAddrErrors": 2, + "IpInHdrErrors": 0, "IcmpInErrors": 0, "IpErrors": 2, + "TcpInSegs": 36, "IpInReceives": 1837, "IcmpInMsgs": 319, + "IP_datagram_error_rate": 0.001, "Udp_datagram_error_rate": 0.0, + "Icmp_message_error_rate": 0.0, "UdpInErrors": 0} + self.assertEqual(result, expected_result) + + def test_nstat_unsuccessful_cmd_error(self, mock_ssh): + + options = { + "duration": 60 + } + sla = { + "IP_datagram_error_rate": 0.1 + } + args = { + "options": options, + "sla": sla + } + n = nstat.Nstat(args, self.ctx) + result = {} + + mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR') + self.assertRaises(RuntimeError, n.run, result) + + +def main(): + unittest.main() + +if __name__ == '__main__': + main() diff --git a/tools/ubuntu-server-cloudimg-dpdk-modify.sh b/tools/ubuntu-server-cloudimg-dpdk-modify.sh index aa4e252ea..9a3857ee3 100755 --- a/tools/ubuntu-server-cloudimg-dpdk-modify.sh +++ b/tools/ubuntu-server-cloudimg-dpdk-modify.sh @@ -63,10 +63,13 @@ linuxheadersversion=`echo ls boot/vmlinuz* | cut -d- -f2-` apt-get update apt-get install -y \ + bc \ fio \ gcc \ git \ iperf3 \ + iproute2 \ + ethtool \ linux-tools-common \ linux-tools-generic \ lmbench \ diff --git a/tools/ubuntu-server-cloudimg-modify.sh b/tools/ubuntu-server-cloudimg-modify.sh index 49c842c97..c0ae774ef 100755 --- a/tools/ubuntu-server-cloudimg-modify.sh +++ b/tools/ubuntu-server-cloudimg-modify.sh @@ -58,10 +58,13 @@ bootcmd: EOF fi apt-get install -y \ + bc \ fio \ git \ gcc \ iperf3 \ + ethtool \ + iproute2 \ linux-tools-common \ linux-tools-generic \ lmbench \ diff --git a/yardstick/__init__.py b/yardstick/__init__.py index fbbc101a9..1ad6eb0b1 100644 --- a/yardstick/__init__.py +++ b/yardstick/__init__.py @@ -39,6 +39,8 @@ def _init_logging(): # don't append to log file, clobber _LOG_FILE_HDLR.setFormatter(_LOG_FORMATTER) + # set log file to store debug info + _LOG_FILE_HDLR.setLevel(logging.DEBUG) del logging.root.handlers[:] logging.root.addHandler(_LOG_STREAM_HDLR) diff --git a/yardstick/benchmark/core/__init__.py b/yardstick/benchmark/core/__init__.py index 161e448cc..79ebc732f 100644 --- a/yardstick/benchmark/core/__init__.py +++ b/yardstick/benchmark/core/__init__.py @@ -33,6 +33,6 @@ class Param(object): def print_hbar(barlen): """print to stdout a horizontal bar""" - print("+"), - print("-" * barlen), + print("+") + print("-" * barlen) print("+") diff --git a/yardstick/benchmark/runners/arithmetic.py b/yardstick/benchmark/runners/arithmetic.py index d5605f755..65fdb9d66 100755 --- a/yardstick/benchmark/runners/arithmetic.py +++ b/yardstick/benchmark/runners/arithmetic.py @@ -139,7 +139,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, sequence += 1 - if (errors and sla_action is None): + if errors and sla_action is None: break benchmark.teardown() diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash index 5c2d6d70e..38dbe0cee 100644 --- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash +++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash @@ -16,4 +16,4 @@ set -e source /root/openrc -nova flavor-create $1 $2 $3 $4 $5 +openstack flavor create $1 --id $2 --ram $3 --disk $4 --vcpus $5 diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash index 67d0c902e..37d2cf6c0 100644 --- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash +++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash @@ -16,4 +16,4 @@ set -e source /root/openrc -nova flavor-delete $1
\ No newline at end of file +openstack flavor delete $1 diff --git a/yardstick/benchmark/scenarios/networking/nstat.py b/yardstick/benchmark/scenarios/networking/nstat.py new file mode 100644 index 000000000..df96dbda7 --- /dev/null +++ b/yardstick/benchmark/scenarios/networking/nstat.py @@ -0,0 +1,130 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from __future__ import print_function +from __future__ import absolute_import + +import time +import logging + +import yardstick.ssh as ssh +from yardstick.benchmark.scenarios import base + +LOG = logging.getLogger(__name__) + +PRECISION = 3 + + +class Nstat(base.Scenario): + """Use nstat to monitor network metrics and measure IP datagram error rate + and etc. + """ + + __scenario_type__ = "Nstat" + + def __init__(self, scenario_cfg, context_cfg): + """Scenario construction""" + self.scenario_cfg = scenario_cfg + self.context_cfg = context_cfg + self.setup_done = False + + def setup(self): + """scenario setup""" + host = self.context_cfg["host"] + user = host.get("user", "ubuntu") + ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT) + ip = host.get("ip", None) + key_filename = host.get('key_filename', "~/.ssh/id_rsa") + + LOG.info("user:%s, host:%s", user, ip) + self.client = ssh.SSH(user, ip, key_filename=key_filename, + port=ssh_port) + self.client.wait(timeout=600) + + self.setup_done = True + + def match(self, key, field, line, results): + """match data in the output""" + if key in line: + results[key] = int(line.split()[field]) + + def calculate_error_rate(self, x, y): + """calculate error rate""" + try: + return round(float(x) / float(y), PRECISION) + except ZeroDivisionError: + # If incoming Errors is non-zero, but incoming data is zero + # consider it as 100% error rate + if x: + return 1 + else: + return 0 + + def process_output(self, out): + """process output""" + results = {} + for line in out.splitlines(): + self.match('IpInReceives', 1, line, results) + self.match('IpInHdrErrors', 1, line, results) + self.match('IpInAddrErrors', 1, line, results) + self.match('IcmpInMsgs', 1, line, results) + self.match('IcmpInErrors', 1, line, results) + self.match('TcpInSegs', 1, line, results) + self.match('TcpInErrs', 1, line, results) + self.match('UdpInDatagrams', 1, line, results) + self.match('UdpInErrors', 1, line, results) + results['IpErrors'] = \ + results['IpInHdrErrors'] + results['IpInAddrErrors'] + results['IP_datagram_error_rate'] = \ + self.calculate_error_rate(results['IpErrors'], + results['IpInReceives']) + results['Icmp_message_error_rate'] = \ + self.calculate_error_rate(results['IcmpInErrors'], + results['IcmpInMsgs']) + results['Tcp_segment_error_rate'] = \ + self.calculate_error_rate(results['TcpInErrs'], + results['TcpInSegs']) + results['Udp_datagram_error_rate'] = \ + self.calculate_error_rate(results['UdpInErrors'], + results['UdpInDatagrams']) + return results + + def run(self, result): + """execute the benchmark""" + + if not self.setup_done: + self.setup() + + options = self.scenario_cfg['options'] + duration = options.get('duration', 60) + + time.sleep(duration) + + cmd = "nstat -z" + + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.client.execute(cmd) + + if status: + raise RuntimeError(stderr) + + results = self.process_output(stdout) + + result.update(results) + + if "sla" in self.scenario_cfg: + sla_error = "" + for i, rate in result.items(): + if i not in self.scenario_cfg['sla']: + continue + sla_rate = float(self.scenario_cfg['sla'][i]) + rate = float(rate) + if rate > sla_rate: + sla_error += "%s rate %f > sla:%s_rate(%f); " % \ + (i, rate, i, sla_rate) + assert sla_error == "", sla_error diff --git a/yardstick/benchmark/scenarios/networking/sfc_openstack.py b/yardstick/benchmark/scenarios/networking/sfc_openstack.py index caaf10060..be24add32 100644 --- a/yardstick/benchmark/scenarios/networking/sfc_openstack.py +++ b/yardstick/benchmark/scenarios/networking/sfc_openstack.py @@ -81,7 +81,7 @@ def create_floating_ips(neutron_client): # pragma: no cover ips = [] props = {'floating_network_id': extnet_id} try: - while (len(ips) < 2): + while len(ips) < 2: ip_json = neutron_client.create_floatingip({'floatingip': props}) fip_addr = ip_json['floatingip']['floating_ip_address'] ips.append(fip_addr) diff --git a/yardstick/cmd/NSBperf.py b/yardstick/cmd/NSBperf.py index dd96b7fc8..c3730f834 100755 --- a/yardstick/cmd/NSBperf.py +++ b/yardstick/cmd/NSBperf.py @@ -115,7 +115,7 @@ class YardstickNSCli(object): def generate_final_report(self, test_case): """ Function will check if partial test results are available and generates final report in rst format. -""" + """ report_caption = '{}\n{} ({})\n{}\n\n'.format( '================================================================', diff --git a/yardstick/common/task_template.py b/yardstick/common/task_template.py index bda8a1b13..9acc21336 100755 --- a/yardstick/common/task_template.py +++ b/yardstick/common/task_template.py @@ -45,11 +45,11 @@ def is_really_missing(mis, task_template): # Removing variables that have default values from # missing. Construction that won't be properly # check is {% set x = x or 1} - if re.search(mis.join(["{%\s*set\s+", "\s*=\s*", "[^\w]+"]), + if re.search(mis.join([r"{%\s*set\s+", "\s*=\s*", r"[^\w]+"]), task_template): return False # Also check for a default filter which can show up as # a missing variable - if re.search(mis + "\s*\|\s*default\(", task_template): + if re.search(mis + r"\s*\|\s*default\(", task_template): return False return True |