aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--api/resources/env_action.py2
-rw-r--r--api/utils/common.py2
-rw-r--r--api/utils/influx.py2
-rw-r--r--samples/nstat.yaml35
-rwxr-xr-xtests/ci/load_images.sh2
-rw-r--r--tests/opnfv/test_cases/opnfv_yardstick_tc052.yaml2
-rw-r--r--tests/opnfv/test_cases/opnfv_yardstick_tc076.yaml56
-rw-r--r--tests/unit/benchmark/scenarios/networking/test_nstat.py118
-rwxr-xr-xtools/ubuntu-server-cloudimg-dpdk-modify.sh3
-rwxr-xr-xtools/ubuntu-server-cloudimg-modify.sh3
-rw-r--r--yardstick/__init__.py2
-rw-r--r--yardstick/benchmark/core/__init__.py4
-rwxr-xr-xyardstick/benchmark/runners/arithmetic.py2
-rw-r--r--yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash2
-rw-r--r--yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash2
-rw-r--r--yardstick/benchmark/scenarios/networking/nstat.py130
-rw-r--r--yardstick/benchmark/scenarios/networking/sfc_openstack.py2
-rwxr-xr-xyardstick/cmd/NSBperf.py2
-rwxr-xr-xyardstick/common/task_template.py4
19 files changed, 361 insertions, 14 deletions
diff --git a/api/resources/env_action.py b/api/resources/env_action.py
index 8955f3cb6..917681c37 100644
--- a/api/resources/env_action.py
+++ b/api/resources/env_action.py
@@ -248,7 +248,7 @@ def _get_remote_rc_file(rc_file, installer_ip, installer_type):
cmd = [os_fetch_script, '-d', rc_file, '-i', installer_type,
'-a', installer_ip]
p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
- p.communicate()[0]
+ p.communicate()
if p.returncode != 0:
logger.debug('Failed to fetch credentials from installer')
diff --git a/api/utils/common.py b/api/utils/common.py
index 1c800ce49..3e9bf8f8b 100644
--- a/api/utils/common.py
+++ b/api/utils/common.py
@@ -33,7 +33,7 @@ def get_command_list(command_list, opts, args):
command_list.append(args)
- command_list.extend(('--{}'.format(k) for k in opts if 'task-args' != k))
+ command_list.extend(('--{}'.format(k) for k in opts if k != 'task-args'))
task_args = opts.get('task-args', '')
if task_args:
diff --git a/api/utils/influx.py b/api/utils/influx.py
index 275c63a24..08996b9c9 100644
--- a/api/utils/influx.py
+++ b/api/utils/influx.py
@@ -24,7 +24,7 @@ def get_data_db_client():
try:
parser.read(conf.OUTPUT_CONFIG_FILE_PATH)
- if 'influxdb' != parser.get('DEFAULT', 'dispatcher'):
+ if parser.get('DEFAULT', 'dispatcher') != 'influxdb':
raise RuntimeError
return _get_client(parser)
diff --git a/samples/nstat.yaml b/samples/nstat.yaml
new file mode 100644
index 000000000..0a5aa80c9
--- /dev/null
+++ b/samples/nstat.yaml
@@ -0,0 +1,35 @@
+---
+
+schema: "yardstick:task:0.1"
+
+description: >
+ Sample benchmark task config file;
+ Monitor network metrics provided by the kernel in a host and calculate
+ IP datagram error rate, ICMP message error rate, TCP segment error rate and
+ UDP datagram error rate.
+
+scenarios:
+-
+ type: Nstat
+ options:
+ duration: 60
+
+ host: poseidon.demo
+
+ runner:
+ type: Iteration
+ iterations: 1
+
+context:
+ name: demo
+ image: yardstick-image
+ flavor: yardstick-flavor
+ user: ubuntu
+
+ servers:
+ poseidon:
+ floating_ip: true
+
+ networks:
+ test:
+ cidr: '10.0.1.0/24'
diff --git a/tests/ci/load_images.sh b/tests/ci/load_images.sh
index e1d717749..6f950ec72 100755
--- a/tests/ci/load_images.sh
+++ b/tests/ci/load_images.sh
@@ -206,7 +206,7 @@ create_nova_flavor()
# Create the nova flavor used by some sample test cases
openstack flavor create --id 100 --ram 512 --disk 3 --vcpus 1 yardstick-flavor
# DPDK-enabled OVS requires guest memory to be backed by large pages
- if [[ "$DEPLOY_SCENARIO" == *"-ovs-"* ]]; then
+ if [[ $DEPLOY_SCENARIO == *[_-]ovs[_-]* ]]; then
openstack flavor set --property hw:mem_page_size=large yardstick-flavor
fi
# VPP requires guest memory to be backed by large pages
diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc052.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc052.yaml
index 67619d3d0..4d3ae93f5 100644
--- a/tests/opnfv/test_cases/opnfv_yardstick_tc052.yaml
+++ b/tests/opnfv/test_cases/opnfv_yardstick_tc052.yaml
@@ -20,7 +20,7 @@ scenarios:
-
monitor_type: "openstack-cmd"
key: "nova-flavor-list"
- command_name: "nova flavor-list"
+ command_name: "openstack flavor list"
monitor_time: 10
sla:
max_outage_time: 5
diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc076.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc076.yaml
new file mode 100644
index 000000000..c23ee97c2
--- /dev/null
+++ b/tests/opnfv/test_cases/opnfv_yardstick_tc076.yaml
@@ -0,0 +1,56 @@
+---
+
+schema: "yardstick:task:0.1"
+
+description: >
+ Yardstick TC076 config file;
+ Monitor network metrics provided by the kernel in a host and calculate
+ IP datagram error rate, ICMP message error rate, TCP segment error rate and
+ UDP datagram error rate.
+
+scenarios:
+-
+ type: Ping
+ run_in_background: true
+ options:
+ packetsize: 200
+
+ host: demeter.yardstick-TC076
+ target: poseidon.yardstick-TC076
+
+-
+ type: Nstat
+ options:
+ duration: 300
+
+ host: poseidon.yardstick-TC076
+
+ runner:
+ type: Iteration
+ iterations: 1
+
+ sla:
+ IP_datagram_error_rate: 0.01
+ action: monitor
+
+context:
+ name: yardstick-TC076
+ image: yardstick-image
+ flavor: yardstick-flavor
+ user: ubuntu
+
+ placement_groups:
+ pgrp1:
+ policy: "availability"
+
+ servers:
+ demeter:
+ floating_ip: true
+ placement: "pgrp1"
+ poseidon:
+ floating_ip: true
+ placement: "pgrp1"
+
+ networks:
+ test:
+ cidr: '10.0.1.0/24'
diff --git a/tests/unit/benchmark/scenarios/networking/test_nstat.py b/tests/unit/benchmark/scenarios/networking/test_nstat.py
new file mode 100644
index 000000000..87a766302
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/networking/test_nstat.py
@@ -0,0 +1,118 @@
+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Unittest for yardstick.benchmark.scenarios.networking.nstat.Nstat
+
+from __future__ import absolute_import
+
+import unittest
+
+import mock
+
+from yardstick.benchmark.scenarios.networking import nstat
+
+@mock.patch('yardstick.benchmark.scenarios.networking.nstat.ssh')
+class NstatTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.ctx = {
+ "host": {
+ "ip": "192.168.50.28",
+ "user": "root",
+ "key_filename": "mykey.key"
+ }
+ }
+
+ def test_nstat_successful_setup(self, mock_ssh):
+
+ n = nstat.Nstat({}, self.ctx)
+ n.setup()
+
+ mock_ssh.SSH().execute.return_value = (0, '', '')
+ self.assertIsNotNone(n.client)
+ self.assertEqual(n.setup_done, True)
+
+ def test_nstat_successful_no_sla(self, mock_ssh):
+
+ options = {
+ "duration": 60
+ }
+ args = {
+ "options": options,
+ }
+ n = nstat.Nstat(args, self.ctx)
+ result = {}
+
+ sample_output = '#kernel\nIpInReceives 1837 0.0\nIpInHdrErrors 0 0.0\nIpInAddrErrors 2 0.0\nIcmpInMsgs 319 0.0\nIcmpInErrors 0 0.0\nTcpInSegs 36 0.0\nTcpInErrs 0 0.0\nUdpInDatagrams 1318 0.0\nUdpInErrors 0 0.0\n'
+
+ mock_ssh.SSH().execute.return_value = (0, sample_output, '')
+
+ n.run(result)
+ expected_result = {"TcpInErrs": 0, "UdpInDatagrams": 1318,
+ "Tcp_segment_error_rate": 0.0, "IpInAddrErrors": 2,
+ "IpInHdrErrors": 0, "IcmpInErrors": 0, "IpErrors": 2,
+ "TcpInSegs": 36, "IpInReceives": 1837, "IcmpInMsgs": 319,
+ "IP_datagram_error_rate": 0.001, "Udp_datagram_error_rate": 0.0,
+ "Icmp_message_error_rate": 0.0, "UdpInErrors": 0}
+ self.assertEqual(result, expected_result)
+
+ def test_nstat_successful_sla(self, mock_ssh):
+
+ options = {
+ "duration": 60
+ }
+ sla = {
+ "IP_datagram_error_rate": 0.1
+ }
+ args = {
+ "options": options,
+ "sla": sla
+ }
+ n = nstat.Nstat(args, self.ctx)
+ result = {}
+
+ sample_output = '#kernel\nIpInReceives 1837 0.0\nIpInHdrErrors 0 0.0\nIpInAddrErrors 2 0.0\nIcmpInMsgs 319 0.0\nIcmpInErrors 0 0.0\nTcpInSegs 36 0.0\nTcpInErrs 0 0.0\nUdpInDatagrams 1318 0.0\nUdpInErrors 0 0.0\n'
+
+ mock_ssh.SSH().execute.return_value = (0, sample_output, '')
+
+ n.run(result)
+ expected_result = {"TcpInErrs": 0, "UdpInDatagrams": 1318,
+ "Tcp_segment_error_rate": 0.0, "IpInAddrErrors": 2,
+ "IpInHdrErrors": 0, "IcmpInErrors": 0, "IpErrors": 2,
+ "TcpInSegs": 36, "IpInReceives": 1837, "IcmpInMsgs": 319,
+ "IP_datagram_error_rate": 0.001, "Udp_datagram_error_rate": 0.0,
+ "Icmp_message_error_rate": 0.0, "UdpInErrors": 0}
+ self.assertEqual(result, expected_result)
+
+ def test_nstat_unsuccessful_cmd_error(self, mock_ssh):
+
+ options = {
+ "duration": 60
+ }
+ sla = {
+ "IP_datagram_error_rate": 0.1
+ }
+ args = {
+ "options": options,
+ "sla": sla
+ }
+ n = nstat.Nstat(args, self.ctx)
+ result = {}
+
+ mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR')
+ self.assertRaises(RuntimeError, n.run, result)
+
+
+def main():
+ unittest.main()
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/ubuntu-server-cloudimg-dpdk-modify.sh b/tools/ubuntu-server-cloudimg-dpdk-modify.sh
index aa4e252ea..9a3857ee3 100755
--- a/tools/ubuntu-server-cloudimg-dpdk-modify.sh
+++ b/tools/ubuntu-server-cloudimg-dpdk-modify.sh
@@ -63,10 +63,13 @@ linuxheadersversion=`echo ls boot/vmlinuz* | cut -d- -f2-`
apt-get update
apt-get install -y \
+ bc \
fio \
gcc \
git \
iperf3 \
+ iproute2 \
+ ethtool \
linux-tools-common \
linux-tools-generic \
lmbench \
diff --git a/tools/ubuntu-server-cloudimg-modify.sh b/tools/ubuntu-server-cloudimg-modify.sh
index 49c842c97..c0ae774ef 100755
--- a/tools/ubuntu-server-cloudimg-modify.sh
+++ b/tools/ubuntu-server-cloudimg-modify.sh
@@ -58,10 +58,13 @@ bootcmd:
EOF
fi
apt-get install -y \
+ bc \
fio \
git \
gcc \
iperf3 \
+ ethtool \
+ iproute2 \
linux-tools-common \
linux-tools-generic \
lmbench \
diff --git a/yardstick/__init__.py b/yardstick/__init__.py
index fbbc101a9..1ad6eb0b1 100644
--- a/yardstick/__init__.py
+++ b/yardstick/__init__.py
@@ -39,6 +39,8 @@ def _init_logging():
# don't append to log file, clobber
_LOG_FILE_HDLR.setFormatter(_LOG_FORMATTER)
+ # set log file to store debug info
+ _LOG_FILE_HDLR.setLevel(logging.DEBUG)
del logging.root.handlers[:]
logging.root.addHandler(_LOG_STREAM_HDLR)
diff --git a/yardstick/benchmark/core/__init__.py b/yardstick/benchmark/core/__init__.py
index 161e448cc..79ebc732f 100644
--- a/yardstick/benchmark/core/__init__.py
+++ b/yardstick/benchmark/core/__init__.py
@@ -33,6 +33,6 @@ class Param(object):
def print_hbar(barlen):
"""print to stdout a horizontal bar"""
- print("+"),
- print("-" * barlen),
+ print("+")
+ print("-" * barlen)
print("+")
diff --git a/yardstick/benchmark/runners/arithmetic.py b/yardstick/benchmark/runners/arithmetic.py
index d5605f755..65fdb9d66 100755
--- a/yardstick/benchmark/runners/arithmetic.py
+++ b/yardstick/benchmark/runners/arithmetic.py
@@ -139,7 +139,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg,
sequence += 1
- if (errors and sla_action is None):
+ if errors and sla_action is None:
break
benchmark.teardown()
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
index 5c2d6d70e..38dbe0cee 100644
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/create_flavor.bash
@@ -16,4 +16,4 @@ set -e
source /root/openrc
-nova flavor-create $1 $2 $3 $4 $5
+openstack flavor create $1 --id $2 --ram $3 --disk $4 --vcpus $5
diff --git a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
index 67d0c902e..37d2cf6c0 100644
--- a/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
+++ b/yardstick/benchmark/scenarios/availability/ha_tools/nova/delete_flavor.bash
@@ -16,4 +16,4 @@ set -e
source /root/openrc
-nova flavor-delete $1 \ No newline at end of file
+openstack flavor delete $1
diff --git a/yardstick/benchmark/scenarios/networking/nstat.py b/yardstick/benchmark/scenarios/networking/nstat.py
new file mode 100644
index 000000000..df96dbda7
--- /dev/null
+++ b/yardstick/benchmark/scenarios/networking/nstat.py
@@ -0,0 +1,130 @@
+##############################################################################
+# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+from __future__ import print_function
+from __future__ import absolute_import
+
+import time
+import logging
+
+import yardstick.ssh as ssh
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+PRECISION = 3
+
+
+class Nstat(base.Scenario):
+ """Use nstat to monitor network metrics and measure IP datagram error rate
+ and etc.
+ """
+
+ __scenario_type__ = "Nstat"
+
+ def __init__(self, scenario_cfg, context_cfg):
+ """Scenario construction"""
+ self.scenario_cfg = scenario_cfg
+ self.context_cfg = context_cfg
+ self.setup_done = False
+
+ def setup(self):
+ """scenario setup"""
+ host = self.context_cfg["host"]
+ user = host.get("user", "ubuntu")
+ ssh_port = host.get("ssh_port", ssh.DEFAULT_PORT)
+ ip = host.get("ip", None)
+ key_filename = host.get('key_filename', "~/.ssh/id_rsa")
+
+ LOG.info("user:%s, host:%s", user, ip)
+ self.client = ssh.SSH(user, ip, key_filename=key_filename,
+ port=ssh_port)
+ self.client.wait(timeout=600)
+
+ self.setup_done = True
+
+ def match(self, key, field, line, results):
+ """match data in the output"""
+ if key in line:
+ results[key] = int(line.split()[field])
+
+ def calculate_error_rate(self, x, y):
+ """calculate error rate"""
+ try:
+ return round(float(x) / float(y), PRECISION)
+ except ZeroDivisionError:
+ # If incoming Errors is non-zero, but incoming data is zero
+ # consider it as 100% error rate
+ if x:
+ return 1
+ else:
+ return 0
+
+ def process_output(self, out):
+ """process output"""
+ results = {}
+ for line in out.splitlines():
+ self.match('IpInReceives', 1, line, results)
+ self.match('IpInHdrErrors', 1, line, results)
+ self.match('IpInAddrErrors', 1, line, results)
+ self.match('IcmpInMsgs', 1, line, results)
+ self.match('IcmpInErrors', 1, line, results)
+ self.match('TcpInSegs', 1, line, results)
+ self.match('TcpInErrs', 1, line, results)
+ self.match('UdpInDatagrams', 1, line, results)
+ self.match('UdpInErrors', 1, line, results)
+ results['IpErrors'] = \
+ results['IpInHdrErrors'] + results['IpInAddrErrors']
+ results['IP_datagram_error_rate'] = \
+ self.calculate_error_rate(results['IpErrors'],
+ results['IpInReceives'])
+ results['Icmp_message_error_rate'] = \
+ self.calculate_error_rate(results['IcmpInErrors'],
+ results['IcmpInMsgs'])
+ results['Tcp_segment_error_rate'] = \
+ self.calculate_error_rate(results['TcpInErrs'],
+ results['TcpInSegs'])
+ results['Udp_datagram_error_rate'] = \
+ self.calculate_error_rate(results['UdpInErrors'],
+ results['UdpInDatagrams'])
+ return results
+
+ def run(self, result):
+ """execute the benchmark"""
+
+ if not self.setup_done:
+ self.setup()
+
+ options = self.scenario_cfg['options']
+ duration = options.get('duration', 60)
+
+ time.sleep(duration)
+
+ cmd = "nstat -z"
+
+ LOG.debug("Executing command: %s", cmd)
+ status, stdout, stderr = self.client.execute(cmd)
+
+ if status:
+ raise RuntimeError(stderr)
+
+ results = self.process_output(stdout)
+
+ result.update(results)
+
+ if "sla" in self.scenario_cfg:
+ sla_error = ""
+ for i, rate in result.items():
+ if i not in self.scenario_cfg['sla']:
+ continue
+ sla_rate = float(self.scenario_cfg['sla'][i])
+ rate = float(rate)
+ if rate > sla_rate:
+ sla_error += "%s rate %f > sla:%s_rate(%f); " % \
+ (i, rate, i, sla_rate)
+ assert sla_error == "", sla_error
diff --git a/yardstick/benchmark/scenarios/networking/sfc_openstack.py b/yardstick/benchmark/scenarios/networking/sfc_openstack.py
index caaf10060..be24add32 100644
--- a/yardstick/benchmark/scenarios/networking/sfc_openstack.py
+++ b/yardstick/benchmark/scenarios/networking/sfc_openstack.py
@@ -81,7 +81,7 @@ def create_floating_ips(neutron_client): # pragma: no cover
ips = []
props = {'floating_network_id': extnet_id}
try:
- while (len(ips) < 2):
+ while len(ips) < 2:
ip_json = neutron_client.create_floatingip({'floatingip': props})
fip_addr = ip_json['floatingip']['floating_ip_address']
ips.append(fip_addr)
diff --git a/yardstick/cmd/NSBperf.py b/yardstick/cmd/NSBperf.py
index dd96b7fc8..c3730f834 100755
--- a/yardstick/cmd/NSBperf.py
+++ b/yardstick/cmd/NSBperf.py
@@ -115,7 +115,7 @@ class YardstickNSCli(object):
def generate_final_report(self, test_case):
""" Function will check if partial test results are available
and generates final report in rst format.
-"""
+ """
report_caption = '{}\n{} ({})\n{}\n\n'.format(
'================================================================',
diff --git a/yardstick/common/task_template.py b/yardstick/common/task_template.py
index bda8a1b13..9acc21336 100755
--- a/yardstick/common/task_template.py
+++ b/yardstick/common/task_template.py
@@ -45,11 +45,11 @@ def is_really_missing(mis, task_template):
# Removing variables that have default values from
# missing. Construction that won't be properly
# check is {% set x = x or 1}
- if re.search(mis.join(["{%\s*set\s+", "\s*=\s*", "[^\w]+"]),
+ if re.search(mis.join([r"{%\s*set\s+", "\s*=\s*", r"[^\w]+"]),
task_template):
return False
# Also check for a default filter which can show up as
# a missing variable
- if re.search(mis + "\s*\|\s*default\(", task_template):
+ if re.search(mis + r"\s*\|\s*default\(", task_template):
return False
return True