diff options
13 files changed, 435 insertions, 9 deletions
diff --git a/docs/userguide/opnfv_yardstick_tc043.rst b/docs/userguide/opnfv_yardstick_tc043.rst new file mode 100644 index 000000000..2f907e9ef --- /dev/null +++ b/docs/userguide/opnfv_yardstick_tc043.rst @@ -0,0 +1,75 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International +.. License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Huawei Technologies Co.,Ltd and others. + +************************************* +Yardstick Test Case Description TC043 +************************************* + +.. _cirros-image: https://download.cirros-cloud.net + ++-----------------------------------------------------------------------------+ +|Network Latency Between NFVI Nodes | +| | ++--------------+--------------------------------------------------------------+ +|test case id | OPNFV_YARDSTICK_TC043_Latency_between_ | +| | NFVI_nodes_measurements | ++--------------+--------------------------------------------------------------+ +|metric | RTT, Round Trip Time | +| | | ++--------------+--------------------------------------------------------------+ +|test purpose | To do a basic verification that network latency is within | +| | acceptable boundaries when packets travel between different | +| | nodes. | +| | | ++--------------+--------------------------------------------------------------+ +|configuration | file: opnfv_yardstick_tc043.yaml | +| | | +| | Packet size 100 bytes. Total test duration 600 seconds. | +| | One ping each 10 seconds. SLA RTT is set to maximum 10 ms. | +| | | ++--------------+--------------------------------------------------------------+ +|test tool | ping | +| | | +| | Ping is normally part of any Linux distribution, hence it | +| | doesn't need to be installed. It is also part of the | +| | Yardstick Docker image. | +| | | ++--------------+--------------------------------------------------------------+ +|references | Ping man page | +| | | +| | ETSI-NFV-TST001 | +| | | ++--------------+--------------------------------------------------------------+ +|applicability | Test case can be configured with different packet sizes, | +| | burst sizes, ping intervals and test duration. | +| | SLA is optional. The SLA in this test case serves as an | +| | example. Considerably lower RTT is expected, and | +| | also normal to achieve in balanced L2 environments. However, | +| | to cover most configurations, both bare metal and fully | +| | virtualized ones, this value should be possible to achieve | +| | and acceptable for black box testing. Many real time | +| | applications start to suffer badly if the RTT time is higher | +| | than this. Some may suffer bad also close to this RTT, while | +| | others may not suffer at all. It is a compromise that may | +| | have to be tuned for different configuration purposes. | +| | | ++--------------+--------------------------------------------------------------+ +|pre_test | Each pod node must have ping included in it. | +|conditions | | +| | | ++--------------+--------------------------------------------------------------+ +|test sequence | description and expected result | +| | | ++--------------+--------------------------------------------------------------+ +|step 1 | The pod is available. Two nodes as server and client. | +| | Ping is invoked and logs are produced and stored. | +| | | +| | Result: Logs are stored. | +| | | ++--------------+--------------------------------------------------------------+ +|test verdict | Test should not PASS if any RTT is above the optional SLA | +| | value, or if there is a test case execution problem. | +| | | ++--------------+--------------------------------------------------------------+ diff --git a/samples/cachestat.yaml b/samples/cachestat.yaml new file mode 100644 index 000000000..5786efa38 --- /dev/null +++ b/samples/cachestat.yaml @@ -0,0 +1,31 @@ +--- +# Sample benchmark task config file +# Reading cache hit/miss ratio and usage statistics + +schema: "yardstick:task:0.1" + +scenarios: +- + type: CACHEstat + options: + interval: 1 + + host: kratos.demo + + runner: + type: Duration + duration: 60 + +context: + name: demo + image: yardstick-trusty-server + flavor: yardstick-flavor + user: ubuntu + + servers: + kratos: + floating_ip: true + + networks: + test: + cidr: '10.0.1.0/24' diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml new file mode 100644 index 000000000..b45e41ebf --- /dev/null +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml @@ -0,0 +1,29 @@ +--- +# Yardstick TC043 config file +# Measure latency between NFVI nodes using ping + +schema: "yardstick:task:0.1" + +scenarios: +- + type: Ping + options: + packetsize: 100 + host: node1.LF + target: node2.LF + + runner: + type: Duration + duration: 60 + interval: 1 + + sla: + max_rtt: 10 + action: monitor + + +context: + type: Node + name: LF + file: etc/yardstick/nodes/compass_sclab_physical/pod.yaml + diff --git a/tests/opnfv/test_suites/opnfv_huawei-pod1_daily.yaml b/tests/opnfv/test_suites/opnfv_huawei-pod1_daily.yaml index 3a3bfccc0..35eeb34a7 100644 --- a/tests/opnfv/test_suites/opnfv_huawei-pod1_daily.yaml +++ b/tests/opnfv/test_suites/opnfv_huawei-pod1_daily.yaml @@ -22,3 +22,5 @@ test_cases: file_name: opnfv_yardstick_tc027.yaml - file_name: opnfv_yardstick_tc037.yaml +- + file_name: opnfv_yardstick_tc043.yaml diff --git a/tests/opnfv/test_suites/opnfv_huawei-pod1_weekly.yaml b/tests/opnfv/test_suites/opnfv_huawei-pod1_weekly.yaml index cda46969d..8a9ce4dc0 100644 --- a/tests/opnfv/test_suites/opnfv_huawei-pod1_weekly.yaml +++ b/tests/opnfv/test_suites/opnfv_huawei-pod1_weekly.yaml @@ -23,4 +23,6 @@ test_cases: - file_name: opnfv_yardstick_tc037.yaml - - file_name: opnfv_yardstick_tc040.yaml
\ No newline at end of file + file_name: opnfv_yardstick_tc040.yaml +- + file_name: opnfv_yardstick_tc043.yaml diff --git a/tests/unit/benchmark/scenarios/compute/cachestat_sample_output.txt b/tests/unit/benchmark/scenarios/compute/cachestat_sample_output.txt new file mode 100644 index 000000000..e2c79a9b1 --- /dev/null +++ b/tests/unit/benchmark/scenarios/compute/cachestat_sample_output.txt @@ -0,0 +1,5 @@ +Counting cache functions... Output every 1 seconds. + HITS MISSES DIRTIES RATIO BUFFERS_MB CACHE_MB + 6462 0 29 100.0% 1157 66782 + +Ending tracing... diff --git a/tests/unit/benchmark/scenarios/compute/test_cachestat.py b/tests/unit/benchmark/scenarios/compute/test_cachestat.py new file mode 100644 index 000000000..f5a6b5ff9 --- /dev/null +++ b/tests/unit/benchmark/scenarios/compute/test_cachestat.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Unittest for yardstick.benchmark.scenarios.compute.cachestat.CACHEstat + +import mock +import unittest +import os + +from yardstick.benchmark.scenarios.compute import cachestat + + +@mock.patch('yardstick.benchmark.scenarios.compute.cachestat.ssh') +class CACHEstatTestCase(unittest.TestCase): + + def setUp(self): + self.ctx = { + 'host': { + 'ip': '172.16.0.137', + 'user': 'root', + 'key_filename': "mykey.key" + } + } + + self.result = {} + + def test_cachestat_successful_setup(self, mock_ssh): + c = cachestat.CACHEstat({}, self.ctx) + mock_ssh.SSH().execute.return_value = (0, '', '') + + c.setup() + self.assertIsNotNone(c.client) + self.assertTrue(c.setup_done) + + def test_execute_command_success(self, mock_ssh): + c = cachestat.CACHEstat({}, self.ctx) + mock_ssh.SSH().execute.return_value = (0, '', '') + c.setup() + + expected_result = 'abcdefg' + mock_ssh.SSH().execute.return_value = (0, expected_result, '') + result = c._execute_command("foo") + self.assertEqual(result, expected_result) + + def test_execute_command_failed(self, mock_ssh): + c = cachestat.CACHEstat({}, self.ctx) + mock_ssh.SSH().execute.return_value = (0, '', '') + c.setup() + + mock_ssh.SSH().execute.return_value = (127, '', 'Failed executing \ + command') + self.assertRaises(RuntimeError, c._execute_command, + "cat /proc/meminfo") + + def test_get_cache_usage_successful(self, mock_ssh): + options = { + "interval": 1, + } + args = {"options": options} + c = cachestat.CACHEstat(args, self.ctx) + mock_ssh.SSH().execute.return_value = (0, '', '') + c.setup() + + output = self._read_file("cachestat_sample_output.txt") + mock_ssh.SSH().execute.return_value = (0, output, '') + result = c._get_cache_usage() + expected_result = {"cachestat": {"cache0": {"HITS": "6462",\ + "DIRTIES": "29", "RATIO": "100.0%", "MISSES": "0", "BUFFERS_MB": "1157",\ + "CACHE_MB": "66782"}}, "average": {"HITS": 6462, "DIRTIES": 29, "RATIO": "100.0%",\ + "MISSES": 0, "BUFFERS_MB":1157, "CACHE_MB": 66782}, "max": {"HITS": 6462,\ + "DIRTIES": 29, "RATIO": 100.0, "MISSES": 0, "BUFFERS_MB": 1157, "CACHE_MB": 66782}} + + self.assertEqual(result, expected_result) + + def _read_file(self, filename): + curr_path = os.path.dirname(os.path.abspath(__file__)) + output = os.path.join(curr_path, filename) + with open(output) as f: + sample_output = f.read() + return sample_output diff --git a/tests/unit/benchmark/scenarios/compute/test_ramspeed.py b/tests/unit/benchmark/scenarios/compute/test_ramspeed.py index 3de84c74c..100102d19 100644 --- a/tests/unit/benchmark/scenarios/compute/test_ramspeed.py +++ b/tests/unit/benchmark/scenarios/compute/test_ramspeed.py @@ -228,7 +228,8 @@ class RamspeedTestCase(unittest.TestCase): args = {'options': options} r = ramspeed.Ramspeed(args, self.ctx) - mock_ssh.SSH().execute.return_value = (1, '', 'No such type_id: 30 for Ramspeed scenario') + mock_ssh.SSH().execute.return_value = (1, '', 'No such type_id: 30 for \ + Ramspeed scenario') self.assertRaises(RuntimeError, r.run, self.result) diff --git a/tools/ubuntu-server-cloudimg-modify.sh b/tools/ubuntu-server-cloudimg-modify.sh index f9e0a2c47..2e8399a9b 100755 --- a/tools/ubuntu-server-cloudimg-modify.sh +++ b/tools/ubuntu-server-cloudimg-modify.sh @@ -1,3 +1,4 @@ +#!/bin/bash ############################################################################## # Copyright (c) 2015 Ericsson AB and others. # @@ -7,8 +8,6 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -#!/bin/bash - # installs required packages # must be run from inside the image (either chrooted or running) @@ -63,11 +62,12 @@ apt-get install -y \ git clone https://github.com/kdlucas/byte-unixbench.git /opt/tempT make --directory /opt/tempT/UnixBench/ - -git clone https://github.com/beefyamoeba5/ramspeed.git /opt/tempT2 -cd /opt/tempT2/ramspeed-2.6.0 +git clone https://github.com/beefyamoeba5/ramspeed.git /opt/tempT/RAMspeed +cd /opt/tempT/RAMspeed/ramspeed-2.6.0 mkdir temp bash build.sh +git clone https://github.com/beefyamoeba5/cachestat.git /opt/tempT/Cachestat + # restore symlink ln -sf /run/resolvconf/resolv.conf /etc/resolv.conf diff --git a/yardstick/benchmark/scenarios/compute/cache_stat.bash b/yardstick/benchmark/scenarios/compute/cache_stat.bash new file mode 100644 index 000000000..393af410c --- /dev/null +++ b/yardstick/benchmark/scenarios/compute/cache_stat.bash @@ -0,0 +1,30 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Run a cachestat cache benchmark in a host + +set -e + +INTERVAL=$1 + +run_cachestat() +{ + cd /opt/tempT/Cachestat + bash cachestat $INTERVAL +} + +main() +{ + # run the test + run_cachestat +} + +main diff --git a/yardstick/benchmark/scenarios/compute/cachestat.py b/yardstick/benchmark/scenarios/compute/cachestat.py new file mode 100644 index 000000000..da4aa754f --- /dev/null +++ b/yardstick/benchmark/scenarios/compute/cachestat.py @@ -0,0 +1,163 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +"""cache hit/miss ratio and usage statistics""" + +import pkg_resources +import logging +import re +import yardstick.ssh as ssh + +from yardstick.benchmark.scenarios import base + +LOG = logging.getLogger(__name__) + + +class CACHEstat(base.Scenario): + '''Collect cache statistics. + + This scenario reads system cache hit/miss ratio and other statistics on + a Linux host. + + cache statistics are read using 'cachestat'. + cachestat - show Linux page cache hit/miss statistics. + Uses Linux ftrace. + + This is a proof of concept using Linux ftrace capabilities on older + kernels, and works by using function profiling for in-kernel counters. + Specifically, four kernel functions are traced: + + mark_page_accessed() for measuring cache accesses + mark_buffer_dirty() for measuring cache writes + add_to_page_cache_lru() for measuring page additions + account_page_dirtied() for measuring page dirties + + It is possible that these functions have been renamed (or are different + logically) for your kernel version, and this script will not work as-is. + This script was written on Linux 3.13. This script is a sandcastle: the + kernel may wash some away, and you'll need to rebuild. + + USAGE: cachestat [-Dht] [interval] + eg, + cachestat 5 # show stats every 5 seconds + + Run "cachestat -h" for full usage. + + WARNING: This uses dynamic tracing of kernel functions, and could cause + kernel panics or freezes. Test, and know what you are doing, before use. + It also traces cache activity, which can be frequent, and cost some + overhead. The statistics should be treated as best-effort: there may be + some error margin depending on unusual workload types. + + REQUIREMENTS: CONFIG_FUNCTION_PROFILER, awk. + ''' + __scenario_type__ = "CACHEstat" + + TARGET_SCRIPT = "cache_stat.bash" + + def __init__(self, scenario_cfg, context_cfg): + """Scenario construction.""" + self.scenario_cfg = scenario_cfg + self.context_cfg = context_cfg + self.setup_done = False + + def setup(self): + """Scenario setup.""" + self.target_script = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.compute", + CACHEstat.TARGET_SCRIPT) + + host = self.context_cfg['host'] + user = host.get('user', 'ubuntu') + ip = host.get('ip', None) + key_filename = host.get('key_filename', '~/.ssh/id_rsa') + + LOG.info("user:%s, host:%s", user, ip) + self.client = ssh.SSH(user, ip, key_filename=key_filename) + self.client.wait(timeout=600) + + # copy scripts to host + self.client.run("cat > ~/cache_stat.sh", + stdin=open(self.target_script, 'rb')) + + self.setup_done = True + + def _execute_command(self, cmd): + """Execute a command on server.""" + LOG.info("Executing: %s" % cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError("Failed executing command: ", + cmd, stderr) + return stdout + + def _filtrate_result(self, result): + fields = [] + cachestat = {} + data_marker = re.compile("\d+") + ite = 0 + average = {'HITS': 0, 'MISSES': 0, 'DIRTIES': 0, 'RATIO': 0, + 'BUFFERS_MB': 0, 'CACHE_MB': 0} + maximum = {'HITS': 0, 'MISSES': 0, 'DIRTIES': 0, 'RATIO': 0, + 'BUFFERS_MB': 0, 'CACHE_MB': 0} + + # Parse cache stats + for row in result.split('\n'): + line = row.split() + + if line and line[0] == 'HITS': + # header fields + fields = line[:] + elif line and re.match(data_marker, line[0]): + cache = 'cache' + str(ite) + ite += 1 + values = line[:] + if values and len(values) == len(fields): + cachestat[cache] = dict(zip(fields, values)) + + for entry in cachestat: + for item in average: + if item != 'RATIO': + average[item] += int(cachestat[entry][item]) + else: + average[item] += float(cachestat[entry][item][:-1]) + + for item in maximum: + if item != 'RATIO': + if int(cachestat[entry][item]) > maximum[item]: + maximum[item] = int(cachestat[entry][item]) + else: + if float(cachestat[entry][item][:-1]) > maximum[item]: + maximum[item] = float(cachestat[entry][item][:-1]) + + for item in average: + if item != 'RATIO': + average[item] = average[item] / len(cachestat) + else: + average[item] = str(average[item] / len(cachestat)) + '%' + + return {'cachestat': cachestat, 'average': average, 'max': maximum} + + def _get_cache_usage(self): + """Get cache statistics.""" + options = self.scenario_cfg['options'] + interval = options.get("interval", 1) + + cmd = "sudo bash cache_stat.sh %s" % (interval) + + result = self._execute_command(cmd) + filtrated_result = self._filtrate_result(result) + + return filtrated_result + + def run(self, result): + if not self.setup_done: + self.setup() + + result.update(self._get_cache_usage()) diff --git a/yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash b/yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash index fcb655968..526d0b9f6 100644 --- a/yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash +++ b/yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash @@ -21,7 +21,7 @@ OUTPUT_FILE=/tmp/ramspeed-out.log run_ramspeed() { - cd /opt/tempT2/ramspeed-2.6.0/ + cd /opt/tempT/RAMspeed/ramspeed-2.6.0/ ./ramspeed -b $TYPE_ID -g $LOAD -m $BLOCK_SIZE > $OUTPUT_FILE } diff --git a/yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash b/yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash index 69c2934af..931a06375 100644 --- a/yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash +++ b/yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash @@ -22,7 +22,7 @@ OUTPUT_FILE=/tmp/ramspeed-out.log run_ramspeed() { - cd /opt/tempT2/ramspeed-2.6.0/ + cd /opt/tempT/RAMspeed/ramspeed-2.6.0/ ./ramspeed -b $TYPE_ID -g $LOAD -m $BLOCK_SIZE -l $ITERATION > $OUTPUT_FILE } |