summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--docs/userguide/opnfv_yardstick_tc043.rst75
-rw-r--r--samples/cachestat.yaml31
-rw-r--r--tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml29
-rw-r--r--tests/opnfv/test_suites/opnfv_huawei-pod1_daily.yaml2
-rw-r--r--tests/opnfv/test_suites/opnfv_huawei-pod1_weekly.yaml4
-rw-r--r--tests/unit/benchmark/scenarios/compute/cachestat_sample_output.txt5
-rw-r--r--tests/unit/benchmark/scenarios/compute/test_cachestat.py88
-rw-r--r--tests/unit/benchmark/scenarios/compute/test_ramspeed.py3
-rwxr-xr-xtools/ubuntu-server-cloudimg-modify.sh10
-rw-r--r--yardstick/benchmark/scenarios/compute/cache_stat.bash30
-rw-r--r--yardstick/benchmark/scenarios/compute/cachestat.py163
-rw-r--r--yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash2
-rw-r--r--yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash2
13 files changed, 435 insertions, 9 deletions
diff --git a/docs/userguide/opnfv_yardstick_tc043.rst b/docs/userguide/opnfv_yardstick_tc043.rst
new file mode 100644
index 000000000..2f907e9ef
--- /dev/null
+++ b/docs/userguide/opnfv_yardstick_tc043.rst
@@ -0,0 +1,75 @@
+.. This work is licensed under a Creative Commons Attribution 4.0 International
+.. License.
+.. http://creativecommons.org/licenses/by/4.0
+.. (c) OPNFV, Huawei Technologies Co.,Ltd and others.
+
+*************************************
+Yardstick Test Case Description TC043
+*************************************
+
+.. _cirros-image: https://download.cirros-cloud.net
+
++-----------------------------------------------------------------------------+
+|Network Latency Between NFVI Nodes |
+| |
++--------------+--------------------------------------------------------------+
+|test case id | OPNFV_YARDSTICK_TC043_Latency_between_ |
+| | NFVI_nodes_measurements |
++--------------+--------------------------------------------------------------+
+|metric | RTT, Round Trip Time |
+| | |
++--------------+--------------------------------------------------------------+
+|test purpose | To do a basic verification that network latency is within |
+| | acceptable boundaries when packets travel between different |
+| | nodes. |
+| | |
++--------------+--------------------------------------------------------------+
+|configuration | file: opnfv_yardstick_tc043.yaml |
+| | |
+| | Packet size 100 bytes. Total test duration 600 seconds. |
+| | One ping each 10 seconds. SLA RTT is set to maximum 10 ms. |
+| | |
++--------------+--------------------------------------------------------------+
+|test tool | ping |
+| | |
+| | Ping is normally part of any Linux distribution, hence it |
+| | doesn't need to be installed. It is also part of the |
+| | Yardstick Docker image. |
+| | |
++--------------+--------------------------------------------------------------+
+|references | Ping man page |
+| | |
+| | ETSI-NFV-TST001 |
+| | |
++--------------+--------------------------------------------------------------+
+|applicability | Test case can be configured with different packet sizes, |
+| | burst sizes, ping intervals and test duration. |
+| | SLA is optional. The SLA in this test case serves as an |
+| | example. Considerably lower RTT is expected, and |
+| | also normal to achieve in balanced L2 environments. However, |
+| | to cover most configurations, both bare metal and fully |
+| | virtualized ones, this value should be possible to achieve |
+| | and acceptable for black box testing. Many real time |
+| | applications start to suffer badly if the RTT time is higher |
+| | than this. Some may suffer bad also close to this RTT, while |
+| | others may not suffer at all. It is a compromise that may |
+| | have to be tuned for different configuration purposes. |
+| | |
++--------------+--------------------------------------------------------------+
+|pre_test | Each pod node must have ping included in it. |
+|conditions | |
+| | |
++--------------+--------------------------------------------------------------+
+|test sequence | description and expected result |
+| | |
++--------------+--------------------------------------------------------------+
+|step 1 | The pod is available. Two nodes as server and client. |
+| | Ping is invoked and logs are produced and stored. |
+| | |
+| | Result: Logs are stored. |
+| | |
++--------------+--------------------------------------------------------------+
+|test verdict | Test should not PASS if any RTT is above the optional SLA |
+| | value, or if there is a test case execution problem. |
+| | |
++--------------+--------------------------------------------------------------+
diff --git a/samples/cachestat.yaml b/samples/cachestat.yaml
new file mode 100644
index 000000000..5786efa38
--- /dev/null
+++ b/samples/cachestat.yaml
@@ -0,0 +1,31 @@
+---
+# Sample benchmark task config file
+# Reading cache hit/miss ratio and usage statistics
+
+schema: "yardstick:task:0.1"
+
+scenarios:
+-
+ type: CACHEstat
+ options:
+ interval: 1
+
+ host: kratos.demo
+
+ runner:
+ type: Duration
+ duration: 60
+
+context:
+ name: demo
+ image: yardstick-trusty-server
+ flavor: yardstick-flavor
+ user: ubuntu
+
+ servers:
+ kratos:
+ floating_ip: true
+
+ networks:
+ test:
+ cidr: '10.0.1.0/24'
diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml
new file mode 100644
index 000000000..b45e41ebf
--- /dev/null
+++ b/tests/opnfv/test_cases/opnfv_yardstick_tc043.yaml
@@ -0,0 +1,29 @@
+---
+# Yardstick TC043 config file
+# Measure latency between NFVI nodes using ping
+
+schema: "yardstick:task:0.1"
+
+scenarios:
+-
+ type: Ping
+ options:
+ packetsize: 100
+ host: node1.LF
+ target: node2.LF
+
+ runner:
+ type: Duration
+ duration: 60
+ interval: 1
+
+ sla:
+ max_rtt: 10
+ action: monitor
+
+
+context:
+ type: Node
+ name: LF
+ file: etc/yardstick/nodes/compass_sclab_physical/pod.yaml
+
diff --git a/tests/opnfv/test_suites/opnfv_huawei-pod1_daily.yaml b/tests/opnfv/test_suites/opnfv_huawei-pod1_daily.yaml
index 3a3bfccc0..35eeb34a7 100644
--- a/tests/opnfv/test_suites/opnfv_huawei-pod1_daily.yaml
+++ b/tests/opnfv/test_suites/opnfv_huawei-pod1_daily.yaml
@@ -22,3 +22,5 @@ test_cases:
file_name: opnfv_yardstick_tc027.yaml
-
file_name: opnfv_yardstick_tc037.yaml
+-
+ file_name: opnfv_yardstick_tc043.yaml
diff --git a/tests/opnfv/test_suites/opnfv_huawei-pod1_weekly.yaml b/tests/opnfv/test_suites/opnfv_huawei-pod1_weekly.yaml
index cda46969d..8a9ce4dc0 100644
--- a/tests/opnfv/test_suites/opnfv_huawei-pod1_weekly.yaml
+++ b/tests/opnfv/test_suites/opnfv_huawei-pod1_weekly.yaml
@@ -23,4 +23,6 @@ test_cases:
-
file_name: opnfv_yardstick_tc037.yaml
-
- file_name: opnfv_yardstick_tc040.yaml \ No newline at end of file
+ file_name: opnfv_yardstick_tc040.yaml
+-
+ file_name: opnfv_yardstick_tc043.yaml
diff --git a/tests/unit/benchmark/scenarios/compute/cachestat_sample_output.txt b/tests/unit/benchmark/scenarios/compute/cachestat_sample_output.txt
new file mode 100644
index 000000000..e2c79a9b1
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/compute/cachestat_sample_output.txt
@@ -0,0 +1,5 @@
+Counting cache functions... Output every 1 seconds.
+ HITS MISSES DIRTIES RATIO BUFFERS_MB CACHE_MB
+ 6462 0 29 100.0% 1157 66782
+
+Ending tracing...
diff --git a/tests/unit/benchmark/scenarios/compute/test_cachestat.py b/tests/unit/benchmark/scenarios/compute/test_cachestat.py
new file mode 100644
index 000000000..f5a6b5ff9
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/compute/test_cachestat.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+
+##############################################################################
+# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Unittest for yardstick.benchmark.scenarios.compute.cachestat.CACHEstat
+
+import mock
+import unittest
+import os
+
+from yardstick.benchmark.scenarios.compute import cachestat
+
+
+@mock.patch('yardstick.benchmark.scenarios.compute.cachestat.ssh')
+class CACHEstatTestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.ctx = {
+ 'host': {
+ 'ip': '172.16.0.137',
+ 'user': 'root',
+ 'key_filename': "mykey.key"
+ }
+ }
+
+ self.result = {}
+
+ def test_cachestat_successful_setup(self, mock_ssh):
+ c = cachestat.CACHEstat({}, self.ctx)
+ mock_ssh.SSH().execute.return_value = (0, '', '')
+
+ c.setup()
+ self.assertIsNotNone(c.client)
+ self.assertTrue(c.setup_done)
+
+ def test_execute_command_success(self, mock_ssh):
+ c = cachestat.CACHEstat({}, self.ctx)
+ mock_ssh.SSH().execute.return_value = (0, '', '')
+ c.setup()
+
+ expected_result = 'abcdefg'
+ mock_ssh.SSH().execute.return_value = (0, expected_result, '')
+ result = c._execute_command("foo")
+ self.assertEqual(result, expected_result)
+
+ def test_execute_command_failed(self, mock_ssh):
+ c = cachestat.CACHEstat({}, self.ctx)
+ mock_ssh.SSH().execute.return_value = (0, '', '')
+ c.setup()
+
+ mock_ssh.SSH().execute.return_value = (127, '', 'Failed executing \
+ command')
+ self.assertRaises(RuntimeError, c._execute_command,
+ "cat /proc/meminfo")
+
+ def test_get_cache_usage_successful(self, mock_ssh):
+ options = {
+ "interval": 1,
+ }
+ args = {"options": options}
+ c = cachestat.CACHEstat(args, self.ctx)
+ mock_ssh.SSH().execute.return_value = (0, '', '')
+ c.setup()
+
+ output = self._read_file("cachestat_sample_output.txt")
+ mock_ssh.SSH().execute.return_value = (0, output, '')
+ result = c._get_cache_usage()
+ expected_result = {"cachestat": {"cache0": {"HITS": "6462",\
+ "DIRTIES": "29", "RATIO": "100.0%", "MISSES": "0", "BUFFERS_MB": "1157",\
+ "CACHE_MB": "66782"}}, "average": {"HITS": 6462, "DIRTIES": 29, "RATIO": "100.0%",\
+ "MISSES": 0, "BUFFERS_MB":1157, "CACHE_MB": 66782}, "max": {"HITS": 6462,\
+ "DIRTIES": 29, "RATIO": 100.0, "MISSES": 0, "BUFFERS_MB": 1157, "CACHE_MB": 66782}}
+
+ self.assertEqual(result, expected_result)
+
+ def _read_file(self, filename):
+ curr_path = os.path.dirname(os.path.abspath(__file__))
+ output = os.path.join(curr_path, filename)
+ with open(output) as f:
+ sample_output = f.read()
+ return sample_output
diff --git a/tests/unit/benchmark/scenarios/compute/test_ramspeed.py b/tests/unit/benchmark/scenarios/compute/test_ramspeed.py
index 3de84c74c..100102d19 100644
--- a/tests/unit/benchmark/scenarios/compute/test_ramspeed.py
+++ b/tests/unit/benchmark/scenarios/compute/test_ramspeed.py
@@ -228,7 +228,8 @@ class RamspeedTestCase(unittest.TestCase):
args = {'options': options}
r = ramspeed.Ramspeed(args, self.ctx)
- mock_ssh.SSH().execute.return_value = (1, '', 'No such type_id: 30 for Ramspeed scenario')
+ mock_ssh.SSH().execute.return_value = (1, '', 'No such type_id: 30 for \
+ Ramspeed scenario')
self.assertRaises(RuntimeError, r.run, self.result)
diff --git a/tools/ubuntu-server-cloudimg-modify.sh b/tools/ubuntu-server-cloudimg-modify.sh
index f9e0a2c47..2e8399a9b 100755
--- a/tools/ubuntu-server-cloudimg-modify.sh
+++ b/tools/ubuntu-server-cloudimg-modify.sh
@@ -1,3 +1,4 @@
+#!/bin/bash
##############################################################################
# Copyright (c) 2015 Ericsson AB and others.
#
@@ -7,8 +8,6 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-#!/bin/bash
-
# installs required packages
# must be run from inside the image (either chrooted or running)
@@ -63,11 +62,12 @@ apt-get install -y \
git clone https://github.com/kdlucas/byte-unixbench.git /opt/tempT
make --directory /opt/tempT/UnixBench/
-
-git clone https://github.com/beefyamoeba5/ramspeed.git /opt/tempT2
-cd /opt/tempT2/ramspeed-2.6.0
+git clone https://github.com/beefyamoeba5/ramspeed.git /opt/tempT/RAMspeed
+cd /opt/tempT/RAMspeed/ramspeed-2.6.0
mkdir temp
bash build.sh
+git clone https://github.com/beefyamoeba5/cachestat.git /opt/tempT/Cachestat
+
# restore symlink
ln -sf /run/resolvconf/resolv.conf /etc/resolv.conf
diff --git a/yardstick/benchmark/scenarios/compute/cache_stat.bash b/yardstick/benchmark/scenarios/compute/cache_stat.bash
new file mode 100644
index 000000000..393af410c
--- /dev/null
+++ b/yardstick/benchmark/scenarios/compute/cache_stat.bash
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+##############################################################################
+# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Run a cachestat cache benchmark in a host
+
+set -e
+
+INTERVAL=$1
+
+run_cachestat()
+{
+ cd /opt/tempT/Cachestat
+ bash cachestat $INTERVAL
+}
+
+main()
+{
+ # run the test
+ run_cachestat
+}
+
+main
diff --git a/yardstick/benchmark/scenarios/compute/cachestat.py b/yardstick/benchmark/scenarios/compute/cachestat.py
new file mode 100644
index 000000000..da4aa754f
--- /dev/null
+++ b/yardstick/benchmark/scenarios/compute/cachestat.py
@@ -0,0 +1,163 @@
+##############################################################################
+# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+"""cache hit/miss ratio and usage statistics"""
+
+import pkg_resources
+import logging
+import re
+import yardstick.ssh as ssh
+
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class CACHEstat(base.Scenario):
+ '''Collect cache statistics.
+
+ This scenario reads system cache hit/miss ratio and other statistics on
+ a Linux host.
+
+ cache statistics are read using 'cachestat'.
+ cachestat - show Linux page cache hit/miss statistics.
+ Uses Linux ftrace.
+
+ This is a proof of concept using Linux ftrace capabilities on older
+ kernels, and works by using function profiling for in-kernel counters.
+ Specifically, four kernel functions are traced:
+
+ mark_page_accessed() for measuring cache accesses
+ mark_buffer_dirty() for measuring cache writes
+ add_to_page_cache_lru() for measuring page additions
+ account_page_dirtied() for measuring page dirties
+
+ It is possible that these functions have been renamed (or are different
+ logically) for your kernel version, and this script will not work as-is.
+ This script was written on Linux 3.13. This script is a sandcastle: the
+ kernel may wash some away, and you'll need to rebuild.
+
+ USAGE: cachestat [-Dht] [interval]
+ eg,
+ cachestat 5 # show stats every 5 seconds
+
+ Run "cachestat -h" for full usage.
+
+ WARNING: This uses dynamic tracing of kernel functions, and could cause
+ kernel panics or freezes. Test, and know what you are doing, before use.
+ It also traces cache activity, which can be frequent, and cost some
+ overhead. The statistics should be treated as best-effort: there may be
+ some error margin depending on unusual workload types.
+
+ REQUIREMENTS: CONFIG_FUNCTION_PROFILER, awk.
+ '''
+ __scenario_type__ = "CACHEstat"
+
+ TARGET_SCRIPT = "cache_stat.bash"
+
+ def __init__(self, scenario_cfg, context_cfg):
+ """Scenario construction."""
+ self.scenario_cfg = scenario_cfg
+ self.context_cfg = context_cfg
+ self.setup_done = False
+
+ def setup(self):
+ """Scenario setup."""
+ self.target_script = pkg_resources.resource_filename(
+ "yardstick.benchmark.scenarios.compute",
+ CACHEstat.TARGET_SCRIPT)
+
+ host = self.context_cfg['host']
+ user = host.get('user', 'ubuntu')
+ ip = host.get('ip', None)
+ key_filename = host.get('key_filename', '~/.ssh/id_rsa')
+
+ LOG.info("user:%s, host:%s", user, ip)
+ self.client = ssh.SSH(user, ip, key_filename=key_filename)
+ self.client.wait(timeout=600)
+
+ # copy scripts to host
+ self.client.run("cat > ~/cache_stat.sh",
+ stdin=open(self.target_script, 'rb'))
+
+ self.setup_done = True
+
+ def _execute_command(self, cmd):
+ """Execute a command on server."""
+ LOG.info("Executing: %s" % cmd)
+ status, stdout, stderr = self.client.execute(cmd)
+ if status:
+ raise RuntimeError("Failed executing command: ",
+ cmd, stderr)
+ return stdout
+
+ def _filtrate_result(self, result):
+ fields = []
+ cachestat = {}
+ data_marker = re.compile("\d+")
+ ite = 0
+ average = {'HITS': 0, 'MISSES': 0, 'DIRTIES': 0, 'RATIO': 0,
+ 'BUFFERS_MB': 0, 'CACHE_MB': 0}
+ maximum = {'HITS': 0, 'MISSES': 0, 'DIRTIES': 0, 'RATIO': 0,
+ 'BUFFERS_MB': 0, 'CACHE_MB': 0}
+
+ # Parse cache stats
+ for row in result.split('\n'):
+ line = row.split()
+
+ if line and line[0] == 'HITS':
+ # header fields
+ fields = line[:]
+ elif line and re.match(data_marker, line[0]):
+ cache = 'cache' + str(ite)
+ ite += 1
+ values = line[:]
+ if values and len(values) == len(fields):
+ cachestat[cache] = dict(zip(fields, values))
+
+ for entry in cachestat:
+ for item in average:
+ if item != 'RATIO':
+ average[item] += int(cachestat[entry][item])
+ else:
+ average[item] += float(cachestat[entry][item][:-1])
+
+ for item in maximum:
+ if item != 'RATIO':
+ if int(cachestat[entry][item]) > maximum[item]:
+ maximum[item] = int(cachestat[entry][item])
+ else:
+ if float(cachestat[entry][item][:-1]) > maximum[item]:
+ maximum[item] = float(cachestat[entry][item][:-1])
+
+ for item in average:
+ if item != 'RATIO':
+ average[item] = average[item] / len(cachestat)
+ else:
+ average[item] = str(average[item] / len(cachestat)) + '%'
+
+ return {'cachestat': cachestat, 'average': average, 'max': maximum}
+
+ def _get_cache_usage(self):
+ """Get cache statistics."""
+ options = self.scenario_cfg['options']
+ interval = options.get("interval", 1)
+
+ cmd = "sudo bash cache_stat.sh %s" % (interval)
+
+ result = self._execute_command(cmd)
+ filtrated_result = self._filtrate_result(result)
+
+ return filtrated_result
+
+ def run(self, result):
+ if not self.setup_done:
+ self.setup()
+
+ result.update(self._get_cache_usage())
diff --git a/yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash b/yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash
index fcb655968..526d0b9f6 100644
--- a/yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash
+++ b/yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash
@@ -21,7 +21,7 @@ OUTPUT_FILE=/tmp/ramspeed-out.log
run_ramspeed()
{
- cd /opt/tempT2/ramspeed-2.6.0/
+ cd /opt/tempT/RAMspeed/ramspeed-2.6.0/
./ramspeed -b $TYPE_ID -g $LOAD -m $BLOCK_SIZE > $OUTPUT_FILE
}
diff --git a/yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash b/yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash
index 69c2934af..931a06375 100644
--- a/yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash
+++ b/yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash
@@ -22,7 +22,7 @@ OUTPUT_FILE=/tmp/ramspeed-out.log
run_ramspeed()
{
- cd /opt/tempT2/ramspeed-2.6.0/
+ cd /opt/tempT/RAMspeed/ramspeed-2.6.0/
./ramspeed -b $TYPE_ID -g $LOAD -m $BLOCK_SIZE -l $ITERATION > $OUTPUT_FILE
}