diff options
| -rw-r--r-- | samples/ramspeed.yaml | 38 | ||||
| -rw-r--r-- | tests/unit/benchmark/scenarios/compute/test_ramspeed.py | 239 | ||||
| -rwxr-xr-x | tools/ubuntu-server-cloudimg-modify.sh | 7 | ||||
| -rw-r--r-- | yardstick/benchmark/scenarios/compute/ramspeed.py | 142 | ||||
| -rw-r--r-- | yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash | 46 | ||||
| -rw-r--r-- | yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash | 47 |
6 files changed, 519 insertions, 0 deletions
diff --git a/samples/ramspeed.yaml b/samples/ramspeed.yaml new file mode 100644 index 000000000..7e1b1aa8c --- /dev/null +++ b/samples/ramspeed.yaml @@ -0,0 +1,38 @@ +--- +# Sample benchmark task config file +# measure cache and memory bandwidth using Ramspeed + +schema: "yardstick:task:0.1" + +scenarios: +- + type: Ramspeed + options: + type_id: 1 + load: 16 + block_size: 32 + iteration: 1 + + host: kratos.demo + + runner: + type: Iteration + iterations: 1 + + sla: + min_bandwidth: 6000 + action: monitor + +context: + name: demo + image: yardstick-trusty-server + flavor: yardstick-flavor + user: ubuntu + + servers: + kratos: + floating_ip: true + + networks: + test: + cidr: '10.0.1.0/24' diff --git a/tests/unit/benchmark/scenarios/compute/test_ramspeed.py b/tests/unit/benchmark/scenarios/compute/test_ramspeed.py new file mode 100644 index 000000000..3de84c74c --- /dev/null +++ b/tests/unit/benchmark/scenarios/compute/test_ramspeed.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python + +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Unittest for yardstick.benchmark.scenarios.compute.ramspeed.Ramspeed + +import mock +import unittest +import json + +from yardstick.benchmark.scenarios.compute import ramspeed + + +@mock.patch('yardstick.benchmark.scenarios.compute.ramspeed.ssh') +class RamspeedTestCase(unittest.TestCase): + + def setUp(self): + self.ctx = { + 'host': { + 'ip': '172.16.0.137', + 'user': 'root', + 'key_filename': "mykey.key" + } + } + + self.result = {} + + def test_ramspeed_successful_setup(self, mock_ssh): + + r = ramspeed.Ramspeed({}, self.ctx) + mock_ssh.SSH().execute.return_value = (0, '', '') + + r.setup() + self.assertIsNotNone(r.client) + self.assertTrue(r.setup_done, True) + + def test_ramspeed_successful__run_no_sla(self, mock_ssh): + + options = { + "test_id": 1, + "load": 16, + "block_size": 32 + } + args = {"options": options} + r = ramspeed.Ramspeed(args, self.ctx) + + sample_output = '{"Result": [{"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 1, "Bandwidth(MBps)": 19909.18}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 2, "Bandwidth(MBps)": 19873.89},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 4, "Bandwidth(MBps)":\ + 19907.56}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 8,\ + "Bandwidth(MBps)": 19906.94}, {"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 16, "Bandwidth(MBps)": 19881.74}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 32, "Bandwidth(MBps)": 19395.65},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 64, "Bandwidth(MBps)":\ + 17623.14}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 128,\ + "Bandwidth(MBps)": 17677.36}, {"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 256, "Bandwidth(MBps)": 16113.49}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 512, "Bandwidth(MBps)": 14659.19},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 1024, "Bandwidth(MBps)":\ + 14680.75}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 2048,\ + "Bandwidth(MBps)": 14756.45}, {"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 4096, "Bandwidth(MBps)": 14604.44}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 8192, "Bandwidth(MBps)": 14159.86},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 16384, "Bandwidth(MBps)":\ + 14128.94}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 32768,\ + "Bandwidth(MBps)": 8340.85}]}' + mock_ssh.SSH().execute.return_value = (0, sample_output, '') + r.run(self.result) + expected_result = json.loads(sample_output) + self.assertEqual(self.result, expected_result) + + def test_ramspeed_successful_run_sla(self, mock_ssh): + + options = { + "test_id": 1, + "load": 16, + "block_size": 32 + } + args = {"options": options, "sla": {"min_bandwidth": 6000}} + r = ramspeed.Ramspeed(args, self.ctx) + + sample_output = '{"Result": [{"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 1, "Bandwidth(MBps)": 19909.18}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 2, "Bandwidth(MBps)": 19873.89},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 4, "Bandwidth(MBps)":\ + 19907.56}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 8,\ + "Bandwidth(MBps)": 19906.94}, {"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 16, "Bandwidth(MBps)": 19881.74}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 32, "Bandwidth(MBps)": 19395.65},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 64, "Bandwidth(MBps)":\ + 17623.14}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 128,\ + "Bandwidth(MBps)": 17677.36}, {"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 256, "Bandwidth(MBps)": 16113.49}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 512, "Bandwidth(MBps)": 14659.19},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 1024, "Bandwidth(MBps)":\ + 14680.75}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 2048,\ + "Bandwidth(MBps)": 14756.45}, {"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 4096, "Bandwidth(MBps)": 14604.44}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 8192, "Bandwidth(MBps)": 14159.86},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 16384, "Bandwidth(MBps)":\ + 14128.94}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 32768,\ + "Bandwidth(MBps)": 8340.85}]}' + mock_ssh.SSH().execute.return_value = (0, sample_output, '') + r.run(self.result) + expected_result = json.loads(sample_output) + self.assertEqual(self.result, expected_result) + + def test_ramspeed_unsuccessful_run_sla(self, mock_ssh): + options = { + "test_id": 1, + "load": 8, + "block_size": 64 + } + args = {"options": options, "sla": {"min_bandwidth": 100000}} + r = ramspeed.Ramspeed(args, self.ctx) + + sample_output = '{"Result": [{"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 1, "Bandwidth(MBps)": 5000.18}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 2, "Bandwidth(MBps)": 5000.89},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 4,\ + "Bandwidth(MBps)": 5000.56}, {"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 8, "Bandwidth(MBps)": 19906.94}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 16, "Bandwidth(MBps)": 19881.74},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 32,\ + "Bandwidth(MBps)": 19395.65}, {"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 64, "Bandwidth(MBps)": 17623.14}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 128, "Bandwidth(MBps)": 17677.36},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 256, "Bandwidth(MBps)":\ + 16113.49}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 512,\ + "Bandwidth(MBps)": 14659.19}, {"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 1024, "Bandwidth(MBps)": 14680.75}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 2048, "Bandwidth(MBps)": 14756.45},\ + {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 4096, "Bandwidth(MBps)":\ + 14604.44}, {"Test_type": "INTEGER & WRITING", "Block_size(kb)": 8192,\ + "Bandwidth(MBps)": 14159.86}, {"Test_type": "INTEGER & WRITING",\ + "Block_size(kb)": 16384, "Bandwidth(MBps)": 14128.94}, {"Test_type":\ + "INTEGER & WRITING", "Block_size(kb)": 32768, "Bandwidth(MBps)": 8340.85}]}' + mock_ssh.SSH().execute.return_value = (0, sample_output, '') + self.assertRaises(AssertionError, r.run, self.result) + + def test_ramspeed_unsuccessful_script_error(self, mock_ssh): + options = { + "test_id": 1, + "load": 16, + "block_size": 32 + } + args = {"options": options} + r = ramspeed.Ramspeed(args, self.ctx) + + mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR') + self.assertRaises(RuntimeError, r.run, self.result) + + def test_ramspeed_mem_successful_run_no_sla(self, mock_ssh): + options = { + "test_id": 3, + "load": 16, + "block_size": 32, + "iteration": 1 + } + args = {"options": options} + r = ramspeed.Ramspeed(args, self.ctx) + + sample_output = '{"Result": [{"Test_type": "INTEGER Copy:",\ + "Bandwidth(MBps)": 8353.97}, {"Test_type": "INTEGER Scale:",\ + "Bandwidth(MBps)": 9078.59}, {"Test_type": "INTEGER Add:",\ + "Bandwidth(MBps)": 10057.48}, {"Test_type": "INTEGER Triad:",\ + "Bandwidth(MBps)": 10116.27}, {"Test_type": "INTEGER AVERAGE:",\ + "Bandwidth(MBps)": 9401.58}]}' + mock_ssh.SSH().execute.return_value = (0, sample_output, '') + r.run(self.result) + expected_result = json.loads(sample_output) + self.assertEqual(self.result, expected_result) + + def test_ramspeed_mem_successful_run_sla(self, mock_ssh): + options = { + "test_id": 3, + "load": 16, + "block_size": 32, + "iteration": 1 + } + args = {"options": options, "sla": {"min_bandwidth": 6000}} + r = ramspeed.Ramspeed(args, self.ctx) + + sample_output = '{"Result": [{"Test_type": "INTEGER Copy:",\ + "Bandwidth(MBps)": 8353.97}, {"Test_type": "INTEGER Scale:",\ + "Bandwidth(MBps)": 9078.59}, {"Test_type": "INTEGER Add:",\ + "Bandwidth(MBps)": 10057.48}, {"Test_type": "INTEGER Triad:",\ + "Bandwidth(MBps)": 10116.27}, {"Test_type": "INTEGER AVERAGE:",\ + "Bandwidth(MBps)": 9401.58}]}' + mock_ssh.SSH().execute.return_value = (0, sample_output, '') + r.run(self.result) + expected_result = json.loads(sample_output) + self.assertEqual(self.result, expected_result) + + def test_ramspeed_mem_unsuccessful_run_sla(self, mock_ssh): + options = { + "test_id": 3, + "load": 16, + "block_size": 32, + "iteration": 1 + } + args = {"options": options, "sla": {"min_bandwidth": 86000}} + r = ramspeed.Ramspeed(args, self.ctx) + + sample_output = '{"Result": [{"Test_type": "INTEGER Copy:",\ + "Bandwidth(MBps)": 4000.97}, {"Test_type": "INTEGER Scale:",\ + "Bandwidth(MBps)": 4400.59}, {"Test_type": "INTEGER Add:",\ + "Bandwidth(MBps)": 4300.48}, {"Test_type": "INTEGER Triad:",\ + "Bandwidth(MBps)": 1300.27}, {"Test_type": "INTEGER AVERAGE:",\ + "Bandwidth(MBps)": 2401.58}]}' + mock_ssh.SSH().execute.return_value = (0, sample_output, '') + self.assertRaises(AssertionError, r.run, self.result) + + def test_ramspeed_unsuccessful_unknown_type_run(self, mock_ssh): + options = { + "test_id": 30, + "load": 16, + "block_size": 32 + } + args = {'options': options} + r = ramspeed.Ramspeed(args, self.ctx) + + mock_ssh.SSH().execute.return_value = (1, '', 'No such type_id: 30 for Ramspeed scenario') + self.assertRaises(RuntimeError, r.run, self.result) + + +def main(): + unittest.main() + +if __name__ == '__main__': + main() diff --git a/tools/ubuntu-server-cloudimg-modify.sh b/tools/ubuntu-server-cloudimg-modify.sh index 78bf098dc..f9e0a2c47 100755 --- a/tools/ubuntu-server-cloudimg-modify.sh +++ b/tools/ubuntu-server-cloudimg-modify.sh @@ -62,5 +62,12 @@ apt-get install -y \ git clone https://github.com/kdlucas/byte-unixbench.git /opt/tempT make --directory /opt/tempT/UnixBench/ + + +git clone https://github.com/beefyamoeba5/ramspeed.git /opt/tempT2 +cd /opt/tempT2/ramspeed-2.6.0 +mkdir temp +bash build.sh + # restore symlink ln -sf /run/resolvconf/resolv.conf /etc/resolv.conf diff --git a/yardstick/benchmark/scenarios/compute/ramspeed.py b/yardstick/benchmark/scenarios/compute/ramspeed.py new file mode 100644 index 000000000..819ef769b --- /dev/null +++ b/yardstick/benchmark/scenarios/compute/ramspeed.py @@ -0,0 +1,142 @@ +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import pkg_resources +import logging +import json + +import yardstick.ssh as ssh +from yardstick.benchmark.scenarios import base + +LOG = logging.getLogger(__name__) + + +class Ramspeed(base.Scenario): + """Execute ramspeed benchmark in a host + The ramspeed script takes a number of options which you can use to + customise a test. The full run time usage + is: + + ramspeed -b ID [-g size] [-m size] [-l runs] [-r speed-format] + + -b runs a specified benchmark (by an ID number): + 1 -- INTmark [writing] 4 -- FLOATmark [writing] + 2 -- INTmark [reading] 5 -- FLOATmark [reading] + 3 -- INTmem 6 -- FLOATmem + 7 -- MMXmark [writing] 10 -- SSEmark [writing] + 8 -- MMXmark [reading] 11 -- SSEmark [reading] + 9 -- MMXmem 12 -- SSEmem + 13 -- MMXmark (nt) [writing] 16 -- SSEmark (nt) [writing] + 14 -- MMXmark (nt) [reading] 17 -- SSEmark (nt) [reading] + 15 -- MMXmem (nt) 18 -- SSEmem (nt) + In this scenario, only the first 6 test type will be used for testing. + + -g specifies a # of Gbytes per pass (default is 8) + -m specifies a # of Mbytes per array (default is 32) + -l enables the BatchRun mode (for *mem benchmarks only), + and specifies a # of runs (suggested is 5) + -r displays speeds in real megabytes per second (default: decimal) + + The -b option is required, others are recommended. + + Parameters + type_id - specifies whether to run *mark benchmark or *mem benchmark + the type_id can be any number from 1 to 19 + type: int + unit: n/a + default: 1 + load - specifies a # of Gbytes per pass + type: int + unit: gigabyte + default: 8 + block_size - specifies a # of Mbytes per array + type: int + unit: megabyte + default: 32 + + Parameters for *mem benchmark + iteration - specifies a # of runs for each test + type: int + unit: n/a + default: 1 + more info http://alasir.com/software/ramspeed + """ + __scenario_type__ = "Ramspeed" + + RAMSPEED_MARK_SCRIPT = "ramspeed_mark_benchmark.bash" + RAMSPEED_MEM_SCRIPT = "ramspeed_mem_benchmark.bash" + + def __init__(self, scenario_cfg, context_cfg): + self.scenario_cfg = scenario_cfg + self.context_cfg = context_cfg + self.setup_done = False + + def setup(self): + """scenario setup""" + self.mark_target_script = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.compute", + Ramspeed.RAMSPEED_MARK_SCRIPT) + self.mem_target_script = pkg_resources.resource_filename( + "yardstick.benchmark.scenarios.compute", + Ramspeed.RAMSPEED_MEM_SCRIPT) + + host = self.context_cfg["host"] + user = host.get("user", "ubuntu") + ip = host.get("ip", None) + key_filename = host.get('key_filename', "~/.ssh/id_rsa") + + LOG.info("user:%s, host:%s", user, ip) + self.client = ssh.SSH(user, ip, key_filename=key_filename) + self.client.wait(timeout=600) + + # copy scripts to host + self.client.run("cat > ~/ramspeed_mark_benchmark.sh", + stdin=open(self.mark_target_script, 'rb')) + self.client.run("cat > ~/ramspeed_mem_benchmark.sh", + stdin=open(self.mem_target_script, 'rb')) + self.setup_done = True + + def run(self, result): + """execute the benchmark""" + + if not self.setup_done: + self.setup() + + options = self.scenario_cfg['options'] + test_id = options.get('type_id', 1) + load = options.get('load', 8) + block_size = options.get('block_size', 32) + + if test_id == 3 or test_id == 6: + iteration = options.get('iteration', 1) + cmd = "sudo bash ramspeed_mem_benchmark.sh %d %d %d %d" % \ + (test_id, load, block_size, iteration) + elif 0 < test_id <= 5: + cmd = "sudo bash ramspeed_mark_benchmark.sh %d %d %d" % \ + (test_id, load, block_size) + # only the test_id 1-6 will be used in this scenario + else: + raise RuntimeError("No such type_id: %s for Ramspeed scenario", + test_id) + + LOG.debug("Executing command: %s", cmd) + status, stdout, stderr = self.client.execute(cmd) + if status: + raise RuntimeError(stderr) + + result.update(json.loads(stdout)) + + if "sla" in self.scenario_cfg: + sla_error = "" + sla_min_bw = int(self.scenario_cfg['sla']['min_bandwidth']) + for i in result["Result"]: + bw = i["Bandwidth(MBps)"] + if bw < sla_min_bw: + sla_error += "Bandwidth %f < " \ + "sla:min_bandwidth(%f)" % (bw, sla_min_bw) + assert sla_error == "", sla_error diff --git a/yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash b/yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash new file mode 100644 index 000000000..fcb655968 --- /dev/null +++ b/yardstick/benchmark/scenarios/compute/ramspeed_mark_benchmark.bash @@ -0,0 +1,46 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Run a ramspeed memory bandwidth benchmark in a host and +# output in json format the memory bandwidth in megabytes per second + +set -e + +TYPE_ID=$1 +LOAD=$2 +BLOCK_SIZE=$3 +OUTPUT_FILE=/tmp/ramspeed-out.log + +run_ramspeed() +{ + cd /opt/tempT2/ramspeed-2.6.0/ + ./ramspeed -b $TYPE_ID -g $LOAD -m $BLOCK_SIZE > $OUTPUT_FILE +} + +# write the result to stdout in json format +output_json() +{ + SCORE=$(awk '/ /{printf "{\"Test_type\": \"%s %s %s\", \"Block_size(kb)\": %s, \"Bandwidth(MBps)\": %s},", $1, $2, $3, $4, $7}' $OUTPUT_FILE) + echo -e "{ \ + \"Result\":[${SCORE%?}] \ + }" +} + +main() +{ + # run the test + run_ramspeed + + #output result + output_json +} + +main diff --git a/yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash b/yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash new file mode 100644 index 000000000..69c2934af --- /dev/null +++ b/yardstick/benchmark/scenarios/compute/ramspeed_mem_benchmark.bash @@ -0,0 +1,47 @@ +#!/bin/bash + +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Run a ramspeed memory bandwidth benchmark in a host and +# output in json format the memory bandwidth in megabytes per second + +set -e + +TYPE_ID=$1 +LOAD=$2 +BLOCK_SIZE=$3 +ITERATION=$4 +OUTPUT_FILE=/tmp/ramspeed-out.log + +run_ramspeed() +{ + cd /opt/tempT2/ramspeed-2.6.0/ + ./ramspeed -b $TYPE_ID -g $LOAD -m $BLOCK_SIZE -l $ITERATION > $OUTPUT_FILE +} + +# write the result to stdout in json format +output_json() +{ + SCORE=$(awk '/ /{printf "{\"Test_type\": \"%s %s\", \"Bandwidth(MBps)\": %s},", $1, $2, $3}' $OUTPUT_FILE) + echo -e "{ \ + \"Result\":[${SCORE%?}] \ + }" +} + +main() +{ + # run the test + run_ramspeed + + # output result + output_json +} + +main |