diff options
20 files changed, 245 insertions, 42 deletions
diff --git a/docker/exec_tests.sh b/docker/exec_tests.sh index db053f7bc..9726e2b4a 100755 --- a/docker/exec_tests.sh +++ b/docker/exec_tests.sh @@ -16,7 +16,8 @@ set -e : ${RELENG_REPO:='https://gerrit.opnfv.org/gerrit/releng'} : ${RELENG_REPO_DIR:='/home/opnfv/repos/releng'} -: ${RELENG_BRANCH:='master'} # branch, tag, sha1 or refspec +# TEMP HACK to freeze releng version to workaround fetch_os_creds.sh problem +: ${RELENG_BRANCH:='abbf19f'} # branch, tag, sha1 or refspec git_checkout() { diff --git a/etc/yardstick/nodes/pod.yaml.ipmi.sample b/etc/yardstick/nodes/pod.yaml.ipmi.sample new file mode 100644 index 000000000..6253c790f --- /dev/null +++ b/etc/yardstick/nodes/pod.yaml.ipmi.sample @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2017 14_ykl@tongji.edu.cn and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +# sample config file about the POD information, including the +# name/IP/user/ssh key of Bare Metal Controllers/Computes/Jumperserver +# and the ipmi information of Controllers/Computes +# +# The options of this config file include: +# name: the name of this node +# role: node's role, support role: Master/Controller/Comupte/BareMetal +# ip: the node's IP address +# user: the username for login +# password: the password for login +# ipmi_ip: the node's IPMI IP address +# ipmi_user: the username for ipmi control +# ipmi_password: the password for ipmi control + +nodes: +- + name: node0 + role: Jumpserver + ip: 10.1.0.2 + user: root + password: root + +- + name: node1 + role: Controller + ip: 10.1.0.50 + user: root + password: root + ipmi_ip: 192.168.10.10 + ipmi_user: root + ipmi_password: root + +- + name: node2 + role: Compute + ip: 10.1.0.51 + user: root + password: root + ipmi_ip: 192.168.10.11 + ipmi_user: root + ipmi_password: root
\ No newline at end of file diff --git a/install.sh b/install.sh index dc2663628..b8212d2f4 100755 --- a/install.sh +++ b/install.sh @@ -22,12 +22,15 @@ deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-updates main unive deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-security main universe multiverse restricted deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ trusty-proposed main universe multiverse restricted" > "${sub_source_file}" echo "vm.mmap_min_addr = 0" > /etc/sysctl.d/mmap_min_addr.conf -dpkg --add-architecture arm64 + +proc_type=$(uname -m) +if [[ $proc_type == "arm"* ]]; then + dpkg --add-architecture arm64 +fi # install tools apt-get update && apt-get install -y \ qemu-user-static/xenial \ - libc6:arm64 \ wget \ expect \ curl \ @@ -49,6 +52,10 @@ apt-get update && apt-get install -y \ python-pip \ vim +if [[ $proc_type == "arm"* ]]; then + apt-get install -y libc6:arm64 +fi + apt-get -y autoremove && apt-get clean git config --global http.sslVerify false diff --git a/tests/ci/load_images.sh b/tests/ci/load_images.sh index 2e22b83c2..487f33e33 100755 --- a/tests/ci/load_images.sh +++ b/tests/ci/load_images.sh @@ -145,12 +145,6 @@ load_yardstick_image() exit 1 fi - if [[ "$DEPLOY_SCENARIO" == *"-lxd-"* ]]; then - sudo rm -f -- "${RAW_IMAGE}" - else - sudo rm -f -- "${QCOW_IMAGE}" - fi - echo "Glance image id: $GLANCE_IMAGE_ID" } @@ -225,7 +219,7 @@ create_nova_flavor() echo echo "========== Creating yardstick-flavor ==========" # Create the nova flavor used by some sample test cases - openstack flavor create --id 100 --ram 512 --disk 3 --vcpus 1 yardstick-flavor + openstack flavor create --id 100 --ram 1024 --disk 3 --vcpus 1 yardstick-flavor # DPDK-enabled OVS requires guest memory to be backed by large pages if [[ $DEPLOY_SCENARIO == *[_-]ovs[_-]* ]]; then openstack flavor set --property hw:mem_page_size=large yardstick-flavor diff --git a/tests/opnfv/test_cases/opnfv_yardstick_tc025.yaml b/tests/opnfv/test_cases/opnfv_yardstick_tc025.yaml index 443395873..c4ee237e5 100644 --- a/tests/opnfv/test_cases/opnfv_yardstick_tc025.yaml +++ b/tests/opnfv/test_cases/opnfv_yardstick_tc025.yaml @@ -11,16 +11,22 @@ schema: "yardstick:task:0.1" description: > Sample test case for the HA of OpenStack Controll Node abnormally shutdown. + In order to power on the shutdown node after testing, the jumphost is + required to install ipmitool. Ipmi jumphost info and each nodes' ipmi ip, + username, password are needed in pod file (There is a sample pod file in + "etc/yardstick/nodes/pod.yaml.ipmi.sample"). {% set file = file or 'etc/yardstick/nodes/fuel_virtual/pod.yaml' %} +{% set jumphost = jumphost or 'node0' %} scenarios: - type: ServiceHA options: attackers: - - fault_type: "host-shutdown" + - fault_type: "bare-metal-down" host: node1 + jump_host: {{jumphost}} monitors: - monitor_type: "openstack-cmd" @@ -42,6 +48,7 @@ scenarios: nodes: node1: node1.LF + node0: node0.LF runner: type: Iteration diff --git a/tests/unit/benchmark/core/test_task.py b/tests/unit/benchmark/core/test_task.py index cd7ffdebb..8034392f4 100644 --- a/tests/unit/benchmark/core/test_task.py +++ b/tests/unit/benchmark/core/test_task.py @@ -64,6 +64,7 @@ class TaskTestCase(unittest.TestCase): t = task.Task() runner = mock.Mock() runner.join.return_value = 0 + runner.get_output.return_value = {} mock_base_runner.Runner.get.return_value = runner t._run([scenario], False, "yardstick.out") self.assertTrue(runner.run.called) @@ -155,6 +156,33 @@ class TaskTestCase(unittest.TestCase): self.assertEqual(task_args_fnames[0], None) self.assertEqual(task_args_fnames[1], None) + def test_parse_options(self): + options = { + 'openstack': { + 'EXTERNAL_NETWORK': '$network' + }, + 'ndoes': ['node1', '$node'], + 'host': '$host' + } + + t = task.Task() + t.outputs = { + 'network': 'ext-net', + 'node': 'node2', + 'host': 'server.yardstick' + } + + idle_result = { + 'openstack': { + 'EXTERNAL_NETWORK': 'ext-net' + }, + 'ndoes': ['node1', 'node2'], + 'host': 'server.yardstick' + } + + actual_result = t._parse_options(options) + self.assertEqual(idle_result, actual_result) + def test_change_server_name_host_str(self): scenario = {'host': 'demo'} suffix = '-8' diff --git a/tests/unit/benchmark/runner/__init__.py b/tests/unit/benchmark/runner/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/tests/unit/benchmark/runner/__init__.py diff --git a/tests/unit/benchmark/runner/test_base.py b/tests/unit/benchmark/runner/test_base.py new file mode 100644 index 000000000..7880fe5a5 --- /dev/null +++ b/tests/unit/benchmark/runner/test_base.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python + +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from __future__ import print_function +from __future__ import absolute_import + +import unittest +import multiprocessing +import time + +from yardstick.benchmark.runners.iteration import IterationRunner + + +class RunnerTestCase(unittest.TestCase): + + def test_get_output(self): + queue = multiprocessing.Queue() + runner = IterationRunner({}, queue) + runner.output_queue.put({'case': 'opnfv_yardstick_tc002'}) + runner.output_queue.put({'criteria': 'PASS'}) + + idle_result = { + 'case': 'opnfv_yardstick_tc002', + 'criteria': 'PASS' + } + + time.sleep(1) + actual_result = runner.get_output() + self.assertEqual(idle_result, actual_result) + + +def main(): + unittest.main() + + +if __name__ == '__main__': + main() diff --git a/tests/unit/dispatcher/test_influxdb.py b/tests/unit/dispatcher/test_influxdb.py index 0c7b58135..dca3c4189 100644 --- a/tests/unit/dispatcher/test_influxdb.py +++ b/tests/unit/dispatcher/test_influxdb.py @@ -14,11 +14,15 @@ from __future__ import absolute_import import unittest + try: from unittest import mock except ImportError: import mock +from yardstick import _init_logging +_init_logging() + from yardstick.dispatcher.influxdb import InfluxdbDispatcher diff --git a/yardstick/benchmark/core/task.py b/yardstick/benchmark/core/task.py index 3a151dbba..c44081b73 100644 --- a/yardstick/benchmark/core/task.py +++ b/yardstick/benchmark/core/task.py @@ -44,6 +44,7 @@ class Task(object): # pragma: no cover def __init__(self): self.config = {} self.contexts = [] + self.outputs = {} def start(self, args, **kwargs): """Start a benchmark scenario.""" @@ -136,6 +137,7 @@ class Task(object): # pragma: no cover # Wait for runners to finish for runner in runners: runner_join(runner) + self.outputs.update(runner.get_output()) print("Runner ended, output in", output_file) else: # run serially @@ -143,6 +145,7 @@ class Task(object): # pragma: no cover if not _is_background_scenario(scenario): runner = self.run_one_scenario(scenario, output_file) runner_join(runner) + self.outputs.update(runner.get_output()) print("Runner ended, output in", output_file) # Abort background runners @@ -155,6 +158,7 @@ class Task(object): # pragma: no cover # Nuke if it did not stop nicely base_runner.Runner.terminate(runner) runner_join(runner) + self.outputs.update(runner.get_output()) else: base_runner.Runner.release(runner) print("Background task ended") @@ -168,11 +172,24 @@ class Task(object): # pragma: no cover for context in self.contexts[::-1]: context.undeploy() + def _parse_options(self, op): + if isinstance(op, dict): + return {k: self._parse_options(v) for k, v in op.items()} + elif isinstance(op, list): + return [self._parse_options(v) for v in op] + elif isinstance(op, str): + return self.outputs.get(op[1:]) if op.startswith('$') else op + else: + return op + def run_one_scenario(self, scenario_cfg, output_file): """run one scenario using context""" runner_cfg = scenario_cfg["runner"] runner_cfg['output_filename'] = output_file + options = scenario_cfg.get('options', {}) + scenario_cfg['options'] = self._parse_options(options) + # TODO support get multi hosts/vms info context_cfg = {} if "host" in scenario_cfg: diff --git a/yardstick/benchmark/runners/arithmetic.py b/yardstick/benchmark/runners/arithmetic.py index 65fdb9d66..7ec593396 100755 --- a/yardstick/benchmark/runners/arithmetic.py +++ b/yardstick/benchmark/runners/arithmetic.py @@ -42,7 +42,7 @@ LOG = logging.getLogger(__name__) def _worker_process(queue, cls, method_name, scenario_cfg, - context_cfg, aborted): + context_cfg, aborted, output_queue): sequence = 1 @@ -108,7 +108,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, errors = "" try: - method(data) + result = method(data) except AssertionError as assertion: # SLA validation failed in scenario, determine what to do now if sla_action == "assert": @@ -119,6 +119,9 @@ def _worker_process(queue, cls, method_name, scenario_cfg, except Exception as e: errors = traceback.format_exc() LOG.exception(e) + else: + if result: + output_queue.put(result) time.sleep(interval) @@ -188,5 +191,5 @@ class ArithmeticRunner(base.Runner): self.process = multiprocessing.Process( target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, - context_cfg, self.aborted)) + context_cfg, self.aborted, self.output_queue)) self.process.start() diff --git a/yardstick/benchmark/runners/base.py b/yardstick/benchmark/runners/base.py index b48ed973a..ebb9a91b5 100755 --- a/yardstick/benchmark/runners/base.py +++ b/yardstick/benchmark/runners/base.py @@ -197,6 +197,7 @@ class Runner(object): self.config = config self.periodic_action_process = None self.result_queue = queue + self.output_queue = multiprocessing.Queue() self.process = None self.aborted = multiprocessing.Event() Runner.runners.append(self) @@ -269,3 +270,9 @@ class Runner(object): self.run_post_stop_action() return self.process.exitcode + + def get_output(self): + result = {} + while not self.output_queue.empty(): + result.update(self.output_queue.get()) + return result diff --git a/yardstick/benchmark/runners/duration.py b/yardstick/benchmark/runners/duration.py index 772983cfd..2bf2cd2fe 100644 --- a/yardstick/benchmark/runners/duration.py +++ b/yardstick/benchmark/runners/duration.py @@ -32,7 +32,7 @@ LOG = logging.getLogger(__name__) def _worker_process(queue, cls, method_name, scenario_cfg, - context_cfg, aborted): + context_cfg, aborted, output_queue): sequence = 1 @@ -66,7 +66,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, errors = "" try: - method(data) + result = method(data) except AssertionError as assertion: # SLA validation failed in scenario, determine what to do now if sla_action == "assert": @@ -77,6 +77,9 @@ def _worker_process(queue, cls, method_name, scenario_cfg, except Exception as e: errors = traceback.format_exc() LOG.exception(e) + else: + if result: + output_queue.put(result) time.sleep(interval) @@ -126,5 +129,5 @@ If the scenario ends before the time has elapsed, it will be started again. self.process = multiprocessing.Process( target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, - context_cfg, self.aborted)) + context_cfg, self.aborted, self.output_queue)) self.process.start() diff --git a/yardstick/benchmark/runners/iteration.py b/yardstick/benchmark/runners/iteration.py index 29daa0d42..973bb9ac4 100644 --- a/yardstick/benchmark/runners/iteration.py +++ b/yardstick/benchmark/runners/iteration.py @@ -32,7 +32,7 @@ LOG = logging.getLogger(__name__) def _worker_process(queue, cls, method_name, scenario_cfg, - context_cfg, aborted): + context_cfg, aborted, output_queue): sequence = 1 @@ -41,10 +41,8 @@ def _worker_process(queue, cls, method_name, scenario_cfg, interval = runner_cfg.get("interval", 1) iterations = runner_cfg.get("iterations", 1) run_step = runner_cfg.get("run_step", "setup,run,teardown") + delta = runner_cfg.get("delta", 2) - options_cfg = scenario_cfg['options'] - initial_rate = options_cfg.get("rate", 100) - scenario_cfg['options']['rate'] = initial_rate LOG.info("worker START, iterations %d times, class %s", iterations, cls) runner_cfg['runner_id'] = os.getpid() @@ -73,7 +71,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, errors = "" try: - method(data) + result = method(data) except AssertionError as assertion: # SLA validation failed in scenario, determine what to do now if sla_action == "assert": @@ -82,12 +80,21 @@ def _worker_process(queue, cls, method_name, scenario_cfg, LOG.warning("SLA validation failed: %s", assertion.args) errors = assertion.args elif sla_action == "rate-control": + try: + scenario_cfg['options']['rate'] + except KeyError: + scenario_cfg.setdefault('options', {}) + scenario_cfg['options']['rate'] = 100 + scenario_cfg['options']['rate'] -= delta sequence = 1 continue except Exception as e: errors = traceback.format_exc() LOG.exception(e) + else: + if result: + output_queue.put(result) time.sleep(interval) @@ -138,5 +145,5 @@ If the scenario ends before the time has elapsed, it will be started again. self.process = multiprocessing.Process( target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, - context_cfg, self.aborted)) + context_cfg, self.aborted, self.output_queue)) self.process.start() diff --git a/yardstick/benchmark/runners/sequence.py b/yardstick/benchmark/runners/sequence.py index af87c006e..74ff82204 100644 --- a/yardstick/benchmark/runners/sequence.py +++ b/yardstick/benchmark/runners/sequence.py @@ -33,7 +33,7 @@ LOG = logging.getLogger(__name__) def _worker_process(queue, cls, method_name, scenario_cfg, - context_cfg, aborted): + context_cfg, aborted, output_queue): sequence = 1 @@ -75,7 +75,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, errors = "" try: - method(data) + result = method(data) except AssertionError as assertion: # SLA validation failed in scenario, determine what to do now if sla_action == "assert": @@ -86,6 +86,9 @@ def _worker_process(queue, cls, method_name, scenario_cfg, except Exception as e: errors = traceback.format_exc() LOG.exception(e) + else: + if result: + output_queue.put(result) time.sleep(interval) @@ -137,5 +140,5 @@ class SequenceRunner(base.Runner): self.process = multiprocessing.Process( target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, - context_cfg, self.aborted)) + context_cfg, self.aborted, self.output_queue)) self.process.start() diff --git a/yardstick/benchmark/scenarios/compute/lmbench_bandwidth_benchmark.bash b/yardstick/benchmark/scenarios/compute/lmbench_bandwidth_benchmark.bash index 09993a088..714207b0d 100644 --- a/yardstick/benchmark/scenarios/compute/lmbench_bandwidth_benchmark.bash +++ b/yardstick/benchmark/scenarios/compute/lmbench_bandwidth_benchmark.bash @@ -19,6 +19,8 @@ SIZE=$1 TEST_NAME=$2 WARMUP=$3 +NODE_CPU_ARCH="$(uname -m)" + # write the result to stdout in json format output_json() { @@ -26,4 +28,8 @@ output_json() echo $DATA | awk '/ /{printf "{\"size(MB)\": %s, \"bandwidth(MBps)\": %s}", $1, $2}' } -/usr/lib/lmbench/bin/x86_64-linux-gnu/bw_mem -W $WARMUP ${SIZE}k $TEST_NAME 2>&1 | output_json
\ No newline at end of file +if [ "${NODE_CPU_ARCH}" == "aarch64" ]; then + /usr/lib/lmbench/bin/bw_mem -W $WARMUP ${SIZE}k $TEST_NAME 2>&1 | output_json +else + /usr/lib/lmbench/bin/x86_64-linux-gnu/bw_mem -W $WARMUP ${SIZE}k $TEST_NAME 2>&1 | output_json +fi diff --git a/yardstick/benchmark/scenarios/compute/lmbench_latency_benchmark.bash b/yardstick/benchmark/scenarios/compute/lmbench_latency_benchmark.bash index 04e3c1a9d..b45470666 100644 --- a/yardstick/benchmark/scenarios/compute/lmbench_latency_benchmark.bash +++ b/yardstick/benchmark/scenarios/compute/lmbench_latency_benchmark.bash @@ -19,6 +19,8 @@ SIZE=$1 shift STRIDE=$1 +NODE_CPU_ARCH="$(uname -m)" + # write the result to stdout in json format output_json() { @@ -37,5 +39,8 @@ output_json() echo ] } -/usr/lib/lmbench/bin/x86_64-linux-gnu/lat_mem_rd $SIZE $STRIDE 2>&1 | output_json - +if [ "${NODE_CPU_ARCH}" == "aarch64" ]; then + /usr/lib/lmbench/bin/lat_mem_rd $SIZE $STRIDE 2>&1 | output_json +else + /usr/lib/lmbench/bin/x86_64-linux-gnu/lat_mem_rd $SIZE $STRIDE 2>&1 | output_json +fi diff --git a/yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash b/yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash index 2ed1bbe14..2fdd0aaaa 100644 --- a/yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash +++ b/yardstick/benchmark/scenarios/compute/lmbench_latency_for_cache.bash @@ -18,6 +18,8 @@ set -e REPETITON=$1 WARMUP=$2 +NODE_CPU_ARCH="$(uname -m)" + # write the result to stdout in json format output_json() { @@ -25,5 +27,9 @@ output_json() echo $DATA | awk '{printf "{\"L1cache\": %s}", $5}' } -/usr/lib/lmbench/bin/x86_64-linux-gnu/cache -W $WARMUP -N $REPETITON 2>&1 | output_json +if [ "${NODE_CPU_ARCH}" == "aarch64" ]; then + /usr/lib/lmbench/bin/cache -W $WARMUP -N $REPETITON 2>&1 | output_json +else + /usr/lib/lmbench/bin/x86_64-linux-gnu/cache -W $WARMUP -N $REPETITON 2>&1 | output_json +fi diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py index f4def8533..7aab46942 100644 --- a/yardstick/common/utils.py +++ b/yardstick/common/utils.py @@ -81,7 +81,10 @@ def import_modules_from_package(package): continue new_package = ".".join(root.split(os.sep)).split("....")[1] module_name = "%s.%s" % (new_package, filename[:-3]) - try_append_module(module_name, sys.modules) + try: + try_append_module(module_name, sys.modules) + except ImportError: + logger.exception("unable to import %s", module_name) def parse_yaml(file_path): diff --git a/yardstick/orchestrator/heat.py b/yardstick/orchestrator/heat.py index ea9bd1b08..864f1f9ec 100644 --- a/yardstick/orchestrator/heat.py +++ b/yardstick/orchestrator/heat.py @@ -149,21 +149,28 @@ class HeatStack(HeatObject): class HeatTemplate(HeatObject): """Describes a Heat template and a method to deploy template to a stack""" - def _init_template(self): - self._template = {} - self._template['heat_template_version'] = '2013-05-23' + DESCRIPTION_TEMPLATE = """\ +Stack built by the yardstick framework for %s on host %s %s. +All referred generated resources are prefixed with the template +name (i.e. %s).\ +""" + def _init_template(self): timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - self._template['description'] = \ - """Stack built by the yardstick framework for %s on host %s %s. - All referred generated resources are prefixed with the template - name (i.e. %s).""" % (getpass.getuser(), socket.gethostname(), - timestamp, self.name) + self._template = { + 'heat_template_version': '2013-05-23', + 'description': self.DESCRIPTION_TEMPLATE % ( + getpass.getuser(), + socket.gethostname(), + timestamp, + self.name + ), + 'resources': {}, + 'outputs': {} + } # short hand for resources part of template - self.resources = self._template['resources'] = {} - - self._template['outputs'] = {} + self.resources = self._template['resources'] def __init__(self, name, template_file=None, heat_parameters=None): super(HeatTemplate, self).__init__() |