diff options
Diffstat (limited to 'yardstick/benchmark')
18 files changed, 149 insertions, 94 deletions
diff --git a/yardstick/benchmark/contexts/heat.py b/yardstick/benchmark/contexts/heat.py index 9a7b3817f..ff3e5f801 100644 --- a/yardstick/benchmark/contexts/heat.py +++ b/yardstick/benchmark/contexts/heat.py @@ -361,6 +361,8 @@ class HeatContext(Context): 'subnet', 'gateway_ip')] return { + # add default port name + "name": port, "private_ip": private_ip, "subnet_id": outputs[h_join(stack_name, "subnet_id")], "subnet_cidr": output_subnet_cidr, diff --git a/yardstick/benchmark/runners/arithmetic.py b/yardstick/benchmark/runners/arithmetic.py index 3ff064ae1..6aaaed888 100755 --- a/yardstick/benchmark/runners/arithmetic.py +++ b/yardstick/benchmark/runners/arithmetic.py @@ -191,7 +191,9 @@ class ArithmeticRunner(base.Runner): __execution_type__ = 'Arithmetic' def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, context_cfg, self.aborted, self.output_queue)) diff --git a/yardstick/benchmark/runners/base.py b/yardstick/benchmark/runners/base.py index 3ecf67736..13718d793 100755 --- a/yardstick/benchmark/runners/base.py +++ b/yardstick/benchmark/runners/base.py @@ -17,13 +17,14 @@ # rally/rally/benchmark/runners/base.py from __future__ import absolute_import -import importlib + import logging import multiprocessing import subprocess import time import traceback +import importlib from six.moves.queue import Empty @@ -36,7 +37,6 @@ log = logging.getLogger(__name__) def _execute_shell_command(command): """execute shell script with error handling""" exitcode = 0 - output = [] try: output = subprocess.check_output(command, shell=True) except Exception: diff --git a/yardstick/benchmark/runners/duration.py b/yardstick/benchmark/runners/duration.py index 75942766d..fbf72a74c 100644 --- a/yardstick/benchmark/runners/duration.py +++ b/yardstick/benchmark/runners/duration.py @@ -31,6 +31,9 @@ from yardstick.benchmark.runners import base LOG = logging.getLogger(__name__) +QUEUE_PUT_TIMEOUT = 10 + + def _worker_process(queue, cls, method_name, scenario_cfg, context_cfg, aborted, output_queue): @@ -79,7 +82,9 @@ def _worker_process(queue, cls, method_name, scenario_cfg, LOG.exception("") else: if result: - output_queue.put(result) + # add timeout for put so we don't block test + # if we do timeout we don't care about dropping individual KPIs + output_queue.put(result, True, QUEUE_PUT_TIMEOUT) time.sleep(interval) @@ -90,7 +95,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, 'errors': errors } - queue.put(benchmark_output) + queue.put(benchmark_output, True, QUEUE_PUT_TIMEOUT) LOG.debug("runner=%(runner)s seq=%(sequence)s END", {"runner": runner_cfg["runner_id"], "sequence": sequence}) @@ -133,7 +138,9 @@ If the scenario ends before the time has elapsed, it will be started again. __execution_type__ = 'Duration' def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, context_cfg, self.aborted, self.output_queue)) diff --git a/yardstick/benchmark/runners/dynamictp.py b/yardstick/benchmark/runners/dynamictp.py index 01e76c6f4..63bfc823a 100755 --- a/yardstick/benchmark/runners/dynamictp.py +++ b/yardstick/benchmark/runners/dynamictp.py @@ -19,11 +19,12 @@ """A runner that searches for the max throughput with binary search """ -import os -import multiprocessing import logging -import traceback +import multiprocessing import time +import traceback + +import os from yardstick.benchmark.runners import base @@ -65,8 +66,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, max_throuput_found = False sequence = 0 - last_min_data = {} - last_min_data['packets_per_second'] = 0 + last_min_data = {'packets_per_second': 0} while True: sequence += 1 @@ -125,7 +125,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, queue.put(record) max_throuput_found = True - if (errors) or aborted.is_set() or max_throuput_found: + if errors or aborted.is_set() or max_throuput_found: LOG.info("worker END") break @@ -155,7 +155,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, class IterationRunner(base.Runner): - '''Run a scenario to find the max throughput + """Run a scenario to find the max throughput If the scenario ends before the time has elapsed, it will be started again. @@ -168,11 +168,13 @@ If the scenario ends before the time has elapsed, it will be started again. type: int unit: pps default: 1000 pps - ''' + """ __execution_type__ = 'Dynamictp' def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, context_cfg, self.aborted)) diff --git a/yardstick/benchmark/runners/iteration.py b/yardstick/benchmark/runners/iteration.py index 4a7439588..cb0424377 100644 --- a/yardstick/benchmark/runners/iteration.py +++ b/yardstick/benchmark/runners/iteration.py @@ -20,17 +20,22 @@ """ from __future__ import absolute_import -import os -import multiprocessing + import logging -import traceback +import multiprocessing import time +import traceback + +import os from yardstick.benchmark.runners import base LOG = logging.getLogger(__name__) +QUEUE_PUT_TIMEOUT = 10 + + def _worker_process(queue, cls, method_name, scenario_cfg, context_cfg, aborted, output_queue): @@ -85,13 +90,14 @@ def _worker_process(queue, cls, method_name, scenario_cfg, scenario_cfg['options']['rate'] -= delta sequence = 1 continue - except Exception as e: + except Exception: errors = traceback.format_exc() - LOG.exception(e) + LOG.exception("") else: if result: - LOG.debug("output_queue.put %s", result) - output_queue.put(result, True, 1) + # add timeout for put so we don't block test + # if we do timeout we don't care about dropping individual KPIs + output_queue.put(result, True, QUEUE_PUT_TIMEOUT) time.sleep(interval) @@ -102,8 +108,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, 'errors': errors } - LOG.debug("queue.put, %s", benchmark_output) - queue.put(benchmark_output, True, 1) + queue.put(benchmark_output, True, QUEUE_PUT_TIMEOUT) LOG.debug("runner=%(runner)s seq=%(sequence)s END", {"runner": runner_cfg["runner_id"], @@ -148,7 +153,9 @@ If the scenario ends before the time has elapsed, it will be started again. __execution_type__ = 'Iteration' def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, context_cfg, self.aborted, self.output_queue)) diff --git a/yardstick/benchmark/runners/search.py b/yardstick/benchmark/runners/search.py index 5948763a7..8037329b5 100644 --- a/yardstick/benchmark/runners/search.py +++ b/yardstick/benchmark/runners/search.py @@ -20,15 +20,16 @@ """ from __future__ import absolute_import -import os -import multiprocessing + import logging -import traceback +import multiprocessing import time - -from collections import Mapping +import traceback from contextlib import contextmanager from itertools import takewhile + +import os +from collections import Mapping from six.moves import zip from yardstick.benchmark.runners import base @@ -173,7 +174,9 @@ If the scenario ends before the time has elapsed, it will be started again. break def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=self._worker_run, args=(cls, method, scenario_cfg, context_cfg)) self.process.start() diff --git a/yardstick/benchmark/runners/sequence.py b/yardstick/benchmark/runners/sequence.py index f08ca5dde..d6e3f7109 100644 --- a/yardstick/benchmark/runners/sequence.py +++ b/yardstick/benchmark/runners/sequence.py @@ -21,11 +21,13 @@ The input value in the sequence is specified in a list in the input file. """ from __future__ import absolute_import -import os -import multiprocessing + import logging -import traceback +import multiprocessing import time +import traceback + +import os from yardstick.benchmark.runners import base @@ -140,7 +142,9 @@ class SequenceRunner(base.Runner): __execution_type__ = 'Sequence' def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, context_cfg, self.aborted, self.output_queue)) diff --git a/yardstick/benchmark/scenarios/availability/actionplayers.py b/yardstick/benchmark/scenarios/availability/actionplayers.py index c5e199ba6..d5a531e8a 100644 --- a/yardstick/benchmark/scenarios/availability/actionplayers.py +++ b/yardstick/benchmark/scenarios/availability/actionplayers.py @@ -20,8 +20,10 @@ class ActionPlayer(object): class AttackerPlayer(ActionPlayer): - def __init__(self, attacker): + def __init__(self, attacker, intermediate_variables): self.underlyingAttacker = attacker + self.underlyingAttacker.intermediate_variables \ + = intermediate_variables def action(self): self.underlyingAttacker.inject_fault() @@ -40,8 +42,10 @@ class OperationPlayer(ActionPlayer): class MonitorPlayer(ActionPlayer): - def __init__(self, monitor): + def __init__(self, monitor, intermediate_variables): self.underlyingmonitor = monitor + self.underlyingmonitor.intermediate_variables \ + = intermediate_variables def action(self): self.underlyingmonitor.start_monitor() @@ -49,8 +53,10 @@ class MonitorPlayer(ActionPlayer): class ResultCheckerPlayer(ActionPlayer): - def __init__(self, resultChecker): + def __init__(self, resultChecker, intermediate_variables): self.underlyingresultChecker = resultChecker + self.underlyingresultChecker.intermediate_variables \ + = intermediate_variables def action(self): self.underlyingresultChecker.verify() diff --git a/yardstick/benchmark/scenarios/availability/attacker/attacker_general.py b/yardstick/benchmark/scenarios/availability/attacker/attacker_general.py index 48863af93..11b02a222 100644 --- a/yardstick/benchmark/scenarios/availability/attacker/attacker_general.py +++ b/yardstick/benchmark/scenarios/availability/attacker/attacker_general.py @@ -13,6 +13,8 @@ import yardstick.ssh as ssh from yardstick.benchmark.scenarios.availability import util from yardstick.benchmark.scenarios.availability.attacker.baseattacker import \ BaseAttacker +from yardstick.benchmark.scenarios.availability.util \ + import read_stdout_item, build_shell_command LOG = logging.getLogger(__name__) @@ -33,13 +35,7 @@ class GeneralAttacker(BaseAttacker): self.attack_key = self._config['attack_key'] if "action_parameter" in self._config: - actionParameter = self._config['action_parameter'] - str = util.buildshellparams(actionParameter) - LOG.debug("inject parameter is: %s", actionParameter) - LOG.debug("inject parameter values are: %s", - list(actionParameter.values())) - l = list(actionParameter.values()) - self.action_param = str.format(*l) + self.actionParameter_config = self._config['action_parameter'] if "rollback_parameter" in self._config: rollbackParameter = self._config['rollback_parameter'] @@ -59,8 +55,12 @@ class GeneralAttacker(BaseAttacker): def inject_fault(self): LOG.debug("%s starting inject!", self.key) LOG.debug("the inject_script path:%s", self.inject_script) - if "action_parameter" in self._config: + self.action_param = \ + build_shell_command( + self.actionParameter_config, + bool(self.connection), + self.intermediate_variables) LOG.debug("the shell command is: %s", self.action_param) with open(self.inject_script, "r") as stdin_file: exit_status, stdout, stderr = self.connection.execute( @@ -75,6 +75,12 @@ class GeneralAttacker(BaseAttacker): LOG.debug("the inject_fault's exit status is: %s", exit_status) if exit_status == 0: LOG.debug("success,the inject_fault's output is: %s", stdout) + if "return_parameter" in self._config: + returnParameter = self._config['return_parameter'] + for key, item in returnParameter.items(): + value = read_stdout_item(stdout, key) + LOG.debug("intermediate variables %s: %s", item, value) + self.intermediate_variables[item] = value else: LOG.error( "the inject_fault's error, stdout:%s, stderr:%s", diff --git a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py index 61698da43..d03d04420 100644 --- a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py +++ b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py @@ -62,6 +62,7 @@ class BaseAttacker(object): self._context = context self.data = {} self.setup_done = False + self.intermediate_variables = {} @staticmethod def get_attacker_cls(attacker_cfg): diff --git a/yardstick/benchmark/scenarios/availability/director.py b/yardstick/benchmark/scenarios/availability/director.py index f152af090..71690c135 100644 --- a/yardstick/benchmark/scenarios/availability/director.py +++ b/yardstick/benchmark/scenarios/availability/director.py @@ -71,12 +71,12 @@ class Director(object): LOG.debug( "the type of current action is %s, the key is %s", type, key) if type == ActionType.ATTACKER: - return actionplayers.AttackerPlayer(self.attackerMgr[key]) + return actionplayers.AttackerPlayer(self.attackerMgr[key], intermediate_variables) if type == ActionType.MONITOR: - return actionplayers.MonitorPlayer(self.monitorMgr[key]) + return actionplayers.MonitorPlayer(self.monitorMgr[key], intermediate_variables) if type == ActionType.RESULTCHECKER: return actionplayers.ResultCheckerPlayer( - self.resultCheckerMgr[key]) + self.resultCheckerMgr[key], intermediate_variables) if type == ActionType.OPERATION: return actionplayers.OperationPlayer(self.operationMgr[key], intermediate_variables) diff --git a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py index a6c1a28bd..50a63f53d 100644 --- a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py +++ b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py @@ -94,6 +94,7 @@ class BaseMonitor(multiprocessing.Process): self.monitor_data = data self.setup_done = False self.tag = "" + self.intermediate_variables = {} @staticmethod def get_monitor_cls(monitor_type): diff --git a/yardstick/benchmark/scenarios/availability/monitor/monitor_general.py b/yardstick/benchmark/scenarios/availability/monitor/monitor_general.py index c16765fe0..b058ae2b1 100644 --- a/yardstick/benchmark/scenarios/availability/monitor/monitor_general.py +++ b/yardstick/benchmark/scenarios/availability/monitor/monitor_general.py @@ -11,8 +11,8 @@ import logging import yardstick.ssh as ssh from yardstick.benchmark.scenarios.availability.monitor import basemonitor -from yardstick.benchmark.scenarios.availability.util import buildshellparams - +from yardstick.benchmark.scenarios.availability.util \ + import build_shell_command, execute_shell_command LOG = logging.getLogger(__name__) @@ -23,37 +23,41 @@ class GeneralMonitor(basemonitor.BaseMonitor): __monitor_type__ = "general-monitor" def setup(self): - host = self._context[self._config["host"]] + host = self._context.get(self._config.get('host', None), None) + self.connection = None + if host: + self.connection = ssh.SSH.from_node( + host, defaults={"user": "root"}) + self.connection.wait(timeout=600) + LOG.debug("ssh host success!") self.key = self._config["key"] self.monitor_key = self._config["monitor_key"] self.monitor_type = self._config["monitor_type"] - if "parameter" in self._config: - parameter = self._config['parameter'] - str = buildshellparams(parameter) - l = list(item for item in parameter.values()) - self.cmd_param = str.format(*l) - + self.parameter_config = self._config['parameter'] self.monitor_cfg = basemonitor.BaseMonitor.monitor_cfgs.get( self.monitor_key) self.monitor_script = self.get_script_fullpath( self.monitor_cfg['monitor_script']) - self.connection = ssh.SSH.from_node(host, defaults={"user": "root"}) - self.connection.wait(timeout=600) LOG.debug("ssh host success!") def monitor_func(self): if "parameter" in self._config: - with open(self.monitor_script, "r") as stdin_file: - exit_status, stdout, stderr = self.connection.execute( - "sudo {}".format(self.cmd_param), - stdin=stdin_file) + self.cmd_param = \ + build_shell_command( + self.parameter_config, + bool(self.connection), + self.intermediate_variables) + cmd_remote = "sudo {}".format(self.cmd_param) + cmd_local = "/bin/bash {0} {1}".format(self.monitor_script, self.cmd_param) else: + cmd_remote = "sudo /bin/sh -s " + cmd_local = "/bin/bash {0}".format(self.monitor_script) + if self.connection: with open(self.monitor_script, "r") as stdin_file: - exit_status, stdout, stderr = self.connection.execute( - "sudo /bin/bash -s ", - stdin=stdin_file) - + exit_status, stdout, stderr = self.connection.execute(cmd_remote, stdin=stdin_file) + else: + exit_status, stdout = execute_shell_command(cmd_local) if exit_status: return False return True diff --git a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py index 05b660105..d1750ab65 100644 --- a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py +++ b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py @@ -66,6 +66,7 @@ class BaseResultChecker(object): self._config = config self._context = context self.setup_done = False + self.intermediate_variables = {} @staticmethod def get_resultchecker_cls(type): diff --git a/yardstick/benchmark/scenarios/availability/result_checker/result_checker_general.py b/yardstick/benchmark/scenarios/availability/result_checker/result_checker_general.py index 454338175..0802aa452 100644 --- a/yardstick/benchmark/scenarios/availability/result_checker/result_checker_general.py +++ b/yardstick/benchmark/scenarios/availability/result_checker/result_checker_general.py @@ -15,7 +15,7 @@ from yardstick.benchmark.scenarios.availability.result_checker \ from yardstick.benchmark.scenarios.availability import Condition import yardstick.ssh as ssh from yardstick.benchmark.scenarios.availability.util \ - import buildshellparams, execute_shell_command + import execute_shell_command, build_shell_command LOG = logging.getLogger(__name__) @@ -40,22 +40,20 @@ class GeneralResultChecker(BaseResultChecker): self.condition = self._config['condition'] self.expectedResult = self._config['expectedValue'] self.actualResult = object() - self.key = self._config['key'] if "parameter" in self._config: - parameter = self._config['parameter'] - str = buildshellparams( - parameter, True if self.connection else False) - l = list(item for item in parameter.values()) - self.shell_cmd = str.format(*l) - - self.resultchecker_cfgs = BaseResultChecker.resultchecker_cfgs.get( - self.resultchecker_key) + self.parameter_config = self._config['parameter'] + self.resultchecker_cfgs = BaseResultChecker.resultchecker_cfgs.get(self.resultchecker_key) self.verify_script = self.get_script_fullpath( self.resultchecker_cfgs['verify_script']) def verify(self): if "parameter" in self._config: + self.shell_cmd = \ + build_shell_command( + self.parameter_config, + bool(self.connection), + self.intermediate_variables) if self.connection: with open(self.verify_script, "r") as stdin_file: exit_status, stdout, stderr = self.connection.execute( diff --git a/yardstick/benchmark/scenarios/availability/util.py b/yardstick/benchmark/scenarios/availability/util.py index d288fcbc1..adc20b844 100644 --- a/yardstick/benchmark/scenarios/availability/util.py +++ b/yardstick/benchmark/scenarios/availability/util.py @@ -33,7 +33,7 @@ def execute_shell_command(command): LOG.error(traceback.format_exc()) return exitcode, output -PREFIX = '$' +PREFIX = '@' def build_shell_command(param_config, remote=True, intermediate_variables=None): diff --git a/yardstick/benchmark/scenarios/networking/vnf_generic.py b/yardstick/benchmark/scenarios/networking/vnf_generic.py index d9cc0eac1..0fab45480 100644 --- a/yardstick/benchmark/scenarios/networking/vnf_generic.py +++ b/yardstick/benchmark/scenarios/networking/vnf_generic.py @@ -19,6 +19,8 @@ import logging import errno import ipaddress + +import copy import os import sys import re @@ -30,6 +32,7 @@ from collections import defaultdict from yardstick.benchmark.scenarios import base from yardstick.common.constants import LOG_DIR +from yardstick.common.process import terminate_children from yardstick.common.utils import import_modules_from_package, itersubclasses from yardstick.common.yaml_loader import yaml_load from yardstick.network_services.collector.subscriber import Collector @@ -375,13 +378,7 @@ class NetworkServiceTestCase(base.Scenario): context_yaml = os.path.join(LOG_DIR, "pod-{}.yaml".format(self.scenario_cfg['task_id'])) # convert OrderedDict to a list # pod.yaml nodes is a list - nodes = [] - for node in self.context_cfg["nodes"].values(): - # name field is required - # remove context suffix - node['name'] = node['name'].split('.')[0] - nodes.append(node) - nodes = self._convert_pkeys_to_string(nodes) + nodes = [self._serialize_node(node) for node in self.context_cfg["nodes"].values()] pod_dict = { "nodes": nodes, "networks": self.context_cfg["networks"] @@ -391,15 +388,16 @@ class NetworkServiceTestCase(base.Scenario): explicit_start=True) @staticmethod - def _convert_pkeys_to_string(nodes): - # make copy because we are mutating - nodes = nodes[:] - for i, node in enumerate(nodes): - try: - nodes[i] = dict(node, pkey=ssh.convert_key_to_str(node["pkey"])) - except KeyError: - pass - return nodes + def _serialize_node(node): + new_node = copy.deepcopy(node) + # name field is required + # remove context suffix + new_node["name"] = node['name'].split('.')[0] + try: + new_node["pkey"] = ssh.convert_key_to_str(node["pkey"]) + except KeyError: + pass + return new_node TOPOLOGY_REQUIRED_KEYS = frozenset({ "vpci", "local_ip", "netmask", "local_mac", "driver"}) @@ -596,7 +594,8 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ vnf.instantiate(self.scenario_cfg, self.context_cfg) LOG.info("Waiting for %s to instantiate", vnf.name) vnf.wait_for_instantiate() - except RuntimeError: + except: + LOG.exception("") for vnf in self.vnfs: vnf.terminate() raise @@ -635,7 +634,19 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ :return """ - self.collector.stop() - for vnf in self.vnfs: - LOG.info("Stopping %s", vnf.name) - vnf.terminate() + try: + try: + self.collector.stop() + for vnf in self.vnfs: + LOG.info("Stopping %s", vnf.name) + vnf.terminate() + LOG.debug("all VNFs terminated: %s", ", ".join(vnf.name for vnf in self.vnfs)) + finally: + terminate_children() + except Exception: + # catch any exception in teardown and convert to simple exception + # never pass exceptions back to multiprocessing, because some exceptions can + # be unpicklable + # https://bugs.python.org/issue9400 + LOG.exception("") + raise RuntimeError("Error in teardown") |