From fc37fef338a99254c64cc5c8cb5e1e29ec7ecdb1 Mon Sep 17 00:00:00 2001 From: xudan Date: Thu, 3 Aug 2017 21:28:03 -0400 Subject: Add resiliency test caes into proposed_tests JIRA: DOVETAIL-474 Add Bottlenecks resiliency test case "posca_factor_ping" into proposed_tests. Change-Id: I76aea651014fb4722ddae07559f417d3f840eee2 Signed-off-by: xudan --- dovetail/compliance/proposed_tests.yml | 2 + dovetail/conf/bottlenecks_config.yml | 20 ++++++++++ dovetail/conf/cmd_config.yml | 11 +++++- dovetail/conf/dovetail_config.yml | 4 ++ dovetail/container.py | 40 +++++++++++++++++--- dovetail/report.py | 69 +++++++++++++++++++++++++++++++++- dovetail/run.py | 14 ++++++- dovetail/test_runner.py | 10 ++++- dovetail/testcase.py | 20 +++++++++- dovetail/testcase/resiliency.tc001.yml | 11 ++++++ dovetail/utils/dovetail_config.py | 5 +++ 11 files changed, 193 insertions(+), 13 deletions(-) create mode 100644 dovetail/conf/bottlenecks_config.yml create mode 100644 dovetail/testcase/resiliency.tc001.yml diff --git a/dovetail/compliance/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml index 018edba0..b148df68 100644 --- a/dovetail/compliance/proposed_tests.yml +++ b/dovetail/compliance/proposed_tests.yml @@ -48,3 +48,5 @@ proposed_tests: - dovetail.sdnvpn.tc002 - dovetail.sdnvpn.tc004 - dovetail.sdnvpn.tc008 + # resiliency + - dovetail.resiliency.tc001 diff --git a/dovetail/conf/bottlenecks_config.yml b/dovetail/conf/bottlenecks_config.yml new file mode 100644 index 00000000..43df8c55 --- /dev/null +++ b/dovetail/conf/bottlenecks_config.yml @@ -0,0 +1,20 @@ +--- +bottlenecks: + image_name: opnfv/bottlenecks + docker_tag: cvp.0.4.0 + opts: '-id --privileged=true' + config: + dir: '/home/opnfv/userconfig' + pre_condition: + - 'echo test for precondition in bottlenecks' + cmds: + - '/home/opnfv/bottlenecks/run_tests.sh -c {{validate_testcase}}' + post_condition: + - 'echo test for postcondition in bottlenecks' + result: + dir: '/tmp' + file_path: 'bottlenecks.log' + openrc: '/tmp/admin_rc.sh' + extra_container: + - 'Bottlenecks-Yardstick' + - 'Bottlenecks-ELK' diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml index a5b262d1..d862a910 100644 --- a/dovetail/conf/cmd_config.yml +++ b/dovetail/conf/cmd_config.yml @@ -21,14 +21,21 @@ cli: - '-y' path: - 'yardstick/docker_tag' - help: 'Overwrite tag for yardstick docker container (e.g. stable or latest)' + help: 'Overwrite tag for yardstick docker container (e.g. danube.3.1)' func_tag: flags: - '--func_tag' - '-f' path: - 'functest/docker_tag' - help: 'Overwrite tag for functest docker container (e.g. stable or latest)' + help: 'Overwrite tag for functest docker container (e.g. cvp.0.2.0)' + bott_tag: + flags: + - '--bott_tag' + - '-b' + path: + - 'bottlenecks/docker_tag' + help: 'Overwrite tag for bottlenecks docker container (e.g. cvp.0.4.0)' control: testsuite: flags: diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml index 36c8e5c1..98f9e814 100644 --- a/dovetail/conf/dovetail_config.yml +++ b/dovetail/conf/dovetail_config.yml @@ -32,6 +32,7 @@ testarea_supported: - ipv6 - sdnvpn - vping + - resiliency functest_testsuite: - refstack_defcore @@ -66,10 +67,12 @@ parameters: include_config: - functest_config.yml - yardstick_config.yml + - bottlenecks_config.yml test_project: - 'yardstick' - 'functest' + - 'bottlenecks' validate_input: valid_docker_tag: @@ -82,3 +85,4 @@ validate_input: - 'cvp.0.1.0' - 'cvp.0.2.0' - 'cvp.0.3.0' + - 'cvp.0.4.0' diff --git a/dovetail/container.py b/dovetail/container.py index 1a5867a7..ede50a80 100644 --- a/dovetail/container.py +++ b/dovetail/container.py @@ -18,7 +18,8 @@ from utils.dovetail_config import DovetailConfig as dt_cfg class Container(object): container_list = {} - has_pull_latest_image = {'yardstick': False, 'functest': False} + has_pull_latest_image = {'yardstick': False, 'functest': False, + 'bottlenecks': False} logger = None @@ -122,6 +123,19 @@ class Container(object): key_vol = '-v %s:%s ' % (key_file, key_container_path) return "%s %s %s" % (envs, log_vol, key_vol) + @classmethod + def set_bottlenecks_config(cls, testcase_name): + dovetail_config = dt_cfg.dovetail_config + yard_tag = dovetail_config['yardstick']['docker_tag'] + docker_vol = '-v /var/run/docker.sock:/var/run/docker.sock' + env = ('-e Yardstick_TAG={} -e OUTPUT_FILE={}.out' + .format(yard_tag, testcase_name)) + report = "" + if dovetail_config['report_dest'].startswith("http"): + report = ("-e BOTTLENECKS_DB_TARGET={}" + .format(dovetail_config['report_dest'])) + return "{} {} {}".format(docker_vol, env, report) + @classmethod def create(cls, type, testcase_name): dovetail_config = dt_cfg.dovetail_config @@ -144,6 +158,8 @@ class Container(object): config = cls.set_functest_config(testcase_name) if type.lower() == "yardstick": config = cls.set_yardstick_config() + if type.lower() == "bottlenecks": + config = cls.set_bottlenecks_config(testcase_name) if not config: return None @@ -284,11 +300,23 @@ class Container(object): return image_id @classmethod - def clean(cls, container_id): - cmd1 = 'sudo docker stop %s' % (container_id) - dt_utils.exec_cmd(cmd1, cls.logger) - cmd2 = 'sudo docker rm %s' % (container_id) - dt_utils.exec_cmd(cmd2, cls.logger) + def check_container_exist(cls, container_name): + cmd = ('sudo docker ps -aq -f name={}'.format(container_name)) + ret, msg = dt_utils.exec_cmd(cmd, cls.logger) + if ret == 0 and msg: + return True + return False + + @classmethod + def clean(cls, container_id, valid_type): + cmd = ('sudo docker rm -f {}'.format(container_id)) + dt_utils.exec_cmd(cmd, cls.logger) + if valid_type.lower() == 'bottlenecks': + containers = dt_cfg.dovetail_config[valid_type]['extra_container'] + for container in containers: + if cls.check_container_exist(container): + cmd = ('sudo docker rm -f {}'.format(container)) + dt_utils.exec_cmd(cmd, cls.logger) @classmethod def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False): diff --git a/dovetail/report.py b/dovetail/report.py index 08780b88..4743c256 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -25,7 +25,7 @@ from testcase import Testcase class Report(object): - results = {'functest': {}, 'yardstick': {}, 'shell': {}} + results = {'functest': {}, 'yardstick': {}, 'bottlenecks': {}, 'shell': {}} logger = None @@ -339,6 +339,53 @@ class YardstickCrawler(object): return None +class BottlenecksCrawler(object): + + logger = None + + def __init__(self): + self.type = 'bottlenecks' + self.logger.debug('Create crawler: {}'.format(self.type)) + + @classmethod + def create_log(cls): + cls.logger = \ + dt_logger.Logger(__name__ + '.BottlenecksCrawler').getLogger() + + def crawl(self, testcase=None): + report_dest = dt_cfg.dovetail_config['report_dest'] + if report_dest.lower() == 'file': + return self.crawl_from_file(testcase) + + if report_dest.lower().startswith('http'): + return self.crawl_from_url(testcase) + + def crawl_from_file(self, testcase=None): + file_path = os.path.join(dt_cfg.dovetail_config['result_dir'], + testcase.name() + '.out') + if not os.path.exists(file_path): + self.logger.error('Result file not found: {}'.format(file_path)) + return None + criteria = 'FAIL' + with open(file_path, 'r') as f: + for jsonfile in f: + data = json.loads(jsonfile) + try: + if 'PASS' == data["data_body"]["result"]: + criteria = 'PASS' + else: + criteria = 'FAIL' + break + except KeyError as e: + self.logger.exception('Pass flag not found {}'.format(e)) + json_results = {'criteria': criteria} + self.logger.debug('Results: {}'.format(str(json_results))) + return json_results + + def crawl_from_url(self, testcase=None): + return None + + class ShellCrawler(object): def __init__(self): @@ -364,6 +411,7 @@ class CrawlerFactory(object): CRAWLER_MAP = {'functest': FunctestCrawler, 'yardstick': YardstickCrawler, + 'bottlenecks': BottlenecksCrawler, 'shell': ShellCrawler} @classmethod @@ -466,6 +514,24 @@ class YardstickChecker(object): return +class BottlenecksChecker(object): + + logger = None + + @classmethod + def create_log(cls): + cls.logger = \ + dt_logger.Logger(__name__ + '.BottlenecksChecker').getLogger() + + @staticmethod + def check(testcase, result): + if not result: + testcase.passed('FAIL') + else: + testcase.passed(result['criteria']) + return + + class ShellChecker(object): @staticmethod @@ -480,6 +546,7 @@ class CheckerFactory(object): CHECKER_MAP = {'functest': FunctestChecker, 'yardstick': YardstickChecker, + 'bottlenecks': BottlenecksChecker, 'shell': ShellChecker} @classmethod diff --git a/dovetail/run.py b/dovetail/run.py index f89671f0..7515023b 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -22,8 +22,8 @@ from container import Container from testcase import Testcase from testcase import Testsuite from report import Report -from report import FunctestCrawler, YardstickCrawler -from report import FunctestChecker, YardstickChecker +from report import FunctestCrawler, YardstickCrawler, BottlenecksCrawler +from report import FunctestChecker, YardstickChecker, BottlenecksChecker from utils.dovetail_config import DovetailConfig as dt_cfg from test_runner import DockerRunner, ShellRunner @@ -90,6 +90,8 @@ def check_tc_result(testcase, logger): result_file = os.path.join(result_dir, testcase.name() + '.out') elif validate_type.lower() == 'functest': result_file = os.path.join(result_dir, functest_result) + elif validate_type.lower() == 'bottlenecks': + result_file = os.path.join(result_dir, testcase.name() + '.out') else: logger.error("Don't support {} now.".format(validate_type)) return @@ -107,6 +109,7 @@ def validate_input(input_dict, check_dict, logger): # for 'func_tag' and 'yard_tag' options func_tag = input_dict['func_tag'] yard_tag = input_dict['yard_tag'] + bott_tag = input_dict['bott_tag'] valid_tag = check_dict['valid_docker_tag'] if func_tag is not None and func_tag not in valid_tag: logger.error("The input option 'func_tag' can't be {}, " @@ -116,6 +119,10 @@ def validate_input(input_dict, check_dict, logger): logger.error("The input option 'yard_tag' can't be {}, " "valid values are {}.".format(yard_tag, valid_tag)) raise SystemExit(1) + if bott_tag is not None and bott_tag not in valid_tag: + logger.error("The input option 'bott_tag' can't be {}, " + "valid values are {}.".format(bott_tag, valid_tag)) + raise SystemExit(1) # for 'report' option report = input_dict['report'] @@ -164,8 +171,10 @@ def create_logs(): Report.create_log() FunctestCrawler.create_log() YardstickCrawler.create_log() + BottlenecksCrawler.create_log() FunctestChecker.create_log() YardstickChecker.create_log() + BottlenecksChecker.create_log() Testcase.create_log() Testsuite.create_log() DockerRunner.create_log() @@ -246,6 +255,7 @@ def main(*args, **kwargs): if(kwargs['report'].endswith('/')): kwargs['report'] = kwargs['report'][0:kwargs['report'].rfind('/')] dt_cfg.dovetail_config['report_dest'] = kwargs['report'] + dt_cfg.update_cmds() if kwargs['offline']: dt_cfg.dovetail_config['offline'] = True diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py index b3fd7a3f..603156fe 100644 --- a/dovetail/test_runner.py +++ b/dovetail/test_runner.py @@ -111,7 +111,7 @@ class DockerRunner(object): ret, msg = Container.exec_cmd(container_id, cmd) self.testcase.cleaned(True) - Container.clean(container_id) + Container.clean(container_id, self.type) class FunctestRunner(DockerRunner): @@ -128,6 +128,13 @@ class YardstickRunner(DockerRunner): super(YardstickRunner, self).__init__(testcase) +class BottlenecksRunner(DockerRunner): + + def __init__(self, testcase): + self.type = 'bottlenecks' + super(BottlenecksRunner, self).__init__(testcase) + + class ShellRunner(object): logger = None @@ -190,6 +197,7 @@ class TestRunnerFactory(object): TEST_RUNNER_MAP = { "functest": FunctestRunner, "yardstick": YardstickRunner, + "bottlenecks": BottlenecksRunner, "shell": ShellRunner, } diff --git a/dovetail/testcase.py b/dovetail/testcase.py index 7b012c88..bdfd3d35 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -254,13 +254,30 @@ class FunctestTestcase(Testcase): class YardstickTestcase(Testcase): - validate_testcae_list = {} + validate_testcase_list = {} def __init__(self, testcase_yaml): super(YardstickTestcase, self).__init__(testcase_yaml) self.type = 'yardstick' +class BottlenecksTestcase(Testcase): + + validate_testcase_list = {} + + def __init__(self, testcase_yaml): + super(BottlenecksTestcase, self).__init__(testcase_yaml) + self.type = 'bottlenecks' + self._update_cmds() + + def _update_cmds(self): + if dt_cfg.dovetail_config['report_dest'].startswith("http"): + try: + self.testcase['validate']['cmds'][0] += ' --report' + except KeyError: + return + + class ShellTestcase(Testcase): validate_testcase_list = {} @@ -274,6 +291,7 @@ class TestcaseFactory(object): TESTCASE_TYPE_MAP = { 'functest': FunctestTestcase, 'yardstick': YardstickTestcase, + 'bottlenecks': BottlenecksTestcase, 'shell': ShellTestcase, } diff --git a/dovetail/testcase/resiliency.tc001.yml b/dovetail/testcase/resiliency.tc001.yml new file mode 100644 index 00000000..86399849 --- /dev/null +++ b/dovetail/testcase/resiliency.tc001.yml @@ -0,0 +1,11 @@ +--- +dovetail.resiliency.tc001: + name: dovetail.resiliency.tc001 + objective: > # This test case verifies the ability of the SUT setting up VM pairs + # for different tenants and providing acceptable capacity after the amount of + # VM pairs reaches certain quantity. + validate: + type: bottlenecks + testcase: posca_factor_ping + report: + sub_testcase_list: diff --git a/dovetail/utils/dovetail_config.py b/dovetail/utils/dovetail_config.py index f8193e5d..5e100d69 100644 --- a/dovetail/utils/dovetail_config.py +++ b/dovetail/utils/dovetail_config.py @@ -54,3 +54,8 @@ class DovetailConfig(object): def update_non_envs(cls, path, value): if value: cls.set_leaf_dict(cls.dovetail_config, path, value) + + @classmethod + def update_cmds(cls): + if cls.dovetail_config['report_dest'].startswith("http"): + cls.dovetail_config['bottlenecks']['cmds'][0] += ' --report' -- cgit 1.2.3-korg