diff options
47 files changed, 380 insertions, 150 deletions
diff --git a/dovetail/compliance/debug.yml b/dovetail/compliance/debug.yml index a0a2d3ee..8cc4b36c 100644 --- a/dovetail/compliance/debug.yml +++ b/dovetail/compliance/debug.yml @@ -4,6 +4,7 @@ debug: name: debug testcases_list: + - dovetail.example.tc002 - dovetail.ipv6.tc001 - dovetail.nfvi.tc001 - dovetail.nfvi.tc002 diff --git a/dovetail/compliance/example_set.yml b/dovetail/compliance/example_set.yml new file mode 100644 index 00000000..1ee1ac50 --- /dev/null +++ b/dovetail/compliance/example_set.yml @@ -0,0 +1,8 @@ +example_set: + name: example_set + testcases_list: + # Temporarily, one test case kept here as default to run + # for use of software development/debug + # TO DO: will amend when compliance set is settled + - dovetail.example.tc001 + - dovetail.example.tc002 diff --git a/dovetail/conf/dovetail_config.py b/dovetail/conf/dovetail_config.py index d812c5c9..6cf3f7af 100644 --- a/dovetail/conf/dovetail_config.py +++ b/dovetail/conf/dovetail_config.py @@ -54,12 +54,12 @@ class DovetailConfig: cls.update_config_envs('yardstick', key, options[item]) @classmethod - def update_config_envs(cls, script_type, key, value): - envs = cls.dovetail_config[script_type]['envs'] + def update_config_envs(cls, validate_type, key, value): + envs = cls.dovetail_config[validate_type]['envs'] old_value = re.findall(r'\s+%s=(.*?)(\s+|$)' % key, envs) if old_value == []: envs += ' -e ' + key + '=' + value else: envs = envs.replace(old_value[0][0], value) - cls.dovetail_config[script_type]['envs'] = envs + cls.dovetail_config[validate_type]['envs'] = envs return envs diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml index be2d075d..5264f140 100644 --- a/dovetail/conf/dovetail_config.yml +++ b/dovetail/conf/dovetail_config.yml @@ -25,8 +25,8 @@ testarea_supported: parameters: - name: testcase path: '("name",)' - - name: script_testcase - path: '("scripts", "testcase")' + - name: validate_testcase + path: '("validate", "testcase")' include_config: - functest_config.yml diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml index d32fe87f..72cdb0dd 100644 --- a/dovetail/conf/functest_config.yml +++ b/dovetail/conf/functest_config.yml @@ -6,15 +6,12 @@ functest: -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal' opts: '-id --privileged=true' pre_condition: - cmds: - - 'echo test for precondition' - testcase: - cmds: - - 'functest env prepare' - - 'functest testcase run {{script_testcase}}' + - 'echo test for precondition' + cmds: + - 'functest env prepare' + - 'functest testcase run {{validate_testcase}}' post_condition: - cmds: - - '' + - '' result: dir: '/home/opnfv/functest/results' store_type: 'file' diff --git a/dovetail/conf/yardstick_config.yml b/dovetail/conf/yardstick_config.yml index f7f05bcc..d13cf2c6 100644 --- a/dovetail/conf/yardstick_config.yml +++ b/dovetail/conf/yardstick_config.yml @@ -7,21 +7,18 @@ yardstick: -e EXTERNAL_NETWORK=ext-net' opts: '-id --privileged=true' pre_condition: - cmds: - - 'source /home/opnfv/repos/yardstick/tests/ci/prepare_env.sh && + - 'source /home/opnfv/repos/yardstick/tests/ci/prepare_env.sh && source /home/opnfv/repos/yardstick/tests/ci/clean_images.sh && cleanup' - - 'source /home/opnfv/repos/yardstick/tests/ci/prepare_env.sh && + - 'source /home/opnfv/repos/yardstick/tests/ci/prepare_env.sh && cd /home/opnfv/repos/yardstick && source tests/ci/load_images.sh' - testcase: - cmds: - - 'mkdir -p /home/opnfv/yardstick/results/' - - 'cd /home/opnfv/repos/yardstick && source tests/ci/prepare_env.sh && - yardstick task start tests/opnfv/test_cases/{{script_testcase}}.yaml - --output-file /home/opnfv/yardstick/results/{{script_testcase}}.out &> + cmds: + - 'mkdir -p /home/opnfv/yardstick/results/' + - 'cd /home/opnfv/repos/yardstick && source tests/ci/prepare_env.sh && + yardstick task start tests/opnfv/test_cases/{{validate_testcase}}.yaml + --output-file /home/opnfv/yardstick/results/{{validate_testcase}}.out &> /home/opnfv/yardstick/results/yardstick.log' post_condition: - cmds: - - '' + - '' result: dir: '/home/opnfv/yardstick/results' store_type: 'file' diff --git a/dovetail/report.py b/dovetail/report.py index 2e2a24f3..1f970b29 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -30,7 +30,7 @@ def get_pass_str(passed): class Report: - results = {'functest': {}, 'yardstick': {}} + results = {'functest': {}, 'yardstick': {}, 'shell': {}} logger = None @@ -40,8 +40,9 @@ class Report: @staticmethod def check_result(testcase, db_result): - checker = CheckerFactory.create(testcase.script_type()) - checker.check(testcase, db_result) + checker = CheckerFactory.create(testcase.validate_type()) + if checker is not None: + checker.check(testcase, db_result) @classmethod def generate_json(cls, testsuite_yaml, testarea, duration): @@ -163,24 +164,26 @@ class Report: @classmethod def get_result(cls, testcase): - script_testcase = testcase.script_testcase() - type = testcase.script_type() + validate_testcase = testcase.validate_testcase() + type = testcase.validate_type() crawler = CrawlerFactory.create(type) + if crawler is None: + return None - if script_testcase in cls.results[type]: - return cls.results[type][script_testcase] + if validate_testcase in cls.results[type]: + return cls.results[type][validate_testcase] - result = crawler.crawl(script_testcase) + result = crawler.crawl(validate_testcase) if result is not None: - cls.results[type][script_testcase] = result + cls.results[type][validate_testcase] = result testcase.script_result_acquired(True) cls.logger.debug('testcase: %s -> result acquired' % - script_testcase) + validate_testcase) else: retry = testcase.increase_retry() cls.logger.debug('testcase: %s -> result acquired retry:%d' % - (script_testcase, retry)) + (validate_testcase, retry)) return result diff --git a/dovetail/run.py b/dovetail/run.py index d9bc0aaa..52a350e5 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -11,7 +11,6 @@ import click import sys import os -import time import utils.dovetail_logger as dt_logger import utils.dovetail_utils as dt_utils @@ -24,6 +23,7 @@ from report import Report from report import FunctestCrawler, YardstickCrawler from report import FunctestChecker, YardstickChecker from conf.dovetail_config import DovetailConfig as dt_cfg +from test_runner import DockerRunner, ShellRunner def load_testsuite(testsuite): @@ -34,9 +34,9 @@ def load_testsuite(testsuite): def set_container_tags(option_str): for script_tag_opt in option_str.split(','): option_str = script_tag_opt.split(':') - script_type = option_str[0].strip() + validate_type = option_str[0].strip() script_tag = option_str[1].strip() - dt_cfg.dovetail_config[script_type]['docker_tag'] = script_tag + dt_cfg.dovetail_config[validate_type]['docker_tag'] = script_tag def load_testcase(): @@ -66,29 +66,7 @@ def run_test(testsuite, testarea, logger): run_testcase = False if run_testcase: - Container.pull_image(testcase.script_type()) - container_id = Container.create(testcase.script_type()) - logger.debug('container id:%s' % container_id) - - if not Testcase.prepared(testcase.script_type()): - cmds = testcase.pre_condition()['cmds'] - if cmds: - for cmd in cmds: - Container.exec_cmd(container_id, cmd) - Testcase.prepared(testcase.script_type(), True) - - if not testcase.prepare_cmd(): - logger.error('failed to prepare testcase:%s' % testcase.name()) - else: - start_time = time.time() - for cmd in testcase.cmds: - Container.exec_cmd(container_id, cmd) - end_time = time.time() - duration = end_time - start_time - - # testcase.post_condition() - - Container.clean(container_id) + testcase.run() db_result = Report.get_result(testcase) Report.check_result(testcase, db_result) @@ -125,6 +103,8 @@ def create_logs(): YardstickChecker.create_log() Testcase.create_log() Testsuite.create_log() + DockerRunner.create_log() + ShellRunner.create_log() def clean_results_dir(): diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py new file mode 100644 index 00000000..bc0e4679 --- /dev/null +++ b/dovetail/test_runner.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# +# grakiss.wanglei@huawei.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +import utils.dovetail_utils as dt_utils +import utils.dovetail_logger as dt_logger + +from container import Container + + +class DockerRunner(object): + + logger = None + + def __init__(self, testcase): + self.testcase = testcase + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__file__).getLogger() + + def run(self): + Container.pull_image(self.testcase.validate_type()) + container_id = Container.create(self.testcase.validate_type()) + self.logger.debug('container id:%s' % container_id) + + if not self.testcase.prepared(): + cmds = self.testcase.pre_condition() + if cmds: + for cmd in cmds: + Container.exec_cmd(container_id, cmd) + self.testcase.prepared(True) + + if not self.testcase.prepare_cmd(): + self.logger.error('failed to prepare testcase:%s', + self.testcase.name()) + else: + for cmd in self.testcase.cmds: + Container.exec_cmd(container_id, cmd) + + cmds = self.testcase.post_condition() + if cmds: + for cmd in cmds: + Container.exec_cmd(container_id, cmd) + self.testcase.cleaned(True) + + Container.clean(container_id) + + +class FunctestRunner(DockerRunner): + + def __init__(self, testcase): + super(FunctestRunner, self).__init__(testcase) + self.name = 'functest' + + +class YardstickRunner(DockerRunner): + + def __init__(self, testcase): + super(YardstickRunner, self).__init__(testcase) + self.name = 'yardstick' + + +class ShellRunner(object): + + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__file__).getLogger() + + def __init__(self, testcase): + super(ShellRunner, self).__init__() + self.testcase = testcase + self.name = 'shell' + + def run(self): + for cmd in self.testcase.cmds: + dt_utils.exec_cmd(cmd, self.logger) + + +class TestRunnerFactory(object): + + TEST_RUNNER_MAP = { + "functest": FunctestRunner, + "yardstick": YardstickRunner, + "shell": ShellRunner, + } + + @classmethod + def create(cls, testcase): + try: + return cls.TEST_RUNNER_MAP[testcase.validate_type()](testcase) + except KeyError: + return None diff --git a/dovetail/testcase.py b/dovetail/testcase.py index 429b9154..79522923 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -14,33 +14,31 @@ import utils.dovetail_logger as dt_logger from parser import Parser from conf.dovetail_config import DovetailConfig as dt_cfg +from test_runner import TestRunnerFactory -class Testcase: +class Testcase(object): logger = None def __init__(self, testcase_yaml): self.testcase = testcase_yaml.values()[0] + self.logger.debug('testcase:%s', self.testcase) self.testcase['passed'] = False self.cmds = [] self.sub_testcase_status = {} - self.update_script_testcase(self.script_type(), - self.script_testcase()) + self.update_validate_testcase(self.validate_testcase()) @classmethod def create_log(cls): cls.logger = dt_logger.Logger(__name__ + '.Testcase').getLogger() def prepare_cmd(self): - script_type = self.script_type() - for cmd in dt_cfg.dovetail_config[script_type]['testcase']['cmds']: - cmd_lines = Parser.parse_cmd(cmd, self) - if not cmd_lines: - return False - self.cmds.append(cmd_lines) - - return True + try: + self.cmds = self.testcase['validate']['cmds'] + return True + except KeyError: + return False def __str__(self): return self.testcase @@ -52,29 +50,31 @@ class Testcase: return self.testcase['objective'] def sub_testcase(self): - return self.testcase['scripts']['sub_testcase_list'] + try: + return self.testcase['report']['sub_testcase_list'] + except KeyError: + return [] def sub_testcase_passed(self, name, passed=None): if passed is not None: - self.logger.debug('sub_testcase_passed:%s %s' % (name, passed)) + self.logger.debug('sub_testcase_passed:%s %s', name, passed) self.sub_testcase_status[name] = passed return self.sub_testcase_status[name] - def script_type(self): - return self.testcase['scripts']['type'] + def validate_type(self): + return self.testcase['validate']['type'] - def script_testcase(self): - return self.testcase['scripts']['testcase'] + def validate_testcase(self): + return self.testcase['validate']['testcase'] def exceed_max_retry_times(self): # logger.debug('retry times:%d' % self.testcase['retry']) - return self._exceed_max_retry_times(self.script_type(), - self.script_testcase()) + return self._exceed_max_retry_times(self.validate_testcase()) def increase_retry(self): # self.testcase['retry'] = self.testcase['retry'] + 1 # return self.testcase['retry'] - return self._increase_retry(self.script_type(), self.script_testcase()) + return self._increase_retry(self.validate_testcase()) def passed(self, passed=None): if passed is not None: @@ -82,75 +82,87 @@ class Testcase: return self.testcase['passed'] def script_result_acquired(self, acquired=None): - return self._result_acquired(self.script_type(), - self.script_testcase(), acquired) + return self._result_acquired(self.validate_testcase(), acquired) def pre_condition(self): - return self.pre_condition_cls(self.script_type()) + return self.pre_condition_cls(self.validate_type()) def post_condition(self): - return self.post_condition_cls(self.script_type()) + return self.post_condition_cls(self.validate_type()) - # testcase in upstream testing project - script_testcase_list = {'functest': {}, 'yardstick': {}} + def run(self): + runner = TestRunnerFactory.create(self) + try: + runner.run() + except AttributeError: + pass + # testcase in upstream testing project + # validate_testcase_list = {'functest': {}, 'yardstick': {}, 'shell': {}} + validate_testcase_list = {} # testcase in dovetail testcase_list = {} @classmethod - def prepared(cls, script_type, prepared=None): + def prepared(cls, prepared=None): if prepared is not None: - cls.script_testcase_list[script_type]['prepared'] = prepared - return cls.script_testcase_list[script_type]['prepared'] + cls.validate_testcase_list['prepared'] = prepared + return cls.validate_testcase_list['prepared'] @classmethod - def cleaned(cls, script_type, cleaned=None): + def cleaned(cls, cleaned=None): if cleaned is not None: - cls.scrpit_testcase_list[script_type]['cleaned'] = cleaned - return cls.script_testcase_list[script_type]['cleaned'] + cls.validate_testcase_list['cleaned'] = cleaned + return cls.validate_testcase_list['cleaned'] @staticmethod - def pre_condition_cls(script_type): - return dt_cfg.dovetail_config[script_type]['pre_condition'] + def pre_condition_cls(validate_type): + return dt_cfg.dovetail_config[validate_type]['pre_condition'] @staticmethod - def post_condition_cls(script_type): - return dt_cfg.dovetail_config[script_type]['post_condition'] + def post_condition_cls(validate_type): + return dt_cfg.dovetail_config[validate_type]['post_condition'] @classmethod - def update_script_testcase(cls, script_type, script_testcase): - if script_testcase not in cls.script_testcase_list[script_type]: - cls.script_testcase_list[script_type][script_testcase] = \ + def update_validate_testcase(cls, testcase_name): + if testcase_name not in cls.validate_testcase_list: + cls.validate_testcase_list[testcase_name] = \ {'retry': 0, 'acquired': False} - cls.script_testcase_list[script_type]['prepared'] = False - cls.script_testcase_list[script_type]['cleaned'] = False + cls.validate_testcase_list['prepared'] = False + cls.validate_testcase_list['cleaned'] = False @classmethod - def _exceed_max_retry_times(cls, script_type, script_testcase): - retry = cls.script_testcase_list[script_type][script_testcase]['retry'] + def _exceed_max_retry_times(cls, validate_testcase): + retry = cls.validate_testcase_list[validate_testcase]['retry'] return retry > 1 @classmethod - def _increase_retry(cls, script_type, script_testcase): - cls.script_testcase_list[script_type][script_testcase]['retry'] += 1 - return cls.script_testcase_list[script_type][script_testcase]['retry'] + def _increase_retry(cls, validate_testcase): + cls.validate_testcase_list[validate_testcase]['retry'] += 1 + return cls.validate_testcase_list[validate_testcase]['retry'] @classmethod - def _result_acquired(cls, script_type, testcase, acquired=None): + def _result_acquired(cls, testcase, acquired=None): if acquired is not None: - cls.script_testcase_list[script_type][testcase]['acquired'] = \ + cls.validate_testcase_list[testcase]['acquired'] = \ acquired - return cls.script_testcase_list[script_type][testcase]['acquired'] + return cls.validate_testcase_list[testcase]['acquired'] @classmethod def load(cls): for root, dirs, files in \ - os.walk(dt_cfg.dovetail_config['TESTCASE_PATH']): + os.walk(dt_cfg.dovetail_config['TESTCASE_PATH']): for testcase_file in files: with open(os.path.join(root, testcase_file)) as f: testcase_yaml = yaml.safe_load(f) - cls.testcase_list[testcase_yaml.keys()[0]] = \ - cls(testcase_yaml) + case_type = testcase_yaml.values()[0]['validate']['type'] + testcase = TestcaseFactory.create(case_type, testcase_yaml) + if testcase is not None: + cls.testcase_list[next(testcase_yaml.iterkeys())] = \ + testcase + else: + cls.logger.error('failed to create testcase: %s', + testcase_file) cls.logger.debug(cls.testcase_list) @classmethod @@ -160,6 +172,60 @@ class Testcase: return None +class FunctestTestcase(Testcase): + + validate_testcase_list = {} + + def __init__(self, testcase_yaml): + super(FunctestTestcase, self).__init__(testcase_yaml) + self.name = 'functest' + + def prepare_cmd(self): + ret = super(FunctestTestcase, self).prepare_cmd() + if not ret: + for cmd in \ + dt_cfg.dovetail_config[self.name]['cmds']: + cmd_lines = Parser.parse_cmd(cmd, self) + if not cmd_lines: + return False + self.cmds.append(cmd_lines) + return True + return ret + + +class YardstickTestcase(Testcase): + + validate_testcae_list = {} + + def __init__(self, testcase_yaml): + super(YardstickTestcase, self).__init__(testcase_yaml) + self.name = 'yardstick' + + +class ShellTestcase(Testcase): + + validate_testcase_list = {} + + def __init__(self, testcase_yaml): + super(ShellTestcase, self).__init__(testcase_yaml) + self.name = 'shell' + + +class TestcaseFactory(object): + TESTCASE_TYPE_MAP = { + 'functest': FunctestTestcase, + 'yardstick': YardstickTestcase, + 'shell': ShellTestcase, + } + + @classmethod + def create(cls, testcase_type, testcase_yaml): + try: + return cls.TESTCASE_TYPE_MAP[testcase_type](testcase_yaml) + except KeyError: + return None + + class Testsuite: logger = None @@ -182,7 +248,7 @@ class Testsuite: @classmethod def load(cls): for root, dirs, files in \ - os.walk(dt_cfg.dovetail_config['COMPLIANCE_PATH']): + os.walk(dt_cfg.dovetail_config['COMPLIANCE_PATH']): for testsuite_yaml in files: with open(os.path.join(root, testsuite_yaml)) as f: testsuite_yaml = yaml.safe_load(f) diff --git a/dovetail/testcase/example.tc001.yml b/dovetail/testcase/example.tc001.yml new file mode 100644 index 00000000..eaf7e754 --- /dev/null +++ b/dovetail/testcase/example.tc001.yml @@ -0,0 +1,18 @@ +dovetail.example.tc001: + name: dovetail.example.tc001 + objective: Bulk creation and deletion of IPv6 networks, ports and subnets + validate: + type: functest + testcase: tempest_smoke_serial + pre_condition: + - 'echo test for precondition' + cmds: + - 'functest env prepare' + - 'functest testcase run {{validate_testcase}}' + post_condition: + - 'echo test for precondition' + report: + sub_testcase_list: + - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network + - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_port + - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_subnet diff --git a/dovetail/testcase/example.tc002.yml b/dovetail/testcase/example.tc002.yml new file mode 100644 index 00000000..89d000c9 --- /dev/null +++ b/dovetail/testcase/example.tc002.yml @@ -0,0 +1,15 @@ +dovetail.example.tc002: + name: dovetail.example.tc002 + objective: VIM ipv6 operations, to create/update/delete an IPv6 network and subnet + validate: + type: shell + testcase: "run shell" + pre_condition: + - "echo pre_condition" + cmds: + - "echo test2" + post_condition: + - "echo post_condition" + report: + sub_testcase_list: + diff --git a/dovetail/testcase/ipv6.tc001.yml b/dovetail/testcase/ipv6.tc001.yml index 1d9a9c38..0bc0baaa 100644 --- a/dovetail/testcase/ipv6.tc001.yml +++ b/dovetail/testcase/ipv6.tc001.yml @@ -1,9 +1,17 @@ dovetail.ipv6.tc001: name: dovetail.ipv6.tc001 objective: Bulk creation and deletion of IPv6 networks, ports and subnets - scripts: + validate: type: functest testcase: tempest_smoke_serial + pre_condition: + - 'echo test for precondition' + cmds: + - 'functest env prepare' + - 'functest testcase run {{validate_testcase}}' + post_condition: + - 'echo test for precondition' + report: sub_testcase_list: - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_port diff --git a/dovetail/testcase/ipv6.tc002.yml b/dovetail/testcase/ipv6.tc002.yml index 86af7300..efdc7dce 100644 --- a/dovetail/testcase/ipv6.tc002.yml +++ b/dovetail/testcase/ipv6.tc002.yml @@ -1,9 +1,11 @@ dovetail.ipv6.tc002: name: dovetail.ipv6.tc002 objective: VIM ipv6 operations, to create/update/delete an IPv6 network and subnet - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_networks.NetworksIpV6Test.test_create_update_delete_network_subnet - tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_update_delete_network_subnet + diff --git a/dovetail/testcase/ipv6.tc003.yml b/dovetail/testcase/ipv6.tc003.yml index 1fedf32c..0b7dc9b2 100644 --- a/dovetail/testcase/ipv6.tc003.yml +++ b/dovetail/testcase/ipv6.tc003.yml @@ -1,9 +1,10 @@ dovetail.ipv6.tc003: name: dovetail.ipv6.tc003 objective: VIM ipv6 operations, to check external network visibility - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_networks.NetworksIpV6Test.test_external_network_visibility - tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_external_network_visibility diff --git a/dovetail/testcase/ipv6.tc004.yml b/dovetail/testcase/ipv6.tc004.yml index 53f9f2ed..10a977e1 100644 --- a/dovetail/testcase/ipv6.tc004.yml +++ b/dovetail/testcase/ipv6.tc004.yml @@ -1,9 +1,10 @@ dovetail.ipv6.tc004: name: dovetail.ipv6.tc004 objective: VIM ipv6 operations, to list IPv6 networks and subnets of a tenant - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_networks.NetworksIpV6Test.test_list_networks - tempest.api.network.test_networks.NetworksIpV6Test.test_list_subnets diff --git a/dovetail/testcase/ipv6.tc005.yml b/dovetail/testcase/ipv6.tc005.yml index 737127ca..8d6deece 100644 --- a/dovetail/testcase/ipv6.tc005.yml +++ b/dovetail/testcase/ipv6.tc005.yml @@ -1,9 +1,10 @@ dovetail.ipv6.tc005: name: dovetail.ipv6.tc005 objective: VIM ipv6 operations, to show information of an IPv6 network and subnet - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_networks.NetworksIpV6Test.test_show_network - tempest.api.network.test_networks.NetworksIpV6Test.test_show_subnet diff --git a/dovetail/testcase/ipv6.tc006.yml b/dovetail/testcase/ipv6.tc006.yml index 2aff3bba..96a902c5 100644 --- a/dovetail/testcase/ipv6.tc006.yml +++ b/dovetail/testcase/ipv6.tc006.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc006: name: dovetail.ipv6.tc006 objective: VIM ipv6 operations, to create an IPv6 port in allowed allocation pools - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_port_in_allowed_allocation_pools diff --git a/dovetail/testcase/ipv6.tc007.yml b/dovetail/testcase/ipv6.tc007.yml index 695ae2e6..9e9de499 100644 --- a/dovetail/testcase/ipv6.tc007.yml +++ b/dovetail/testcase/ipv6.tc007.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc007: name: dovetail.ipv6.tc007 objective: VIM ipv6 operations, to create an IPv6 port without security groups - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_port_with_no_securitygroups diff --git a/dovetail/testcase/ipv6.tc008.yml b/dovetail/testcase/ipv6.tc008.yml index f1889446..e057a570 100644 --- a/dovetail/testcase/ipv6.tc008.yml +++ b/dovetail/testcase/ipv6.tc008.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc008: name: dovetail.ipv6.tc008 objective: VIM ipv6 operations, to create/update/delete an IPv6 port - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_update_delete_port diff --git a/dovetail/testcase/ipv6.tc009.yml b/dovetail/testcase/ipv6.tc009.yml index 790c0ec7..56e8015b 100644 --- a/dovetail/testcase/ipv6.tc009.yml +++ b/dovetail/testcase/ipv6.tc009.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc009: name: dovetail.ipv6.tc009 objective: VIM ipv6 operations, to list IPv6 ports of a tenant - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_ports.PortsIpV6TestJSON.test_list_ports diff --git a/dovetail/testcase/ipv6.tc010.yml b/dovetail/testcase/ipv6.tc010.yml index 35d8ee91..eb055ec6 100644 --- a/dovetail/testcase/ipv6.tc010.yml +++ b/dovetail/testcase/ipv6.tc010.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc010: name: dovetail.ipv6.tc010 objective: VIM ipv6 operations, to show information of an IPv6 port - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_ports.PortsIpV6TestJSON.test_show_port diff --git a/dovetail/testcase/ipv6.tc011.yml b/dovetail/testcase/ipv6.tc011.yml index db3d155e..caf44cf0 100644 --- a/dovetail/testcase/ipv6.tc011.yml +++ b/dovetail/testcase/ipv6.tc011.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc011: name: dovetail.ipv6.tc011 objective: VIM ipv6 operations, to add multiple interfaces for an IPv6 router - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_routers.RoutersIpV6Test.test_add_multiple_router_interfaces diff --git a/dovetail/testcase/ipv6.tc012.yml b/dovetail/testcase/ipv6.tc012.yml index 2b471342..ab8e7c48 100644 --- a/dovetail/testcase/ipv6.tc012.yml +++ b/dovetail/testcase/ipv6.tc012.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc012: name: dovetail.ipv6.tc012 objective: VIM ipv6 operations, to add and remove an IPv6 router interface with port_id - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_routers.RoutersIpV6Test.test_add_remove_router_interface_with_port_id diff --git a/dovetail/testcase/ipv6.tc013.yml b/dovetail/testcase/ipv6.tc013.yml index ca52a434..58732624 100644 --- a/dovetail/testcase/ipv6.tc013.yml +++ b/dovetail/testcase/ipv6.tc013.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc013: name: dovetail.ipv6.tc013 objective: VIM ipv6 operations, to add and remove an IPv6 router interface with subnet_id - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_routers.RoutersIpV6Test.test_add_remove_router_interface_with_subnet_id diff --git a/dovetail/testcase/ipv6.tc014.yml b/dovetail/testcase/ipv6.tc014.yml index 0982ad09..c8c6de1c 100644 --- a/dovetail/testcase/ipv6.tc014.yml +++ b/dovetail/testcase/ipv6.tc014.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc014: name: dovetail.ipv6.tc014 objective: VIM ipv6 operations, to create, update, delete, list and show an IPv6 router - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_routers.RoutersIpV6Test.test_create_show_list_update_delete_router diff --git a/dovetail/testcase/ipv6.tc015.yml b/dovetail/testcase/ipv6.tc015.yml index f4c38ac4..e3a47261 100644 --- a/dovetail/testcase/ipv6.tc015.yml +++ b/dovetail/testcase/ipv6.tc015.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc015: name: dovetail.ipv6.tc015 objective: VIM ipv6 operations, to create, update, delete, list and show an IPv6 security group - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_list_update_show_delete_security_group diff --git a/dovetail/testcase/ipv6.tc016.yml b/dovetail/testcase/ipv6.tc016.yml index 0bc17c1b..8195e59c 100644 --- a/dovetail/testcase/ipv6.tc016.yml +++ b/dovetail/testcase/ipv6.tc016.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc016: name: dovetail.ipv6.tc016 objective: VIM ipv6 operations, to create, delete and show security group rules - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_show_delete_security_group_rule diff --git a/dovetail/testcase/ipv6.tc017.yml b/dovetail/testcase/ipv6.tc017.yml index 1df7caf4..8b7ae5fc 100644 --- a/dovetail/testcase/ipv6.tc017.yml +++ b/dovetail/testcase/ipv6.tc017.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc017: name: dovetail.ipv6.tc017 objective: VIM ipv6 operations, to list all security groups - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_security_groups.SecGroupIPv6Test.test_list_security_groups diff --git a/dovetail/testcase/ipv6.tc018.yml b/dovetail/testcase/ipv6.tc018.yml index 01c4d3f7..be888b4f 100644 --- a/dovetail/testcase/ipv6.tc018.yml +++ b/dovetail/testcase/ipv6.tc018.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc018: name: dovetail.ipv6.tc018 objective: VIM ipv6 operations, to show information of an IPv6 port - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_dhcp6_stateless_from_os diff --git a/dovetail/testcase/ipv6.tc019.yml b/dovetail/testcase/ipv6.tc019.yml index d44b9309..524d0a4c 100644 --- a/dovetail/testcase/ipv6.tc019.yml +++ b/dovetail/testcase/ipv6.tc019.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc019: name: dovetail.ipv6.tc019 objective: VIM ipv6 operations, to do IPv6 address assignment - dual stack, DHCPv6 stateless - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_dhcp6_stateless_from_os diff --git a/dovetail/testcase/ipv6.tc020.yml b/dovetail/testcase/ipv6.tc020.yml index e974e083..4cc65e1a 100644 --- a/dovetail/testcase/ipv6.tc020.yml +++ b/dovetail/testcase/ipv6.tc020.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc020: name: dovetail.ipv6.tc020 objective: VIM ipv6 operations, to do IPv6 Address Assignment - Multiple Prefixes, DHCPv6 Stateless - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_dhcpv6_stateless diff --git a/dovetail/testcase/ipv6.tc021.yml b/dovetail/testcase/ipv6.tc021.yml index 20544530..199336aa 100644 --- a/dovetail/testcase/ipv6.tc021.yml +++ b/dovetail/testcase/ipv6.tc021.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc021: name: dovetail.ipv6.tc021 objective: VIM ipv6 operations, to do IPv6 Address Assignment - Dual Stack, Multiple Prefixes, DHCPv6 Stateless - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_dhcpv6_stateless diff --git a/dovetail/testcase/ipv6.tc022.yml b/dovetail/testcase/ipv6.tc022.yml index e01c5b6f..1e6cf0bd 100644 --- a/dovetail/testcase/ipv6.tc022.yml +++ b/dovetail/testcase/ipv6.tc022.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc022: name: dovetail.ipv6.tc022 objective: VIM ipv6 operations, to do IPv6 Address Assignment - SLAAC - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_slaac_from_os diff --git a/dovetail/testcase/ipv6.tc023.yml b/dovetail/testcase/ipv6.tc023.yml index cd17501d..1c735f0a 100644 --- a/dovetail/testcase/ipv6.tc023.yml +++ b/dovetail/testcase/ipv6.tc023.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc023: name: dovetail.ipv6.tc023 objective: VIM ipv6 operations, to do IPv6 Address Assignment - Dual Stack, SLAAC - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_dhcp6_stateless_from_os diff --git a/dovetail/testcase/ipv6.tc024.yml b/dovetail/testcase/ipv6.tc024.yml index 1c8a93f8..010008db 100644 --- a/dovetail/testcase/ipv6.tc024.yml +++ b/dovetail/testcase/ipv6.tc024.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc024: name: dovetail.ipv6.tc024 objective: VIM ipv6 operations, to do IPv6 address assignment - multiple prefixes, SLAAC - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_slaac diff --git a/dovetail/testcase/ipv6.tc025.yml b/dovetail/testcase/ipv6.tc025.yml index 3f9d97b8..d735a60f 100644 --- a/dovetail/testcase/ipv6.tc025.yml +++ b/dovetail/testcase/ipv6.tc025.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc025: name: dovetail.ipv6.tc025 objective: VIM ipv6 operations, to do IPv6 address assignment - dual stack, multiple prefixes, SLAAC - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_slaac diff --git a/dovetail/testcase/nfvi.tc001.yml b/dovetail/testcase/nfvi.tc001.yml index 136fd9d1..b796e803 100644 --- a/dovetail/testcase/nfvi.tc001.yml +++ b/dovetail/testcase/nfvi.tc001.yml @@ -1,7 +1,8 @@ dovetail.nfvi.tc001: name: dovetail.nfvi.tc001 objective: testing for vping using ssh - scripts: + validate: type: functest testcase: vping_ssh + report: sub_testcase_list: diff --git a/dovetail/testcase/nfvi.tc002.yml b/dovetail/testcase/nfvi.tc002.yml index f5724c56..d9413477 100644 --- a/dovetail/testcase/nfvi.tc002.yml +++ b/dovetail/testcase/nfvi.tc002.yml @@ -1,7 +1,8 @@ dovetail.nfvi.tc002: name: dovetail.nfvi.tc002 objective: testing for vping using userdata - scripts: + validate: type: functest testcase: vping_userdata + report: sub_testcase_list: diff --git a/dovetail/testcase/vimops.tc001.yml b/dovetail/testcase/vimops.tc001.yml index 3d2ba0c0..5f8da6a1 100644 --- a/dovetail/testcase/vimops.tc001.yml +++ b/dovetail/testcase/vimops.tc001.yml @@ -1,8 +1,9 @@ dovetail.vimops.tc001: name: dovetail.vimops.tc001 objective: Glance images v2 index - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.ListImagesTest.test_list_no_params diff --git a/dovetail/testcase/vimops.tc002.yml b/dovetail/testcase/vimops.tc002.yml index 15f5bf08..90201f56 100644 --- a/dovetail/testcase/vimops.tc002.yml +++ b/dovetail/testcase/vimops.tc002.yml @@ -1,9 +1,10 @@ dovetail.vimops.tc002: name: dovetail.vimops.tc002 objective: Glance Images v2 Delete - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_delete_image - tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_image_null_id diff --git a/dovetail/testcase/vimops.tc003.yml b/dovetail/testcase/vimops.tc003.yml index 418c041c..d63a17c4 100644 --- a/dovetail/testcase/vimops.tc003.yml +++ b/dovetail/testcase/vimops.tc003.yml @@ -1,9 +1,10 @@ dovetail.vimops.tc003: name: dovetail.vimops.tc003 objective: Glance images v2 list - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.ListImagesTest.test_get_image_schema - tempest.api.image.v2.test_images.ListImagesTest.test_get_images_schema diff --git a/dovetail/testcase/vimops.tc004.yml b/dovetail/testcase/vimops.tc004.yml index 3d205a38..3f924a4c 100644 --- a/dovetail/testcase/vimops.tc004.yml +++ b/dovetail/testcase/vimops.tc004.yml @@ -1,9 +1,10 @@ dovetail.vimops.tc004: name: dovetail.vimops.tc004 objective: Glance images v2 list - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_container_format - tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_disk_format diff --git a/dovetail/testcase/vimops.tc005.yml b/dovetail/testcase/vimops.tc005.yml index d9413849..acaad685 100644 --- a/dovetail/testcase/vimops.tc005.yml +++ b/dovetail/testcase/vimops.tc005.yml @@ -1,9 +1,10 @@ dovetail.vimops.tc005: name: dovetail.vimops.tc005 objective: Glance images v2 import - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_register_upload_get_image_file - tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_register_with_invalid_container_format diff --git a/dovetail/testcase/vimops.tc006.yml b/dovetail/testcase/vimops.tc006.yml index 0676c317..68e2ca05 100644 --- a/dovetail/testcase/vimops.tc006.yml +++ b/dovetail/testcase/vimops.tc006.yml @@ -1,9 +1,10 @@ dovetail.vimops.tc006: name: dovetail.vimops.tc006 objective: Glance images v2 update - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_update_image - tempest.api.image.v2.test_images_tags.ImagesTagsTest.test_update_delete_tags_for_image diff --git a/dovetail/tests/unit/test_parser.py b/dovetail/tests/unit/test_parser.py index 8d32995c..410a6629 100644 --- a/dovetail/tests/unit/test_parser.py +++ b/dovetail/tests/unit/test_parser.py @@ -32,7 +32,8 @@ class TestParser(unittest.TestCase): def test_parser_cmd(self): """Test whether the command is correctly parsed.""" - mock_cmd = "python /functest/ci/run_tests.py -t {{script_testcase}} -r" + mock_cmd = "python /functest/ci/run_tests.py "\ + "-t {{validate_testcase}} -r" with open(os.path.join(self.test_path, 'test_testcase.yaml')) as f: mock_testcase_yaml = yaml.safe_load(f) MockTestcase = type('Testcase', (object,), {}) @@ -45,7 +46,8 @@ class TestParser(unittest.TestCase): def test_parser_cmd_fail(self): """Test whether the command is correctly parsed.""" - mock_cmd = "python /functest/ci/run_tests.py -t {{script_testcase}} -r" + mock_cmd = "python /functest/ci/run_tests.py "\ + "-t {{validate_testcase}} -r" mock_testcase_yaml = {} MockTestcase = type('Testcase', (object,), {}) mock_testcase = MockTestcase() @@ -55,5 +57,6 @@ class TestParser(unittest.TestCase): "None -r") self.assertEqual(expected_output, output) + if __name__ == '__main__': unittest.main() diff --git a/dovetail/tests/unit/test_testcase.yaml b/dovetail/tests/unit/test_testcase.yaml index 1b03262f..735219c5 100644 --- a/dovetail/tests/unit/test_testcase.yaml +++ b/dovetail/tests/unit/test_testcase.yaml @@ -1,9 +1,10 @@ dovetail.ipv6.tc001: name: dovetail.ipv6.tc001 objective: VIM ipv6 operations, to create/delete network, port and subnet in bulk operation - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network - tempest.api.network.test_networks.BulkNetworkOpsIpV7Test.test_bulk_create_delete_port |