From 186cdc71bf9bdd00be1df87d90d1a4c57790208c Mon Sep 17 00:00:00 2001 From: xudan Date: Mon, 25 Sep 2017 23:30:08 -0400 Subject: Support to run mandatory or optional test cases separately 1. Currently, dovetail can run a test suite or a test area in that test suite. 2. The test areas in one test suite are not divided into optional and mandatory. 3. Split them and support to run just mandatory or optional. 4. Support to run multiple test areas. JIRA: DOVETAIL-505 Change-Id: I42cd7b4e11c3e3674c806e9bc999b782bf5c85c6 Signed-off-by: xudan --- dovetail/compliance/proposed_tests.yml | 21 +++++++++++-------- dovetail/conf/cmd_config.yml | 2 +- dovetail/conf/dovetail_config.yml | 13 ++++++++++++ dovetail/report.py | 7 ++----- dovetail/run.py | 17 +++++---------- dovetail/testcase.py | 38 ++++++++++++++++++++++++++++++++++ 6 files changed, 71 insertions(+), 27 deletions(-) diff --git a/dovetail/compliance/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml index 83b97d53..15fa7ba9 100644 --- a/dovetail/compliance/proposed_tests.yml +++ b/dovetail/compliance/proposed_tests.yml @@ -2,11 +2,22 @@ proposed_tests: name: proposed_tests testcases_list: + # mandatory test cases # osinterop - dovetail.osinterop.tc001 # vping - dovetail.vping.tc001 - dovetail.vping.tc002 + # HA + - dovetail.ha.tc001 + - dovetail.ha.tc002 + - dovetail.ha.tc003 + - dovetail.ha.tc004 + - dovetail.ha.tc005 + - dovetail.ha.tc006 + - dovetail.ha.tc007 + - dovetail.ha.tc008 + # optional test cases # ipv6 - dovetail.ipv6.tc001 - dovetail.ipv6.tc002 @@ -33,15 +44,6 @@ proposed_tests: - dovetail.ipv6.tc023 - dovetail.ipv6.tc024 - dovetail.ipv6.tc025 - # HA - - dovetail.ha.tc001 - - dovetail.ha.tc002 - - dovetail.ha.tc003 - - dovetail.ha.tc004 - - dovetail.ha.tc005 - - dovetail.ha.tc006 - - dovetail.ha.tc007 - - dovetail.ha.tc008 # sdnvpn - dovetail.sdnvpn.tc001 - dovetail.sdnvpn.tc002 @@ -53,5 +55,6 @@ proposed_tests: - dovetail.tempest.tc003 - dovetail.tempest.tc004 - dovetail.tempest.tc005 + # other test cases # resiliency # - dovetail.resiliency.tc001 diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml index 58409f2b..470c7108 100644 --- a/dovetail/conf/cmd_config.yml +++ b/dovetail/conf/cmd_config.yml @@ -45,7 +45,7 @@ cli: testarea: flags: - '--testarea' - default: 'full' + multiple: 'True' help: 'compliance testarea within testsuite' debug: flags: diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml index e93bd64a..b95c6d37 100644 --- a/dovetail/conf/dovetail_config.yml +++ b/dovetail/conf/dovetail_config.yml @@ -37,6 +37,9 @@ testarea_supported: - vping - resiliency - tempest + - optional + - mandatory + - full functest_testsuite: - refstack_defcore @@ -93,3 +96,13 @@ validate_input: - 'cvp.0.4.0' - 'cvp.0.5.0' - 'cvp.0.6.0' + +mandatory: + - osinterop + - ha + - vping + +optional: + - ipv6 + - tempest + - sdnvpn diff --git a/dovetail/report.py b/dovetail/report.py index fa6a0ba4..bd74ad1c 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -54,11 +54,8 @@ class Report(object): report_obj['duration'] = duration report_obj['testcases_list'] = [] - testarea_list = [] - for value in testsuite_yaml['testcases_list']: - if value is not None and (testarea == 'full' or testarea in value): - testarea_list.append(value) - for testcase_name in testarea_list: + testcase_list = Testcase.get_testcase_list(testsuite_yaml, testarea) + for testcase_name in testcase_list: testcase = Testcase.get(testcase_name) testcase_inreport = {} testcase_inreport['name'] = testcase_name diff --git a/dovetail/run.py b/dovetail/run.py index 1133b864..eb2cc968 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -38,14 +38,10 @@ def load_testcase(): def run_test(testsuite, testarea, logger): - testarea_list = [] - for value in testsuite['testcases_list']: - if value is not None and (testarea == 'full' or testarea in value): - testarea_list.append(value) - + testcase_list = Testcase.get_testcase_list(testsuite, testarea) duration = 0 start_time = time.time() - for testcase_name in testarea_list: + for testcase_name in testcase_list: logger.info('>>[testcase]: {}'.format(testcase_name)) testcase = Testcase.get(testcase_name) if testcase is None: @@ -275,14 +271,11 @@ def main(*args, **kwargs): else: dt_cfg.dovetail_config['offline'] = False - testarea = kwargs['testarea'] + origin_testarea = kwargs['testarea'] testsuite_validation = False - testarea_validation = False - if (testarea == 'full') or \ - (testarea in dt_cfg.dovetail_config['testarea_supported']): - testarea_validation = True if kwargs['testsuite'] in dt_cfg.dovetail_config['testsuite_supported']: testsuite_validation = True + testarea_validation, testarea = Testcase.check_testarea(origin_testarea) if testsuite_validation and testarea_validation: testsuite_yaml = load_testsuite(kwargs['testsuite']) load_testcase() @@ -292,7 +285,7 @@ def main(*args, **kwargs): Report.save_logs() else: logger.error('Invalid input commands, testsuite {} testarea {}' - .format(kwargs['testsuite'], testarea)) + .format(kwargs['testsuite'], origin_testarea)) dt_cfg.load_config_files() diff --git a/dovetail/testcase.py b/dovetail/testcase.py index b6819964..99845484 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -243,6 +243,44 @@ class Testcase(object): return cls.testcase_list[testcase_name] return None + @classmethod + def check_testarea(cls, testarea): + area_no_duplicate = [] + area_full = ['full'] + + # testarea is empty, run full testsuite + if not testarea: + return True, area_full + + mandatory_list = dt_cfg.dovetail_config['mandatory'] + optional_list = dt_cfg.dovetail_config['optional'] + for area in testarea: + if area not in dt_cfg.dovetail_config['testarea_supported']: + return False, None + if area == 'full': + return True, area_full + if area == 'mandatory': + for mandatory_area in mandatory_list: + area_no_duplicate.append(mandatory_area) + continue + if area == 'optional': + for optional_area in optional_list: + area_no_duplicate.append(optional_area) + continue + area_no_duplicate.append(area) + area_no_duplicate = list(set(area_no_duplicate)) + return True, area_no_duplicate + + @classmethod + def get_testcase_list(cls, testsuite, testarea): + testcase_list = [] + for value in testsuite['testcases_list']: + for area in testarea: + if value is not None and (area == 'full' or area in value): + testcase_list.append(value) + break + return testcase_list + class FunctestTestcase(Testcase): -- cgit 1.2.3-korg