summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorxudan <xudan16@huawei.com>2017-09-25 23:30:08 -0400
committerxudan <xudan16@huawei.com>2017-09-25 23:30:08 -0400
commit186cdc71bf9bdd00be1df87d90d1a4c57790208c (patch)
tree7849337a1bd5da7f036183903189000dba741bbb
parent6f47c82d4b6f830863ff1330163d869baa6c7eec (diff)
Support to run mandatory or optional test cases separately
1. Currently, dovetail can run a test suite or a test area in that test suite. 2. The test areas in one test suite are not divided into optional and mandatory. 3. Split them and support to run just mandatory or optional. 4. Support to run multiple test areas. JIRA: DOVETAIL-505 Change-Id: I42cd7b4e11c3e3674c806e9bc999b782bf5c85c6 Signed-off-by: xudan <xudan16@huawei.com>
-rw-r--r--dovetail/compliance/proposed_tests.yml21
-rw-r--r--dovetail/conf/cmd_config.yml2
-rw-r--r--dovetail/conf/dovetail_config.yml13
-rw-r--r--dovetail/report.py7
-rwxr-xr-xdovetail/run.py17
-rw-r--r--dovetail/testcase.py38
6 files changed, 71 insertions, 27 deletions
diff --git a/dovetail/compliance/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml
index 83b97d53..15fa7ba9 100644
--- a/dovetail/compliance/proposed_tests.yml
+++ b/dovetail/compliance/proposed_tests.yml
@@ -2,11 +2,22 @@
proposed_tests:
name: proposed_tests
testcases_list:
+ # mandatory test cases
# osinterop
- dovetail.osinterop.tc001
# vping
- dovetail.vping.tc001
- dovetail.vping.tc002
+ # HA
+ - dovetail.ha.tc001
+ - dovetail.ha.tc002
+ - dovetail.ha.tc003
+ - dovetail.ha.tc004
+ - dovetail.ha.tc005
+ - dovetail.ha.tc006
+ - dovetail.ha.tc007
+ - dovetail.ha.tc008
+ # optional test cases
# ipv6
- dovetail.ipv6.tc001
- dovetail.ipv6.tc002
@@ -33,15 +44,6 @@ proposed_tests:
- dovetail.ipv6.tc023
- dovetail.ipv6.tc024
- dovetail.ipv6.tc025
- # HA
- - dovetail.ha.tc001
- - dovetail.ha.tc002
- - dovetail.ha.tc003
- - dovetail.ha.tc004
- - dovetail.ha.tc005
- - dovetail.ha.tc006
- - dovetail.ha.tc007
- - dovetail.ha.tc008
# sdnvpn
- dovetail.sdnvpn.tc001
- dovetail.sdnvpn.tc002
@@ -53,5 +55,6 @@ proposed_tests:
- dovetail.tempest.tc003
- dovetail.tempest.tc004
- dovetail.tempest.tc005
+ # other test cases
# resiliency
# - dovetail.resiliency.tc001
diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml
index 58409f2b..470c7108 100644
--- a/dovetail/conf/cmd_config.yml
+++ b/dovetail/conf/cmd_config.yml
@@ -45,7 +45,7 @@ cli:
testarea:
flags:
- '--testarea'
- default: 'full'
+ multiple: 'True'
help: 'compliance testarea within testsuite'
debug:
flags:
diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml
index e93bd64a..b95c6d37 100644
--- a/dovetail/conf/dovetail_config.yml
+++ b/dovetail/conf/dovetail_config.yml
@@ -37,6 +37,9 @@ testarea_supported:
- vping
- resiliency
- tempest
+ - optional
+ - mandatory
+ - full
functest_testsuite:
- refstack_defcore
@@ -93,3 +96,13 @@ validate_input:
- 'cvp.0.4.0'
- 'cvp.0.5.0'
- 'cvp.0.6.0'
+
+mandatory:
+ - osinterop
+ - ha
+ - vping
+
+optional:
+ - ipv6
+ - tempest
+ - sdnvpn
diff --git a/dovetail/report.py b/dovetail/report.py
index fa6a0ba4..bd74ad1c 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -54,11 +54,8 @@ class Report(object):
report_obj['duration'] = duration
report_obj['testcases_list'] = []
- testarea_list = []
- for value in testsuite_yaml['testcases_list']:
- if value is not None and (testarea == 'full' or testarea in value):
- testarea_list.append(value)
- for testcase_name in testarea_list:
+ testcase_list = Testcase.get_testcase_list(testsuite_yaml, testarea)
+ for testcase_name in testcase_list:
testcase = Testcase.get(testcase_name)
testcase_inreport = {}
testcase_inreport['name'] = testcase_name
diff --git a/dovetail/run.py b/dovetail/run.py
index 1133b864..eb2cc968 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -38,14 +38,10 @@ def load_testcase():
def run_test(testsuite, testarea, logger):
- testarea_list = []
- for value in testsuite['testcases_list']:
- if value is not None and (testarea == 'full' or testarea in value):
- testarea_list.append(value)
-
+ testcase_list = Testcase.get_testcase_list(testsuite, testarea)
duration = 0
start_time = time.time()
- for testcase_name in testarea_list:
+ for testcase_name in testcase_list:
logger.info('>>[testcase]: {}'.format(testcase_name))
testcase = Testcase.get(testcase_name)
if testcase is None:
@@ -275,14 +271,11 @@ def main(*args, **kwargs):
else:
dt_cfg.dovetail_config['offline'] = False
- testarea = kwargs['testarea']
+ origin_testarea = kwargs['testarea']
testsuite_validation = False
- testarea_validation = False
- if (testarea == 'full') or \
- (testarea in dt_cfg.dovetail_config['testarea_supported']):
- testarea_validation = True
if kwargs['testsuite'] in dt_cfg.dovetail_config['testsuite_supported']:
testsuite_validation = True
+ testarea_validation, testarea = Testcase.check_testarea(origin_testarea)
if testsuite_validation and testarea_validation:
testsuite_yaml = load_testsuite(kwargs['testsuite'])
load_testcase()
@@ -292,7 +285,7 @@ def main(*args, **kwargs):
Report.save_logs()
else:
logger.error('Invalid input commands, testsuite {} testarea {}'
- .format(kwargs['testsuite'], testarea))
+ .format(kwargs['testsuite'], origin_testarea))
dt_cfg.load_config_files()
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index b6819964..99845484 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -243,6 +243,44 @@ class Testcase(object):
return cls.testcase_list[testcase_name]
return None
+ @classmethod
+ def check_testarea(cls, testarea):
+ area_no_duplicate = []
+ area_full = ['full']
+
+ # testarea is empty, run full testsuite
+ if not testarea:
+ return True, area_full
+
+ mandatory_list = dt_cfg.dovetail_config['mandatory']
+ optional_list = dt_cfg.dovetail_config['optional']
+ for area in testarea:
+ if area not in dt_cfg.dovetail_config['testarea_supported']:
+ return False, None
+ if area == 'full':
+ return True, area_full
+ if area == 'mandatory':
+ for mandatory_area in mandatory_list:
+ area_no_duplicate.append(mandatory_area)
+ continue
+ if area == 'optional':
+ for optional_area in optional_list:
+ area_no_duplicate.append(optional_area)
+ continue
+ area_no_duplicate.append(area)
+ area_no_duplicate = list(set(area_no_duplicate))
+ return True, area_no_duplicate
+
+ @classmethod
+ def get_testcase_list(cls, testsuite, testarea):
+ testcase_list = []
+ for value in testsuite['testcases_list']:
+ for area in testarea:
+ if value is not None and (area == 'full' or area in value):
+ testcase_list.append(value)
+ break
+ return testcase_list
+
class FunctestTestcase(Testcase):