Support to run mandatory or optional test cases separately

1. Currently, dovetail can run a test suite or a test area in that test suite.
2. The test areas in one test suite are not divided into optional and mandatory.
3. Split them and support to run just mandatory or optional.
4. Support to run multiple test areas.

JIRA: DOVETAIL-505

Change-Id: I42cd7b4e11c3e3674c806e9bc999b782bf5c85c6
Signed-off-by: xudan <xudan16@huawei.com>

6 files changed, 71 insertions, 27 deletions

diff --git a/dovetail/compliance/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml
index 83b97d53..15fa7ba9 100644
--- a/dovetail/compliance/proposed_tests.yml
+++ b/dovetail/compliance/proposed_tests.yml
@@ -2,11 +2,22 @@
 proposed_tests:
   name: proposed_tests
   testcases_list:
+  # mandatory test cases
     # osinterop
     - dovetail.osinterop.tc001
     # vping
     - dovetail.vping.tc001
     - dovetail.vping.tc002
+    # HA
+    - dovetail.ha.tc001
+    - dovetail.ha.tc002
+    - dovetail.ha.tc003
+    - dovetail.ha.tc004
+    - dovetail.ha.tc005
+    - dovetail.ha.tc006
+    - dovetail.ha.tc007
+    - dovetail.ha.tc008
+  # optional test cases
     # ipv6
     - dovetail.ipv6.tc001
     - dovetail.ipv6.tc002
@@ -33,15 +44,6 @@ proposed_tests:
     - dovetail.ipv6.tc023
     - dovetail.ipv6.tc024
     - dovetail.ipv6.tc025
-    # HA
-    - dovetail.ha.tc001
-    - dovetail.ha.tc002
-    - dovetail.ha.tc003
-    - dovetail.ha.tc004
-    - dovetail.ha.tc005
-    - dovetail.ha.tc006
-    - dovetail.ha.tc007
-    - dovetail.ha.tc008
     # sdnvpn
     - dovetail.sdnvpn.tc001
     - dovetail.sdnvpn.tc002
@@ -53,5 +55,6 @@ proposed_tests:
     - dovetail.tempest.tc003
     - dovetail.tempest.tc004
     - dovetail.tempest.tc005
+  # other test cases
     # resiliency
     # - dovetail.resiliency.tc001
diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml
index 58409f2b..470c7108 100644
--- a/dovetail/conf/cmd_config.yml
+++ b/dovetail/conf/cmd_config.yml
@@ -45,7 +45,7 @@ cli:
       testarea:
         flags:
           - '--testarea'
-        default: 'full'
+        multiple: 'True'
         help: 'compliance testarea within testsuite'
       debug:
         flags:
diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml
index e93bd64a..b95c6d37 100644
--- a/dovetail/conf/dovetail_config.yml
+++ b/dovetail/conf/dovetail_config.yml
@@ -37,6 +37,9 @@ testarea_supported:
   - vping
   - resiliency
   - tempest
+  - optional
+  - mandatory
+  - full
 
 functest_testsuite:
   - refstack_defcore
@@ -93,3 +96,13 @@ validate_input:
     - 'cvp.0.4.0'
     - 'cvp.0.5.0'
     - 'cvp.0.6.0'
+
+mandatory:
+  - osinterop
+  - ha
+  - vping
+
+optional:
+  - ipv6
+  - tempest
+  - sdnvpn
diff --git a/dovetail/report.py b/dovetail/report.py
index fa6a0ba4..bd74ad1c 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -54,11 +54,8 @@ class Report(object):
         report_obj['duration'] = duration
 
         report_obj['testcases_list'] = []
-        testarea_list = []
-        for value in testsuite_yaml['testcases_list']:
-            if value is not None and (testarea == 'full' or testarea in value):
-                testarea_list.append(value)
-        for testcase_name in testarea_list:
+        testcase_list = Testcase.get_testcase_list(testsuite_yaml, testarea)
+        for testcase_name in testcase_list:
             testcase = Testcase.get(testcase_name)
             testcase_inreport = {}
             testcase_inreport['name'] = testcase_name
diff --git a/dovetail/run.py b/dovetail/run.py
index 1133b864..eb2cc968 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -38,14 +38,10 @@ def load_testcase():
 
 
 def run_test(testsuite, testarea, logger):
-    testarea_list = []
-    for value in testsuite['testcases_list']:
-        if value is not None and (testarea == 'full' or testarea in value):
-            testarea_list.append(value)
-
+    testcase_list = Testcase.get_testcase_list(testsuite, testarea)
     duration = 0
     start_time = time.time()
-    for testcase_name in testarea_list:
+    for testcase_name in testcase_list:
         logger.info('>>[testcase]: {}'.format(testcase_name))
         testcase = Testcase.get(testcase_name)
         if testcase is None:
@@ -275,14 +271,11 @@ def main(*args, **kwargs):
     else:
         dt_cfg.dovetail_config['offline'] = False
 
-    testarea = kwargs['testarea']
+    origin_testarea = kwargs['testarea']
     testsuite_validation = False
-    testarea_validation = False
-    if (testarea == 'full') or \
-       (testarea in dt_cfg.dovetail_config['testarea_supported']):
-        testarea_validation = True
     if kwargs['testsuite'] in dt_cfg.dovetail_config['testsuite_supported']:
         testsuite_validation = True
+    testarea_validation, testarea = Testcase.check_testarea(origin_testarea)
     if testsuite_validation and testarea_validation:
         testsuite_yaml = load_testsuite(kwargs['testsuite'])
         load_testcase()
@@ -292,7 +285,7 @@ def main(*args, **kwargs):
         Report.save_logs()
     else:
         logger.error('Invalid input commands, testsuite {} testarea {}'
-                     .format(kwargs['testsuite'], testarea))
+                     .format(kwargs['testsuite'], origin_testarea))
 
 
 dt_cfg.load_config_files()
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index b6819964..99845484 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -243,6 +243,44 @@ class Testcase(object):
             return cls.testcase_list[testcase_name]
         return None
 
+    @classmethod
+    def check_testarea(cls, testarea):
+        area_no_duplicate = []
+        area_full = ['full']
+
+        # testarea is empty, run full testsuite
+        if not testarea:
+            return True, area_full
+
+        mandatory_list = dt_cfg.dovetail_config['mandatory']
+        optional_list = dt_cfg.dovetail_config['optional']
+        for area in testarea:
+            if area not in dt_cfg.dovetail_config['testarea_supported']:
+                return False, None
+            if area == 'full':
+                return True, area_full
+            if area == 'mandatory':
+                for mandatory_area in mandatory_list:
+                    area_no_duplicate.append(mandatory_area)
+                continue
+            if area == 'optional':
+                for optional_area in optional_list:
+                    area_no_duplicate.append(optional_area)
+                continue
+            area_no_duplicate.append(area)
+        area_no_duplicate = list(set(area_no_duplicate))
+        return True, area_no_duplicate
+
+    @classmethod
+    def get_testcase_list(cls, testsuite, testarea):
+        testcase_list = []
+        for value in testsuite['testcases_list']:
+            for area in testarea:
+                if value is not None and (area == 'full' or area in value):
+                    testcase_list.append(value)
+                    break
+        return testcase_list
+
 
 class FunctestTestcase(Testcase):