Merge "Support to run mandatory or optional test cases separately"

6 files changed, 71 insertions, 27 deletions

diff --git a/dovetail/compliance/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml
index 83b97d53..15fa7ba9 100644
--- a/dovetail/compliance/proposed_tests.yml
+++ b/dovetail/compliance/proposed_tests.yml
@@ -2,11 +2,22 @@
 proposed_tests:
   name: proposed_tests
   testcases_list:
+  # mandatory test cases
     # osinterop
     - dovetail.osinterop.tc001
     # vping
     - dovetail.vping.tc001
     - dovetail.vping.tc002
+    # HA
+    - dovetail.ha.tc001
+    - dovetail.ha.tc002
+    - dovetail.ha.tc003
+    - dovetail.ha.tc004
+    - dovetail.ha.tc005
+    - dovetail.ha.tc006
+    - dovetail.ha.tc007
+    - dovetail.ha.tc008
+  # optional test cases
     # ipv6
     - dovetail.ipv6.tc001
     - dovetail.ipv6.tc002
@@ -33,15 +44,6 @@ proposed_tests:
     - dovetail.ipv6.tc023
     - dovetail.ipv6.tc024
     - dovetail.ipv6.tc025
-    # HA
-    - dovetail.ha.tc001
-    - dovetail.ha.tc002
-    - dovetail.ha.tc003
-    - dovetail.ha.tc004
-    - dovetail.ha.tc005
-    - dovetail.ha.tc006
-    - dovetail.ha.tc007
-    - dovetail.ha.tc008
     # sdnvpn
     - dovetail.sdnvpn.tc001
     - dovetail.sdnvpn.tc002
@@ -53,5 +55,6 @@ proposed_tests:
     - dovetail.tempest.tc003
     - dovetail.tempest.tc004
     - dovetail.tempest.tc005
+  # other test cases
     # resiliency
     # - dovetail.resiliency.tc001
diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml
index 58409f2b..470c7108 100644
--- a/dovetail/conf/cmd_config.yml
+++ b/dovetail/conf/cmd_config.yml
@@ -45,7 +45,7 @@ cli:
       testarea:
         flags:
           - '--testarea'
-        default: 'full'
+        multiple: 'True'
         help: 'compliance testarea within testsuite'
       debug:
         flags:
diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml
index e93bd64a..b95c6d37 100644
--- a/dovetail/conf/dovetail_config.yml
+++ b/dovetail/conf/dovetail_config.yml
@@ -37,6 +37,9 @@ testarea_supported:
   - vping
   - resiliency
   - tempest
+  - optional
+  - mandatory
+  - full
 
 functest_testsuite:
   - refstack_defcore
@@ -93,3 +96,13 @@ validate_input:
     - 'cvp.0.4.0'
     - 'cvp.0.5.0'
     - 'cvp.0.6.0'
+
+mandatory:
+  - osinterop
+  - ha
+  - vping
+
+optional:
+  - ipv6
+  - tempest
+  - sdnvpn
diff --git a/dovetail/report.py b/dovetail/report.py
index 535e5a25..28ac544f 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -55,11 +55,8 @@ class Report(object):
         report_obj['duration'] = duration
 
         report_obj['testcases_list'] = []
-        testarea_list = []
-        for value in testsuite_yaml['testcases_list']:
-            if value is not None and (testarea == 'full' or testarea in value):
-                testarea_list.append(value)
-        for testcase_name in testarea_list:
+        testcase_list = Testcase.get_testcase_list(testsuite_yaml, testarea)
+        for testcase_name in testcase_list:
             testcase = Testcase.get(testcase_name)
             testcase_inreport = {}
             testcase_inreport['name'] = testcase_name
diff --git a/dovetail/run.py b/dovetail/run.py
index 7de4c814..0306efe4 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -38,14 +38,10 @@ def load_testcase():
 
 
 def run_test(testsuite, testarea, logger):
-    testarea_list = []
-    for value in testsuite['testcases_list']:
-        if value is not None and (testarea == 'full' or testarea in value):
-            testarea_list.append(value)
-
+    testcase_list = Testcase.get_testcase_list(testsuite, testarea)
     duration = 0
     start_time = time.time()
-    for testcase_name in testarea_list:
+    for testcase_name in testcase_list:
         logger.info('>>[testcase]: {}'.format(testcase_name))
         testcase = Testcase.get(testcase_name)
         if testcase is None:
@@ -284,14 +280,11 @@ def main(*args, **kwargs):
     else:
         dt_cfg.dovetail_config['offline'] = False
 
-    testarea = kwargs['testarea']
+    origin_testarea = kwargs['testarea']
     testsuite_validation = False
-    testarea_validation = False
-    if (testarea == 'full') or \
-       (testarea in dt_cfg.dovetail_config['testarea_supported']):
-        testarea_validation = True
     if kwargs['testsuite'] in dt_cfg.dovetail_config['testsuite_supported']:
         testsuite_validation = True
+    testarea_validation, testarea = Testcase.check_testarea(origin_testarea)
     if testsuite_validation and testarea_validation:
         testsuite_yaml = load_testsuite(kwargs['testsuite'])
         load_testcase()
@@ -302,7 +295,7 @@ def main(*args, **kwargs):
             Report.save_logs()
     else:
         logger.error('Invalid input commands, testsuite {} testarea {}'
-                     .format(kwargs['testsuite'], testarea))
+                     .format(kwargs['testsuite'], origin_testarea))
 
 
 dt_cfg.load_config_files()
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index b6819964..99845484 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -243,6 +243,44 @@ class Testcase(object):
             return cls.testcase_list[testcase_name]
         return None
 
+    @classmethod
+    def check_testarea(cls, testarea):
+        area_no_duplicate = []
+        area_full = ['full']
+
+        # testarea is empty, run full testsuite
+        if not testarea:
+            return True, area_full
+
+        mandatory_list = dt_cfg.dovetail_config['mandatory']
+        optional_list = dt_cfg.dovetail_config['optional']
+        for area in testarea:
+            if area not in dt_cfg.dovetail_config['testarea_supported']:
+                return False, None
+            if area == 'full':
+                return True, area_full
+            if area == 'mandatory':
+                for mandatory_area in mandatory_list:
+                    area_no_duplicate.append(mandatory_area)
+                continue
+            if area == 'optional':
+                for optional_area in optional_list:
+                    area_no_duplicate.append(optional_area)
+                continue
+            area_no_duplicate.append(area)
+        area_no_duplicate = list(set(area_no_duplicate))
+        return True, area_no_duplicate
+
+    @classmethod
+    def get_testcase_list(cls, testsuite, testarea):
+        testcase_list = []
+        for value in testsuite['testcases_list']:
+            for area in testarea:
+                if value is not None and (area == 'full' or area in value):
+                    testcase_list.append(value)
+                    break
+        return testcase_list
+
 
 class FunctestTestcase(Testcase):