dovetail tool: testarea argument added

JIRA: DOVETAIL-71

1) argument testarea is added
2) terminology "scenario" change to "testsuite"
3) related documents amended
4) all word "certification" changed to "compliance"

Change-Id: I038a9e04bf83cfdac20f0c59adb1841bf5470584
Signed-off-by: MatthewLi <matthew.lijun@huawei.com>

9 files changed, 88 insertions, 55 deletions

diff --git a/docs/dovetailtool/dovetail.tool.installation.rst b/docs/dovetailtool/dovetail.tool.installation.rst
index 17290a35..2f544480 100644
--- a/docs/dovetailtool/dovetail.tool.installation.rst
+++ b/docs/dovetailtool/dovetail.tool.installation.rst
@@ -69,22 +69,19 @@ At this point the environment is now ready for Dovetail execution.
 Compliance and certification test cases
 ----------------------------------------
 
-The compliance and certification test cases can be defined under the ``/dovetail/cert``
+The compliance and certification test cases can be defined under the ``/dovetail/compliance``
 directory, which is defined in yaml format.
 A sample file named ``compliance_set.yml`` is provided as follows:
 
 ::
 
-  certification_compliance_set:
-    name: certification_compliance_set
+  compliance_set:
+    name: compliance_set
     testcases_list:
       - dovetail.ipv6.tc001
 
 The testcase listed here is dovetail.ipv6.tc001, defined within ``dovetail/testcase``.
 
-Note: if a new test case yaml file is created, its name should start with ``certification_``,
-in similiar fashion as the sample file ``certification_compliance_set``.
-
 Running Dovetail tool
 ---------------------
 
@@ -92,10 +89,17 @@ After environment preparation is complete and test cases added, the Dovetail too
 
 ::
 
-  python run.py --scenario compliance_set
+  python run.py --testsuite compliance_set
+
+The value ``compliance_set`` passed to the ``testsuite`` flag can be replaced with the test cases yaml file.
+If not argument is given, the compliance_set testsuite will be run as the default.
+
+Moreover, the testcases in given testarea can be run with ``testarea`` command line argument, such as
+testarea ``ipv6`` in ``compliance_set``
+
+::
 
-The value ``compliance_set`` passed to the ``scenario`` flag can be replaced with the test cases yaml file.
-If not argument is given, the compliance_set scenario will be run as the default.
+  python run.py --testsuite compliance_set --testarea ipv6
 
 Running Dovetail in a Docker container
 ########################################
@@ -145,10 +149,10 @@ Attach to the container by starting it and obtaining a bash prompt with ::
 Inside the container the following commands can be executed to trigger the testcases ::
 
    cd /home/opnfv/dovetail/dovetail
-   python run.py --scenario compliance_set
+   python run.py --testsuite compliance_set
 
 Results Output
 ###############
 
 The running log is stored in ``/home/opnfv/dovetail/results/dovetail.log``.
-The certification report is stored in ``/home/opnfv/dovetail/results/dovetail_report.txt``.
+The compliance report is stored in ``/home/opnfv/dovetail/results/dovetail_report.txt``.
diff --git a/docs/dovetailtool/dovetail.tool.overview.rst b/docs/dovetailtool/dovetail.tool.overview.rst
index a1133f34..c4b7fa22 100644
--- a/docs/dovetailtool/dovetail.tool.overview.rst
+++ b/docs/dovetailtool/dovetail.tool.overview.rst
@@ -10,7 +10,7 @@ Dovetail Tool
 What is Dovetail tool
 #####################
 
-A toolset for providing an evaluation of a deployed cloud environment for OPNFV certification.
+A toolset for providing an evaluation of a deployed cloud environment for OPNFV compliance.
 
 Overview
 ########
diff --git a/dovetail/cert/compliance_set.yml b/dovetail/compliance/compliance_set.yml
index b4108b4f..df0d427d 100644
--- a/dovetail/cert/compliance_set.yml
+++ b/dovetail/compliance/compliance_set.yml
@@ -1,5 +1,5 @@
-certification_compliance_set:
-  name: certification_compliance_set
+compliance_set:
+  name: compliance_set
   testcases_list:
     # Temporarily, one test case kept here as default to run
     # for use of software development/debug
diff --git a/dovetail/cert/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml
index 6d6c8d19..fcb7e03a 100644
--- a/dovetail/cert/proposed_tests.yml
+++ b/dovetail/compliance/proposed_tests.yml
@@ -1,5 +1,5 @@
-certification_proposed_tests:
-  name: certification_proposed_tests
+proposed_tests:
+  name: proposed_tests
   testcases_list:
     # TO DO: will adjust the dovetail tool to support in later patches
     # run.py --name1 {**/proposed/compliance} --name2 {**/vim/ipv6,etc}
diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml
index c2108c58..04001774 100644
--- a/dovetail/conf/cmd_config.yml
+++ b/dovetail/conf/cmd_config.yml
@@ -37,12 +37,16 @@ cli:
           - '-d'
         help: 'CI_DEBUG for showing debug log.'
     non-envs:
-      scenario:
+      testsuite:
         flags:
-          - '--scenario'
-          - '-s'
+          - '--testsuite'
         default: 'compliance_set'
-        help: 'certification scenario.'
+        help: 'compliance testsuite.'
+      testarea:
+        flags:
+          - '--testarea'
+        default: 'full'
+        help: 'compliance testarea within testsuite'
       tag:
         flags:
           - '--tag'
diff --git a/dovetail/conf/dovetail_config.py b/dovetail/conf/dovetail_config.py
index 5ac23c43..5fe1e7b5 100644
--- a/dovetail/conf/dovetail_config.py
+++ b/dovetail/conf/dovetail_config.py
@@ -14,9 +14,12 @@ import re
 
 class DovetailConfig:
 
-    CERT_PATH = './cert/'
+    COMPLIANCE_PATH = './compliance/'
     TESTCASE_PATH = './testcase/'
-    SCENARIO_NAMING_FMT = 'certification_%s'
+    # testsuite supported tuple, should adjust accordingly
+    testsuite_supported = ('compliance_set', 'proposed_tests')
+    # testarea supported tuple, should adjust accordingly
+    testarea_supported = ('vimops', 'nfvi', 'ipv6')
 
     curr_path = os.path.dirname(os.path.abspath(__file__))
 
diff --git a/dovetail/report.py b/dovetail/report.py
index 7dd8ba2d..2e544ad3 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -36,11 +36,15 @@ class Report:
         checker.check(testcase, db_result)
 
     @classmethod
-    def generate_json(cls, scenario_yaml):
+    def generate_json(cls, testsuite_yaml, testarea):
         report_obj = {}
-        report_obj['scenario'] = scenario_yaml['name']
+        report_obj['testsuite'] = testsuite_yaml['name']
         report_obj['testcases_list'] = []
-        for testcase_name in scenario_yaml['testcases_list']:
+        testarea_list = []
+        for value in testsuite_yaml['testcases_list']:
+            if value is not None and (testarea == 'full' or testarea in value):
+                testarea_list.append(value)
+        for testcase_name in testarea_list:
             testcase = Testcase.get(testcase_name)
             testcase_in_rpt = {}
             testcase_in_rpt['name'] = testcase_name
@@ -66,8 +70,8 @@ class Report:
         return report_obj
 
     @classmethod
-    def generate(cls, scenario_yaml):
-        rpt_data = cls.generate_json(scenario_yaml)
+    def generate(cls, testsuite_yaml, testarea):
+        rpt_data = cls.generate_json(testsuite_yaml, testarea)
         rpt_text = ''
         split_line = '+-----------------------------------------------------'
         split_line += '---------------------+\n'
@@ -76,7 +80,7 @@ class Report:
 +==========================================================================+\n\
 |                                   report                                 |\n'
         rpt_text += split_line
-        rpt_text += '|scenario: %s\n' % rpt_data['scenario']
+        rpt_text += '|testsuite: %s\n' % rpt_data['testsuite']
         for testcase in rpt_data['testcases_list']:
             rpt_text += '|   [testcase]: %s\t\t\t\t[%s]\n' % \
                 (testcase['name'], testcase['result'])
diff --git a/dovetail/run.py b/dovetail/run.py
index 86194389..fcc9da1a 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -16,16 +16,16 @@ import utils.dovetail_logger as dt_logger
 
 from container import Container
 from testcase import Testcase
-from testcase import Scenario
+from testcase import Testsuite
 from report import Report
 from conf.dovetail_config import DovetailConfig as dt_config
 
 logger = dt_logger.Logger('run.py').getLogger()
 
 
-def load_scenario(scenario):
-    Scenario.load()
-    return Scenario.get(dt_config.SCENARIO_NAMING_FMT % scenario)
+def load_testsuite(testsuite):
+    Testsuite.load()
+    return Testsuite.get(testsuite)
 
 
 def set_container_tags(option_str):
@@ -40,8 +40,13 @@ def load_testcase():
     Testcase.load()
 
 
-def run_test(scenario):
-    for testcase_name in scenario['testcases_list']:
+def run_test(testsuite, testarea):
+    testarea_list = []
+    for value in testsuite['testcases_list']:
+        if value is not None and (testarea == 'full' or testarea in value):
+            testarea_list.append(value)
+
+    for testcase_name in testarea_list:
         logger.info('>>[testcase]: %s' % (testcase_name))
         testcase = Testcase.get(testcase_name)
         if testcase is None:
@@ -102,10 +107,10 @@ def filter_env_options(input_dict):
 
 
 def main(*args, **kwargs):
-    """Dovetail certification test entry!"""
-    logger.info('=======================================')
-    logger.info('Dovetail certification: %s!' % (kwargs['scenario']))
-    logger.info('=======================================')
+    """Dovetail compliance test entry!"""
+    logger.info('================================================')
+    logger.info('Dovetail compliance: %s!' % (kwargs['testsuite']))
+    logger.info('================================================')
     validate_options(kwargs)
     envs_options = filter_env_options(kwargs)
     dt_config.update_envs(envs_options)
@@ -113,12 +118,25 @@ def main(*args, **kwargs):
                 dt_config.dovetail_config['functest']['envs'])
     logger.info('Your new envs for yardstick: %s' %
                 dt_config.dovetail_config['yardstick']['envs'])
-    load_testcase()
-    scenario_yaml = load_scenario(kwargs['scenario'])
+
     if 'tag' in kwargs and kwargs['tag'] is not None:
         set_container_tags(kwargs['tag'])
-    run_test(scenario_yaml)
-    Report.generate(scenario_yaml)
+
+    load_testcase()
+    testsuite_yaml = load_testsuite(kwargs['testsuite'])
+    testarea = kwargs['testarea']
+    testsuite_validation = False
+    testarea_validation = False
+    if (testarea == 'full') or (testarea in dt_config.testarea_supported):
+        testarea_validation = True
+    if kwargs['testsuite'] in dt_config.testsuite_supported:
+        testsuite_validation = True
+    if testsuite_validation and testarea_validation:
+        run_test(testsuite_yaml, testarea)
+        Report.generate(testsuite_yaml, testarea)
+    else:
+        logger.error('invalid input commands, testsuite %s testarea %s' %
+                     (kwargs['testsuite'], testarea))
 
 
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index 6f0552b6..b5d28aa1 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -155,10 +155,10 @@ class Testcase:
         return None
 
 
-class Scenario:
+class Testsuite:
 
-    def __init__(self, scenario):
-        self.scenario = scenario
+    def __init__(self, testsuite):
+        self.testsuite = testsuite
         self.testcase_list = {}
 
     def get_test(self, testcase_name):
@@ -166,20 +166,20 @@ class Scenario:
             return self.testcase_list[testcase_name]
         return None
 
-    scenario_list = {}
+    testsuite_list = {}
 
     @classmethod
     def load(cls):
-        for root, dirs, files in os.walk(dt_config.CERT_PATH):
-            for scenario_yaml in files:
-                with open(os.path.join(root, scenario_yaml)) as f:
-                    scenario_yaml = yaml.safe_load(f)
-                    cls.scenario_list.update(scenario_yaml)
+        for root, dirs, files in os.walk(dt_config.COMPLIANCE_PATH):
+            for testsuite_yaml in files:
+                with open(os.path.join(root, testsuite_yaml)) as f:
+                    testsuite_yaml = yaml.safe_load(f)
+                    cls.testsuite_list.update(testsuite_yaml)
 
-        logger.debug(cls.scenario_list)
+        logger.debug(cls.testsuite_list)
 
     @classmethod
-    def get(cls, scenario_name):
-        if scenario_name in cls.scenario_list:
-            return cls.scenario_list[scenario_name]
+    def get(cls, testsuite_name):
+        if testsuite_name in cls.testsuite_list:
+            return cls.testsuite_list[testsuite_name]
         return None