Improve the CLI of Dovetail tool

The CLI of dovetail tool needs some improvement.
1. Add an option '--testcase' to support specify test case.
2. Add an option '--no-clean' to keep functest/yardstick/bottlenecks containers
   for debugging after running the test cases.
3. Improve the local report section.
    a. Remove some info that is not necessary
    b. Make the info more clearly

JIRA: DOVETAIL-588

Change-Id: Ic769a6e2788b407c3df08ccd593a0ecdaa35e5ea
Signed-off-by: xudan <xudan16@huawei.com>

4 files changed, 86 insertions, 73 deletions

diff --git a/dovetail/report.py b/dovetail/report.py
index 835fed3f..20475a3b 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -42,20 +42,19 @@ class Report(object):
             checker.check(testcase, db_result)
 
     @classmethod
-    def generate_json(cls, testsuite_yaml, testarea, duration):
+    def generate_json(cls, testcase_list, duration):
         report_obj = {}
         report_obj['version'] = \
             version.VersionInfo('dovetail').version_string()
-        report_obj['testsuite'] = testsuite_yaml['name']
-        # TO DO: once dashboard url settled, adjust accordingly
-        report_obj['dashboard'] = None
         report_obj['build_tag'] = dt_cfg.dovetail_config['build_tag']
         report_obj['upload_date'] =\
             datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
         report_obj['duration'] = duration
 
         report_obj['testcases_list'] = []
-        testcase_list = Testcase.get_testcase_list(testsuite_yaml, testarea)
+        if not testcase_list:
+            return report_obj
+
         for testcase_name in testcase_list:
             testcase = Testcase.get(testcase_name)
             testcase_inreport = {}
@@ -81,19 +80,14 @@ class Report(object):
         return report_obj
 
     @classmethod
-    def generate(cls, testsuite_yaml, testarea, duration):
-        report_data = cls.generate_json(testsuite_yaml, testarea, duration)
+    def generate(cls, testcase_list, duration):
+        report_data = cls.generate_json(testcase_list, duration)
         report_txt = ''
         report_txt += '\n\nDovetail Report\n'
         report_txt += 'Version: %s\n' % report_data['version']
-        report_txt += 'TestSuite: %s\n' % report_data['testsuite']
-        report_txt += 'Result Dashboard: %s\n' % report_data['dashboard']
         report_txt += 'Build Tag: %s\n' % report_data['build_tag']
         report_txt += 'Upload Date: %s\n' % report_data['upload_date']
-        if report_data['duration'] == 0:
-            report_txt += 'Duration: %s\n\n' % 'N/A'
-        else:
-            report_txt += 'Duration: %.2f s\n\n' % report_data['duration']
+        report_txt += 'Duration: %.2f s\n\n' % report_data['duration']
 
         total_num = 0
         pass_num = 0
@@ -135,21 +129,13 @@ class Report(object):
             pass_rate = pass_num / total_num
             report_txt += 'Pass Rate: %.2f%% (%s/%s)\n' %\
                 (pass_rate * 100, pass_num, total_num)
-            report_txt += 'Assessed test areas:\n'
         else:
             report_txt += \
-                'no testcase or all testcases are skipped in this testsuite'
+                'no testcase or all testcases are skipped in this testsuite\n'
 
         for key in sub_report:
             if testcase_num[key] != 0:
                 pass_rate = testcase_passnum[key] / testcase_num[key]
-                report_txt += '-%-25s pass %.2f%%\n' %\
-                    (key + ' results:', pass_rate * 100)
-            elif key in testarea_scope:
-                report_txt += '-%-25s all skipped\n' % key
-        for key in sub_report:
-            if testcase_num[key] != 0:
-                pass_rate = testcase_passnum[key] / testcase_num[key]
                 report_txt += '%-25s  pass rate %.2f%%\n' %\
                     (key + ':', pass_rate * 100)
                 report_txt += sub_report[key]
@@ -158,7 +144,6 @@ class Report(object):
                 report_txt += sub_report[key]
 
         cls.logger.info(report_txt)
-        # cls.save(report_txt)
         return report_txt
 
     @classmethod
@@ -292,7 +277,6 @@ class FunctestCrawler(Crawler):
                         'timestop': timestop, 'duration': duration,
                         'details': details}
 
-        self.logger.debug('Results: {}'.format(str(json_results)))
         return json_results
 
 
@@ -334,7 +318,6 @@ class YardstickCrawler(Crawler):
                 except KeyError as e:
                     self.logger.exception('Pass flag not found {}'.format(e))
         json_results = {'criteria': criteria}
-        self.logger.debug('Results: {}'.format(str(json_results)))
         return json_results
 
     def add_result_to_file(self, result, tc_name):
@@ -379,7 +362,6 @@ class BottlenecksCrawler(Crawler):
                 except KeyError as e:
                     self.logger.exception('Pass flag not found {}'.format(e))
         json_results = {'criteria': criteria}
-        self.logger.debug('Results: {}'.format(str(json_results)))
         return json_results
 
 
@@ -436,7 +418,6 @@ class VnftestCrawler(Crawler):
                 except KeyError as e:
                     self.logger.exception('Pass flag not found {}'.format(e))
         json_results = {'criteria': criteria}
-        self.logger.debug('Results: {}'.format(str(json_results)))
         return json_results
 
 
diff --git a/dovetail/run.py b/dovetail/run.py
index 21e7b4c2..512565c3 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -38,21 +38,16 @@ def load_testsuite(testsuite):
     return Testsuite.get(testsuite)
 
 
-def load_testcase():
-    Testcase.load()
-
-
-def run_test(testsuite, testarea, logger, kwargs):
-    testcase_list = Testcase.get_testcase_list(testsuite, testarea)
+def run_test(testcase_list, logger):
     duration = 0
+    if not testcase_list:
+        logger.warning("No test case will be executed.")
+        return duration
+
     start_time = time.time()
     for testcase_name in testcase_list:
         logger.info('>>[testcase]: {}'.format(testcase_name))
         testcase = Testcase.get(testcase_name)
-        if testcase is None:
-            logger.error('Test case {} is not defined in testcase folder, '
-                         'skipping.'.format(testcase_name))
-            continue
         run_testcase = True
 
         # if testcase.exceed_max_retry_times():
@@ -67,7 +62,7 @@ def run_test(testsuite, testarea, logger, kwargs):
         stop_on_fail = check_tc_result(testcase, logger)
         try:
             if (not stop_on_fail or stop_on_fail['criteria'] == "FAIL") \
-                and kwargs['stop']:
+                and dt_cfg.dovetail_config['stop']:
                 return "stop_on_fail"
         except KeyError as e:
             logger.error("There is no key {}.".format(e))
@@ -235,6 +230,57 @@ def check_hosts_file(logger):
                     "domain name resolution.".format(hosts_file))
 
 
+def parse_cli(logger=None, **kwargs):
+    validate_input(kwargs, dt_cfg.dovetail_config['validate_input'], logger)
+    configs = filter_config(kwargs, logger)
+    if configs is not None:
+        dt_cfg.update_config(configs)
+    dt_cfg.dovetail_config['offline'] = True if kwargs['offline'] else False
+    dt_cfg.dovetail_config['noclean'] = True if kwargs['no_clean'] else False
+    dt_cfg.dovetail_config['stop'] = True if kwargs['stop'] else False
+    if kwargs['no_api_validation']:
+        dt_cfg.dovetail_config['no_api_validation'] = True
+        logger.warning('Strict API response validation DISABLED.')
+    else:
+        dt_cfg.dovetail_config['no_api_validation'] = False
+
+
+def check_testcase_list(testcase_list, logger=None):
+    if testcase_list:
+        for tc in testcase_list:
+            if tc not in Testcase.testcase_list:
+                logger.error('Test case {} is not defined.'.format(tc))
+                return None
+        return testcase_list
+    return None
+
+
+# If specify 'testcase' with CLI, ignore 'testsuite' and 'testarea'
+# If not specify 'testcase', check combination of 'testsuite' and 'testarea'
+def get_testcase_list(logger=None, **kwargs):
+    Testcase.load()
+    testcase_list = kwargs['testcase']
+    if testcase_list:
+        return check_testcase_list(testcase_list, logger)
+
+    testsuite_validation = False
+    testsuite = kwargs['testsuite']
+    if testsuite in dt_cfg.dovetail_config['testsuite_supported']:
+        testsuite_validation = True
+    origin_testarea = kwargs['testarea']
+    testarea_validation, testarea = Testcase.check_testarea(origin_testarea)
+
+    if testsuite_validation and testarea_validation:
+        testsuite_yaml = load_testsuite(testsuite)
+        testcase_list = Testcase.get_testcase_list(testsuite_yaml, testarea)
+        return check_testcase_list(testcase_list, logger)
+    elif not testsuite_validation:
+        logger.error('Test suite {} is not defined.'.format(testsuite))
+    else:
+        logger.error('Test area {} is not defined.'.format(origin_testarea))
+    return None
+
+
 def main(*args, **kwargs):
     """Dovetail compliance test entry!"""
     build_tag = "daily-master-%s" % str(uuid.uuid1())
@@ -251,47 +297,21 @@ def main(*args, **kwargs):
     logger.info('Dovetail compliance: {}!'.format(kwargs['testsuite']))
     logger.info('================================================')
     logger.info('Build tag: {}'.format(dt_cfg.dovetail_config['build_tag']))
+    parse_cli(logger, **kwargs)
     env_init(logger)
     copy_userconfig_files(logger)
     copy_patch_files(logger)
     dt_utils.check_docker_version(logger)
     dt_utils.get_openstack_endpoint(logger)
-    validate_input(kwargs, dt_cfg.dovetail_config['validate_input'], logger)
     check_hosts_file(logger)
-    configs = filter_config(kwargs, logger)
-
-    if configs is not None:
-        dt_cfg.update_config(configs)
-
-    if kwargs['offline']:
-        dt_cfg.dovetail_config['offline'] = True
-    else:
-        dt_cfg.dovetail_config['offline'] = False
-
-    if kwargs['no_api_validation']:
-        dt_cfg.dovetail_config['no_api_validation'] = True
-        logger.warning('Strict API response validation DISABLED.')
-    else:
-        dt_cfg.dovetail_config['no_api_validation'] = False
-
     dt_utils.get_hardware_info(logger)
 
-    origin_testarea = kwargs['testarea']
-    testsuite_validation = False
-    if kwargs['testsuite'] in dt_cfg.dovetail_config['testsuite_supported']:
-        testsuite_validation = True
-    testarea_validation, testarea = Testcase.check_testarea(origin_testarea)
-    if testsuite_validation and testarea_validation:
-        testsuite_yaml = load_testsuite(kwargs['testsuite'])
-        load_testcase()
-        duration = run_test(testsuite_yaml, testarea, logger, kwargs)
-        if (duration != "stop_on_fail"):
-            Report.generate(testsuite_yaml, testarea, duration)
-            if (kwargs['report']):
-                Report.save_logs()
-    else:
-        logger.error('Invalid input commands, testsuite {} testarea {}'
-                     .format(kwargs['testsuite'], origin_testarea))
+    testcase_list = get_testcase_list(logger, **kwargs)
+    duration = run_test(testcase_list, logger)
+    if (duration != "stop_on_fail"):
+        Report.generate(testcase_list, duration)
+        if (kwargs['report']):
+            Report.save_logs()
 
 
 dt_cfg.load_config_files(constants.CONF_PATH)
diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py
index f43b4739..7370c81b 100644
--- a/dovetail/test_runner.py
+++ b/dovetail/test_runner.py
@@ -113,7 +113,8 @@ class DockerRunner(object):
                 ret, msg = Container.exec_cmd(container_id, cmd)
         self.testcase.cleaned(True)
 
-        Container.clean(container_id, self.type)
+        if not dt_cfg.dovetail_config['noclean']:
+            Container.clean(container_id, self.type)
 
     def save_logs(self):
         pass
diff --git a/etc/conf/cmd_config.yml b/etc/conf/cmd_config.yml
index 52192acd..ae097185 100644
--- a/etc/conf/cmd_config.yml
+++ b/etc/conf/cmd_config.yml
@@ -53,7 +53,12 @@ cli:
         flags:
           - '--testarea'
         multiple: 'True'
-        help: 'compliance testarea within testsuite'
+        help: 'Compliance testarea within testsuite. Specify option multiple times to include multiple test areas.'
+      testcase:
+        flags:
+          - '--testcase'
+        multiple: 'True'
+        help: 'Compliance testcase. Specify option multiple times to include multiple test cases.'
       debug:
         flags:
           - '--debug'
@@ -82,3 +87,9 @@ cli:
           - '--no-api-validation'
         is_flag: 'True'
         help: 'disable strict API response validation'
+      noclean:
+        flags:
+          - '--no-clean'
+          - '-n'
+        is_flag: 'True'
+        help: 'Keep all Containers created for debuging.'