Archive all test case result files

1. All detailed tempest test cases results are recorded with
   file tempest-report.html rather than tempest.log now.
2. Archive all results files include functest.log, yardstick.log ...
   for each test case. Otherwise, they will all be in one log file.
3. Support to set the source files and dest files need to be archived with
   the test case config files rather than hardcode with the source code.
4. Give the file which used to be parse the results (PASS/FAIL) with the
   test case config files rather than hardcode with the source code.

Change-Id: I30f435d91ab90cf20def47007f177fe98187484d
Signed-off-by: xudan <xudan16@huawei.com>

4 files changed, 66 insertions, 83 deletions

diff --git a/dovetail/report.py b/dovetail/report.py
index 20475a3b..9808c7a5 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -35,8 +35,31 @@ class Report(object):
     def create_log(cls):
         cls.logger = dt_logger.Logger(__name__ + '.Report').getLogger()
 
+    @classmethod
+    def check_tc_result(cls, testcase):
+        result_path = dt_cfg.dovetail_config['result_dir']
+        check_results_file = dt_utils.get_value_from_dict(
+            'report.check_results_file', testcase.testcase)
+        if not check_results_file:
+            cls.logger.error("Failed to get 'check_results_file' from config "
+                             "file of test case {}".format(testcase.name()))
+            cls.check_result(testcase)
+            return None
+        result_file = os.path.join(result_path, check_results_file)
+        if os.path.isfile(result_file):
+            cls.logger.info(
+                "Results have been stored with file {}.".format(result_file))
+            result = cls.get_result(testcase, result_file)
+            cls.check_result(testcase, result)
+            return result
+        else:
+            cls.logger.error(
+                "Failed to store results with file {}.".format(result_file))
+            cls.check_result(testcase)
+            return None
+
     @staticmethod
-    def check_result(testcase, db_result):
+    def check_result(testcase, db_result=None):
         checker = CheckerFactory.create(testcase.validate_type())
         if checker is not None:
             checker.check(testcase, db_result)
@@ -163,7 +186,7 @@ class Report(object):
         os.chdir(cwd)
 
     @classmethod
-    def get_result(cls, testcase):
+    def get_result(cls, testcase, check_results_file):
         validate_testcase = testcase.validate_testcase()
         type = testcase.validate_type()
         crawler = CrawlerFactory.create(type)
@@ -174,7 +197,7 @@ class Report(object):
         # if validate_testcase in cls.results[type]:
         #    return cls.results[type][validate_testcase]
 
-        result = crawler.crawl(testcase)
+        result = crawler.crawl(testcase, check_results_file)
 
         if result is not None:
             cls.results[type][validate_testcase] = result
@@ -216,10 +239,10 @@ class FunctestCrawler(Crawler):
         cls.logger = \
             dt_logger.Logger(__name__ + '.FunctestCrawler').getLogger()
 
-    def crawl(self, testcase=None):
-        return self.crawl_from_file(testcase)
+    def crawl(self, testcase, file_path):
+        return self.crawl_from_file(testcase, file_path)
 
-    def crawl_from_file(self, testcase=None):
+    def crawl_from_file(self, testcase, file_path):
         dovetail_config = dt_cfg.dovetail_config
         criteria = 'FAIL'
         details = {}
@@ -228,9 +251,6 @@ class FunctestCrawler(Crawler):
         duration = 0
         testcase_name = testcase.validate_testcase()
         build_tag = '%s-%s' % (dovetail_config['build_tag'], testcase.name())
-        file_path = \
-            os.path.join(dovetail_config['result_dir'],
-                         dovetail_config[self.type]['result']['file_path'])
         if not os.path.exists(file_path):
             self.logger.error('Result file not found: {}'.format(file_path))
             return None
@@ -293,12 +313,10 @@ class YardstickCrawler(Crawler):
         cls.logger = \
             dt_logger.Logger(__name__ + '.YardstickCrawler').getLogger()
 
-    def crawl(self, testcase=None):
-        return self.crawl_from_file(testcase)
+    def crawl(self, testcase, file_path):
+        return self.crawl_from_file(testcase, file_path)
 
-    def crawl_from_file(self, testcase=None):
-        file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
-                                 testcase.name() + '.out')
+    def crawl_from_file(self, testcase, file_path):
         if not os.path.exists(file_path):
             self.logger.error('Result file not found: {}'.format(file_path))
             return None
@@ -340,12 +358,10 @@ class BottlenecksCrawler(Crawler):
         cls.logger = \
             dt_logger.Logger(__name__ + '.BottlenecksCrawler').getLogger()
 
-    def crawl(self, testcase=None):
-        return self.crawl_from_file(testcase)
+    def crawl(self, testcase, file_path):
+        return self.crawl_from_file(testcase, file_path)
 
-    def crawl_from_file(self, testcase=None):
-        file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
-                                 testcase.name() + '.out')
+    def crawl_from_file(self, testcase, file_path):
         if not os.path.exists(file_path):
             self.logger.error('Result file not found: {}'.format(file_path))
             return None
@@ -370,12 +386,10 @@ class ShellCrawler(Crawler):
     def __init__(self):
         self.type = 'shell'
 
-    def crawl(self, testcase=None):
-        return self.crawl_from_file(testcase)
+    def crawl(self, testcase, file_path):
+        return self.crawl_from_file(testcase, file_path)
 
-    def crawl_from_file(self, testcase=None):
-        file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
-                                 testcase.name()) + '.out'
+    def crawl_from_file(self, testcase, file_path):
         if not os.path.exists(file_path):
             return None
         try:
@@ -399,13 +413,10 @@ class VnftestCrawler(Crawler):
         cls.logger = \
             dt_logger.Logger(__name__ + '.VnftestCrawler').getLogger()
 
-    def crawl(self, testcase):
-        return self.crawl_from_file(testcase)
-
-    def crawl_from_file(self, testcase):
+    def crawl(self, testcase, file_path):
+        return self.crawl_from_file(testcase, file_path)
 
-        file_path = os.path.join(dt_cfg.dovetail_config['result_dir'],
-                                 testcase.name() + '.out')
+    def crawl_from_file(self, testcase, file_path):
         if not os.path.exists(file_path):
             self.logger.error('Result file not found: {}'.format(file_path))
             return None
diff --git a/dovetail/run.py b/dovetail/run.py
index 512565c3..19d0acb6 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -59,7 +59,7 @@ def run_test(testcase_list, logger):
         if run_testcase:
             testcase.run()
 
-        stop_on_fail = check_tc_result(testcase, logger)
+        stop_on_fail = Report.check_tc_result(testcase)
         try:
             if (not stop_on_fail or stop_on_fail['criteria'] == "FAIL") \
                 and dt_cfg.dovetail_config['stop']:
@@ -72,28 +72,6 @@ def run_test(testcase_list, logger):
     return duration
 
 
-def check_tc_result(testcase, logger):
-    result_dir = dt_cfg.dovetail_config['result_dir']
-    validate_type = testcase.validate_type()
-    functest_result = dt_cfg.dovetail_config['functest']['result']['file_path']
-    if validate_type.lower() in ['yardstick', 'bottlenecks', 'vnftest']:
-        result_file = os.path.join(result_dir, testcase.name() + '.out')
-    elif validate_type.lower() == 'functest':
-        result_file = os.path.join(result_dir, functest_result)
-    else:
-        logger.error("Don't support {} now.".format(validate_type))
-        return
-    if os.path.isfile(result_file):
-        logger.info(
-            "Results have been stored with file {}.".format(result_file))
-    else:
-        logger.error(
-            "Failed to store results with file {}.".format(result_file))
-    result = Report.get_result(testcase)
-    Report.check_result(testcase, result)
-    return result
-
-
 def validate_input(input_dict, check_dict, logger):
     func_tag = input_dict['functest_tag']
     yard_tag = input_dict['yardstick_tag']
diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py
index aba457e5..44eddabd 100644
--- a/dovetail/test_runner.py
+++ b/dovetail/test_runner.py
@@ -116,8 +116,29 @@ class DockerRunner(object):
         if not dt_cfg.dovetail_config['noclean']:
             Container.clean(container_id, self.type)
 
-    def save_logs(self):
-        pass
+    def archive_logs(self):
+        result_path = os.path.join(os.environ["DOVETAIL_HOME"], 'results')
+        src_files = dt_utils.get_value_from_dict(
+            'report.source_archive_files', self.testcase.testcase)
+        dest_files = dt_utils.get_value_from_dict(
+            'report.dest_archive_files', self.testcase.testcase)
+        if not src_files and not dest_files:
+            return True
+        if not (src_files and dest_files) or len(src_files) != len(dest_files):
+            self.logger.error("Can't find corresponding 'result_dest_files' "
+                              "for 'result_source_files' with testcase {}"
+                              .format(self.testcase.name()))
+            return False
+        res = True
+        for index in range(0, len(src_files)):
+            src_file_path = os.path.join(result_path, src_files[index])
+            dest_file_path = os.path.join(result_path, dest_files[index])
+            if os.path.isfile(src_file_path):
+                os.renames(src_file_path, dest_file_path)
+            else:
+                self.logger.error("Can't find file {}.".format(src_file_path))
+                res = False
+        return res
 
     @staticmethod
     def _render(task_template, **kwargs):
@@ -170,33 +191,6 @@ class FunctestRunner(DockerRunner):
         self.type = 'functest'
         super(FunctestRunner, self).__init__(testcase)
 
-    def save_logs(self):
-        validate_testcase = self.testcase.validate_testcase()
-        test_area = self.testcase.name().split(".")[1]
-        result_path = os.path.join(os.environ["DOVETAIL_HOME"], 'results')
-        dest_path = os.path.join(result_path, test_area + '_logs')
-        dest_file = os.path.join(dest_path, self.testcase.name() + '.log')
-        if validate_testcase == 'tempest_custom':
-            source_file = os.path.join(result_path, 'tempest', 'tempest.log')
-        elif validate_testcase == 'refstack_defcore':
-            source_file = os.path.join(result_path, 'refstack', 'tempest.log')
-        elif validate_testcase == 'bgpvpn':
-            source_file = os.path.join(result_path, 'bgpvpn.log')
-        elif validate_testcase == 'patrole':
-            source_file = os.path.join(result_path, 'patrole', 'tempest.log')
-        elif validate_testcase == 'neutron_trunk':
-            source_file = os.path.join(result_path,
-                                       'neutron_trunk',
-                                       'tempest.log')
-        else:
-            source_file = None
-        if source_file:
-            if os.path.isfile(source_file):
-                os.renames(source_file, dest_file)
-            else:
-                self.logger.error("Tempest log file for test case {} is not "
-                                  "found.".format(self.testcase.name()))
-
 
 class YardstickRunner(DockerRunner):
 
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index 9680f192..272cf43c 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -159,7 +159,7 @@ class Testcase(object):
         runner = TestRunnerFactory.create(self)
         try:
             runner.run()
-            runner.save_logs()
+            runner.archive_logs()
         except AttributeError as e:
             self.logger.exception(
                 'Test case: {} Exception: {}'.format(self.name, e))