Provide a list of check results files instead of one

This patchs gives the ability to provide more than one check results
file. Furthermore, it does not alter the behavior of the existing tests
but prepares dovetail to be able to fetch information from many
sources.

Change-Id: I23cd23d2908ad6bdddd94b977813d0e81dc0c05a
Signed-off-by: Stamatis Katsaounis <mokats@intracom-telecom.com>

2 files changed, 80 insertions, 72 deletions

diff --git a/dovetail/report.py b/dovetail/report.py
index 305129c1..44555f46 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -40,25 +40,29 @@ class Report(object):
 
     def check_tc_result(self, testcase):
         result_path = dt_cfg.dovetail_config['result_dir']
-        check_results_file = dt_utils.get_value_from_dict(
-            'report.check_results_file', testcase.testcase)
-        if not check_results_file:
-            self.logger.error("Failed to get 'check_results_file' from config "
-                              "file of test case {}".format(testcase.name()))
-            self.check_result(testcase)
-            return None
-        result_file = os.path.join(result_path, check_results_file)
-        if os.path.isfile(result_file):
-            self.logger.info(
-                'Results have been stored with file {}.'.format(result_file))
-            result = self.get_result(testcase, result_file)
-            self.check_result(testcase, result)
-            return result
-        else:
-            self.logger.error(
-                'Failed to store results with file {}.'.format(result_file))
+        check_results_files = dt_utils.get_value_from_dict(
+            'report.check_results_files', testcase.testcase)
+        if not check_results_files:
+            self.logger.error("Failed to get 'check_results_files' from config"
+                              " file of test case {}".format(testcase.name()))
             self.check_result(testcase)
             return None
+        result_files = []
+        for check_results_file in check_results_files:
+            result_file = os.path.join(result_path, check_results_file)
+            if not os.path.isfile(result_file):
+                self.logger.error(
+                    'Failed to store results with file {}.'.
+                    format(result_file))
+                self.check_result(testcase)
+                return None
+            else:
+                result_files.append(result_file)
+        self.logger.info(
+            'Results have been stored with files: {}.'.format(result_files))
+        result = self.get_result(testcase, result_files)
+        self.check_result(testcase, result)
+        return result
 
     @staticmethod
     def check_result(testcase, db_result=None):
@@ -197,7 +201,7 @@ class Report(object):
                     f_out.add(os.path.join('results', f))
         os.chdir(cwd)
 
-    def get_result(self, testcase, check_results_file):
+    def get_result(self, testcase, check_results_files):
         validate_testcase = testcase.validate_testcase()
         type = testcase.validate_type()
         crawler = CrawlerFactory.create(type)
@@ -205,7 +209,7 @@ class Report(object):
             self.logger.error('Crawler is None: {}'.format(testcase.name()))
             return None
 
-        result = crawler.crawl(testcase, check_results_file)
+        result = crawler.crawl(testcase, check_results_files)
 
         if result is not None:
             self.results[type][validate_testcase] = result
@@ -235,8 +239,8 @@ class FunctestCrawler(Crawler):
         cls.logger = \
             dt_logger.Logger(__name__ + '.FunctestCrawler').getLogger()
 
-    def crawl(self, testcase, file_path):
-        return self.crawl_from_file(testcase, file_path)
+    def crawl(self, testcase, file_paths):
+        return self.crawl_from_file(testcase, file_paths[0])
 
     def crawl_from_file(self, testcase, file_path):
         dovetail_config = dt_cfg.dovetail_config
@@ -266,16 +270,7 @@ class FunctestCrawler(Crawler):
                         duration = dt_utils.get_duration(timestart, timestop,
                                                          self.logger)
                         if complex_testcase:
-                            tests = data['details']['tests_number']
-                            failed_num = data['details']['failures_number']
-                            success_case = data['details']['success']
-                            error_case = data['details']['failures']
-                            skipped_case = data['details']['skipped']
-                            details = {'tests': tests,
-                                       'failures': failed_num,
-                                       'success': success_case,
-                                       'errors': error_case,
-                                       'skipped': skipped_case}
+                            details = self.get_details(data)
                 except KeyError as e:
                     self.logger.exception(
                         "Result data don't have key {}.".format(e))
@@ -290,6 +285,19 @@ class FunctestCrawler(Crawler):
         testcase.set_results(json_results)
         return json_results
 
+    def get_details(self, data):
+        tests = data['details']['tests_number']
+        failed_num = data['details']['failures_number']
+        success_case = data['details']['success']
+        error_case = data['details']['failures']
+        skipped_case = data['details']['skipped']
+        details = {'tests': tests,
+                   'failures': failed_num,
+                   'success': success_case,
+                   'errors': error_case,
+                   'skipped': skipped_case}
+        return details
+
 
 class FunctestK8sCrawler(FunctestCrawler):
 
@@ -318,8 +326,8 @@ class YardstickCrawler(Crawler):
         cls.logger = \
             dt_logger.Logger(__name__ + '.YardstickCrawler').getLogger()
 
-    def crawl(self, testcase, file_path):
-        return self.crawl_from_file(testcase, file_path)
+    def crawl(self, testcase, file_paths):
+        return self.crawl_from_file(testcase, file_paths[0])
 
     def crawl_from_file(self, testcase, file_path):
         if not os.path.exists(file_path):
@@ -359,8 +367,8 @@ class BottlenecksCrawler(Crawler):
         cls.logger = \
             dt_logger.Logger(__name__ + '.BottlenecksCrawler').getLogger()
 
-    def crawl(self, testcase, file_path):
-        return self.crawl_from_file(testcase, file_path)
+    def crawl(self, testcase, file_paths):
+        return self.crawl_from_file(testcase, file_paths[0])
 
     def crawl_from_file(self, testcase, file_path):
         if not os.path.exists(file_path):
@@ -389,8 +397,8 @@ class ShellCrawler(Crawler):
     def __init__(self):
         self.type = 'shell'
 
-    def crawl(self, testcase, file_path):
-        return self.crawl_from_file(testcase, file_path)
+    def crawl(self, testcase, file_paths):
+        return self.crawl_from_file(testcase, file_paths[0])
 
     def crawl_from_file(self, testcase, file_path):
         if not os.path.exists(file_path):
@@ -415,8 +423,8 @@ class OnapVtpCrawler(Crawler):
     def create_log(cls):
         cls.logger = dt_logger.Logger(__name__ + '.OnapVtpCrawler').getLogger()
 
-    def crawl(self, testcase, file_path):
-        return self.crawl_from_file(testcase, file_path)
+    def crawl(self, testcase, file_paths):
+        return self.crawl_from_file(testcase, file_paths[0])
 
     # The pass result looks like
     # {
diff --git a/dovetail/tests/unit/test_report.py b/dovetail/tests/unit/test_report.py
index acd44141..9d3acbfd 100644
--- a/dovetail/tests/unit/test_report.py
+++ b/dovetail/tests/unit/test_report.py
@@ -82,7 +82,7 @@ class ReportTesting(unittest.TestCase):
         inner_testcase_obj = Mock()
         testcase_obj.testcase = inner_testcase_obj
         mock_config.dovetail_config = {'result_dir': 'result_dir'}
-        mock_utils.get_value_from_dict.return_value = 'check_results_file'
+        mock_utils.get_value_from_dict.return_value = ['check_results_file']
         mock_path.join.return_value = 'results_file'
         mock_path.isfile.return_value = True
         mock_get.return_value = 'result'
@@ -90,13 +90,13 @@ class ReportTesting(unittest.TestCase):
         result = report.check_tc_result(testcase_obj)
 
         mock_utils.get_value_from_dict.assert_called_once_with(
-            'report.check_results_file', inner_testcase_obj)
+            'report.check_results_files', inner_testcase_obj)
         mock_path.join.assert_called_once_with(
             'result_dir', 'check_results_file')
         mock_path.isfile.assert_called_once_with('results_file')
         logger_obj.info.assert_called_once_with(
-            'Results have been stored with file results_file.')
-        mock_get.assert_called_once_with(testcase_obj, 'results_file')
+            'Results have been stored with files: [\'results_file\'].')
+        mock_get.assert_called_once_with(testcase_obj, ['results_file'])
         mock_check.assert_called_once_with(testcase_obj, 'result')
         self.assertEquals('result', result)
 
@@ -114,14 +114,14 @@ class ReportTesting(unittest.TestCase):
         inner_testcase_obj = Mock()
         testcase_obj.testcase = inner_testcase_obj
         mock_config.dovetail_config = {'result_dir': 'result_dir'}
-        mock_utils.get_value_from_dict.return_value = 'check_results_file'
+        mock_utils.get_value_from_dict.return_value = ['check_results_file']
         mock_path.join.return_value = 'results_file'
         mock_path.isfile.return_value = False
 
         result = report.check_tc_result(testcase_obj)
 
         mock_utils.get_value_from_dict.assert_called_once_with(
-            'report.check_results_file', inner_testcase_obj)
+            'report.check_results_files', inner_testcase_obj)
         mock_path.join.assert_called_once_with(
             'result_dir', 'check_results_file')
         mock_path.isfile.assert_called_once_with('results_file')
@@ -150,9 +150,9 @@ class ReportTesting(unittest.TestCase):
         result = report.check_tc_result(testcase_obj)
 
         mock_utils.get_value_from_dict.assert_called_once_with(
-            'report.check_results_file', inner_testcase_obj)
+            'report.check_results_files', inner_testcase_obj)
         logger_obj.error.assert_called_once_with(
-            "Failed to get 'check_results_file' from config "
+            "Failed to get 'check_results_files' from config "
             "file of test case name")
         mock_check.assert_called_once_with(testcase_obj)
         self.assertEquals(None, result)
@@ -411,13 +411,13 @@ class ReportTesting(unittest.TestCase):
         mock_crawler.create.return_value = crawler_obj
         crawler_obj.crawl.return_value = 'result'
 
-        result = report.get_result(testcase_obj, 'check_results_file')
+        result = report.get_result(testcase_obj, 'check_results_files')
 
         testcase_obj.validate_testcase.assert_called_once_with()
         testcase_obj.validate_type.assert_called_once_with()
         mock_crawler.create.assert_called_once_with('functest')
         crawler_obj.crawl.assert_called_once_with(
-            testcase_obj, 'check_results_file')
+            testcase_obj, 'check_results_files')
         logger_obj.debug.assert_called_once_with(
             'Test case: validate -> result acquired')
         self.assertEquals({'validate': 'result'},
@@ -437,13 +437,13 @@ class ReportTesting(unittest.TestCase):
         mock_crawler.create.return_value = crawler_obj
         crawler_obj.crawl.return_value = None
 
-        result = report.get_result(testcase_obj, 'check_results_file')
+        result = report.get_result(testcase_obj, 'check_results_files')
 
         testcase_obj.validate_testcase.assert_called_once_with()
         testcase_obj.validate_type.assert_called_once_with()
         mock_crawler.create.assert_called_once_with('functest')
         crawler_obj.crawl.assert_called_once_with(
-            testcase_obj, 'check_results_file')
+            testcase_obj, 'check_results_files')
         testcase_obj.increase_retry.assert_called_once_with()
         logger_obj.debug.assert_called_once_with(
             'Test case: validate -> result acquired retry: retry')
@@ -460,7 +460,7 @@ class ReportTesting(unittest.TestCase):
         testcase_obj.validate_type.return_value = 'functest'
         mock_crawler.create.return_value = None
 
-        result = report.get_result(testcase_obj, 'check_results_file')
+        result = report.get_result(testcase_obj, 'check_results_files')
 
         testcase_obj.validate_testcase.assert_called_once_with()
         testcase_obj.validate_type.assert_called_once_with()
@@ -493,7 +493,7 @@ class ReportTesting(unittest.TestCase):
         testcase_obj.name.return_value = 'name'
 
         crawler = dt_report.FunctestCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
 
         mock_path.exists.assert_called_once_with(file_path)
         testcase_obj.validate_testcase.assert_called_once_with()
@@ -538,7 +538,7 @@ class ReportTesting(unittest.TestCase):
         mock_utils.get_duration.return_value = 'duration'
 
         crawler = dt_report.FunctestCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
         expected = {'criteria': 'criteria', 'timestart': 'start_date',
                     'timestop': 'stop_date', 'duration': 'duration',
                     'details': {
@@ -580,7 +580,7 @@ class ReportTesting(unittest.TestCase):
         mock_utils.get_duration.return_value = 'duration'
 
         crawler = dt_report.FunctestCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
 
         mock_path.exists.assert_called_once_with(file_path)
         mock_open.assert_called_once_with(file_path, 'r')
@@ -616,7 +616,7 @@ class ReportTesting(unittest.TestCase):
 
         crawler = dt_report.FunctestK8sCrawler()
 
-        result = crawler.crawl(testcase, file_path)
+        result = crawler.crawl(testcase, [file_path])
 
         dt_report.FunctestK8sCrawler.crawl_from_file.assert_called_once_with(
             'testcase', 'file_path')
@@ -641,7 +641,7 @@ class ReportTesting(unittest.TestCase):
         file_path = 'file_path'
 
         crawler = dt_report.YardstickCrawler()
-        result = crawler.crawl(None, file_path)
+        result = crawler.crawl(None, [file_path])
 
         mock_path.exists.assert_called_once_with(file_path)
         logger_obj.error.assert_called_once_with(
@@ -678,7 +678,7 @@ class ReportTesting(unittest.TestCase):
         mock_utils.get_value_from_dict.return_value = 'PASS'
 
         crawler = dt_report.YardstickCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
         expected = {'criteria': 'FAIL'}
 
         mock_path.exists.assert_called_once_with(file_path)
@@ -709,7 +709,7 @@ class ReportTesting(unittest.TestCase):
         mock_utils.get_value_from_dict.return_value = 'PASS'
 
         crawler = dt_report.YardstickCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
         expected = {'criteria': 'PASS'}
 
         mock_path.exists.assert_called_once_with(file_path)
@@ -742,7 +742,7 @@ class ReportTesting(unittest.TestCase):
         file_path = 'file_path'
 
         crawler = dt_report.BottlenecksCrawler()
-        result = crawler.crawl(None, file_path)
+        result = crawler.crawl(None, [file_path])
 
         mock_path.exists.assert_called_once_with(file_path)
         logger_obj.error.assert_called_once_with(
@@ -768,7 +768,7 @@ class ReportTesting(unittest.TestCase):
         mock_loads.return_value = data_dict
 
         crawler = dt_report.BottlenecksCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
         expected = {'criteria': 'PASS'}
 
         mock_path.exists.assert_called_once_with(file_path)
@@ -796,7 +796,7 @@ class ReportTesting(unittest.TestCase):
         mock_loads.return_value = data_dict
 
         crawler = dt_report.BottlenecksCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
         expected = {'criteria': 'FAIL'}
 
         mock_path.exists.assert_called_once_with(file_path)
@@ -821,7 +821,7 @@ class ReportTesting(unittest.TestCase):
         mock_loads.return_value = {}
 
         crawler = dt_report.BottlenecksCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
         expected = {'criteria': 'FAIL'}
 
         mock_path.exists.assert_called_once_with(file_path)
@@ -838,7 +838,7 @@ class ReportTesting(unittest.TestCase):
         file_path = 'file_path'
 
         crawler = dt_report.ShellCrawler()
-        result = crawler.crawl(None, file_path)
+        result = crawler.crawl(None, [file_path])
 
         mock_path.exists.assert_called_once_with(file_path)
         self.assertEquals(None, result)
@@ -851,7 +851,7 @@ class ReportTesting(unittest.TestCase):
         mock_open.return_value.__enter__.return_value = Exception()
 
         crawler = dt_report.ShellCrawler()
-        result = crawler.crawl(None, file_path)
+        result = crawler.crawl(None, [file_path])
 
         mock_path.exists.assert_called_once_with(file_path)
         mock_open.assert_called_once_with(file_path, 'r')
@@ -869,7 +869,7 @@ class ReportTesting(unittest.TestCase):
         mock_load.return_value = 'result'
 
         crawler = dt_report.ShellCrawler()
-        result = crawler.crawl(None, file_path)
+        result = crawler.crawl(None, [file_path])
 
         mock_path.exists.assert_called_once_with(file_path)
         mock_open.assert_called_once_with(file_path, 'r')
@@ -906,7 +906,7 @@ class ReportTesting(unittest.TestCase):
         file_path = 'file_path'
 
         crawler = dt_report.OnapVtpCrawler()
-        result = crawler.crawl(None, file_path)
+        result = crawler.crawl(None, [file_path])
 
         mock_path.exists.assert_called_once_with(file_path)
         logger_obj.error.assert_called_once_with(
@@ -1041,7 +1041,7 @@ class ReportTesting(unittest.TestCase):
         mock_loads.return_value = data_dict
 
         crawler = dt_report.OnapVtpCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
         expected = {'criteria': 'PASS'}
 
         mock_path.exists.assert_called_once_with(file_path)
@@ -1071,7 +1071,7 @@ class ReportTesting(unittest.TestCase):
         mock_loads.return_value = data_dict
 
         crawler = dt_report.OnapVtpCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
         expected = {'criteria': 'FAIL'}
 
         mock_path.exists.assert_called_once_with(file_path)
@@ -1101,7 +1101,7 @@ class ReportTesting(unittest.TestCase):
         mock_loads.return_value = data_dict
 
         crawler = dt_report.OnapVtpCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
         expected = {'criteria': 'FAIL'}
 
         mock_path.exists.assert_called_once_with(file_path)
@@ -1133,7 +1133,7 @@ class ReportTesting(unittest.TestCase):
         mock_loads.return_value = data_dict
 
         crawler = dt_report.OnapVtpCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
         expected = {'criteria': 'FAIL'}
 
         mock_path.exists.assert_called_once_with(file_path)
@@ -1158,7 +1158,7 @@ class ReportTesting(unittest.TestCase):
         mock_loads.side_effect = ValueError('No JSON object could be decoded')
 
         crawler = dt_report.OnapVtpCrawler()
-        result = crawler.crawl(testcase_obj, file_path)
+        result = crawler.crawl(testcase_obj, [file_path])
         expected = {'criteria': 'FAIL'}
 
         mock_path.exists.assert_called_once_with(file_path)