Making the results file a proper json file

The results.json file is currently not a properly formated json file,
but contains one json structure per line/testcase. Whit this change, all
test results are being collected at runtime and then written to a
properly formated json file at the end of a test run.

Moreover, this patch introduces 'mandatory' and 'optional' sections to
the test suites definition files in order to clearly indicate if a test
is part of the corresponding test scope.

The web portal needs to be adapted accordingly to be able to read the
new results file.

JIRA: DOVETAIL-672

Change-Id: Id0706c7e82622fc9c2c2fece26b2d6c07c1580be
Signed-off-by: Georg Kunz <georg.kunz@ericsson.com>

5 files changed, 51 insertions, 37 deletions

diff --git a/dovetail/report.py b/dovetail/report.py
index 8d157559..f1f47c25 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -18,8 +18,6 @@ import datetime
 import tarfile
 import time
 
-from pbr import version
-
 import utils.dovetail_logger as dt_logger
 
 from utils.dovetail_config import DovetailConfig as dt_cfg
@@ -70,17 +68,18 @@ class Report(object):
     @classmethod
     def generate_json(cls, testcase_list, duration):
         report_obj = {}
-        report_obj['version'] = \
-            version.VersionInfo('dovetail').version_string()
+        # egeokun: using a hardcoded string instead of pbr version for
+        # versioning the result file. The version of the results.json is
+        # logically independent of the release of Dovetail.
+        report_obj['version'] = '2018.08'
         report_obj['build_tag'] = dt_cfg.dovetail_config['build_tag']
-        report_obj['upload_date'] =\
+        report_obj['test_date'] =\
             datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
         report_obj['duration'] = duration
 
         report_obj['testcases_list'] = []
         if not testcase_list:
             return report_obj
-
         for testcase_name in testcase_list:
             testcase = Testcase.get(testcase_name)
             testcase_inreport = {}
@@ -89,11 +88,13 @@ class Report(object):
                 testcase_inreport['result'] = 'Undefined'
                 testcase_inreport['objective'] = ''
                 testcase_inreport['sub_testcase'] = []
+                testcase_inreport['mandatory'] = False
                 report_obj['testcases_list'].append(testcase_inreport)
                 continue
 
             testcase_inreport['result'] = testcase.passed()
             testcase_inreport['objective'] = testcase.objective()
+            testcase_inreport['mandatory'] = testcase.is_mandatory
             testcase_inreport['sub_testcase'] = []
             if testcase.sub_testcase() is not None:
                 for sub_test in testcase.sub_testcase():
@@ -108,11 +109,13 @@ class Report(object):
     @classmethod
     def generate(cls, testcase_list, duration):
         report_data = cls.generate_json(testcase_list, duration)
+        cls.save_json_results(report_data)
+
         report_txt = ''
         report_txt += '\n\nDovetail Report\n'
         report_txt += 'Version: %s\n' % report_data['version']
         report_txt += 'Build Tag: %s\n' % report_data['build_tag']
-        report_txt += 'Upload Date: %s\n' % report_data['upload_date']
+        report_txt += 'Test Date: %s\n' % report_data['test_date']
         report_txt += 'Duration: %.2f s\n\n' % report_data['duration']
 
         total_num = 0
@@ -173,6 +176,18 @@ class Report(object):
         return report_txt
 
     @classmethod
+    def save_json_results(cls, results):
+        result_file = os.path.join(dt_cfg.dovetail_config['result_dir'],
+                                   dt_cfg.dovetail_config['result_file'])
+
+        try:
+            with open(result_file, 'w') as f:
+                f.write(json.dumps(results) + '\n')
+        except Exception as e:
+            cls.logger.exception("Failed to add result to file {}, "
+                                 "exception: {}".format(result_file, e))
+
+    @classmethod
     def save_logs(cls):
         file_suffix = time.strftime('%Y%m%d_%H%M', time.localtime())
         logs_gz = "logs_{}.tar.gz".format(file_suffix)
@@ -197,14 +212,10 @@ class Report(object):
             cls.logger.error('Crawler is None: {}'.format(testcase.name()))
             return None
 
-        # if validate_testcase in cls.results[type]:
-        #    return cls.results[type][validate_testcase]
-
         result = crawler.crawl(testcase, check_results_file)
 
         if result is not None:
             cls.results[type][validate_testcase] = result
-            # testcase.script_result_acquired(True)
             cls.logger.debug(
                 'Test case: {} -> result acquired'.format(validate_testcase))
         else:
@@ -215,18 +226,7 @@ class Report(object):
 
 
 class Crawler(object):
-
-    def add_result_to_file(self, result):
-        result_file = os.path.join(dt_cfg.dovetail_config['result_dir'],
-                                   dt_cfg.dovetail_config['result_file'])
-        try:
-            with open(result_file, 'a') as f:
-                f.write(json.dumps(result) + '\n')
-                return True
-        except Exception as e:
-            self.logger.exception("Failed to add result to file {}, "
-                                  "exception: {}".format(result_file, e))
-            return False
+    pass
 
 
 class FunctestCrawler(Crawler):
@@ -268,7 +268,6 @@ class FunctestCrawler(Crawler):
                     if (testcase_name == data['case_name'] or
                         data['project_name'] == "sdnvpn") and \
                         build_tag == data['build_tag']:
-                        self.add_result_to_file(data)
                         criteria = data['criteria']
                         timestart = data['start_date']
                         timestop = data['stop_date']
@@ -296,6 +295,7 @@ class FunctestCrawler(Crawler):
                         'timestop': timestop, 'duration': duration,
                         'details': details}
 
+        testcase.set_results(json_results)
         return json_results
 
 
@@ -323,7 +323,6 @@ class YardstickCrawler(Crawler):
         with open(file_path, 'r') as f:
             for jsonfile in f:
                 data = json.loads(jsonfile)
-                self.add_result_to_file(data, testcase.name())
                 try:
                     criteria = data['result']['criteria']
                     if criteria == 'PASS':
@@ -335,13 +334,9 @@ class YardstickCrawler(Crawler):
                 except KeyError as e:
                     self.logger.exception('Pass flag not found {}'.format(e))
         json_results = {'criteria': criteria}
-        return json_results
 
-    def add_result_to_file(self, result, tc_name):
-        build_tag = '{}-{}'.format(dt_cfg.dovetail_config['build_tag'],
-                                   tc_name)
-        result['build_tag'] = build_tag
-        super(YardstickCrawler, self).add_result_to_file(result)
+        testcase.set_results(json_results)
+        return json_results
 
 
 class BottlenecksCrawler(Crawler):
@@ -377,6 +372,8 @@ class BottlenecksCrawler(Crawler):
                 except KeyError as e:
                     self.logger.exception('Pass flag not found {}'.format(e))
         json_results = {'criteria': criteria}
+
+        testcase.set_results(json_results)
         return json_results
 
 
diff --git a/dovetail/run.py b/dovetail/run.py
index 84a448f6..8d4b7f8d 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -221,11 +221,15 @@ def check_testcase_list(testcase_list, logger=None):
     return None
 
 
-# If specify 'testcase' with CLI, ignore 'testsuite' and 'testarea'
-# If not specify 'testcase', check combination of 'testsuite' and 'testarea'
 def get_testcase_list(logger=None, **kwargs):
     Testcase.load()
     testcase_list = kwargs['testcase']
+
+    # If specify 'testcase' on the CLI, ignore 'testsuite' and 'testarea'. In
+    # this case, all test cases are marked as mandatory=false in the result
+    # file because there is no testsuite to relate to.
+    # If 'testcase' is not specified on the CLI, check the combination of
+    # 'testsuite' and 'testarea'
     if testcase_list:
         return check_testcase_list(testcase_list, logger)
 
@@ -238,7 +242,8 @@ def get_testcase_list(logger=None, **kwargs):
 
     if testsuite_validation and testarea_validation:
         testsuite_yaml = load_testsuite(testsuite)
-        testcase_list = Testcase.get_testcase_list(testsuite_yaml, testarea)
+        testcase_list = Testcase.get_testcases_for_testsuite(testsuite_yaml,
+                                                             testarea)
         return check_testcase_list(testcase_list, logger)
     elif not testsuite_validation:
         logger.error('Test suite {} is not defined.'.format(testsuite))
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index ff716292..8eec9388 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -30,6 +30,8 @@ class Testcase(object):
         self.cmds = []
         self.sub_testcase_status = {}
         self.update_validate_testcase(self.validate_testcase())
+        self.is_mandatory = False
+        self.results = None
 
     @classmethod
     def create_log(cls):
@@ -101,6 +103,12 @@ class Testcase(object):
             self.testcase['passed'] = passed
         return self.testcase['passed']
 
+    def set_results(self, results):
+        self.results = results
+
+    def get_results(self):
+        return self.results
+
     def script_result_acquired(self, acquired=None):
         return self._result_acquired(self.validate_testcase(), acquired)
 
@@ -270,7 +278,7 @@ class Testcase(object):
             return False
 
     @classmethod
-    def get_testcase_list(cls, testsuite, testarea):
+    def get_testcases_for_testsuite(cls, testsuite, testarea):
         testcase_list = []
         selected_tests = []
         testcases = dt_utils.get_value_from_dict('testcases_list', testsuite)
@@ -303,6 +311,10 @@ class Testcase(object):
             for area in testarea:
                 if cls.check_testcase_area(value, area):
                     testcase_list.append(value)
+                    if value in mandatory:
+                        Testcase.testcase_list[value].is_mandatory = True
+                    else:
+                        Testcase.testcase_list[value].is_mandatory = False
                     break
         return testcase_list
 
diff --git a/etc/compliance/proposed_tests.yml b/etc/compliance/proposed_tests.yml
index e893fa19..2ded05d8 100644
--- a/etc/compliance/proposed_tests.yml
+++ b/etc/compliance/proposed_tests.yml
@@ -2,7 +2,7 @@
 proposed_tests:
   name: proposed_tests
   testcases_list:
-  # proposed test cases for 2nd release
+    # proposed test cases for 2nd release
     mandatory:
       # tempest
       - dovetail.tempest.compute
diff --git a/setup.cfg b/setup.cfg
index c8fbc812..12eb926f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,6 +1,6 @@
 [metadata]
 name = dovetail
-version = 1.0.0
+version = 2018.08.0
 home-page = https://wiki.opnfv.org/display/dovetail
 
 [files]