Add validation disabled/enabled to results.json

Change-Id: I918f85c5ac89cf2745efa8513946a2b5ab8638b1
Signed-off-by: xudan <xudan16@huawei.com>

6 files changed, 28 insertions, 11 deletions

diff --git a/docs/testing/user/testspecification/vnf/index.rst b/docs/testing/user/testspecification/vnf/index.rst
index 4b43e7d3..a9219982 100644
--- a/docs/testing/user/testspecification/vnf/index.rst
+++ b/docs/testing/user/testspecification/vnf/index.rst
@@ -39,10 +39,6 @@ This test area references the following specifications and guides:
 
   - https://www.etsi.org/deliver/etsi_ts/124300_124399/124301/10.03.00_60/ts_124301v100300p.pdf
 
-- ABoT : Test Orchestration Solution
-
-  - https://www.rebaca.com/abot-test-orchestration-tool/
-
 - Cloudify clearwater: opnfv-cloudify-clearwater [1]
 
   - https://github.com/Orange-OpenSource/opnfv-cloudify-clearwater
diff --git a/dovetail/report.py b/dovetail/report.py
index 864f1165..643f7db2 100644
--- a/dovetail/report.py
+++ b/dovetail/report.py
@@ -104,6 +104,8 @@ class Report(object):
         if vnf_type:
             report_obj['vnf_type'] = vnf_type
             report_obj['vnf_checksum'] = self.get_checksum(vnf_type)
+        else:
+            report_obj['validation'] = os.getenv('validation')
 
         report_obj['testcases_list'] = []
         if not testcase_list:
diff --git a/dovetail/run.py b/dovetail/run.py
index 0ea3cb11..193efbc4 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -262,6 +262,8 @@ def main(*args, **kwargs):
     clean_results_dir()
     os.environ['DEBUG'] = 'true' if kwargs['debug'] else 'false'
     os.environ['OPNFV_CI'] = 'true' if kwargs['opnfv_ci'] else 'false'
+    os.environ['validation'] = 'disabled' \
+        if kwargs['no_api_validation'] else 'enabled'
     create_logs()
     logger = dt_logger.Logger('run').getLogger()
 
diff --git a/dovetail/tests/unit/cmd_config.yml b/dovetail/tests/unit/cmd_config.yml
index cc15d64a..405aabce 100644
--- a/dovetail/tests/unit/cmd_config.yml
+++ b/dovetail/tests/unit/cmd_config.yml
@@ -35,3 +35,7 @@ cli:
         flags:
           - '--opnfv-ci'
         is_flag: 'True'
+      noapivalidation:
+        flags:
+          - '--no-api-validation'
+        is_flag: 'True'
diff --git a/dovetail/tests/unit/test_report.py b/dovetail/tests/unit/test_report.py
index 849b31d8..84bd1f38 100644
--- a/dovetail/tests/unit/test_report.py
+++ b/dovetail/tests/unit/test_report.py
@@ -170,12 +170,13 @@ class ReportTesting(unittest.TestCase):
         mock_factory.create.assert_called_once_with('type')
         checker_obj.check.assert_called_once_with(testcase_obj, None)
 
+    @patch('dovetail.report.os.getenv')
     @patch.object(dt_report.Report, 'get_checksum')
     @patch('dovetail.report.Testcase')
     @patch('dovetail.report.datetime.datetime')
     @patch('dovetail.report.dt_cfg')
     def test_generate_json(self, mock_config, mock_datetime, mock_testcase,
-                           mock_checksum):
+                           mock_checksum, mock_env):
         logger_obj = Mock()
         report = dt_report.Report()
         report.logger = logger_obj
@@ -185,6 +186,7 @@ class ReportTesting(unittest.TestCase):
             'build_tag': 'build_tag',
             'version': '2018.09'
         }
+        mock_env.return_value = 'enabled'
         utc_obj = Mock()
         utc_obj.strftime.return_value = '2018-01-13 13:13:13 UTC'
         mock_datetime.utcnow.return_value = utc_obj
@@ -206,6 +208,7 @@ class ReportTesting(unittest.TestCase):
             'vnf_checksum': 'da39a3ee5e6b4b0d3255bfef95601890afd80709',
             'test_date': '2018-01-13 13:13:13 UTC',
             'duration': duration,
+            'validation': 'enabled',
             'testcases_list': [
                 {
                     'name': 'ta.tb.tc',
@@ -229,11 +232,12 @@ class ReportTesting(unittest.TestCase):
 
         self.assertEquals(expected, result)
 
+    @patch('dovetail.report.os.getenv')
     @patch('dovetail.report.Testcase')
     @patch('dovetail.report.datetime.datetime')
     @patch('dovetail.report.dt_cfg')
     def test_generate_json_noVNF(self, mock_config, mock_datetime,
-                                 mock_testcase):
+                                 mock_testcase, mock_env):
         logger_obj = Mock()
         report = dt_report.Report()
         report.logger = logger_obj
@@ -243,6 +247,7 @@ class ReportTesting(unittest.TestCase):
             'build_tag': 'build_tag',
             'version': '2018.09'
         }
+        mock_env.return_value = 'disabled'
         utc_obj = Mock()
         utc_obj.strftime.return_value = '2018-01-13 13:13:13 UTC'
         mock_datetime.utcnow.return_value = utc_obj
@@ -261,6 +266,7 @@ class ReportTesting(unittest.TestCase):
             'build_tag': 'build_tag',
             'test_date': '2018-01-13 13:13:13 UTC',
             'duration': duration,
+            'validation': 'disabled',
             'testcases_list': [
                 {
                     'name': 'ta.tb.tc',
@@ -284,11 +290,12 @@ class ReportTesting(unittest.TestCase):
 
         self.assertEquals(expected, result)
 
+    @patch('dovetail.report.os.getenv')
     @patch('dovetail.report.Testcase')
     @patch('dovetail.report.datetime.datetime')
     @patch('dovetail.report.dt_cfg')
     def test_generate_json_noVNF_inTestCase(self, mock_config, mock_datetime,
-                                            mock_testcase):
+                                            mock_testcase, mock_env):
         logger_obj = Mock()
         report = dt_report.Report()
         report.logger = logger_obj
@@ -298,6 +305,7 @@ class ReportTesting(unittest.TestCase):
             'build_tag': 'build_tag',
             'version': '2018.09'
         }
+        mock_env.return_value = 'enabled'
         utc_obj = Mock()
         utc_obj.strftime.return_value = '2018-01-13 13:13:13 UTC'
         mock_datetime.utcnow.return_value = utc_obj
@@ -316,6 +324,7 @@ class ReportTesting(unittest.TestCase):
             'build_tag': 'build_tag',
             'test_date': '2018-01-13 13:13:13 UTC',
             'duration': duration,
+            'validation': 'enabled',
             'testcases_list': [
                 {
                     'name': 'ta.tb.tc',
diff --git a/dovetail/tests/unit/test_run.py b/dovetail/tests/unit/test_run.py
index 497cd06c..0604c8ed 100644
--- a/dovetail/tests/unit/test_run.py
+++ b/dovetail/tests/unit/test_run.py
@@ -528,7 +528,8 @@ class RunTesting(unittest.TestCase):
             'opnfv_ci': True,
             'report': True,
             'testsuite': 'testsuite',
-            'docker_tag': '2.0.0'
+            'docker_tag': '2.0.0',
+            'no_api_validation': False
         }
 
         with self.assertRaises(SystemExit) as cm:
@@ -544,7 +545,8 @@ class RunTesting(unittest.TestCase):
         mock_get_result.assert_called_once_with()
         mock_clean.assert_called_once_with()
         self.assertEquals({'DOVETAIL_HOME': 'dovetail_home', 'DEBUG': 'true',
-                           'OPNFV_CI': 'true'}, mock_os.environ)
+                           'OPNFV_CI': 'true', 'validation': 'enabled'},
+                          mock_os.environ)
         mock_create_logs.assert_called_once_with()
         logger_obj.info.assert_has_calls([
             call('================================================'),
@@ -619,7 +621,8 @@ class RunTesting(unittest.TestCase):
             'opnfv_ci': False,
             'report': True,
             'testsuite': 'testsuite',
-            'docker_tag': '2.0.0'
+            'docker_tag': '2.0.0',
+            'no_api_validation': False
         }
 
         with self.assertRaises(SystemExit) as cm:
@@ -636,7 +639,8 @@ class RunTesting(unittest.TestCase):
         mock_get_result.assert_called_once_with()
         mock_clean.assert_called_once_with()
         self.assertEquals({'DOVETAIL_HOME': 'dovetail_home', 'DEBUG': 'true',
-                           'OPNFV_CI': 'false'}, mock_os.environ)
+                           'OPNFV_CI': 'false', 'validation': 'enabled'},
+                          mock_os.environ)
         mock_create_logs.assert_called_once_with()
         logger_obj.info.assert_has_calls([
             call('================================================'),