Conform ODL with last framework updates

Now ODL TestCase calculates the ratio between critical tests passed
and failed and saves it in result. Non-critical test cases can fail as
result doesn't take them into account [1].

It also updates default attribute values in TestCase and allows result
to be float.

[1] http://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#setting-criticality

Change-Id: Id4a89271b5b1a90cd3c1e2b08591ff26ffaffee0
Signed-off-by: Cédric Ollivier <cedric.ollivier@orange.com>

3 files changed, 38 insertions, 9 deletions

diff --git a/functest/core/testcase.py b/functest/core/testcase.py
index 624655424..b993183cc 100644
--- a/functest/core/testcase.py
+++ b/functest/core/testcase.py
@@ -39,9 +39,9 @@ class TestCase(object):
         self.project_name = kwargs.get('project_name', 'functest')
         self.case_name = kwargs.get('case_name', '')
         self.criteria = kwargs.get('criteria', 100)
-        self.result = ""
-        self.start_time = ""
-        self.stop_time = ""
+        self.result = 0
+        self.start_time = 0
+        self.stop_time = 0
 
     def get_duration(self):
         """Return the duration of the test case.
@@ -75,7 +75,8 @@ class TestCase(object):
         """
         try:
             assert self.criteria
-            if isinstance(self.result, int) and isinstance(self.criteria, int):
+            if (not isinstance(self.result, str) and
+                    not isinstance(self.criteria, str)):
                 if self.result >= self.criteria:
                     return TestCase.EX_OK
             else:
diff --git a/functest/opnfv_tests/sdn/odl/odl.py b/functest/opnfv_tests/sdn/odl/odl.py
index f92cb95da..e50d9c130 100755
--- a/functest/opnfv_tests/sdn/odl/odl.py
+++ b/functest/opnfv_tests/sdn/odl/odl.py
@@ -16,6 +16,8 @@ Example:
         $ python odl.py
 """
 
+from __future__ import division
+
 import argparse
 import errno
 import fileinput
@@ -100,7 +102,12 @@ class ODLTests(testcase.TestCase):
         result = robot.api.ExecutionResult(xml_file)
         visitor = ODLResultVisitor()
         result.visit(visitor)
-        self.result = result.suite.status
+        try:
+            self.result = 100 * (
+                result.suite.statistics.critical.passed /
+                result.suite.statistics.critical.total)
+        except ZeroDivisionError:
+            self.__logger.error("No test has been ran")
         self.start_time = timestamp_to_secs(result.suite.starttime)
         self.stop_time = timestamp_to_secs(result.suite.endtime)
         self.details = {}
diff --git a/functest/tests/unit/odl/test_odl.py b/functest/tests/unit/odl/test_odl.py
index f3d37c650..d7ce70c79 100644
--- a/functest/tests/unit/odl/test_odl.py
+++ b/functest/tests/unit/odl/test_odl.py
@@ -109,6 +109,9 @@ class ODLParseResultTesting(ODLTesting):
     """The class testing ODLTests.parse_results()."""
     # pylint: disable=missing-docstring
 
+    _config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000',
+               'endtime': '20161216 16:00:01.000'}
+
     @mock.patch('robot.api.ExecutionResult', side_effect=DataError)
     def test_raises_exc(self, mock_method):
         with self.assertRaises(DataError):
@@ -116,15 +119,13 @@ class ODLParseResultTesting(ODLTesting):
         mock_method.assert_called_once_with(
             os.path.join(odl.ODLTests.res_dir, 'output.xml'))
 
-    def test_ok(self):
-        config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000',
-                  'endtime': '20161216 16:00:01.000', 'status': 'PASS'}
+    def _test_result(self, config, result):
         suite = mock.Mock()
         suite.configure_mock(**config)
         with mock.patch('robot.api.ExecutionResult',
                         return_value=mock.Mock(suite=suite)):
             self.test.parse_results()
-            self.assertEqual(self.test.result, config['status'])
+            self.assertEqual(self.test.result, result)
             self.assertEqual(self.test.start_time,
                              timestamp_to_secs(config['starttime']))
             self.assertEqual(self.test.stop_time,
@@ -132,6 +133,26 @@ class ODLParseResultTesting(ODLTesting):
             self.assertEqual(self.test.details,
                              {'description': config['name'], 'tests': []})
 
+    def test_null_passed(self):
+        self._config.update({'statistics.critical.passed': 0,
+                             'statistics.critical.total': 20})
+        self._test_result(self._config, 0)
+
+    def test_no_test(self):
+        self._config.update({'statistics.critical.passed': 20,
+                             'statistics.critical.total': 0})
+        self._test_result(self._config, 0)
+
+    def test_half_success(self):
+        self._config.update({'statistics.critical.passed': 10,
+                             'statistics.critical.total': 20})
+        self._test_result(self._config, 50)
+
+    def test_success(self):
+        self._config.update({'statistics.critical.passed': 20,
+                             'statistics.critical.total': 20})
+        self._test_result(self._config, 100)
+
 
 class ODLRobotTesting(ODLTesting):