diff options
| -rw-r--r-- | functest/ci/generate_report.py | 6 | ||||
| -rwxr-xr-x | functest/ci/run_tests.py | 3 | ||||
| -rw-r--r-- | functest/core/pytest_suite_runner.py | 1 | ||||
| -rw-r--r-- | functest/core/testcase.py | 26 | ||||
| -rw-r--r-- | functest/core/vnf.py (renamed from functest/core/vnf_base.py) | 4 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/snaps/snaps_test_runner.py | 3 | ||||
| -rwxr-xr-x | functest/opnfv_tests/sdn/odl/odl.py | 9 | ||||
| -rwxr-xr-x | functest/opnfv_tests/vnf/aaa/aaa.py | 4 | ||||
| -rw-r--r-- | functest/opnfv_tests/vnf/ims/clearwater_ims_base.py | 4 | ||||
| -rwxr-xr-x | functest/opnfv_tests/vnf/ims/orchestra_ims.py | 6 | ||||
| -rw-r--r-- | functest/tests/unit/ci/test_generate_report.py | 8 | ||||
| -rw-r--r-- | functest/tests/unit/core/test_testcase.py | 33 | ||||
| -rw-r--r-- | functest/tests/unit/core/test_vnf.py (renamed from functest/tests/unit/core/test_vnf_base.py) | 6 | ||||
| -rw-r--r-- | functest/tests/unit/odl/test_odl.py | 29 |
14 files changed, 111 insertions, 31 deletions
diff --git a/functest/ci/generate_report.py b/functest/ci/generate_report.py index 17240e8c0..e400b1b64 100644 --- a/functest/ci/generate_report.py +++ b/functest/ci/generate_report.py @@ -39,8 +39,8 @@ def init(tiers_to_run=[]): def get_results_from_db(): - url = "%s/results?build_tag=%s" % (ft_utils.get_db_url(), - CONST.BUILD_TAG) + url = "%s?build_tag=%s" % (ft_utils.get_db_url(), + CONST.BUILD_TAG) logger.debug("Query to rest api: %s" % url) try: data = json.load(urllib2.urlopen(url)) @@ -56,7 +56,7 @@ def get_data(test, results): for test_db in results: if test['test_name'] in test_db['case_name']: id = test_db['_id'] - url = ft_utils.get_db_url() + '/results/' + id + url = ft_utils.get_db_url() + '/' + id test_result = test_db['criteria'] return {"url": url, "result": test_result} diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py index ae002c2e3..1396644f7 100755 --- a/functest/ci/run_tests.py +++ b/functest/ci/run_tests.py @@ -155,6 +155,7 @@ def run_test(test, tier_name, testcases=None): test_case.push_to_db() result = test_case.is_successful() duration = test_case.get_duration() + logger.info("\n%s\n", test_case) except ImportError: logger.exception("Cannot import module {}".format( run_dict['module'])) @@ -167,8 +168,6 @@ def run_test(test, tier_name, testcases=None): if test.needs_clean() and GlobalVariables.CLEAN_FLAG: cleanup() - logger.info("Test execution time: %s", duration) - if result != testcase.TestCase.EX_OK: logger.error("The test case '%s' failed. " % test_name) GlobalVariables.OVERALL_RESULT = Result.EX_ERROR diff --git a/functest/core/pytest_suite_runner.py b/functest/core/pytest_suite_runner.py index 8b5da05ef..21edc1874 100644 --- a/functest/core/pytest_suite_runner.py +++ b/functest/core/pytest_suite_runner.py @@ -18,6 +18,7 @@ class PyTestSuiteRunner(base.TestCase): def __init__(self, **kwargs): super(PyTestSuiteRunner, self).__init__(**kwargs) self.suite = None + self.logger = None def run(self, **kwargs): """ diff --git a/functest/core/testcase.py b/functest/core/testcase.py index 624655424..49fae6097 100644 --- a/functest/core/testcase.py +++ b/functest/core/testcase.py @@ -39,9 +39,24 @@ class TestCase(object): self.project_name = kwargs.get('project_name', 'functest') self.case_name = kwargs.get('case_name', '') self.criteria = kwargs.get('criteria', 100) - self.result = "" - self.start_time = "" - self.stop_time = "" + self.result = 0 + self.start_time = 0 + self.stop_time = 0 + + def __str__(self): + try: + assert self.project_name + assert self.case_name + result = 'PASS' if(self.is_successful( + ) == TestCase.EX_OK) else 'FAIL' + return ('| {0:<23} | {1:<13} | {2:<10} | {3:<13} |' + '\n{4:-<26}{4:-<16}{4:-<13}{4:-<16}{4}'.format( + self.case_name, self.project_name, + self.get_duration(), result, '+')) + except AssertionError: + self.__logger.error("We cannot print invalid objects") + return '| {0:^68} |\n{1:-<26}{1:-<16}{1:-<13}{1:-<16}{1}'.format( + 'INVALID OBJECT', '+') def get_duration(self): """Return the duration of the test case. @@ -57,7 +72,7 @@ class TestCase(object): return "XX:XX" return "{0[0]:02.0f}:{0[1]:02.0f}".format(divmod( self.stop_time - self.start_time, 60)) - except Exception: + except Exception: # pylint: disable=broad-except self.__logger.error("Please run test before getting the duration") return "XX:XX" @@ -75,7 +90,8 @@ class TestCase(object): """ try: assert self.criteria - if isinstance(self.result, int) and isinstance(self.criteria, int): + if (not isinstance(self.result, str) and + not isinstance(self.criteria, str)): if self.result >= self.criteria: return TestCase.EX_OK else: diff --git a/functest/core/vnf_base.py b/functest/core/vnf.py index bc5bf9a71..5667b2997 100644 --- a/functest/core/vnf_base.py +++ b/functest/core/vnf.py @@ -17,12 +17,12 @@ import functest.utils.functest_utils as ft_utils import functest.utils.openstack_utils as os_utils -class VnfOnBoardingBase(base.TestCase): +class VnfOnBoarding(base.TestCase): __logger = logging.getLogger(__name__) def __init__(self, **kwargs): - super(VnfOnBoardingBase, self).__init__(**kwargs) + super(VnfOnBoarding, self).__init__(**kwargs) self.repo = kwargs.get('repo', '') self.cmd = kwargs.get('cmd', '') self.details = {} diff --git a/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py b/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py index 8a68cad9e..2a1b3a39b 100644 --- a/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py +++ b/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py @@ -5,6 +5,8 @@ # # http://www.apache.org/licenses/LICENSE-2.0 +import logging + from functest.core.pytest_suite_runner import PyTestSuiteRunner from functest.opnfv_tests.openstack.snaps import snaps_utils from functest.utils import functest_utils @@ -22,6 +24,7 @@ class SnapsTestRunner(PyTestSuiteRunner): """ def __init__(self, **kwargs): super(SnapsTestRunner, self).__init__(**kwargs) + self.logger = logging.getLogger(__name__) self.os_creds = openstack_tests.get_credentials( os_env_file=CONST.__getattribute__('openstack_creds'), diff --git a/functest/opnfv_tests/sdn/odl/odl.py b/functest/opnfv_tests/sdn/odl/odl.py index f92cb95da..e50d9c130 100755 --- a/functest/opnfv_tests/sdn/odl/odl.py +++ b/functest/opnfv_tests/sdn/odl/odl.py @@ -16,6 +16,8 @@ Example: $ python odl.py """ +from __future__ import division + import argparse import errno import fileinput @@ -100,7 +102,12 @@ class ODLTests(testcase.TestCase): result = robot.api.ExecutionResult(xml_file) visitor = ODLResultVisitor() result.visit(visitor) - self.result = result.suite.status + try: + self.result = 100 * ( + result.suite.statistics.critical.passed / + result.suite.statistics.critical.total) + except ZeroDivisionError: + self.__logger.error("No test has been ran") self.start_time = timestamp_to_secs(result.suite.starttime) self.stop_time = timestamp_to_secs(result.suite.endtime) self.details = {} diff --git a/functest/opnfv_tests/vnf/aaa/aaa.py b/functest/opnfv_tests/vnf/aaa/aaa.py index 1a484ddfb..0030256c2 100755 --- a/functest/opnfv_tests/vnf/aaa/aaa.py +++ b/functest/opnfv_tests/vnf/aaa/aaa.py @@ -13,10 +13,10 @@ import sys import argparse import functest.core.testcase as testcase -import functest.core.vnf_base as vnf_base +import functest.core.vnf as vnf -class AaaVnf(vnf_base.VnfOnBoardingBase): +class AaaVnf(vnf.VnfOnBoarding): logger = logging.getLogger(__name__) diff --git a/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py b/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py index f3bb30126..42d31e3bb 100644 --- a/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py +++ b/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py @@ -13,12 +13,12 @@ import shutil import requests -import functest.core.vnf_base as vnf_base +import functest.core.vnf as vnf from functest.utils.constants import CONST import functest.utils.functest_utils as ft_utils -class ClearwaterOnBoardingBase(vnf_base.VnfOnBoardingBase): +class ClearwaterOnBoardingBase(vnf.VnfOnBoarding): def __init__(self, **kwargs): self.logger = logging.getLogger(__name__) diff --git a/functest/opnfv_tests/vnf/ims/orchestra_ims.py b/functest/opnfv_tests/vnf/ims/orchestra_ims.py index 95751d47f..6f341970d 100755 --- a/functest/opnfv_tests/vnf/ims/orchestra_ims.py +++ b/functest/opnfv_tests/vnf/ims/orchestra_ims.py @@ -9,15 +9,15 @@ import json import logging +import os import socket import sys import time import yaml -import functest.core.vnf_base as vnf_base +import functest.core.vnf as vnf import functest.utils.functest_utils as ft_utils import functest.utils.openstack_utils as os_utils -import os from functest.utils.constants import CONST from org.openbaton.cli.agents.agents import MainAgent @@ -76,7 +76,7 @@ def servertest(host, port): return True -class ImsVnf(vnf_base.VnfOnBoardingBase): +class ImsVnf(vnf.VnfOnBoarding): def __init__(self, project='functest', case_name='orchestra_ims', repo='', cmd=''): diff --git a/functest/tests/unit/ci/test_generate_report.py b/functest/tests/unit/ci/test_generate_report.py index 13361c1d1..2c5ce2ea8 100644 --- a/functest/tests/unit/ci/test_generate_report.py +++ b/functest/tests/unit/ci/test_generate_report.py @@ -28,16 +28,16 @@ class GenerateReportTesting(unittest.TestCase): @mock.patch('functest.ci.generate_report.urllib2.urlopen', side_effect=urllib2.URLError('no host given')) def test_get_results_from_db_fail(self, mock_method): - url = "%s/results?build_tag=%s" % (ft_utils.get_db_url(), - CONST.__getattribute__('BUILD_TAG')) + url = "%s?build_tag=%s" % (ft_utils.get_db_url(), + CONST.__getattribute__('BUILD_TAG')) self.assertIsNone(gen_report.get_results_from_db()) mock_method.assert_called_once_with(url) @mock.patch('functest.ci.generate_report.urllib2.urlopen', return_value={'results': []}) def test_get_results_from_db_success(self, mock_method): - url = "%s/results?build_tag=%s" % (ft_utils.get_db_url(), - CONST.__getattribute__('BUILD_TAG')) + url = "%s?build_tag=%s" % (ft_utils.get_db_url(), + CONST.__getattribute__('BUILD_TAG')) self.assertEqual(gen_report.get_results_from_db(), None) mock_method.assert_called_once_with(url) diff --git a/functest/tests/unit/core/test_testcase.py b/functest/tests/unit/core/test_testcase.py index 17329ea32..b25ce2260 100644 --- a/functest/tests/unit/core/test_testcase.py +++ b/functest/tests/unit/core/test_testcase.py @@ -189,6 +189,39 @@ class TestCaseTesting(unittest.TestCase): self.test.stop_time = 180 self.assertEqual(self.test.get_duration(), "02:59") + def test_str_project_name_ko(self): + self.test.project_name = None + self.assertIn("INVALID OBJECT", str(self.test)) + + def test_str_case_name_ko(self): + self.test.case_name = None + self.assertIn("INVALID OBJECT", str(self.test)) + + def test_str_pass(self): + duration = '01:01' + with mock.patch.object(self.test, 'get_duration', + return_value=duration), \ + mock.patch.object(self.test, 'is_successful', + return_value=testcase.TestCase.EX_OK): + message = str(self.test) + self.assertIn(self._project_name, message) + self.assertIn(self._case_name, message) + self.assertIn(duration, message) + self.assertIn('PASS', message) + + def test_str_fail(self): + duration = '00:59' + with mock.patch.object(self.test, 'get_duration', + return_value=duration), \ + mock.patch.object( + self.test, 'is_successful', + return_value=testcase.TestCase.EX_TESTCASE_FAILED): + message = str(self.test) + self.assertIn(self._project_name, message) + self.assertIn(self._case_name, message) + self.assertIn(duration, message) + self.assertIn('FAIL', message) + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/functest/tests/unit/core/test_vnf_base.py b/functest/tests/unit/core/test_vnf.py index 540cf610d..f348c0dbf 100644 --- a/functest/tests/unit/core/test_vnf_base.py +++ b/functest/tests/unit/core/test_vnf.py @@ -10,7 +10,7 @@ import logging import unittest -from functest.core import vnf_base +from functest.core import vnf class VnfBaseTesting(unittest.TestCase): @@ -18,8 +18,8 @@ class VnfBaseTesting(unittest.TestCase): logging.disable(logging.CRITICAL) def setUp(self): - self.test = vnf_base.VnfOnBoardingBase(project='functest', - case_name='aaa') + self.test = vnf.VnfOnBoarding(project='functest', + case_name='aaa') self.test.project = "functest" self.test.start_time = "1" self.test.stop_time = "5" diff --git a/functest/tests/unit/odl/test_odl.py b/functest/tests/unit/odl/test_odl.py index f3d37c650..d7ce70c79 100644 --- a/functest/tests/unit/odl/test_odl.py +++ b/functest/tests/unit/odl/test_odl.py @@ -109,6 +109,9 @@ class ODLParseResultTesting(ODLTesting): """The class testing ODLTests.parse_results().""" # pylint: disable=missing-docstring + _config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000', + 'endtime': '20161216 16:00:01.000'} + @mock.patch('robot.api.ExecutionResult', side_effect=DataError) def test_raises_exc(self, mock_method): with self.assertRaises(DataError): @@ -116,15 +119,13 @@ class ODLParseResultTesting(ODLTesting): mock_method.assert_called_once_with( os.path.join(odl.ODLTests.res_dir, 'output.xml')) - def test_ok(self): - config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000', - 'endtime': '20161216 16:00:01.000', 'status': 'PASS'} + def _test_result(self, config, result): suite = mock.Mock() suite.configure_mock(**config) with mock.patch('robot.api.ExecutionResult', return_value=mock.Mock(suite=suite)): self.test.parse_results() - self.assertEqual(self.test.result, config['status']) + self.assertEqual(self.test.result, result) self.assertEqual(self.test.start_time, timestamp_to_secs(config['starttime'])) self.assertEqual(self.test.stop_time, @@ -132,6 +133,26 @@ class ODLParseResultTesting(ODLTesting): self.assertEqual(self.test.details, {'description': config['name'], 'tests': []}) + def test_null_passed(self): + self._config.update({'statistics.critical.passed': 0, + 'statistics.critical.total': 20}) + self._test_result(self._config, 0) + + def test_no_test(self): + self._config.update({'statistics.critical.passed': 20, + 'statistics.critical.total': 0}) + self._test_result(self._config, 0) + + def test_half_success(self): + self._config.update({'statistics.critical.passed': 10, + 'statistics.critical.total': 20}) + self._test_result(self._config, 50) + + def test_success(self): + self._config.update({'statistics.critical.passed': 20, + 'statistics.critical.total': 20}) + self._test_result(self._config, 100) + class ODLRobotTesting(ODLTesting): |