aboutsummaryrefslogtreecommitdiffstats
path: root/functest
diff options
context:
space:
mode:
Diffstat (limited to 'functest')
-rw-r--r--functest/ci/generate_report.py6
-rwxr-xr-xfunctest/ci/run_tests.py3
-rw-r--r--functest/core/pytest_suite_runner.py1
-rw-r--r--functest/core/testcase.py26
-rw-r--r--functest/core/vnf.py (renamed from functest/core/vnf_base.py)4
-rw-r--r--functest/opnfv_tests/openstack/snaps/snaps_test_runner.py3
-rwxr-xr-xfunctest/opnfv_tests/sdn/odl/odl.py9
-rwxr-xr-xfunctest/opnfv_tests/vnf/aaa/aaa.py4
-rw-r--r--functest/opnfv_tests/vnf/ims/clearwater_ims_base.py4
-rwxr-xr-xfunctest/opnfv_tests/vnf/ims/orchestra_ims.py6
-rw-r--r--functest/tests/unit/ci/test_generate_report.py8
-rw-r--r--functest/tests/unit/core/test_testcase.py33
-rw-r--r--functest/tests/unit/core/test_vnf.py (renamed from functest/tests/unit/core/test_vnf_base.py)6
-rw-r--r--functest/tests/unit/odl/test_odl.py29
14 files changed, 111 insertions, 31 deletions
diff --git a/functest/ci/generate_report.py b/functest/ci/generate_report.py
index 17240e8c..e400b1b6 100644
--- a/functest/ci/generate_report.py
+++ b/functest/ci/generate_report.py
@@ -39,8 +39,8 @@ def init(tiers_to_run=[]):
def get_results_from_db():
- url = "%s/results?build_tag=%s" % (ft_utils.get_db_url(),
- CONST.BUILD_TAG)
+ url = "%s?build_tag=%s" % (ft_utils.get_db_url(),
+ CONST.BUILD_TAG)
logger.debug("Query to rest api: %s" % url)
try:
data = json.load(urllib2.urlopen(url))
@@ -56,7 +56,7 @@ def get_data(test, results):
for test_db in results:
if test['test_name'] in test_db['case_name']:
id = test_db['_id']
- url = ft_utils.get_db_url() + '/results/' + id
+ url = ft_utils.get_db_url() + '/' + id
test_result = test_db['criteria']
return {"url": url, "result": test_result}
diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py
index ae002c2e..1396644f 100755
--- a/functest/ci/run_tests.py
+++ b/functest/ci/run_tests.py
@@ -155,6 +155,7 @@ def run_test(test, tier_name, testcases=None):
test_case.push_to_db()
result = test_case.is_successful()
duration = test_case.get_duration()
+ logger.info("\n%s\n", test_case)
except ImportError:
logger.exception("Cannot import module {}".format(
run_dict['module']))
@@ -167,8 +168,6 @@ def run_test(test, tier_name, testcases=None):
if test.needs_clean() and GlobalVariables.CLEAN_FLAG:
cleanup()
- logger.info("Test execution time: %s", duration)
-
if result != testcase.TestCase.EX_OK:
logger.error("The test case '%s' failed. " % test_name)
GlobalVariables.OVERALL_RESULT = Result.EX_ERROR
diff --git a/functest/core/pytest_suite_runner.py b/functest/core/pytest_suite_runner.py
index 8b5da05e..21edc187 100644
--- a/functest/core/pytest_suite_runner.py
+++ b/functest/core/pytest_suite_runner.py
@@ -18,6 +18,7 @@ class PyTestSuiteRunner(base.TestCase):
def __init__(self, **kwargs):
super(PyTestSuiteRunner, self).__init__(**kwargs)
self.suite = None
+ self.logger = None
def run(self, **kwargs):
"""
diff --git a/functest/core/testcase.py b/functest/core/testcase.py
index 62465542..49fae609 100644
--- a/functest/core/testcase.py
+++ b/functest/core/testcase.py
@@ -39,9 +39,24 @@ class TestCase(object):
self.project_name = kwargs.get('project_name', 'functest')
self.case_name = kwargs.get('case_name', '')
self.criteria = kwargs.get('criteria', 100)
- self.result = ""
- self.start_time = ""
- self.stop_time = ""
+ self.result = 0
+ self.start_time = 0
+ self.stop_time = 0
+
+ def __str__(self):
+ try:
+ assert self.project_name
+ assert self.case_name
+ result = 'PASS' if(self.is_successful(
+ ) == TestCase.EX_OK) else 'FAIL'
+ return ('| {0:<23} | {1:<13} | {2:<10} | {3:<13} |'
+ '\n{4:-<26}{4:-<16}{4:-<13}{4:-<16}{4}'.format(
+ self.case_name, self.project_name,
+ self.get_duration(), result, '+'))
+ except AssertionError:
+ self.__logger.error("We cannot print invalid objects")
+ return '| {0:^68} |\n{1:-<26}{1:-<16}{1:-<13}{1:-<16}{1}'.format(
+ 'INVALID OBJECT', '+')
def get_duration(self):
"""Return the duration of the test case.
@@ -57,7 +72,7 @@ class TestCase(object):
return "XX:XX"
return "{0[0]:02.0f}:{0[1]:02.0f}".format(divmod(
self.stop_time - self.start_time, 60))
- except Exception:
+ except Exception: # pylint: disable=broad-except
self.__logger.error("Please run test before getting the duration")
return "XX:XX"
@@ -75,7 +90,8 @@ class TestCase(object):
"""
try:
assert self.criteria
- if isinstance(self.result, int) and isinstance(self.criteria, int):
+ if (not isinstance(self.result, str) and
+ not isinstance(self.criteria, str)):
if self.result >= self.criteria:
return TestCase.EX_OK
else:
diff --git a/functest/core/vnf_base.py b/functest/core/vnf.py
index bc5bf9a7..5667b299 100644
--- a/functest/core/vnf_base.py
+++ b/functest/core/vnf.py
@@ -17,12 +17,12 @@ import functest.utils.functest_utils as ft_utils
import functest.utils.openstack_utils as os_utils
-class VnfOnBoardingBase(base.TestCase):
+class VnfOnBoarding(base.TestCase):
__logger = logging.getLogger(__name__)
def __init__(self, **kwargs):
- super(VnfOnBoardingBase, self).__init__(**kwargs)
+ super(VnfOnBoarding, self).__init__(**kwargs)
self.repo = kwargs.get('repo', '')
self.cmd = kwargs.get('cmd', '')
self.details = {}
diff --git a/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py b/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py
index 8a68cad9..2a1b3a39 100644
--- a/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py
+++ b/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py
@@ -5,6 +5,8 @@
#
# http://www.apache.org/licenses/LICENSE-2.0
+import logging
+
from functest.core.pytest_suite_runner import PyTestSuiteRunner
from functest.opnfv_tests.openstack.snaps import snaps_utils
from functest.utils import functest_utils
@@ -22,6 +24,7 @@ class SnapsTestRunner(PyTestSuiteRunner):
"""
def __init__(self, **kwargs):
super(SnapsTestRunner, self).__init__(**kwargs)
+ self.logger = logging.getLogger(__name__)
self.os_creds = openstack_tests.get_credentials(
os_env_file=CONST.__getattribute__('openstack_creds'),
diff --git a/functest/opnfv_tests/sdn/odl/odl.py b/functest/opnfv_tests/sdn/odl/odl.py
index f92cb95d..e50d9c13 100755
--- a/functest/opnfv_tests/sdn/odl/odl.py
+++ b/functest/opnfv_tests/sdn/odl/odl.py
@@ -16,6 +16,8 @@ Example:
$ python odl.py
"""
+from __future__ import division
+
import argparse
import errno
import fileinput
@@ -100,7 +102,12 @@ class ODLTests(testcase.TestCase):
result = robot.api.ExecutionResult(xml_file)
visitor = ODLResultVisitor()
result.visit(visitor)
- self.result = result.suite.status
+ try:
+ self.result = 100 * (
+ result.suite.statistics.critical.passed /
+ result.suite.statistics.critical.total)
+ except ZeroDivisionError:
+ self.__logger.error("No test has been ran")
self.start_time = timestamp_to_secs(result.suite.starttime)
self.stop_time = timestamp_to_secs(result.suite.endtime)
self.details = {}
diff --git a/functest/opnfv_tests/vnf/aaa/aaa.py b/functest/opnfv_tests/vnf/aaa/aaa.py
index 1a484ddf..0030256c 100755
--- a/functest/opnfv_tests/vnf/aaa/aaa.py
+++ b/functest/opnfv_tests/vnf/aaa/aaa.py
@@ -13,10 +13,10 @@ import sys
import argparse
import functest.core.testcase as testcase
-import functest.core.vnf_base as vnf_base
+import functest.core.vnf as vnf
-class AaaVnf(vnf_base.VnfOnBoardingBase):
+class AaaVnf(vnf.VnfOnBoarding):
logger = logging.getLogger(__name__)
diff --git a/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py b/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py
index f3bb3012..42d31e3b 100644
--- a/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py
+++ b/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py
@@ -13,12 +13,12 @@ import shutil
import requests
-import functest.core.vnf_base as vnf_base
+import functest.core.vnf as vnf
from functest.utils.constants import CONST
import functest.utils.functest_utils as ft_utils
-class ClearwaterOnBoardingBase(vnf_base.VnfOnBoardingBase):
+class ClearwaterOnBoardingBase(vnf.VnfOnBoarding):
def __init__(self, **kwargs):
self.logger = logging.getLogger(__name__)
diff --git a/functest/opnfv_tests/vnf/ims/orchestra_ims.py b/functest/opnfv_tests/vnf/ims/orchestra_ims.py
index 95751d47..6f341970 100755
--- a/functest/opnfv_tests/vnf/ims/orchestra_ims.py
+++ b/functest/opnfv_tests/vnf/ims/orchestra_ims.py
@@ -9,15 +9,15 @@
import json
import logging
+import os
import socket
import sys
import time
import yaml
-import functest.core.vnf_base as vnf_base
+import functest.core.vnf as vnf
import functest.utils.functest_utils as ft_utils
import functest.utils.openstack_utils as os_utils
-import os
from functest.utils.constants import CONST
from org.openbaton.cli.agents.agents import MainAgent
@@ -76,7 +76,7 @@ def servertest(host, port):
return True
-class ImsVnf(vnf_base.VnfOnBoardingBase):
+class ImsVnf(vnf.VnfOnBoarding):
def __init__(self, project='functest', case_name='orchestra_ims',
repo='', cmd=''):
diff --git a/functest/tests/unit/ci/test_generate_report.py b/functest/tests/unit/ci/test_generate_report.py
index 13361c1d..2c5ce2ea 100644
--- a/functest/tests/unit/ci/test_generate_report.py
+++ b/functest/tests/unit/ci/test_generate_report.py
@@ -28,16 +28,16 @@ class GenerateReportTesting(unittest.TestCase):
@mock.patch('functest.ci.generate_report.urllib2.urlopen',
side_effect=urllib2.URLError('no host given'))
def test_get_results_from_db_fail(self, mock_method):
- url = "%s/results?build_tag=%s" % (ft_utils.get_db_url(),
- CONST.__getattribute__('BUILD_TAG'))
+ url = "%s?build_tag=%s" % (ft_utils.get_db_url(),
+ CONST.__getattribute__('BUILD_TAG'))
self.assertIsNone(gen_report.get_results_from_db())
mock_method.assert_called_once_with(url)
@mock.patch('functest.ci.generate_report.urllib2.urlopen',
return_value={'results': []})
def test_get_results_from_db_success(self, mock_method):
- url = "%s/results?build_tag=%s" % (ft_utils.get_db_url(),
- CONST.__getattribute__('BUILD_TAG'))
+ url = "%s?build_tag=%s" % (ft_utils.get_db_url(),
+ CONST.__getattribute__('BUILD_TAG'))
self.assertEqual(gen_report.get_results_from_db(), None)
mock_method.assert_called_once_with(url)
diff --git a/functest/tests/unit/core/test_testcase.py b/functest/tests/unit/core/test_testcase.py
index 17329ea3..b25ce226 100644
--- a/functest/tests/unit/core/test_testcase.py
+++ b/functest/tests/unit/core/test_testcase.py
@@ -189,6 +189,39 @@ class TestCaseTesting(unittest.TestCase):
self.test.stop_time = 180
self.assertEqual(self.test.get_duration(), "02:59")
+ def test_str_project_name_ko(self):
+ self.test.project_name = None
+ self.assertIn("INVALID OBJECT", str(self.test))
+
+ def test_str_case_name_ko(self):
+ self.test.case_name = None
+ self.assertIn("INVALID OBJECT", str(self.test))
+
+ def test_str_pass(self):
+ duration = '01:01'
+ with mock.patch.object(self.test, 'get_duration',
+ return_value=duration), \
+ mock.patch.object(self.test, 'is_successful',
+ return_value=testcase.TestCase.EX_OK):
+ message = str(self.test)
+ self.assertIn(self._project_name, message)
+ self.assertIn(self._case_name, message)
+ self.assertIn(duration, message)
+ self.assertIn('PASS', message)
+
+ def test_str_fail(self):
+ duration = '00:59'
+ with mock.patch.object(self.test, 'get_duration',
+ return_value=duration), \
+ mock.patch.object(
+ self.test, 'is_successful',
+ return_value=testcase.TestCase.EX_TESTCASE_FAILED):
+ message = str(self.test)
+ self.assertIn(self._project_name, message)
+ self.assertIn(self._case_name, message)
+ self.assertIn(duration, message)
+ self.assertIn('FAIL', message)
+
if __name__ == "__main__":
unittest.main(verbosity=2)
diff --git a/functest/tests/unit/core/test_vnf_base.py b/functest/tests/unit/core/test_vnf.py
index 540cf610..f348c0db 100644
--- a/functest/tests/unit/core/test_vnf_base.py
+++ b/functest/tests/unit/core/test_vnf.py
@@ -10,7 +10,7 @@
import logging
import unittest
-from functest.core import vnf_base
+from functest.core import vnf
class VnfBaseTesting(unittest.TestCase):
@@ -18,8 +18,8 @@ class VnfBaseTesting(unittest.TestCase):
logging.disable(logging.CRITICAL)
def setUp(self):
- self.test = vnf_base.VnfOnBoardingBase(project='functest',
- case_name='aaa')
+ self.test = vnf.VnfOnBoarding(project='functest',
+ case_name='aaa')
self.test.project = "functest"
self.test.start_time = "1"
self.test.stop_time = "5"
diff --git a/functest/tests/unit/odl/test_odl.py b/functest/tests/unit/odl/test_odl.py
index f3d37c65..d7ce70c7 100644
--- a/functest/tests/unit/odl/test_odl.py
+++ b/functest/tests/unit/odl/test_odl.py
@@ -109,6 +109,9 @@ class ODLParseResultTesting(ODLTesting):
"""The class testing ODLTests.parse_results()."""
# pylint: disable=missing-docstring
+ _config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000',
+ 'endtime': '20161216 16:00:01.000'}
+
@mock.patch('robot.api.ExecutionResult', side_effect=DataError)
def test_raises_exc(self, mock_method):
with self.assertRaises(DataError):
@@ -116,15 +119,13 @@ class ODLParseResultTesting(ODLTesting):
mock_method.assert_called_once_with(
os.path.join(odl.ODLTests.res_dir, 'output.xml'))
- def test_ok(self):
- config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000',
- 'endtime': '20161216 16:00:01.000', 'status': 'PASS'}
+ def _test_result(self, config, result):
suite = mock.Mock()
suite.configure_mock(**config)
with mock.patch('robot.api.ExecutionResult',
return_value=mock.Mock(suite=suite)):
self.test.parse_results()
- self.assertEqual(self.test.result, config['status'])
+ self.assertEqual(self.test.result, result)
self.assertEqual(self.test.start_time,
timestamp_to_secs(config['starttime']))
self.assertEqual(self.test.stop_time,
@@ -132,6 +133,26 @@ class ODLParseResultTesting(ODLTesting):
self.assertEqual(self.test.details,
{'description': config['name'], 'tests': []})
+ def test_null_passed(self):
+ self._config.update({'statistics.critical.passed': 0,
+ 'statistics.critical.total': 20})
+ self._test_result(self._config, 0)
+
+ def test_no_test(self):
+ self._config.update({'statistics.critical.passed': 20,
+ 'statistics.critical.total': 0})
+ self._test_result(self._config, 0)
+
+ def test_half_success(self):
+ self._config.update({'statistics.critical.passed': 10,
+ 'statistics.critical.total': 20})
+ self._test_result(self._config, 50)
+
+ def test_success(self):
+ self._config.update({'statistics.critical.passed': 20,
+ 'statistics.critical.total': 20})
+ self._test_result(self._config, 100)
+
class ODLRobotTesting(ODLTesting):