diff options
Diffstat (limited to 'functest')
| -rw-r--r-- | functest/ci/testcases.yaml | 10 | ||||
| -rw-r--r-- | functest/core/robotframework.py | 125 | ||||
| -rw-r--r-- | functest/opnfv_tests/sdn/odl/odl.py | 113 | ||||
| -rw-r--r-- | functest/tests/unit/core/test_robotframework.py | 191 | ||||
| -rw-r--r-- | functest/tests/unit/odl/test_odl.py | 117 |
5 files changed, 350 insertions, 206 deletions
diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 2a708052..1d20a920 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -162,8 +162,8 @@ tiers: class: 'ODLTests' args: suites: - - /src/odl_test/csit/suites/integration/basic - - /src/odl_test/csit/suites/openstack/neutron + - /src/odl_test/csit/suites/integration/basic + - /src/odl_test/csit/suites/openstack/neutron - case_name: odl_netvirt @@ -183,9 +183,9 @@ tiers: class: 'ODLTests' args: suites: - - /src/odl_test/csit/suites/integration/basic - - /src/odl_test/csit/suites/openstack/neutron - - /src/odl_test/csit/suites/openstack/connectivity + - /src/odl_test/csit/suites/integration/basic + - /src/odl_test/csit/suites/openstack/neutron + - /src/odl_test/csit/suites/openstack/connectivity - case_name: snaps_smoke diff --git a/functest/core/robotframework.py b/functest/core/robotframework.py new file mode 100644 index 00000000..689d9d94 --- /dev/null +++ b/functest/core/robotframework.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python + +# Copyright (c) 2017 Orange and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +"""Define classes required to run any Robot suites.""" + +from __future__ import division + +import errno +import logging +import os + +import robot.api +from robot.errors import RobotError +import robot.run +from robot.utils.robottime import timestamp_to_secs +from six import StringIO + +from functest.core import testcase +from functest.utils import constants + +__author__ = "Cedric Ollivier <cedric.ollivier@orange.com>" + + +class ResultVisitor(robot.api.ResultVisitor): + """Visitor to get result details.""" + + def __init__(self): + self._data = [] + + def visit_test(self, test): + output = {} + output['name'] = test.name + output['parent'] = test.parent.name + output['status'] = test.status + output['starttime'] = test.starttime + output['endtime'] = test.endtime + output['critical'] = test.critical + output['text'] = test.message + output['elapsedtime'] = test.elapsedtime + self._data.append(output) + + def get_data(self): + """Get the details of the result.""" + return self._data + + +class RobotFramework(testcase.TestCase): + """RobotFramework runner.""" + + __logger = logging.getLogger(__name__) + + def __init__(self, **kwargs): + self.res_dir = os.path.join( + constants.CONST.__getattribute__('dir_results'), 'robot') + self.xml_file = os.path.join(self.res_dir, 'output.xml') + super(RobotFramework, self).__init__(**kwargs) + + def parse_results(self): + """Parse output.xml and get the details in it.""" + result = robot.api.ExecutionResult(self.xml_file) + visitor = ResultVisitor() + result.visit(visitor) + try: + self.result = 100 * ( + result.suite.statistics.critical.passed / + result.suite.statistics.critical.total) + except ZeroDivisionError: + self.__logger.error("No test has been run") + self.start_time = timestamp_to_secs(result.suite.starttime) + self.stop_time = timestamp_to_secs(result.suite.endtime) + self.details = {} + self.details['description'] = result.suite.name + self.details['tests'] = visitor.get_data() + + def run(self, **kwargs): + """Run the RobotFramework suites + + Here are the steps: + * create the output directories if required, + * get the results in output.xml, + * delete temporary files. + + Args: + kwargs: Arbitrary keyword arguments. + + Returns: + EX_OK if all suites ran well. + EX_RUN_ERROR otherwise. + """ + try: + suites = kwargs["suites"] + variable = kwargs.get("variable", []) + except KeyError: + self.__logger.exception("Mandatory args were not passed") + return self.EX_RUN_ERROR + try: + os.makedirs(self.res_dir) + except OSError as ex: + if ex.errno != errno.EEXIST: + self.__logger.exception("Cannot create %s", self.res_dir) + return self.EX_RUN_ERROR + except Exception: # pylint: disable=broad-except + self.__logger.exception("Cannot create %s", self.res_dir) + return self.EX_RUN_ERROR + stream = StringIO() + robot.run(*suites, variable=variable, output=self.xml_file, + log='NONE', report='NONE', stdout=stream) + self.__logger.info("\n" + stream.getvalue()) + self.__logger.info("Results were successfully generated") + try: + self.parse_results() + self.__logger.info("Results were successfully parsed") + except RobotError as ex: + self.__logger.error("Run suites before publishing: %s", ex.message) + return self.EX_RUN_ERROR + except Exception: # pylint: disable=broad-except + self.__logger.exception("Cannot parse results") + return self.EX_RUN_ERROR + return self.EX_OK diff --git a/functest/opnfv_tests/sdn/odl/odl.py b/functest/opnfv_tests/sdn/odl/odl.py index a8312ab3..de723d19 100644 --- a/functest/opnfv_tests/sdn/odl/odl.py +++ b/functest/opnfv_tests/sdn/odl/odl.py @@ -19,51 +19,22 @@ Example: from __future__ import division import argparse -import errno import fileinput import logging import os import re import sys -import robot.api -from robot.errors import RobotError -import robot.run -from robot.utils.robottime import timestamp_to_secs -from six import StringIO from six.moves import urllib -from functest.core import testcase +from functest.core import robotframework from functest.utils import constants import functest.utils.openstack_utils as op_utils __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>" -class ODLResultVisitor(robot.api.ResultVisitor): - """Visitor to get result details.""" - - def __init__(self): - self._data = [] - - def visit_test(self, test): - output = {} - output['name'] = test.name - output['parent'] = test.parent.name - output['status'] = test.status - output['starttime'] = test.starttime - output['endtime'] = test.endtime - output['critical'] = test.critical - output['text'] = test.message - output['elapsedtime'] = test.elapsedtime - self._data.append(output) - - def get_data(self): - """Get the details of the result.""" - return self._data - - -class ODLTests(testcase.TestCase): +class ODLTests(robotframework.RobotFramework): """ODL test runner.""" odl_test_repo = constants.CONST.__getattribute__('dir_repo_odl_test') @@ -72,10 +43,14 @@ class ODLTests(testcase.TestCase): basic_suite_dir = os.path.join(odl_test_repo, "csit/suites/integration/basic") default_suites = [basic_suite_dir, neutron_suite_dir] - res_dir = os.path.join( - constants.CONST.__getattribute__('dir_results'), 'odl') __logger = logging.getLogger(__name__) + def __init__(self, **kwargs): + super(ODLTests, self).__init__(**kwargs) + self.res_dir = os.path.join( + constants.CONST.__getattribute__('dir_results'), 'odl') + self.xml_file = os.path.join(self.res_dir, 'output.xml') + @classmethod def set_robotframework_vars(cls, odlusername="admin", odlpassword="admin"): """Set credentials in csit/variables/Variables.robot. @@ -98,24 +73,6 @@ class ODLTests(testcase.TestCase): cls.__logger.error("Cannot set ODL creds: %s", str(ex)) return False - def parse_results(self): - """Parse output.xml and get the details in it.""" - xml_file = os.path.join(self.res_dir, 'output.xml') - result = robot.api.ExecutionResult(xml_file) - visitor = ODLResultVisitor() - result.visit(visitor) - try: - self.result = 100 * ( - result.suite.statistics.critical.passed / - result.suite.statistics.critical.total) - except ZeroDivisionError: - self.__logger.error("No test has been run") - self.start_time = timestamp_to_secs(result.suite.starttime) - self.stop_time = timestamp_to_secs(result.suite.endtime) - self.details = {} - self.details['description'] = result.suite.name - self.details['tests'] = visitor.get_data() - def run_suites(self, suites=None, **kwargs): """Run the test suites @@ -155,44 +112,24 @@ class ODLTests(testcase.TestCase): keystoneurl = "{}://{}".format( urllib.parse.urlparse(osauthurl).scheme, urllib.parse.urlparse(osauthurl).netloc) - variables = ['KEYSTONEURL:' + keystoneurl, - 'NEUTRONURL:' + kwargs['neutronurl'], - 'OS_AUTH_URL:"' + osauthurl + '"', - 'OSUSERNAME:"' + kwargs['osusername'] + '"', - ('OSUSERDOMAINNAME:"' + - kwargs['osuserdomainname'] + '"'), - 'OSTENANTNAME:"' + kwargs['osprojectname'] + '"', - ('OSPROJECTDOMAINNAME:"' + - kwargs['osprojectdomainname'] + '"'), - 'OSPASSWORD:"' + kwargs['ospassword'] + '"', - 'ODL_SYSTEM_IP:' + kwargs['odlip'], - 'PORT:' + kwargs['odlwebport'], - 'RESTCONFPORT:' + kwargs['odlrestconfport']] - except KeyError as ex: + variable = ['KEYSTONEURL:' + keystoneurl, + 'NEUTRONURL:' + kwargs['neutronurl'], + 'OS_AUTH_URL:"' + osauthurl + '"', + 'OSUSERNAME:"' + kwargs['osusername'] + '"', + ('OSUSERDOMAINNAME:"' + + kwargs['osuserdomainname'] + '"'), + 'OSTENANTNAME:"' + kwargs['osprojectname'] + '"', + ('OSPROJECTDOMAINNAME:"' + + kwargs['osprojectdomainname'] + '"'), + 'OSPASSWORD:"' + kwargs['ospassword'] + '"', + 'ODL_SYSTEM_IP:' + kwargs['odlip'], + 'PORT:' + kwargs['odlwebport'], + 'RESTCONFPORT:' + kwargs['odlrestconfport']] + except KeyError: self.__logger.exception("Cannot run ODL testcases. Please check") return self.EX_RUN_ERROR if self.set_robotframework_vars(odlusername, odlpassword): - try: - os.makedirs(self.res_dir) - except OSError as ex: - if ex.errno != errno.EEXIST: - self.__logger.exception( - "Cannot create %s", self.res_dir) - return self.EX_RUN_ERROR - output_dir = os.path.join(self.res_dir, 'output.xml') - stream = StringIO() - robot.run(*suites, variable=variables, output=output_dir, - log='NONE', report='NONE', stdout=stream) - self.__logger.info("\n" + stream.getvalue()) - self.__logger.info("ODL results were successfully generated") - try: - self.parse_results() - self.__logger.info("ODL results were successfully parsed") - except RobotError as ex: - self.__logger.error("Run tests before publishing: %s", - ex.message) - return self.EX_RUN_ERROR - return self.EX_OK + return super(ODLTests, self).run(variable=variable, suites=suites) else: return self.EX_RUN_ERROR @@ -330,11 +267,11 @@ def main(): args = parser.parse_args(sys.argv[1:]) try: result = odl.run_suites(ODLTests.default_suites, **args) - if result != testcase.TestCase.EX_OK: + if result != robotframework.RobotFramework.EX_OK: return result if args['pushtodb']: return odl.push_to_db() else: return result except Exception: # pylint: disable=broad-except - return testcase.TestCase.EX_RUN_ERROR + return robotframework.RobotFramework.EX_RUN_ERROR diff --git a/functest/tests/unit/core/test_robotframework.py b/functest/tests/unit/core/test_robotframework.py new file mode 100644 index 00000000..38e9039b --- /dev/null +++ b/functest/tests/unit/core/test_robotframework.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python + +# Copyright (c) 2017 Orange and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +"""Define the classes required to fully cover robot.""" + +import errno +import logging +import os +import unittest + +import mock +from robot.errors import DataError, RobotError +from robot.result import model +from robot.utils.robottime import timestamp_to_secs + +from functest.core import robotframework + +__author__ = "Cedric Ollivier <cedric.ollivier@orange.com>" + + +class ResultVisitorTesting(unittest.TestCase): + + """The class testing ResultVisitor.""" + # pylint: disable=missing-docstring + + def setUp(self): + self.visitor = robotframework.ResultVisitor() + + def test_empty(self): + self.assertFalse(self.visitor.get_data()) + + def test_ok(self): + data = {'name': 'foo', + 'parent': 'bar', + 'status': 'PASS', + 'starttime': "20161216 16:00:00.000", + 'endtime': "20161216 16:00:01.000", + 'elapsedtime': 1000, + 'text': 'Hello, World!', + 'critical': True} + test = model.TestCase( + name=data['name'], status=data['status'], message=data['text'], + starttime=data['starttime'], endtime=data['endtime']) + test.parent = mock.Mock() + config = {'name': data['parent'], + 'criticality.test_is_critical.return_value': data[ + 'critical']} + test.parent.configure_mock(**config) + self.visitor.visit_test(test) + self.assertEqual(self.visitor.get_data(), [data]) + + +class ParseResultTesting(unittest.TestCase): + + """The class testing RobotFramework.parse_results().""" + # pylint: disable=missing-docstring + + _config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000', + 'endtime': '20161216 16:00:01.000'} + + def setUp(self): + self.test = robotframework.RobotFramework( + case_name='robot', project_name='functest') + + @mock.patch('robot.api.ExecutionResult', side_effect=DataError) + def test_raises_exc(self, mock_method): + with self.assertRaises(DataError): + self.test.parse_results() + mock_method.assert_called_once_with( + os.path.join(self.test.res_dir, 'output.xml')) + + def _test_result(self, config, result): + suite = mock.Mock() + suite.configure_mock(**config) + with mock.patch('robot.api.ExecutionResult', + return_value=mock.Mock(suite=suite)): + self.test.parse_results() + self.assertEqual(self.test.result, result) + self.assertEqual(self.test.start_time, + timestamp_to_secs(config['starttime'])) + self.assertEqual(self.test.stop_time, + timestamp_to_secs(config['endtime'])) + self.assertEqual(self.test.details, + {'description': config['name'], 'tests': []}) + + def test_null_passed(self): + self._config.update({'statistics.critical.passed': 0, + 'statistics.critical.total': 20}) + self._test_result(self._config, 0) + + def test_no_test(self): + self._config.update({'statistics.critical.passed': 20, + 'statistics.critical.total': 0}) + self._test_result(self._config, 0) + + def test_half_success(self): + self._config.update({'statistics.critical.passed': 10, + 'statistics.critical.total': 20}) + self._test_result(self._config, 50) + + def test_success(self): + self._config.update({'statistics.critical.passed': 20, + 'statistics.critical.total': 20}) + self._test_result(self._config, 100) + + +class RunTesting(unittest.TestCase): + + """The class testing RobotFramework.run().""" + # pylint: disable=missing-docstring + + suites = ["foo"] + variable = [] + + def setUp(self): + self.test = robotframework.RobotFramework( + case_name='robot', project_name='functest') + + def test_exc_key_error(self): + self.assertEqual(self.test.run(), self.test.EX_RUN_ERROR) + + @mock.patch('robot.run') + def _test_makedirs_exc(self, *args): + with mock.patch.object(self.test, 'parse_results') as mock_method: + self.assertEqual( + self.test.run(suites=self.suites, variable=self.variable), + self.test.EX_RUN_ERROR) + args[0].assert_not_called() + mock_method.asser_not_called() + + @mock.patch('os.makedirs', side_effect=Exception) + def test_makedirs_exc(self, *args): + self._test_makedirs_exc() + args[0].assert_called_once_with(self.test.res_dir) + + @mock.patch('os.makedirs', side_effect=OSError) + def test_makedirs_oserror(self, *args): + self._test_makedirs_exc() + args[0].assert_called_once_with(self.test.res_dir) + + @mock.patch('robot.run') + def _test_makedirs(self, *args): + with mock.patch.object(self.test, 'parse_results') as mock_method: + self.assertEqual( + self.test.run(suites=self.suites, variable=self.variable), + self.test.EX_OK) + args[0].assert_called_once_with( + *self.suites, log='NONE', output=self.test.xml_file, + report='NONE', stdout=mock.ANY, variable=self.variable) + mock_method.assert_called_once_with() + + @mock.patch('os.makedirs', side_effect=OSError(errno.EEXIST, '')) + def test_makedirs_oserror17(self, *args): + self._test_makedirs() + args[0].assert_called_once_with(self.test.res_dir) + + @mock.patch('os.makedirs') + def test_makedirs(self, *args): + self._test_makedirs() + args[0].assert_called_once_with(self.test.res_dir) + + @mock.patch('robot.run') + def _test_parse_results(self, status, *args): + self.assertEqual( + self.test.run(suites=self.suites, variable=self.variable), status) + args[0].assert_called_once_with( + *self.suites, log='NONE', output=self.test.xml_file, + report='NONE', stdout=mock.ANY, variable=self.variable) + + def test_parse_results_exc(self): + with mock.patch.object(self.test, 'parse_results', + side_effect=Exception) as mock_method: + self._test_parse_results(self.test.EX_RUN_ERROR) + mock_method.assert_called_once_with() + + def test_parse_results_robot_error(self): + with mock.patch.object(self.test, 'parse_results', + side_effect=RobotError('foo')) as mock_method: + self._test_parse_results(self.test.EX_RUN_ERROR) + mock_method.assert_called_once_with() + + +if __name__ == "__main__": + logging.disable(logging.CRITICAL) + unittest.main(verbosity=2) diff --git a/functest/tests/unit/odl/test_odl.py b/functest/tests/unit/odl/test_odl.py index 7258cd5e..a331a6f3 100644 --- a/functest/tests/unit/odl/test_odl.py +++ b/functest/tests/unit/odl/test_odl.py @@ -9,16 +9,13 @@ """Define the classes required to fully cover odl.""" -import errno import logging import os import unittest from keystoneauth1.exceptions import auth_plugins import mock -from robot.errors import DataError, RobotError -from robot.result import model -from robot.utils.robottime import timestamp_to_secs +from robot.errors import RobotError import six from six.moves import urllib @@ -28,38 +25,6 @@ from functest.opnfv_tests.sdn.odl import odl __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>" -class ODLVisitorTesting(unittest.TestCase): - - """The class testing ODLResultVisitor.""" - # pylint: disable=missing-docstring - - def setUp(self): - self.visitor = odl.ODLResultVisitor() - - def test_empty(self): - self.assertFalse(self.visitor.get_data()) - - def test_ok(self): - data = {'name': 'foo', - 'parent': 'bar', - 'status': 'PASS', - 'starttime': "20161216 16:00:00.000", - 'endtime': "20161216 16:00:01.000", - 'elapsedtime': 1000, - 'text': 'Hello, World!', - 'critical': True} - test = model.TestCase( - name=data['name'], status=data['status'], message=data['text'], - starttime=data['starttime'], endtime=data['endtime']) - test.parent = mock.Mock() - config = {'name': data['parent'], - 'criticality.test_is_critical.return_value': data[ - 'critical']} - test.parent.configure_mock(**config) - self.visitor.visit_test(test) - self.assertEqual(self.visitor.get_data(), [data]) - - class ODLTesting(unittest.TestCase): """The super class which testing classes could inherit.""" @@ -109,56 +74,6 @@ class ODLTesting(unittest.TestCase): 'pushtodb': False} -class ODLParseResultTesting(ODLTesting): - - """The class testing ODLTests.parse_results().""" - # pylint: disable=missing-docstring - - _config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000', - 'endtime': '20161216 16:00:01.000'} - - @mock.patch('robot.api.ExecutionResult', side_effect=DataError) - def test_raises_exc(self, mock_method): - with self.assertRaises(DataError): - self.test.parse_results() - mock_method.assert_called_once_with( - os.path.join(odl.ODLTests.res_dir, 'output.xml')) - - def _test_result(self, config, result): - suite = mock.Mock() - suite.configure_mock(**config) - with mock.patch('robot.api.ExecutionResult', - return_value=mock.Mock(suite=suite)): - self.test.parse_results() - self.assertEqual(self.test.result, result) - self.assertEqual(self.test.start_time, - timestamp_to_secs(config['starttime'])) - self.assertEqual(self.test.stop_time, - timestamp_to_secs(config['endtime'])) - self.assertEqual(self.test.details, - {'description': config['name'], 'tests': []}) - - def test_null_passed(self): - self._config.update({'statistics.critical.passed': 0, - 'statistics.critical.total': 20}) - self._test_result(self._config, 0) - - def test_no_test(self): - self._config.update({'statistics.critical.passed': 20, - 'statistics.critical.total': 0}) - self._test_result(self._config, 0) - - def test_half_success(self): - self._config.update({'statistics.critical.passed': 10, - 'statistics.critical.total': 20}) - self._test_result(self._config, 50) - - def test_success(self): - self._config.update({'statistics.critical.passed': 20, - 'statistics.critical.total': 20}) - self._test_result(self._config, 100) - - class ODLRobotTesting(ODLTesting): """The class testing ODLTests.set_robotframework_vars().""" @@ -239,8 +154,7 @@ class ODLMainTesting(ODLTesting): kwargs = self._get_run_suites_kwargs() self.assertEqual(self.test.run_suites(**kwargs), status) if len(args) > 0: - args[0].assert_called_once_with( - odl.ODLTests.res_dir) + args[0].assert_called_once_with(self.test.res_dir) if len(args) > 1: variable = [ 'KEYSTONEURL:{}://{}'.format( @@ -260,13 +174,13 @@ class ODLMainTesting(ODLTesting): odl.ODLTests.basic_suite_dir, odl.ODLTests.neutron_suite_dir, log='NONE', - output=os.path.join(odl.ODLTests.res_dir, 'output.xml'), + output=os.path.join(self.test.res_dir, 'output.xml'), report='NONE', stdout=mock.ANY, variable=variable) if len(args) > 2: args[2].assert_called_with( - os.path.join(odl.ODLTests.res_dir, 'stdout.txt')) + os.path.join(self.test.res_dir, 'stdout.txt')) def _test_no_keyword(self, key): kwargs = self._get_run_suites_kwargs(key) @@ -310,21 +224,6 @@ class ODLMainTesting(ODLTesting): mock_object.assert_called_once_with( self._odl_username, self._odl_password) - @mock.patch('os.makedirs', side_effect=Exception) - def test_makedirs_exc(self, mock_method): - with mock.patch.object(self.test, 'set_robotframework_vars', - return_value=True), \ - self.assertRaises(Exception): - self._test_run_suites(testcase.TestCase.EX_RUN_ERROR, - mock_method) - - @mock.patch('os.makedirs', side_effect=OSError) - def test_makedirs_oserror(self, mock_method): - with mock.patch.object(self.test, 'set_robotframework_vars', - return_value=True): - self._test_run_suites(testcase.TestCase.EX_RUN_ERROR, - mock_method) - @mock.patch('robot.run', side_effect=RobotError) @mock.patch('os.makedirs') def test_run_ko(self, *args): @@ -350,14 +249,6 @@ class ODLMainTesting(ODLTesting): mock.patch.object(self.test, 'parse_results'): self._test_run_suites(testcase.TestCase.EX_OK, *args) - @mock.patch('robot.run') - @mock.patch('os.makedirs', side_effect=OSError(errno.EEXIST, '')) - def test_makedirs_oserror17(self, *args): - with mock.patch.object(self.test, 'set_robotframework_vars', - return_value=True), \ - mock.patch.object(self.test, 'parse_results'): - self._test_run_suites(testcase.TestCase.EX_OK, *args) - @mock.patch('robot.run', return_value=1) @mock.patch('os.makedirs') def test_testcases_in_failure(self, *args): |