diff options
| -rw-r--r-- | INFO | 31 | ||||
| -rw-r--r-- | docs/testing/user/configguide/configguide.rst | 1 | ||||
| -rw-r--r-- | docs/testing/user/userguide/index.rst | 120 | ||||
| -rw-r--r-- | functest/ci/testcases.yaml | 6 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/refstack_client/defcore.txt | 9 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/snaps/health_check.py | 3 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/snaps/smoke.py | 5 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/snaps/snaps_test_runner.py | 10 | ||||
| -rw-r--r-- | functest/tests/unit/odl/test_odl.py | 44 | ||||
| -rwxr-xr-x | run_unit_tests.sh | 13 |
10 files changed, 177 insertions, 65 deletions
@@ -11,19 +11,24 @@ IRC: Server:freenode.net Channel:#opnfv-functest Repository: functest Committers: -yaohelan@huawei.com -feng.xiaowei@zte.com.cn -ollivier.cedric@gmail.com -jose.lausuch@ericsson.com -morgan.richomme@orange.com -meimei@huawei.com -valentin.boucher@orange.com -lanqinglong@huawei.com -viktor.tikkanen@nokia.com -juha.kosonen@nokia.com -zhanghaoyu7@huawei.com -raghavendrachari.kamsali@hpe.com -lixiaoguang5@huawei.com +Morgan Richomme <morgan.richomme@orange.com> +Jose Lausuch <jose.lausuch@ericsson.com> +Cedric Ollivier <ollivier.cedric@gmail.com> +Helen Yao <yaohelan@huawei.com> +Serena Feng <feng.xiaowei@zte.com.cn> +Juha Kosonen <juha.kosonen@nokia.com> +Valentin Boucher <valentin.boucher@orange.com> +Viktor Tikkanen <viktor.tikkanen@nokia.com> +Mei Mei <meimei@huawei.com> + +Additional contributors: +Linda Wang <wangwulin@huawei.com> +Georgios Paraskevopoulos <georgepar.91@gmail.com> +Romanos Skiadas <rom.skiad@gmail.com> +Michael Polenchuk <mpolenchuk@mirantis.com> +Cristina Pauna <cristina.pauna@enea.com> +Matthew Li <matthew.lijun@huawei.com> +Steven Pisarski <s.pisarski@cablelabs.com> Link to TSC approval of the project: http://meetbot.opnfv.org/meetings/opnfv-meeting/2015/opnfv-meeting.2015-01-20-14.57.html diff --git a/docs/testing/user/configguide/configguide.rst b/docs/testing/user/configguide/configguide.rst index e3485be40..9a1749789 100644 --- a/docs/testing/user/configguide/configguide.rst +++ b/docs/testing/user/configguide/configguide.rst @@ -336,6 +336,7 @@ should now be in place:: |-- parser |-- promise |-- rally + |-- refstack-client |-- releng |-- sdnvpn |-- securityscanning diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index e050cf14b..c1faecdaa 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -247,13 +247,125 @@ The Rally testcases are distributed accross two Tiers: NOTE: Test case 'rally_sanity' executes a limited number of Rally smoke test cases. Test case 'rally_full' executes the full defined set of Rally tests. + +Refstack-client to run Defcore testcases +----------------------------------------- + +Refstack-client `[8]`_ is a command line utility that allows you to +execute Tempest test runs based on configurations you specify. +It is the official tool to run Defcore `[9]`_ testcases, +which focuses on testing interoperability between OpenStack clouds. + +Refstack-client is integrated in Functest, consumed by Dovetail, which +intends to define and provide a set of OPNFV related validation criteria +that will provide input for the evaluation of the use of OPNFV trademarks. +This progress is under the guideline of Compliance Verification Program(CVP). + +Defcore testcases +^^^^^^^^^^^^^^^^^^ + +*Danube Release* + +Set of DefCore tempest test cases not flagged and required. +According to `[10]`_, some tests are still flagged due to outstanding bugs +in the Tempest library, particularly tests that require SSH. Refstack developers +are working on correcting these bugs upstream. Please note that although some tests +are flagged because of bugs, there is still an expectation that the capabilities +covered by the tests are available. It only contains Openstack core compute +(no object storage). The approved guidelines (2016.08) are valid for Kilo, +Liberty, Mitaka and Newton releases of OpenStack. +The list can be generated using the Rest API from RefStack project: +https://refstack.openstack.org/api/v1/guidelines/2016.08/tests?target=compute&type=required&alias=true&flag=false + +Running methods +^^^^^^^^^^^^^^^ + +Two running methods are provided after refstack-client integrated into +Functest, Functest command line and manually, respectively. + +By default, for Defcore test cases run by Functest command line, +are run followed with automatically generated +configuration file, i.e., refstack_tempest.conf. In some circumstances, +the automatic configuration file may not quite satisfied with the SUT, +Functest also inherits the refstack-client command line and provides a way +for users to set its configuration file according to its own SUT manually. + +*command line* + +Inside the Functest container, first to prepare Functest environment: + +:: + + cd /home/opnfv/repos/functest + pip install -e . + functest env prepare + +then to run default defcore testcases by using refstack-client: + +:: + + functest testcase run refstack_defcore + +In OPNFV Continuous Integration(CI) system, the command line method is used. + +*manually* + +Inside the Functest container, first to prepare the refstack virtualenv: + +:: + + cd /home/opnfv/repos/refstack-client + source .venv/bin/activate + +then prepare the tempest configuration file and the testcases want to run with the SUT, +run the testcases with: + +:: + + ./refstack-client test -c <Path of the tempest configuration file to use> -v --test-list <Path or URL of test list> + +using help for more information: + +:: + + ./refstack-client --help + ./refstack-client test --help + +Reference tempest configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +*command line method* + +When command line method is used, the default tempest configuration file +is generated by Rally. + +*manually* + +When running manually is used, recommended way to generate tempest configuration +file is: + +:: + + cd /home/opnfv/repos/functest/functest/opnfv_tests/openstack/refstack_client + python tempest_conf.py + +a file called tempest.conf is stored in the current path by default, users can do +some adjustment according to the SUT: + +:: + + vim refstack_tempest.conf + +a reference article can be used `[15]`_. + + snaps_smoke ------------ This test case contains tests that setup and destroy environments with VMs with and without Floating IPs with a newly created user and project. Set the config value snaps.use_floating_ips (True|False) to toggle this functionality. When -the config value of snaps.use_keystone is True, functest must have access +the config value of snaps.use_keystone is True, Functest must have access the cloud's private network. This suite consists in 38 tests (test duration < 10 minutes) @@ -362,7 +474,7 @@ The test cases are described as follows: Features -------- -In Danube, functest supports the integration of: +In Danube, Functest supports the integration of: * barometer * bgpvpn @@ -520,10 +632,14 @@ References .. _`[5]`: https://github.com/Orange-OpenSource/opnfv-cloudify-clearwater/blob/master/openstack-blueprint.yaml .. _`[6]`: https://scap.nist.gov/ .. _`[7]`: https://github.com/OpenSCAP/openscap +.. _`[8]`: https://github.com/openstack/refstack-client +.. _`[9]`: https://github.com/openstack/defcore +.. _`[10]`: https://github.com/openstack/interop/blob/master/2016.08/procedure.rst .. _`[11]`: http://robotframework.org/ .. _`[12]`: http://artifacts.opnfv.org/parser/colorado/docs/userguide/index.html .. _`[13]`: https://wiki.opnfv.org/display/PROJ/SNAPS-OO .. _`[14]`: https://github.com/oolorg/opnfv-functest-vrouter +.. _`[15]`: https://aptira.com/testing-openstack-tempest-part-1/ `OPNFV main site`_ diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 8f2cc4bc2..7009e910c 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -567,7 +567,7 @@ tiers: class: 'ImsVnf' - - case_name: opera-vims + case_name: opera_vims project_name: opera criteria: 100 blocking: false @@ -575,8 +575,8 @@ tiers: description: >- VNF deployment with OPEN-O dependencies: - installer: 'unknown' - scenario: 'unknown' + installer: 'compass' + scenario: 'os-nosdn-openo-ha' run: module: 'functest.opnfv_tests.vnf.ims.opera_ims' class: 'OperaIms' diff --git a/functest/opnfv_tests/openstack/refstack_client/defcore.txt b/functest/opnfv_tests/openstack/refstack_client/defcore.txt index be8fd8998..0a1787ef3 100644 --- a/functest/opnfv_tests/openstack/refstack_client/defcore.txt +++ b/functest/opnfv_tests/openstack/refstack_client/defcore.txt @@ -1,4 +1,11 @@ -# Set of DefCore tempest test cases not flagged and required. It only contains OpenStack core (no object storage) +# Set of DefCore tempest test cases not flagged and required. +# According to https://github.com/openstack/interop/blob/master/2016.08/procedure.rst, +# some tests are still flagged due to outstanding bugs in the Tempest library, +# particularly tests that require SSH. Refstack developers +# are working on correcting these bugs upstream. Please note that although some tests +# are flagged because of bugs, there is still an expectation that the capabilities +# covered by the tests are available. +# It only contains Openstack core compute (no object storage) # The approved guidelines (2016.08) are valid for Kilo, Liberty, Mitaka and Newton releases of OpenStack # The list can be generated using the Rest API from RefStack project: # https://refstack.openstack.org/api/v1/guidelines/2016.08/tests?target=compute&type=required&alias=true&flag=false diff --git a/functest/opnfv_tests/openstack/snaps/health_check.py b/functest/opnfv_tests/openstack/snaps/health_check.py index 44e3b876b..c057eb2b0 100644 --- a/functest/opnfv_tests/openstack/snaps/health_check.py +++ b/functest/opnfv_tests/openstack/snaps/health_check.py @@ -30,8 +30,7 @@ class HealthCheck(SnapsTestRunner): image_custom_config = None if hasattr(CONST, 'snaps_health_check'): - image_custom_config = CONST.snaps_health_check - + image_custom_config = CONST.__getattribute__('snaps_health_check') self.suite.addTest( OSIntegrationTestCase.parameterize( SimpleHealthCheck, os_creds=self.os_creds, diff --git a/functest/opnfv_tests/openstack/snaps/smoke.py b/functest/opnfv_tests/openstack/snaps/smoke.py index 5a637f288..2c6fc2553 100644 --- a/functest/opnfv_tests/openstack/snaps/smoke.py +++ b/functest/opnfv_tests/openstack/snaps/smoke.py @@ -32,12 +32,13 @@ class SnapsSmoke(SnapsTestRunner): # snaps_health_check suite, so reuse it here image_custom_config = None if hasattr(CONST, 'snaps_health_check'): - image_custom_config = CONST.snaps_health_check + image_custom_config = CONST.__getattribute__('snaps_health_check') # Tests requiring floating IPs leverage files contained within the # SNAPS repository and are found relative to that path if self.use_fip: - snaps_dir = CONST.dir_repo_snaps + '/snaps' + snaps_dir = os.path.join(CONST.__getattribute__('dir_repo_snaps'), + 'snaps') os.chdir(snaps_dir) test_suite_builder.add_openstack_integration_tests( diff --git a/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py b/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py index b17aab0c0..8a68cad9e 100644 --- a/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py +++ b/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py @@ -24,16 +24,16 @@ class SnapsTestRunner(PyTestSuiteRunner): super(SnapsTestRunner, self).__init__(**kwargs) self.os_creds = openstack_tests.get_credentials( - os_env_file=CONST.openstack_creds, proxy_settings_str=None, - ssh_proxy_cmd=None) + os_env_file=CONST.__getattribute__('openstack_creds'), + proxy_settings_str=None, ssh_proxy_cmd=None) self.ext_net_name = snaps_utils.get_ext_net_name(self.os_creds) - self.use_fip = CONST.snaps_use_floating_ips - self.use_keystone = CONST.snaps_use_keystone + self.use_fip = CONST.__getattribute__('snaps_use_floating_ips') + self.use_keystone = CONST.__getattribute__('snaps_use_keystone') scenario = functest_utils.get_scenario() self.flavor_metadata = create_flavor.MEM_PAGE_SIZE_ANY if 'ovs' in scenario or 'fdio' in scenario: self.flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE - self.logger.info("Using flavor metatdata '%s'" % self.flavor_metadata) + self.logger.info("Using flavor metadata '%s'", self.flavor_metadata) diff --git a/functest/tests/unit/odl/test_odl.py b/functest/tests/unit/odl/test_odl.py index 804693463..54d6da72c 100644 --- a/functest/tests/unit/odl/test_odl.py +++ b/functest/tests/unit/odl/test_odl.py @@ -113,7 +113,8 @@ class ODLParseResultTesting(ODLTesting): def test_raises_exc(self, mock_method): with self.assertRaises(DataError): self.test.parse_results() - mock_method.assert_called_once_with() + mock_method.assert_called_once_with( + os.path.join(odl.ODLTests.res_dir, 'output.xml')) def test_ok(self): config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000', @@ -368,19 +369,10 @@ class ODLRunTesting(ODLTesting): """The class testing ODLTests.run().""" # pylint: disable=missing-docstring - @classmethod - def _fake_url_for(cls, service_type='identity'): - if service_type == 'identity': - return "http://{}:5000/v2.0".format( - ODLTesting._keystone_ip) - elif service_type == 'network': - return "http://{}:9696".format(ODLTesting._neutron_ip) - else: - return None - def _test_no_env_var(self, var): with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): del os.environ[var] self.assertEqual(self.test.run(), testcase.TestCase.EX_RUN_ERROR) @@ -393,7 +385,8 @@ class ODLRunTesting(ODLTesting): if 'odlrestconfport' in kwargs else '8181') with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): if exception: self.test.main = mock.Mock(side_effect=exception) else: @@ -410,18 +403,15 @@ class ODLRunTesting(ODLTesting): osusername=self._os_username) def _test_multiple_suites(self, suites, - status=testcase.TestCase.EX_OK, - exception=None, **kwargs): + status=testcase.TestCase.EX_OK, **kwargs): odlip = kwargs['odlip'] if 'odlip' in kwargs else '127.0.0.3' odlwebport = kwargs['odlwebport'] if 'odlwebport' in kwargs else '8080' odlrestconfport = (kwargs['odlrestconfport'] if 'odlrestconfport' in kwargs else '8181') with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): - if exception: - self.test.main = mock.Mock(side_effect=exception) - else: - self.test.main = mock.Mock(return_value=status) + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): + self.test.main = mock.Mock(return_value=status) self.assertEqual(self.test.run(suites=suites), status) self.test.main.assert_called_once_with( suites, @@ -467,7 +457,8 @@ class ODLRunTesting(ODLTesting): def test_no_sdn_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): self.assertEqual(self.test.run(), testcase.TestCase.EX_RUN_ERROR) @@ -492,7 +483,8 @@ class ODLRunTesting(ODLTesting): def test_apex_no_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): os.environ["INSTALLER_TYPE"] = "apex" self.assertEqual(self.test.run(), testcase.TestCase.EX_RUN_ERROR) @@ -506,7 +498,8 @@ class ODLRunTesting(ODLTesting): def test_netvirt_no_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): os.environ["INSTALLER_TYPE"] = "netvirt" self.assertEqual(self.test.run(), testcase.TestCase.EX_RUN_ERROR) @@ -520,7 +513,8 @@ class ODLRunTesting(ODLTesting): def test_joid_no_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): os.environ["INSTALLER_TYPE"] = "joid" self.assertEqual(self.test.run(), testcase.TestCase.EX_RUN_ERROR) @@ -563,7 +557,7 @@ class ODLArgParserTesting(ODLTesting): self.defaultargs['foo'] = 'bar' with self.assertRaises(SystemExit): self.parser.parse_args(["--foo=bar"]) - mock_method.assert_called_once_with() + self.assertTrue(mock_method.getvalue().startswith("usage:")) def _test_arg(self, arg, value): self.defaultargs[arg] = value diff --git a/run_unit_tests.sh b/run_unit_tests.sh index 917c8eee8..9780de7ac 100755 --- a/run_unit_tests.sh +++ b/run_unit_tests.sh @@ -31,21 +31,10 @@ pip install releng-unittests/modules/ rm -fr releng-unittests export CONFIG_FUNCTEST_YAML=$(pwd)/functest/ci/config_functest.yaml -# unit tests -# TODO: remove cover-erase -# To be deleted when all functest packages will be listed nosetests --with-xunit \ --with-coverage \ - --cover-erase \ --cover-tests \ - --cover-package=functest.ci \ - --cover-package=functest.cli \ - --cover-package=functest.core \ - --cover-package=functest.opnfv_tests.sdn.odl.odl \ - --cover-package=functest.opnfv_tests.vnf.ims \ - --cover-package=functest.utils \ - --cover-package=functest.opnfv_tests.features \ - --cover-package=functest.opnfv_tests.openstack \ + --cover-package=functest \ --cover-xml \ --cover-html \ --log-config=$(pwd)/functest/tests/unit/test_logging.ini \ |