diff options
24 files changed, 575 insertions, 298 deletions
@@ -11,19 +11,24 @@ IRC: Server:freenode.net Channel:#opnfv-functest Repository: functest Committers: -yaohelan@huawei.com -feng.xiaowei@zte.com.cn -ollivier.cedric@gmail.com -jose.lausuch@ericsson.com -morgan.richomme@orange.com -meimei@huawei.com -valentin.boucher@orange.com -lanqinglong@huawei.com -viktor.tikkanen@nokia.com -juha.kosonen@nokia.com -zhanghaoyu7@huawei.com -raghavendrachari.kamsali@hpe.com -lixiaoguang5@huawei.com +Morgan Richomme <morgan.richomme@orange.com> +Jose Lausuch <jose.lausuch@ericsson.com> +Cedric Ollivier <ollivier.cedric@gmail.com> +Helen Yao <yaohelan@huawei.com> +Serena Feng <feng.xiaowei@zte.com.cn> +Juha Kosonen <juha.kosonen@nokia.com> +Valentin Boucher <valentin.boucher@orange.com> +Viktor Tikkanen <viktor.tikkanen@nokia.com> +Mei Mei <meimei@huawei.com> + +Additional contributors: +Linda Wang <wangwulin@huawei.com> +Georgios Paraskevopoulos <georgepar.91@gmail.com> +Romanos Skiadas <rom.skiad@gmail.com> +Michael Polenchuk <mpolenchuk@mirantis.com> +Cristina Pauna <cristina.pauna@enea.com> +Matthew Li <matthew.lijun@huawei.com> +Steven Pisarski <s.pisarski@cablelabs.com> Link to TSC approval of the project: http://meetbot.opnfv.org/meetings/opnfv-meeting/2015/opnfv-meeting.2015-01-20-14.57.html diff --git a/docs/com/pres/framework/framework.html b/docs/com/pres/framework/framework.html new file mode 100644 index 000000000..950c2bebf --- /dev/null +++ b/docs/com/pres/framework/framework.html @@ -0,0 +1,52 @@ +<html> +<head> +<title>OPNFV Functest Framework</title> +<meta name="author" content="Cédric Ollivier"> +<meta name="viewport" + content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no"> +<link rel="stylesheet" href="../reveal.js/css/reveal.css"> +<link rel="stylesheet" href="../reveal.js/css/theme/white.css"> +<link rel="stylesheet" href="../reveal.js/lib/css/zenburn.css"> +<script> +var link = document.createElement( 'link' ); +link.rel = 'stylesheet'; +link.type = 'text/css'; +link.href = window.location.search.match( /print-pdf/gi ) ? '../reveal.js/css/print/pdf.css' : '../reveal.js/css/print/paper.css'; +document.getElementsByTagName( 'head' )[0].appendChild( link ); +</script> +</head> +<body> + <div class="reveal"> + <div class="slides"> + <section data-markdown="framework.md" data-separator="^\n\n\n" + data-separator-vertical="^\n\n" data-separator-notes="^Note:"></section> + </div> + </div> + <script src="../reveal.js/lib/js/head.min.js"></script> + <script src="../reveal.js/js/reveal.js"></script> + <script> + Reveal.initialize({ + dependencies : [ { + src : '../reveal.js/plugin/markdown/marked.js', + condition : function() { + return !!document.querySelector('[data-markdown]'); + } + }, { + src : '../reveal.js/plugin/markdown/markdown.js', + condition : function() { + return !!document.querySelector('[data-markdown]'); + } + }, { + src: '../reveal.js/plugin/highlight/highlight.js', + async: true, + callback: function() { + hljs.initHighlightingOnLoad(); + } + }, { + src: '../reveal.js/plugin/notes/notes.js', + async: true + } ] + }); + </script> +</body> +</html> diff --git a/docs/com/pres/framework/framework.md b/docs/com/pres/framework/framework.md new file mode 100644 index 000000000..b80ad3dd7 --- /dev/null +++ b/docs/com/pres/framework/framework.md @@ -0,0 +1,266 @@ +# Functest Framework + +created by [Cédric Ollivier](mailto:cedric.ollivier@orange.com) + +2017/04/24 + +Note: + +- Functest integrates lots of heteregeounous testcases: + - python vs bash + - internal vs external +- it aims to benefit from object programming + - to define common operations + - to avoid conditional instructions regarding the testcases + - to avoid duplicating code + - to ease the integration of third-party testcases (written in Bash or Python) + + + +## Quick overview + + +### Functest function calls + +- **CI** calls *run_tests.py* (please see [jenkins jobs](https://gerrit.opnfv.org/gerrit/gitweb?p=releng.git;a=tree;f=jjb/functest)) +- *run_tests.py* parses *functest/ci/testcases.yaml* to: + - check which testcase(s) must be run + - execute the common operations on every testcase (run, push its results to db...) +<!-- .element: class="fragment highlight-red"--> + - return the right status code to **CI** + + +### Our target + +- limit run_tests.py instructions by defining: + - the basic testcase attritutes + - all common operations + - the status codes expected +- avoid duplicating codes between testcases +- ease the developpement of third-party testcases (aka features) + + + +## class TestCase + +base model for single test case + + +### instance attributes + +- project_name (default: 'functest') +- case_name +- criteria +- start_time +- stop_time +- details + + +### methods + +| Method | Purpose | +|-------------------|--------------------------------------------| +| run(**kwargs) | run the test case | +| check_criteria() | interpret the results of the test case | +| push_to_db() | push the results of the test case to the DB| + + +### run(**kwargs) + +- the subclasses must override the default implementation which is false on purpose +- the new implementation must set the following attributes to push the results to DB: + - criteria + - start_time + - stop_time + + +### class attributes + +| Status code | Returned when | +|--------------------|---------------------| +| EX_OK | everything is OK | +| EX_RUN_ERROR | run() failed | +| EX_TESTCASE_FAILED | results are false | +| EX_PUSH_TO_DB_ERROR| push_to_db() failed | + + +### run_tests.py + +```python +module = importlib.import_module(run_dict['module']) +cls = getattr(module, run_dict['class']) +test_dict = ft_utils.get_dict_by_test(test_name) +test_case = cls(**test_dict) +try: + kwargs = run_dict['args'] + result = test_case.run(**kwargs) +except KeyError: + result = test_case.run() +if result == testcase.TestCase.EX_OK: + if GlobalVariables.REPORT_FLAG: + test_case.push_to_db() + result = test_case.check_criteria() +``` + + + +## Your first test case + + +### first.py + +```python +#!/usr/bin/env python + +import time + +from functest.core import testcase + +class Test(testcase.TestCase): + + def run(self, **kwargs): + self.start_time = time.time() + print "Hello World" + self.criteria = 'PASS' + self.stop_time = time.time() + return testcase.TestCase.EX_OK +``` + + +### functest/ci/testcases.yaml + +```yaml +case_name: first +project_name: functest +criteria: 'status == "PASS"' +blocking: true +clean_flag: false +description: '' +dependencies: + installer: '' + scenario: '' +run: + module: 'first' + class: 'Test' +``` + + + +## class Feature +bases: TestCase + +base model for single feature + + +### methods + +| Method | Purpose | +|-------------------|---------------------------| +| run(**kwargs) | run the feature | +| execute(**kwargs) | execute the Python method | + + +### run(**kwargs) + +- allows executing any Python method by calling execute() +- sets the following attributes required to push the results to DB: + - criteria + - start_time + - stop_time +- doesn't fulfill details when pushing the results to the DB. + + +### execute(**kwargs) + +- the subclasses must override the default implementation which is false on purpose +- the new implementation must return 0 if success or anything else if failure. + + + +## Your second test case + + +### second.py + +```python +#!/usr/bin/env python + +from functest.core import feature + +class Test(feature.Feature): + + def execute(self, **kwargs): + print "Hello World" + return 0 +``` + + +### functest/ci/testcases.yaml + +```yaml +case_name: second +project_name: functest +criteria: 'status == "PASS"' +blocking: true +clean_flag: false +description: '' +dependencies: + installer: '' + scenario: '' +run: + module: 'second' + class: 'Test' +``` + + + +## class BashFeature +bases: Feature + +class designed to run any bash command + + +### execute(**kwargs) + +execute the cmd passed as arg. + + + +## Your third test case + + +### functest/ci/testcases.yaml + +``` +case_name: third +project_name: functest +criteria: 'status == "PASS"' +blocking: true +clean_flag: false +description: '' +dependencies: + installer: '' + scenario: '' +run: + module: 'functest.core.feature' + class: 'BashFeature' + args: + cmd: 'echo Hello World; exit 0' +``` + + + +## Euphrates + + +### Next actions + +- __to finish VNF abstraction (coverage + pylint)__ +- to publish doc API +- to manage criteria as written in testcases.yaml + +Please see [Functest Euphrates page](https://wiki.opnfv.org/display/functest/Functest+Euphrates+page) for more details + + + +## Thank You! diff --git a/docs/testing/user/configguide/configguide.rst b/docs/testing/user/configguide/configguide.rst index e3485be40..9a1749789 100644 --- a/docs/testing/user/configguide/configguide.rst +++ b/docs/testing/user/configguide/configguide.rst @@ -336,6 +336,7 @@ should now be in place:: |-- parser |-- promise |-- rally + |-- refstack-client |-- releng |-- sdnvpn |-- securityscanning diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index e050cf14b..c1faecdaa 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -247,13 +247,125 @@ The Rally testcases are distributed accross two Tiers: NOTE: Test case 'rally_sanity' executes a limited number of Rally smoke test cases. Test case 'rally_full' executes the full defined set of Rally tests. + +Refstack-client to run Defcore testcases +----------------------------------------- + +Refstack-client `[8]`_ is a command line utility that allows you to +execute Tempest test runs based on configurations you specify. +It is the official tool to run Defcore `[9]`_ testcases, +which focuses on testing interoperability between OpenStack clouds. + +Refstack-client is integrated in Functest, consumed by Dovetail, which +intends to define and provide a set of OPNFV related validation criteria +that will provide input for the evaluation of the use of OPNFV trademarks. +This progress is under the guideline of Compliance Verification Program(CVP). + +Defcore testcases +^^^^^^^^^^^^^^^^^^ + +*Danube Release* + +Set of DefCore tempest test cases not flagged and required. +According to `[10]`_, some tests are still flagged due to outstanding bugs +in the Tempest library, particularly tests that require SSH. Refstack developers +are working on correcting these bugs upstream. Please note that although some tests +are flagged because of bugs, there is still an expectation that the capabilities +covered by the tests are available. It only contains Openstack core compute +(no object storage). The approved guidelines (2016.08) are valid for Kilo, +Liberty, Mitaka and Newton releases of OpenStack. +The list can be generated using the Rest API from RefStack project: +https://refstack.openstack.org/api/v1/guidelines/2016.08/tests?target=compute&type=required&alias=true&flag=false + +Running methods +^^^^^^^^^^^^^^^ + +Two running methods are provided after refstack-client integrated into +Functest, Functest command line and manually, respectively. + +By default, for Defcore test cases run by Functest command line, +are run followed with automatically generated +configuration file, i.e., refstack_tempest.conf. In some circumstances, +the automatic configuration file may not quite satisfied with the SUT, +Functest also inherits the refstack-client command line and provides a way +for users to set its configuration file according to its own SUT manually. + +*command line* + +Inside the Functest container, first to prepare Functest environment: + +:: + + cd /home/opnfv/repos/functest + pip install -e . + functest env prepare + +then to run default defcore testcases by using refstack-client: + +:: + + functest testcase run refstack_defcore + +In OPNFV Continuous Integration(CI) system, the command line method is used. + +*manually* + +Inside the Functest container, first to prepare the refstack virtualenv: + +:: + + cd /home/opnfv/repos/refstack-client + source .venv/bin/activate + +then prepare the tempest configuration file and the testcases want to run with the SUT, +run the testcases with: + +:: + + ./refstack-client test -c <Path of the tempest configuration file to use> -v --test-list <Path or URL of test list> + +using help for more information: + +:: + + ./refstack-client --help + ./refstack-client test --help + +Reference tempest configuration +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +*command line method* + +When command line method is used, the default tempest configuration file +is generated by Rally. + +*manually* + +When running manually is used, recommended way to generate tempest configuration +file is: + +:: + + cd /home/opnfv/repos/functest/functest/opnfv_tests/openstack/refstack_client + python tempest_conf.py + +a file called tempest.conf is stored in the current path by default, users can do +some adjustment according to the SUT: + +:: + + vim refstack_tempest.conf + +a reference article can be used `[15]`_. + + snaps_smoke ------------ This test case contains tests that setup and destroy environments with VMs with and without Floating IPs with a newly created user and project. Set the config value snaps.use_floating_ips (True|False) to toggle this functionality. When -the config value of snaps.use_keystone is True, functest must have access +the config value of snaps.use_keystone is True, Functest must have access the cloud's private network. This suite consists in 38 tests (test duration < 10 minutes) @@ -362,7 +474,7 @@ The test cases are described as follows: Features -------- -In Danube, functest supports the integration of: +In Danube, Functest supports the integration of: * barometer * bgpvpn @@ -520,10 +632,14 @@ References .. _`[5]`: https://github.com/Orange-OpenSource/opnfv-cloudify-clearwater/blob/master/openstack-blueprint.yaml .. _`[6]`: https://scap.nist.gov/ .. _`[7]`: https://github.com/OpenSCAP/openscap +.. _`[8]`: https://github.com/openstack/refstack-client +.. _`[9]`: https://github.com/openstack/defcore +.. _`[10]`: https://github.com/openstack/interop/blob/master/2016.08/procedure.rst .. _`[11]`: http://robotframework.org/ .. _`[12]`: http://artifacts.opnfv.org/parser/colorado/docs/userguide/index.html .. _`[13]`: https://wiki.opnfv.org/display/PROJ/SNAPS-OO .. _`[14]`: https://github.com/oolorg/opnfv-functest-vrouter +.. _`[15]`: https://aptira.com/testing-openstack-tempest-part-1/ `OPNFV main site`_ diff --git a/functest/ci/check_os.sh b/functest/ci/check_os.sh index 83f9f476d..ce0bc20c6 100755 --- a/functest/ci/check_os.sh +++ b/functest/ci/check_os.sh @@ -86,30 +86,6 @@ if [ $RETVAL -ne 0 ]; then fi echo " ...OK" -adminURL=$(openstack catalog show identity |awk '/admin/ {print $4}') -if [ -z ${adminURL} ]; then - echo "ERROR: Cannot determine the admin URL." - openstack catalog show identity - exit 1 -fi -adminIP=$(echo $adminURL|sed 's/^.*http.*\:\/\///'|sed 's/.[^:]*$//') -adminPort=$(echo $adminURL|grep -Po '(?<=:)\d+') -https_enabled=$(echo $adminURL | grep 'https') -if [[ -n $https_enabled ]]; then - echo ">>Verifying SSL connectivity to the admin endpoint $adminIP:$adminPort..." - verify_SSL_connectivity $adminIP $adminPort -else - echo ">>Verifying connectivity to the admin endpoint $adminIP:$adminPort..." - verify_connectivity $adminIP $adminPort -fi -RETVAL=$? -if [ $RETVAL -ne 0 ]; then - echo "ERROR: Cannot talk to the admin endpoint $adminIP:$adminPort ." - echo "$adminURL" - exit 1 -fi -echo " ...OK" - echo "Checking Required OpenStack services:" for service in $MANDATORY_SERVICES; do diff --git a/functest/ci/config_functest.yaml b/functest/ci/config_functest.yaml index a9ad15cab..f291cf1f6 100644 --- a/functest/ci/config_functest.yaml +++ b/functest/ci/config_functest.yaml @@ -125,7 +125,7 @@ rally: router_name: rally-router refstack: - tempest_conf_path: openstack/refstack_client/tempest.conf + tempest_conf_path: openstack/refstack_client/refstack_tempest.conf defcore_list: openstack/refstack_client/defcore.txt vnf: diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 8f2cc4bc2..7009e910c 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -567,7 +567,7 @@ tiers: class: 'ImsVnf' - - case_name: opera-vims + case_name: opera_vims project_name: opera criteria: 100 blocking: false @@ -575,8 +575,8 @@ tiers: description: >- VNF deployment with OPEN-O dependencies: - installer: 'unknown' - scenario: 'unknown' + installer: 'compass' + scenario: 'os-nosdn-openo-ha' run: module: 'functest.opnfv_tests.vnf.ims.opera_ims' class: 'OperaIms' diff --git a/functest/opnfv_tests/openstack/examples/create_instance_and_ip.py b/functest/opnfv_tests/openstack/examples/create_instance_and_ip.py deleted file mode 100755 index b44008642..000000000 --- a/functest/opnfv_tests/openstack/examples/create_instance_and_ip.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/python -# -# Copyright (c) 2015 All rights reserved -# This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# This script boots an instance and assigns a floating ip -# - -import argparse -import os -import sys - -from functest.utils.constants import CONST -import functest.utils.functest_logger as ft_logger -import functest.utils.openstack_utils as os_utils - -parser = argparse.ArgumentParser() - -parser.add_argument("-r", "--report", - help="Create json result file", - action="store_true") - -args = parser.parse_args() - -""" logging configuration """ -logger = ft_logger.Logger("create_instance_and_ip").getLogger() - -HOME = CONST.dir_home + "/" - -VM_BOOT_TIMEOUT = 180 - -EXAMPLE_INSTANCE_NAME = CONST.example_vm_name -EXAMPLE_FLAVOR = CONST.example_flavor -EXAMPLE_IMAGE_NAME = CONST.example_image_name -IMAGE_FILENAME = CONST.openstack_image_file_name -IMAGE_FORMAT = CONST.openstack_image_disk_format -IMAGE_PATH = os.path.join(CONST.dir_functest_data, IMAGE_FILENAME) - -# NEUTRON Private Network parameters - -EXAMPLE_PRIVATE_NET_NAME = CONST.example_private_net_name -EXAMPLE_PRIVATE_SUBNET_NAME = CONST.example_private_subnet_name -EXAMPLE_PRIVATE_SUBNET_CIDR = CONST.example_private_subnet_cidr -EXAMPLE_ROUTER_NAME = CONST.example_router_name - -EXAMPLE_SECGROUP_NAME = CONST.example_sg_name -EXAMPLE_SECGROUP_DESCR = CONST.example_sg_desc - - -def main(): - - nova_client = os_utils.get_nova_client() - neutron_client = os_utils.get_neutron_client() - glance_client = os_utils.get_glance_client() - - image_id = os_utils.create_glance_image(glance_client, - EXAMPLE_IMAGE_NAME, - IMAGE_PATH, - disk=IMAGE_FORMAT, - container="bare", - public=True) - - network_dic = os_utils.create_network_full( - neutron_client, - EXAMPLE_PRIVATE_NET_NAME, - EXAMPLE_PRIVATE_SUBNET_NAME, - EXAMPLE_ROUTER_NAME, - EXAMPLE_PRIVATE_SUBNET_CIDR) - if not network_dic: - logger.error( - "There has been a problem when creating the neutron network") - sys.exit(-1) - - network_id = network_dic["net_id"] - - sg_id = os_utils.create_security_group_full(neutron_client, - EXAMPLE_SECGROUP_NAME, - EXAMPLE_SECGROUP_DESCR) - - # boot INTANCE - logger.info("Creating instance '%s'..." % EXAMPLE_INSTANCE_NAME) - logger.debug( - "Configuration:\n name=%s \n flavor=%s \n image=%s \n " - "network=%s \n" - % (EXAMPLE_INSTANCE_NAME, EXAMPLE_FLAVOR, image_id, network_id)) - instance = os_utils.create_instance_and_wait_for_active( - EXAMPLE_FLAVOR, - image_id, - network_id, - EXAMPLE_INSTANCE_NAME) - - if instance is None: - logger.error("Error while booting instance.") - sys.exit(-1) - # Retrieve IP of INSTANCE - instance_ip = instance.networks.get(EXAMPLE_PRIVATE_NET_NAME)[0] - logger.debug("Instance '%s' got private ip '%s'." % - (EXAMPLE_INSTANCE_NAME, instance_ip)) - - logger.info("Adding '%s' to security group '%s'..." - % (EXAMPLE_INSTANCE_NAME, EXAMPLE_SECGROUP_NAME)) - os_utils.add_secgroup_to_instance(nova_client, instance.id, sg_id) - - logger.info("Creating floating IP for VM '%s'..." % EXAMPLE_INSTANCE_NAME) - floatip_dic = os_utils.create_floating_ip(neutron_client) - floatip = floatip_dic['fip_addr'] - # floatip_id = floatip_dic['fip_id'] - - if floatip is None: - logger.error("Cannot create floating IP.") - sys.exit(-1) - logger.info("Floating IP created: '%s'" % floatip) - - logger.info("Associating floating ip: '%s' to VM '%s' " - % (floatip, EXAMPLE_INSTANCE_NAME)) - if not os_utils.add_floating_ip(nova_client, instance.id, floatip): - logger.error("Cannot associate floating IP to VM.") - sys.exit(-1) - - sys.exit(0) - - -if __name__ == '__main__': - main() diff --git a/functest/opnfv_tests/openstack/refstack_client/defcore.txt b/functest/opnfv_tests/openstack/refstack_client/defcore.txt index be8fd8998..0a1787ef3 100644 --- a/functest/opnfv_tests/openstack/refstack_client/defcore.txt +++ b/functest/opnfv_tests/openstack/refstack_client/defcore.txt @@ -1,4 +1,11 @@ -# Set of DefCore tempest test cases not flagged and required. It only contains OpenStack core (no object storage) +# Set of DefCore tempest test cases not flagged and required. +# According to https://github.com/openstack/interop/blob/master/2016.08/procedure.rst, +# some tests are still flagged due to outstanding bugs in the Tempest library, +# particularly tests that require SSH. Refstack developers +# are working on correcting these bugs upstream. Please note that although some tests +# are flagged because of bugs, there is still an expectation that the capabilities +# covered by the tests are available. +# It only contains Openstack core compute (no object storage) # The approved guidelines (2016.08) are valid for Kilo, Liberty, Mitaka and Newton releases of OpenStack # The list can be generated using the Rest API from RefStack project: # https://refstack.openstack.org/api/v1/guidelines/2016.08/tests?target=compute&type=required&alias=true&flag=false diff --git a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py index 7aff251b9..2f2fc00f7 100755 --- a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py +++ b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py @@ -171,7 +171,7 @@ class RefstackClient(testcase.TestCase): '''used for manually running, python refstack_client.py -c <tempest_conf_path> --testlist <testlist_path> - can generate a reference tempest.conf by + can generate a reference refstack_tempest.conf by python tempest_conf.py ''' try: @@ -206,7 +206,7 @@ class RefstackClientParser(object): self.parser = argparse.ArgumentParser() self.parser.add_argument( '-c', '--config', - help='the file path of tempest.conf', + help='the file path of refstack_tempest.conf', default=self.confpath) self.parser.add_argument( '-t', '--testlist', diff --git a/functest/opnfv_tests/openstack/snaps/health_check.py b/functest/opnfv_tests/openstack/snaps/health_check.py index 44e3b876b..c057eb2b0 100644 --- a/functest/opnfv_tests/openstack/snaps/health_check.py +++ b/functest/opnfv_tests/openstack/snaps/health_check.py @@ -30,8 +30,7 @@ class HealthCheck(SnapsTestRunner): image_custom_config = None if hasattr(CONST, 'snaps_health_check'): - image_custom_config = CONST.snaps_health_check - + image_custom_config = CONST.__getattribute__('snaps_health_check') self.suite.addTest( OSIntegrationTestCase.parameterize( SimpleHealthCheck, os_creds=self.os_creds, diff --git a/functest/opnfv_tests/openstack/snaps/smoke.py b/functest/opnfv_tests/openstack/snaps/smoke.py index 5a637f288..2c6fc2553 100644 --- a/functest/opnfv_tests/openstack/snaps/smoke.py +++ b/functest/opnfv_tests/openstack/snaps/smoke.py @@ -32,12 +32,13 @@ class SnapsSmoke(SnapsTestRunner): # snaps_health_check suite, so reuse it here image_custom_config = None if hasattr(CONST, 'snaps_health_check'): - image_custom_config = CONST.snaps_health_check + image_custom_config = CONST.__getattribute__('snaps_health_check') # Tests requiring floating IPs leverage files contained within the # SNAPS repository and are found relative to that path if self.use_fip: - snaps_dir = CONST.dir_repo_snaps + '/snaps' + snaps_dir = os.path.join(CONST.__getattribute__('dir_repo_snaps'), + 'snaps') os.chdir(snaps_dir) test_suite_builder.add_openstack_integration_tests( diff --git a/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py b/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py index b17aab0c0..8a68cad9e 100644 --- a/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py +++ b/functest/opnfv_tests/openstack/snaps/snaps_test_runner.py @@ -24,16 +24,16 @@ class SnapsTestRunner(PyTestSuiteRunner): super(SnapsTestRunner, self).__init__(**kwargs) self.os_creds = openstack_tests.get_credentials( - os_env_file=CONST.openstack_creds, proxy_settings_str=None, - ssh_proxy_cmd=None) + os_env_file=CONST.__getattribute__('openstack_creds'), + proxy_settings_str=None, ssh_proxy_cmd=None) self.ext_net_name = snaps_utils.get_ext_net_name(self.os_creds) - self.use_fip = CONST.snaps_use_floating_ips - self.use_keystone = CONST.snaps_use_keystone + self.use_fip = CONST.__getattribute__('snaps_use_floating_ips') + self.use_keystone = CONST.__getattribute__('snaps_use_keystone') scenario = functest_utils.get_scenario() self.flavor_metadata = create_flavor.MEM_PAGE_SIZE_ANY if 'ovs' in scenario or 'fdio' in scenario: self.flavor_metadata = create_flavor.MEM_PAGE_SIZE_LARGE - self.logger.info("Using flavor metatdata '%s'" % self.flavor_metadata) + self.logger.info("Using flavor metadata '%s'", self.flavor_metadata) diff --git a/functest/opnfv_tests/sdn/onos/onos.py b/functest/opnfv_tests/sdn/onos/onos.py index cbe1b9ebe..4d489d674 100644 --- a/functest/opnfv_tests/sdn/onos/onos.py +++ b/functest/opnfv_tests/sdn/onos/onos.py @@ -21,16 +21,21 @@ import functest.utils.functest_utils as ft_utils import functest.utils.openstack_utils as openstack_utils -logger = ft_logger.Logger(__name__).getLogger() - - class OnosBase(testcase.TestCase): - onos_repo_path = CONST.dir_repo_onos - onos_sfc_image_name = CONST.onos_sfc_image_name - onos_sfc_image_path = os.path.join(CONST.dir_functest_data, - CONST.onos_sfc_image_file_name) - onos_sfc_path = os.path.join(CONST.dir_repo_functest, - CONST.dir_onos_sfc) + onos_repo_path = CONST.__getattribute__('dir_repo_onos') + onos_sfc_image_name = CONST.__getattribute__('onos_sfc_image_name') + onos_sfc_image_path = os.path.join( + CONST.__getattribute__('dir_functest_data'), + CONST.__getattribute__('onos_sfc_image_file_name')) + onos_sfc_path = os.path.join(CONST.__getattribute__('dir_repo_functest'), + CONST.__getattribute__('dir_onos_sfc')) + installer_type = CONST.__getattribute__('INSTALLER_TYPE') + logger = ft_logger.Logger(__name__).getLogger() + + def __init__(self, **kwargs): + if "case_name" not in kwargs: + kwargs["case_name"] = "onos_base" + super(OnosBase, self).__init__(**kwargs) def run(self): self.start_time = time.time() @@ -38,7 +43,7 @@ class OnosBase(testcase.TestCase): self._run() res = testcase.TestCase.EX_OK except Exception as e: - logger.error('Error with run: %s', e) + self.logger.error('Error with run: %s', e) res = testcase.TestCase.EX_RUN_ERROR self.stop_time = time.time() @@ -56,20 +61,20 @@ class Onos(OnosBase): self.log_path = os.path.join(self.onos_repo_path, 'TestON/logs') def set_onos_ip(self): - if (CONST.INSTALLER_TYPE and - CONST.INSTALLER_TYPE.lower() == 'joid'): + if (self.installer_type and + self.installer_type.lower() == 'joid'): sdn_controller_env = os.getenv('SDN_CONTROLLER') OC1 = re.search(r"\d+\.\d+\.\d+\.\d+", sdn_controller_env).group() else: neutron_url = openstack_utils.get_endpoint(service_type='network') OC1 = urlparse.urlparse(neutron_url).hostname os.environ['OC1'] = OC1 - logger.debug("ONOS IP is %s" % OC1) + self.logger.debug("ONOS IP is %s", OC1) def run_onos_script(self, testname): cli_dir = os.path.join(self.onos_repo_path, 'TestON/bin/cli.py') cmd = '{0} run {1}'.format(cli_dir, testname) - logger.debug("Run script: %s" % testname) + self.logger.debug("Run script: %s", testname) ft_utils.execute_command_raise( cmd, error_msg=('Error when running ONOS script: %s' @@ -84,8 +89,8 @@ class Onos(OnosBase): elif os.path.isfile(log): os.remove(log) except OSError as e: - logger.error('Error with deleting file %s: %s', - log, e.strerror) + self.logger.error('Error with deleting file %s: %s', + log, e.strerror) def get_result(self): cmd = 'grep -rnh Fail {0}'.format(self.log_path) @@ -95,9 +100,9 @@ class Onos(OnosBase): stderr=subprocess.STDOUT) for line in p.stdout: - logger.debug(line) + self.logger.debug(line) if re.search("\s+[1-9]+\s+", line): - logger.debug("Testcase Fails\n" + line) + self.logger.debug("Testcase Fails\n" + line) cmd = "grep -rnh 'Execution Time' {0}".format(self.log_path) result_buffer = os.popen(cmd).read() @@ -155,8 +160,8 @@ class Onos(OnosBase): if (result['FUNCvirNet']['result'] == "Success" and result['FUNCvirNetL3']['result'] == "Success"): status = "PASS" - except: - logger.error("Unable to set ONOS result") + except Exception: + self.logger.error("Unable to set ONOS result") self.result = status self.details = result @@ -170,13 +175,14 @@ class Onos(OnosBase): class OnosSfc(OnosBase): - def __init__(self): - super(OnosSfc, self).__init__() - self.case_name = 'onos_sfc' + def __init__(self, **kwargs): + if "case_name" not in kwargs: + kwargs["case_name"] = "onos_sfc" + super(OnosSfc, self).__init__(**kwargs) - def get_ip(type): + def get_ip(self, type): url = openstack_utils.get_endpoint(service_type=type) - logger.debug('get_ip for %s: %s' % (type, url)) + self.logger.debug('get_ip for %s: %s', type, url) return urlparse.urlparse(url).hostname def update_sfc_onos_file(self, before, after): @@ -190,6 +196,7 @@ class OnosSfc(OnosBase): % (before, after))) def create_image(self): + self.logger.warn('inside create_image') glance_client = openstack_utils.get_glance_client() image_id = openstack_utils.create_glance_image( glance_client, @@ -198,19 +205,20 @@ class OnosSfc(OnosBase): if image_id is None: raise Exception('Failed to create image') - logger.debug("Image '%s' with ID=%s is created successfully." - % (self.onos_sfc_image_name, image_id)) + self.logger.debug("Image '%s' with ID=%s is created successfully.", + self.onos_sfc_image_name, image_id) def set_sfc_conf(self): self.update_sfc_onos_file("keystone_ip", self.get_ip("keystone")) self.update_sfc_onos_file("neutron_ip", self.get_ip("neutron")) self.update_sfc_onos_file("nova_ip", self.get_ip("nova")) self.update_sfc_onos_file("glance_ip", self.get_ip("glance")) - self.update_sfc_onos_file("console", CONST.OS_PASSWORD) + self.update_sfc_onos_file("console", + CONST.__getattribute__('OS_PASSWORD')) neutron_client = openstack_utils.get_neutron_client() ext_net = openstack_utils.get_external_net(neutron_client) self.update_sfc_onos_file("admin_floating_net", ext_net) - logger.debug("SFC configuration is modified") + self.logger.debug("SFC configuration is modified") def sfc_test(self): cmd = 'python {0}'.format(os.path.join(self.onos_sfc_path, 'sfc.py')) diff --git a/functest/tests/unit/odl/test_odl.py b/functest/tests/unit/odl/test_odl.py index 804693463..54d6da72c 100644 --- a/functest/tests/unit/odl/test_odl.py +++ b/functest/tests/unit/odl/test_odl.py @@ -113,7 +113,8 @@ class ODLParseResultTesting(ODLTesting): def test_raises_exc(self, mock_method): with self.assertRaises(DataError): self.test.parse_results() - mock_method.assert_called_once_with() + mock_method.assert_called_once_with( + os.path.join(odl.ODLTests.res_dir, 'output.xml')) def test_ok(self): config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000', @@ -368,19 +369,10 @@ class ODLRunTesting(ODLTesting): """The class testing ODLTests.run().""" # pylint: disable=missing-docstring - @classmethod - def _fake_url_for(cls, service_type='identity'): - if service_type == 'identity': - return "http://{}:5000/v2.0".format( - ODLTesting._keystone_ip) - elif service_type == 'network': - return "http://{}:9696".format(ODLTesting._neutron_ip) - else: - return None - def _test_no_env_var(self, var): with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): del os.environ[var] self.assertEqual(self.test.run(), testcase.TestCase.EX_RUN_ERROR) @@ -393,7 +385,8 @@ class ODLRunTesting(ODLTesting): if 'odlrestconfport' in kwargs else '8181') with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): if exception: self.test.main = mock.Mock(side_effect=exception) else: @@ -410,18 +403,15 @@ class ODLRunTesting(ODLTesting): osusername=self._os_username) def _test_multiple_suites(self, suites, - status=testcase.TestCase.EX_OK, - exception=None, **kwargs): + status=testcase.TestCase.EX_OK, **kwargs): odlip = kwargs['odlip'] if 'odlip' in kwargs else '127.0.0.3' odlwebport = kwargs['odlwebport'] if 'odlwebport' in kwargs else '8080' odlrestconfport = (kwargs['odlrestconfport'] if 'odlrestconfport' in kwargs else '8181') with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): - if exception: - self.test.main = mock.Mock(side_effect=exception) - else: - self.test.main = mock.Mock(return_value=status) + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): + self.test.main = mock.Mock(return_value=status) self.assertEqual(self.test.run(suites=suites), status) self.test.main.assert_called_once_with( suites, @@ -467,7 +457,8 @@ class ODLRunTesting(ODLTesting): def test_no_sdn_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): self.assertEqual(self.test.run(), testcase.TestCase.EX_RUN_ERROR) @@ -492,7 +483,8 @@ class ODLRunTesting(ODLTesting): def test_apex_no_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): os.environ["INSTALLER_TYPE"] = "apex" self.assertEqual(self.test.run(), testcase.TestCase.EX_RUN_ERROR) @@ -506,7 +498,8 @@ class ODLRunTesting(ODLTesting): def test_netvirt_no_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): os.environ["INSTALLER_TYPE"] = "netvirt" self.assertEqual(self.test.run(), testcase.TestCase.EX_RUN_ERROR) @@ -520,7 +513,8 @@ class ODLRunTesting(ODLTesting): def test_joid_no_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', - side_effect=self._fake_url_for): + return_value="http://{}:9696".format( + ODLTesting._neutron_ip)): os.environ["INSTALLER_TYPE"] = "joid" self.assertEqual(self.test.run(), testcase.TestCase.EX_RUN_ERROR) @@ -563,7 +557,7 @@ class ODLArgParserTesting(ODLTesting): self.defaultargs['foo'] = 'bar' with self.assertRaises(SystemExit): self.parser.parse_args(["--foo=bar"]) - mock_method.assert_called_once_with() + self.assertTrue(mock_method.getvalue().startswith("usage:")) def _test_arg(self, arg, value): self.defaultargs[arg] = value diff --git a/functest/tests/unit/utils/test_openstack_utils.py b/functest/tests/unit/utils/test_openstack_utils.py index 7f3995d03..a7df264c9 100644 --- a/functest/tests/unit/utils/test_openstack_utils.py +++ b/functest/tests/unit/utils/test_openstack_utils.py @@ -418,21 +418,45 @@ class OSUtilsTesting(unittest.TestCase): mock_logger_info.assert_called_once_with("OS_IDENTITY_API_VERSION is " "set in env as '%s'", '3') - def test_get_keystone_client(self): + @mock.patch('functest.utils.openstack_utils.get_session') + @mock.patch('functest.utils.openstack_utils.keystoneclient.Client') + @mock.patch('functest.utils.openstack_utils.get_keystone_client_version', + return_value='3') + @mock.patch('functest.utils.openstack_utils.os.getenv', + return_value='public') + def test_get_keystone_client_with_interface(self, mock_os_getenv, + mock_keystoneclient_version, + mock_key_client, + mock_get_session): mock_keystone_obj = mock.Mock() mock_session_obj = mock.Mock() - with mock.patch('functest.utils.openstack_utils' - '.get_keystone_client_version', return_value='3'), \ - mock.patch('functest.utils.openstack_utils' - '.keystoneclient.Client', - return_value=mock_keystone_obj) \ - as mock_key_client, \ - mock.patch('functest.utils.openstack_utils.get_session', - return_value=mock_session_obj): - self.assertEqual(openstack_utils.get_keystone_client(), - mock_keystone_obj) - mock_key_client.assert_called_once_with('3', - session=mock_session_obj) + mock_key_client.return_value = mock_keystone_obj + mock_get_session.return_value = mock_session_obj + self.assertEqual(openstack_utils.get_keystone_client(), + mock_keystone_obj) + mock_key_client.assert_called_once_with('3', + session=mock_session_obj, + interface='public') + + @mock.patch('functest.utils.openstack_utils.get_session') + @mock.patch('functest.utils.openstack_utils.keystoneclient.Client') + @mock.patch('functest.utils.openstack_utils.get_keystone_client_version', + return_value='3') + @mock.patch('functest.utils.openstack_utils.os.getenv', + return_value='admin') + def test_get_keystone_client_no_interface(self, mock_os_getenv, + mock_keystoneclient_version, + mock_key_client, + mock_get_session): + mock_keystone_obj = mock.Mock() + mock_session_obj = mock.Mock() + mock_key_client.return_value = mock_keystone_obj + mock_get_session.return_value = mock_session_obj + self.assertEqual(openstack_utils.get_keystone_client(), + mock_keystone_obj) + mock_key_client.assert_called_once_with('3', + session=mock_session_obj, + interface='admin') @mock.patch('functest.utils.openstack_utils.os.getenv', return_value=None) diff --git a/functest/utils/openstack/cinder.py b/functest/utils/openstack/cinder.py deleted file mode 100644 index f966468a3..000000000 --- a/functest/utils/openstack/cinder.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# diff --git a/functest/utils/openstack/glance.py b/functest/utils/openstack/glance.py deleted file mode 100644 index f966468a3..000000000 --- a/functest/utils/openstack/glance.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# diff --git a/functest/utils/openstack/keystone.py b/functest/utils/openstack/keystone.py deleted file mode 100644 index f966468a3..000000000 --- a/functest/utils/openstack/keystone.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# diff --git a/functest/utils/openstack/neutron.py b/functest/utils/openstack/neutron.py deleted file mode 100644 index f966468a3..000000000 --- a/functest/utils/openstack/neutron.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# diff --git a/functest/utils/openstack/nova.py b/functest/utils/openstack/nova.py deleted file mode 100644 index f966468a3..000000000 --- a/functest/utils/openstack/nova.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# diff --git a/functest/utils/openstack_utils.py b/functest/utils/openstack_utils.py index 702acc025..3b83c3f0c 100644 --- a/functest/utils/openstack_utils.py +++ b/functest/utils/openstack_utils.py @@ -206,7 +206,9 @@ def get_keystone_client_version(): def get_keystone_client(other_creds={}): sess = get_session(other_creds) - return keystoneclient.Client(get_keystone_client_version(), session=sess) + return keystoneclient.Client(get_keystone_client_version(), + session=sess, + interface=os.getenv('OS_INTERFACE', 'admin')) def get_nova_client_version(): diff --git a/run_unit_tests.sh b/run_unit_tests.sh index 917c8eee8..9780de7ac 100755 --- a/run_unit_tests.sh +++ b/run_unit_tests.sh @@ -31,21 +31,10 @@ pip install releng-unittests/modules/ rm -fr releng-unittests export CONFIG_FUNCTEST_YAML=$(pwd)/functest/ci/config_functest.yaml -# unit tests -# TODO: remove cover-erase -# To be deleted when all functest packages will be listed nosetests --with-xunit \ --with-coverage \ - --cover-erase \ --cover-tests \ - --cover-package=functest.ci \ - --cover-package=functest.cli \ - --cover-package=functest.core \ - --cover-package=functest.opnfv_tests.sdn.odl.odl \ - --cover-package=functest.opnfv_tests.vnf.ims \ - --cover-package=functest.utils \ - --cover-package=functest.opnfv_tests.features \ - --cover-package=functest.opnfv_tests.openstack \ + --cover-package=functest \ --cover-xml \ --cover-html \ --log-config=$(pwd)/functest/tests/unit/test_logging.ini \ |