diff options
58 files changed, 2333 insertions, 1874 deletions
diff --git a/docker/Dockerfile b/docker/Dockerfile index 4c0995b97..a2033a49a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -92,6 +92,7 @@ RUN git clone --depth 1 -b $BRANCH https://gerrit.opnfv.org/gerrit/sfc ${REPOS_D RUN git clone --depth 1 -b $BRANCH https://gerrit.opnfv.org/gerrit/snaps ${REPOS_DIR}/snaps RUN git clone --depth 1 https://gerrit.opnfv.org/gerrit/securityscanning ${REPOS_DIR}/securityscanning RUN git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng ${REPOS_DIR}/releng +RUN git clone --depth 1 https://gerrit.opnfv.org/gerrit/opera ${REPOS_DIR}/opera # OpenStack repositories RUN git clone --depth 1 -b $OPENSTACK_TAG https://github.com/openstack/networking-bgpvpn ${REPOS_DIR}/bgpvpn @@ -119,6 +120,8 @@ RUN cd ${RELENG_MODULE_DIR} \ RUN cd ${REPOS_DIR}/barometer \ && pip install . +RUN pip install ${REPOS_DIR}/opera + RUN find ${FUNCTEST_REPO_DIR} -name "*.py" \ -not -path "*tests/unit*" \ -not -path "*functest_venv*" \ diff --git a/docker/Dockerfile.aarch64 b/docker/Dockerfile.aarch64 index 17bf00027..83df2d0cb 100644 --- a/docker/Dockerfile.aarch64 +++ b/docker/Dockerfile.aarch64 @@ -88,6 +88,7 @@ RUN git clone --depth 1 -b $BRANCH https://gerrit.opnfv.org/gerrit/sfc ${REPOS_D RUN git clone --depth 1 -b $BRANCH https://gerrit.opnfv.org/gerrit/snaps ${REPOS_DIR}/snaps RUN git clone --depth 1 https://gerrit.opnfv.org/gerrit/securityscanning ${REPOS_DIR}/securityscanning RUN git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng ${REPOS_DIR}/releng +RUN git clone --depth 1 https://gerrit.opnfv.org/gerrit/opera ${REPOS_DIR}/opera # OpenStack repositories RUN git clone --depth 1 -b $OPENSTACK_TAG https://github.com/openstack/networking-bgpvpn ${REPOS_DIR}/bgpvpn @@ -138,6 +139,8 @@ RUN curl -L https://get.rvm.io | bash -s stable # SNAPS integration RUN pip install -e ${REPOS_DIR}/snaps/ +RUN pip install ${REPOS_DIR}/opera + # SFC integration RUN /bin/bash -c ". ${REPOS_DIR}/sfc/sfc/tests/functest/setup_scripts/tacker_client_install.sh" RUN cd ${REPOS_DIR}/sfc && pip install -e . diff --git a/docs/images/FunctestDashboardDanube.png b/docs/images/FunctestDashboardDanube.png Binary files differnew file mode 100644 index 000000000..b934d7e8e --- /dev/null +++ b/docs/images/FunctestDashboardDanube.png diff --git a/docs/images/concepts_mapping_final.png b/docs/images/concepts_mapping_final.png Binary files differindex 5ad0fc569..91d8fa1a5 100644 --- a/docs/images/concepts_mapping_final.png +++ b/docs/images/concepts_mapping_final.png diff --git a/docs/images/functest_overview.odg b/docs/images/functest_overview.odg Binary files differnew file mode 100644 index 000000000..9f6fe345f --- /dev/null +++ b/docs/images/functest_overview.odg diff --git a/docs/release/release-notes/functest-release.rst b/docs/release/release-notes/functest-release.rst index ce03047f6..423b9f9b8 100644 --- a/docs/release/release-notes/functest-release.rst +++ b/docs/release/release-notes/functest-release.rst @@ -1,17 +1,20 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 + ======= License ======= -OPNFV Colorado release note for Functest Docs -(c) by Morgan Richomme (Orange) +OPNFV Danube release note for Functest Docs +(c) by Jose Lausuch (Ericsson) -OPNFV Colorado release note for Functest Docs +OPNFV Danube release note for Functest Docs are licensed under a Creative Commons Attribution 4.0 International License. You should have received a copy of the license along with this. If not, see <http://creativecommons.org/licenses/by/4.0/>. =========================================== -OPNFV Colorado1.0 release note for Functest +OPNFV Danube1.0 release note for Functest =========================================== Abstract @@ -30,8 +33,11 @@ Version history | 2016-08-17 | 1.0.0 | Morgan Richomme | Functest for Colorado | | | | (Orange) | release | +------------+----------+------------------+------------------------+ +| 2017-03-27 | 4.0.0 | Jose Lausuch | Functest for Danube | +| | | (Ericsson) | release | ++------------+----------+------------------+------------------------+ -OPNFV Colorado Release +OPNFV Danube Release ====================== Functest deals with functional testing of the OPNFV solution. @@ -40,36 +46,44 @@ other OPNFV projects and other upstream communities. The internal test cases are: - * healthcheck - * vPing ssh - * vPing userdata - * Tempest Smoke Serial - * Rally Sanity - * ODL - * Tempest full parallel - * Rally full - * vIMS + * connection_check + * api_check + * snaps_health_check + * vping_ssh + * vping_userdata + * tempest_smoke_serial + * refstack_defcore + * snaps_smoke + * rally_sanity + * odl + * tempest_full_parallel + * rally_full + * cloudify_ims The OPNFV projects integrated into Functest framework for automation are: + * barometer * bgpvpn - * Copper - * Doctor - * Domino - * Moon - * Multisite - * ONOSFW - * ONOS-sfc - * ODL-sfc - * Parser - * Promise - * Security scan + * doctor + * domino + * fds + * multisite + * netready + * onos + * odl-sfc + * odl-netvirt + * orchestra_ims + * parser + * promise + * security scan + * vyos_vrouter The validation of a scenario requires a subset of these tests depending on the installer and the scenario. -The 3 last internal test cases (tempest full parallel, Rally full and -vIMS) are not considered for scenario validation. +The test cases from vnf (cloudify_ims, orchestra_ims, vyos_vrouter) and +component categories (tempest full parallel, Rally full) are not considered for +scenario validation. Release Data ============ @@ -78,16 +92,16 @@ Release Data | **Project** | functest | | | | +--------------------------------------+--------------------------------------+ -| **Repo/tag** | colorado.1.0 | +| **Repo/tag** | danube.1.0 | | | | +--------------------------------------+--------------------------------------+ -| **Release designation** | Colorado base release | +| **Release designation** | Danube base release | | | | +--------------------------------------+--------------------------------------+ -| **Release date** | September 22 2016 | +| **Release date** | April 3rd 2017 | | | | +--------------------------------------+--------------------------------------+ -| **Purpose of the delivery** | Colorado base release | +| **Purpose of the delivery** | Danube base release | | | | +--------------------------------------+--------------------------------------+ @@ -97,19 +111,19 @@ Deliverables Software -------- - - The Functest Docker image: https://hub.docker.com/r/opnfv/functest (tag: colorado.1.0) + - The Functest Docker image: https://hub.docker.com/r/opnfv/functest (tag: danube.1.0) - - The TestAPI Docker image: https://hub.docker.com/r/opnfv/testapi (tag:colorado.1.0) + - The TestAPI Docker image: https://hub.docker.com/r/opnfv/testapi (tag:danube.1.0) Documents --------- - - Installation/configuration guide: http://artifacts.opnfv.org/functest/colorado/docs/configguide/index.html + - Installation/configuration guide: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/configguide/index.html - - User Guide: http://artifacts.opnfv.org/functest/colorado/docs/userguide/index.html + - User Guide: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/userguide/index.html - - Developer Guide: http://artifacts.opnfv.org/functest/colorado/docs/devguide/index.html + - Developer Guide: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/developer/devguide/index.html Version change @@ -118,137 +132,140 @@ Version change Feature evolution ----------------- - - refactoring of ODL functional tests (with upstream modifications) +- Adoption of SNAPS as middleware in 4 new test cases (connection_check, api_check, +snaps_health_check and snaps_smoke) + +- Introduction of refstack suite + +- Support new odl suites (odl-netvirt, fds) + +- Introduction of VNF onboarding capabilities + +- Support of new feature projects (fds, netready, barometer, orchestra, vyos_vrouter) - - refactoring of TestAPI (update, swagger documentation, dockerization) - - jenkins logs improvement - - update integration of Doctor, Promise and SDNVPN projects +Framework +--------- - - split Tempest and rally into 2 different tests: smoke and full + - Harmonization of the naming, better adoption of OpenStack coding conventions - - vIMS test suite integration + - Pythonization, rewritting of bash scripts - - adoption of Kibana for dashboarding + - Introduction of abstraction classes to ease and harmonize the integration of + test cases (internal or from feature projects) + - New management of logger, env variables and configuration files -New features ------------- + - Creation of unit tests on the whole framework to ensure stability - - Functest CLI to prepare and run the tests + - Creation or ARM Functest docker - - creation of the healthcheck test case - - support new scenarios (ocl, odl_l2-sfc, onos-sfc, lxd, moon, fdio, multisite) +Test API +--------- - - integration of new OPNFV feature projects (copper, domino, multisite, - moon, parser, onos-sfc, odl-sfc, security scan) +- automatic documentation (html & pdf) - - introduction of test tiers in functest framework +- full dockerization and automation of the deployment on testresults.opnfv.org - - automatic reporting +- automation of test database backup on artifact - - introduction of a jenkins summary table - - support of ARM architecture Scenario Matrix =============== -For Colorado 1.0, Functest was tested on the following scenarios (if not -precised, the scenario is a HA scenario): +For Danube 1.0, Functest was tested on the following HA scenarios (new +dabube scenarios in bold): +---------------------+---------+---------+---------+---------+ | Scenario | Apex | Compass | Fuel | Joid | +=====================+=========+=========+=========+=========+ | nosdn | X | X | X | X | +---------------------+---------+---------+---------+---------+ -| nosdn-noha | | | X | X | -+---------------------+---------+---------+---------+---------+ -| kvm | | | X | | +| **fdio** | X | | | | +---------------------+---------+---------+---------+---------+ -| kvm-noha | | | X | | +| kvm | X | | X | | +---------------------+---------+---------+---------+---------+ -| lxd | | | | X | +| **kvm_ovs_dpdk** | | | X | | +---------------------+---------+---------+---------+---------+ -| lxd-noha | | | | X | +| **kvm_ovs_dpdk-bar**| | | X | | +---------------------+---------+---------+---------+---------+ -| ovs-noha (dpdk) | | | X | | +| lxd | | | | X | +---------------------+---------+---------+---------+---------+ -| odl_l2 | X | X | X | X | +| **ovs** | | | X | | +---------------------+---------+---------+---------+---------+ -| odl_l2-noha | | | X | | +| **openo** | | X | | | +---------------------+---------+---------+---------+---------+ -| odl_l2-bgpvpn | X | | X | | +| odl_l2 | | X | X | X | +---------------------+---------+---------+---------+---------+ -| odl_l2-bgpvpn-noha| | | X | | +| odl-bgpvpn | X | | | | +---------------------+---------+---------+---------+---------+ -| odl_l2-fdio-noha | X | | | | +| odl_l2-bgpvpn | | | X | | +---------------------+---------+---------+---------+---------+ -| odl_l2-moon | | X | | | +| **odl_l2-fdio** | X | | | | +---------------------+---------+---------+---------+---------+ -| odl_l2-sfc-noha | X | | X | | +| odl_l2-sfc | | | X | | +---------------------+---------+---------+---------+---------+ | odl_l3 | X | X | X | | +---------------------+---------+---------+---------+---------+ -| odl_l3-noha | | | X | | +| **ocl** | | X | | | +---------------------+---------+---------+---------+---------+ -| onos | | X | X | X | -+---------------------+---------+---------+---------+---------+ -| onos-noha | | | X | | -+---------------------+---------+---------+---------+---------+ -| onos-sfc | | X | X | X | -+---------------------+---------+---------+---------+---------+ -| onos-sfc-noha | | | X | | +| onos | | X | | | +---------------------+---------+---------+---------+---------+ | multisite | | | X | | +---------------------+---------+---------+---------+---------+ -For Colorado 2.0, the following scenarios have been re-released: +Non HA scenarios: +---------------------+---------+---------+---------+---------+ | Scenario | Apex | Compass | Fuel | Joid | +=====================+=========+=========+=========+=========+ -| onos | X | X | X | X | -+---------------------+---------+---------+---------+---------+ -| odl_l2-sfc-ha | | | X | | -+---------------------+---------+---------+---------+---------+ - - -In Colorado, the functional tests have been sliced in 6 different -categories: - -+----------------+-----------------------------------------------+ -| Category | Description | -+================+===============================================+ -| healthcheck | Basic OpenStack commands | -+----------------+-----------------------------------------------+ -| smoke | vPings, Tempest and rally smoke tests | -+----------------+-----------------------------------------------+ -| sdn_suites | Specific SDN feature tests | -+----------------+-----------------------------------------------+ -| features | OPNFV feature project functional test suites | -+----------------+-----------------------------------------------+ -| openstack | Advanced, long duration OpenStack tests | -| | (Tempest and Rally full suite). Each test may | -| | last several hours | -+----------------+-----------------------------------------------+ -| vnf | Complex scenarios dealing with orchestration, | -| | VNF deployment and tests (e.g. vIMS) | -+----------------+-----------------------------------------------+ - -For the scenario validation, we consider only the healthcheck, smoke, -sdn_suites and features categories. These tests are run systematically -in the CI daily loops. +| nosdn | | | X | X | ++---------------------+---------+---------+---------+---------+ +| kvm | | | X | | ++---------------------+---------+---------+---------+---------+ +| **kvm_ovs_dpdk** | | | X | | ++---------------------+---------+---------+---------+---------+ +| **kvm_ovs_dpdk-bar**| | | X | | ++---------------------+---------+---------+---------+---------+ +| lxd | | | | X | ++---------------------+---------+---------+---------+---------+ +| ovs | | | X | | ++---------------------+---------+---------+---------+---------+ +| odl_l2 | | | X | | ++---------------------+---------+---------+---------+---------+ +| odl_l2-bgpvpn | | | X | | ++---------------------+---------+---------+---------+---------+ +| odl_l2-fdio | X | | | | ++---------------------+---------+---------+---------+---------+ +| **odl_l3-fdio** | X | | | | ++---------------------+---------+---------+---------+---------+ +| odl_l2-sfc | | | X | | ++---------------------+---------+---------+---------+---------+ +| odl_l3 | | | X | | ++---------------------+---------+---------+---------+---------+ +| **odl_gluon** | X | | | | ++---------------------+---------+---------+---------+---------+ + +Colorado deprecated scenarios: + + * odl_l2-moon + * onos-sfc + * onos-noha + * onos-sfc-noha + +For the scenario validation, we consider only the healthcheck, smoke and +features categories. These tests are run systematically in the CI daily loops. Success criteria have been defined for these test cases, they can be PASS/FAIL or a success rate may be declared (100%, > 90%) All the criteria, as well as the test dependencies are declared in the ci/testcases.yaml file. -The scoring for the Colorado release per installer can be described as +The scoring for the Danube release per installer can be described as follows. The scoring is an indicator showing how many feature project test suites @@ -266,7 +283,7 @@ By default, if not specified, the scenarios are HA. HA means OpenStack High Availability (main services). Note that not all VIM (e.g. OpenStack) services are deployed in HA mode, as that depends upon support of the specific service for HA deployment. -For example, in the Colorado release, the Congress service +For example, in the Danube release, the Congress service is deployed in non-HA mode even for HA OPNFV scenarios, as explicit support for HA operation has not yet been verified. See the release notes of the installers for more details. @@ -279,6 +296,7 @@ Apex | Scenario | Scoring | Success | Results | | | | rate | | +==================+=========+=========+===================+ +************* TODO ***************************************** | nosdn | 17/18 | 95% | `apex-res-174`_ | +------------------+---------+---------+-------------------+ | odl_l2 | 21/21 | 100% | `apex-res-175`_ | @@ -302,6 +320,7 @@ Compass | Scenario | Scoring | Success | Results | | | | rate | | +==================+=========+=========+===================+ +************* TODO ***************************************** | nosdn | 12/12 | 100% | `compass-res-55`_ | +------------------+---------+---------+-------------------+ | odl_l2 | 15/15 | 100% | `compass-res-59`_ | @@ -315,7 +334,7 @@ Compass | onos-sfc-ha | 17/18 | 95% | `compass-res-76`_ | +------------------+---------+---------+-------------------+ -Note: all the Compass tests for Colorado have been executed on virtual +Note: all the Compass tests for Danube have been executed on virtual environment. Bare metal resources were used for Master branch. @@ -326,6 +345,7 @@ Fuel | Scenario | Scoring | Success | Results | | | | rate | | +=====================+=========+=========+===================+ +************* TODO ***************************************** | nosdn | 18/18 | 100% | `fuel-res-129`_ | +---------------------+---------+---------+-------------------+ | nosdn-noha | 15/15 | 100% | `fuel-res-154`_ | @@ -367,12 +387,13 @@ Fuel *: all results passed, lacking iterations to reach the full score -Results of Functest on AArch64 Colorado 3.0 +Results of Functest on AArch64 Danube 3.0 +---------------------+---------+---------+----------------------+ | Scenario | Scoring | Success | Results | | | | rate | | +=====================+=========+=========+======================+ +************* TODO ***************************************** | nosdn | 18/18 | 100% | `fuel-arm-res-128`_ | +---------------------+---------+---------+----------------------+ | odl_l2 | 21/21 | 100% | `fuel-arm-res-122`_ | @@ -390,6 +411,7 @@ Joid | Scenario | Scoring | Success | Results | | | | rate | | +=====================+=========+=========+=================+ +************* TODO ***************************************** | nosdn | 18/18 | 100% | `joid-res-102`_ | +---------------------+---------+---------+-----------------+ | nosdn-noha | 17/18 | 95% | `joid-res-93`_ | @@ -416,14 +438,16 @@ for the date of the test you are interested in. The reporting pages can be found at: - * apex: http://testresults.opnfv.org/reporting/functest/release/colorado/index-status-apex.html - * compass: http://testresults.opnfv.org/reporting/functest/release/colorado/index-status-compass.html - * fuel: http://testresults.opnfv.org/reporting/functest/release/colorado/index-status-fuel.html - * joid: http://testresults.opnfv.org/reporting/functest/release/colorado/index-status-joid.html + * apex: http://testresults.opnfv.org/reporting/functest/release/danube/index-status-apex.html + * compass: http://testresults.opnfv.org/reporting/functest/release/danube/index-status-compass.html + * fuel: http://testresults.opnfv.org/reporting/functest/release/danube/index-status-fuel.html + * joid: http://testresults.opnfv.org/reporting/functest/release/danube/index-status-joid.html -Colorado known restrictions/issues +Danube known restrictions/issues ================================== +************* TODO ***************************************** + +-----------+-----------+----------------------------------------------+ | Installer | Scenario | Issue | +===========+===========+==============================================+ @@ -443,43 +467,19 @@ Colorado known restrictions/issues | | | Rally sanity test case has been disabled. | | | | Performance issues seem to be connected to | | | | the ODL version. It is planned to reintroduce| -| | | Rally sanity in Colorado 2.0 with the | +| | | Rally sanity in Danube 2.0 with the | | | | adoption of ODL Boron release. | +-----------+-----------+----------------------------------------------+ -| apex | *-fdio | Due to late integration, fdio decided to | -| | | focus on mandatory tests and exclude feature | -| | | tests (copper, doctor, security_scan) from | -| | | its scenarios | -+-----------+-----------+----------------------------------------------+ -| compass | moon | First ODL test FAILS because ODL/Openstack | -| | | federation done in moon is partial. Only | -| | | MD-SAL is federated (not AD-SAL) | -+-----------+-----------+----------------------------------------------+ -| fuel | any | TestServerBasicOps test case skipped | -| | | https://gerrit.opnfv.org/gerrit/#/c/19635/ | -+-----------+-----------+----------------------------------------------+ -| fuel | kvm | vPing_ssh and vIMS excluded (metadata related| -| | | scenarios) | -+-----------+-----------+----------------------------------------------+ -| fuel | multisite | random errors running multisite. A known bug | -| | | in keystone mitaka, due to which memcache | -| | | raises exception and keystone becomes | -| | | unresponsive | -| | | bugs.launchpad.net/keystone/+bug/1600394 | -| | | workaround consists in restarting memcache on| -| | | server | +| fuel | odl | TestServerBasicOps test case skipped | +| | | Tempest ssh client is hanging on opendaylight| +| | | enabled envs (getting deadlock in paramiko | +| | | recv_exit_status method) while trying to | +| | | execute a command on a vm. | +-----------+-----------+----------------------------------------------+ | joid | any | Tempest cases related to object storage | | | | excluded | | | | https://gerrit.opnfv.org/gerrit/#/c/17871/ | +-----------+-----------+----------------------------------------------+ -| joid | domino | Domino tests are skipped in CI. However the | -| | | test case can be run by manually setting | -| | | IS_IPandKEY_CONFIGURED=true after manually | -| | | setting properly the IP addresses of the 3 | -| | | Controller nodes in the configuration file | -| | | /repos/domino/tests/run_multinode.sh | -+-----------+-----------+----------------------------------------------+ Test and installer/scenario dependencies @@ -509,11 +509,9 @@ Open JIRA tickets +------------------+-----------------------------------------------+ | JIRA | Description | +==================+===============================================+ -| `FUNCTEST-446`_ | Cleanup ODL-SFC output in Functest execution | -| | Impact on odl_l2-sfc scenarios | -+------------------+-----------------------------------------------+ -| `FUNCTEST-454`_ | Cleanup failures when using HA networks in | -| | Neutron | +************* TODO ***************************************** +| | | +| | | +------------------+-----------------------------------------------+ Useful links @@ -521,7 +519,7 @@ Useful links - wiki project page: https://wiki.opnfv.org/opnfv_functional_testing - - wiki Functest Colorado page: https://wiki.opnfv.org/display/functest/Functest+Colorado + - wiki Functest Danube page: https://wiki.opnfv.org/display/functest/Functest+Danube - Functest repo: https://git.opnfv.org/cgit/functest @@ -531,107 +529,6 @@ Useful links - Functest IRC chan: #opnfv-functest - - Functest reporting: http://testresults.opnfv.org/reporting - - - Functest test configuration: https://git.opnfv.org/cgit/functest/tree/ci/testcases.yaml - - - Functest Colorado user guide: http://artifacts.opnfv.org/functest/colorado/docs/userguide/index.html - - - Functest installation/configuration guide: http://artifacts.opnfv.org/functest/colorado/docs/configguide/index.html - - - Functest developer guide: http://artifacts.opnfv.org/functest/colorado/docs/devguide/index.html - -.. _`FUNCTEST-446` : https://jira.opnfv.org/browse/FUNCTEST-446 - -.. _`FUNCTEST-454` : https://jira.opnfv.org/browse/FUNCTEST-454 - -.. _`apex-res-6` : http://testresults.opnfv.org/test/api/v1/results?build_tag=bd-push-daily-colorado-6 - -.. _`apex-res-174` : http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-apex-apex-daily-colorado-daily-colorado-174 - -.. _`apex-res-175`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-apex-apex-daily-colorado-daily-colorado-175 - -.. _`apex-res-176`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-apex-apex-daily-colorado-daily-colorado-176 - -.. _`apex-res-217`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-apex-apex-daily-colorado-daily-colorado-217 - -.. _`apex-res-235`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-apex-apex-daily-colorado-daily-colorado-235 - -.. _`apex-res-423`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-apex-apex-daily-colorado-daily-colorado-423 - -.. _`compass-res-55`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-compass-virtual-daily-colorado-55 - -.. _`compass-res-59`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-compass-virtual-daily-colorado-59 - -.. _`compass-res-73`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-compass-virtual-daily-colorado-73 - -.. _`compass-res-76`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-compass-virtual-daily-colorado-76 - -.. _`compass-res-77`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-compass-virtual-daily-colorado-77 - -.. _`compass-res-567`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-compass-virtual-daily-master-567 - -.. _`compass-res-285`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-compass-virtual-daily-master-285 - -.. _`fuel-res-8`: https://build.opnfv.org/ci/view/functest/job/functest-fuel-virtual-suite-colorado/lastSuccessfulBuild/console - -.. _`fuel-res-115`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-baremetal-daily-colorado-115 - -.. _`fuel-res-117`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-baremetal-daily-colorado-117 - -.. _`fuel-res-119`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-baremetal-daily-colorado-119 - -.. _`fuel-res-123`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-baremetal-daily-colorado-123 - -.. _`fuel-res-124`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-baremetal-daily-colorado-124 - -.. _`fuel-res-128`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-baremetal-daily-colorado-128 - -.. _`fuel-res-129`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-baremetal-daily-colorado-129 - -.. _`fuel-res-154`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-virtual-daily-colorado-154 - -.. _`fuel-res-155`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-virtual-daily-colorado-155 - -.. _`fuel-res-160`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-virtual-daily-colorado-160 - -.. _`fuel-res-161`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-virtual-daily-colorado-161 - -.. _`fuel-res-162`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-virtual-daily-colorado-162 - -.. _`fuel-res-164`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-virtual-daily-colorado-164 - -.. _`fuel-res-166`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-virtual-daily-colorado-166 - -.. _`fuel-res-213`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-virtual-daily-colorado-213 - -.. _`fuel-res-219`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-virtual-daily-colorado-219 - -.. _`fuel-res-376`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-virtual-daily-colorado-376 - -.. _`fuel-res-492`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-fuel-virtual-daily-colorado-492 - -.. _`fuel-arm-res-128`: https://build.opnfv.org/ci/view/armband/job/functest-fuel-armband-baremetal-daily-colorado/128/console - -.. _`fuel-arm-res-122`: https://build.opnfv.org/ci/view/armband/job/functest-fuel-armband-baremetal-daily-colorado/122/console - -.. _`fuel-arm-res-129`: https://build.opnfv.org/ci/view/armband/job/functest-fuel-armband-baremetal-daily-colorado/129/console - -.. _`fuel-arm-res-135`: https://build.opnfv.org/ci/view/armband/job/functest-fuel-armband-baremetal-daily-colorado/135/console - -.. _`joid-res-93`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-joid-baremetal-daily-colorado-93 - -.. _`joid-res-91`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-joid-baremetal-daily-colorado-91 - -.. _`joid-res-99`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-joid-baremetal-daily-colorado-99 - -.. _`joid-res-97`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-joid-baremetal-daily-colorado-97 - -.. _`joid-res-102`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-joid-baremetal-daily-colorado-102 - -.. _`joid-res-103`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-joid-baremetal-daily-colorado-103 - -.. _`joid-res-104`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-joid-baremetal-daily-colorado-104 - -.. _`joid-res-345`: http://testresults.opnfv.org/test/api/v1/results?build_tag=jenkins-functest-joid-baremetal-daily-colorado-345 + - Reporting page: http://testresults.opnfv.org/reporting/danube.html + - Functest test configuration: https://git.opnfv.org/cgit/functest/tree/functest/ci/testcases.yaml diff --git a/docs/release/release-notes/index.rst b/docs/release/release-notes/index.rst index 25ee47713..a1a2aa1a7 100644 --- a/docs/release/release-notes/index.rst +++ b/docs/release/release-notes/index.rst @@ -1,3 +1,6 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 + .. _functest-releasenotes: ********************** diff --git a/docs/testing/developer/devguide/index.rst b/docs/testing/developer/devguide/index.rst index ce5dc77be..43f0804d7 100644 --- a/docs/testing/developer/devguide/index.rst +++ b/docs/testing/developer/devguide/index.rst @@ -1,3 +1,6 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. SPDX-License-Identifier: CC-BY-4.0 + ****************************** OPNFV FUNCTEST developer guide ****************************** @@ -62,18 +65,18 @@ Functest internal test cases ============================ The internal test cases in Danube are: - * healthcheck - * connection_check + * api_check + * cloudify_ims + * connection_check * vping_ssh * vping_userdata * odl - * snaps_smoke - * tempest_smoke_serial + * rally_full * rally_sanity + * snaps_health_check * tempest_full_parallel - * rally_full - * cloudify_ims + * tempest_smoke_serial By internal, we mean that this particular test cases have been developped and/or integrated by functest contributors and the associated @@ -86,7 +89,7 @@ The main internal test cases are in the opnfv_tests subfolder of the repository, the internal test cases are: * sdn: odl, onos - * openstack: healthcheck, vping_ssh, vping_userdata, tempest_*, rally_*, connection_check, api_check, snaps_smoke + * openstack: api_check, connection_check, snaps_health_check, vping_ssh, vping_userdata, tempest_*, rally_*, snaps_smoke * vnf: cloudify_ims If you want to create a new test case you will have to create a new @@ -99,19 +102,23 @@ especially the feature projects. The external test cases are: - * promise - * doctor - * onos + * barometer * bgpvpn - * copper - * security_scan - * sfc-odl - * sfc-onos - * parser + * doctor * domino + * odl-netvirt + * onos + * fds * multisite - * opera_ims + * netready * orchestra_ims + * parser + * promise + * refstack_defcore + * security_scan + * snaps_smoke + * sfc-odl + * vyos_vrouter The code to run these test cases may be directly in the repository of @@ -149,17 +156,17 @@ Functest abstraction classes In order to harmonize test integration, 3 abstraction classes have been introduced in Danube: - * testcase_base: base for any test case - * feature_base: abstraction for feature project + * testcase: base for any test case + * feature: abstraction for feature project * vnf_base: abstraction for vnf onboarding The goal is to unify the way to run test from Functest. -feature_base and vnf_base inherit from testcase_base. +feature and vnf_base inherit from testcase:: +-----------------------------------------+ | | - | Testcase_base | + | TestCase | | | | - init() | | - run() | @@ -171,12 +178,12 @@ feature_base and vnf_base inherit from testcase_base. V V +--------------------+ +--------------------------+ | | | | - | feature_base | | vnf_base | + | feature | | vnf_base | | | | | | - prepare() | | - prepare() | - | - post() | | - deploy_orchestrator() | - | - parse_results() | | - deploy_vnf() | - | | | - test_vnf() | + | - execute() | | - deploy_orchestrator() | + | - post() | | - deploy_vnf() | + | - parse_results() | | - test_vnf() | | | | - clean() | | | | - execute() | | | | | @@ -270,232 +277,13 @@ the API will return an error message. An additional method dashboard has been added to post-process the raw results in release Brahmaputra (deprecated in Colorado). -The data model is very basic, 4 objects are created: +The data model is very basic, 5 objects are created: * Pods * Projects * Testcases * Results - -Pods:: - - { - "id": <ID>, - "details": <URL description of the POD>, - "creation_date": "YYYY-MM-DD HH:MM:SS", - "name": <The POD Name>, - "mode": <metal or virtual>, - "role": <ci-pod or community-pod or single-node> - }, - -Projects:: - - { - "id": <ID>, - "name": <Name of the Project>, - "creation_date": "YYYY-MM-DD HH:MM:SS", - "description": <Short description> - }, - -Testcases:: - - { - "id": <ID>, - "name":<Name of the test case>, - "project_name":<Name of belonged project>, - "creation_date": "YYYY-MM-DD HH:MM:SS", - "description": <short description>, - "url":<URL for longer description> - }, - -Results:: - - { - "_id": <ID>, - "case_name": <Reference to the test case>, - "project_name": <Reference to project>, - "pod_name": <Reference to POD where the test was executed>, - "installer": <Installer Apex or Compass or Fuel or Joid>, - "version": <master or Colorado or Brahmaputra>, - "start_date": "YYYY-MM-DD HH:MM:SS", - "stop_date": "YYYY-MM-DD HH:MM:SS", - "build_tag": <such as "jenkins-functest-fuel-baremetal-daily-master-108">, - "scenario": <Scenario on which the test was executed>, - "criteria": <PASS or FAILED>, - "trust_indicator": { - "current": 0, - "histories": [] - } - } - -The API can described as follows. For detailed information, please go to - - http://testresults.opnfv.org/test/swagger/spec.html - - Authentication: opnfv/api@opnfv - -Version: - - +--------+--------------------------+-----------------------------------------+ - | Method | Path | Description | - +========+==========================+=========================================+ - | GET | /versions | Get all supported API versions | - +--------+--------------------------+-----------------------------------------+ - - -Pods: - - +--------+----------------------------+-----------------------------------------+ - | Method | Path | Description | - +========+============================+=========================================+ - | GET | /api/v1/pods | Get the list of declared Labs (PODs) | - +--------+----------------------------+-----------------------------------------+ - | POST | /api/v1/pods | Declare a new POD | - | | | Content-Type: application/json | - | | | { | - | | | "name": "pod_foo", | - | | | "mode": "metal", | - | | | "role": "ci-pod", | - | | | "details": "it is a ci pod" | - | | | } | - +--------+----------------------------+-----------------------------------------+ - | GET | /api/v1/pods/{pod_name} | Get a declared POD | - +--------+----------------------------+-----------------------------------------+ - -Projects: - - +--------+----------------------------+-----------------------------------------+ - | Method | Path | Description | - +========+============================+=========================================+ - | GET | /api/v1/projects | Get the list of declared projects | - +--------+----------------------------+-----------------------------------------+ - | POST | /api/v1/projects | Declare a new test project | - | | | Content-Type: application/json | - | | | { | - | | | "name": "project_foo", | - | | | "description": "whatever you want" | - | | | } | - +--------+----------------------------+-----------------------------------------+ - | DELETE | /api/v1/projects/{project} | Delete a test project | - +--------+----------------------------+-----------------------------------------+ - | GET | /api/v1/projects/{project} | Get details on a {project} | - | | | | - +--------+----------------------------+-----------------------------------------+ - | PUT | /api/v1/projects/{project} | Update a test project | - | | | | - | | | Content-Type: application/json | - | | | { | - | | | <the field(s) you want to modify> | - | | | } | - +--------+----------------------------+-----------------------------------------+ - - -Testcases: - - +--------+----------------------------+-----------------------------------------+ - | Method | Path | Description | - +========+============================+=========================================+ - | GET | /api/v1/projects/{project}/| Get the list of testcases of {project} | - | | cases | | - +--------+----------------------------+-----------------------------------------+ - | POST | /api/v1/projects/{project}/| Add a new test case to {project} | - | | cases | Content-Type: application/json | - | | | { | - | | | "name": "case_foo", | - | | | "description": "whatever you want" | - | | | "url": "whatever you want" | - | | | } | - +--------+----------------------------+-----------------------------------------+ - | DELETE | /api/v1/projects/{project}/| Delete a test case | - | | cases/{case} | | - +--------+----------------------------+-----------------------------------------+ - | GET | /api/v1/projects/{project}/| Get a declared test case | - | | cases/{case} | | - +--------+----------------------------+-----------------------------------------+ - | PUT | /api/v1/projects/{project}?| Modify a test case of {project} | - | | cases/{case} | | - | | | Content-Type: application/json | - | | | { | - | | | <the field(s) you want to modify> | - | | | } | - +--------+----------------------------+-----------------------------------------+ - -Results: - - +--------+----------------------------+------------------------------------------+ - | Method | Path | Description | - +========+============================+==========================================+ - | GET | /api/v1/results | Get all the test results | - +--------+----------------------------+------------------------------------------+ - | POST | /api/v1/results | Add a new test results | - | | | Content-Type: application/json | - | | | { | - | | | "project_name": "project_foo", | - | | | "scenario": "odl-l2", | - | | | "stop_date": "2016-05-28T14:42:58.384Z", | - | | | "trust_indicator": 0.5, | - | | | "case_name": "vPing", | - | | | "build_tag": "", | - | | | "version": "Colorado", | - | | | "pod_name": "pod_foo", | - | | | "criteria": "PASS", | - | | | "installer": "fuel", | - | | | "start_date": "2016-05-28T14:41:58.384Z",| - | | | "details": <your results> | - | | | } | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | Get the test results of {case} | - | | case={case} | | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | Get the test results of build_tag | - | | build_tag={tag} | {tag}. | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | Get last {N} records of test results | - | | last={N} | | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | Get the test results of scenario | - | | scenario={scenario} | {scenario}. | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | Get the test results of trust_indicator | - | | trust_indicator={ind} | {ind}. | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | Get the test results of last days | - | | period={period} | {period}. | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | Get the test results of {project} | - | | project={project} | | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | Get the test results of version | - | | version={version} | {version}. | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | Get the test results of criteria | - | | criteria={criteria} | {criteria}. | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | get the results on pod {pod} | - | | pod={pod} | | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | Get the test results of installer {inst} | - | | installer={inst} | | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results? | Get the results according to combined | - | | <query conditions> | query conditions supported above | - +--------+----------------------------+------------------------------------------+ - | GET | /api/v1/results/{result_id}| Get the test result by result_id | - +--------+----------------------------+------------------------------------------+ - -Scenarios: - - +--------+----------------------------+-----------------------------------------+ - | Method | Path | Description | - +========+============================+=========================================+ - | GET | /api/v1/scenarios | Get the list of declared scenarios | - +--------+----------------------------+-----------------------------------------+ - | POST | /api/v1/scenario | Declare a new scenario | - +--------+----------------------------+-----------------------------------------+ - | GET | /api/v1/scenario? | Get a declared scenario | - | | <query conditions> | | - +--------+----------------------------+-----------------------------------------+ - + * Scenarios The code of the API is hosted in the releng repository `[6]`_. The static documentation of the API can be found at `[17]`_. @@ -574,6 +362,7 @@ Please note that currently token authorization is implemented but is not yet ena +---------------------+---------+---------+---------+---------+ | copper | X | | | X | +---------------------+---------+---------+---------+---------+ + src: colorado (see release note for the last matrix version) All the testcases listed in the table are runnable on os-odl_l2-nofeature scenarios. @@ -736,8 +525,7 @@ Regex are standard regex. You can have a look at `[11]`_ You can also easily test your regex via an online regex checker such as `[12]`_. Put your scenario in the TEST STRING window (e.g. os-odl_l3-ovs-ha), put -your regex in the REGULAR EXPRESSION window, then you can test your rule -. +your regex in the REGULAR EXPRESSION window, then you can test your rule. How to know which test I can run? @@ -941,6 +729,233 @@ You can also reuse a python function defined in functest_utils.py:: return False +Where can I find the documentation on the test API? +=================================================== + +http://artifacts.opnfv.org/releng/docs/testapi.html + + +How to exclude Tempest case from default Tempest smoke suite? +============================================================= + +Tempest default smoke suite deals with 165 test cases. +Since Colorado the success criteria is 100%, i.e. if 1 test is failed the +success criteria is not matched for the scenario. + +It is necessary to exclude some test cases that are expected to fail due to +known upstream bugs (see release notes). + +A file has been created for such operation: https://git.opnfv.org/cgit/functest/tree/functest/opnfv_tests/openstack/tempest/custom_tests/blacklist.txt. + +It can be described as follows:: + + - + scenarios: + - os-odl_l2-bgpvpn-ha + - os-odl_l2-bgpvpn-noha + installers: + - fuel + - apex + tests: + - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers + - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details + - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers + - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details + - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard + - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops + - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops + - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern + - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern + +Please note that each exclusion must be justified. the goal is not to exclude +test cases because they do not pass. Several scenarios reached the 100% criteria. +So it is expected in the patch submited to exclude the cases to indicate the +reasons of the exclusion. + + +How do I know the Functest status of a scenario? +================================================ + +A Functest automatic reporting page is generated daily. +This page is dynamically created through a cron job and is based on the results +stored in the Test DB. +You can access this reporting page: http://testresults.opnfv.org/reporting + +See https://wiki.opnfv.org/pages/viewpage.action?pageId=6828617 for details. + + +I have tests, to which category should I declare them? +====================================================== + +CATEGORIES/TIERS description: + ++----------------+-------------------------------------------------------------+ +| healthcheck | Simple OpenStack healtcheck tests case that validates the | +| | basic operations in OpenStack | ++----------------+-------------------------------------------------------------+ +| Smoke | Set of smoke test cases/suites to validate the most common | +| | OpenStack and SDN Controller operations | ++----------------+-------------------------------------------------------------+ +| Features | Test cases that validate a specific feature on top of OPNFV.| +| | Those come from Feature projects and need a bit of support | +| | for integration | ++----------------+-------------------------------------------------------------+ +| Components | Advanced Openstack tests: Full Tempest, Full Rally | ++----------------+-------------------------------------------------------------+ +| Performance | Out of Functest Scope | ++----------------+-------------------------------------------------------------+ +| VNF | Test cases related to deploy an open source VNF including | +| | an orchestrator | ++----------------+-------------------------------------------------------------+ + +The main ambiguity could be between features and VNF. +In fact sometimes you have to spawn VMs to demonstrate the capabilities of the +feature you introduced. +We recommend to declare your test in the feature category. + +VNF category is really dedicated to test including: + + * creation of resources + * deployement of an orchestrator/VNFM + * deployment of the VNF + * test of the VNFM + * free resources + +The goal is not to study a particular feature on the infrastructure but to have +a whole end to end test of a VNF automatically deployed in CI. +Moreover VNF are run in weekly jobs (one a week), feature tests are in daily +jobs and use to get a scenario score. + +Where are the logs? +=================== + +Functest deals with internal and external testcases. Each testcase can generate +logs. + +Since Colorado we introduce the possibility to push the logs to the artifact. +A new script (https://git.opnfv.org/releng/tree/utils/push-test-logs.sh) has +been created for CI. + +When called, and assuming that the POD is authorized to push the logs to +artifacts, the script will push all the results or logs locally stored under +/home/opnfv/functest/results/. + +If the POD is not connected to CI, logs are not pushed. +But in both cases, logs are stored in /home/opnfv/functest/results in the +container. +Projects are encouraged to push their logs here. + +Since Colorado it is also easy for feature project to integrate this feature by +adding the log file as output_file parameter when calling execute_command from +functest_utils library + + ret_val = functest_utils.execute_command(cmd, output_file=log_file) + + +How does Functest deal with VNF onboarding? +=========================================== + +VNF onboarding has been introduced in Brahmaputra through the automation of a +clearwater vIMS deployed thanks to cloudify orchestrator. + +This automation has been described at OpenStack summit Barcelona: +https://youtu.be/Jr4nG74glmY + +The goal of Functest consists in testing OPNFV from a functional perspective: +the NFVI and/or the features developed in OPNFV. Feature test suites are +provided by the feature project. Functest just simplifies the integration of +the suite into the CI and gives a consolidated view of the tests per scenario. + +Functest does not develop VNFs. + +Functest does not test any MANO stack. + +OPNFV projects dealing with VNF onboarding +------------------------------------------ + +Testing VNF is not the main goal however it gives interesting and realistic +feedback on OPNFV as a Telco cloud. + +Onboarding VNF also allows to test a full stack: orchestrator + VNF. + +Functest is VNF and MANO stack agnostic. + +An internship has been initiated to reference the Open Source VNF: Intern +Project Open Source VNF catalog + +New projects dealing with orchestrators or VNFs are candidate for Danube. + +The 2 projects dealing with orchestration are: + + * orchestra (Openbaton) + * opera (Open-O) + +The Models project address various goals for promoting availability and +convergence of information and/or data models related to NFV service/VNF +management, as being defined in standards (SDOs) and as developed in open +source projects. + +Functest VNF onboarding +----------------------- + +In order to simplify VNF onboarding a new abstraction class has been developed +in Functest. + +This class is based on vnf_base and can be described as follow: + + +------------+ +--------------+ + | test_base |------------>| vnf_base | + +------------+ +--------------+ + |_ prepare + |_ deploy_orchestrator (optional) + |_ deploy_vnf + |_ test_vnf + |_ clean + + +Several methods are declared in vnf_base: + + * prepare + * deploy_orchestrator + * deploy_vnf + * test_vnf + * clean + +deploy_vnf and test_vnf are mandatory. + +prepare will create a user and a project. + +How to declare your orchestrator/VNF? +------------------------------------- +1) test declaration + +You must declare your testcase in the file <Functest repo>/functest/ci/testcases.yaml + +2) configuration + +You can precise some configuration parameters in config_functest.yaml + +3) implement your test + +Create your own VnfOnboarding file + +you must create your entry point through a python clase as referenced in the +configuration file + +e.g. aaa => creation of the file <Functest repo>/functest/opnfv_tests/vnf/aaa/aaa.py + +the class shall inherit vnf_base. +You must implement the methods deploy_vnf() and test_vnf() and may implement +deploy_orchestrator() + +you can call the code from your repo (but need to add the repo in Functest if +it is not the case) + +4) success criteria + +So far we considered the test as PASS if vnf_deploy and test_vnf is PASS +(see example in aaa). + ========== References ========== @@ -989,4 +1004,4 @@ _`OpenRC`: http://docs.openstack.org/user-guide/common/cli_set_environment_varia _`Rally installation procedure`: https://rally.readthedocs.org/en/latest/tutorial/step_0_installation.html -_`config_functest.yaml` : https://git.opnfv.org/cgit/functest/tree/testcases/config_functest.yaml +_`config_functest.yaml` : https://git.opnfv.org/cgit/functest/tree/functest/ci/config_functest.yaml diff --git a/docs/testing/user/configguide/configguide.rst b/docs/testing/user/configguide/configguide.rst index 56a414d36..e3485be40 100644 --- a/docs/testing/user/configguide/configguide.rst +++ b/docs/testing/user/configguide/configguide.rst @@ -1,5 +1,5 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 +.. SPDX-License-Identifier: CC-BY-4.0 Pulling the Docker image @@ -16,8 +16,8 @@ created automatically by the CI mechanisms, and you must ensure you pull an image with the **correct tag** to match the OPNFV software release installed in your environment. All available tagged images can be seen from location [FunctestDockerTags_]. For example, when running -on the first official release of the OPNFV Colorado system platform, -tag "colorado.1.0" is needed. Pulling other tags might cause some +on the first official release of the OPNFV Danube system platform, +tag "danube.1.0" is needed. Pulling other tags might cause some problems while running the tests. Docker images pulled without a tag specifier bear the implicitly assigned label "latest". If you need to specifically pull the latest @@ -201,7 +201,7 @@ when performing manual test scenarios:: tests are activated under Jenkins CI control. It enables the correlation of test results, which - are independently pushed to the results datbase + are independently pushed to the results database from different Jenkins jobs. DO NOT USE THIS OPTION IN MANUAL TEST SCENARIOS. <DebugTraceValue> = "true" or "false" @@ -405,9 +405,9 @@ follows:: | `-- setup.py |-- core | |-- __init__.py - | |-- feature_base.py + | |-- feature.py | |-- pytest_suite_runner.py - | |-- testcase_base.py + | |-- testcase.py | |-- vnf_base.py |-- opnfv_tests | |-- __init__.py @@ -444,8 +444,7 @@ We may distinguish several directories, the first level has 4 directories: * **docker**: This directory includes the needed files and tools to build the Funtest Docker image. * **docs**: This directory includes documentation: Release Notes, - User Guide, Configuration Guide and Developer Guide. Test results - are also located in a sub--directory called 'results'. + User Guide, Configuration Guide and Developer Guide. * **functest**: This directory contains all the code needed to run functest internal cases and OPNFV onboarded feature or VNF test cases. @@ -592,7 +591,7 @@ prompt:: - here = "false" NOTE: In Jenkins CI runs, an additional item "BUILD TAG" - would also be listed. The valaue is set by Jenkins CI. + would also be listed. The value is set by Jenkins CI. Finally, the **functest** CLI has a **--help** options: diff --git a/docs/testing/user/configguide/index.rst b/docs/testing/user/configguide/index.rst index f12739e31..737f0ba7c 100644 --- a/docs/testing/user/configguide/index.rst +++ b/docs/testing/user/configguide/index.rst @@ -27,8 +27,8 @@ Version history Introduction ============ This document describes how to install and configure Functest in OPNFV. -The Functest CLI is utilized during the Functest environment preparation -step. The given example commands should work in both virtual and bare +The Functest CLI is used during the Functest environment preparation +phase. The given example commands should work in both virtual and bare metal cases alike. High level architecture @@ -52,21 +52,12 @@ follows:: | | | - Rally | | | | | | | | | | - Robot | | | | | | | | | | - TestON | | | | | | | + | | | - RefStack | | | | | | | | | | | |-------------------------+ | | | | | Testcases | | | | | | | | | | - VIM | | | | | | | - | | | -- healthcheck | | | | | | | - | | | -- vPing_ssh | | | | | | | - | | | -- vPing_userdata | | | | | | | - | | | -- SNAPS_cases | | | | | | | - | | | -- Tempest_smoke | | | | | | | - | | | -- Rally_sanity | | | | | | | - | | | -- Tempest_full | | | | | | | - | | | -- Rally_full | | | | | | | | | | | | | | | | | | | | - SDN Controller | | | | | | | - | | | -- odl | | | | | | | - | | | -- onos | | | | | | | | | | | | | | | | | | | | - Features | | | | | | | | | | | | | | | | | @@ -109,16 +100,16 @@ on any platform on any Operating System. The automated mechanisms inside the Functest Docker container will: * Retrieve OpenStack credentials - * Prepare the environment according to the SUT + * Prepare the environment according to the System Under Test (SUT) * Perform the appropriate functional tests * Push the test results into the OPNFV test result database This Docker image can be integrated into CI or deployed independently. -Please note that the Functest Docker container has been designed for -OPNFV, however, it would be possible to adapt it to any VIM + controller -environment, since most of the test cases are integrated from upstream -communities. +Please note that the Functest Docker container has been designed for OPNFV, +however, it would be possible to adapt it to any OpenStack based VIM + +controller environment, since most of the test cases are integrated from +upstream communities. The functional test cases are described in the Functest User Guide `[2]`_ @@ -127,8 +118,7 @@ Prerequisites ============= The OPNFV deployment is out of the scope of this document but it can be found in http://docs.opnfv.org. -The OPNFV platform is considered as the System Under Test (SUT) in this -document. +The OPNFV platform is considered as the SUT in this document. Several prerequisites are needed for Functest: @@ -149,7 +139,7 @@ requirements. Normally it is the same server from where the OPNFV deployment has been triggered previously. NOTE: If your Jumphost is operating behind a company http proxy and/or -Firewall, please consult first the section `Proxy Support`_, towards +firewall, please consult first the section `Proxy Support`_, towards the end of this document. The section details some tips/tricks which *may* be of help in a proxified environment. @@ -158,7 +148,7 @@ Docker installation Docker installation and configuration is only needed to be done once through the life cycle of Jumphost. -If your Jumphost is based on Ubuntu, RHEL or CentOS linux, please +If your Jumphost is based on Ubuntu, SUSE, RHEL or CentOS linux, please consult the references below for more detailed instructions. The commands below are offered as a short reference. @@ -182,7 +172,7 @@ Add your user to docker group to be able to run commands without sudo:: sudo usermod -aG docker <your_user> -A reconnect is needed. There are 2 ways for this: +A reconnection is needed. There are 2 ways for this: #. Re-login to your account #. su - <username> @@ -190,10 +180,12 @@ References - Installing Docker Engine on different Linux Operating Systems: * Ubuntu_ * RHEL_ * CentOS_ + * SUSE_ .. _Ubuntu: https://docs.docker.com/engine/installation/linux/ubuntulinux/ .. _RHEL: https://docs.docker.com/engine/installation/linux/rhel/ .. _CentOS: https://docs.docker.com/engine/installation/linux/centos/ +.. _SUSE: https://docs.docker.com/engine/installation/linux/suse/ Public/External network on SUT ------------------------------ @@ -286,11 +278,11 @@ References .. _`[4]`: http://artifacts.opnfv.org/functest/danube/docs/configguide/index.html -OPNFV main site: opnfvmain_. +`OPNFV main site`_ -OPNFV functional test page: opnfvfunctest_. +`Functest page`_ IRC support channel: #opnfv-functest -.. _opnfvmain: http://www.opnfv.org -.. _opnfvfunctest: https://wiki.opnfv.org/functest +.. _`OPNFV main site`: http://www.opnfv.org +.. _`Functest page`: https://wiki.opnfv.org/functest diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index 9436de2b9..0459ce5f1 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -1,5 +1,5 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. -.. http://creativecommons.org/licenses/by/4.0 +.. SPDX-License-Identifier: CC-BY-4.0 ========================= OPNFV FUNCTEST user guide @@ -28,7 +28,7 @@ Introduction The goal of this document is to describe the OPNFV Functest test cases and to provide a procedure to execute them. In the OPNFV Danube system release, -a Functest CLI utility is introduced for easier execution of test procedures. +a Functest CLI utility is introduced for an easier execution of test procedures. **IMPORTANT**: It is assumed here that the Functest Docker container is already properly deployed and that all instructions described in this guide are to be @@ -41,21 +41,37 @@ The different test cases are described in the remaining sections of this documen VIM (Virtualized Infrastructure Manager) ---------------------------------------- -Healthcheck -^^^^^^^^^^^ -In Colorado release a new Tier 'healthcheck' with one testcase 'healthcheck' -was introduced. The healthcheck testcase verifies that some basic IP connectivity -and essential operations of OpenStack functionality over the command line are -working correctly. +Healthcheck tests +^^^^^^^^^^^^^^^^^ +In Danube, healthcheck tests have been refactored and rely on SNAPS, a +OPNFV middleware project. + +SNAPS stands for "SDN/NFV Application development Platform and Stack". +SNAPS is an object-oriented OpenStack library packaged with tests that exercise +OpenStack. +More information on SNAPS can be found in  `[13]`_ + +Three tests are declared as healthcheck tests and can be used for gating by the +installer, they cover functionally the tests previously done by healthcheck +test case. + +The tests are: -In particular, the following verifications are performed: - * DHCP agent functionality for IP address allocation - * Openstack Authentication management functionality via the Keystone API - * OpenStack Image management functionality via the Glance API - * OpenStack Block Storage management functionality via the Cinder API - * OpenStack Networking management functionality via the Neutron API - * Openstack Compute management functionality via the NOVA API + * *connection_check* + * *api_check* + * *snaps_health_check* + +Connection_check consists in 9 test cases (test duration < 5s) checking the +connectivity with Glance, Keystone, Neutron, Nova and the external network. + +Api_check verifies the retrieval of OpenStack clients: Keystone, Glance, +Neutron and Nova and may perform some simple queries. When the config value of +snaps.use_keystone is True, functest must have access to the cloud's private +network. This suite consists in 49 tests (test duration < 2 minutes). + +snaps_health_check creates instance, allocate floating IP, connect to the VM. +This test replaced the previous Colorado healthcheck test. Self-obviously, successful completion of the 'healthcheck' testcase is a necessary pre-requisite for the execution of all other test Tiers. @@ -229,30 +245,9 @@ The Rally testcases are distributed accross two Tiers: NOTE: Test case 'rally_sanity' executes a limited number of Rally smoke test cases. Test case 'rally_full' executes the full defined set of Rally tests. -SNAPS ------ - -SNAPS stands for "SNA/NFV Application development Platform and Stack". -This project seeks to develop baseline OpenStack NFV installations. It has been -developed by Steven Pisarski and provided an object oriented library to perform -functional and performance tests. It has been declined in several test suites in -Functest, 2 are part of healthcheck tier, one belongs to smoke tier. - -connection check -^^^^^^^^^^^^^^^^ -Connection_check consists in 9 test cases (test duration < 5s) checking the -connectivity with Glance, Keystone, Neutron, Nova and the external network. - -api_check -^^^^^^^^^ -This test case verifies the retrieval of OpenStack clients: Keystone, Glance, -Neutron and Nova and may perform some simple queries. When the config value of -snaps.use_keystone is True, functest must have access to the cloud's private -network. -This suite consists in 49 tests (test duration< 2 minutes) - snaps_smoke -^^^^^^^^^^^ +------------ + This test case contains tests that setup and destroy environments with VMs with and without Floating IPs with a newly created user and project. Set the config value snaps.use_floating_ips (True|False) to toggle this functionality. When @@ -260,16 +255,15 @@ the config value of snaps.use_keystone is True, functest must have access the cloud's private network. This suite consists in 38 tests (test duration < 10 minutes) -More information on SNAPS can be found in  `[13]`_ - SDN Controllers --------------- -There are currently 2 available controllers: +There are currently 3 available controllers: * OpenDaylight (ODL) * ONOS + * OpenContrail (OCL) OpenDaylight ^^^^^^^^^^^^ @@ -366,60 +360,30 @@ The test cases are described as follows: Features -------- -Please refer to the dedicated feature user guides for details: - - * bgpvpn: http://artifacts.opnfv.org/sdnvpn/danube/docs/userguide/index.html - * copper: http://artifacts.opnfv.org/copper/danube/docs/userguide/index.html - * doctor: http://artifacts.opnfv.org/doctor/danube/userguide/index.html - * domino: http://artifacts.opnfv.org/domino/docs/userguide-single/index.html - * multisites: http://artifacts.opnfv.org/multisite/docs/userguide/index.html - * onos-sfc: http://artifacts.opnfv.org/onosfw/danube/userguide/index.html - * odl-sfc: http://artifacts.opnfv.org/sfc/danube/userguide/index.html - * promise: http://artifacts.opnfv.org/danube/colorado/docs/userguide/index.html - * security_scan: http://artifacts.opnfv.org/security_scan/colorado/docs/userguide/index.html - * TODO +In Danube, functest supports the integration of: -security_scan -^^^^^^^^^^^^^ + * barometer + * bgpvpn + * doctor + * domino + * fds + * multisite + * netready + * odl-sfc + * promise + * security_scan -Security Scanning, is a project to insure security compliance and vulnerability -checks, as part of an automated CI / CD platform delivery process. - -The project makes use of the existing SCAP format `[6]`_ to perform deep -scanning of NFVI nodes, to insure they are hardened and free of known CVE -reported vulnerabilities. - -The SCAP content itself, is then consumed and run using an upstream opensource tool -known as OpenSCAP `[7]`_. - -The OPNFV Security Group have developed the code that will called by the OPNFV Jenkins -build platform, to perform a complete scan. Resulting reports are then copied to the -OPNFV functest dashboard. - -The current work flow is as follows: - - * Jenkins Build Initiated - * security_scan.py script is called, and a config file is passed to the script as - an argument. - * The IP addresses of each NFVi node (compute / control) are gathered - * A scan profile is matched to the node type. - * The OpenSCAP application is remotely installed onto each target node gathered - on step 3, using upstream packaging (rpm and .deb). - * A scan is made against each node gathered within step 3. - * HTML Reports are downloaded for rendering on a dashboard. - * If the config file value 'clean' is set to 'True' then the application installed in - step 5 is removed, and all reports created at step 6 are deleted. - -Security scan is supported by Apex, TODO.... +Note: copper is not supported in Danube. +Please refer to the dedicated feature user guides for details. VNF --- -vIMS -^^^^ +cloudify_ims +^^^^^^^^^^^^ The IP Multimedia Subsystem or IP Multimedia Core Network Subsystem (IMS) is an architectural framework for delivering IP multimedia services. @@ -437,10 +401,14 @@ The goal of this test suite consists of: The Clearwater architecture is described as follows: -.. figure:: ../images/clearwater-architecture.png +.. figure:: ../../../images/clearwater-architecture.png :align: center :alt: vIMS architecture +orchestra_ims +^^^^^^^^^^^^^ +Orchestra test case deals with the deployment of OpenIMS with OpenBaton +orchestrator. parser ^^^^^^ @@ -448,6 +416,28 @@ parser See parser user guide for details: `[12]`_ +vyos-vrouter +^^^^^^^^^^^^ + +This test case deals with the deployment and the test of vyos vrouter with +Cloudify orchestrator. The test case can do testing for interchangeability of +BGP Protocol using vyos. + +The Workflow is as follows: + * Deploy + Deploy VNF Testing topology by Cloudify using blueprint. + * Configuration + Setting configuration to Target VNF and reference VNF using ssh + * Run + Execution of test command for test item written YAML format file. + Check VNF status and behavior. + * Reporting + Output of report based on result using JSON format. + +The vyos-vrouter architecture is described in `[14]`_ + + + .. include:: ./runfunctest.rst @@ -479,20 +469,27 @@ at the end of the job and can be described as follow:: +=========================+===============+============+===============+===========================================================================+ | TEST CASE | TIER | DURATION | RESULT | URL | +=========================+===============+============+===============+===========================================================================+ - | healthcheck | healthcheck | 03:07 | PASS | | + | connection_check | healthcheck | 00:02 | PASS | http://testresults.opnfv.org/test/api/v1/results/58cb62b34079ac000a42e3fe | + +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+ + | api_check | healthcheck | 01:15 | PASS | http://testresults.opnfv.org/test/api/v1/results/58cb62fe4079ac000a42e3ff | + +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+ + | snaps_health_check | healthcheck | 00:50 | PASS | http://testresults.opnfv.org/test/api/v1/results/58cb63314079ac000a42e400 | +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+ - | vping_ssh | smoke | 00:56 | PASS | http://testresults.opnfv.org/test/api/v1/results/57ac13d79377c54b278bd4c1 | + | vping_ssh | smoke | 01:10 | PASS | http://testresults.opnfv.org/test/api/v1/results/58cb63654079ac000a42e401 | +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+ - | vping_userdata | smoke | 00:41 | PASS | http://testresults.opnfv.org/test/api/v1/results/57ac14019377c54b278bd4c2 | + | vping_userdata | smoke | 00:59 | PASS | http://testresults.opnfv.org/test/api/v1/results/58cb63a14079ac000a42e403 | +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+ - | tempest_smoke_serial | smoke | 16:05 | FAIL | http://testresults.opnfv.org/test/api/v1/results/57ac17ca9377c54b278bd4c3 | + | tempest_smoke_serial | smoke | 12:57 | PASS | http://testresults.opnfv.org/test/api/v1/results/58cb66bd4079ac000a42e408 | +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+ - | rally_sanity | smoke | 12:19 | PASS | http://testresults.opnfv.org/test/api/v1/results/57ac1aad9377c54b278bd4cd | + | rally_sanity | smoke | 10:22 | PASS | http://testresults.opnfv.org/test/api/v1/results/58cb692b4079ac000a42e40a | +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+ - | odl | sdn_suites | 00:24 | PASS | http://testresults.opnfv.org/test/api/v1/results/57ac1ad09377c54b278bd4ce | + | refstack_defcore | smoke | 12:28 | PASS | http://testresults.opnfv.org/test/api/v1/results/58cb6c184079ac000a42e40c | +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+ - | promise | features | 00:41 | PASS | http://testresults.opnfv.org/test/api/v1/results/57ac1ae59377c54b278bd4cf | + | snaps_smoke | smoke | 12:04 | PASS | http://testresults.opnfv.org/test/api/v1/results/58cb6eec4079ac000a42e40e | +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+ + | domino | features | 00:29 | PASS | http://testresults.opnfv.org/test/api/v1/results/58cb6f044079ac000a42e40f | + +-------------------------+---------------+------------+---------------+---------------------------------------------------------------------------+ + Results are automatically pushed to the test results database, some additional result files are pushed to OPNFV artifact web sites. @@ -504,17 +501,10 @@ portal is also automatically updated. This portal provides information on: * Tempest: Tempest test case including reported errors per scenario and installer * vIMS: vIMS details per scenario and installer -.. figure:: ../images/functest-reporting-status.png +.. figure:: ../../../images/functest-reporting-status.png :align: center :alt: Functest reporting portal Fuel status page - -Test Dashboard -============== - -Based on results collected in CI, a test dashboard is dynamically generated. - - .. include:: ./troubleshooting.rst @@ -528,21 +518,20 @@ References .. _`[5]`: https://github.com/Orange-OpenSource/opnfv-cloudify-clearwater/blob/master/openstack-blueprint.yaml .. _`[6]`: https://scap.nist.gov/ .. _`[7]`: https://github.com/OpenSCAP/openscap -.. _`[9]`: https://git.opnfv.org/cgit/functest/tree/testcases/VIM/OpenStack/CI/libraries/os_defaults.yaml .. _`[11]`: http://robotframework.org/ .. _`[12]`: http://artifacts.opnfv.org/parser/colorado/docs/userguide/index.html -.. _`[13]`: TODO URL doc SNAPS +.. _`[13]`: https://wiki.opnfv.org/display/PROJ/SNAPS-OO +.. _`[14]`: https://github.com/oolorg/opnfv-functest-vrouter -OPNFV main site: opnfvmain_. +`OPNFV main site`_ -OPNFV functional test page: opnfvfunctest_. +`Functest page`_ IRC support chan: #opnfv-testperf -.. _opnfvmain: http://www.opnfv.org -.. _opnfvfunctest: https://wiki.opnfv.org/opnfv_functional_testing +.. _`OPNFV main site`: http://www.opnfv.org +.. _`Functest page`: https://wiki.opnfv.org/functest .. _`OpenRC`: http://docs.openstack.org/user-guide/common/cli_set_environment_variables_using_openstack_rc.html .. _`Rally installation procedure`: https://rally.readthedocs.org/en/latest/tutorial/step_0_installation.html -.. _`config_test.py` : https://git.opnfv.org/cgit/functest/tree/testcases/config_functest.py -.. _`config_functest.yaml` : https://git.opnfv.org/cgit/functest/tree/testcases/config_functest.yaml -.. _`Functest reporting`: http://testresults.opnfv.org/reporting/functest/release/colorado/index-status-fuel.html +.. _`config_functest.yaml` : https://git.opnfv.org/cgit/functest/tree/functest/ci/config_functest.yaml +.. _`Functest reporting`: http://testresults.opnfv.org/reporting/functest/release/danube/index-status-fuel.html diff --git a/docs/testing/user/userguide/introduction.rst b/docs/testing/user/userguide/introduction.rst index 4dfe79375..d1b7d23e0 100644 --- a/docs/testing/user/userguide/introduction.rst +++ b/docs/testing/user/userguide/introduction.rst @@ -22,15 +22,16 @@ validate the scenario for the release. +-------------+---------------+----------------+----------------------------------+ | Domain | Tier | Test case | Comments | +=============+===============+================+==================================+ -| VIM | healthcheck | healthcheck | Verify basic operation in VIM | -| | +----------------+----------------------------------+ -| | | connection | Check OpenStack connectivity | +| VIM | healthcheck | connection | Check OpenStack connectivity | | | | _check | through SNAPS framework | | | +----------------+----------------------------------+ | | | api_check | Check OpenStack API through | | | | | SNAPS framework | +| | +----------------+----------------------------------+ +| | | snaps_health | basic instance creation, check | +| | | \_check | DHCP | | +---------------+----------------+----------------------------------+ -| | smoke | vPing_SSH | NFV "Hello World" using an SSH | +| | smoke | vping_ssh | NFV "Hello World" using an SSH | | | | | connection to a destination VM | | | | | over a created floating IP | | | | | address on the SUT Public / | @@ -42,7 +43,7 @@ validate the scenario for the release. | | | | VM on a specified IP address over| | | | | the SUT Private Tenant network. | | | +----------------+----------------------------------+ -| | | vPing_userdata | Uses Ping with given userdata | +| | | vping_userdata | Uses Ping with given userdata | | | | | to test intra-VM connectivity | | | | | over the SUT Private Tenant | | | | | network. The correct operation | @@ -60,6 +61,9 @@ validate the scenario for the release. | | +----------------+----------------------------------+ | | | snaps_smoke | Run a subset of the OpenStack | | | | | Rally Test Suite in smoke mode | +| | +----------------+----------------------------------+ +| | | refstack | Reference RefStack suite | +| | | \_defcore | tempest selection for NFV | | +---------------+----------------+----------------------------------+ | | components | tempest_full | Generate and run a full set of | | | | \_parallel | the OpenStack Tempest Test Suite.| @@ -71,6 +75,9 @@ validate the scenario for the release. | | | rally_full | Run the OpenStack testing tool | | | | | benchmarking OpenStack modules | | | | | See the Rally documents `[3]`_. | +| | +----------------+----------------------------------+ +| | | tempest_custom | Allow to run a customized list | +| | | | of Tempest cases | +-------------+---------------+----------------+----------------------------------+ | Controllers | smoke | odl | Opendaylight Test suite | | | | | Limited test suite to check the | @@ -83,17 +90,27 @@ validate the scenario for the release. | | | | functions. | | | | | See `ONOSFW User Guide`_ for | | | | | details. | +| | +----------------+----------------------------------+ +| | | odl_netvirt | Test Suite for the OpenDaylight | +| | | | SDN Controller when the NetVirt | +| | | | features are installed. It | +| | | | integrates some test suites from | +| | | | upstream using Robot as the test | +| | | | framework | +| | +----------------+----------------------------------+ +| | | fds | Test Suite for the OpenDaylight | +| | | | SDN Controller when the GBP | +| | | | features are installed. It | +| | | | integrates some test suites from | +| | | | upstream using Robot as the test | +| | | | framework | +-------------+---------------+----------------+----------------------------------+ -| Features | features | promise | Resource reservation and | -| | | | management project to identify | -| | | | NFV related requirements and | -| | | | realize resource reservation for | -| | | | future usage by capacity | -| | | | management of resource pools | -| | | | regarding compute, network and | -| | | | storage. | -| | | | See `Promise User Guide`_ for | -| | | | details. | +| Features | features | bgpvpn | Implementation of the OpenStack | +| | | | bgpvpn API from the SDNVPN | +| | | | feature project. It allows for | +| | | | the creation of BGP VPNs. | +| | | | See `SDNVPN User Guide`_ for | +| | | | details | | | +----------------+----------------------------------+ | | | doctor | Doctor platform, as of Colorado | | | | | release, provides the three | @@ -106,25 +123,6 @@ validate the scenario for the release. | | | | See `Doctor User Guide`_ for | | | | | details | | | +----------------+----------------------------------+ -| | | bgpvpn | Implementation of the OpenStack | -| | | | bgpvpn API from the SDNVPN | -| | | | feature project. It allows for | -| | | | the creation of BGP VPNs. | -| | | | See `SDNVPN User Guide`_ for | -| | | | details | -| | +----------------+----------------------------------+ -| | | security_scan | Implementation of a simple | -| | | | security scan. (Currently | -| | | | available only for the Apex | -| | | | installer environment) | -| | +----------------+----------------------------------+ -| | | onos-sfc | SFC testing for onos scenarios | -| | | | See `ONOSFW User Guide`_ for | -| | | | details | -| | +----------------+----------------------------------+ -| | | odl-sfc | SFC testing for odl scenarios | -| | | | See `SFC User Guide`_ for details| -| | +----------------+----------------------------------+ | | | domino | Domino provides TOSCA template | | | | | distribution service for network | | | | | service and VNF descriptors | @@ -134,21 +132,41 @@ validate the scenario for the release. | | | | See `Domino User Guide`_ for | | | | | details | | | +----------------+----------------------------------+ -| | | copper | Copper develops OPNFV platform | -| | | | support for policy management, | -| | | | using open source projects such | -| | | | as OpenStack Congress, focused | -| | | | on helping ensure that virtual | -| | | | infrastructure and the apps that | -| | | | execute on it comply with the | -| | | | configuration policy intent of | -| | | | service providers, developers, | -| | | | and end users. See more detail | -| | | | in the `Copper User Guide`_. | -| | +----------------+----------------------------------+ -| | | multisites | Multisites | +| | | multisite | Multisite | | | | | See `Multisite User Guide`_ for | | | | | details | +| | +----------------+----------------------------------+ +| | | netready | Testing from netready project | +| | | | ping using gluon | +| | +----------------+----------------------------------+ +| | | odl-sfc | SFC testing for odl scenarios | +| | | | See `SFC User Guide`_ for details| +| | +----------------+----------------------------------+ +| | | parser | Parser is an integration project | +| | | | which aims to provide | +| | | | placement/deployment templates | +| | | | translation for OPNFV platform, | +| | | | including TOSCA -> HOT, POLICY ->| +| | | | TOSCA and YANG -> TOSCA. it | +| | | | deals with a fake vRNC. | +| | | | See `Parser User Guide`_ for | +| | | | details | +| | +----------------+----------------------------------+ +| | | promise | Resource reservation and | +| | | | management project to identify | +| | | | NFV related requirements and | +| | | | realize resource reservation for | +| | | | future usage by capacity | +| | | | management of resource pools | +| | | | regarding compute, network and | +| | | | storage. | +| | | | See `Promise User Guide`_ for | +| | | | details. | +| | +----------------+----------------------------------+ +| | | security_scan | Implementation of a simple | +| | | | security scan. (Currently | +| | | | available only for the Apex | +| | | | installer environment) | +-------------+---------------+----------------+----------------------------------+ | VNF | vnf | cloudify_ims | Example of a real VNF deployment | | | | | to show the NFV capabilities of | @@ -158,18 +176,10 @@ validate the scenario for the release. | | | | It provides a fully functional | | | | | VoIP System | | | +----------------+----------------------------------+ -| | | opera_ims | vIMS deployment using openBaton | +| | | orchestra_ims | OpenIMS deployment using | +| | | | Openbaton orchestrator | | | +----------------+----------------------------------+ -| | | orchestra_ims | vIMS deployment using open-O | -+ +---------------+----------------+----------------------------------+ -| | | parser | Parser is an integration project | -| | | | which aims to provide | -| | | | placement/deployment templates | -| | | | translation for OPNFV platform, | -| | | | including TOSCA -> HOT, POLICY ->| -| | | | TOSCA and YANG -> TOSCA. | -| | | | See `Parser User Guide`_ for | -| | | | details | +| | | vyos_vrouter | vRouter testing | +-------------+---------------+----------------+----------------------------------+ @@ -185,7 +195,7 @@ In Danube, we merged smoke and sdn controller tiers in smoke tier. An overview of the Functest Structural Concept is depicted graphically below: -.. figure:: ../images/concepts_mapping_final.png +.. figure:: ../../../images/concepts_mapping_final.png :align: center :alt: Functest Concepts Structure @@ -203,8 +213,7 @@ NoSQL database. The goal is to populate the database with results from different sources and scenarios and to show them on a `Functest Dashboard`_. A screenshot of a live Functest Dashboard is shown below: -** TODO ** -.. figure:: ../images/FunctestDashboardDanube.png +.. figure:: ../../../images/FunctestDashboardDanube.png :align: center :alt: Functest Dashboard diff --git a/docs/testing/user/userguide/runfunctest.rst b/docs/testing/user/userguide/runfunctest.rst index e7ab84b26..8a3ba8c73 100644 --- a/docs/testing/user/userguide/runfunctest.rst +++ b/docs/testing/user/userguide/runfunctest.rst @@ -48,33 +48,34 @@ command:: root@22e436918db0:~/repos/functest/ci# functest tier list - 0. healthcheck: - ['healthcheck', 'connection_check', 'api_check',] + ['connection_check', 'api_check', 'snaps_health_check',] - 1. smoke: - ['vping_ssh', 'vping_userdata', 'tempest_smoke_serial', 'rally_sanity', 'snaps_smoke', 'odl'] + ['vping_ssh', 'vping_userdata', 'tempest_smoke_serial', 'odl', 'rally_sanity', 'refstack_defcore', 'snaps_smoke'] - 2. features: - ['doctor', 'security_scan'] + ['doctor', 'domino', 'promise', security_scan'] - 3. components: ['tempest_full_parallel', 'rally_full'] - 4. vnf: - ['cloudify_ims'] + ['cloudify_ims', 'orchestra_ims', 'vyos_vrouter'] and root@22e436918db0:~/repos/functest/ci# functest testcase list - healthcheck api_check connection_check + snaps_health_check vping_ssh vping_userdata snaps_smoke + refstack_defcore tempest_smoke_serial rally_sanity odl - doctor - security_scan tempest_full_parallel rally_full - cloudify_ims + vyos_vrouter + +Note the list of test cases depend on the installer and the scenario. More specific details on specific Tiers or Test Cases can be seen wih the 'show' command:: @@ -117,55 +118,260 @@ More specific details on specific Tiers or Test Cases can be seen wih the To execute a Test Tier or Test Case, the 'run' command is used:: root@22e436918db0:~/repos/functest/ci# functest tier run healthcheck - Executing command: 'python /home/opnfv/repos/functest/ci/run_tests.py -t healthcheck' - 2016-06-30 11:44:56,933 - run_tests - INFO - Sourcing the OpenStack RC file... - 2016-06-30 11:44:56,937 - run_tests - INFO - ############################################ - 2016-06-30 11:44:56,938 - run_tests - INFO - Running tier 'healthcheck' - 2016-06-30 11:44:56,938 - run_tests - INFO - ############################################ - 2016-06-30 11:44:56,938 - run_tests - INFO - ============================================ - 2016-06-30 11:44:56,938 - run_tests - INFO - Running test case 'healthcheck'... - 2016-06-30 11:44:56,938 - run_tests - INFO - ============================================ - 2016-06-30 11:44:56,953 - healtcheck - INFO - Testing Keystone API... - 2016-06-30 11:45:05,351 - healtcheck - INFO - ...Keystone OK! - 2016-06-30 11:45:05,354 - healtcheck - INFO - Testing Glance API... - 2016-06-30 11:45:29,746 - healtcheck - INFO - ... Glance OK! - 2016-06-30 11:45:29,749 - healtcheck - INFO - Testing Cinder API... - 2016-06-30 11:45:37,502 - healtcheck - INFO - ...Cinder OK! - 2016-06-30 11:45:37,505 - healtcheck - INFO - Testing Neutron API... - 2016-06-30 11:45:39,664 - healtcheck - INFO - External network found. ccd98ad6-d34a-4768-b03c-e28ecfcd51ca - 2016-06-30 11:45:39,667 - healtcheck - INFO - 1. Create Networks... - 2016-06-30 11:45:44,227 - healtcheck - INFO - 2. Create subnets... - 2016-06-30 11:45:46,805 - healtcheck - INFO - 4. Create Routers... - 2016-06-30 11:45:54,261 - healtcheck - INFO - ...Neutron OK! - 2016-06-30 11:45:54,264 - healtcheck - INFO - Testing Nova API... - 2016-06-30 11:47:12,272 - healtcheck - INFO - ...Nova OK! - 2016-06-30 11:47:12,274 - healtcheck - INFO - Checking if instances get an IP from DHCP... - : - : - 2016-06-30 11:48:17,832 - healtcheck - INFO - ...DHCP OK! - 2016-06-30 11:48:17,835 - healtcheck - INFO - Health check passed! - 2016-06-30 11:48:17,837 - clean_openstack - INFO - +++++++++++++++++++++++++++++++ - 2016-06-30 11:48:17,837 - clean_openstack - INFO - Cleaning OpenStack resources... - 2016-06-30 11:48:17,837 - clean_openstack - INFO - +++++++++++++++++++++++++++++++ - Version 1 is deprecated, use alternative version 2 instead. - WARNING:cinderclient.api_versions:Version 1 is deprecated, use alternative version 2 instead. - 2016-06-30 11:48:18,272 - clean_openstack - INFO - Removing Nova instances... - 2016-06-30 11:48:24,439 - clean_openstack - INFO - ------------------------------------------- - 2016-06-30 11:48:24,440 - clean_openstack - INFO - Removing Glance images... - 2016-06-30 11:48:35,853 - clean_openstack - INFO - ------------------------------------------- - 2016-06-30 11:48:35,854 - clean_openstack - INFO - Removing Cinder volumes... - 2016-06-30 11:48:37,344 - clean_openstack - INFO - ------------------------------------------- - 2016-06-30 11:48:37,344 - clean_openstack - INFO - Removing floating IPs... - 2016-06-30 11:48:37,467 - clean_openstack - INFO - ------------------------------------------- - 2016-06-30 11:48:37,467 - clean_openstack - INFO - Removing Neutron objects - 2016-06-30 11:48:53,633 - clean_openstack - INFO - ------------------------------------------- - 2016-06-30 11:48:53,633 - clean_openstack - INFO - Removing Security groups... - 2016-06-30 11:48:53,689 - clean_openstack - INFO - ------------------------------------------- - 2016-06-30 11:48:53,689 - clean_openstack - INFO - Removing Users... - 2016-06-30 11:48:54,444 - clean_openstack - INFO - ------------------------------------------- - 2016-06-30 11:48:54,444 - clean_openstack - INFO - Removing Tenants... - 2016-06-30 11:48:54,711 - clean_openstack - INFO - ------------------------------------------- - + 2017-03-21 13:34:21,400 - run_tests - INFO - ############################################ + 2017-03-21 13:34:21,400 - run_tests - INFO - Running tier 'healthcheck' + 2017-03-21 13:34:21,400 - run_tests - INFO - ############################################ + 2017-03-21 13:34:21,401 - run_tests - INFO - + + 2017-03-21 13:34:21,401 - run_tests - INFO - ============================================ + 2017-03-21 13:34:21,401 - run_tests - INFO - Running test case 'connection_check'... + 2017-03-21 13:34:21,401 - run_tests - INFO - ============================================ + test_glance_connect_fail (snaps.openstack.utils.tests.glance_utils_tests.GlanceSmokeTests) ... ok + test_glance_connect_success (snaps.openstack.utils.tests.glance_utils_tests.GlanceSmokeTests) ... ok + test_keystone_connect_fail (snaps.openstack.utils.tests.keystone_utils_tests.KeystoneSmokeTests) ... ok + test_keystone_connect_success (snaps.openstack.utils.tests.keystone_utils_tests.KeystoneSmokeTests) ... ok + test_neutron_connect_fail (snaps.openstack.utils.tests.neutron_utils_tests.NeutronSmokeTests) ... ok + test_neutron_connect_success (snaps.openstack.utils.tests.neutron_utils_tests.NeutronSmokeTests) ... ok + test_retrieve_ext_network_name (snaps.openstack.utils.tests.neutron_utils_tests.NeutronSmokeTests) ... ok + test_nova_connect_fail (snaps.openstack.utils.tests.nova_utils_tests.NovaSmokeTests) ... ok + test_nova_connect_success (snaps.openstack.utils.tests.nova_utils_tests.NovaSmokeTests) ... ok + + ---------------------------------------------------------------------- + Ran 9 tests in 3.768s + + OK + 2017-03-21 13:34:26,570 - functest.core.testcase_base - INFO - connection_check OK + 2017-03-21 13:34:26,918 - functest.core.testcase_base - INFO - The results were successfully pushed to DB + 2017-03-21 13:34:26,918 - run_tests - INFO - Test execution time: 00:05 + 2017-03-21 13:34:26,918 - run_tests - INFO - + + 2017-03-21 13:34:26,918 - run_tests - INFO - ============================================ + 2017-03-21 13:34:26,918 - run_tests - INFO - Running test case 'api_check'... + 2017-03-21 13:34:26,919 - run_tests - INFO - ============================================ + test_create_project_minimal (snaps.openstack.utils.tests.keystone_utils_tests.KeystoneUtilsTests) ... ok + test_create_user_minimal (snaps.openstack.utils.tests.keystone_utils_tests.KeystoneUtilsTests) ... ok + test_create_delete_user (snaps.openstack.tests.create_user_tests.CreateUserSuccessTests) ... ok + test_create_user (snaps.openstack.tests.create_user_tests.CreateUserSuccessTests) ... ok + test_create_user_2x (snaps.openstack.tests.create_user_tests.CreateUserSuccessTests) ... + 2017-03-21 13:34:32,684 - create_user - INFO - Found user with name - CreateUserSuccessTests-7e741e11-c9fd-489-name ok + test_create_delete_project (snaps.openstack.tests.create_project_tests.CreateProjectSuccessTests) ... ok + test_create_project (snaps.openstack.tests.create_project_tests.CreateProjectSuccessTests) ... ok + test_create_project_2x (snaps.openstack.tests.create_project_tests.CreateProjectSuccessTests) ... + 2017-03-21 13:34:35,922 - create_image - INFO - Found project with name - CreateProjectSuccessTests-b38e08ce-2862-48a-name ok + test_create_project_sec_grp_one_user (snaps.openstack.tests.create_project_tests.CreateProjectUserTests) ... + 2017-03-21 13:34:37,907 - OpenStackSecurityGroup - INFO - Creating security group CreateProjectUserTests-ab8801f6-dad8-4f9-name... + 2017-03-21 13:34:37,907 - neutron_utils - INFO - Retrieving security group with name - CreateProjectUserTests-ab8801f6-dad8-4f9-name + 2017-03-21 13:34:38,376 - neutron_utils - INFO - Creating security group with name - CreateProjectUserTests-ab8801f6-dad8-4f9-name + 2017-03-21 13:34:38,716 - neutron_utils - INFO - Retrieving security group rules associate with the security group - CreateProjectUserTests-ab8801f6-dad8-4f9-name + 2017-03-21 13:34:38,762 - neutron_utils - INFO - Retrieving security group with ID - 821419cb-c54c-41b4-a61b-fb30e5dd2ec5 + 2017-03-21 13:34:38,886 - neutron_utils - INFO - Retrieving security group with ID - 821419cb-c54c-41b4-a61b-fb30e5dd2ec5 + 2017-03-21 13:34:39,000 - neutron_utils - INFO - Retrieving security group with name - CreateProjectUserTests-ab8801f6-dad8-4f9-name + 2017-03-21 13:34:39,307 - neutron_utils - INFO - Deleting security group rule with ID - d85fafc0-9649-45c9-a00e-452f3d5c09a6 + 2017-03-21 13:34:39,531 - neutron_utils - INFO - Deleting security group rule with ID - 69d79c09-bc3b-4975-9353-5f43aca51237 + 2017-03-21 13:34:39,762 - neutron_utils - INFO - Deleting security group with name - CreateProjectUserTests-ab8801f6-dad8-4f9-name ok + test_create_project_sec_grp_two_users (snaps.openstack.tests.create_project_tests.CreateProjectUserTests) ... + 2017-03-21 13:34:43,511 - OpenStackSecurityGroup - INFO - Creating security group CreateProjectUserTests-4d9261a6-e008-44b-name... + 2017-03-21 13:34:43,511 - neutron_utils - INFO - Retrieving security group with name - CreateProjectUserTests-4d9261a6-e008-44b-name + 2017-03-21 13:34:44,090 - neutron_utils - INFO - Creating security group with name - CreateProjectUserTests-4d9261a6-e008-44b-name + 2017-03-21 13:34:44,784 - neutron_utils - INFO - Retrieving security group rules associate with the security group - CreateProjectUserTests-4d9261a6-e008-44b-name + 2017-03-21 13:34:44,864 - neutron_utils - INFO - Retrieving security group with ID - 780193e4-9bd2-4f2e-a14d-b01abf74c832 + 2017-03-21 13:34:45,233 - neutron_utils - INFO - Retrieving security group with ID - 780193e4-9bd2-4f2e-a14d-b01abf74c832 + 2017-03-21 13:34:45,332 - neutron_utils - INFO - Retrieving security group with name - CreateProjectUserTests-4d9261a6-e008-44b-name + 2017-03-21 13:34:45,779 - OpenStackSecurityGroup - INFO - Creating security group CreateProjectUserTests-4d9261a6-e008-44b-name... + 2017-03-21 13:34:45,779 - neutron_utils - INFO - Retrieving security group with name - CreateProjectUserTests-4d9261a6-e008-44b-name + 2017-03-21 13:34:46,112 - neutron_utils - INFO - Retrieving security group rules associate with the security group - CreateProjectUserTests-4d9261a6-e008-44b-name + 2017-03-21 13:34:46,184 - neutron_utils - INFO - Retrieving security group with ID - 780193e4-9bd2-4f2e-a14d-b01abf74c832 + 2017-03-21 13:34:46,296 - neutron_utils - INFO - Retrieving security group with ID - 780193e4-9bd2-4f2e-a14d-b01abf74c832 + 2017-03-21 13:34:46,387 - neutron_utils - INFO - Deleting security group rule with ID - 2320a573-ec56-47c5-a1ba-ec514d30114b + 2017-03-21 13:34:46,636 - neutron_utils - INFO - Deleting security group rule with ID - 6186282b-db37-4e47-becc-a3886079c069 + 2017-03-21 13:34:46,780 - neutron_utils - INFO - Deleting security group with name - CreateProjectUserTests-4d9261a6-e008-44b-name + 2017-03-21 13:34:47,006 - neutron_utils - INFO - Deleting security group rule with ID - 2320a573-ec56-47c5-a1ba-ec514d30114b + 2017-03-21 13:34:47,072 - OpenStackSecurityGroup - WARNING - Rule not found, cannot delete - Security group rule 2320a573-ec56-47c5-a1ba-ec514d30114b does not exist + Neutron server returns request_ids: ['req-d74eb2e2-b26f-4236-87dc-7255866141d9'] + 2017-03-21 13:34:47,072 - neutron_utils - INFO - Deleting security group rule with ID - 6186282b-db37-4e47-becc-a3886079c069 + 2017-03-21 13:34:47,118 - OpenStackSecurityGroup - WARNING - Rule not found, cannot delete - Security group rule 6186282b-db37-4e47-becc-a3886079c069 does not exist + Neutron server returns request_ids: ['req-8c0a5a24-be90-4844-a9ed-2a85cc6f59a5'] + 2017-03-21 13:34:47,118 - neutron_utils - INFO - Deleting security group with name - CreateProjectUserTests-4d9261a6-e008-44b-name + 2017-03-21 13:34:47,172 - OpenStackSecurityGroup - WARNING - Security Group not found, cannot delete - Security group 780193e4-9bd2-4f2e-a14d-b01abf74c832 does not exist + Neutron server returns request_ids: ['req-c6e1a6b5-43e0-4d46-bb68-c2e1672d4d21'] ok + test_create_image_minimal_file (snaps.openstack.utils.tests.glance_utils_tests.GlanceUtilsTests) ... ok + test_create_image_minimal_url (snaps.openstack.utils.tests.glance_utils_tests.GlanceUtilsTests) ... ok + test_create_network (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsNetworkTests) ... + 2017-03-21 13:35:22,275 - neutron_utils - INFO - Creating network with name NeutronUtilsNetworkTests-c06c20e0-d78f-4fa4-8401-099a7a6cab2e-pub-net + 2017-03-21 13:35:23,965 - neutron_utils - INFO - Deleting network with name NeutronUtilsNetworkTests-c06c20e0-d78f-4fa4-8401-099a7a6cab2e-pub-net ok + test_create_network_empty_name (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsNetworkTests) ... ok + test_create_network_null_name (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsNetworkTests) ... ok + test_create_subnet (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsSubnetTests) ... + 2017-03-21 13:35:25,495 - neutron_utils - INFO - Creating network with name NeutronUtilsSubnetTests-4f440a5f-54e3-4455-ab9b-39dfe06f6d21-pub-net + 2017-03-21 13:35:26,841 - neutron_utils - INFO - Creating subnet with name NeutronUtilsSubnetTests-4f440a5f-54e3-4455-ab9b-39dfe06f6d21-pub-subnet + 2017-03-21 13:35:28,311 - neutron_utils - INFO - Deleting subnet with name NeutronUtilsSubnetTests-4f440a5f-54e3-4455-ab9b-39dfe06f6d21-pub-subnet + 2017-03-21 13:35:29,585 - neutron_utils - INFO - Deleting network with name NeutronUtilsSubnetTests-4f440a5f-54e3-4455-ab9b-39dfe06f6d21-pub-net ok + test_create_subnet_empty_cidr (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsSubnetTests) ... + 2017-03-21 13:35:31,013 - neutron_utils - INFO - Creating network with name NeutronUtilsSubnetTests-41fc0db4-71ee-47e6-bec9-316273e5bcc0-pub-net + 2017-03-21 13:35:31,652 - neutron_utils - INFO - Deleting network with name NeutronUtilsSubnetTests-41fc0db4-71ee-47e6-bec9-316273e5bcc0-pub-net ok + test_create_subnet_empty_name (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsSubnetTests) ... + 2017-03-21 13:35:32,379 - neutron_utils - INFO - Creating network with name NeutronUtilsSubnetTests-1030e0cb-1714-4d18-8619-a03bac0d0257-pub-net + 2017-03-21 13:35:33,516 - neutron_utils - INFO - Creating subnet with name NeutronUtilsSubnetTests-1030e0cb-1714-4d18-8619-a03bac0d0257-pub-subnet + 2017-03-21 13:35:34,160 - neutron_utils - INFO - Deleting network with name NeutronUtilsSubnetTests-1030e0cb-1714-4d18-8619-a03bac0d0257-pub-net ok + test_create_subnet_null_cidr (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsSubnetTests) ... + 2017-03-21 13:35:35,784 - neutron_utils - INFO - Creating network with name NeutronUtilsSubnetTests-1d7522fd-3fb5-4b1c-8741-97d7c47a5f7d-pub-net + 2017-03-21 13:35:36,367 - neutron_utils - INFO - Deleting network with name NeutronUtilsSubnetTests-1d7522fd-3fb5-4b1c-8741-97d7c47a5f7d-pub-net ok + test_create_subnet_null_name (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsSubnetTests) ... + 2017-03-21 13:35:37,055 - neutron_utils - INFO - Creating network with name NeutronUtilsSubnetTests-0a8ac1b2-e5d4-4522-a079-7e17945e482e-pub-net + 2017-03-21 13:35:37,691 - neutron_utils - INFO - Deleting network with name NeutronUtilsSubnetTests-0a8ac1b2-e5d4-4522-a079-7e17945e482e-pub-net ok + test_add_interface_router (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:35:38,994 - neutron_utils - INFO - Creating network with name NeutronUtilsRouterTests-433818c9-4472-49a8-9241-791ad0a71d3f-pub-net + 2017-03-21 13:35:40,311 - neutron_utils - INFO - Creating subnet with name NeutronUtilsRouterTests-433818c9-4472-49a8-9241-791ad0a71d3f-pub-subnet + 2017-03-21 13:35:41,713 - neutron_utils - INFO - Creating router with name - NeutronUtilsRouterTests-433818c9-4472-49a8-9241-791ad0a71d3f-pub-router + 2017-03-21 13:35:44,131 - neutron_utils - INFO - Adding interface to router with name NeutronUtilsRouterTests-433818c9-4472-49a8-9241-791ad0a71d3f-pub-router + 2017-03-21 13:35:45,725 - neutron_utils - INFO - Removing router interface from router named NeutronUtilsRouterTests-433818c9-4472-49a8-9241-791ad0a71d3f-pub-router + 2017-03-21 13:35:47,464 - neutron_utils - INFO - Deleting router with name - NeutronUtilsRouterTests-433818c9-4472-49a8-9241-791ad0a71d3f-pub-router + 2017-03-21 13:35:48,670 - neutron_utils - INFO - Deleting subnet with name NeutronUtilsRouterTests-433818c9-4472-49a8-9241-791ad0a71d3f-pub-subnet + 2017-03-21 13:35:50,921 - neutron_utils - INFO - Deleting network with name NeutronUtilsRouterTests-433818c9-4472-49a8-9241-791ad0a71d3f-pub-net ok + test_add_interface_router_null_router (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:35:52,230 - neutron_utils - INFO - Creating network with name NeutronUtilsRouterTests-1fc2de16-2d3e-497b-b947-022b1bf9d90c-pub-net + 2017-03-21 13:35:53,662 - neutron_utils - INFO - Creating subnet with name NeutronUtilsRouterTests-1fc2de16-2d3e-497b-b947-022b1bf9d90c-pub-subnet + 2017-03-21 13:35:55,203 - neutron_utils - INFO - Deleting subnet with name NeutronUtilsRouterTests-1fc2de16-2d3e-497b-b947-022b1bf9d90c-pub-subnet + 2017-03-21 13:35:55,694 - neutron_utils - INFO - Deleting network with name NeutronUtilsRouterTests-1fc2de16-2d3e-497b-b947-022b1bf9d90c-pub-net ok + test_add_interface_router_null_subnet (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:35:57,392 - neutron_utils - INFO - Creating network with name NeutronUtilsRouterTests-2e4fb9f3-312b-4954-8015-435464fdc8b0-pub-net + 2017-03-21 13:35:58,215 - neutron_utils - INFO - Creating router with name - NeutronUtilsRouterTests-2e4fb9f3-312b-4954-8015-435464fdc8b0-pub-router + 2017-03-21 13:36:00,369 - neutron_utils - INFO - Adding interface to router with name NeutronUtilsRouterTests-2e4fb9f3-312b-4954-8015-435464fdc8b0-pub-router + 2017-03-21 13:36:00,369 - neutron_utils - INFO - Deleting router with name - NeutronUtilsRouterTests-2e4fb9f3-312b-4954-8015-435464fdc8b0-pub-router + 2017-03-21 13:36:02,742 - neutron_utils - INFO - Deleting network with name NeutronUtilsRouterTests-2e4fb9f3-312b-4954-8015-435464fdc8b0-pub-net ok + test_create_port (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:36:05,010 - neutron_utils - INFO - Creating network with name NeutronUtilsRouterTests-dde05ce1-a2f8-4c5e-a028-e1ca0e11a05b-pub-net + 2017-03-21 13:36:05,996 - neutron_utils - INFO - Creating subnet with name NeutronUtilsRouterTests-dde05ce1-a2f8-4c5e-a028-e1ca0e11a05b-pub-subnet + 2017-03-21 13:36:09,103 - neutron_utils - INFO - Creating port for network with name - NeutronUtilsRouterTests-dde05ce1-a2f8-4c5e-a028-e1ca0e11a05b-pub-net + 2017-03-21 13:36:10,312 - neutron_utils - INFO - Deleting port with name NeutronUtilsRouterTests-dde05ce1-a2f8-4c5e-a028-e1ca0e11a05b-port + 2017-03-21 13:36:11,045 - neutron_utils - INFO - Deleting subnet with name NeutronUtilsRouterTests-dde05ce1-a2f8-4c5e-a028-e1ca0e11a05b-pub-subnet + 2017-03-21 13:36:14,265 - neutron_utils - INFO - Deleting network with name NeutronUtilsRouterTests-dde05ce1-a2f8-4c5e-a028-e1ca0e11a05b-pub-net ok + test_create_port_empty_name (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:36:16,250 - neutron_utils - INFO - Creating network with name NeutronUtilsRouterTests-b986a259-e873-431c-bde4-b2771ace4549-pub-net + 2017-03-21 13:36:16,950 - neutron_utils - INFO - Creating subnet with name NeutronUtilsRouterTests-b986a259-e873-431c-bde4-b2771ace4549-pub-subnet + 2017-03-21 13:36:17,798 - neutron_utils - INFO - Creating port for network with name - NeutronUtilsRouterTests-b986a259-e873-431c-bde4-b2771ace4549-pub-net + 2017-03-21 13:36:18,544 - neutron_utils - INFO - Deleting port with name NeutronUtilsRouterTests-b986a259-e873-431c-bde4-b2771ace4549-port + 2017-03-21 13:36:19,582 - neutron_utils - INFO - Deleting subnet with name NeutronUtilsRouterTests-b986a259-e873-431c-bde4-b2771ace4549-pub-subnet + 2017-03-21 13:36:21,606 - neutron_utils - INFO - Deleting network with name NeutronUtilsRouterTests-b986a259-e873-431c-bde4-b2771ace4549-pub-net ok + test_create_port_invalid_ip (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:36:23,779 - neutron_utils - INFO - Creating network with name NeutronUtilsRouterTests-7ab3a329-9dd8-4e6f-9d52-aafb47ea5122-pub-net + 2017-03-21 13:36:25,201 - neutron_utils - INFO - Creating subnet with name NeutronUtilsRouterTests-7ab3a329-9dd8-4e6f-9d52-aafb47ea5122-pub-subnet + 2017-03-21 13:36:25,599 - neutron_utils - INFO - Deleting subnet with name NeutronUtilsRouterTests-7ab3a329-9dd8-4e6f-9d52-aafb47ea5122-pub-subnet + 2017-03-21 13:36:26,220 - neutron_utils - INFO - Deleting network with name NeutronUtilsRouterTests-7ab3a329-9dd8-4e6f-9d52-aafb47ea5122-pub-net ok + test_create_port_invalid_ip_to_subnet (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:36:27,112 - neutron_utils - INFO - Creating network with name NeutronUtilsRouterTests-c016821d-cd4f-4e0f-8f8c-d5cef3392e64-pub-net + 2017-03-21 13:36:28,720 - neutron_utils - INFO - Creating subnet with name NeutronUtilsRouterTests-c016821d-cd4f-4e0f-8f8c-d5cef3392e64-pub-subnet + 2017-03-21 13:36:29,457 - neutron_utils - INFO - Deleting subnet with name NeutronUtilsRouterTests-c016821d-cd4f-4e0f-8f8c-d5cef3392e64-pub-subnet + 2017-03-21 13:36:29,909 - neutron_utils - INFO - Deleting network with name NeutronUtilsRouterTests-c016821d-cd4f-4e0f-8f8c-d5cef3392e64-pub-net ok + test_create_port_null_ip (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:36:31,037 - neutron_utils - INFO - Creating network with name NeutronUtilsRouterTests-9a86227f-6041-4b04-86a7-1701fb86baa3-pub-net + 2017-03-21 13:36:31,695 - neutron_utils - INFO - Creating subnet with name NeutronUtilsRouterTests-9a86227f-6041-4b04-86a7-1701fb86baa3-pub-subnet + 2017-03-21 13:36:32,305 - neutron_utils - INFO - Deleting subnet with name NeutronUtilsRouterTests-9a86227f-6041-4b04-86a7-1701fb86baa3-pub-subnet + 2017-03-21 13:36:33,553 - neutron_utils - INFO - Deleting network with name NeutronUtilsRouterTests-9a86227f-6041-4b04-86a7-1701fb86baa3-pub-net ok + test_create_port_null_name (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:36:34,593 - neutron_utils - INFO - Creating network with name NeutronUtilsRouterTests-42efa897-4f65-4d9b-b19d-fbc61f97c966-pub-net + 2017-03-21 13:36:35,217 - neutron_utils - INFO - Creating subnet with name NeutronUtilsRouterTests-42efa897-4f65-4d9b-b19d-fbc61f97c966-pub-subnet + 2017-03-21 13:36:36,648 - neutron_utils - INFO - Deleting subnet with name NeutronUtilsRouterTests-42efa897-4f65-4d9b-b19d-fbc61f97c966-pub-subnet + 2017-03-21 13:36:37,251 - neutron_utils - INFO - Deleting network with name NeutronUtilsRouterTests-42efa897-4f65-4d9b-b19d-fbc61f97c966-pub-net ok + test_create_port_null_network_object (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:36:37,885 - neutron_utils - INFO - Creating network with name NeutronUtilsRouterTests-617f4110-45c1-4900-bad1-a6204f34dd64-pub-net + 2017-03-21 13:36:38,468 - neutron_utils - INFO - Creating subnet with name NeutronUtilsRouterTests-617f4110-45c1-4900-bad1-a6204f34dd64-pub-subnet + 2017-03-21 13:36:40,005 - neutron_utils - INFO - Deleting subnet with name NeutronUtilsRouterTests-617f4110-45c1-4900-bad1-a6204f34dd64-pub-subnet + 2017-03-21 13:36:41,637 - neutron_utils - INFO - Deleting network with name NeutronUtilsRouterTests-617f4110-45c1-4900-bad1-a6204f34dd64-pub-net ok + test_create_router_empty_name (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... ok + test_create_router_null_name (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... ok + test_create_router_simple (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:36:43,424 - neutron_utils - INFO - Creating router with name - NeutronUtilsRouterTests-b6a2dafc-38d4-4c46-bb41-2ba9e1c0084e-pub-router + 2017-03-21 13:36:45,013 - neutron_utils - INFO - Deleting router with name - NeutronUtilsRouterTests-b6a2dafc-38d4-4c46-bb41-2ba9e1c0084e-pub-router ok + test_create_router_with_public_interface (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsRouterTests) ... + 2017-03-21 13:36:47,829 - neutron_utils - INFO - Creating router with name - NeutronUtilsRouterTests-d268dda2-7a30-4d3d-a008-e5aa4592637d-pub-router + 2017-03-21 13:36:49,448 - neutron_utils - INFO - Deleting router with name - NeutronUtilsRouterTests-d268dda2-7a30-4d3d-a008-e5aa4592637d-pub-router ok + test_create_delete_simple_sec_grp (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsSecurityGroupTests) ... + 2017-03-21 13:36:51,067 - neutron_utils - INFO - Creating security group with name - NeutronUtilsSecurityGroupTests-1543e861-ea38-4fbe-9723-c27552e3eb7aname + 2017-03-21 13:36:51,493 - neutron_utils - INFO - Retrieving security group with name - NeutronUtilsSecurityGroupTests-1543e861-ea38-4fbe-9723-c27552e3eb7aname + 2017-03-21 13:36:51,568 - neutron_utils - INFO - Deleting security group with name - NeutronUtilsSecurityGroupTests-1543e861-ea38-4fbe-9723-c27552e3eb7aname + 2017-03-21 13:36:51,772 - neutron_utils - INFO - Retrieving security group with name - NeutronUtilsSecurityGroupTests-1543e861-ea38-4fbe-9723-c27552e3eb7aname ok + test_create_sec_grp_no_name (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsSecurityGroupTests) ... ok + test_create_sec_grp_no_rules (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsSecurityGroupTests) ... + 2017-03-21 13:36:52,253 - neutron_utils - INFO - Creating security group with name - NeutronUtilsSecurityGroupTests-57c60864-f46c-4391-ba99-6acc4dd123ddname + 2017-03-21 13:36:52,634 - neutron_utils - INFO - Retrieving security group with name - NeutronUtilsSecurityGroupTests-57c60864-f46c-4391-ba99-6acc4dd123ddname + 2017-03-21 13:36:52,718 - neutron_utils - INFO - Deleting security group with name - NeutronUtilsSecurityGroupTests-57c60864-f46c-4391-ba99-6acc4dd123ddname ok + test_create_sec_grp_one_rule (snaps.openstack.utils.tests.neutron_utils_tests.NeutronUtilsSecurityGroupTests) ... + 2017-03-21 13:36:53,082 - neutron_utils - INFO - Creating security group with name - NeutronUtilsSecurityGroupTests-a3ac62bb-a7e8-4fc2-ba4c-e656f1f3c9a1name + 2017-03-21 13:36:53,483 - neutron_utils - INFO - Retrieving security group rules associate with the security group - NeutronUtilsSecurityGroupTests-a3ac62bb-a7e8-4fc2-ba4c-e656f1f3c9a1name + 2017-03-21 13:36:53,548 - neutron_utils - INFO - Creating security group to security group - NeutronUtilsSecurityGroupTests-a3ac62bb-a7e8-4fc2-ba4c-e656f1f3c9a1name + 2017-03-21 13:36:53,548 - neutron_utils - INFO - Retrieving security group with name - NeutronUtilsSecurityGroupTests-a3ac62bb-a7e8-4fc2-ba4c-e656f1f3c9a1name + 2017-03-21 13:36:53,871 - neutron_utils - INFO - Retrieving security group with name - NeutronUtilsSecurityGroupTests-a3ac62bb-a7e8-4fc2-ba4c-e656f1f3c9a1name + 2017-03-21 13:36:53,944 - neutron_utils - INFO - Retrieving security group rules associate with the security group - NeutronUtilsSecurityGroupTests-a3ac62bb-a7e8-4fc2-ba4c-e656f1f3c9a1name + 2017-03-21 13:36:53,991 - neutron_utils - INFO - Retrieving security group with name - NeutronUtilsSecurityGroupTests-a3ac62bb-a7e8-4fc2-ba4c-e656f1f3c9a1name + 2017-03-21 13:36:54,069 - neutron_utils - INFO - Deleting security group rule with ID - 7f76046c-d043-46e0-9d12-4b983525810b + 2017-03-21 13:36:54,185 - neutron_utils - INFO - Deleting security group rule with ID - f18a9ed1-466f-4373-a6b2-82bd317bc838 + 2017-03-21 13:36:54,338 - neutron_utils - INFO - Deleting security group rule with ID - fe34a3d0-948e-47c1-abad-c3ec8d33b2fb + 2017-03-21 13:36:54,444 - neutron_utils - INFO - Deleting security group with name - NeutronUtilsSecurityGroupTests-a3ac62bb-a7e8-4fc2-ba4c-e656f1f3c9a1name ok + test_create_delete_keypair (snaps.openstack.utils.tests.nova_utils_tests.NovaUtilsKeypairTests) ... + 2017-03-21 13:36:54,637 - nova_utils - INFO - Creating keypair with name - NovaUtilsKeypairTests-5ce69b6f-d8d0-4b66-bd25-30a22cf3bda0 ok + test_create_key_from_file (snaps.openstack.utils.tests.nova_utils_tests.NovaUtilsKeypairTests) ... + 2017-03-21 13:36:58,989 - nova_utils - INFO - Saved public key to - tmp/NovaUtilsKeypairTests-df3e848d-a467-4cc4-99d5-022eb67eee94.pub + 2017-03-21 13:36:58,990 - nova_utils - INFO - Saved private key to - tmp/NovaUtilsKeypairTests-df3e848d-a467-4cc4-99d5-022eb67eee94 + 2017-03-21 13:36:58,990 - nova_utils - INFO - Saving keypair to - tmp/NovaUtilsKeypairTests-df3e848d-a467-4cc4-99d5-022eb67eee94.pub + 2017-03-21 13:36:58,990 - nova_utils - INFO - Creating keypair with name - NovaUtilsKeypairTests-df3e848d-a467-4cc4-99d5-022eb67eee94 ok + test_create_keypair (snaps.openstack.utils.tests.nova_utils_tests.NovaUtilsKeypairTests) ... + 2017-03-21 13:36:59,807 - nova_utils - INFO - Creating keypair with name - NovaUtilsKeypairTests-fc7f7ffd-80f6-43df-bd41-a3c014ba8c3d ok + test_floating_ips (snaps.openstack.utils.tests.nova_utils_tests.NovaUtilsKeypairTests) ... + 2017-03-21 13:37:02,765 - nova_utils - INFO - Creating floating ip to external network - admin_floating_net ok + test_create_delete_flavor (snaps.openstack.utils.tests.nova_utils_tests.NovaUtilsFlavorTests) ... ok + test_create_flavor (snaps.openstack.utils.tests.nova_utils_tests.NovaUtilsFlavorTests) ... ok + test_create_clean_flavor (snaps.openstack.tests.create_flavor_tests.CreateFlavorTests) ... ok + test_create_delete_flavor (snaps.openstack.tests.create_flavor_tests.CreateFlavorTests) ... ok + test_create_flavor (snaps.openstack.tests.create_flavor_tests.CreateFlavorTests) ... ok + test_create_flavor_existing (snaps.openstack.tests.create_flavor_tests.CreateFlavorTests) ... + 2017-03-21 13:37:18,545 - create_image - INFO - Found flavor with name - CreateFlavorTests-3befc152-4319-4f9c-82d4-75f8941d9533name ok + + ---------------------------------------------------------------------- + Ran 48 tests in 171.000s + + OK + 2017-03-21 13:37:18,620 - functest.core.testcase_base - INFO - api_check OK + 2017-03-21 13:37:18,977 - functest.core.testcase_base - INFO - The results were successfully pushed to DB + 2017-03-21 13:37:18,977 - run_tests - INFO - Test execution time: 02:52 + 2017-03-21 13:37:18,981 - run_tests - INFO - + + 2017-03-21 13:37:18,981 - run_tests - INFO - ============================================ + 2017-03-21 13:37:18,981 - run_tests - INFO - Running test case 'snaps_health_check'... + 2017-03-21 13:37:18,981 - run_tests - INFO - ============================================ + 2017-03-21 13:37:19,098 - file_utils - INFO - Attempting to read OS environment file - /home/opnfv/functest/conf/openstack.creds + 2017-03-21 13:37:19,099 - openstack_tests - INFO - OS Credentials = OSCreds - username=admin, password=admin, auth_url=http://192.168.10.7:5000/v3, project_name=admin, identity_api_version=3, image_api_version=1, network_api_version=2, compute_api_version=2, user_domain_id=default, proxy_settings=None + 2017-03-21 13:37:19,434 - file_utils - INFO - Attempting to read OS environment file - /home/opnfv/functest/conf/openstack.creds + 2017-03-21 13:37:19,435 - openstack_tests - INFO - OS Credentials = OSCreds - username=admin, password=admin, auth_url=http://192.168.10.7:5000/v3, project_name=admin, identity_api_version=3, image_api_version=1, network_api_version=2, compute_api_version=2, user_domain_id=default, proxy_settings=None + test_check_vm_ip_dhcp (snaps.openstack.tests.create_instance_tests.SimpleHealthCheck) ... + 2017-03-21 13:37:26,082 - create_image - INFO - Creating image + 2017-03-21 13:37:28,793 - create_image - INFO - Image is active with name - SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5d-image + 2017-03-21 13:37:28,793 - create_image - INFO - Image is now active with name - SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5d-image + 2017-03-21 13:37:28,794 - OpenStackNetwork - INFO - Creating neutron network SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5d-priv-net... + 2017-03-21 13:37:29,308 - neutron_utils - INFO - Creating network with name SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5d-priv-net + 2017-03-21 13:37:30,771 - neutron_utils - INFO - Creating subnet with name SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5d-priv-subnet + 2017-03-21 13:37:36,974 - neutron_utils - INFO - Creating port for network with name - SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5d-priv-net + 2017-03-21 13:37:38,188 - create_instance - INFO - Creating VM with name - SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5d-inst + 2017-03-21 13:37:41,538 - create_instance - INFO - Created instance with name - SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5d-inst + 2017-03-21 13:37:59,577 - create_instance - INFO - VM is - ACTIVE + 2017-03-21 13:37:59,577 - create_instance_tests - INFO - Looking for expression Lease of.*obtained in the console log + 2017-03-21 13:37:59,830 - create_instance_tests - INFO - DHCP lease obtained logged in console + 2017-03-21 13:37:59,830 - create_instance_tests - INFO - With correct IP address + 2017-03-21 13:37:59,830 - create_instance - INFO - Deleting Port - SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5dport-1 + 2017-03-21 13:37:59,830 - neutron_utils - INFO - Deleting port with name SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5dport-1 + 2017-03-21 13:38:00,705 - create_instance - INFO - Deleting VM instance - SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5d-inst + 2017-03-21 13:38:01,412 - create_instance - INFO - Checking deletion status + 2017-03-21 13:38:04,938 - create_instance - INFO - VM has been properly deleted VM with name - SimpleHealthCheck-23244728-5a5a-4545-9b16-50257a595e5d-inst + ok + + ---------------------------------------------------------------------- + Ran 1 test in 46.982s + + OK + 2017-03-21 13:38:06,417 - functest.core.testcase_base - INFO - snaps_health_check OK + 2017-03-21 13:38:06,778 - functest.core.testcase_base - INFO - The results were successfully pushed to DB + 2017-03-21 13:38:06,779 - run_tests - INFO - Test execution time: 00:47 + 2017-03-21 13:38:06,779 - run_tests - INFO - and root@22e436918db0:~/repos/functest/ci# functest testcase run vping_ssh @@ -206,12 +412,12 @@ command is used with 'functest tier':: root@22e436918db0:~/repos/functest/ci# functest tier get-tests healthcheck Test cases in tier 'healthcheck': - ['healthcheck'] + ['connection_check', 'api_check', 'snaps_health_check'] Please note that for some scenarios some test cases might not be launched. For example, the last example displayed only the 'odl' testcase for the given -environment. In this particular system the deployment does not support the 'onos' SDN +environment. In this particular system the deployment does not support the 'ocl' SDN Controller Test Case; for example. **Important** If you use the command 'functest tier run <tier_name>', then the @@ -229,32 +435,35 @@ two possibilities:: Functest includes a cleaning mechanism in order to remove all the OpenStack resources except those present before running any test. The script -*$REPOS_DIR/functest/functest/utils/generate_defaults.py* is called once when setting up +*$REPOS_DIR/functest/functest/utils/openstack_snapshot.py* is called once when setting up the Functest environment (i.e. CLI command 'functest env prepare') to snapshot all the OpenStack resources (images, networks, volumes, security groups, tenants, users) so that an eventual cleanup does not remove any of these defaults. -The script **clean_openstack.py** which is located in +It is also called before running a test except if it is disabled by configuration +in the testcases.yaml file (clean_flag=false). This flag has been added as some +upstream tests already include their own cleaning mechanism (e.g. Rally). + +The script **openstack_clean.py** which is located in *$REPOS_DIR/functest/functest/utils/* is normally called after a test execution. It is in charge of cleaning the OpenStack resources that are not specified in the defaults file generated previously which is stored in -*/home/opnfv/functest/conf/os_defaults.yaml* in the Functest docker container. +*/home/opnfv/functest/conf/openstack_snapshot.yaml* in the Functest docker container. It is important to mention that if there are new OpenStack resources created -manually after preparing the Functest environment, they will be removed, unless -you use the special method of invoking the test case with specific suppression -of clean up. (See the `Troubleshooting`_ section). +manually after the snapshot done before running the tests, they will be removed, +unless you use the special method of invoking the test case with specific +suppression of clean up. (See the `Troubleshooting`_ section). The reason to include this cleanup meachanism in Functest is because some -test suites such as Tempest or Rally create a lot of resources (users, -tenants, networks, volumes etc.) that are not always properly cleaned, so this -function has been set to keep the system as clean as it was before a -full Functest execution. +test suites create a lot of resources (users, tenants, networks, volumes etc.) +that are not always properly cleaned, so this function has been set to keep the +system as clean as it was before a full Functest execution. Although the Functest CLI provides an easy way to run any test, it is possible to do a direct call to the desired test script. For example: - python $REPOS_DIR/functest/functest/opnfv_tests/OpenStack/vPing/vPing_ssh.py -d + python $REPOS_DIR/functest/functest/opnfv_tests/openstack/vping/vping_ssh.py Automated testing @@ -282,38 +491,18 @@ The mechanism remains however as part of the CI evolution. CI provides some useful information passed to the container as environment variables: - * Installer (apex|compass|daisy|fuel|joid), stored in INSTALLER_TYPE + * Installer (apex|compass|fuel|joid), stored in INSTALLER_TYPE * Installer IP of the engine or VM running the actual deployment, stored in INSTALLER_IP * The scenario [controller]-[feature]-[mode], stored in DEPLOY_SCENARIO with - * controller = (odl|onos|ocl|nosdn) - * feature = (ovs(dpdk)|kvm|sfc|bgpvpn|multisites) + * controller = (odl|ocl|nosdn|onos) + * feature = (ovs(dpdk)|kvm|sfc|bgpvpn|multisites|netready|ovs_dpdk_bar) * mode = (ha|noha) The constraints per test case are defined in the Functest configuration file */home/opnfv/repos/functest/functest/ci/testcases.yaml*:: tiers: - - - name: healthcheck - order: 0 - ci_loop: '(daily)|(weekly)' - description : >- - First tier to be executed to verify the basic - operations in the VIM. - testcases: - - - name: healthcheck - criteria: 'status == "PASS"' - blocking: true - description: >- - This test case verifies the basic OpenStack services like - Keystone, Glance, Cinder, Neutron and Nova. - - dependencies: - installer: '' - scenario: '' - - name: smoke order: 1 @@ -343,13 +532,14 @@ We may distinguish 2 levels in the test case description: At the tier level, we define the following parameters: - * ci_loop: indicate if in automated mode, the test case must be run in daily and/or weekly jobs + * ci_loop: indicate if in automated mode, the test case must be run in dail and/or weekly jobs * description: a high level view of the test case For a given test case we defined: * the name of the test case * the criteria (experimental): a criteria used to declare the test case as PASS or FAIL * blocking: if set to true, if the test is failed, the execution of the following tests is canceled + * clean_flag: shall the functect internal mechanism be invoked after the test * the description of the test case * the dependencies: a combination of 2 regex on the scenario and the installer name * run: In Danube we introduced the notion of abstract class in order to harmonize the way to run internal, feature or vnf tests diff --git a/docs/testing/user/userguide/troubleshooting.rst b/docs/testing/user/userguide/troubleshooting.rst index 845501916..bcd9e1c84 100644 --- a/docs/testing/user/userguide/troubleshooting.rst +++ b/docs/testing/user/userguide/troubleshooting.rst @@ -44,8 +44,8 @@ These test cases can be run inside the container, using new Functest CLI as foll The Functest CLI is designed to route a call to the corresponding internal python scripts, located in paths: -*$REPOS_DIR/functest/functest/opnfv_tests/vPing/CI/libraries/vPing_ssh.py* and -*$REPOS_DIR/functest/functest/opnfv_tests/vPing/CI/libraries/vPing_userdata.py* +*$REPOS_DIR/functest/functest/opnfv_tests/openstack/vping/vping_ssh.py* and +*$REPOS_DIR/functest/functest/opnfv_tests/openstack/vping/vping_userdata.py* Notes: @@ -122,7 +122,7 @@ This test case creates a floating IP on the external network and assigns it to the second instance **opnfv-vping-2**. The purpose of this is to establish a SSH connection to that instance and SCP a script that will ping the first instance. This script is located in the repository under -*$REPOS_DIR/functest/functest/opnfv_tests/OpenStack/vPing/ping.sh* and takes an IP as +*$REPOS_DIR/functest/functest/opnfv_tests/openstack/vping/ping.sh* and takes an IP as a parameter. When the SCP is completed, the test will do an SSH call to that script inside the second instance. Some problems can happen here:: @@ -238,7 +238,8 @@ of the following | execution test cases are | Rally in Functest, does not contain all the needed | | missing | parameters or some parameters are not set properly. | | | The tempest.conf file is located in directory | -| | '/home/opnfv/.rally/tempest/for-deployment-<UUID>' | +| | '/home/opnfv/.rally/verification/verifier-<UUID> | +| | /for-deployment-<UUID>' | | | in the Functest Docker container. Use the "rally | | | deployment list" command in order to check the UUID | | | the UUID of the current deployment. | @@ -257,30 +258,6 @@ Rally The same error causes which were mentioned above for Tempest test cases, may also lead to errors in Rally as well. -It is possible to run only one Rally scenario, instead of the whole suite. -To do that, call the alternative python script as follows:: - - python $REPOS_DIR/functest/functest/opnfv_tests/OpenStack/rally/run_rally-cert.py -h - usage: run_rally-cert.py [-h] [-d] [-r] [-s] [-v] [-n] test_name - - positional arguments: - test_name Module name to be tested. Possible values are : [ - authenticate | glance | cinder | heat | keystone | neutron | - nova | quotas | requests | vm | all ] The 'all' value - performs all possible test scenarios - - optional arguments: - -h, --help show this help message and exit - -d, --debug Debug mode - -r, --report Create json result file - -s, --smoke Smoke test mode - -v, --verbose Print verbose info about the progress - -n, --noclean Don't clean the created resources for this test. - -For example, to run the Glance scenario with debug information:: - - python $REPOS_DIR/functest/functest/opnfv_tests/OpenStack/rally/run_rally-cert.py -d glance - Possible scenarios are: * authenticate * glance @@ -294,14 +271,10 @@ Possible scenarios are: * vm To know more about what those scenarios are doing, they are defined in directory: -*$REPOS_DIR/functest/functest/opnfv_tests/OpenStack/rally/scenario* +*$REPOS_DIR/functest/functest/opnfv_tests/openstack/rally/scenario* For more info about Rally scenario definition please refer to the Rally official documentation. `[3]`_ -If the flag *all* is specified, it will run all the scenarios one by one. Please -note that this might take some time (~1,5hr), taking around 1 hour alone to -complete the Nova scenario. - To check any possible problems with Rally, the logs are stored under */home/opnfv/functest/results/rally/* in the Functest Docker container. @@ -322,10 +295,8 @@ If any of the other test cases fails, check that Neutron and ODL have been correctly configured to work together. Check Neutron configuration files, accounts, IP addresses etc.). - ONOS ^^^^ - Please refer to the ONOS documentation. `ONOSFW User Guide`_ . @@ -335,14 +306,8 @@ Features Please refer to the dedicated feature user guides for details. -security_scan -^^^^^^^^^^^^^ - -See OpenSCAP web site: https://www.open-scap.org/ - - -NFV +VNF --- cloudify_ims @@ -378,10 +343,5 @@ described in the following table: +-----------------------------------+------------------------------------+ -parser -^^^^^^ - -For now log info is the only way to do trouble shooting - .. _`OPNFV Functest Developer Guide`: http://artifacts.opnfv.org/functest/docs/devguide/# diff --git a/functest/ci/config_functest.yaml b/functest/ci/config_functest.yaml index 78f6257c8..95a4408a1 100755 --- a/functest/ci/config_functest.yaml +++ b/functest/ci/config_functest.yaml @@ -27,6 +27,7 @@ general: repo_parser: /home/opnfv/repos/parser repo_domino: /home/opnfv/repos/domino repo_snaps: /home/opnfv/repos/snaps + repo_opera: /home/opnfv/repos/opera repo_fds: /home/opnfv/repos/fds repo_securityscan: /home/opnfv/repos/securityscanning repo_vrouter: /home/opnfv/repos/vnfs/vrouter diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py index 4a47ba570..37b90f922 100755 --- a/functest/ci/run_tests.py +++ b/functest/ci/run_tests.py @@ -18,7 +18,7 @@ import sys import functest.ci.generate_report as generate_report import functest.ci.tier_builder as tb -import functest.core.testcase_base as testcase_base +import functest.core.testcase as testcase import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils import functest.utils.openstack_clean as os_clean @@ -138,7 +138,7 @@ def run_test(test, tier_name, testcases=None): if GlobalVariables.REPORT_FLAG: flags += " -r" - result = testcase_base.TestcaseBase.EX_RUN_ERROR + result = testcase.TestCase.EX_RUN_ERROR run_dict = get_run_dict(test_name) if run_dict: try: @@ -151,7 +151,7 @@ def run_test(test, tier_name, testcases=None): result = test_case.run(**kwargs) except KeyError: result = test_case.run() - if result == testcase_base.TestcaseBase.EX_OK: + if result == testcase.TestCase.EX_OK: if GlobalVariables.REPORT_FLAG: test_case.push_to_db() result = test_case.check_criteria() diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 5f54b975d..1c33b9962 100755 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -78,7 +78,7 @@ tiers: over a private network. dependencies: installer: '' - scenario: '^((?!odl_l3).)*$' + scenario: '^((?!odl_l3|odl-bgpvpn|gluon).)*$' run: module: 'functest.opnfv_tests.openstack.vping.vping_ssh' class: 'VPingSSH' @@ -487,7 +487,7 @@ tiers: scenario: '(ocl)|(nosdn)|^(os-odl)((?!bgpvpn).)*$' run: module: 'functest.opnfv_tests.vnf.ims.cloudify_ims' - class: 'ImsVnf' + class: 'CloudifyIms' # - # name: aaa # criteria: 'ret == 0' @@ -517,17 +517,17 @@ tiers: - name: opera_ims - criteria: 'ret == 0' + criteria: 'status == "PASS"' blocking: false clean_flag: true description: >- - Evolution of vIMS + VNF deployment with OPEN-O dependencies: installer: 'unknown' scenario: 'unknown' run: module: 'functest.opnfv_tests.vnf.ims.opera_ims' - class: 'ImsVnf' + class: 'OperaIms' - name: vyos_vrouter diff --git a/functest/core/feature_base.py b/functest/core/feature.py index 2bd1ec83d..325c10d49 100644 --- a/functest/core/feature_base.py +++ b/functest/core/feature.py @@ -1,15 +1,15 @@ import time -import testcase_base as base +import testcase as base import functest.utils.functest_utils as ft_utils import functest.utils.functest_logger as ft_logger from functest.utils.constants import CONST -class FeatureBase(base.TestcaseBase): +class Feature(base.TestCase): def __init__(self, project='functest', case='', repo='', cmd=''): - super(FeatureBase, self).__init__() + super(Feature, self).__init__() self.project_name = project self.case_name = case self.cmd = cmd @@ -26,7 +26,7 @@ class FeatureBase(base.TestcaseBase): self.parse_results(ret) self.log_results() self.logger.info("Test result is stored in '%s'" % self.result_file) - return base.TestcaseBase.EX_OK + return base.TestCase.EX_OK def execute(self): ''' @@ -42,13 +42,13 @@ class FeatureBase(base.TestcaseBase): pass def parse_results(self, ret): - exit_code = base.TestcaseBase.EX_OK + exit_code = base.TestCase.EX_OK if ret == 0: self.logger.info("{} OK".format(self.project_name)) self.criteria = 'PASS' else: self.logger.info("{} FAILED".format(self.project_name)) - exit_code = base.TestcaseBase.EX_RUN_ERROR + exit_code = base.TestCase.EX_RUN_ERROR self.criteria = "FAIL" return exit_code diff --git a/functest/core/pytest_suite_runner.py b/functest/core/pytest_suite_runner.py index f0ae265a4..4f777628a 100644 --- a/functest/core/pytest_suite_runner.py +++ b/functest/core/pytest_suite_runner.py @@ -5,12 +5,12 @@ # # http://www.apache.org/licenses/LICENSE-2.0 -import testcase_base as base +import testcase as base import unittest import time -class PyTestSuiteRunner(base.TestcaseBase): +class PyTestSuiteRunner(base.TestCase): """ This superclass is designed to execute pre-configured unittest.TestSuite() objects @@ -42,9 +42,9 @@ class PyTestSuiteRunner(base.TestcaseBase): # a result can be PASS or FAIL # But in this case it means that the Execution was OK # we shall distinguish Execution Error from FAIL results - # TestcaseBase.EX_RUN_ERROR means that the test case was not run + # TestCase.EX_RUN_ERROR means that the test case was not run # not that it was run but the result was FAIL - exit_code = base.TestcaseBase.EX_OK + exit_code = base.TestCase.EX_OK if ((result.errors and len(result.errors) > 0) or (result.failures and len(result.failures) > 0)): self.logger.info("%s FAILED" % self.case_name) diff --git a/functest/core/testcase.py b/functest/core/testcase.py new file mode 100644 index 000000000..b540cfb5a --- /dev/null +++ b/functest/core/testcase.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python + +# Copyright (c) 2016 Orange and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +"""Define the parent class of Functest TestCase.""" + +import os + +import functest.utils.functest_logger as ft_logger +import functest.utils.functest_utils as ft_utils + +__author__ = "Cedric Ollivier <cedric.ollivier@orange.com>" + + +class TestCase(object): + """Parent class of Functest TestCase.""" + + EX_OK = os.EX_OK + EX_RUN_ERROR = os.EX_SOFTWARE + EX_PUSH_TO_DB_ERROR = os.EX_SOFTWARE - 1 + EX_TESTCASE_FAILED = os.EX_SOFTWARE - 2 + + logger = ft_logger.Logger(__name__).getLogger() + + def __init__(self): + self.details = {} + self.project_name = "functest" + self.case_name = "" + self.criteria = "" + self.start_time = "" + self.stop_time = "" + + def check_criteria(self): + """Interpret the results of TestCase. + + It allows getting the results of TestCase. It completes run() + which only returns the execution status. + + It can be overriden if checking criteria is not suitable. + + Returns: + TestCase.EX_OK if criteria is 'PASS'. + TestCase.EX_TESTCASE_FAILED otherwise. + """ + try: + assert self.criteria + if self.criteria == 'PASS': + return TestCase.EX_OK + except AssertionError: + self.logger.error("Please run test before checking the results") + return TestCase.EX_TESTCASE_FAILED + + def run(self, **kwargs): + """Run TestCase. + + It allows running TestCase and getting its execution + status. + + The subclasses must override the default implementation which + is false on purpose. The only prerequisite is to set the + following attributes to push the results to DB: + * case_name, + * criteria, + * start_time, + * stop_time. + + Args: + **kwargs: Arbitrary keyword arguments. + + Returns: + TestCase.EX_RUN_ERROR. + """ + # pylint: disable=unused-argument + self.logger.error("Run must be implemented") + return TestCase.EX_RUN_ERROR + + def push_to_db(self): + """Push the results of TestCase to the DB. + + It allows publishing the results and to check the status. + + It could be overriden if the common implementation is not + suitable. The following attributes must be set before pushing + the results to DB: + * case_name, + * criteria, + * start_time, + * stop_time. + + Returns: + TestCase.EX_OK if results were pushed to DB. + TestCase.EX_PUSH_TO_DB_ERROR otherwise. + """ + try: + assert self.project_name + assert self.case_name + assert self.criteria + assert self.start_time + assert self.stop_time + if ft_utils.push_results_to_db( + self.project_name, self.case_name, self.start_time, + self.stop_time, self.criteria, self.details): + self.logger.info("The results were successfully pushed to DB") + return TestCase.EX_OK + else: + self.logger.error("The results cannot be pushed to DB") + return TestCase.EX_PUSH_TO_DB_ERROR + except Exception: # pylint: disable=broad-except + self.logger.exception("The results cannot be pushed to DB") + return TestCase.EX_PUSH_TO_DB_ERROR diff --git a/functest/core/testcase_base.py b/functest/core/testcase_base.py deleted file mode 100644 index 838b63983..000000000 --- a/functest/core/testcase_base.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2016 Orange and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 - -import os - -import functest.utils.functest_logger as ft_logger -import functest.utils.functest_utils as ft_utils - - -class TestcaseBase(object): - - EX_OK = os.EX_OK - EX_RUN_ERROR = os.EX_SOFTWARE - EX_PUSH_TO_DB_ERROR = os.EX_SOFTWARE - 1 - EX_TESTCASE_FAILED = os.EX_SOFTWARE - 2 - - logger = ft_logger.Logger(__name__).getLogger() - - def __init__(self): - self.details = {} - self.project_name = "functest" - self.case_name = "" - self.criteria = "" - self.start_time = "" - self.stop_time = "" - - def check_criteria(self): - try: - assert self.criteria - if self.criteria == 'PASS': - return TestcaseBase.EX_OK - except: - self.logger.error("Please run test before checking the results") - return TestcaseBase.EX_TESTCASE_FAILED - - def run(self, **kwargs): - self.logger.error("Run must be implemented") - return TestcaseBase.EX_RUN_ERROR - - def push_to_db(self): - try: - assert self.project_name - assert self.case_name - assert self.criteria - assert self.start_time - assert self.stop_time - if ft_utils.push_results_to_db( - self.project_name, self.case_name, self.start_time, - self.stop_time, self.criteria, self.details): - self.logger.info("The results were successfully pushed to DB") - return TestcaseBase.EX_OK - else: - self.logger.error("The results cannot be pushed to DB") - return TestcaseBase.EX_PUSH_TO_DB_ERROR - except Exception: - self.logger.exception("The results cannot be pushed to DB") - return TestcaseBase.EX_PUSH_TO_DB_ERROR diff --git a/functest/core/vnf_base.py b/functest/core/vnf_base.py index f5e86054f..3f0adcc66 100644 --- a/functest/core/vnf_base.py +++ b/functest/core/vnf_base.py @@ -7,18 +7,17 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 -import time - import inspect +import time +import functest.core.testcase as base +from functest.utils.constants import CONST import functest.utils.functest_logger as ft_logger -import functest.utils.openstack_utils as os_utils import functest.utils.functest_utils as ft_utils -import testcase_base as base -from functest.utils.constants import CONST +import functest.utils.openstack_utils as os_utils -class VnfOnBoardingBase(base.TestcaseBase): +class VnfOnBoardingBase(base.TestCase): logger = ft_logger.Logger(__name__).getLogger() @@ -29,7 +28,7 @@ class VnfOnBoardingBase(base.TestcaseBase): self.case_name = case self.cmd = cmd self.details = {} - self.data_dir = CONST.dir_functest_data + self.result_dir = CONST.dir_results self.details['orchestrator'] = {} self.details['vnf'] = {} self.details['test_vnf'] = {} @@ -39,14 +38,14 @@ class VnfOnBoardingBase(base.TestcaseBase): 'vnf_{}_tenant_name'.format(self.case_name)) self.tenant_description = CONST.__getattribute__( 'vnf_{}_tenant_description'.format(self.case_name)) - except: + except Exception: # raise Exception("Unknown VNF case=" + self.case_name) self.logger.error("Unknown VNF case={}".format(self.case_name)) try: self.images = CONST.__getattribute__( 'vnf_{}_tenant_images'.format(self.case_name)) - except: + except Exception: self.logger.warn("No tenant image defined for this VNF") def execute(self): @@ -58,7 +57,7 @@ class VnfOnBoardingBase(base.TestcaseBase): except Exception: self.logger.error("Error during VNF Onboarding environment" + "creation", exc_info=True) - return base.TestcaseBase.EX_TESTCASE_FAILED + return base.TestCase.EX_TESTCASE_FAILED # Deploy orchestrator try: @@ -87,7 +86,7 @@ class VnfOnBoardingBase(base.TestcaseBase): vnf_ready_time - orchestrator_ready_time, 1) except Exception: self.logger.error("Error during VNF deployment", exc_info=True) - return base.TestcaseBase.EX_TESTCASE_FAILED + return base.TestCase.EX_TESTCASE_FAILED # Test VNF try: @@ -100,7 +99,7 @@ class VnfOnBoardingBase(base.TestcaseBase): test_vnf_done_time - vnf_ready_time, 1) except Exception: self.logger.error("Error when running VNF tests", exc_info=True) - return base.TestcaseBase.EX_TESTCASE_FAILED + return base.TestCase.EX_TESTCASE_FAILED # Clean the system self.clean() @@ -234,7 +233,12 @@ class VnfOnBoardingBase(base.TestcaseBase): def step_failure(self, error_msg): part = inspect.stack()[1][3] - self.details[part]['status'] = 'FAIL' - self.details[part]['result'] = error_msg - self.logger.error("Step failure:{}".format(error_msg)) + self.logger.error("Step '%s' failed: %s", part, error_msg) + try: + part_info = self.details[part] + except KeyError: + self.details[part] = {} + part_info = self.details[part] + part_info['status'] = 'FAIL' + part_info['result'] = error_msg raise Exception(error_msg) diff --git a/functest/opnfv_tests/features/barometer.py b/functest/opnfv_tests/features/barometer.py index 32067284b..6011340f8 100644 --- a/functest/opnfv_tests/features/barometer.py +++ b/functest/opnfv_tests/features/barometer.py @@ -8,10 +8,10 @@ from baro_tests import collectd -import functest.core.feature_base as base +import functest.core.feature as base -class BarometerCollectd(base.FeatureBase): +class BarometerCollectd(base.Feature): ''' Class for executing barometercollectd testcase. ''' diff --git a/functest/opnfv_tests/features/copper.py b/functest/opnfv_tests/features/copper.py index 735b315d2..689341eab 100644 --- a/functest/opnfv_tests/features/copper.py +++ b/functest/opnfv_tests/features/copper.py @@ -14,10 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import functest.core.feature_base as base +import functest.core.feature as base -class Copper(base.FeatureBase): +class Copper(base.Feature): def __init__(self): super(Copper, self).__init__(project='copper', case='copper-notification', diff --git a/functest/opnfv_tests/features/doctor.py b/functest/opnfv_tests/features/doctor.py index 4d295a674..d32bbfc99 100644 --- a/functest/opnfv_tests/features/doctor.py +++ b/functest/opnfv_tests/features/doctor.py @@ -13,10 +13,10 @@ # 0.2: measure test duration and publish results under json format # # -import functest.core.feature_base as base +import functest.core.feature as base -class Doctor(base.FeatureBase): +class Doctor(base.Feature): def __init__(self): super(Doctor, self).__init__(project='doctor', case='doctor-notification', diff --git a/functest/opnfv_tests/features/domino.py b/functest/opnfv_tests/features/domino.py index b36220fa0..e34429bc2 100644 --- a/functest/opnfv_tests/features/domino.py +++ b/functest/opnfv_tests/features/domino.py @@ -14,10 +14,10 @@ # 0.3: add report flag to push results when needed # 0.4: refactoring to match Test abstraction class -import functest.core.feature_base as base +import functest.core.feature as base -class Domino(base.FeatureBase): +class Domino(base.Feature): def __init__(self): super(Domino, self).__init__(project='domino', case='domino-multinode', diff --git a/functest/opnfv_tests/features/netready.py b/functest/opnfv_tests/features/netready.py index dec2a23ce..88f377c26 100644 --- a/functest/opnfv_tests/features/netready.py +++ b/functest/opnfv_tests/features/netready.py @@ -8,10 +8,10 @@ # # -import functest.core.feature_base as base +import functest.core.feature as base -class GluonVping(base.FeatureBase): +class GluonVping(base.Feature): def __init__(self): super(GluonVping, self).__init__(project='netready', diff --git a/functest/opnfv_tests/features/odl_sfc.py b/functest/opnfv_tests/features/odl_sfc.py index 431cd47e4..fff7f2b0d 100644 --- a/functest/opnfv_tests/features/odl_sfc.py +++ b/functest/opnfv_tests/features/odl_sfc.py @@ -7,16 +7,14 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -import functest.core.feature_base as base -from sfc.tests.functest import run_tests +import functest.core.feature as base -class OpenDaylightSFC(base.FeatureBase): +class OpenDaylightSFC(base.Feature): def __init__(self): super(OpenDaylightSFC, self).__init__(project='sfc', case='functest-odl-sfc', repo='dir_repo_sfc') - - def execute(self): - return run_tests.main() + dir_sfc_functest = '{}/sfc/tests/functest'.format(self.repo) + self.cmd = 'cd %s && python ./run_tests.py' % dir_sfc_functest diff --git a/functest/opnfv_tests/features/promise.py b/functest/opnfv_tests/features/promise.py index 15636fbfe..a7f4e6283 100644 --- a/functest/opnfv_tests/features/promise.py +++ b/functest/opnfv_tests/features/promise.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import functest.core.feature_base as base +import functest.core.feature as base -class Promise(base.FeatureBase): +class Promise(base.Feature): def __init__(self): super(Promise, self).__init__(project='promise', case='promise', diff --git a/functest/opnfv_tests/features/sdnvpn.py b/functest/opnfv_tests/features/sdnvpn.py index 1919a03c2..10e3146c8 100644 --- a/functest/opnfv_tests/features/sdnvpn.py +++ b/functest/opnfv_tests/features/sdnvpn.py @@ -7,10 +7,10 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -import functest.core.feature_base as base +import functest.core.feature as base -class SdnVpnTests(base.FeatureBase): +class SdnVpnTests(base.Feature): def __init__(self): super(SdnVpnTests, self).__init__(project='sdnvpn', diff --git a/functest/opnfv_tests/features/security_scan.py b/functest/opnfv_tests/features/security_scan.py index 58f0ec748..2374b39fe 100644 --- a/functest/opnfv_tests/features/security_scan.py +++ b/functest/opnfv_tests/features/security_scan.py @@ -8,11 +8,11 @@ # http://www.apache.org/licenses/LICENSE-2.0 # -import functest.core.feature_base as base +import functest.core.feature as base from functest.utils.constants import CONST -class SecurityScan(base.FeatureBase): +class SecurityScan(base.Feature): def __init__(self): super(SecurityScan, self).__init__(project='securityscanning', case='security_scan', diff --git a/functest/opnfv_tests/mano/orchestra.py b/functest/opnfv_tests/mano/orchestra.py index fd5e40d05..a9cf0ae66 100644 --- a/functest/opnfv_tests/mano/orchestra.py +++ b/functest/opnfv_tests/mano/orchestra.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import functest.core.feature_base as base +import functest.core.feature as base -class Orchestra(base.FeatureBase): +class Orchestra(base.Feature): def __init__(self): super(Orchestra, self).__init__(project='orchestra', case='orchestra', diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index 46d6a5706..8c6abc157 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -17,7 +17,7 @@ import time import iniparse import yaml -from functest.core import testcase_base +from functest.core import testcase from functest.utils.constants import CONST import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils @@ -26,7 +26,7 @@ import functest.utils.openstack_utils as os_utils logger = ft_logger.Logger('Rally').getLogger() -class RallyBase(testcase_base.TestcaseBase): +class RallyBase(testcase.TestCase): TESTS = ['authenticate', 'glance', 'cinder', 'heat', 'keystone', 'neutron', 'nova', 'quotas', 'requests', 'vm', 'all'] GLANCE_IMAGE_NAME = CONST.openstack_image_name @@ -526,10 +526,10 @@ class RallyBase(testcase_base.TestcaseBase): self._run_tests() self._generate_report() self._clean_up() - res = testcase_base.TestcaseBase.EX_OK + res = testcase.TestCase.EX_OK except Exception as e: logger.error('Error with run: %s' % e) - res = testcase_base.TestcaseBase.EX_RUN_ERROR + res = testcase.TestCase.EX_RUN_ERROR self.stop_time = time.time() return res diff --git a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py index 4312b7f9e..37aa9e399 100755 --- a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py +++ b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py @@ -12,18 +12,18 @@ import sys import subprocess import time -from functest.core import testcase_base +from functest.core import testcase from functest.opnfv_tests.openstack.tempest import conf_utils -from functest.utils import openstack_utils from functest.utils.constants import CONST import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils +from tempest_conf import TempestConf """ logging configuration """ logger = ft_logger.Logger("refstack_defcore").getLogger() -class RefstackClient(testcase_base.TestcaseBase): +class RefstackClient(testcase.TestCase): def __init__(self): super(RefstackClient, self).__init__() @@ -35,12 +35,6 @@ class RefstackClient(testcase_base.TestcaseBase): self.CONF_PATH) self.defcorelist = os.path.join(self.FUNCTEST_TEST, self.DEFCORE_LIST) - self.VERIFIER_ID = conf_utils.get_verifier_id() - self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir( - self.VERIFIER_ID) - self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id() - self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir( - self.VERIFIER_ID, self.DEPLOYMENT_ID) def source_venv(self): @@ -143,59 +137,62 @@ class RefstackClient(testcase_base.TestcaseBase): logger.info("Testcase %s success_rate is %s%%, is marked as %s" % (self.case_name, success_rate, self.criteria)) - def defcore_env_prepare(self): - try: - img_flavor_dict = conf_utils.create_tempest_resources( - use_custom_images=True, use_custom_flavors=True) - conf_utils.configure_tempest_defcore( - self.DEPLOYMENT_DIR, img_flavor_dict) - self.source_venv() - res = testcase_base.TestcaseBase.EX_OK - except KeyError as e: - logger.error("defcore prepare env error with: %s", e) - res = testcase_base.TestcaseBase.EX_RUN_ERROR - - return res - def run(self): + '''used for functest command line, + functest testcase run refstack_defcore''' self.start_time = time.time() if not os.path.exists(conf_utils.REFSTACK_RESULTS_DIR): os.makedirs(conf_utils.REFSTACK_RESULTS_DIR) try: - self.defcore_env_prepare() + tempestconf = TempestConf() + tempestconf.generate_tempestconf() + self.source_venv() self.run_defcore_default() self.parse_refstack_result() - res = testcase_base.TestcaseBase.EX_OK + res = testcase.TestCase.EX_OK except Exception as e: logger.error('Error with run: %s', e) - res = testcase_base.TestcaseBase.EX_RUN_ERROR + res = testcase.TestCase.EX_RUN_ERROR self.stop_time = time.time() return res + def _prep_test(self): + '''Check that the config file exists.''' + if not os.path.isfile(self.confpath): + logger.error("Conf file not valid: %s" % self.confpath) + if not os.path.isfile(self.testlist): + logger.error("testlist file not valid: %s" % self.testlist) + def main(self, **kwargs): + '''used for manually running, + python refstack_client.py -c <tempest_conf_path> + --testlist <testlist_path> + can generate a reference tempest.conf by + python tempest_conf.py + ''' try: - tempestconf = kwargs['config'] - testlist = kwargs['testlist'] + self.confpath = kwargs['config'] + self.testlist = kwargs['testlist'] except KeyError as e: logger.error("Cannot run refstack client. Please check " "%s", e) return self.EX_RUN_ERROR try: - openstack_utils.source_credentials(CONST.openstack_creds) - self.defcore_env_prepare() - self.run_defcore(tempestconf, testlist) - res = testcase_base.TestcaseBase.EX_OK + self.source_venv() + self._prep_test() + self.run_defcore(self.confpath, self.testlist) + res = testcase.TestCase.EX_OK except Exception as e: logger.error('Error with run: %s', e) - res = testcase_base.TestcaseBase.EX_RUN_ERROR + res = testcase.TestCase.EX_RUN_ERROR return res -class RefstackClientParser(testcase_base.TestcaseBase): +class RefstackClientParser(testcase.TestCase): def __init__(self): super(RefstackClientParser, self).__init__() @@ -228,7 +225,7 @@ if __name__ == '__main__': args = parser.parse_args(sys.argv[1:]) try: result = refstackclient.main(**args) - if result != testcase_base.TestcaseBase.EX_OK: + if result != testcase.TestCase.EX_OK: sys.exit(result) except Exception: - sys.exit(testcase_base.TestcaseBase.EX_RUN_ERROR) + sys.exit(testcase.TestCase.EX_RUN_ERROR) diff --git a/functest/opnfv_tests/openstack/refstack_client/tempest_conf.py b/functest/opnfv_tests/openstack/refstack_client/tempest_conf.py new file mode 100755 index 000000000..5624ed79b --- /dev/null +++ b/functest/opnfv_tests/openstack/refstack_client/tempest_conf.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# matthew.lijun@huawei.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +import sys + +from functest.core import testcase +from functest.opnfv_tests.openstack.tempest import conf_utils +from functest.utils import openstack_utils +from functest.utils.constants import CONST +import functest.utils.functest_logger as ft_logger + +""" logging configuration """ +logger = ft_logger.Logger("refstack_defcore").getLogger() + + +class TempestConf(object): + def __init__(self): + self.VERIFIER_ID = conf_utils.get_verifier_id() + self.VERIFIER_REPO_DIR = conf_utils.get_verifier_repo_dir( + self.VERIFIER_ID) + self.DEPLOYMENT_ID = conf_utils.get_verifier_deployment_id() + self.DEPLOYMENT_DIR = conf_utils.get_verifier_deployment_dir( + self.VERIFIER_ID, self.DEPLOYMENT_ID) + + def generate_tempestconf(self): + try: + openstack_utils.source_credentials(CONST.openstack_creds) + img_flavor_dict = conf_utils.create_tempest_resources( + use_custom_images=True, use_custom_flavors=True) + conf_utils.configure_tempest_defcore( + self.DEPLOYMENT_DIR, img_flavor_dict) + except KeyError as e: + logger.error("defcore prepare env error with: %s", e) + + def main(self): + try: + self.generate_tempestconf() + res = testcase.TestCase.EX_OK + except Exception as e: + logger.error('Error with run: %s', e) + res = testcase.TestCase.EX_RUN_ERROR + + return res + +if __name__ == '__main__': + tempestconf = TempestConf() + result = tempestconf.main() + if result != testcase.TestCase.EX_OK: + sys.exit(result) diff --git a/functest/opnfv_tests/openstack/tempest/custom_tests/blacklist.txt b/functest/opnfv_tests/openstack/tempest/custom_tests/blacklist.txt index 43edabc10..0da92cd8a 100644 --- a/functest/opnfv_tests/openstack/tempest/custom_tests/blacklist.txt +++ b/functest/opnfv_tests/openstack/tempest/custom_tests/blacklist.txt @@ -27,11 +27,7 @@ scenarios: - os-onos-nofeature-ha - os-onos-nofeature-noha - - os-onos-sfc-ha - - os-onos-sfc-noha installers: - - fuel - - apex - compass tests: - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard @@ -41,22 +37,6 @@ - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern - - scenarios: - - os-onos-nofeature-ha - - os-onos-nofeature-noha - - os-onos-sfc-ha - - os-onos-sfc-noha - installers: - - joid - tests: - - tempest.api.object_storage - - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard - - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops - - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops - - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern - - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern - -- # https://bugs.launchpad.net/tempest/+bug/1577632 scenarios: - os-odl_l2-nofeature-ha @@ -71,3 +51,15 @@ - fuel tests: - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops + + + # https://bugs.opendaylight.org/show_bug.cgi?id=5586 + scenarios: + - os-odl-bgpvpn-ha + - os-odl-gluon-noha + - os-odl_l2-bgpvpn-ha + installers: + - apex + - fuel + tests: + - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index 54556a32f..0addbd172 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -16,7 +16,7 @@ import time import yaml -from functest.core import testcase_base +from functest.core import testcase from functest.opnfv_tests.openstack.tempest import conf_utils from functest.utils.constants import CONST import functest.utils.functest_logger as ft_logger @@ -26,7 +26,7 @@ import functest.utils.functest_utils as ft_utils logger = ft_logger.Logger("Tempest").getLogger() -class TempestCommon(testcase_base.TestcaseBase): +class TempestCommon(testcase.TestCase): def __init__(self): super(TempestCommon, self).__init__() @@ -223,10 +223,10 @@ class TempestCommon(testcase_base.TestcaseBase): self.apply_tempest_blacklist() self.run_verifier_tests() self.parse_verifier_result() - res = testcase_base.TestcaseBase.EX_OK + res = testcase.TestCase.EX_OK except Exception as e: logger.error('Error with run: %s' % e) - res = testcase_base.TestcaseBase.EX_RUN_ERROR + res = testcase.TestCase.EX_RUN_ERROR self.stop_time = time.time() return res diff --git a/functest/opnfv_tests/openstack/vping/vping_base.py b/functest/opnfv_tests/openstack/vping/vping_base.py index 9d57cfaea..584ded386 100644 --- a/functest/opnfv_tests/openstack/vping/vping_base.py +++ b/functest/opnfv_tests/openstack/vping/vping_base.py @@ -12,12 +12,12 @@ import pprint import time from datetime import datetime -import functest.core.testcase_base as testcase_base +import functest.core.testcase as testcase import functest.utils.openstack_utils as os_utils from functest.utils.constants import CONST -class VPingBase(testcase_base.TestcaseBase): +class VPingBase(testcase.TestCase): def __init__(self): super(VPingBase, self).__init__() self.logger = None @@ -52,23 +52,23 @@ class VPingBase(testcase_base.TestcaseBase): def run(self, **kwargs): if not self.check_repo_exist(): - return testcase_base.TestcaseBase.EX_RUN_ERROR + return testcase.TestCase.EX_RUN_ERROR image_id = self.create_image() if not image_id: - return testcase_base.TestcaseBase.EX_RUN_ERROR + return testcase.TestCase.EX_RUN_ERROR flavor = self.get_flavor() if not flavor: - return testcase_base.TestcaseBase.EX_RUN_ERROR + return testcase.TestCase.EX_RUN_ERROR network_id = self.create_network_full() if not network_id: - return testcase_base.TestcaseBase.EX_RUN_ERROR + return testcase.TestCase.EX_RUN_ERROR sg_id = self.create_security_group() if not sg_id: - return testcase_base.TestcaseBase.EX_RUN_ERROR + return testcase.TestCase.EX_RUN_ERROR self.delete_exist_vms() @@ -84,7 +84,7 @@ class VPingBase(testcase_base.TestcaseBase): None, sg_id) if not vm1: - return testcase_base.TestcaseBase.EX_RUN_ERROR + return testcase.TestCase.EX_RUN_ERROR test_ip = self.get_test_ip(vm1) vm2 = self.boot_vm(self.vm2_name, @@ -94,17 +94,17 @@ class VPingBase(testcase_base.TestcaseBase): test_ip, sg_id) if not vm2: - return testcase_base.TestcaseBase.EX_RUN_ERROR + return testcase.TestCase.EX_RUN_ERROR EXIT_CODE = self.do_vping(vm2, test_ip) - if EXIT_CODE == testcase_base.TestcaseBase.EX_RUN_ERROR: + if EXIT_CODE == testcase.TestCase.EX_RUN_ERROR: return EXIT_CODE self.stop_time = time.time() self.parse_result(EXIT_CODE, self.start_time, self.stop_time) - return testcase_base.TestcaseBase.EX_OK + return testcase.TestCase.EX_OK def boot_vm_preparation(self, config, vmname, test_ip): pass diff --git a/functest/opnfv_tests/openstack/vping/vping_ssh.py b/functest/opnfv_tests/openstack/vping/vping_ssh.py index 7a58a41fa..fc2f01c64 100755 --- a/functest/opnfv_tests/openstack/vping/vping_ssh.py +++ b/functest/opnfv_tests/openstack/vping/vping_ssh.py @@ -19,7 +19,7 @@ from scp import SCPClient import functest.utils.functest_logger as ft_logger import functest.utils.openstack_utils as os_utils import vping_base -import functest.core.testcase_base as testcase_base +import functest.core.testcase as testcase class VPingSSH(vping_base.VPingBase): @@ -32,12 +32,12 @@ class VPingSSH(vping_base.VPingBase): def do_vping(self, vm, test_ip): floatip = self.add_float_ip(vm) if not floatip: - return testcase_base.TestcaseBase.EX_RUN_ERROR + return testcase.TestCase.EX_RUN_ERROR ssh = self.establish_ssh(vm, floatip) if not ssh: - return testcase_base.TestcaseBase.EX_RUN_ERROR + return testcase.TestCase.EX_RUN_ERROR if not self.transfer_ping_script(ssh, floatip): - return testcase_base.TestcaseBase.EX_RUN_ERROR + return testcase.TestCase.EX_RUN_ERROR return self.do_vping_ssh(ssh, test_ip) def add_float_ip(self, vm): diff --git a/functest/opnfv_tests/sdn/odl/odl.py b/functest/opnfv_tests/sdn/odl/odl.py index c8e9c4920..ccc1101a5 100755 --- a/functest/opnfv_tests/sdn/odl/odl.py +++ b/functest/opnfv_tests/sdn/odl/odl.py @@ -7,6 +7,15 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 +"""Define classes required to run ODL suites. + +It has been designed for any context. But helpers are given for +running test suites in OPNFV environment. + +Example: + $ python odl.py +""" + import argparse import errno import fileinput @@ -20,12 +29,15 @@ from robot.errors import RobotError import robot.run from robot.utils.robottime import timestamp_to_secs -from functest.core import testcase_base +from functest.core import testcase import functest.utils.functest_logger as ft_logger import functest.utils.openstack_utils as op_utils +__author__ = "Cedric Ollivier <cedric.ollivier@orange.com>" + class ODLResultVisitor(robot.api.ResultVisitor): + """Visitor to get result details.""" def __init__(self): self._data = [] @@ -43,10 +55,12 @@ class ODLResultVisitor(robot.api.ResultVisitor): self._data.append(output) def get_data(self): + """Get the details of the result.""" return self._data -class ODLTests(testcase_base.TestcaseBase): +class ODLTests(testcase.TestCase): + """ODL test runner.""" repos = "/home/opnfv/repos/" odl_test_repo = os.path.join(repos, "odl_test") @@ -59,11 +73,17 @@ class ODLTests(testcase_base.TestcaseBase): logger = ft_logger.Logger("opendaylight").getLogger() def __init__(self): - testcase_base.TestcaseBase.__init__(self) + testcase.TestCase.__init__(self) self.case_name = "odl" @classmethod def set_robotframework_vars(cls, odlusername="admin", odlpassword="admin"): + """Set credentials in csit/variables/Variables.py. + + Returns: + True if credentials are set. + False otherwise. + """ odl_variables_files = os.path.join(cls.odl_test_repo, 'csit/variables/Variables.py') try: @@ -74,11 +94,12 @@ class ODLTests(testcase_base.TestcaseBase): odlpassword + "']"), line.rstrip()) return True - except Exception as e: - cls.logger.error("Cannot set ODL creds: %s" % str(e)) + except Exception as ex: # pylint: disable=broad-except + cls.logger.error("Cannot set ODL creds: %s", str(ex)) return False def parse_results(self): + """Parse output.xml and get the details in it.""" xml_file = os.path.join(self.res_dir, 'output.xml') result = robot.api.ExecutionResult(xml_file) visitor = ODLResultVisitor() @@ -90,8 +111,38 @@ class ODLTests(testcase_base.TestcaseBase): self.details['description'] = result.suite.name self.details['tests'] = visitor.get_data() - def main(self, suites=default_suites, **kwargs): + def main(self, suites=None, **kwargs): + """Run the test suites + + It has been designed to be called in any context. + It requires the following keyword arguments: + * odlusername, + * odlpassword, + * osauthurl, + * neutronip, + * osusername, + * ostenantname, + * ospassword, + * odlip, + * odlwebport, + * odlrestconfport. + + Here are the steps: + * set all RobotFramework_variables, + * create the output directories if required, + * get the results in output.xml, + * delete temporary files. + + Args: + **kwargs: Arbitrary keyword arguments. + + Returns: + EX_OK if all suites ran well. + EX_RUN_ERROR otherwise. + """ try: + if not suites: + suites = self.default_suites odlusername = kwargs['odlusername'] odlpassword = kwargs['odlpassword'] osauthurl = kwargs['osauthurl'] @@ -105,17 +156,17 @@ class ODLTests(testcase_base.TestcaseBase): 'ODL_SYSTEM_IP:' + kwargs['odlip'], 'PORT:' + kwargs['odlwebport'], 'RESTCONFPORT:' + kwargs['odlrestconfport']] - except KeyError as e: + except KeyError as ex: self.logger.error("Cannot run ODL testcases. Please check " - "%s" % str(e)) + "%s", str(ex)) return self.EX_RUN_ERROR if self.set_robotframework_vars(odlusername, odlpassword): try: os.makedirs(self.res_dir) - except OSError as e: - if e.errno != errno.EEXIST: + except OSError as ex: + if ex.errno != errno.EEXIST: self.logger.exception( - "Cannot create {}".format(self.res_dir)) + "Cannot create %s", self.res_dir) return self.EX_RUN_ERROR stdout_file = os.path.join(self.res_dir, 'stdout.txt') output_dir = os.path.join(self.res_dir, 'output.xml') @@ -131,19 +182,31 @@ class ODLTests(testcase_base.TestcaseBase): try: self.parse_results() self.logger.info("ODL results were successfully parsed") - except RobotError as e: - self.logger.error("Run tests before publishing: %s" % - e.message) + except RobotError as ex: + self.logger.error("Run tests before publishing: %s", + ex.message) return self.EX_RUN_ERROR try: os.remove(stdout_file) except OSError: - self.logger.warning("Cannot remove {}".format(stdout_file)) + self.logger.warning("Cannot remove %s", stdout_file) return self.EX_OK else: return self.EX_RUN_ERROR def run(self, **kwargs): + """Run suites in OPNFV environment + + It basically check env vars to call main() with the keywords + required. + + Args: + **kwargs: Arbitrary keyword arguments. + + Returns: + EX_OK if all suites ran well. + EX_RUN_ERROR otherwise. + """ try: suites = self.default_suites try: @@ -176,19 +239,20 @@ class ODLTests(testcase_base.TestcaseBase): kwargs['odlwebport'] = '8181' else: kwargs['odlip'] = os.environ['SDN_CONTROLLER_IP'] - except KeyError as e: + except KeyError as ex: self.logger.error("Cannot run ODL testcases. " "Please check env var: " - "%s" % str(e)) + "%s", str(ex)) return self.EX_RUN_ERROR - except Exception: + except Exception: # pylint: disable=broad-except self.logger.exception("Cannot run ODL testcases.") return self.EX_RUN_ERROR return self.main(suites, **kwargs) -class ODLParser(object): +class ODLParser(object): # pylint: disable=too-few-public-methods + """Parser to run ODL test suites.""" def __init__(self): self.parser = argparse.ArgumentParser() @@ -226,19 +290,28 @@ class ODLParser(object): '-p', '--pushtodb', help='Push results to DB', action='store_true') - def parse_args(self, argv=[]): + def parse_args(self, argv=None): + """Parse arguments. + + It can call sys.exit if arguments are incorrect. + + Returns: + the arguments from cmdline + """ + if not argv: + argv = [] return vars(self.parser.parse_args(argv)) if __name__ == '__main__': - odl = ODLTests() - parser = ODLParser() - args = parser.parse_args(sys.argv[1:]) + ODL = ODLTests() + PARSER = ODLParser() + ARGS = PARSER.parse_args(sys.argv[1:]) try: - result = odl.main(ODLTests.default_suites, **args) - if result != testcase_base.TestcaseBase.EX_OK: - sys.exit(result) - if args['pushtodb']: - sys.exit(odl.push_to_db()) - except Exception: - sys.exit(testcase_base.TestcaseBase.EX_RUN_ERROR) + RESULT = ODL.main(ODLTests.default_suites, **ARGS) + if RESULT != testcase.TestCase.EX_OK: + sys.exit(RESULT) + if ARGS['pushtodb']: + sys.exit(ODL.push_to_db()) + except Exception: # pylint: disable=broad-except + sys.exit(testcase.TestCase.EX_RUN_ERROR) diff --git a/functest/opnfv_tests/sdn/onos/onos.py b/functest/opnfv_tests/sdn/onos/onos.py index 8bc738325..d482ae320 100644 --- a/functest/opnfv_tests/sdn/onos/onos.py +++ b/functest/opnfv_tests/sdn/onos/onos.py @@ -14,7 +14,7 @@ import shutil import time import urlparse -from functest.core import testcase_base +from functest.core import testcase from functest.utils.constants import CONST import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils @@ -24,7 +24,7 @@ import functest.utils.openstack_utils as openstack_utils logger = ft_logger.Logger(__name__).getLogger() -class OnosBase(testcase_base.TestcaseBase): +class OnosBase(testcase.TestCase): onos_repo_path = CONST.dir_repo_onos onos_sfc_image_name = CONST.onos_sfc_image_name onos_sfc_image_path = os.path.join(CONST.dir_functest_data, @@ -39,10 +39,10 @@ class OnosBase(testcase_base.TestcaseBase): self.start_time = time.time() try: self._run() - res = testcase_base.TestcaseBase.EX_OK + res = testcase.TestCase.EX_OK except Exception as e: logger.error('Error with run: %s', e) - res = testcase_base.TestcaseBase.EX_RUN_ERROR + res = testcase.TestCase.EX_RUN_ERROR self.stop_time = time.time() return res diff --git a/functest/opnfv_tests/vnf/aaa/aaa.py b/functest/opnfv_tests/vnf/aaa/aaa.py index f1c265f47..bdedcf7c5 100755 --- a/functest/opnfv_tests/vnf/aaa/aaa.py +++ b/functest/opnfv_tests/vnf/aaa/aaa.py @@ -11,7 +11,7 @@ import sys import argparse -import functest.core.testcase_base as testcase_base +import functest.core.testcase as testcase import functest.core.vnf_base as vnf_base import functest.utils.functest_logger as ft_logger @@ -62,9 +62,9 @@ if __name__ == '__main__': aaa_vnf = AaaVnf() try: result = aaa_vnf.main(**args) - if result != testcase_base.TestcaseBase.EX_OK: + if result != testcase.TestCase.EX_OK: sys.exit(result) if args['pushtodb']: sys.exit(aaa_vnf.push_to_db()) except Exception: - sys.exit(testcase_base.TestcaseBase.EX_RUN_ERROR) + sys.exit(testcase.TestCase.EX_RUN_ERROR) diff --git a/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py b/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py new file mode 100644 index 000000000..f21ce3f9a --- /dev/null +++ b/functest/opnfv_tests/vnf/ims/clearwater_ims_base.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python +# +# Copyright (c) 2017 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +import json +import os +import shutil + +import requests + +import functest.core.vnf_base as vnf_base +from functest.utils.constants import CONST +import functest.utils.functest_logger as ft_logger +import functest.utils.functest_utils as ft_utils + + +class ClearwaterOnBoardingBase(vnf_base.VnfOnBoardingBase): + + def __init__(self, project='functest', case='', repo='', cmd=''): + self.logger = ft_logger.Logger(__name__).getLogger() + super(ClearwaterOnBoardingBase, self).__init__( + project, case, repo, cmd) + self.case_dir = os.path.join(CONST.dir_functest_test, 'vnf', 'ims') + self.data_dir = CONST.dir_ims_data + self.result_dir = os.path.join(CONST.dir_results, case) + self.test_dir = CONST.dir_repo_vims_test + + if not os.path.exists(self.data_dir): + os.makedirs(self.data_dir) + if not os.path.exists(self.result_dir): + os.makedirs(self.result_dir) + + def config_ellis(self, ellis_ip, signup_code='secret', two_numbers=False): + output_dict = {} + self.logger.info('Configure Ellis: %s', ellis_ip) + output_dict['ellis_ip'] = ellis_ip + account_url = 'http://{0}/accounts'.format(ellis_ip) + params = {"password": "functest", + "full_name": "opnfv functest user", + "email": "functest@opnfv.org", + "signup_code": signup_code} + rq = requests.post(account_url, data=params) + output_dict['login'] = params + if rq.status_code != 201 and rq.status_code != 409: + raise Exception("Unable to create an account for number" + " provision: %s" % rq.json()['reason']) + self.logger.info('Account is created on Ellis: %s', params) + + session_url = 'http://{0}/session'.format(ellis_ip) + session_data = { + 'username': params['email'], + 'password': params['password'], + 'email': params['email'] + } + rq = requests.post(session_url, data=session_data) + if rq.status_code != 201: + raise Exception('Failed to get cookie for Ellis') + cookies = rq.cookies + self.logger.info('Cookies: %s' % cookies) + + number_url = 'http://{0}/accounts/{1}/numbers'.format( + ellis_ip, + params['email']) + self.logger.info('Create 1st calling number on Ellis') + number_res = self.create_ellis_number(number_url, cookies) + output_dict['number'] = number_res + + if two_numbers: + self.logger.info('Create 2nd calling number on Ellis') + number_res = self.create_ellis_number(number_url, cookies) + output_dict['number2'] = number_res + + return output_dict + + def create_ellis_number(self, number_url, cookies): + rq = requests.post(number_url, cookies=cookies) + + if rq.status_code != 200: + if rq and rq.json(): + reason = rq.json()['reason'] + else: + reason = rq + raise Exception("Unable to create a number: %s" % reason) + number_res = rq.json() + self.logger.info('Calling number is created: %s', number_res) + return number_res + + def run_clearwater_live_test(self, dns_ip, public_domain, + bono_ip=None, ellis_ip=None, + signup_code='secret'): + self.logger.info('Run Clearwater live test') + nameservers = ft_utils.get_resolvconf_ns() + resolvconf = ['{0}{1}{2}'.format(os.linesep, 'nameserver ', ns) + for ns in nameservers] + self.logger.debug('resolvconf: %s', resolvconf) + dns_file = '/etc/resolv.conf' + dns_file_bak = '/etc/resolv.conf.bak' + shutil.copy(dns_file, dns_file_bak) + script = ('echo -e "nameserver {0}{1}" > {2};' + 'source /etc/profile.d/rvm.sh;' + 'cd {3};' + 'rake test[{4}] SIGNUP_CODE={5}' + .format(dns_ip, + ''.join(resolvconf), + dns_file, + self.test_dir, + public_domain, + signup_code)) + if bono_ip and ellis_ip: + subscript = ' PROXY={0} ELLIS={1}'.format(bono_ip, ellis_ip) + script = '{0}{1}'.format(script, subscript) + script = ('{0}{1}'.format(script, ' --trace')) + cmd = "/bin/bash -c '{0}'".format(script) + self.logger.info('Live test cmd: %s', cmd) + output_file = os.path.join(self.result_dir, "ims_test_output.txt") + ft_utils.execute_command(cmd, + error_msg='Clearwater live test failed', + output_file=output_file) + + with open(dns_file_bak, 'r') as bak_file: + result = bak_file.read() + with open(dns_file, 'w') as f: + f.write(result) + + f = open(output_file, 'r') + result = f.read() + if result != "": + self.logger.debug(result) + + vims_test_result = "" + tempFile = os.path.join(self.test_dir, "temp.json") + try: + self.logger.debug("Trying to load test results") + with open(tempFile) as f: + vims_test_result = json.load(f) + f.close() + except Exception: + self.logger.error("Unable to retrieve test results") + + try: + os.remove(tempFile) + except Exception: + self.logger.error("Deleting file failed") + + return vims_test_result diff --git a/functest/opnfv_tests/vnf/ims/cloudify_ims.py b/functest/opnfv_tests/vnf/ims/cloudify_ims.py index f7dfd532f..404f208eb 100644 --- a/functest/opnfv_tests/vnf/ims/cloudify_ims.py +++ b/functest/opnfv_tests/vnf/ims/cloudify_ims.py @@ -7,42 +7,37 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 -import json import os -import requests -import subprocess import sys import time + +import requests import yaml -import functest.core.vnf_base as vnf_base +from functest.opnfv_tests.vnf.ims.clearwater import Clearwater +import functest.opnfv_tests.vnf.ims.clearwater_ims_base as clearwater_ims_base +from functest.opnfv_tests.vnf.ims.orchestrator_cloudify import Orchestrator +from functest.utils.constants import CONST import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils import functest.utils.openstack_utils as os_utils -from clearwater import Clearwater -from functest.utils.constants import CONST -from orchestrator_cloudify import Orchestrator - -class ImsVnf(vnf_base.VnfOnBoardingBase): +class CloudifyIms(clearwater_ims_base.ClearwaterOnBoardingBase): def __init__(self, project='functest', case='cloudify_ims', repo='', cmd=''): - super(ImsVnf, self).__init__(project, case, repo, cmd) - self.logger = ft_logger.Logger("vIMS").getLogger() - self.case_dir = os.path.join(CONST.dir_functest_test, 'vnf/ims/') - self.data_dir = CONST.dir_ims_data - self.test_dir = CONST.dir_repo_vims_test + super(CloudifyIms, self).__init__(project, case, repo, cmd) + self.logger = ft_logger.Logger(__name__).getLogger() # Retrieve the configuration try: self.config = CONST.__getattribute__( 'vnf_{}_config'.format(self.case_name)) - except: + except Exception: raise Exception("VNF config file not found") - config_file = self.case_dir + self.config + config_file = os.path.join(self.case_dir, self.config) self.orchestrator = dict( requirements=get_config("cloudify.requirements", config_file), blueprint=get_config("cloudify.blueprint", config_file), @@ -61,10 +56,6 @@ class ImsVnf(vnf_base.VnfOnBoardingBase): self.images = get_config("tenant_images", config_file) self.logger.info("Images needed for vIMS: %s" % self.images) - # vIMS Data directory creation - if not os.path.exists(self.data_dir): - os.makedirs(self.data_dir) - def deploy_orchestrator(self, **kwargs): self.logger.info("Additional pre-configuration steps") @@ -82,7 +73,7 @@ class ImsVnf(vnf_base.VnfOnBoardingBase): image_id = os_utils.get_image_id(self.glance_client, image_name) self.logger.debug("image_id: %s" % image_id) - except: + except Exception: self.logger.error("Unexpected error: %s" % sys.exc_info()[0]) if image_id == '': @@ -177,10 +168,11 @@ class ImsVnf(vnf_base.VnfOnBoardingBase): self.logger.debug("Resolvconf set") self.logger.info("Prepare virtualenv for cloudify-cli") - cmd = "chmod +x " + self.case_dir + "create_venv.sh" + venv_scrit_dir = os.path.join(self.case_dir, "create_venv.sh") + cmd = "chmod +x " + venv_scrit_dir ft_utils.execute_command(cmd) time.sleep(3) - cmd = self.case_dir + "create_venv.sh " + self.data_dir + cmd = venv_scrit_dir + " " + self.data_dir ft_utils.execute_command(cmd) cfy.download_manager_blueprint( @@ -251,99 +243,25 @@ class ImsVnf(vnf_base.VnfOnBoardingBase): self.logger.debug("Trying to get clearwater manager IP ... ") mgr_ip = os.popen(cmd).read() mgr_ip = mgr_ip.splitlines()[0] - except: + except Exception: self.step_failure("Unable to retrieve the IP of the " "cloudify manager server !") - api_url = "http://" + mgr_ip + "/api/v2" - dep_outputs = requests.get(api_url + "/deployments/" + - self.vnf['deployment_name'] + "/outputs") - dns_ip = dep_outputs.json()['outputs']['dns_ip'] - ellis_ip = dep_outputs.json()['outputs']['ellis_ip'] - - self.logger.debug("DNS ip : %s" % dns_ip) - self.logger.debug("ELLIS ip : %s" % ellis_ip) - - ellis_url = "http://" + ellis_ip + "/" - url = ellis_url + "accounts" - - params = {"password": "functest", - "full_name": "opnfv functest user", - "email": "functest@opnfv.fr", - "signup_code": "secret"} - - rq = requests.post(url, data=params) - i = 30 - while rq.status_code != 201 and i > 0: - rq = requests.post(url, data=params) - self.logger.debug("Account creation http status code: %s" - % rq.status_code) - i = i - 1 - time.sleep(10) - - if rq.status_code == 201: - url = ellis_url + "session" - rq = requests.post(url, data=params) - cookies = rq.cookies - else: - self.step_failure("Unable to create an account") - - url = ellis_url + "accounts/" + params['email'] + "/numbers" - if cookies != "": - rq = requests.post(url, cookies=cookies) - i = 24 - while rq.status_code != 200 and i > 0: - rq = requests.post(url, cookies=cookies) - self.logger.debug("Number creation http status code: %s" - % rq.status_code) - i = i - 1 - time.sleep(25) - - if rq.status_code != 200: - self.step_failure("Unable to create a number: %s" - % rq.json()['reason']) - - nameservers = ft_utils.get_resolvconf_ns() - resolvconf = "" - for ns in nameservers: - resolvconf += "\nnameserver " + ns + self.logger.info('Cloudify Manager: %s', mgr_ip) + api_url = 'http://{0}/api/v2/deployments/{1}/outputs'.format( + mgr_ip, self.vnf['deployment_name']) + dep_outputs = requests.get(api_url) + self.logger.info(api_url) + outputs = dep_outputs.json()['outputs'] + self.logger.info("Deployment outputs: %s", outputs) + dns_ip = outputs['dns_ip'] + ellis_ip = outputs['ellis_ip'] + self.config_ellis(ellis_ip) if dns_ip != "": - script = ('echo -e "nameserver ' + dns_ip + resolvconf + - '" > /etc/resolv.conf; ') - script += 'source /etc/profile.d/rvm.sh; ' - script += 'cd {0}; ' - script += ('rake test[{1}] SIGNUP_CODE="secret"') - - cmd = ("/bin/bash -c '" + - script.format(self.data_dir, self.inputs["public_domain"]) + - "'") - output_file = "output.txt" - f = open(output_file, 'w+') - subprocess.call(cmd, shell=True, stdout=f, - stderr=subprocess.STDOUT) - f.close() - - f = open(output_file, 'r') - result = f.read() - if result != "": - self.logger.debug(result) - - vims_test_result = "" - tempFile = os.path.join(self.test_dir, "temp.json") - try: - self.logger.debug("Trying to load test results") - with open(tempFile) as f: - vims_test_result = json.load(f) - f.close() - except: - self.logger.error("Unable to retrieve test results") - - try: - os.remove(tempFile) - except: - self.logger.error("Deleting file failed") - + vims_test_result = self.run_clearwater_live_test( + dns_ip=dns_ip, + public_domain=self.inputs["public_domain"]) if vims_test_result != '': return {'status': 'PASS', 'result': vims_test_result} else: @@ -352,7 +270,7 @@ class ImsVnf(vnf_base.VnfOnBoardingBase): def clean(self): self.vnf['object'].undeploy_vnf() self.orchestrator['object'].undeploy_manager() - super(ImsVnf, self).clean() + super(CloudifyIms, self).clean() def main(self, **kwargs): self.logger.info("Cloudify IMS VNF onboarding test starting") diff --git a/functest/opnfv_tests/vnf/ims/opera_ims.py b/functest/opnfv_tests/vnf/ims/opera_ims.py index 7ead401fe..d022b3c7f 100755..100644 --- a/functest/opnfv_tests/vnf/ims/opera_ims.py +++ b/functest/opnfv_tests/vnf/ims/opera_ims.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -# Copyright (c) 2016 Orange and others. +# Copyright (c) 2017 HUAWEI TECHNOLOGIES CO.,LTD and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -8,381 +8,117 @@ # http://www.apache.org/licenses/LICENSE-2.0 import json -import socket -import sys +import os import time -import yaml -import functest.core.vnf_base as vnf_base -import functest.utils.functest_logger as ft_logger -import functest.utils.functest_utils as ft_utils -import functest.utils.openstack_utils as os_utils -import os +from opera import openo_connect +import requests + +import functest.opnfv_tests.vnf.ims.clearwater_ims_base as clearwater_ims_base from functest.utils.constants import CONST +import functest.utils.functest_logger as ft_logger -from org.openbaton.cli.agents.agents import MainAgent -from org.openbaton.cli.errors.errors import NfvoException +class OperaIms(clearwater_ims_base.ClearwaterOnBoardingBase): -def servertest(host, port): - args = socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM) - for family, socktype, proto, canonname, sockaddr in args: - s = socket.socket(family, socktype, proto) + def __init__(self, project='functest', case='opera_ims', + repo=CONST.dir_repo_opera, cmd=''): + super(OperaIms, self).__init__(project, case, repo, cmd) + self.logger = ft_logger.Logger(__name__).getLogger() + self.ellis_file = os.path.join(self.result_dir, 'ellis.info') + self.live_test_file = os.path.join(self.result_dir, + 'live_test_report.json') try: - s.connect(sockaddr) - except socket.error: - return False + self.openo_msb_endpoint = os.environ['OPENO_MSB_ENDPOINT'] + except KeyError: + raise Exception('OPENO_MSB_ENDPOINT is not specified,' + ' put it as <OPEN-O ip>:<port>') else: - s.close() - return True - -# ---------------------------------------------------------- -# -# UTILS -# -# ----------------------------------------------------------- - - -def get_config(parameter, file): - """ - Returns the value of a given parameter in file.yaml - parameter must be given in string format with dots - Example: general.openstack.image_name - """ - with open(file) as f: - file_yaml = yaml.safe_load(f) - f.close() - value = file_yaml - for element in parameter.split("."): - value = value.get(element) - if value is None: - raise ValueError("The parameter %s is not defined in" - " reporting.yaml" % parameter) - return value - - -def download_and_add_image_on_glance(glance, image_name, - image_url, data_dir): - dest_path = data_dir - if not os.path.exists(dest_path): - os.makedirs(dest_path) - file_name = image_url.rsplit('/')[-1] - if not ft_utils.download_url(image_url, dest_path): - return False - image = os_utils.create_glance_image( - glance, image_name, dest_path + file_name) - if not image: - return False - return image - - -class ImsVnf(vnf_base.VnfOnBoardingBase): - - def __init__(self, project='functest', case='orchestra_ims', - repo='', cmd=''): - super(ImsVnf, self).__init__(project, case, repo, cmd) - self.ob_password = "openbaton" - self.ob_username = "admin" - self.ob_https = False - self.ob_port = "8080" - self.ob_ip = "localhost" - self.ob_instance_id = "" - self.logger = ft_logger.Logger("orchestra_ims").getLogger() - self.case_dir = os.path.join(CONST.dir_functest_test, 'vnf/ims/') - self.data_dir = CONST.dir_ims_data - self.test_dir = CONST.dir_repo_vims_test - self.ob_projectid = "" - self.keystone_client = os_utils.get_keystone_client() - self.ob_nsr_id = "" - self.main_agent = None - # vIMS Data directory creation - if not os.path.exists(self.data_dir): - os.makedirs(self.data_dir) - # Retrieve the configuration - try: - self.config = CONST.__getattribute__( - 'vnf_{}_config'.format(self.case_name)) - except: - raise Exception("Orchestra VNF config file not found") - config_file = self.case_dir + self.config - self.imagename = get_config("openbaton.imagename", config_file) - self.market_link = get_config("openbaton.marketplace_link", - config_file) - self.images = get_config("tenant_images", config_file) - - def deploy_orchestrator(self, **kwargs): - self.logger.info("Additional pre-configuration steps") - nova_client = os_utils.get_nova_client() - neutron_client = os_utils.get_neutron_client() - glance_client = os_utils.get_glance_client() - - # needs some images - self.logger.info("Upload some OS images if it doesn't exist") - temp_dir = os.path.join(self.data_dir, "tmp/") - for image_name, image_url in self.images.iteritems(): - self.logger.info("image: %s, url: %s" % (image_name, image_url)) - try: - image_id = os_utils.get_image_id(glance_client, - image_name) - self.logger.info("image_id: %s" % image_id) - except: - self.logger.error("Unexpected error: %s" % sys.exc_info()[0]) - - if image_id == '': - self.logger.info("""%s image doesn't exist on glance repository. Try - downloading this image and upload on glance !""" % image_name) - image_id = download_and_add_image_on_glance(glance_client, - image_name, - image_url, - temp_dir) - if image_id == '': - self.step_failure( - "Failed to find or upload required OS " - "image for this deployment") - network_dic = os_utils.create_network_full(neutron_client, - "openbaton_mgmt", - "openbaton_mgmt_subnet", - "openbaton_router", - "192.168.100.0/24") - - # orchestrator VM flavor - self.logger.info("Check medium Flavor is available, if not create one") - flavor_exist, flavor_id = os_utils.get_or_create_flavor( - "m1.medium", - "4096", - '1', - '2', - public=True) - self.logger.debug("Flavor id: %s" % flavor_id) - - if not network_dic: - self.logger.error("There has been a problem when creating the " - "neutron network") - - network_id = network_dic["net_id"] - - self.logger.info("Creating floating IP for VM in advance...") - floatip_dic = os_utils.create_floating_ip(neutron_client) - floatip = floatip_dic['fip_addr'] - - if floatip is None: - self.logger.error("Cannot create floating IP.") - - userdata = "#!/bin/bash\n" - userdata += "set -x\n" - userdata += "set -e\n" - userdata += "echo \"nameserver 8.8.8.8\" >> /etc/resolv.conf\n" - userdata += "apt-get install curl\n" - userdata += ("echo \"rabbitmq_broker_ip=%s\" > ./config_file\n" - % floatip) - userdata += "echo \"mysql=no\" >> ./config_file\n" - userdata += ("echo \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCuPXrV3" - "geeHc6QUdyUr/1Z+yQiqLcOskiEGBiXr4z76MK4abiFmDZ18OMQlc" - "fl0p3kS0WynVgyaOHwZkgy/DIoIplONVr2CKBKHtPK+Qcme2PVnCtv" - "EqItl/FcD+1h5XSQGoa+A1TSGgCod/DPo+pes0piLVXP8Ph6QS1k7S" - "ic7JDeRQ4oT1bXYpJ2eWBDMfxIWKZqcZRiGPgMIbJ1iEkxbpeaAd9O" - "4MiM9nGCPESmed+p54uYFjwEDlAJZShcAZziiZYAvMZhvAhe6USljc" - "7YAdalAnyD/jwCHuwIrUw/lxo7UdNCmaUxeobEYyyFA1YVXzpNFZya" - "XPGAAYIJwEq/ openbaton@opnfv\" >> /home/ubuntu/.ssh/aut" - "horized_keys\n") - userdata += "cat ./config_file\n" - userdata += ("curl -s http://get.openbaton.org/bootstrap " - "> ./bootstrap\n") - userdata += "export OPENBATON_COMPONENT_AUTOSTART=false\n" - bootstrap = "sh ./bootstrap release -configFile=./config_file" - userdata += bootstrap + "\n" - - userdata += ("echo \"nfvo.plugin.timeout=300000\" >> " - "/etc/openbaton/openbaton-nfvo.properties\n") - userdata += "service openbaton-nfvo restart\n" - userdata += "service openbaton-vnfm-generic restart\n" - - sg_id = os_utils.create_security_group_full(neutron_client, - "orchestra-sec-group", - "allowall") - - os_utils.create_secgroup_rule(neutron_client, sg_id, "ingress", - "icmp", 0, 255) - os_utils.create_secgroup_rule(neutron_client, sg_id, "egress", - "icmp", 0, 255) - os_utils.create_secgroup_rule(neutron_client, sg_id, "ingress", - "tcp", 1, 65535) - os_utils.create_secgroup_rule(neutron_client, sg_id, "ingress", - "udp", 1, 65535) - os_utils.create_secgroup_rule(neutron_client, sg_id, "egress", - "tcp", 1, 65535) - os_utils.create_secgroup_rule(neutron_client, sg_id, "egress", - "udp", 1, 65535) - - self.logger.info("Security group set") - - self.logger.info("Create instance....") - self.logger.info("flavor: m1.medium\n" - "image: %s\n" - "network_id: %s\n" - "userdata: %s\n" - % (self.imagename, network_id, userdata)) - - instance = os_utils.create_instance_and_wait_for_active( - "m1.medium", - os_utils.get_image_id(glance_client, self.imagename), - network_id, - "orchestra-openbaton", - config_drive=False, - userdata=userdata) + self.logger.info('OPEN-O endpoint is: %s', self.openo_msb_endpoint) - self.ob_instance_id = instance.id + def prepare(self): + pass - self.logger.info("Adding sec group to orchestra instance") - os_utils.add_secgroup_to_instance(nova_client, - self.ob_instance_id, sg_id) - - self.logger.info("Associating floating ip: '%s' to VM '%s' " - % (floatip, "orchestra-openbaton")) - if not os_utils.add_floating_ip(nova_client, instance.id, floatip): - self.logger.error("Cannot associate floating IP to VM.") - self.step_failure("Cannot associate floating IP to VM.") - - self.logger.info("Waiting for nfvo to be up and running...") - x = 0 - while x < 100: - if servertest(floatip, "8080"): - break - else: - self.logger.debug("openbaton is not started yet") - time.sleep(5) - x += 1 - - if x == 100: - self.logger.error("Openbaton is not started correctly") - self.step_failure("Openbaton is not started correctly") - - self.ob_ip = floatip - self.ob_password = "openbaton" - self.ob_username = "admin" - self.ob_https = False - self.ob_port = "8080" - - self.logger.info("Deploy orchestrator: OK") + def clean(self): + pass def deploy_vnf(self): - self.logger.info("vIMS Deployment") - - self.main_agent = MainAgent(nfvo_ip=self.ob_ip, - nfvo_port=self.ob_port, - https=self.ob_https, - version=1, - username=self.ob_username, - password=self.ob_password) - - project_agent = self.main_agent.get_agent("project", self.ob_projectid) - for p in json.loads(project_agent.find()): - if p.get("name") == "default": - self.ob_projectid = p.get("id") - break - - self.logger.debug("project id: %s" % self.ob_projectid) - if self.ob_projectid == "": - self.logger.error("Default project id was not found!") - self.step_failure("Default project id was not found!") - - vim_json = { - "name": "vim-instance", - "authUrl": os_utils.get_credentials().get("auth_url"), - "tenant": os_utils.get_credentials().get("tenant_name"), - "username": os_utils.get_credentials().get("username"), - "password": os_utils.get_credentials().get("password"), - # "keyPair": "opnfv", - # TODO change the keypair to correct value - # or upload a correct one or remove it - "securityGroups": [ - "default", - "orchestra-sec-group" - ], - "type": "openstack", - "location": { - "name": "opnfv", - "latitude": "52.525876", - "longitude": "13.314400" - } - } - - self.logger.debug("vim: %s" % vim_json) - - self.main_agent.get_agent( - "vim", - project_id=self.ob_projectid).create(entity=json.dumps(vim_json)) - - market_agent = self.main_agent.get_agent("market", - project_id=self.ob_projectid) - - nsd = {} - try: - self.logger.info("sending: %s" % self.market_link) - nsd = market_agent.create(entity=self.market_link) - self.logger.info("Onboarded nsd: " + nsd.get("name")) - except NfvoException as e: - self.step_failure(e.message) - - nsr_agent = self.main_agent.get_agent("nsr", - project_id=self.ob_projectid) - nsd_id = nsd.get('id') - if nsd_id is None: - self.step_failure("NSD not onboarded correctly") - - nsr = None try: - nsr = nsr_agent.create(nsd_id) - except NfvoException as e: - self.step_failure(e.message) - - if nsr.get('code') is not None: - self.logger.error( - "vIMS cannot be deployed: %s -> %s" % - (nsr.get('code'), nsr.get('message'))) - self.step_failure("vIMS cannot be deployed") - - i = 0 - self.logger.info("waiting NSR to go to active...") - while nsr.get("status") != 'ACTIVE' and nsr.get("status") != 'ERROR': - i += 1 - if i == 100: - self.step_failure("After %s sec the nsr did not go to active.." - % 5 * 100) - time.sleep(5) - nsr = json.loads(nsr_agent.find(nsr.get('id'))) - - if nsr.get("status") == 'ACTIVE': - deploy_vnf = {'status': "PASS", 'result': nsr} - self.logger.info("Deploy VNF: OK") + openo_connect.create_service(self.openo_msb_endpoint, + 'functest_opera', + 'VNF for functest testing') + except Exception as e: + self.logger.error(e) + return {'status': 'FAIL', 'result': e} else: - deploy_vnf = {'status': "FAIL", 'result': nsr} - self.logger.error("Deploy VNF: ERROR") - self.step_failure("Deploy vIMS failed") - self.ob_nsr_id = nsr.get("id") - return deploy_vnf + self.logger.info('vIMS deployment is kicked off') + return {'status': 'PASS', 'result': ''} - def test_vnf(self): - # Adaptations probably needed - # code used for cloudify_ims - # ruby client on jumphost calling the vIMS on the SUT - return + def dump_info(self, info_file, result): + with open(info_file, 'w') as f: + self.logger.debug('Save information to file: %s', info_file) + json.dump(result, f) - def clean(self): - self.main_agent.get_agent( - "nsr", - project_id=self.ob_projectid).delete(self.ob_nsr_id) - time.sleep(5) - os_utils.delete_instance(nova_client=os_utils.get_nova_client(), - instance_id=self.ob_instance_id) - # TODO question is the clean removing also the VM? - # I think so since is goinf to remove the tenant... - super(ImsVnf, self).clean() + def test_vnf(self): + vnfm_ip = openo_connect.get_vnfm_ip(self.openo_msb_endpoint) + self.logger.info('VNFM IP: %s', vnfm_ip) + vnf_status_url = 'http://{0}:5000/api/v1/model/status'.format(vnfm_ip) + vnf_alive = False + retry = 15 + + self.logger.info('Check the VNF status') + while retry > 0: + rq = requests.get(vnf_status_url) + response = rq.json() + vnf_alive = response['vnf_alive'] + msg = response['msg'] + self.logger.info(msg) + if vnf_alive: + break + self.logger.info('check again in one minute...') + retry = retry - 1 + time.sleep(60) + + if not vnf_alive: + raise Exception('VNF failed to start: {0}'.format(msg)) + + ellis_config_url = ('http://{0}:5000/api/v1/model/ellis/configure' + .format(vnfm_ip)) + rq = requests.get(ellis_config_url, timeout=60) + if rq.json() and not rq.json()['ellis_ok']: + self.logger.error(rq.json()['data']) + raise Exception('Failed to configure Ellis') + + self.logger.info('Get Clearwater deployment detail') + vnf_info_url = ('http://{0}:5000/api/v1/model/output' + .format(vnfm_ip)) + rq = requests.get(vnf_info_url, timeout=60) + data = rq.json()['data'] + self.logger.info(data) + bono_ip = data['bono_ip'] + ellis_ip = data['ellis_ip'] + dns_ip = data['dns_ip'] + result = self.config_ellis(ellis_ip, 'signup', True) + self.logger.debug('Ellis Result: %s', result) + self.dump_info(self.ellis_file, result) + + if dns_ip: + vims_test_result = self.run_clearwater_live_test( + dns_ip, + 'clearwater.local', + bono_ip, + ellis_ip, + 'signup') + if vims_test_result != '': + self.dump_info(self.live_test_file, vims_test_result) + return {'status': 'PASS', 'result': vims_test_result} + else: + return {'status': 'FAIL', 'result': ''} def main(self, **kwargs): - self.logger.info("Orchestra IMS VNF onboarding test starting") + self.logger.info("Start to run Opera vIMS VNF onboarding test") self.execute() - self.logger.info("Orchestra IMS VNF onboarding test executed") + self.logger.info("Opera vIMS VNF onboarding test finished") if self.criteria is "PASS": return self.EX_OK else: @@ -391,10 +127,3 @@ class ImsVnf(vnf_base.VnfOnBoardingBase): def run(self): kwargs = {} return self.main(**kwargs) - - -if __name__ == '__main__': - test = ImsVnf() - test.deploy_orchestrator() - test.deploy_vnf() - test.clean() diff --git a/functest/opnfv_tests/vnf/rnc/parser.py b/functest/opnfv_tests/vnf/rnc/parser.py index 1cff72209..133145d74 100644 --- a/functest/opnfv_tests/vnf/rnc/parser.py +++ b/functest/opnfv_tests/vnf/rnc/parser.py @@ -15,10 +15,10 @@ # limitations under the License. # -import functest.core.feature_base as base +import functest.core.feature as base -class Parser(base.FeatureBase): +class Parser(base.Feature): def __init__(self): super(Parser, self).__init__(project='parser', case='parser-basics', diff --git a/functest/opnfv_tests/vnf/router/vyos_vrouter.py b/functest/opnfv_tests/vnf/router/vyos_vrouter.py index 94a3ecfd8..e188c3fbb 100644 --- a/functest/opnfv_tests/vnf/router/vyos_vrouter.py +++ b/functest/opnfv_tests/vnf/router/vyos_vrouter.py @@ -6,16 +6,16 @@ # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 -import functest.core.feature_base as base +import functest.core.feature as base import json import os RESULT_DETAILS_FILE = "test_result.json" -class VrouterVnf(base.FeatureBase): +class VrouterVnf(base.Feature): def __init__(self): - super(VrouterVnf, self).__init__(project='vRouter', + super(VrouterVnf, self).__init__(project='functest', case='vyos_vrouter', repo='dir_repo_vrouter') self.cmd = 'cd %s && ./run.sh' % self.repo diff --git a/functest/tests/unit/core/test_testcase_base.py b/functest/tests/unit/core/test_testcase.py index b7c81d87c..32104194b 100644 --- a/functest/tests/unit/core/test_testcase_base.py +++ b/functest/tests/unit/core/test_testcase.py @@ -7,19 +7,27 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 +"""Define the classe required to fully cover testcase.""" + import logging -import mock import unittest -from functest.core import testcase_base +import mock + +from functest.core import testcase + +__author__ = "Cedric Ollivier <cedric.ollivier@orange.com>" + +class TestCaseTesting(unittest.TestCase): -class TestcaseBaseTesting(unittest.TestCase): + """The class testing TestCase.""" + # pylint: disable=missing-docstring logging.disable(logging.CRITICAL) def setUp(self): - self.test = testcase_base.TestcaseBase() + self.test = testcase.TestCase() self.test.project = "functest" self.test.case_name = "base" self.test.start_time = "1" @@ -29,13 +37,13 @@ class TestcaseBaseTesting(unittest.TestCase): def test_run_unimplemented(self): self.assertEqual(self.test.run(), - testcase_base.TestcaseBase.EX_RUN_ERROR) + testcase.TestCase.EX_RUN_ERROR) @mock.patch('functest.utils.functest_utils.push_results_to_db', return_value=False) - def _test_missing_attribute(self, mock_function): + def _test_missing_attribute(self, mock_function=None): self.assertEqual(self.test.push_to_db(), - testcase_base.TestcaseBase.EX_PUSH_TO_DB_ERROR) + testcase.TestCase.EX_PUSH_TO_DB_ERROR) mock_function.assert_not_called() def test_missing_case_name(self): @@ -56,28 +64,28 @@ class TestcaseBaseTesting(unittest.TestCase): @mock.patch('functest.utils.functest_utils.push_results_to_db', return_value=True) - def test_missing_details(self, mock_function): + def test_missing_details(self, mock_function=None): self.test.details = None self.assertEqual(self.test.push_to_db(), - testcase_base.TestcaseBase.EX_OK) + testcase.TestCase.EX_OK) mock_function.assert_called_once_with( self.test.project, self.test.case_name, self.test.start_time, self.test.stop_time, self.test.criteria, self.test.details) @mock.patch('functest.utils.functest_utils.push_results_to_db', return_value=False) - def test_push_to_db_failed(self, mock_function): + def test_push_to_db_failed(self, mock_function=None): self.assertEqual(self.test.push_to_db(), - testcase_base.TestcaseBase.EX_PUSH_TO_DB_ERROR) + testcase.TestCase.EX_PUSH_TO_DB_ERROR) mock_function.assert_called_once_with( self.test.project, self.test.case_name, self.test.start_time, self.test.stop_time, self.test.criteria, self.test.details) @mock.patch('functest.utils.functest_utils.push_results_to_db', return_value=True) - def test_push_to_db(self, mock_function): + def test_push_to_db(self, mock_function=None): self.assertEqual(self.test.push_to_db(), - testcase_base.TestcaseBase.EX_OK) + testcase.TestCase.EX_OK) mock_function.assert_called_once_with( self.test.project, self.test.case_name, self.test.start_time, self.test.stop_time, self.test.criteria, self.test.details) @@ -85,17 +93,17 @@ class TestcaseBaseTesting(unittest.TestCase): def test_check_criteria_missing(self): self.test.criteria = None self.assertEqual(self.test.check_criteria(), - testcase_base.TestcaseBase.EX_TESTCASE_FAILED) + testcase.TestCase.EX_TESTCASE_FAILED) def test_check_criteria_failed(self): self.test.criteria = 'FAILED' self.assertEqual(self.test.check_criteria(), - testcase_base.TestcaseBase.EX_TESTCASE_FAILED) + testcase.TestCase.EX_TESTCASE_FAILED) def test_check_criteria_pass(self): self.test.criteria = 'PASS' self.assertEqual(self.test.check_criteria(), - testcase_base.TestcaseBase.EX_OK) + testcase.TestCase.EX_OK) if __name__ == "__main__": diff --git a/functest/tests/unit/odl/test_odl.py b/functest/tests/unit/odl/test_odl.py index 5961940f1..e08deb27d 100644 --- a/functest/tests/unit/odl/test_odl.py +++ b/functest/tests/unit/odl/test_odl.py @@ -7,24 +7,67 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 +"""Define the classes required to fully cover odl.""" + import errno import logging -import mock import os import StringIO import unittest from keystoneauth1.exceptions import auth_plugins +import mock from robot.errors import DataError, RobotError -from robot.result import testcase +from robot.result import testcase as result_testcase from robot.utils.robottime import timestamp_to_secs -from functest.core import testcase_base +from functest.core import testcase from functest.opnfv_tests.sdn.odl import odl +__author__ = "Cedric Ollivier <cedric.ollivier@orange.com>" + + +class ODLVisitorTesting(unittest.TestCase): + + """The class testing ODLResultVisitor.""" + # pylint: disable=missing-docstring + + logging.disable(logging.CRITICAL) + + def setUp(self): + self.visitor = odl.ODLResultVisitor() + + def test_empty(self): + self.assertFalse(self.visitor.get_data()) + + def test_ok(self): + data = {'name': 'foo', + 'parent': 'bar', + 'status': 'PASS', + 'starttime': "20161216 16:00:00.000", + 'endtime': "20161216 16:00:01.000", + 'elapsedtime': 1000, + 'text': 'Hello, World!', + 'critical': True} + test = result_testcase.TestCase(name=data['name'], + status=data['status'], + message=data['text'], + starttime=data['starttime'], + endtime=data['endtime']) + test.parent = mock.Mock() + config = {'name': data['parent'], + 'criticality.test_is_critical.return_value': data[ + 'critical']} + test.parent.configure_mock(**config) + self.visitor.visit_test(test) + self.assertEqual(self.visitor.get_data(), [data]) + class ODLTesting(unittest.TestCase): + """The super class which testing classes could inherit.""" + # pylint: disable=missing-docstring + logging.disable(logging.CRITICAL) _keystone_ip = "127.0.0.1" @@ -60,39 +103,19 @@ class ODLTesting(unittest.TestCase): 'odlrestconfport': self._odl_restconfport, 'pushtodb': False} - def test_empty_visitor(self): - visitor = odl.ODLResultVisitor() - self.assertFalse(visitor.get_data()) - def test_visitor(self): - visitor = odl.ODLResultVisitor() - data = {'name': 'foo', - 'parent': 'bar', - 'status': 'PASS', - 'starttime': "20161216 16:00:00.000", - 'endtime': "20161216 16:00:01.000", - 'elapsedtime': 1000, - 'text': 'Hello, World!', - 'critical': True} - test = testcase.TestCase(name=data['name'], - status=data['status'], - message=data['text'], - starttime=data['starttime'], - endtime=data['endtime']) - test.parent = mock.Mock() - config = {'name': data['parent'], - 'criticality.test_is_critical.return_value': data[ - 'critical']} - test.parent.configure_mock(**config) - visitor.visit_test(test) - self.assertEqual(visitor.get_data(), [data]) +class ODLParseResultTesting(ODLTesting): + + """The class testing ODLTests.parse_results().""" + # pylint: disable=missing-docstring @mock.patch('robot.api.ExecutionResult', side_effect=DataError) - def test_parse_results_raises_exceptions(self, *args): + def test_raises_exc(self, mock_method): with self.assertRaises(DataError): self.test.parse_results() + mock_method.assert_called_once_with() - def test_parse_results(self, *args): + def test_ok(self): config = {'name': 'dummy', 'starttime': '20161216 16:00:00.000', 'endtime': '20161216 16:00:01.000', 'status': 'PASS'} suite = mock.Mock() @@ -108,16 +131,28 @@ class ODLTesting(unittest.TestCase): self.assertEqual(self.test.details, {'description': config['name'], 'tests': []}) + +class ODLRobotTesting(ODLTesting): + + """The class testing ODLTests.set_robotframework_vars().""" + # pylint: disable=missing-docstring + @mock.patch('fileinput.input', side_effect=Exception()) - def test_set_robotframework_vars_failed(self, *args): + def test_set_vars_ko(self, mock_method): self.assertFalse(self.test.set_robotframework_vars()) + mock_method.assert_called_once_with( + os.path.join(odl.ODLTests.odl_test_repo, + 'csit/variables/Variables.py'), inplace=True) @mock.patch('fileinput.input', return_value=[]) - def test_set_robotframework_vars_empty(self, args): + def test_set_vars_empty(self, mock_method): self.assertTrue(self.test.set_robotframework_vars()) + mock_method.assert_called_once_with( + os.path.join(odl.ODLTests.odl_test_repo, + 'csit/variables/Variables.py'), inplace=True) @mock.patch('sys.stdout', new_callable=StringIO.StringIO) - def _test_set_robotframework_vars(self, msg1, msg2, *args): + def _test_set_vars(self, msg1, msg2, *args): line = mock.MagicMock() line.__iter__.return_value = [msg1] with mock.patch('fileinput.input', return_value=line) as mock_method: @@ -127,15 +162,15 @@ class ODLTesting(unittest.TestCase): 'csit/variables/Variables.py'), inplace=True) self.assertEqual(args[0].getvalue(), "{}\n".format(msg2)) - def test_set_robotframework_vars_auth_default(self): - self._test_set_robotframework_vars("AUTH = []", - "AUTH = [u'admin', u'admin']") + def test_set_vars_auth_default(self): + self._test_set_vars("AUTH = []", + "AUTH = [u'admin', u'admin']") - def test_set_robotframework_vars_auth1(self): - self._test_set_robotframework_vars("AUTH1 = []", "AUTH1 = []") + def test_set_vars_auth1(self): + self._test_set_vars("AUTH1 = []", "AUTH1 = []") @mock.patch('sys.stdout', new_callable=StringIO.StringIO) - def test_set_robotframework_vars_auth_foo(self, *args): + def test_set_vars_auth_foo(self, *args): line = mock.MagicMock() line.__iter__.return_value = ["AUTH = []"] with mock.patch('fileinput.input', return_value=line) as mock_method: @@ -146,15 +181,11 @@ class ODLTesting(unittest.TestCase): self.assertEqual(args[0].getvalue(), "AUTH = [u'{}', u'{}']\n".format('foo', 'bar')) - @classmethod - def _fake_url_for(cls, service_type='identity', **kwargs): - if service_type == 'identity': - return "http://{}:5000/v2.0".format( - ODLTesting._keystone_ip) - elif service_type == 'network': - return "http://{}:9696".format(ODLTesting._neutron_ip) - else: - return None + +class ODLMainTesting(ODLTesting): + + """The class testing ODLTests.main().""" + # pylint: disable=missing-docstring def _get_main_kwargs(self, key=None): kwargs = {'odlusername': self._odl_username, @@ -199,148 +230,168 @@ class ODLTesting(unittest.TestCase): args[2].assert_called_with( os.path.join(odl.ODLTests.res_dir, 'stdout.txt')) - def _test_main_missing_keyword(self, key): + def _test_no_keyword(self, key): kwargs = self._get_main_kwargs(key) self.assertEqual(self.test.main(**kwargs), - testcase_base.TestcaseBase.EX_RUN_ERROR) + testcase.TestCase.EX_RUN_ERROR) - def test_main_missing_odlusername(self): - self._test_main_missing_keyword('odlusername') + def test_no_odlusername(self): + self._test_no_keyword('odlusername') - def test_main_missing_odlpassword(self): - self._test_main_missing_keyword('odlpassword') + def test_no_odlpassword(self): + self._test_no_keyword('odlpassword') - def test_main_missing_neutronip(self): - self._test_main_missing_keyword('neutronip') + def test_no_neutronip(self): + self._test_no_keyword('neutronip') - def test_main_missing_osauthurl(self): - self._test_main_missing_keyword('osauthurl') + def test_no_osauthurl(self): + self._test_no_keyword('osauthurl') - def test_main_missing_osusername(self): - self._test_main_missing_keyword('osusername') + def test_no_osusername(self): + self._test_no_keyword('osusername') - def test_main_missing_ostenantname(self): - self._test_main_missing_keyword('ostenantname') + def test_no_ostenantname(self): + self._test_no_keyword('ostenantname') - def test_main_missing_ospassword(self): - self._test_main_missing_keyword('ospassword') + def test_no_ospassword(self): + self._test_no_keyword('ospassword') - def test_main_missing_odlip(self): - self._test_main_missing_keyword('odlip') + def test_no_odlip(self): + self._test_no_keyword('odlip') - def test_main_missing_odlwebport(self): - self._test_main_missing_keyword('odlwebport') + def test_no_odlwebport(self): + self._test_no_keyword('odlwebport') - def test_main_missing_odlrestconfport(self): - self._test_main_missing_keyword('odlrestconfport') + def test_no_odlrestconfport(self): + self._test_no_keyword('odlrestconfport') - def test_main_set_robotframework_vars_failed(self): + def test_set_vars_ko(self): with mock.patch.object(self.test, 'set_robotframework_vars', - return_value=False): - self._test_main(testcase_base.TestcaseBase.EX_RUN_ERROR) - self.test.set_robotframework_vars.assert_called_once_with( + return_value=False) as mock_object: + self._test_main(testcase.TestCase.EX_RUN_ERROR) + mock_object.assert_called_once_with( self._odl_username, self._odl_password) @mock.patch('os.makedirs', side_effect=Exception) - def test_main_makedirs_exception(self, mock_method): + def test_makedirs_exc(self, mock_method): with mock.patch.object(self.test, 'set_robotframework_vars', return_value=True), \ self.assertRaises(Exception): - self._test_main(testcase_base.TestcaseBase.EX_RUN_ERROR, + self._test_main(testcase.TestCase.EX_RUN_ERROR, mock_method) @mock.patch('os.makedirs', side_effect=OSError) - def test_main_makedirs_oserror(self, mock_method): + def test_makedirs_oserror(self, mock_method): with mock.patch.object(self.test, 'set_robotframework_vars', return_value=True): - self._test_main(testcase_base.TestcaseBase.EX_RUN_ERROR, + self._test_main(testcase.TestCase.EX_RUN_ERROR, mock_method) @mock.patch('robot.run', side_effect=RobotError) @mock.patch('os.makedirs') - def test_main_robot_run_failed(self, *args): + def test_run_ko(self, *args): with mock.patch.object(self.test, 'set_robotframework_vars', return_value=True), \ mock.patch.object(odl, 'open', mock.mock_open(), create=True), \ self.assertRaises(RobotError): - self._test_main(testcase_base.TestcaseBase.EX_RUN_ERROR, *args) + self._test_main(testcase.TestCase.EX_RUN_ERROR, *args) @mock.patch('robot.run') @mock.patch('os.makedirs') - def test_main_parse_results_failed(self, *args): + def test_parse_results_ko(self, *args): with mock.patch.object(self.test, 'set_robotframework_vars', return_value=True), \ mock.patch.object(odl, 'open', mock.mock_open(), create=True), \ mock.patch.object(self.test, 'parse_results', side_effect=RobotError): - self._test_main(testcase_base.TestcaseBase.EX_RUN_ERROR, *args) + self._test_main(testcase.TestCase.EX_RUN_ERROR, *args) @mock.patch('os.remove', side_effect=Exception) @mock.patch('robot.run') @mock.patch('os.makedirs') - def test_main_remove_exception(self, *args): + def test_remove_exc(self, *args): with mock.patch.object(self.test, 'set_robotframework_vars', return_value=True), \ mock.patch.object(self.test, 'parse_results'), \ self.assertRaises(Exception): - self._test_main(testcase_base.TestcaseBase.EX_OK, *args) + self._test_main(testcase.TestCase.EX_OK, *args) @mock.patch('os.remove') @mock.patch('robot.run') @mock.patch('os.makedirs') - def test_main(self, *args): + def test_ok(self, *args): with mock.patch.object(self.test, 'set_robotframework_vars', return_value=True), \ mock.patch.object(odl, 'open', mock.mock_open(), create=True), \ mock.patch.object(self.test, 'parse_results'): - self._test_main(testcase_base.TestcaseBase.EX_OK, *args) + self._test_main(testcase.TestCase.EX_OK, *args) @mock.patch('os.remove') @mock.patch('robot.run') @mock.patch('os.makedirs', side_effect=OSError(errno.EEXIST, '')) - def test_main_makedirs_oserror17(self, *args): + def test_makedirs_oserror17(self, *args): with mock.patch.object(self.test, 'set_robotframework_vars', return_value=True), \ mock.patch.object(odl, 'open', mock.mock_open(), create=True), \ mock.patch.object(self.test, 'parse_results'): - self._test_main(testcase_base.TestcaseBase.EX_OK, *args) + self._test_main(testcase.TestCase.EX_OK, *args) @mock.patch('os.remove') @mock.patch('robot.run', return_value=1) @mock.patch('os.makedirs') - def test_main_testcases_in_failure(self, *args): + def test_testcases_in_failure(self, *args): with mock.patch.object(self.test, 'set_robotframework_vars', return_value=True), \ mock.patch.object(odl, 'open', mock.mock_open(), create=True), \ mock.patch.object(self.test, 'parse_results'): - self._test_main(testcase_base.TestcaseBase.EX_OK, *args) + self._test_main(testcase.TestCase.EX_OK, *args) @mock.patch('os.remove', side_effect=OSError) @mock.patch('robot.run') @mock.patch('os.makedirs') - def test_main_remove_oserror(self, *args): + def test_remove_oserror(self, *args): with mock.patch.object(self.test, 'set_robotframework_vars', return_value=True), \ mock.patch.object(odl, 'open', mock.mock_open(), create=True), \ mock.patch.object(self.test, 'parse_results'): - self._test_main(testcase_base.TestcaseBase.EX_OK, *args) + self._test_main(testcase.TestCase.EX_OK, *args) + - def _test_run_missing_env_var(self, var): +class ODLRunTesting(ODLTesting): + + """The class testing ODLTests.run().""" + # pylint: disable=missing-docstring + + @classmethod + def _fake_url_for(cls, service_type='identity'): + if service_type == 'identity': + return "http://{}:5000/v2.0".format( + ODLTesting._keystone_ip) + elif service_type == 'network': + return "http://{}:9696".format(ODLTesting._neutron_ip) + else: + return None + + def _test_no_env_var(self, var): with mock.patch('functest.utils.openstack_utils.get_endpoint', side_effect=self._fake_url_for): del os.environ[var] self.assertEqual(self.test.run(), - testcase_base.TestcaseBase.EX_RUN_ERROR) + testcase.TestCase.EX_RUN_ERROR) + + def _test_run(self, status=testcase.TestCase.EX_OK, + exception=None, **kwargs): + odlip = kwargs['odlip'] if 'odlip' in kwargs else '127.0.0.3' + odlwebport = kwargs['odlwebport'] if 'odlwebport' in kwargs else '8080' + odlrestconfport = (kwargs['odlrestconfport'] + if 'odlrestconfport' in kwargs else '8181') - def _test_run(self, status=testcase_base.TestcaseBase.EX_OK, - exception=None, odlip="127.0.0.3", odlwebport="8080", - odlrestconfport="8181"): with mock.patch('functest.utils.openstack_utils.get_endpoint', side_effect=self._fake_url_for): if exception: @@ -358,11 +409,13 @@ class ODLTesting(unittest.TestCase): ospassword=self._os_password, ostenantname=self._os_tenantname, osusername=self._os_username) - def _test_run_defining_multiple_suites( - self, suites, - status=testcase_base.TestcaseBase.EX_OK, - exception=None, odlip="127.0.0.3", odlwebport="8080", - odlrestconfport="8181"): + def _test_multiple_suites(self, suites, + status=testcase.TestCase.EX_OK, + exception=None, **kwargs): + odlip = kwargs['odlip'] if 'odlip' in kwargs else '127.0.0.3' + odlwebport = kwargs['odlwebport'] if 'odlwebport' in kwargs else '8080' + odlrestconfport = (kwargs['odlrestconfport'] + if 'odlrestconfport' in kwargs else '8181') with mock.patch('functest.utils.openstack_utils.get_endpoint', side_effect=self._fake_url_for): if exception: @@ -380,179 +433,187 @@ class ODLTesting(unittest.TestCase): ospassword=self._os_password, ostenantname=self._os_tenantname, osusername=self._os_username) - def test_run_exception(self): + def test_exc(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', side_effect=auth_plugins.MissingAuthPlugin()): self.assertEqual(self.test.run(), - testcase_base.TestcaseBase.EX_RUN_ERROR) + testcase.TestCase.EX_RUN_ERROR) - def test_run_missing_os_auth_url(self): - self._test_run_missing_env_var("OS_AUTH_URL") + def test_no_os_auth_url(self): + self._test_no_env_var("OS_AUTH_URL") - def test_run_missing_os_username(self): - self._test_run_missing_env_var("OS_USERNAME") + def test_no_os_username(self): + self._test_no_env_var("OS_USERNAME") - def test_run_missing_os_password(self): - self._test_run_missing_env_var("OS_PASSWORD") + def test_no_os_password(self): + self._test_no_env_var("OS_PASSWORD") - def test_run_missing_os_tenant_name(self): - self._test_run_missing_env_var("OS_TENANT_NAME") + def test_no_os_tenant_name(self): + self._test_no_env_var("OS_TENANT_NAME") - def test_run_main_false(self): + def test_main_false(self): os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip - self._test_run(testcase_base.TestcaseBase.EX_RUN_ERROR, + self._test_run(testcase.TestCase.EX_RUN_ERROR, odlip=self._sdn_controller_ip, odlwebport=self._odl_webport) - def test_run_main_exception(self): + def test_main_exc(self): with self.assertRaises(Exception): os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip - self._test_run(status=testcase_base.TestcaseBase.EX_RUN_ERROR, + self._test_run(status=testcase.TestCase.EX_RUN_ERROR, exception=Exception(), odlip=self._sdn_controller_ip, odlwebport=self._odl_webport) - def test_run_missing_sdn_controller_ip(self): + def test_no_sdn_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', side_effect=self._fake_url_for): self.assertEqual(self.test.run(), - testcase_base.TestcaseBase.EX_RUN_ERROR) + testcase.TestCase.EX_RUN_ERROR) - def test_run_without_installer_type(self): + def test_without_installer_type(self): os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip - self._test_run(testcase_base.TestcaseBase.EX_OK, + self._test_run(testcase.TestCase.EX_OK, odlip=self._sdn_controller_ip, odlwebport=self._odl_webport) - def test_run_redefining_suites(self): + def test_suites(self): os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip - self._test_run_defining_multiple_suites( + self._test_multiple_suites( [odl.ODLTests.basic_suite_dir], - testcase_base.TestcaseBase.EX_OK, + testcase.TestCase.EX_OK, odlip=self._sdn_controller_ip, odlwebport=self._odl_webport) - def test_run_fuel(self): + def test_fuel(self): os.environ["INSTALLER_TYPE"] = "fuel" - self._test_run(testcase_base.TestcaseBase.EX_OK, + self._test_run(testcase.TestCase.EX_OK, odlip=self._neutron_ip, odlwebport='8282') - def test_run_apex_missing_sdn_controller_ip(self): + def test_apex_no_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', side_effect=self._fake_url_for): os.environ["INSTALLER_TYPE"] = "apex" self.assertEqual(self.test.run(), - testcase_base.TestcaseBase.EX_RUN_ERROR) + testcase.TestCase.EX_RUN_ERROR) - def test_run_apex(self): + def test_apex(self): os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip os.environ["INSTALLER_TYPE"] = "apex" - self._test_run(testcase_base.TestcaseBase.EX_OK, + self._test_run(testcase.TestCase.EX_OK, odlip=self._sdn_controller_ip, odlwebport='8081', odlrestconfport='8081') - def test_run_netvirt_missing_sdn_controller_ip(self): + def test_netvirt_no_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', side_effect=self._fake_url_for): os.environ["INSTALLER_TYPE"] = "netvirt" self.assertEqual(self.test.run(), - testcase_base.TestcaseBase.EX_RUN_ERROR) + testcase.TestCase.EX_RUN_ERROR) - def test_run_netvirt(self): + def test_netvirt(self): os.environ["SDN_CONTROLLER_IP"] = self._sdn_controller_ip os.environ["INSTALLER_TYPE"] = "netvirt" - self._test_run(testcase_base.TestcaseBase.EX_OK, + self._test_run(testcase.TestCase.EX_OK, odlip=self._sdn_controller_ip, odlwebport='8081', odlrestconfport='8081') - def test_run_joid_missing_sdn_controller(self): + def test_joid_no_controller_ip(self): with mock.patch('functest.utils.openstack_utils.get_endpoint', side_effect=self._fake_url_for): os.environ["INSTALLER_TYPE"] = "joid" self.assertEqual(self.test.run(), - testcase_base.TestcaseBase.EX_RUN_ERROR) + testcase.TestCase.EX_RUN_ERROR) - def test_run_joid(self): + def test_joid(self): os.environ["SDN_CONTROLLER"] = self._sdn_controller_ip os.environ["INSTALLER_TYPE"] = "joid" - self._test_run(testcase_base.TestcaseBase.EX_OK, + self._test_run(testcase.TestCase.EX_OK, odlip=self._sdn_controller_ip, odlwebport='8080') - def test_run_compass(self, *args): + def test_compass(self): os.environ["INSTALLER_TYPE"] = "compass" - self._test_run(testcase_base.TestcaseBase.EX_OK, + self._test_run(testcase.TestCase.EX_OK, odlip=self._neutron_ip, odlwebport='8181') - def test_argparser_default(self): - parser = odl.ODLParser() - self.assertEqual(parser.parse_args(), self.defaultargs) - def test_argparser_basic(self): +class ODLArgParserTesting(ODLTesting): + + """The class testing ODLParser.""" + # pylint: disable=missing-docstring + + def setUp(self): + self.parser = odl.ODLParser() + super(ODLArgParserTesting, self).setUp() + + def test_default(self): + self.assertEqual(self.parser.parse_args(), self.defaultargs) + + def test_basic(self): self.defaultargs['neutronip'] = self._neutron_ip self.defaultargs['odlip'] = self._sdn_controller_ip - parser = odl.ODLParser() - self.assertEqual(parser.parse_args( - ["--neutronip={}".format(self._neutron_ip), - "--odlip={}".format(self._sdn_controller_ip) - ]), self.defaultargs) + self.assertEqual( + self.parser.parse_args( + ["--neutronip={}".format(self._neutron_ip), + "--odlip={}".format(self._sdn_controller_ip)]), + self.defaultargs) @mock.patch('sys.stderr', new_callable=StringIO.StringIO) - def test_argparser_fail(self, *args): + def test_fail(self, mock_method): self.defaultargs['foo'] = 'bar' - parser = odl.ODLParser() with self.assertRaises(SystemExit): - parser.parse_args(["--foo=bar"]) + self.parser.parse_args(["--foo=bar"]) + mock_method.assert_called_once_with() - def _test_argparser(self, arg, value): + def _test_arg(self, arg, value): self.defaultargs[arg] = value - parser = odl.ODLParser() - self.assertEqual(parser.parse_args(["--{}={}".format(arg, value)]), - self.defaultargs) + self.assertEqual( + self.parser.parse_args(["--{}={}".format(arg, value)]), + self.defaultargs) - def test_argparser_odlusername(self): - self._test_argparser('odlusername', 'foo') + def test_odlusername(self): + self._test_arg('odlusername', 'foo') - def test_argparser_odlpassword(self): - self._test_argparser('odlpassword', 'foo') + def test_odlpassword(self): + self._test_arg('odlpassword', 'foo') - def test_argparser_osauthurl(self): - self._test_argparser('osauthurl', 'http://127.0.0.4:5000/v2') + def test_osauthurl(self): + self._test_arg('osauthurl', 'http://127.0.0.4:5000/v2') - def test_argparser_neutronip(self): - self._test_argparser('neutronip', '127.0.0.4') + def test_neutronip(self): + self._test_arg('neutronip', '127.0.0.4') - def test_argparser_osusername(self): - self._test_argparser('osusername', 'foo') + def test_osusername(self): + self._test_arg('osusername', 'foo') - def test_argparser_ostenantname(self): - self._test_argparser('ostenantname', 'foo') + def test_ostenantname(self): + self._test_arg('ostenantname', 'foo') - def test_argparser_ospassword(self): - self._test_argparser('ospassword', 'foo') + def test_ospassword(self): + self._test_arg('ospassword', 'foo') - def test_argparser_odlip(self): - self._test_argparser('odlip', '127.0.0.4') + def test_odlip(self): + self._test_arg('odlip', '127.0.0.4') - def test_argparser_odlwebport(self): - self._test_argparser('odlwebport', '80') + def test_odlwebport(self): + self._test_arg('odlwebport', '80') - def test_argparser_odlrestconfport(self): - self._test_argparser('odlrestconfport', '80') + def test_odlrestconfport(self): + self._test_arg('odlrestconfport', '80') - def test_argparser_pushtodb(self): + def test_pushtodb(self): self.defaultargs['pushtodb'] = True - parser = odl.ODLParser() - self.assertEqual(parser.parse_args(["--{}".format('pushtodb')]), + self.assertEqual(self.parser.parse_args(["--{}".format('pushtodb')]), self.defaultargs) - def test_argparser_multiple_args(self): + def test_multiple_args(self): self.defaultargs['neutronip'] = self._neutron_ip self.defaultargs['odlip'] = self._sdn_controller_ip - parser = odl.ODLParser() - self.assertEqual(parser.parse_args( - ["--neutronip={}".format(self._neutron_ip), - "--odlip={}".format(self._sdn_controller_ip) - ]), self.defaultargs) + self.assertEqual( + self.parser.parse_args( + ["--neutronip={}".format(self._neutron_ip), + "--odlip={}".format(self._sdn_controller_ip)]), + self.defaultargs) if __name__ == "__main__": diff --git a/functest/tests/unit/opnfv_tests/openstack/rally/test_rally.py b/functest/tests/unit/opnfv_tests/openstack/rally/test_rally.py index ad39be48c..fe25dfcf9 100644 --- a/functest/tests/unit/opnfv_tests/openstack/rally/test_rally.py +++ b/functest/tests/unit/opnfv_tests/openstack/rally/test_rally.py @@ -12,7 +12,7 @@ import unittest import mock -from functest.core import testcase_base +from functest.core import testcase from functest.opnfv_tests.openstack.rally import rally from functest.utils.constants import CONST @@ -378,13 +378,13 @@ class OSRallyTesting(unittest.TestCase): mock.patch.object(self.rally_base, '_generate_report'), \ mock.patch.object(self.rally_base, '_clean_up'): self.assertEqual(self.rally_base.run(), - testcase_base.TestcaseBase.EX_OK) + testcase.TestCase.EX_OK) def test_run_exception(self): with mock.patch.object(self.rally_base, '_prepare_env', side_effect=Exception): self.assertEqual(self.rally_base.run(), - testcase_base.TestcaseBase.EX_RUN_ERROR) + testcase.TestCase.EX_RUN_ERROR) if __name__ == "__main__": diff --git a/functest/tests/unit/opnfv_tests/openstack/refstack_client/test_refstack_client.py b/functest/tests/unit/opnfv_tests/openstack/refstack_client/test_refstack_client.py index 4e83f6bf3..60e180c9e 100644 --- a/functest/tests/unit/opnfv_tests/openstack/refstack_client/test_refstack_client.py +++ b/functest/tests/unit/opnfv_tests/openstack/refstack_client/test_refstack_client.py @@ -10,7 +10,7 @@ import mock import os import unittest -from functest.core import testcase_base +from functest.core import testcase from functest.opnfv_tests.openstack.refstack_client import refstack_client from functest.utils.constants import CONST @@ -70,7 +70,7 @@ class OSRefstackClientTesting(unittest.TestCase): def _test_main_missing_keyword(self, key): kwargs = self._get_main_kwargs(key) self.assertEqual(self.refstackclient.main(**kwargs), - testcase_base.TestcaseBase.EX_RUN_ERROR) + testcase.TestCase.EX_RUN_ERROR) def test_main_missing_conf(self): self._test_main_missing_keyword('config') diff --git a/functest/tests/unit/opnfv_tests/openstack/tempest/test_conf_utils.py b/functest/tests/unit/opnfv_tests/openstack/tempest/test_conf_utils.py index caf21925a..8ca5cc5b6 100644 --- a/functest/tests/unit/opnfv_tests/openstack/tempest/test_conf_utils.py +++ b/functest/tests/unit/opnfv_tests/openstack/tempest/test_conf_utils.py @@ -37,7 +37,6 @@ class OSTempestConfUtilsTesting(unittest.TestCase): self.assertTrue(msg in context) def test_create_tempest_resources_missing_image(self): - CONST.tempest_use_custom_images = 'test_image' with mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' 'os_utils.get_keystone_client', return_value=mock.Mock()), \ @@ -54,13 +53,18 @@ class OSTempestConfUtilsTesting(unittest.TestCase): 'os_utils.get_or_create_image', return_value=(mock.Mock(), None)), \ self.assertRaises(Exception) as context: + + CONST.tempest_use_custom_images = True conf_utils.create_tempest_resources() msg = 'Failed to create image' self.assertTrue(msg in context) + CONST.tempest_use_custom_images = False + conf_utils.create_tempest_resources(use_custom_images=True) + msg = 'Failed to create image' + self.assertTrue(msg in context) + def test_create_tempest_resources_missing_flavor(self): - CONST.tempest_use_custom_images = 'test_image' - CONST.tempest_use_custom_flavors = 'test_flavour' with mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' 'os_utils.get_keystone_client', return_value=mock.Mock()), \ @@ -80,10 +84,18 @@ class OSTempestConfUtilsTesting(unittest.TestCase): 'os_utils.get_or_create_flavor', return_value=(mock.Mock(), None)), \ self.assertRaises(Exception) as context: + CONST.tempest_use_custom_images = True + CONST.tempest_use_custom_flavors = True conf_utils.create_tempest_resources() msg = 'Failed to create flavor' self.assertTrue(msg in context) + CONST.tempest_use_custom_images = True + CONST.tempest_use_custom_flavors = False + conf_utils.create_tempest_resources(use_custom_flavors=False) + msg = 'Failed to create flavor' + self.assertTrue(msg in context) + def test_get_verifier_id_missing_verifier(self): CONST.tempest_deployment_name = 'test_deploy_name' with mock.patch('functest.opnfv_tests.openstack.tempest.' @@ -153,6 +165,210 @@ class OSTempestConfUtilsTesting(unittest.TestCase): self.assertTrue(m1.called) self.assertTrue(m2.called) + def test_get_repo_tag_default(self): + mock_popen = mock.Mock() + attrs = {'stdout.readline.return_value': 'test_tag'} + mock_popen.configure_mock(**attrs) + + with mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.subprocess.Popen', + return_value=mock_popen): + self.assertEqual(conf_utils.get_repo_tag('test_repo'), + 'test_tag') + + def test_backup_tempest_config_default(self): + with mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.os.path.exists', + return_value=False), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.os.makedirs') as m1, \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.shutil.copyfile') as m2: + conf_utils.backup_tempest_config('test_conf_file') + self.assertTrue(m1.called) + self.assertTrue(m2.called) + + with mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.os.path.exists', + return_value=True), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.shutil.copyfile') as m2: + conf_utils.backup_tempest_config('test_conf_file') + self.assertTrue(m2.called) + + def test_configure_tempest_default(self): + with mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.configure_verifier', + return_value='test_conf_file'), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.configure_tempest_update_params') as m1, \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.configure_tempest_multisite_params') as m2: + conf_utils.configure_tempest('test_dep_dir', + MODE='feature_multisite') + self.assertTrue(m1.called) + self.assertTrue(m2.called) + + with mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.configure_verifier', + return_value='test_conf_file'), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.configure_tempest_update_params') as m1: + conf_utils.configure_tempest('test_dep_dir') + self.assertTrue(m1.called) + self.assertTrue(m2.called) + + def test_configure_tempest_defcore_default(self): + img_flavor_dict = {'image_id': 'test_image_id', + 'flavor_id': 'test_flavor_id', + 'image_id_alt': 'test_image_alt_id', + 'flavor_id_alt': 'test_flavor_alt_id'} + with mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.configure_verifier', + return_value='test_conf_file'), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.configure_tempest_update_params'), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ConfigParser.RawConfigParser.' + 'set') as mset, \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ConfigParser.RawConfigParser.' + 'read') as mread, \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ConfigParser.RawConfigParser.' + 'write') as mwrite, \ + mock.patch('__builtin__.open', mock.mock_open()), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.shutil.copyfile'): + CONST.dir_functest_test = 'test_dir' + CONST.refstack_tempest_conf_path = 'test_path' + conf_utils.configure_tempest_defcore('test_dep_dir', + img_flavor_dict) + mset.assert_any_call('compute', 'image_ref', 'test_image_id') + mset.assert_any_call('compute', 'image_ref_alt', + 'test_image_alt_id') + mset.assert_any_call('compute', 'flavor_ref', 'test_flavor_id') + mset.assert_any_call('compute', 'flavor_ref_alt', + 'test_flavor_alt_id') + self.assertTrue(mread.called) + self.assertTrue(mwrite.called) + + def _test_missing_param(self, params, image_id, flavor_id): + with mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ConfigParser.RawConfigParser.' + 'set') as mset, \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ConfigParser.RawConfigParser.' + 'read') as mread, \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ConfigParser.RawConfigParser.' + 'write') as mwrite, \ + mock.patch('__builtin__.open', mock.mock_open()), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.backup_tempest_config'): + CONST.dir_functest_test = 'test_dir' + CONST.OS_ENDPOINT_TYPE = None + conf_utils.\ + configure_tempest_update_params('test_conf_file', + IMAGE_ID=image_id, + FLAVOR_ID=flavor_id) + mset.assert_any_call(params[0], params[1], params[2]) + self.assertTrue(mread.called) + self.assertTrue(mwrite.called) + + def test_configure_tempest_update_params_missing_image_id(self): + CONST.tempest_use_custom_images = True + self._test_missing_param(('compute', 'image_ref', + 'test_image_id'), 'test_image_id', + None) + + def test_configure_tempest_update_params_missing_image_id_alt(self): + CONST.tempest_use_custom_images = True + conf_utils.IMAGE_ID_ALT = 'test_image_id_alt' + self._test_missing_param(('compute', 'image_ref_alt', + 'test_image_id_alt'), None, None) + + def test_configure_tempest_update_params_missing_flavor_id(self): + CONST.tempest_use_custom_flavors = True + self._test_missing_param(('compute', 'flavor_ref', + 'test_flavor_id'), None, + 'test_flavor_id') + + def test_configure_tempest_update_params_missing_flavor_id_alt(self): + CONST.tempest_use_custom_flavors = True + conf_utils.FLAVOR_ID_ALT = 'test_flavor_id_alt' + self._test_missing_param(('compute', 'flavor_ref_alt', + 'test_flavor_id_alt'), None, + None) + + def test_configure_verifier_missing_temp_conf_file(self): + with mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.os.path.isfile', + return_value=False), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ft_utils.execute_command') as mexe, \ + self.assertRaises(Exception) as context: + conf_utils.configure_verifier('test_dep_dir') + mexe.assert_any_call("rally verify configure-verifier") + msg = ("Tempest configuration file 'test_dep_dir/tempest.conf'" + " NOT found.") + self.assertTrue(msg in context) + + def test_configure_verifier_default(self): + with mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.os.path.isfile', + return_value=True), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ft_utils.execute_command') as mexe: + self.assertEqual(conf_utils.configure_verifier('test_dep_dir'), + 'test_dep_dir/tempest.conf') + mexe.assert_any_call("rally verify configure-verifier " + "--reconfigure") + + def test_configure_tempest_multisite_params_without_fuel(self): + conf_utils.CI_INSTALLER_TYPE = 'not_fuel' + with mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.os_utils.get_endpoint', + return_value='kingbird_endpoint_url'), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ConfigParser.RawConfigParser.' + 'set') as mset, \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ConfigParser.RawConfigParser.' + 'read') as mread, \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ConfigParser.RawConfigParser.' + 'add_section') as msection, \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ConfigParser.RawConfigParser.' + 'write') as mwrite, \ + mock.patch('__builtin__.open', mock.mock_open()), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.backup_tempest_config'): + + conf_utils.configure_tempest_multisite_params('test_conf_file') + msection.assert_any_call("kingbird") + mset.assert_any_call('service_available', 'kingbird', 'true') + mset.assert_any_call('kingbird', 'endpoint_type', 'publicURL') + mset.assert_any_call('kingbird', 'TIME_TO_SYNC', '120') + mset.assert_any_call('kingbird', 'endpoint_url', + 'kingbird_endpoint_url') + self.assertTrue(mread.called) + self.assertTrue(mwrite.called) + + def test_install_verifier_ext_default(self): + with mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.get_repo_tag', + return_value='test_tag'), \ + mock.patch('functest.opnfv_tests.openstack.tempest.' + 'conf_utils.ft_utils.' + 'execute_command_raise') as mexe: + conf_utils.install_verifier_ext('test_path') + cmd = ("rally verify add-verifier-ext --source test_path " + "--version test_tag") + error_msg = ("Problem while adding verifier extension from" + " test_path") + mexe.assert_called_once_with(cmd, error_msg=error_msg) if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/functest/tests/unit/opnfv_tests/openstack/tempest/test_tempest.py b/functest/tests/unit/opnfv_tests/openstack/tempest/test_tempest.py index 856cd1434..34031b404 100644 --- a/functest/tests/unit/opnfv_tests/openstack/tempest/test_tempest.py +++ b/functest/tests/unit/opnfv_tests/openstack/tempest/test_tempest.py @@ -10,9 +10,10 @@ import unittest import mock -from functest.core import testcase_base +from functest.core import testcase from functest.opnfv_tests.openstack.tempest import tempest from functest.opnfv_tests.openstack.tempest import conf_utils +from functest.utils.constants import CONST class OSTempestTesting(unittest.TestCase): @@ -33,6 +34,14 @@ class OSTempestTesting(unittest.TestCase): 'conf_utils.get_verifier_deployment_dir', return_value='test_verifier_deploy_dir'): self.tempestcommon = tempest.TempestCommon() + self.tempestsmoke_serial = tempest.TempestSmokeSerial() + self.tempestsmoke_parallel = tempest.TempestSmokeParallel() + self.tempestfull_parallel = tempest.TempestFullParallel() + with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' + 'conf_utils.install_verifier_ext'): + self.tempestmultisite = tempest.TempestMultisite() + self.tempestcustom = tempest.TempestCustom() + self.tempestdefcore = tempest.TempestDefcore() @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.logger.debug') def test_generate_test_list_defcore_mode(self, mock_logger_debug): @@ -100,6 +109,48 @@ class OSTempestTesting(unittest.TestCase): with self.assertRaises(Exception): self.tempestcommon.parse_verifier_result() + def test_apply_tempest_blacklist_no_blacklist(self): + with mock.patch('__builtin__.open', mock.mock_open()) as m, \ + mock.patch.object(self.tempestcommon, 'read_file', + return_value=['test1', 'test2']): + conf_utils.TEMPEST_BLACKLIST = Exception + CONST.INSTALLER_TYPE = 'installer_type' + CONST.DEPLOY_SCENARIO = 'deploy_scenario' + self.tempestcommon.apply_tempest_blacklist() + obj = m() + obj.write.assert_any_call('test1\n') + obj.write.assert_any_call('test2\n') + + def test_apply_tempest_blacklist_default(self): + item_dict = {'scenarios': ['deploy_scenario'], + 'installers': ['installer_type'], + 'tests': ['test2']} + with mock.patch('__builtin__.open', mock.mock_open()) as m, \ + mock.patch.object(self.tempestcommon, 'read_file', + return_value=['test1', 'test2']), \ + mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' + 'yaml.safe_load', return_value=item_dict): + CONST.INSTALLER_TYPE = 'installer_type' + CONST.DEPLOY_SCENARIO = 'deploy_scenario' + self.tempestcommon.apply_tempest_blacklist() + obj = m() + obj.write.assert_any_call('test1\n') + self.assertFalse(obj.write.assert_any_call('test2\n')) + + @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.logger.info') + def test_run_verifier_tests_default(self, mock_logger_info): + with mock.patch('__builtin__.open', mock.mock_open()), \ + mock.patch('__builtin__.iter', return_value=['\} tempest\.']), \ + mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' + 'subprocess.Popen'): + conf_utils.TEMPEST_LIST = 'test_tempest_list' + cmd_line = ("rally verify start --load-list " + "test_tempest_list --detailed") + self.tempestcommon.run_verifier_tests() + mock_logger_info. \ + assert_any_call("Starting Tempest test suite: '%s'." + % cmd_line) + @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.logger.info') def test_parse_verifier_result_default(self, mock_logger_info): self.tempestcommon.VERIFICATION_ID = 'test_uuid' @@ -119,7 +170,7 @@ class OSTempestTesting(unittest.TestCase): mock_method.assert_any_call('test_case_name', 100) def test_run_missing_create_tempest_dir(self): - ret = testcase_base.TestcaseBase.EX_RUN_ERROR + ret = testcase.TestCase.EX_RUN_ERROR with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'os.path.exists', return_value=False), \ mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' @@ -132,8 +183,8 @@ class OSTempestTesting(unittest.TestCase): self.assertTrue(mock_os_makedirs.called) def test_run_missing_configure_tempest(self): - ret = testcase_base.TestcaseBase.EX_RUN_ERROR - ret_ok = testcase_base.TestcaseBase.EX_OK + ret = testcase.TestCase.EX_RUN_ERROR + ret_ok = testcase.TestCase.EX_OK with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'os.path.exists', return_value=False), \ mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' @@ -149,8 +200,8 @@ class OSTempestTesting(unittest.TestCase): self.assertTrue(mock_os_makedirs.called) def test_run_missing_generate_test_list(self): - ret = testcase_base.TestcaseBase.EX_RUN_ERROR - ret_ok = testcase_base.TestcaseBase.EX_OK + ret = testcase.TestCase.EX_RUN_ERROR + ret_ok = testcase.TestCase.EX_OK with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'os.path.exists', return_value=False), \ mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' @@ -168,8 +219,8 @@ class OSTempestTesting(unittest.TestCase): self.assertTrue(mock_os_makedirs.called) def test_run_missing_apply_tempest_blacklist(self): - ret = testcase_base.TestcaseBase.EX_RUN_ERROR - ret_ok = testcase_base.TestcaseBase.EX_OK + ret = testcase.TestCase.EX_RUN_ERROR + ret_ok = testcase.TestCase.EX_OK with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'os.path.exists', return_value=False), \ mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' @@ -189,8 +240,8 @@ class OSTempestTesting(unittest.TestCase): self.assertTrue(mock_os_makedirs.called) def test_run_missing_parse_verifier_result(self): - ret = testcase_base.TestcaseBase.EX_RUN_ERROR - ret_ok = testcase_base.TestcaseBase.EX_OK + ret = testcase.TestCase.EX_RUN_ERROR + ret_ok = testcase.TestCase.EX_OK with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'os.path.exists', return_value=False), \ mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' diff --git a/functest/tests/unit/opnfv_tests/vnf/ims/test_clearwater.py b/functest/tests/unit/opnfv_tests/vnf/ims/test_clearwater.py index 527f12e59..18bebfdff 100644 --- a/functest/tests/unit/opnfv_tests/vnf/ims/test_clearwater.py +++ b/functest/tests/unit/opnfv_tests/vnf/ims/test_clearwater.py @@ -39,16 +39,48 @@ class ClearwaterTesting(unittest.TestCase): with mock.patch.object(self.clearwater.orchestrator, 'download_upload_and_deploy_blueprint', return_value=''): - self.clearwater.deploy_vnf(self.bp), + self.clearwater.deploy_vnf(self.bp) self.assertEqual(self.clearwater.deploy, True) def test_undeploy_vnf_deployment_passed(self): with mock.patch.object(self.clearwater.orchestrator, 'undeploy_deployment'): self.clearwater.deploy = True + self.clearwater.undeploy_vnf() + self.assertEqual(self.clearwater.deploy, False) + + def test_undeploy_vnf_deployment_with_undeploy(self): + with mock.patch.object(self.clearwater.orchestrator, + 'undeploy_deployment') as m: + self.clearwater.deploy = False + self.clearwater.undeploy_vnf(), + self.assertEqual(self.clearwater.deploy, False) + self.assertFalse(m.called) + + self.clearwater.orchestrator = None + self.clearwater.deploy = True + self.clearwater.undeploy_vnf(), + self.assertEqual(self.clearwater.deploy, True) + + self.clearwater.deploy = False self.clearwater.undeploy_vnf(), self.assertEqual(self.clearwater.deploy, False) + def test_set_methods(self): + self.clearwater.set_orchestrator(self.orchestrator) + self.assertTrue(self.clearwater.orchestrator, self.orchestrator) + self.clearwater.set_flavor_id('test_flavor_id') + self.assertTrue(self.clearwater.config['flavor_id'], 'test_flavor_id') + self.clearwater.set_image_id('test_image_id') + self.assertTrue(self.clearwater.config['image_id'], 'test_image_id') + self.clearwater.set_agent_user('test_user') + self.assertTrue(self.clearwater.config['agent_user'], 'test_user') + self.clearwater.set_external_network_name('test_network') + self.assertTrue(self.clearwater.config['external_network_name'], + 'test_network') + self.clearwater.set_public_domain('test_domain') + self.assertTrue(self.clearwater.config['public_domain'], + 'test_domain') if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/functest/tests/unit/opnfv_tests/vnf/ims/test_cloudify_ims.py b/functest/tests/unit/opnfv_tests/vnf/ims/test_cloudify_ims.py index e25816f01..f47ea865e 100644 --- a/functest/tests/unit/opnfv_tests/vnf/ims/test_cloudify_ims.py +++ b/functest/tests/unit/opnfv_tests/vnf/ims/test_cloudify_ims.py @@ -13,7 +13,7 @@ import mock from functest.opnfv_tests.vnf.ims import cloudify_ims -class ImsVnfTesting(unittest.TestCase): +class CloudifyImsTesting(unittest.TestCase): logging.disable(logging.CRITICAL) @@ -22,7 +22,7 @@ class ImsVnfTesting(unittest.TestCase): 'os.makedirs'), \ mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' 'get_config', return_value='config_value'): - self.ims_vnf = cloudify_ims.ImsVnf() + self.ims_vnf = cloudify_ims.CloudifyIms() self.neutron_client = mock.Mock() self.glance_client = mock.Mock() self.keystone_client = mock.Mock() @@ -419,94 +419,46 @@ class ImsVnfTesting(unittest.TestCase): self.ims_vnf.test_vnf() self.assertTrue(msg in context.exception) - def test_test_vnf_create_number_failure(self): - with mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'os.popen') as m, \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'requests.get'), \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'requests.post', - return_value=self.mock_post), \ - self.assertRaises(Exception) as context: - mock_obj = mock.Mock() - attrs = {'read.return_value': 'test_ip\n'} - mock_obj.configure_mock(**attrs) - m.return_value = mock_obj - - self.ims_vnf.test_vnf() - - msg = "Unable to create a number:" - self.assertTrue(msg in context.exception) - - def _get_post_status(self, url, cookies='', data=''): - ellis_url = "http://test_ellis_ip/session" - if url == ellis_url: - return self.mock_post_200 - return self.mock_post - def test_test_vnf_fail(self): with mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'os.popen') as m, \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'requests.get') as mock_get, \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'requests.post', - side_effect=self._get_post_status), \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'ft_utils.get_resolvconf_ns'), \ - mock.patch('__builtin__.open', mock.mock_open()), \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'subprocess.call'), \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'os.remove'), \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'json.load', return_value=''): - mock_obj = mock.Mock() - attrs = {'read.return_value': 'test_ip\n'} - mock_obj.configure_mock(**attrs) - m.return_value = mock_obj + 'os.popen'), \ + mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' + 'requests.get') as mock_get, \ + mock.patch.object(self.ims_vnf, 'config_ellis'), \ + mock.patch.object(self.ims_vnf, + 'run_clearwater_live_test') as clearwater_obj: + clearwater_obj.return_value = '' - mock_obj2 = mock.Mock() - attrs = {'json.return_value': {'outputs': - {'dns_ip': 'test_dns_ip', - 'ellis_ip': 'test_ellis_ip'}}} - mock_obj2.configure_mock(**attrs) - mock_get.return_value = mock_obj2 + mock_obj2 = mock.Mock() + attrs = {'json.return_value': {'outputs': + {'dns_ip': 'test_dns_ip', + 'ellis_ip': 'test_ellis_ip'}}} + mock_obj2.configure_mock(**attrs) + mock_get.return_value = mock_obj2 - self.assertEqual(self.ims_vnf.test_vnf(), - {'status': 'FAIL', 'result': ''}) + self.assertEqual(self.ims_vnf.test_vnf(), + {'status': 'FAIL', 'result': ''}) def test_test_vnf_pass(self): with mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'os.popen') as m, \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'requests.get') as mock_get, \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'requests.post', - side_effect=self._get_post_status), \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'ft_utils.get_resolvconf_ns'), \ - mock.patch('__builtin__.open', mock.mock_open()), \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'subprocess.call'), \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'os.remove'), \ - mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' - 'json.load', return_value='vims_test_result'): - mock_obj = mock.Mock() - attrs = {'read.return_value': 'test_ip\n'} - mock_obj.configure_mock(**attrs) - m.return_value = mock_obj - - mock_obj2 = mock.Mock() - attrs = {'json.return_value': {'outputs': - {'dns_ip': 'test_dns_ip', - 'ellis_ip': 'test_ellis_ip'}}} - mock_obj2.configure_mock(**attrs) - mock_get.return_value = mock_obj2 - - self.assertEqual(self.ims_vnf.test_vnf(), - {'status': 'PASS', 'result': 'vims_test_result'}) + 'os.popen'), \ + mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' + 'requests.get') as mock_get, \ + mock.patch.object(self.ims_vnf, 'config_ellis'), \ + mock.patch.object(self.ims_vnf, + 'run_clearwater_live_test') as clearwater_obj: + clearwater_obj.return_value = 'vims_test_result' + + mock_obj2 = mock.Mock() + attrs = {'json.return_value': {'outputs': + {'dns_ip': 'test_dns_ip', + 'ellis_ip': 'test_ellis_ip'}}} + mock_obj2.configure_mock(**attrs) + mock_get.return_value = mock_obj2 + + self.assertEqual(self.ims_vnf.test_vnf(), + {'status': 'PASS', + 'result': 'vims_test_result'}) def test_download_and_add_image_on_glance_incorrect_url(self): with mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' diff --git a/functest/tests/unit/opnfv_tests/vnf/ims/test_ims_base.by b/functest/tests/unit/opnfv_tests/vnf/ims/test_ims_base.by new file mode 100644 index 000000000..9440bcdf3 --- /dev/null +++ b/functest/tests/unit/opnfv_tests/vnf/ims/test_ims_base.by @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +import logging +import unittest + +import mock + +from functest.opnfv_tests.vnf.ims import ims_base + + +class ClearwaterOnBoardingBaseTesting(unittest.TestCase): + + logging.disable(logging.CRITICAL) + + def setUp(self): + with mock.patch('functest.opnfv_tests.vnf.ims.cloudify_ims.' + 'os.makedirs'): + self.ims_vnf = ims_base.ClearwaterOnBoardingBase() + + self.mock_post = mock.Mock() + attrs = {'status_code': 201, + 'cookies': ""} + self.mock_post.configure_mock(**attrs) + + self.mock_post_200 = mock.Mock() + attrs = {'status_code': 200, + 'cookies': ""} + self.mock_post_200.configure_mock(**attrs) + + self.mock_post_500 = mock.Mock() + attrs = {'status_code': 500, + 'cookies': ""} + self.mock_post_200.configure_mock(**attrs) + + def test_create_ellis_number_failure(self): + with mock.patch('functest.opnfv_tests.vnf.ims.ims_base.' + 'requests.post', + return_value=self.mock_post_500), \ + self.assertRaises(Exception) as context: + self.ims_vnf.create_ellis_number() + + msg = "Unable to create a number:" + self.assertTrue(msg in context.exception) + + def _get_post_status(self, url, cookies='', data=''): + ellis_url = "http://test_ellis_ip/session" + if url == ellis_url: + return self.mock_post_200 + return self.mock_post + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/functest/tests/unit/opnfv_tests/vnf/ims/test_orchestrator_cloudify.py b/functest/tests/unit/opnfv_tests/vnf/ims/test_orchestrator_cloudify.py index 620b0216f..bf6d483f7 100644 --- a/functest/tests/unit/opnfv_tests/vnf/ims/test_orchestrator_cloudify.py +++ b/functest/tests/unit/opnfv_tests/vnf/ims/test_orchestrator_cloudify.py @@ -6,6 +6,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 import logging +import subprocess32 as subprocess import unittest import mock @@ -117,6 +118,60 @@ class ImsVnfTesting(unittest.TestCase): 'dest'), True) + def test_execute_command_failed(self): + with mock.patch('__builtin__.open', + mock.mock_open(read_data='test_data\n')): + subprocess.call = mock.create_autospec(subprocess.call, + return_value=0) + mock_log = mock.Mock() + cmd = 'test_cmd -e test_env bash_script' + ret = orchestrator_cloudify.execute_command(cmd, mock_log, + timeout=100) + self.assertEqual(ret, False) + + def test_execute_command_default(self): + with mock.patch('__builtin__.open', + mock.mock_open(read_data='test_data\n')): + subprocess.call = mock. \ + create_autospec(subprocess.call, + return_value=subprocess.TimeoutExpired) + mock_log = mock.Mock() + cmd = 'test_cmd -e test_env bash_script' + ret = orchestrator_cloudify.execute_command(cmd, mock_log, + timeout=100) + self.assertEqual(ret, ['test_data\n']) + + def test_set_methods(self): + self.orchestrator.set_credentials('test_username', 'test_password', + 'test_tenant_name', 'test_auth_url') + self.assertTrue(self.orchestrator.config['keystone_username'], + 'test_username') + self.assertTrue(self.orchestrator.config['keystone_password'], + 'test_password') + self.assertTrue(self.orchestrator.config['keystone_url'], + 'test_auth_url') + self.assertTrue(self.orchestrator.config['keystone_tenant_name'], + 'test_tenant_name') + self.orchestrator.set_flavor_id('test_flavor_id') + self.assertTrue(self.orchestrator.config['flavor_id'], + 'test_flavor_id') + self.orchestrator.set_image_id('test_image_id') + self.assertTrue(self.orchestrator.config['image_id'], 'test_image_id') + self.orchestrator.set_external_network_name('test_network') + self.assertTrue(self.orchestrator.config['external_network_name'], + 'test_network') + self.orchestrator.set_ssh_user('test_user') + self.assertTrue(self.orchestrator.config['ssh_user'], + 'test_user') + self.orchestrator.set_nova_url('test_nova_url') + self.assertTrue(self.orchestrator.config['nova_url'], + 'test_nova_url') + self.orchestrator.set_neutron_url('test_neutron_url') + self.assertTrue(self.orchestrator.config['neutron_url'], + 'test_neutron_url') + self.orchestrator.set_nameservers(['test_subnet']) + self.assertTrue(self.orchestrator.config['dns_subnet_1'], + 'test_subnet') if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/run_unit_tests.sh b/run_unit_tests.sh index 32076dabe..3de9b36f9 100755 --- a/run_unit_tests.sh +++ b/run_unit_tests.sh @@ -40,7 +40,7 @@ nosetests --with-xunit \ --cover-tests \ --cover-package=functest.ci \ --cover-package=functest.cli \ - --cover-package=functest.core.testcase_base \ + --cover-package=functest.core.testcase \ --cover-package=functest.opnfv_tests.sdn.odl.odl \ --cover-package=functest.opnfv_tests.vnf.ims \ --cover-package=functest.utils \ |
