diff options
45 files changed, 642 insertions, 328 deletions
@@ -13,9 +13,7 @@ docker/features \ docker/components \ docker/vnf"} arm64_dirs=${arm64_dirs-${amd64_dirs}} -build_opts=(--pull=true --no-cache --force-rm=true \ - --build-arg OPENSTACK_TAG="${OPENSTACK_TAG:-stable/queens}" \ - --build-arg RALLY_OPENSTACK_TAG="${RALLY_OPENSTACK_TAG:-1.2.0}") +build_opts=(--pull=true --no-cache --force-rm=true) find . -name Dockerfile -exec sed -i \ -e "s|opnfv/functest-core|${repo}/functest-core:amd64-latest|g" {} + diff --git a/ci/globals.yaml b/ci/globals.yaml index 4be1b37a2..20217921f 100644 --- a/ci/globals.yaml +++ b/ci/globals.yaml @@ -10,3 +10,4 @@ prefix: '/home/opnfv/functest' nameserver: '8.8.8.8' external_network: 'ext-net' + storage_protocol: 'iSCSI' diff --git a/ci/run.yaml b/ci/run.yaml index 9cc374c0c..c7e27eccc 100644 --- a/ci/run.yaml +++ b/ci/run.yaml @@ -7,6 +7,7 @@ docker run \ -e NAMESERVER={nameserver} \ -e EXTERNAL_NETWORK={external_network} \ + -e STORAGE_PROTOCOL={storage_protocol} \ -v \ {prefix}/results/$JOB_NAME-$BUILD_ID:/home/opnfv/functest/results \ -v {prefix}/openstack.creds:/home/opnfv/functest/conf/env_file \ @@ -66,6 +67,7 @@ - tempest_smoke - neutron-tempest-plugin-api - rally_sanity + - rally_jobs - refstack_defcore - patrole - snaps_smoke @@ -86,6 +88,7 @@ - name: '{repo}-functest-smoke-tempest_smoke-{tag}-run' - name: '{repo}-functest-smoke-neutron-tempest-plugin-api-{tag}-run' - name: '{repo}-functest-smoke-rally_sanity-{tag}-run' + - name: '{repo}-functest-smoke-rally_jobs-{tag}-run' - name: '{repo}-functest-smoke-refstack_defcore-{tag}-run' - name: '{repo}-functest-smoke-patrole-{tag}-run' - name: '{repo}-functest-smoke-snaps_smoke-{tag}-run' diff --git a/docker/benchmarking/Dockerfile b/docker/benchmarking/Dockerfile index 7b98eade9..ff87f56d6 100644 --- a/docker/benchmarking/Dockerfile +++ b/docker/benchmarking/Dockerfile @@ -1,7 +1,7 @@ FROM opnfv/functest-core ARG BRANCH=master -ARG OPENSTACK_TAG=stable/queens +ARG OPENSTACK_TAG=master ARG VMTP_TAG=99b261ccccc2f8a08ee2d8fca9f54ef9d69899d7 RUN apk --no-cache add --update libxml2 libxslt && \ diff --git a/docker/core/Dockerfile b/docker/core/Dockerfile index c1e7ede03..eaedf2a29 100644 --- a/docker/core/Dockerfile +++ b/docker/core/Dockerfile @@ -1,7 +1,7 @@ FROM alpine:3.8 ARG BRANCH=master -ARG OPENSTACK_TAG=stable/queens +ARG OPENSTACK_TAG=master ARG PIP_TAG=18.0 RUN apk --no-cache add --update \ diff --git a/docker/features/Dockerfile b/docker/features/Dockerfile index 99de1eda5..01e0cb2bb 100644 --- a/docker/features/Dockerfile +++ b/docker/features/Dockerfile @@ -1,7 +1,7 @@ FROM opnfv/functest-tempest ARG BRANCH=master -ARG OPENSTACK_TAG=stable/queens +ARG OPENSTACK_TAG=master ARG FDS_TAG=master COPY thirdparty-requirements.txt thirdparty-requirements.txt @@ -10,7 +10,7 @@ RUN apk --no-cache add --update python3 sshpass && \ python-dev python3-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev && \ wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ - sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest@\\1#egg=tempest/ \ + sed -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ \ > upper-constraints.txt && \ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ diff --git a/docker/features/testcases.yaml b/docker/features/testcases.yaml index a2f71f8c1..f2793fdd1 100644 --- a/docker/features/testcases.yaml +++ b/docker/features/testcases.yaml @@ -10,6 +10,7 @@ tiers: - case_name: doctor-notification project_name: doctor + enabled: false criteria: 100 blocking: false description: >- @@ -25,6 +26,7 @@ tiers: - case_name: bgpvpn project_name: sdnvpn + enabled: false criteria: 100 blocking: false description: >- @@ -37,6 +39,7 @@ tiers: - case_name: functest-odl-sfc project_name: sfc + enabled: false criteria: 100 blocking: false description: >- @@ -50,6 +53,7 @@ tiers: - case_name: barometercollectd project_name: barometer + enabled: false criteria: 100 blocking: false description: >- @@ -64,6 +68,7 @@ tiers: - case_name: fds project_name: fastdatastacks + enabled: false criteria: 100 blocking: false description: >- @@ -92,6 +97,7 @@ tiers: - case_name: stor4nfv_os project_name: stor4nfv + enabled: false criteria: 100 blocking: false description: >- diff --git a/docker/features/thirdparty-requirements.txt b/docker/features/thirdparty-requirements.txt index 510b3e6f6..e9b16b933 100644 --- a/docker/features/thirdparty-requirements.txt +++ b/docker/features/thirdparty-requirements.txt @@ -1,8 +1,8 @@ robotframework-httplibrary robotframework-requests robotframework-sshlibrary;python_version=='2.7' -baro_tests -sdnvpn -sfc -stor4nfv_tests -doctor-tests;python_version>='3.0' +# baro_tests +# sdnvpn +# sfc +# stor4nfv_tests +# doctor-tests;python_version>='3.0' diff --git a/docker/healthcheck/Dockerfile b/docker/healthcheck/Dockerfile index 4a1c255b3..aba2bbdee 100644 --- a/docker/healthcheck/Dockerfile +++ b/docker/healthcheck/Dockerfile @@ -1,7 +1,7 @@ FROM opnfv/functest-core ARG BRANCH=master -ARG OPENSTACK_TAG=stable/queens +ARG OPENSTACK_TAG=master ARG ODL_TAG=85448c9d97b89989488e675b29b38ac42d8674e4 COPY thirdparty-requirements.txt thirdparty-requirements.txt diff --git a/docker/smoke/Dockerfile b/docker/smoke/Dockerfile index c405330a6..19fa9d9c1 100644 --- a/docker/smoke/Dockerfile +++ b/docker/smoke/Dockerfile @@ -1,17 +1,18 @@ FROM opnfv/functest-tempest ARG BRANCH=master -ARG OPENSTACK_TAG=stable/queens +ARG OPENSTACK_TAG=master ARG REFSTACK_TARGET=2018.02 -ARG PATROLE_TAG=0.3.0 -ARG NEUTRON_TAG=0.2.0 +ARG PATROLE_TAG=0.4.0 +ARG NEUTRON_TAG=master ARG BARBICAN_TAG=0.1.0 +ARG NEUTRON_RALLY_TAG=master RUN apk --no-cache add --virtual .build-deps --update \ python-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev && \ wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ - sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest@\\1#egg=tempest/ \ + sed -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ \ > upper-constraints.txt && \ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ @@ -41,6 +42,11 @@ RUN apk --no-cache add --virtual .build-deps --update \ mkdir -p /etc/neutron /etc/glance && \ wget -q -O /etc/neutron/policy.json https://git.openstack.org/cgit/openstack/neutron/plain/etc/policy.json?h=$OPENSTACK_TAG && \ wget -q -O /etc/glance/policy.json https://git.openstack.org/cgit/openstack/glance/plain/etc/policy.json?h=$OPENSTACK_TAG && \ + git clone --depth 1 -b $NEUTRON_RALLY_TAG https://git.openstack.org/openstack/neutron.git /src/neutron && \ + (cd /src/neutron && git fetch --tags origin $NEUTRON_RALLY_TAG && git checkout FETCH_HEAD) && \ + mkdir -p /home/opnfv/functest/data/rally/neutron && \ + cp -r /src/neutron/rally-jobs /home/opnfv/functest/data/rally/neutron/rally-jobs && \ + rm -r /src/neutron && \ apk del .build-deps COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml CMD ["run_tests", "-t", "all"] diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml index a510c6474..55f7a102a 100644 --- a/docker/smoke/testcases.yaml +++ b/docker/smoke/testcases.yaml @@ -53,6 +53,17 @@ tiers: name: rally_sanity - + case_name: rally_jobs + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs a group of Rally jobs used in + OpenStack gating + run: + name: rally_jobs + + - case_name: refstack_defcore project_name: functest criteria: 100 @@ -82,6 +93,7 @@ tiers: - 'network' excludes: - 'test_networks_multiprovider_rbac' + - 'test_auto_allocated_topology_rbac' - case_name: snaps_smoke diff --git a/docker/tempest/Dockerfile b/docker/tempest/Dockerfile index 0cc947939..9174a495c 100644 --- a/docker/tempest/Dockerfile +++ b/docker/tempest/Dockerfile @@ -1,24 +1,17 @@ FROM opnfv/functest-core ARG BRANCH=master -ARG OPENSTACK_TAG=stable/queens +ARG OPENSTACK_TAG=master ARG RALLY_TAG=1.1.0 -ARG RALLY_OPENSTACK_TAG=1.2.0 +ARG RALLY_OPENSTACK_TAG=67882d55ca453d6aa863d279ec4cfc2b52c80f0a ARG OS_FAULTS_TAG=0.1.17 RUN apk --no-cache add --virtual .build-deps --update \ python-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev && \ - case $OPENSTACK_TAG in \ - master) \ - wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ - sed -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ \ - > upper-constraints.txt ;; \ - *) \ - wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ - sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest@\\1#egg=tempest/ \ - > upper-constraints.txt ;; \ - esac && \ + wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ + sed -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ \ + > upper-constraints.txt && \ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ git init /src/os-faults && \ diff --git a/docker/vnf/Dockerfile b/docker/vnf/Dockerfile index d081ff17f..0b39b480e 100644 --- a/docker/vnf/Dockerfile +++ b/docker/vnf/Dockerfile @@ -1,15 +1,16 @@ FROM opnfv/functest-core ARG BRANCH=master -ARG OPENSTACK_TAG=stable/queens +ARG OPENSTACK_TAG=master ARG VIMS_TEST_TAG=release-129 ARG QUAFF_TAG=59213d6d8ee29433552bb75f505cdc96b0b18909 ARG CLOUDIFY_VIMS_TAG=fraser ARG HEAT_VIMS_TAG=release-129 ARG VROUTER_TAG=fraser -ARG JUJU_TAG=tags/juju-2.2.5 +ARG JUJU_TAG=tags/juju-2.3.9 ARG JUJU_WAIT_TAG=2.6.4 ARG ABOT_CHARM=opnfv-fraser +ARG GODEPS_TAG=404a7e748cd352bb0d7449dedc645546eebbfc6e ENV GOPATH /src/epc-requirements/go ENV GOBIN /src/epc-requirements/go/bin @@ -47,13 +48,19 @@ RUN apk --no-cache add --update \ (cd /src/opnfv-vnf-vyos-blueprint && \ git fetch --tags https://github.com/oolorg/opnfv-vnf-vyos-blueprint.git $VROUTER_TAG && \ git checkout FETCH_HEAD) && \ + mkdir -p /home/opnfv/functest/data/router && \ + git init /home/opnfv/functest/data/router/opnfv-vnf-data && \ + (cd /home/opnfv/functest/data/router/opnfv-vnf-data && \ + git fetch --tags https://github.com/oolorg/opnfv-vnf-data.git $VROUTER_TAG && \ + git checkout FETCH_HEAD) && \ git init /src/epc-requirements/abot_charm && \ (cd /src/epc-requirements/abot_charm && \ git fetch --tags https://github.com/RebacaInc/abot_charm.git $ABOT_CHARM && \ git checkout FETCH_HEAD) && \ python3 -m pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ juju-wait==$JUJU_WAIT_TAG && \ - go get github.com/rogpeppe/godeps && \ + go get -d github.com/rogpeppe/godeps && \ + (cd $GOPATH/src/github.com/rogpeppe/godeps && git checkout $GODEPS_TAG && go install -v github.com/rogpeppe/godeps) && \ go get -d -v github.com/juju/juju/... || true && \ (cd $GOPATH/src/github.com/juju/juju && git checkout $JUJU_TAG && godeps -u dependencies.tsv) && \ go install -v github.com/juju/juju/... && \ diff --git a/docs/release/release-notes/functest-release.rst b/docs/release/release-notes/functest-release.rst index bb7600d35..67fa3dcbb 100644 --- a/docs/release/release-notes/functest-release.rst +++ b/docs/release/release-notes/functest-release.rst @@ -23,22 +23,37 @@ OpenStack The internal test cases are: * connection_check - * api_check - * snaps_health_check + * tenantnetwork1 + * tenantnetwork2 + * vmready1 + * vmready2 + * singlevm1 + * singlevm2 * vping_ssh * vping_userdata - * tempest_smoke_serial + * cinder_test + * api_check + * snaps_health_check + * odl + * tempest_smoke + * neutron-tempest-plugin-api * rally_sanity * refstack_defcore * patrole - * odl * snaps_smoke * neutron_trunk + * networking-bgpvpn + * networking-sfc + * barbican * tempest_full + * tempest_scenario * rally_full + * cloudify * cloudify_ims + * heat_ims * vyos_vrouter * juju_epc + * vgpu The OPNFV projects integrated into Functest framework for automation are: @@ -47,6 +62,7 @@ The OPNFV projects integrated into Functest framework for automation are: * odl-sfc * barometer * fds + * stor4nfv_os Kubernetes ---------- @@ -80,6 +96,7 @@ Software * https://hub.docker.com/r/opnfv/functest-healthcheck * https://hub.docker.com/r/opnfv/functest-smoke + * https://hub.docker.com/r/opnfv/functest-benchmarking * https://hub.docker.com/r/opnfv/functest-features * https://hub.docker.com/r/opnfv/functest-components * https://hub.docker.com/r/opnfv/functest-vnf @@ -95,9 +112,9 @@ Docker tag for master: latest Documents --------- - * Config Guide: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/configguide/index.html - * User Guide: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/user/userguide/index.html - * Developer Guide: http://docs.opnfv.org/en/latest/submodules/functest/docs/testing/developer/devguide/index.html + * Config Guide: https://functest.readthedocs.io/projects/configguide/en/latest/ + * User Guide: https://functest.readthedocs.io/projects/userguide/en/latest/ + * Developer Guide: https://functest.readthedocs.io/projects/devguide/en/latest/ * API Docs: http://functest.readthedocs.io/en/latest/ Version change @@ -106,44 +123,51 @@ Version change New test cases -------------- - * neutron_trunk - * patrole - * juju_epc - * k8s_smoke - * k8s_conformance - * stor4nfv - * clover + * tenantnetwork1 + * tenantnetwork2 + * vmready1 + * vmready2 + * singlevm1 + * singlevm2 + * cinder_test + * neutron-tempest-plugin-api + * rally_jobs + * networking-bgpvpn + * networking-sfc + * barbican + * vmtp + * shaker + * tempest_scenario + * cloudify + * heat_ims + * vgpu Key changes ----------- - * update test cases and containers to `OpenStack Pike`_ - * move the framework into a separate project: Xtesting_ - * ease running all containers thanks to default values - * clean interfaces with OPNFV Installers and Features - * rewrite all vnfs to allow multiple tests in parallel - * fully support non-default region names and Keystone v3 domains - * refactor all tempest-based test cases (e.g. refstack_defcore) - * remove obsolete OpenStack and Functest utils - * verify all changes via doc8 and yamllint too - * generate reports for all tempest-based test cases - -.. _`OpenStack Pike`: https://raw.githubusercontent.com/openstack/requirements/stable/pike/upper-constraints.txt -.. _Xtesting: http://xtesting.readthedocs.io/en/latest/ + * update test cases and containers to `OpenStack master`_ and to + `Kubernetes master`_ + * define new scenarios to ease writing testcases vs OpenStack + * isolate all resources created in different tenants + * fully remove all OPNFV logics + * publish new Jenkins jobs + * support VIO (VMware Integrated OpenStack) and arm64 for Kubernetes + * reduce Functest Kubernetes image sizes + * add tempest_full and tempest_scenario in all daily jobs + * include benchmarking tools such as Vmtp ans Shaker + * increase functional scope by adding bgpvpn and sfc tempest plugins + +.. _`OpenStack master`: https://github.com/openstack/requirements/blob/master/upper-constraints.txt +.. _`Kubernetes master`: https://github.com/kubernetes/kubernetes Key benefits ------------ - * the enduser can run all tests by setting only one input (EXTERNAL_NETWORK) - * the developer can only work on the test suites without diving into CI/CD - integration - * both OpenStack and Kubernetes deployments can be verified - * Functest test cases are trustable as they meet the best coding rules (unit - tests, coverage, linters, etc.) - * Functest can be reused in other projects (e.g. - `ONS: Re-using OPNFV framework tests for LFN projects`_) - -.. _`ONS: Re-using OPNFV framework tests for LFN projects`: https://wiki.lfnetworking.org/display/LN/LFN+Developer+Forum+Schedule?preview=/328197/328329/ONS-OPNFV%20framework%20tests%20for%20LFN%20projects.pdf + * the enduser can easily build its own toolchains by loading our Jenkins jobs + * all developpers can easily verify their changes before merge + * our testcases may be run vs VIM in production + * all testcases can run in parallel to decrease the overall duration + * Functest includes most of the OpenStack gate jobs Code quality ------------ diff --git a/docs/testing/developer/devguide/index.rst b/docs/testing/developer/devguide/index.rst index caa1ccb3b..d26a537a2 100644 --- a/docs/testing/developer/devguide/index.rst +++ b/docs/testing/developer/devguide/index.rst @@ -211,35 +211,6 @@ Feature, unit and vnf_base inherit from testcase:: +--------------------+ +---------+ +------------------------+ +-----------------+ -Testcase --------- -.. raw:: html - :url: http://artifacts.opnfv.org/functest/docs/apidoc/functest.core.testcase.html - -Feature -------- -.. raw:: html - :url: http://artifacts.opnfv.org/functest/docs/apidoc/functest.core.feature.html - -Unit ----- -.. raw:: html - :url: http://artifacts.opnfv.org/functest/docs/apidoc/functest.core.unit.html - -VNF ---- -.. raw:: html - :url: http://artifacts.opnfv.org/functest/docs/apidoc/functest.core.vnf.html - -Robotframework --------------- -.. raw:: html - :url: http://artifacts.opnfv.org/functest/docs/apidoc/functest.core.robotframework.html - - -see `[5]`_ to get code samples - - Functest util classes ===================== @@ -266,7 +237,7 @@ instances. SNAPS `[4]`_ is an OPNFV project providing OpenStack utils. TestAPI ======= Functest is using the Test collection framework and the TestAPI developed by -the OPNFV community. See `[6]`_ for details. +the OPNFV community. See `[5]`_ for details. Reporting @@ -278,7 +249,7 @@ jinja2 templates `[3]`_. Dashboard ========= -Additional dashboarding is managed at the testing group level, see `[7]`_ for +Additional dashboarding is managed at the testing group level, see `[6]`_ for details. @@ -286,7 +257,7 @@ details. How TOs ======= -See How to section on Functest wiki `[8]`_ +See How to section on Functest wiki `[7]`_ ========== @@ -301,12 +272,10 @@ _`[3]`: https://git.opnfv.org/cgit/releng/tree/utils/test/reporting _`[4]`: https://git.opnfv.org/snaps/ -_`[5]` : http://testresults.opnfv.org/functest/framework/index.html - -_`[6]`: http://docs.opnfv.org/en/latest/testing/testing-dev.html +_`[5]`: http://docs.opnfv.org/en/latest/testing/testing-dev.html -_`[7]`: https://opnfv.biterg.io/goto/283dba93ca18e95964f852c63af1d1ba +_`[6]`: https://opnfv.biterg.io/goto/283dba93ca18e95964f852c63af1d1ba -_`[8]`: https://wiki.opnfv.org/pages/viewpage.action?pageId=7768932 +_`[7]`: https://wiki.opnfv.org/pages/viewpage.action?pageId=7768932 IRC support chan: #opnfv-functest diff --git a/docs/testing/user/configguide/configguide.rst b/docs/testing/user/configguide/configguide.rst index c04689191..09fca19b5 100644 --- a/docs/testing/user/configguide/configguide.rst +++ b/docs/testing/user/configguide/configguide.rst @@ -16,19 +16,11 @@ Docker images are available on the dockerhub: * opnfv/functest-core * opnfv/functest-healthcheck * opnfv/functest-smoke + * opnfv/functest-benchmarking * opnfv/functest-features * opnfv/functest-components * opnfv/functest-vnf -The tag "opnfv-6.0.0" is the official release image in Fraser, but you can -also pull "fraser" tag as it is being maintained by Functest team and might -include bugfixes. - -The Functest docker container environment can -in principle- be also -used with non-OPNFV official installers (e.g. 'devstack'), with the -**disclaimer** that support for such environments is outside of the -scope and responsibility of the OPNFV project. - Preparing your environment ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -62,10 +54,9 @@ Create a directory for the different images (attached as a Docker volume):: images/CentOS-7-x86_64-GenericCloud.qcow2 images/cirros-0.4.0-x86_64-disk.img images/cirros-0.4.0-x86_64-lxc.tar.gz - images/cirros-d161201-aarch64-disk.img - images/cirros-d161201-aarch64-initramfs - images/cirros-d161201-aarch64-kernel images/cloudify-manager-premium-4.0.1.qcow2 + images/shaker-image-arm64.qcow2 + images/shaker-image.qcow images/ubuntu-14.04-server-cloudimg-amd64-disk1.img images/ubuntu-14.04-server-cloudimg-arm64-uefi1.img images/ubuntu-16.04-server-cloudimg-amd64-disk1.img @@ -86,11 +77,22 @@ Results shall be displayed as follows:: +----------------------------+------------------+---------------------+------------------+----------------+ | TEST CASE | PROJECT | TIER | DURATION | RESULT | +----------------------------+------------------+---------------------+------------------+----------------+ - | connection_check | functest | healthcheck | 00:07 | PASS | - | api_check | functest | healthcheck | 07:46 | PASS | - | snaps_health_check | functest | healthcheck | 00:36 | PASS | + | connection_check | functest | healthcheck | 00:09 | PASS | + | tenantnetwork1 | functest | healthcheck | 00:14 | PASS | + | tenantnetwork2 | functest | healthcheck | 00:11 | PASS | + | vmready1 | functest | healthcheck | 00:19 | PASS | + | vmready2 | functest | healthcheck | 00:16 | PASS | + | singlevm1 | functest | healthcheck | 00:41 | PASS | + | singlevm2 | functest | healthcheck | 00:36 | PASS | + | vping_ssh | functest | healthcheck | 00:46 | PASS | + | vping_userdata | functest | healthcheck | 00:41 | PASS | + | cinder_test | functest | healthcheck | 01:18 | PASS | + | api_check | functest | healthcheck | 10:33 | PASS | + | snaps_health_check | functest | healthcheck | 00:44 | PASS | + | odl | functest | healthcheck | 00:00 | SKIP | +----------------------------+------------------+---------------------+------------------+----------------+ - NOTE: the duration is a reference and it might vary depending on your SUT. + +NOTE: the duration is a reference and it might vary depending on your SUT. Testing smoke suite ^^^^^^^^^^^^^^^^^^^ @@ -104,21 +106,46 @@ Run smoke suite:: Results shall be displayed as follows:: - +------------------------------+------------------+---------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +------------------------------+------------------+---------------+------------------+----------------+ - | vping_ssh | functest | smoke | 00:57 | PASS | - | vping_userdata | functest | smoke | 00:33 | PASS | - | tempest_smoke_serial | functest | smoke | 13:22 | PASS | - | rally_sanity | functest | smoke | 24:07 | PASS | - | refstack_defcore | functest | smoke | 05:21 | PASS | - | patrole | functest | smoke | 04:29 | PASS | - | snaps_smoke | functest | smoke | 46:54 | PASS | - | odl | functest | smoke | 00:00 | SKIP | - | neutron_trunk | functest | smoke | 00:00 | SKIP | - +------------------------------+------------------+---------------+------------------+----------------+ - Note: if the scenario does not support some tests, they are indicated as SKIP. - See User guide for details. + +------------------------------------+------------------+---------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +------------------------------------+------------------+---------------+------------------+----------------+ + | tempest_smoke | functest | smoke | 06:13 | PASS | + | neutron-tempest-plugin-api | functest | smoke | 09:32 | PASS | + | rally_sanity | functest | smoke | 29:34 | PASS | + | rally_jobs | functest | smoke | 24:02 | PASS | + | refstack_defcore | functest | smoke | 13:07 | PASS | + | patrole | functest | smoke | 05:17 | PASS | + | snaps_smoke | functest | smoke | 90:13 | PASS | + | neutron_trunk | functest | smoke | 00:00 | SKIP | + | networking-bgpvpn | functest | smoke | 00:00 | SKIP | + | networking-sfc | functest | smoke | 00:00 | SKIP | + | barbican | functest | smoke | 05:01 | PASS | + +------------------------------------+------------------+---------------+------------------+----------------+ + +Note: if the scenario does not support some tests, they are indicated as SKIP. +See User guide for details. + +Testing benchmarking suite +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Run benchmarking suite:: + + sudo docker run --env-file env \ + -v $(pwd)/openstack.creds:/home/opnfv/functest/conf/env_file \ + -v $(pwd)/images:/home/opnfv/functest/images \ + opnfv/functest-benchmarking + +Results shall be displayed as follows:: + + +-------------------+------------------+----------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------+------------------+----------------------+------------------+----------------+ + | vmtp | functest | benchmarking | 18:43 | PASS | + | shaker | functest | benchmarking | 29:45 | PASS | + +-------------------+------------------+----------------------+------------------+----------------+ + +Note: if the scenario does not support some tests, they are indicated as SKIP. +See User guide for details. Testing features suite ^^^^^^^^^^^^^^^^^^^^^^ @@ -140,9 +167,12 @@ Results shall be displayed as follows:: | functest-odl-sfc | sfc | features | 00:00 | SKIP | | barometercollectd | barometer | features | 00:00 | SKIP | | fds | fastdatastacks | features | 00:00 | SKIP | + | vgpu | functest | features | 00:00 | SKIP | + | stor4nfv_os | stor4nfv | features | 00:00 | SKIP | +-----------------------------+------------------------+------------------+------------------+----------------+ - Note: if the scenario does not support some tests, they are indicated as SKIP. - See User guide for details. + +Note: if the scenario does not support some tests, they are indicated as SKIP. +See User guide for details. Testing components suite ^^^^^^^^^^^^^^^^^^^^^^^^ @@ -156,12 +186,13 @@ Run components suite:: Results shall be displayed as follows:: - +-------------------------------+------------------+--------------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +-------------------------------+------------------+--------------------+------------------+----------------+ - | tempest_full_parallel | functest | components | 48:28 | PASS | - | rally_full | functest | components | 126:02 | PASS | - +-------------------------------+------------------+--------------------+------------------+----------------+ + +--------------------------+------------------+--------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +--------------------------+------------------+--------------------+------------------+----------------+ + | tempest_full | functest | components | 49:51 | PASS | + | tempest_scenario | functest | components | 18:50 | PASS | + | rally_full | functest | components | 167:13 | PASS | + +--------------------------+------------------+--------------------+------------------+----------------+ Testing vnf suite ^^^^^^^^^^^^^^^^^ @@ -178,12 +209,13 @@ Results shall be displayed as follows:: +----------------------+------------------+--------------+------------------+----------------+ | TEST CASE | PROJECT | TIER | DURATION | RESULT | +----------------------+------------------+--------------+------------------+----------------+ - | cloudify_ims | functest | vnf | 28:15 | PASS | - | vyos_vrouter | functest | vnf | 17:59 | PASS | - | juju_epc | functest | vnf | 46:44 | PASS | + | cloudify | functest | vnf | 04:05 | PASS | + | cloudify_ims | functest | vnf | 24:07 | PASS | + | heat_ims | functest | vnf | 18:15 | PASS | + | vyos_vrouter | functest | vnf | 15:48 | PASS | + | juju_epc | functest | vnf | 29:38 | PASS | +----------------------+------------------+--------------+------------------+----------------+ - Functest Dockers for Kubernetes deployment ------------------------------------------ Docker images are available on the dockerhub: @@ -198,14 +230,7 @@ Preparing your environment cat env:: - INSTALLER_TYPE=XXX - DEPLOY_SCENARIO=XXX - -cat k8s.creds:: - - export KUBERNETES_PROVIDER=local - export KUBE_MASTER_URL=XXX - export KUBE_MASTER_IP=XXX + DEPLOY_SCENARIO=k8s-XXX Testing healthcheck suite ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -213,7 +238,6 @@ Testing healthcheck suite Run healthcheck suite:: sudo docker run -it --env-file env \ - -v $(pwd)/k8s.creds:/home/opnfv/functest/conf/env_file \ -v $(pwd)/config:/root/.kube/config \ opnfv/functest-kubernetes-healthcheck @@ -222,11 +246,11 @@ volume mapped to ~/.kube/config inside kubernetes container. Results shall be displayed as follows:: - +-------------------------+------------------+-----------------+------------------+----------------+ - | TEST CASE | PROJECT | TIER | DURATION | RESULT | - +-------------------------+------------------+-----------------+------------------+----------------+ - | k8s_smoke | functest | healthcheck | 01:54 | PASS | - +-------------------------+------------------+-----------------+------------------+----------------+ + +-------------------+------------------+---------------------+------------------+----------------+ + | TEST CASE | PROJECT | TIER | DURATION | RESULT | + +-------------------+------------------+---------------------+------------------+----------------+ + | k8s_smoke | functest | healthcheck | 02:27 | PASS | + +-------------------+------------------+---------------------+------------------+----------------+ Testing smoke suite ^^^^^^^^^^^^^^^^^^^ @@ -234,7 +258,6 @@ Testing smoke suite Run smoke suite:: sudo docker run -it --env-file env \ - -v $(pwd)/k8s.creds:/home/opnfv/functest/conf/env_file \ -v $(pwd)/config:/root/.kube/config \ opnfv/functest-kubernetes-smoke @@ -243,7 +266,7 @@ Results shall be displayed as follows:: +-------------------------+------------------+---------------+------------------+----------------+ | TEST CASE | PROJECT | TIER | DURATION | RESULT | +-------------------------+------------------+---------------+------------------+----------------+ - | k8s_conformance | functest | smoke | 57:47 | PASS | + | k8s_conformance | functest | smoke | 57:14 | PASS | +-------------------------+------------------+---------------+------------------+----------------+ Testing features suite @@ -252,7 +275,6 @@ Testing features suite Run features suite:: sudo docker run -it --env-file env \ - -v $(pwd)/k8s.creds:/home/opnfv/functest/conf/env_file \ -v $(pwd)/config:/root/.kube/config \ opnfv/functest-kubernetes-features @@ -269,10 +291,13 @@ Environment variables ===================== Several environement variables may be specified: - * INSTALLER_TYPE=(apex|compass|daisy|fuel|joid) + * INSTALLER_IP=<Specific IP Address> * DEPLOY_SCENARIO=<vim>-<controller>-<nfv_feature>-<ha_mode> - + * NAMESERVER=XXX # if not 8.8.8.8 + * VOLUME_DEVICE_NAME=XXX # if not vdb + * EXTERNAL_NETWORK=XXX # if not first network with router:external=True + * NEW_USER_ROLE=XXX # if not member INSTALLER_IP is required by Barometer in order to access the installer node and the deployment. @@ -305,8 +330,9 @@ to the Functest Docker Container, using the -e only relevant to Jenkins CI invoked testing and **should not be used** when performing manual test scenarios: - * NODE_NAME = <Test POD Name> - * BUILD_TAG = <Jenkins Build Tag> + * INSTALLER_TYPE=(apex|compass|daisy|fuel) + * NODE_NAME=<Test POD Name> + * BUILD_TAG=<Jenkins Build Tag> where: diff --git a/functest/ci/add_proxy.sh b/functest/ci/add_proxy.sh new file mode 100644 index 000000000..3244f246a --- /dev/null +++ b/functest/ci/add_proxy.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -e + +pushd "${1:-/home/opnfv/functest/images}" > /dev/null + +images=${images-"\ +cloudify-manager-premium-4.0.1.qcow2 \ +ubuntu-14.04-server-cloudimg-amd64-disk1.img \ +ubuntu-16.04-server-cloudimg-amd64-disk1.img"} + +add_proxy () { + cat << EOF >> "$1" +http_proxy=${http_proxy:-http://proxy:8080} +HTTP_PROXY=${http_proxy:-http://proxy:8080} +https_proxy=${https_proxy:-${http_proxy:-http://proxy:8080}} +HTTPS_PROXY=${https_proxy:-${http_proxy:-http://proxy:8080}} +ftp_proxy=${ftp_proxy:-${http_proxy:-http://proxy:8080}} +FTP_PROXY=${ftp_proxy:-${http_proxy:-http://proxy:8080}} +no_proxy=${no_proxy:-"10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"} +NO_PROXY=${no_proxy:-"10.0.0.0/8,172.16.0.0/12,192.168.0.0/16"} +EOF +} + +add_proxy_apt () { + cat << EOF >> "$1" +Acquire::http::Proxy "${http_proxy:-http://proxy:8080}"; +Acquire::https::Proxy "${https_proxy:-http://proxy:8080}"; +EOF +} + +tmpdir=$(mktemp -d) +for image in $images; do + if [ ! -f "$image" ]; then + echo "skip ${image} ($(pwd)/${image} not found)" + continue + fi + guestmount -a "${image}" -i --rw "${tmpdir}" + add_proxy "${tmpdir}/etc/environment" + if [[ ${image} == *"cloudify-manager"* ]]; then + echo >> "${tmpdir}/etc/sysconfig/cloudify-mgmtworker" + add_proxy "${tmpdir}/etc/sysconfig/cloudify-mgmtworker" + echo >> "${tmpdir}/etc/sysconfig/cloudify-restservice" + add_proxy "${tmpdir}/etc/sysconfig/cloudify-restservice" + fi + if [[ ${image} == "ubuntu"* ]]; then + add_proxy_apt "${tmpdir}/etc/apt/apt.conf" + fi + guestunmount "${tmpdir}" +done + +rmdir "${tmpdir}" +popd > /dev/null diff --git a/functest/ci/config_functest.yaml b/functest/ci/config_functest.yaml index 7e51ee790..dbea320c9 100644 --- a/functest/ci/config_functest.yaml +++ b/functest/ci/config_functest.yaml @@ -11,8 +11,9 @@ general: functest_conf: /home/opnfv/functest/conf functest_data: /home/opnfv/functest/data ims_data: /home/opnfv/functest/data/ims/ + rally_data: /home/opnfv/functest/data/rally refstack_data: /home/opnfv/functest/data/refstack - router_data: /home/opnfv/functest/data/router/ + router_data: /home/opnfv/functest/data/router/opnfv-vnf-data functest_images: /home/opnfv/functest/images rally_inst: /root/.rally diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 2413287ac..16365b0b7 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -225,6 +225,17 @@ tiers: name: rally_sanity - + case_name: rally_jobs + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs a group of Rally jobs used in + OpenStack gating + run: + name: rally_jobs + + - case_name: refstack_defcore project_name: functest criteria: 100 @@ -254,6 +265,7 @@ tiers: - 'network' excludes: - 'test_networks_multiprovider_rbac' + - 'test_auto_allocated_topology_rbac' - case_name: snaps_smoke @@ -386,6 +398,7 @@ tiers: - case_name: doctor-notification project_name: doctor + enabled: false criteria: 100 blocking: false description: >- @@ -401,6 +414,7 @@ tiers: - case_name: bgpvpn project_name: sdnvpn + enabled: false criteria: 100 blocking: false description: >- @@ -413,6 +427,7 @@ tiers: - case_name: functest-odl-sfc project_name: sfc + enabled: false criteria: 100 blocking: false description: >- @@ -426,6 +441,7 @@ tiers: - case_name: barometercollectd project_name: barometer + enabled: false criteria: 100 blocking: false description: >- @@ -440,6 +456,7 @@ tiers: - case_name: fds project_name: fastdatastacks + enabled: false criteria: 100 blocking: false description: >- @@ -468,6 +485,7 @@ tiers: - case_name: stor4nfv_os project_name: stor4nfv + enabled: false criteria: 100 blocking: false description: >- diff --git a/functest/core/tenantnetwork.py b/functest/core/tenantnetwork.py index 9f5913c70..e03740767 100644 --- a/functest/core/tenantnetwork.py +++ b/functest/core/tenantnetwork.py @@ -106,6 +106,12 @@ class NewProject(object): self.orig_cloud.delete_project(self.project.id) if self.role: self.orig_cloud.delete_role(self.role.id) + secgroups = self.orig_cloud.list_security_groups( + filters={'name': 'default', + 'project_id': self.project.id}) + if secgroups: + sec_id = secgroups[0].id + self.orig_cloud.delete_security_group(sec_id) except Exception: # pylint: disable=broad-except self.__logger.exception("Cannot clean all resources") diff --git a/functest/opnfv_tests/openstack/rally/blacklist.txt b/functest/opnfv_tests/openstack/rally/blacklist.txt index 0900f21c1..1ade43eaf 100644 --- a/functest/opnfv_tests/openstack/rally/blacklist.txt +++ b/functest/opnfv_tests/openstack/rally/blacklist.txt @@ -17,3 +17,8 @@ functionality: - NovaServers.boot_server_attach_created_volume_and_live_migrate - NovaServers.boot_server_from_volume_and_live_migrate - NovaServers.boot_and_migrate_server + - + functions: + - no_net_trunk_service + tests: + - '^NeutronTrunk' diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index a91059a2b..ac44de118 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -16,11 +16,14 @@ import json import logging import os import re +import shutil import subprocess import time import pkg_resources import prettytable +from ruamel.yaml import YAML +from six.moves import configparser from xtesting.core import testcase from xtesting.energy import energy import yaml @@ -52,7 +55,6 @@ class RallyBase(singlevm.VmReady2): TENANTS_AMOUNT = 3 ITERATIONS_AMOUNT = 10 CONCURRENCY = 4 - RESULTS_DIR = os.path.join(getattr(config.CONF, 'dir_results'), 'rally') BLACKLIST_FILE = os.path.join(RALLY_DIR, "blacklist.txt") TEMP_DIR = os.path.join(RALLY_DIR, "var") @@ -74,6 +76,9 @@ class RallyBase(singlevm.VmReady2): role_name, user=self.project.user.id, project=self.project.project.id, domain=self.project.domain.id) + self.results_dir = os.path.join( + getattr(config.CONF, 'dir_results'), self.case_name) + self.task_file = os.path.join(self.RALLY_DIR, 'task.yaml') self.creators = [] self.summary = [] self.scenario_dir = '' @@ -85,8 +90,8 @@ class RallyBase(singlevm.VmReady2): self.compute_cnt = 0 self.flavor_alt = None self.tests = [] - self.task_file = '' self.run_cmd = '' + self.network_extensions = [] def _build_task_args(self, test_file_name): """Build arguments for the Rally task.""" @@ -137,10 +142,34 @@ class RallyBase(singlevm.VmReady2): if not os.path.exists(self.TEMP_DIR): os.makedirs(self.TEMP_DIR) - self._apply_blacklist(scenario_file_name, test_file_name) + self.apply_blacklist(scenario_file_name, test_file_name) return test_file_name @staticmethod + def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'): + """Set keystone_default_role in rally.conf""" + if env.get("NEW_USER_ROLE").lower() != "member": + rconfig = configparser.RawConfigParser() + rconfig.read(rally_conf) + if not rconfig.has_section('openstack'): + rconfig.add_section('openstack') + rconfig.set( + 'openstack', 'keystone_default_role', env.get("NEW_USER_ROLE")) + with open(rally_conf, 'wb') as config_file: + rconfig.write(config_file) + + @staticmethod + def clean_rally_conf(rally_conf='/etc/rally/rally.conf'): + """Clean Rally config""" + if env.get("NEW_USER_ROLE").lower() != "member": + rconfig = configparser.RawConfigParser() + rconfig.read(rally_conf) + if rconfig.has_option('openstack', 'keystone_default_role'): + rconfig.remove_option('openstack', 'keystone_default_role') + with open(rally_conf, 'wb') as config_file: + rconfig.write(config_file) + + @staticmethod def get_task_id(cmd_raw): """ Get task id from command rally result. @@ -179,7 +208,12 @@ class RallyBase(singlevm.VmReady2): """Determine if migration is supported.""" if self.compute_cnt > 1: return True + return False + def _network_trunk_supported(self): + """Determine if network trunk service is available""" + if 'trunk' in self.network_extensions: + return True return False @staticmethod @@ -236,6 +270,8 @@ class RallyBase(singlevm.VmReady2): if not self._migration_supported(): func_list.append("no_migration") + if not self._network_trunk_supported(): + func_list.append("no_net_trunk_service") if 'functionality' in black_list_yaml.keys(): for item in black_list_yaml['functionality']: @@ -249,7 +285,7 @@ class RallyBase(singlevm.VmReady2): return black_tests - def _apply_blacklist(self, case_file_name, result_file_name): + def apply_blacklist(self, case_file_name, result_file_name): """Apply blacklist.""" LOGGER.debug("Applying blacklist...") cases_file = open(case_file_name, 'r') @@ -292,10 +328,10 @@ class RallyBase(singlevm.VmReady2): def _save_results(self, test_name, task_id): """ Generate and save task execution results""" # check for result directory and create it otherwise - if not os.path.exists(self.RESULTS_DIR): + if not os.path.exists(self.results_dir): LOGGER.debug('%s does not exist, we create it.', - self.RESULTS_DIR) - os.makedirs(self.RESULTS_DIR) + self.results_dir) + os.makedirs(self.results_dir) # put detailed result to log cmd = (["rally", "task", "detailed", "--uuid", task_id]) @@ -304,8 +340,8 @@ class RallyBase(singlevm.VmReady2): LOGGER.info("%s\n%s", " ".join(cmd), output) # save report as JSON - report_json_name = 'opnfv-{}.json'.format(test_name) - report_json_dir = os.path.join(self.RESULTS_DIR, report_json_name) + report_json_name = '{}.json'.format(test_name) + report_json_dir = os.path.join(self.results_dir, report_json_name) cmd = (["rally", "task", "report", "--json", "--uuid", task_id, "--out", report_json_dir]) LOGGER.debug('running command: %s', cmd) @@ -313,8 +349,8 @@ class RallyBase(singlevm.VmReady2): LOGGER.info("%s\n%s", " ".join(cmd), output) # save report as HTML - report_html_name = 'opnfv-{}.html'.format(test_name) - report_html_dir = os.path.join(self.RESULTS_DIR, report_html_name) + report_html_name = '{}.html'.format(test_name) + report_html_dir = os.path.join(self.results_dir, report_html_name) cmd = (["rally", "task", "report", "--html", "--uuid", task_id, "--out", report_html_dir]) LOGGER.debug('running command: %s', cmd) @@ -384,13 +420,14 @@ class RallyBase(singlevm.VmReady2): else: raise Exception("Test name '%s' is invalid" % self.test_name) - self.task_file = os.path.join(self.RALLY_DIR, 'task.yaml') if not os.path.exists(self.task_file): LOGGER.error("Task file '%s' does not exist.", self.task_file) raise Exception("Task file '{}' does not exist.". format(self.task_file)) + self.update_keystone_default_role() self.compute_cnt = len(self.cloud.list_hypervisors()) + self.network_extensions = self.cloud.get_network_extensions() self.flavor_alt = self.create_flavor_alt() LOGGER.debug("flavor: %s", self.flavor_alt) @@ -438,7 +475,7 @@ class RallyBase(singlevm.VmReady2): except ZeroDivisionError: success_avg = 0 success_str = str("{:0.2f}".format(success_avg)) + '%' - duration_str = time.strftime("%M:%S", + duration_str = time.strftime("%H:%M:%S", time.gmtime(item['overall_duration'])) res_table.add_row([item['test_name'], duration_str, item['nb_tests'], success_str]) @@ -470,6 +507,7 @@ class RallyBase(singlevm.VmReady2): def clean(self): """Cleanup of OpenStack resources. Should be called on completion.""" + self.clean_rally_conf() if self.flavor_alt: self.orig_cloud.delete_flavor(self.flavor_alt.id) super(RallyBase, self).clean() @@ -495,6 +533,11 @@ class RallyBase(singlevm.VmReady2): OS_PROJECT_NAME=self.project.project.name, OS_PROJECT_ID=self.project.project.id, OS_PASSWORD=self.project.password) + try: + del environ['OS_TENANT_NAME'] + del environ['OS_TENANT_ID'] + except Exception: # pylint: disable=broad-except + pass conf_utils.create_rally_deployment(environ=environ) self.prepare_run() self.run_tests() @@ -532,3 +575,109 @@ class RallyFull(RallyBase): self.test_name = 'all' self.smoke = False self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full') + + +class RallyJobs(RallyBase): + """Rally OpenStack CI testcase implementation.""" + + TESTS = ["neutron"] + + def __init__(self, **kwargs): + """Initialize RallyJobs object.""" + if "case_name" not in kwargs: + kwargs["case_name"] = "rally_jobs" + super(RallyJobs, self).__init__(**kwargs) + self.test_name = 'all' + self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml') + self.task_yaml = None + + def prepare_run(self): + """Create resources needed by test scenarios.""" + super(RallyJobs, self).prepare_run() + with open(os.path.join(self.RALLY_DIR, + 'rally_jobs.yaml'), 'r') as task_file: + self.task_yaml = yaml.safe_load(task_file) + + if not all(task in self.task_yaml for task in self.tests): + raise Exception("Test '%s' not in '%s'" % + (self.test_name, self.tests)) + + def apply_blacklist(self, case_file_name, result_file_name): + # pylint: disable=too-many-branches + """Apply blacklist.""" + LOGGER.debug("Applying blacklist...") + black_tests = list(set(self.excl_func() + + self.excl_scenario())) + if black_tests: + LOGGER.debug("Blacklisted tests: %s", str(black_tests)) + + template = YAML(typ='jinja2') + with open(case_file_name, 'r') as fname: + cases = template.load(fname) + if cases.get("version", 1) == 1: + # scenarios in dictionary + for name in cases.keys(): + if self.in_iterable_re(name, black_tests): + cases.pop(name) + else: + # workloads in subtasks + for sind, subtask in enumerate(cases.get('subtasks', [])): + idx = [] + for wind, workload in enumerate(subtask.get('workloads', [])): + scenario = workload.get('scenario', {}) + for name in scenario.keys(): + if self.in_iterable_re(name, black_tests): + idx.append(wind) + break + for wind in reversed(idx): + cases['subtasks'][sind]['workloads'].pop(wind) + # scenarios in subtasks + idx = [] + for sind, subtask in enumerate(cases.get('subtasks', [])): + scenario = subtask.get('scenario', {}) + for name in scenario.keys(): + if self.in_iterable_re(name, black_tests): + idx.append(sind) + break + for sind in reversed(idx): + cases['subtasks'].pop(sind) + + with open(result_file_name, 'w') as fname: + template.dump(cases, fname) + + @staticmethod + def _remove_plugins_extra(): + inst_dir = getattr(config.CONF, 'dir_rally_inst') + try: + shutil.rmtree(os.path.join(inst_dir, 'plugins')) + shutil.rmtree(os.path.join(inst_dir, 'extra')) + except Exception: # pylint: disable=broad-except + pass + + def prepare_task(self, test_name): + """Prepare resources for test run.""" + self._remove_plugins_extra() + jobs_dir = os.path.join( + getattr(config.CONF, 'dir_rally_data'), test_name, 'rally-jobs') + inst_dir = getattr(config.CONF, 'dir_rally_inst') + shutil.copytree(os.path.join(jobs_dir, 'plugins'), + os.path.join(inst_dir, 'plugins')) + shutil.copytree(os.path.join(jobs_dir, 'extra'), + os.path.join(inst_dir, 'extra')) + + task_name = self.task_yaml.get(test_name).get("task") + task = os.path.join(jobs_dir, task_name) + if not os.path.exists(task): + raise Exception("The scenario '%s' does not exist." % task) + LOGGER.debug('Scenario fetched from : %s', task) + + if not os.path.exists(self.TEMP_DIR): + os.makedirs(self.TEMP_DIR) + task_file_name = os.path.join(self.TEMP_DIR, task_name) + self.apply_blacklist(task, task_file_name) + self.run_cmd = (["rally", "task", "start", "--task", task_file_name]) + return True + + def clean(self): + self._remove_plugins_extra() + super(RallyJobs, self).clean() diff --git a/functest/opnfv_tests/openstack/rally/rally_jobs.yaml b/functest/opnfv_tests/openstack/rally/rally_jobs.yaml new file mode 100644 index 000000000..2092fb4cf --- /dev/null +++ b/functest/opnfv_tests/openstack/rally/rally_jobs.yaml @@ -0,0 +1,3 @@ +--- +neutron: + task: task-neutron.yaml diff --git a/functest/opnfv_tests/openstack/rally/scenario/full/opnfv-nova.yaml b/functest/opnfv_tests/openstack/rally/scenario/full/opnfv-nova.yaml index 512448fd4..0e65ac15f 100644 --- a/functest/opnfv_tests/openstack/rally/scenario/full/opnfv-nova.yaml +++ b/functest/opnfv_tests/openstack/rally/scenario/full/opnfv-nova.yaml @@ -381,6 +381,40 @@ sla: {{ no_failures_sla() }} + NovaServers.boot_server_associate_and_dissociate_floating_ip: + - + args: + {{ vm_params(image_name, flavor_name) }} + create_floating_ip_args: + ext_network: {{ floating_network }} + nics: + - net-id: {{ netid }} + context: + {% call user_context(tenants_amount, users_amount, use_existing_users) %} + network: {} + {% endcall %} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + + NovaServers.boot_and_associate_floating_ip: + - + args: + {{ vm_params(image_name, flavor_name) }} + create_floating_ip_args: + ext_network: {{ floating_network }} + nics: + - net-id: {{ netid }} + context: + {% call user_context(tenants_amount, users_amount, use_existing_users) %} + network: {} + {% endcall %} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + NovaServerGroups.create_and_delete_server_group: - args: diff --git a/functest/opnfv_tests/openstack/rally/scenario/sanity/opnfv-nova.yaml b/functest/opnfv_tests/openstack/rally/scenario/sanity/opnfv-nova.yaml index 801938c4e..7dd252da5 100644 --- a/functest/opnfv_tests/openstack/rally/scenario/sanity/opnfv-nova.yaml +++ b/functest/opnfv_tests/openstack/rally/scenario/sanity/opnfv-nova.yaml @@ -132,6 +132,23 @@ sla: {{ no_failures_sla() }} + NovaServers.boot_server_associate_and_dissociate_floating_ip: + - + args: + {{ vm_params(image_name, flavor_name) }} + create_floating_ip_args: + ext_network: {{ floating_network }} + nics: + - net-id: {{ netid }} + context: + {% call user_context(tenants_amount, users_amount, use_existing_users) %} + network: {} + {% endcall %} + runner: + {{ constant_runner(concurrency=concurrency, times=iterations, is_smoke=smoke) }} + sla: + {{ no_failures_sla() }} + NovaServerGroups.create_and_delete_server_group: - args: diff --git a/functest/opnfv_tests/openstack/refstack/refstack.py b/functest/opnfv_tests/openstack/refstack/refstack.py index 22e09bc82..c709e5626 100644 --- a/functest/opnfv_tests/openstack/refstack/refstack.py +++ b/functest/opnfv_tests/openstack/refstack/refstack.py @@ -40,15 +40,18 @@ class Refstack(tempest.TempestCommon): return yaml.load(yaml_data) def _extract_tempest_data(self): + olddir = os.getcwd() try: - cmd = ['stestr', '--here', self.verifier_repo_dir, 'list', - '^tempest.'] + os.chdir(self.verifier_repo_dir) + cmd = ['stestr', 'list', '^tempest.'] output = subprocess.check_output(cmd) except subprocess.CalledProcessError as cpe: self.__logger.error( "Exception when listing tempest tests: %s\n%s", cpe.cmd, cpe.output) raise + finally: + os.chdir(olddir) yaml_data2 = "" for line in output.splitlines(): try: diff --git a/functest/opnfv_tests/openstack/tempest/conf_utils.py b/functest/opnfv_tests/openstack/tempest/conf_utils.py index 80f78e979..fc36a72ed 100644 --- a/functest/opnfv_tests/openstack/tempest/conf_utils.py +++ b/functest/opnfv_tests/openstack/tempest/conf_utils.py @@ -81,6 +81,7 @@ def create_rally_deployment(environ=None): cmd = ['rally', 'deployment', 'check'] output = subprocess.check_output(cmd) LOGGER.info("%s\n%s", " ".join(cmd), output) + return get_verifier_deployment_id() def create_verifier(): @@ -101,6 +102,7 @@ def create_verifier(): '--type', 'tempest', '--system-wide'] output = subprocess.check_output(cmd) LOGGER.info("%s\n%s", " ".join(cmd), output) + return get_verifier_id() def get_verifier_id(): @@ -114,9 +116,6 @@ def get_verifier_id(): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) verifier_uuid = proc.stdout.readline().rstrip() - if verifier_uuid == "": - LOGGER.error("Tempest verifier not found.") - raise Exception('Error with command:%s' % cmd) return verifier_uuid @@ -131,9 +130,6 @@ def get_verifier_deployment_id(): stdout=subprocess.PIPE, stderr=subprocess.STDOUT) deployment_uuid = proc.stdout.readline().rstrip() - if deployment_uuid == "": - LOGGER.error("Rally deployment not found.") - raise Exception('Error with command:%s' % cmd) return deployment_uuid @@ -175,17 +171,17 @@ def update_tempest_conf_file(conf_file, rconfig): def configure_tempest_update_params( - tempest_conf_file, network_name=None, image_id=None, flavor_id=None, + tempest_conf_file, image_id=None, flavor_id=None, compute_cnt=1, image_alt_id=None, flavor_alt_id=None, - domain_name="Default"): - # pylint: disable=too-many-branches, too-many-arguments + admin_role_name='admin', cidr='192.168.120.0/24', + domain_id='default'): + # pylint: disable=too-many-branches,too-many-arguments,too-many-statements """ Add/update needed parameters into tempest.conf file """ LOGGER.debug("Updating selected tempest.conf parameters...") rconfig = configparser.RawConfigParser() rconfig.read(tempest_conf_file) - rconfig.set('compute', 'fixed_network_name', network_name) rconfig.set('compute', 'volume_device_name', env.get('VOLUME_DEVICE_NAME')) if image_id is not None: rconfig.set('compute', 'image_ref', image_id) @@ -200,16 +196,14 @@ def configure_tempest_update_params( rconfig.set('compute', 'min_compute_nodes', compute_cnt) rconfig.set('compute-feature-enabled', 'live_migration', True) rconfig.set('compute-feature-enabled', 'shelve', False) + filters = ['RetryFilter', 'AvailabilityZoneFilter', 'ComputeFilter', + 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', + 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter'] + rconfig.set( + 'compute-feature-enabled', 'scheduler_available_filters', + convert_list_to_ini(filters)) if os.environ.get('OS_REGION_NAME'): rconfig.set('identity', 'region', os.environ.get('OS_REGION_NAME')) - identity_api_version = os.environ.get("OS_IDENTITY_API_VERSION", '3') - rconfig.set('auth', 'admin_domain_scope', True) - rconfig.set('auth', 'default_credentials_domain_name', domain_name) - if identity_api_version == '3': - auth_version = 'v3' - rconfig.set('identity-feature-enabled', 'api_v2', False) - else: - auth_version = 'v2' if env.get("NEW_USER_ROLE").lower() != "member": rconfig.set( 'auth', 'tempest_roles', @@ -221,13 +215,25 @@ def configure_tempest_update_params( assert os.path.exists( account_file), "{} doesn't exist".format(account_file) rconfig.set('auth', 'test_accounts_file', account_file) - rconfig.set('identity', 'auth_version', auth_version) + rconfig.set('identity', 'auth_version', 'v3') + rconfig.set('identity', 'admin_role', admin_role_name) + rconfig.set('identity', 'admin_domain_scope', True) + rconfig.set('identity', 'default_domain_id', domain_id) + rconfig.set('identity-feature-enabled', 'api_v2', False) + rconfig.set('identity-feature-enabled', 'api_v2_admin', False) + if not rconfig.has_section('network'): + rconfig.add_section('network') + rconfig.set('network', 'default_network', cidr) + rconfig.set('network', 'project_network_cidr', cidr) + rconfig.set('network', 'project_networks_reachable', False) rconfig.set( 'validation', 'ssh_timeout', getattr(config.CONF, 'tempest_validation_ssh_timeout')) rconfig.set('object-storage', 'operator_role', getattr(config.CONF, 'tempest_object_storage_operator_role')) - + if not rconfig.has_section('volume'): + rconfig.add_section('volume') + rconfig.set('volume', 'storage_protocol', env.get('STORAGE_PROTOCOL')) rconfig.set( 'identity', 'v3_endpoint_type', os.environ.get('OS_INTERFACE', 'public')) @@ -261,16 +267,14 @@ def configure_verifier(deployment_dir): if not os.path.isfile(tempest_conf_file): LOGGER.error("Tempest configuration file %s NOT found.", tempest_conf_file) - raise Exception("Tempest configuration file %s NOT found." - % tempest_conf_file) - else: - return tempest_conf_file + return None + return tempest_conf_file def convert_dict_to_ini(value): "Convert dict to oslo.conf input" assert isinstance(value, dict) - return ",".join("{}={}".format( + return ",".join("{}:{}".format( key, val) for (key, val) in six.iteritems(value)) diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index ddd0d640d..4c5308bdf 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -36,7 +36,6 @@ class TempestCommon(singlevm.VmReady2): """TempestCommon testcases implementation class.""" visibility = 'public' - shared_network = True filename_alt = '/home/opnfv/functest/images/cirros-0.4.0-x86_64-disk.img' def __init__(self, **kwargs): @@ -47,27 +46,38 @@ class TempestCommon(singlevm.VmReady2): assert self.cloud assert self.project if self.orig_cloud.get_role("admin"): - role_name = "admin" + self.role_name = "admin" elif self.orig_cloud.get_role("Admin"): - role_name = "Admin" + self.role_name = "Admin" else: raise Exception("Cannot detect neither admin nor Admin") self.orig_cloud.grant_role( - role_name, user=self.project.user.id, + self.role_name, user=self.project.user.id, project=self.project.project.id, domain=self.project.domain.id) + self.orig_cloud.grant_role( + self.role_name, user=self.project.user.id, + domain=self.project.domain.id) environ = dict( os.environ, OS_USERNAME=self.project.user.name, OS_PROJECT_NAME=self.project.project.name, OS_PROJECT_ID=self.project.project.id, OS_PASSWORD=self.project.password) - conf_utils.create_rally_deployment(environ=environ) - conf_utils.create_verifier() - self.verifier_id = conf_utils.get_verifier_id() + try: + del environ['OS_TENANT_NAME'] + del environ['OS_TENANT_ID'] + except Exception: # pylint: disable=broad-except + pass + self.deployment_id = conf_utils.create_rally_deployment( + environ=environ) + if not self.deployment_id: + raise Exception("Deployment create failed") + self.verifier_id = conf_utils.create_verifier() + if not self.verifier_id: + raise Exception("Verifier create failed") self.verifier_repo_dir = conf_utils.get_verifier_repo_dir( self.verifier_id) - self.deployment_id = conf_utils.get_verifier_deployment_id() self.deployment_dir = conf_utils.get_verifier_deployment_dir( self.verifier_id, self.deployment_id) self.verification_id = None @@ -90,6 +100,9 @@ class TempestCommon(singlevm.VmReady2): except Exception: # pylint: disable=broad-except pass + def create_network_resources(self): + pass + def check_services(self): """Check the mandatory services.""" for service in self.services: @@ -112,6 +125,8 @@ class TempestCommon(singlevm.VmReady2): def check_requirements(self): self.check_services() self.check_extensions() + if self.is_skipped: + self.project.clean() @staticmethod def read_file(filename): @@ -394,14 +409,17 @@ class TempestCommon(singlevm.VmReady2): LOGGER.debug("flavor: %s", self.flavor_alt) self.conf_file = conf_utils.configure_verifier(self.deployment_dir) + if not self.conf_file: + raise Exception("Tempest verifier configuring failed") conf_utils.configure_tempest_update_params( - self.conf_file, network_name=self.network.name, + self.conf_file, image_id=self.image.id, flavor_id=self.flavor.id, compute_cnt=compute_cnt, image_alt_id=self.image_alt.id, flavor_alt_id=self.flavor_alt.id, - domain_name=self.cloud.auth.get("project_domain_name", "Default")) + admin_role_name=self.role_name, cidr=self.cidr, + domain_id=self.project.domain.id) self.update_scenario_section() self.backup_tempest_config(self.conf_file, self.res_dir) diff --git a/functest/opnfv_tests/vnf/epc/juju_epc.py b/functest/opnfv_tests/vnf/epc/juju_epc.py index fb3c587ca..644911bd7 100644 --- a/functest/opnfv_tests/vnf/epc/juju_epc.py +++ b/functest/opnfv_tests/vnf/epc/juju_epc.py @@ -230,7 +230,7 @@ class JujuEpc(singlevm.VmReady2): self.cloud.region_name if self.cloud.region_name else ( 'RegionOne')), 'abot-controller', - '--agent-version', '2.2.9', + '--agent-version', '2.3.9', '--metadata-source', '/root', '--constraints', 'mem=2G', '--bootstrap-series', 'xenial', diff --git a/functest/opnfv_tests/vnf/ims/cloudify_ims.py b/functest/opnfv_tests/vnf/ims/cloudify_ims.py index f7931b7b7..08699e4bb 100644 --- a/functest/opnfv_tests/vnf/ims/cloudify_ims.py +++ b/functest/opnfv_tests/vnf/ims/cloudify_ims.py @@ -169,8 +169,18 @@ class CloudifyIms(cloudify.Cloudify): """Deploy Clearwater IMS.""" start_time = time.time() + secgroups = self.cloud.list_security_groups( + filters={'name': 'default', + 'project_id': self.project.project.id}) + if secgroups: + secgroup = secgroups[0] + else: + self.__logger.error("No 'default' security group in project %s", + self.project.project.name) + return False + self.cloud.create_security_group_rule( - 'default', port_range_min=22, port_range_max=22, + secgroup.id, port_range_min=22, port_range_max=22, protocol='tcp', direction='ingress') self.__logger.info("Upload VNFD") diff --git a/functest/opnfv_tests/vnf/router/cloudify_vrouter.py b/functest/opnfv_tests/vnf/router/cloudify_vrouter.py index f9266ff17..e8c8632bd 100644 --- a/functest/opnfv_tests/vnf/router/cloudify_vrouter.py +++ b/functest/opnfv_tests/vnf/router/cloudify_vrouter.py @@ -167,11 +167,6 @@ class CloudifyVrouter(cloudify.Cloudify): def deploy_vnf(self): start_time = time.time() - - self.cloud.create_security_group_rule( - 'default', port_range_min=22, port_range_max=22, - protocol='tcp', direction='ingress') - self.__logger.info("Upload VNFD") descriptor = self.vnf['descriptor'] self.util_info["cfy"] = self.cfy_client diff --git a/functest/opnfv_tests/vnf/router/cloudify_vrouter.yaml b/functest/opnfv_tests/vnf/router/cloudify_vrouter.yaml index 649cd6ccd..2d98dffa5 100644 --- a/functest/opnfv_tests/vnf/router/cloudify_vrouter.yaml +++ b/functest/opnfv_tests/vnf/router/cloudify_vrouter.yaml @@ -3,9 +3,6 @@ tenant_images: cloudify_manager_4.0: /home/opnfv/functest/images/cloudify-manager-premium-4.0.1.qcow2 vyos1.1.7: /home/opnfv/functest/images/vyos-1.1.7.img -test_data: - url: 'https://github.com/oolorg/opnfv-vnf-data.git' - branch: 'fraser' orchestrator: name: cloudify version: '4.0' diff --git a/functest/opnfv_tests/vnf/router/test_controller/function_test_exec.py b/functest/opnfv_tests/vnf/router/test_controller/function_test_exec.py index 9d484cb56..0b8a69b73 100644 --- a/functest/opnfv_tests/vnf/router/test_controller/function_test_exec.py +++ b/functest/opnfv_tests/vnf/router/test_controller/function_test_exec.py @@ -12,6 +12,7 @@ """vrouter function test execution module""" import logging +import os import time import yaml @@ -31,11 +32,11 @@ class FunctionTestExec(object): credentials = util_info["credentials"] self.vnf_ctrl = VnfController(util_info) - test_cmd_map_file = open(self.util.vnf_data_dir + - self.util.opnfv_vnf_data_dir + - self.util.command_template_dir + - self.util.test_cmd_map_yaml_file, - 'r') + test_cmd_map_file = open( + os.path.join( + self.util.vnf_data_dir, self.util.command_template_dir, + self.util.test_cmd_map_yaml_file), + 'r') self.test_cmd_map_yaml = yaml.safe_load(test_cmd_map_file) test_cmd_map_file.close() diff --git a/functest/opnfv_tests/vnf/router/utilvnf.py b/functest/opnfv_tests/vnf/router/utilvnf.py index 4bb277a8d..a54f6cb0b 100644 --- a/functest/opnfv_tests/vnf/router/utilvnf.py +++ b/functest/opnfv_tests/vnf/router/utilvnf.py @@ -14,12 +14,9 @@ import json import logging import os -import pkg_resources import requests import yaml -from git import Repo - from functest.utils import config RESULT_SPRIT_INDEX = { @@ -53,14 +50,12 @@ class Utilvnf(object): # pylint: disable=too-many-instance-attributes def __init__(self): self.vnf_data_dir = getattr(config.CONF, 'dir_router_data') - self.opnfv_vnf_data_dir = "opnfv-vnf-data/" self.command_template_dir = "command_template/" self.test_scenario_yaml = "test_scenario.yaml" test_env_config_yaml_file = "test_env_config.yaml" self.test_cmd_map_yaml_file = "test_cmd_map.yaml" self.test_env_config_yaml = os.path.join( self.vnf_data_dir, - self.opnfv_vnf_data_dir, test_env_config_yaml_file) self.blueprint_dir = "opnfv-vnf-vyos-blueprint/" @@ -69,28 +64,6 @@ class Utilvnf(object): # pylint: disable=too-many-instance-attributes if not os.path.exists(self.vnf_data_dir): os.makedirs(self.vnf_data_dir) - case_dir = pkg_resources.resource_filename( - 'functest', 'opnfv_tests/vnf/router') - - config_file_name = getattr( - config.CONF, 'vnf_{}_config'.format("vyos_vrouter")) - - config_file = os.path.join(case_dir, config_file_name) - - with open(config_file) as file_fd: - vrouter_config_yaml = yaml.safe_load(file_fd) - file_fd.close() - - test_data = vrouter_config_yaml.get("test_data") - - self.logger.debug("Downloading the test data.") - vrouter_data_path = self.vnf_data_dir + self.opnfv_vnf_data_dir - - if not os.path.exists(vrouter_data_path): - Repo.clone_from(test_data['url'], - vrouter_data_path, - branch=test_data['branch']) - with open(self.test_env_config_yaml) as file_fd: test_env_config_yaml = yaml.safe_load(file_fd) file_fd.close() diff --git a/functest/opnfv_tests/vnf/router/vrouter_base.py b/functest/opnfv_tests/vnf/router/vrouter_base.py index fe3058f01..e8a933ff5 100644 --- a/functest/opnfv_tests/vnf/router/vrouter_base.py +++ b/functest/opnfv_tests/vnf/router/vrouter_base.py @@ -32,15 +32,12 @@ class VrouterOnBoardingBase(object): self.logger = logging.getLogger(__name__) self.case_dir = pkg_resources.resource_filename( 'functest', 'opnfv_tests/vnf/router') - self.data_dir = getattr(config.CONF, 'dir_router_data') self.result_dir = os.path.join( getattr(config.CONF, 'dir_results'), case_name) self.util = util self.util_info = util_info self.vnf_list = [] - if not os.path.exists(self.data_dir): - os.makedirs(self.data_dir) if not os.path.exists(self.result_dir): os.makedirs(self.result_dir) diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py index d89ebd634..22ddb1cc5 100644 --- a/functest/tests/unit/openstack/rally/test_rally.py +++ b/functest/tests/unit/openstack/rally/test_rally.py @@ -71,7 +71,7 @@ class OSRallyTesting(unittest.TestCase): @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists') @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs') @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' - '_apply_blacklist') + 'apply_blacklist') def test_prepare_test_list_missing_temp_dir( self, mock_method, mock_os_makedirs, mock_path_exists): mock_path_exists.side_effect = self.check_temp_dir @@ -153,10 +153,13 @@ class OSRallyTesting(unittest.TestCase): {'functions': ['no_migration'], 'tests': ['test']}]}) @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' '_migration_supported', return_value=False) - def test_excl_func_default(self, mock_func, mock_yaml_load): + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' + '_network_trunk_supported', return_value=False) + def test_excl_func_default(self, mock_trunk, mock_func, mock_yaml_load): os.environ['DEPLOY_SCENARIO'] = 'test_scenario' self.assertEqual(self.rally_base.excl_func(), ['test']) mock_func.assert_called() + mock_trunk.assert_called() mock_yaml_load.assert_called() @mock.patch('six.moves.builtins.open', side_effect=Exception) @@ -263,8 +266,7 @@ class OSRallyTesting(unittest.TestCase): with self.assertRaises(Exception): self.rally_base.prepare_run() - @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' - 'get_external_network') + @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists') def test_prepare_run_flavor_alt_creation_failed(self, *args): # pylint: disable=unused-argument self.rally_base.TESTS = ['test1', 'test2'] diff --git a/functest/tests/unit/openstack/tempest/test_conf_utils.py b/functest/tests/unit/openstack/tempest/test_conf_utils.py index 7c493372f..316f21be2 100644 --- a/functest/tests/unit/openstack/tempest/test_conf_utils.py +++ b/functest/tests/unit/openstack/tempest/test_conf_utils.py @@ -20,9 +20,12 @@ from functest.utils import config class OSTempestConfUtilsTesting(unittest.TestCase): # pylint: disable=too-many-public-methods + @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils' + '.get_verifier_deployment_id', return_value='foo') @mock.patch('subprocess.check_output') - def test_create_rally_deployment(self, mock_exec): - self.assertEqual(conf_utils.create_rally_deployment(), None) + def test_create_rally_deployment(self, mock_exec, mock_get_id): + # pylint: disable=unused-argument + self.assertEqual(conf_utils.create_rally_deployment(), 'foo') calls = [ mock.call(['rally', 'deployment', 'destroy', '--deployment', str(getattr(config.CONF, 'rally_deployment_name'))]), @@ -51,22 +54,6 @@ class OSTempestConfUtilsTesting(unittest.TestCase): 'create_verifier', return_value=mock.Mock()) @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' 'create_rally_deployment', return_value=mock.Mock()) - def test_get_verif_id_missing_verif(self, mock_rally, mock_tempest): - # pylint: disable=unused-argument - setattr(config.CONF, 'tempest_verifier_name', 'test_verifier_name') - with mock.patch('functest.opnfv_tests.openstack.tempest.' - 'conf_utils.subprocess.Popen') as mock_popen, \ - self.assertRaises(Exception): - mock_stdout = mock.Mock() - attrs = {'stdout.readline.return_value': ''} - mock_stdout.configure_mock(**attrs) - mock_popen.return_value = mock_stdout - conf_utils.get_verifier_id() - - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' - 'create_verifier', return_value=mock.Mock()) - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' - 'create_rally_deployment', return_value=mock.Mock()) def test_get_verifier_id_default(self, mock_rally, mock_tempest): # pylint: disable=unused-argument setattr(config.CONF, 'tempest_verifier_name', 'test_verifier_name') @@ -80,17 +67,6 @@ class OSTempestConfUtilsTesting(unittest.TestCase): self.assertEqual(conf_utils.get_verifier_id(), 'test_deploy_id') - def test_get_depl_id_missing_rally(self): - setattr(config.CONF, 'tempest_verifier_name', 'test_deploy_name') - with mock.patch('functest.opnfv_tests.openstack.tempest.' - 'conf_utils.subprocess.Popen') as mock_popen, \ - self.assertRaises(Exception): - mock_stdout = mock.Mock() - attrs = {'stdout.readline.return_value': ''} - mock_stdout.configure_mock(**attrs) - mock_popen.return_value = mock_stdout - conf_utils.get_verifier_deployment_id() - def test_get_depl_id_default(self): setattr(config.CONF, 'tempest_verifier_name', 'test_deploy_name') with mock.patch('functest.opnfv_tests.openstack.tempest.' @@ -186,9 +162,9 @@ class OSTempestConfUtilsTesting(unittest.TestCase): self.assertEqual( conf_utils.convert_dict_to_ini({}), "") self.assertEqual( - conf_utils.convert_dict_to_ini({"a": "b"}), "a=b") + conf_utils.convert_dict_to_ini({"a": "b"}), "a:b") value = conf_utils.convert_dict_to_ini({"a": "b", "c": "d"}) - self.assertTrue(value == "a=b,c=d" or value == "c=d,a=b") + self.assertTrue(value == "a:b,c:d" or value == "c:d,a:b") with self.assertRaises(AssertionError): conf_utils.convert_list_to_ini("") diff --git a/functest/tests/unit/utils/test_functest_utils.py b/functest/tests/unit/utils/test_functest_utils.py index 2b98267c4..ab90de595 100644 --- a/functest/tests/unit/utils/test_functest_utils.py +++ b/functest/tests/unit/utils/test_functest_utils.py @@ -261,7 +261,7 @@ class FunctestUtilsTesting(unittest.TestCase): def test_openstack_version1(self, *args): cloud = mock.Mock() self.assertEqual(functest_utils.get_openstack_version( - cloud), "Rocky or newer") + cloud), "Rocky") args[0].assert_called_once_with(cloud) @mock.patch('functest.utils.functest_utils.get_nova_version', @@ -327,7 +327,7 @@ class FunctestUtilsTesting(unittest.TestCase): def test_openstack_version10(self, *args): cloud = mock.Mock() self.assertEqual( - functest_utils.get_openstack_version(cloud), "Rocky or newer") + functest_utils.get_openstack_version(cloud), "Master") args[0].assert_called_once_with(cloud) @mock.patch('functest.utils.functest_utils.get_nova_version', diff --git a/functest/utils/env.py b/functest/utils/env.py index 87940ce11..26efbb196 100644 --- a/functest/utils/env.py +++ b/functest/utils/env.py @@ -35,7 +35,8 @@ INPUTS = { 'IMG_PROP': '', 'NAMESERVER': '8.8.8.8', 'NEW_USER_ROLE': 'Member', - 'USE_DYNAMIC_CREDENTIALS': 'True' + 'USE_DYNAMIC_CREDENTIALS': 'True', + 'STORAGE_PROTOCOL': 'iSCSI' } diff --git a/functest/utils/functest_utils.py b/functest/utils/functest_utils.py index 1993f1e83..dd2bf1932 100644 --- a/functest/utils/functest_utils.py +++ b/functest/utils/functest_utils.py @@ -109,8 +109,10 @@ def get_openstack_version(cloud): version = get_nova_version(cloud) try: assert version - if version > (2, 60): - osversion = "Rocky or newer" + if version > (2, 65): + osversion = "Master" + elif version > (2, 60): + osversion = "Rocky" elif version > (2, 53): osversion = "Queens" elif version > (2, 42): diff --git a/requirements.txt b/requirements.txt index 3b82370b4..298cc38ad 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,19 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -pbr!=2.1.0,>=2.0.0 # Apache-2.0 -PyYAML>=3.12 # MIT -GitPython>=1.0.1 # BSD License (3 clause) -requests>=2.14.2 # Apache-2.0 +pbr!=2.1.0 # Apache-2.0 +PyYAML # MIT +requests # Apache-2.0 robotframework>=3.0 scp cloudify-rest-client -mock>=2.0.0 # BSD -PrettyTable<0.8,>=0.7.1 # BSD -six>=1.10.0 # MIT +mock # BSD +PrettyTable<0.8 # BSD +six # MIT snaps -paramiko>=2.0.0 # LGPLv2.1+ -Jinja2>=2.10 # BSD License (3 clause) +paramiko # LGPLv2.1+ +Jinja2 # BSD License (3 clause) xtesting -os-client-config>=1.28.0 # Apache-2.0 -shade>=1.17.0 # Apache-2.0 +os-client-config # Apache-2.0 +shade # Apache-2.0 +ruamel.yaml.jinja2 # MIT @@ -6,6 +6,7 @@ home-page = https://wiki.opnfv.org/display/functest [files] packages = functest scripts = + functest/ci/add_proxy.sh functest/ci/convert_images.sh functest/ci/download_images.sh @@ -42,6 +43,7 @@ xtesting.testcase = heat_ims = functest.opnfv_tests.vnf.ims.heat_ims:HeatIms vyos_vrouter = functest.opnfv_tests.vnf.router.cloudify_vrouter:CloudifyVrouter juju_epc = functest.opnfv_tests.vnf.epc.juju_epc:JujuEpc + rally_jobs = functest.opnfv_tests.openstack.rally.rally:RallyJobs [build_sphinx] all_files = 1 diff --git a/test-requirements.txt b/test-requirements.txt index ae4cfac37..b855f2f06 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,14 +1,14 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -coverage!=4.4,>=4.0 # Apache-2.0 -mock>=2.0.0 # BSD -nose>=1.3.7 # LGPL +coverage!=4.4 # Apache-2.0 +mock # BSD +nose # LGPL flake8<2.6.0,>=2.5.4 # MIT pylint # GPLv2 -sphinx!=1.6.6,!=1.6.7,>=1.6.2 # BSD +sphinx!=1.6.6,!=1.6.7 # BSD sphinx-rtd-theme yamllint -doc8>=0.6.0 # Apache-2.0 -bashate>=0.5.1 # Apache-2.0 -nosehtmloutput # Apache-2.0 +doc8 # Apache-2.0 +bashate # Apache-2.0 +nosehtmloutput # Apache-2.0 @@ -5,7 +5,7 @@ envlist = docs,pep8,pylint,yamllint,bashate,py35,py27,perm,cover usedevelop = True deps = -c{toxinidir}/upper-constraints.txt - -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/queens} + -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=master} -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt install_command = pip install {opts} {packages} @@ -93,6 +93,7 @@ files = ci docker functest/ci + functest/opnfv_tests/openstack/rally/rally_jobs.yaml functest/opnfv_tests/vnf commands = yamllint -s {[testenv:yamllint]files} @@ -111,6 +112,7 @@ basepython = python2.7 files = functest/opnfv_tests/openstack/cinder/write_data.sh functest/opnfv_tests/openstack/cinder/read_data.sh + functest/ci/add_proxy.sh functest/ci/convert_images.sh functest/ci/download_images.sh build.sh diff --git a/upper-constraints.txt b/upper-constraints.txt index 8ac4fd069..16200d0b7 100644 --- a/upper-constraints.txt +++ b/upper-constraints.txt @@ -1,6 +1,6 @@ git+https://gerrit.opnfv.org/gerrit/functest#egg=functest git+https://gerrit.opnfv.org/gerrit/releng#egg=opnfv&subdirectory=modules -git+https://gerrit.opnfv.org/gerrit/snaps@d869e27230bf390c8f4fe68f5682584d4b9dbc49#egg=snaps +git+https://gerrit.opnfv.org/gerrit/snaps@0dacfaa2fbd2dfe7fc9d438b9350a0187506e61c#egg=snaps git+https://gerrit.opnfv.org/gerrit/barometer#egg=baro_tests git+https://gerrit.opnfv.org/gerrit/sdnvpn#egg=sdnvpn git+https://gerrit.opnfv.org/gerrit/sfc#egg=sfc @@ -17,5 +17,5 @@ robotframework-requests===0.4.7 robotframework-sshlibrary===2.1.3;python_version=='2.7' ansible===2.3.2.0 xtesting===0.60.0 -networking-bgpvpn===8.0.0 -networking-sfc===6.0.0 +git+https://git.openstack.org/openstack/networking-bgpvpn#egg=networking_bgpvpn +git+https://git.openstack.org/openstack/networking-sfc#egg=networking_sfc |