diff options
| -rw-r--r-- | .travis.yml | 5 | ||||
| -rw-r--r-- | build.sh | 1 | ||||
| -rw-r--r-- | ci/daily.yaml | 3 | ||||
| -rw-r--r-- | ci/gate.yaml | 4 | ||||
| -rw-r--r-- | ci/run.yaml | 28 | ||||
| -rw-r--r-- | docker/benchmarking/Dockerfile | 24 | ||||
| -rw-r--r-- | docker/benchmarking/hooks/post_checkout | 6 | ||||
| -rw-r--r-- | docker/benchmarking/testcases.yaml | 35 | ||||
| -rw-r--r-- | docker/components/testcases.yaml | 11 | ||||
| -rw-r--r-- | docker/core/Dockerfile | 4 | ||||
| -rw-r--r-- | docker/features/hooks/post_checkout | 2 | ||||
| -rw-r--r-- | docker/features/testcases.yaml | 23 | ||||
| -rw-r--r-- | docker/healthcheck/testcases.yaml | 41 | ||||
| -rw-r--r-- | docker/smoke/Dockerfile | 17 | ||||
| -rw-r--r-- | docker/smoke/hooks/post_checkout | 2 | ||||
| -rw-r--r-- | docker/smoke/testcases.yaml | 39 | ||||
| -rw-r--r-- | docker/tempest/Dockerfile | 2 | ||||
| -rw-r--r-- | docker/vnf/testcases.yaml | 17 | ||||
| -rw-r--r-- | functest/ci/testcases.yaml | 188 | ||||
| -rw-r--r-- | functest/opnfv_tests/openstack/tempest/tempest.py | 20 | ||||
| -rw-r--r-- | functest/tests/unit/odl/test_odl.py | 2 | ||||
| -rw-r--r-- | setup.cfg | 28 | ||||
| -rw-r--r-- | upper-constraints.txt | 2 |
23 files changed, 272 insertions, 232 deletions
diff --git a/.travis.yml b/.travis.yml index ea13625e3..e4060757f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,6 +52,11 @@ jobs: - script: sudo -E bash build.sh env: - REPO="${DOCKER_USERNAME}" + - amd64_dirs="docker/benchmarking" + - arm64_dirs="" + - script: sudo -E bash build.sh + env: + - REPO="${DOCKER_USERNAME}" - amd64_dirs="docker/features" - arm64_dirs="" - script: sudo -E bash build.sh @@ -8,6 +8,7 @@ docker/core \ docker/tempest \ docker/healthcheck \ docker/smoke \ +docker/benchmarking \ docker/features \ docker/components \ docker/vnf"} diff --git a/ci/daily.yaml b/ci/daily.yaml index 2ae1b68c2..89b404f77 100644 --- a/ci/daily.yaml +++ b/ci/daily.yaml @@ -9,6 +9,7 @@ container: - healthcheck - smoke + - benchmarking - components - vnf - features @@ -24,6 +25,7 @@ projects: - name: '{repo}-functest-healthcheck-{tag}-pull' - name: '{repo}-functest-smoke-{tag}-pull' + - name: '{repo}-functest-benchmarking-{tag}-pull' - name: '{repo}-functest-components-{tag}-pull' - name: '{repo}-functest-vnf-{tag}-pull' - name: '{repo}-functest-features-{tag}-pull' @@ -49,6 +51,7 @@ name: advanced tests projects: - name: '{repo}-functest-smoke-{tag}-run' + - name: '{repo}-functest-benchmarking-{tag}-run' - name: '{repo}-functest-components-{tag}-run' - name: '{repo}-functest-vnf-{tag}-run' - name: '{repo}-functest-features-{tag}-run' diff --git a/ci/gate.yaml b/ci/gate.yaml index ea0c76e09..1abfc67ac 100644 --- a/ci/gate.yaml +++ b/ci/gate.yaml @@ -39,6 +39,7 @@ - tempest - healthcheck - smoke + - benchmarking - components - vnf - features @@ -67,6 +68,8 @@ <<: *functest-projectparameters - name: '{repo}-functest-smoke-{tag}-build' <<: *functest-projectparameters + - name: '{repo}-functest-benchmarking-{tag}-build' + <<: *functest-projectparameters - name: '{repo}-functest-components-{tag}-build' <<: *functest-projectparameters - name: '{repo}-functest-vnf-{tag}-build' @@ -97,6 +100,7 @@ name: advanced tests projects: - name: '{repo}-functest-smoke-{tag}-run' + - name: '{repo}-functest-benchmarking-{tag}-run' - name: '{repo}-functest-components-{tag}-run' - name: '{repo}-functest-vnf-{tag}-run' - name: '{repo}-functest-features-{tag}-run' diff --git a/ci/run.yaml b/ci/run.yaml index 558956f6f..9cc374c0c 100644 --- a/ci/run.yaml +++ b/ci/run.yaml @@ -68,8 +68,6 @@ - rally_sanity - refstack_defcore - patrole - - vmtp - - shaker - snaps_smoke - neutron_trunk - networking-bgpvpn @@ -90,8 +88,6 @@ - name: '{repo}-functest-smoke-rally_sanity-{tag}-run' - name: '{repo}-functest-smoke-refstack_defcore-{tag}-run' - name: '{repo}-functest-smoke-patrole-{tag}-run' - - name: '{repo}-functest-smoke-vmtp-{tag}-run' - - name: '{repo}-functest-smoke-shaker-{tag}-run' - name: '{repo}-functest-smoke-snaps_smoke-{tag}-run' - name: '{repo}-functest-smoke-neutron_trunk-{tag}-run' - name: '{repo}-functest-smoke-networking-bgpvpn-{tag}-run' @@ -104,6 +100,30 @@ - '{repo}-functest-smoke-{tag}-run' - project: + name: '{repo}-functest-benchmarking-{test}-{tag}-run' + container: benchmarking + test: + - vmtp + - shaker + jobs: + - '{repo}-functest-{container}-{test}-{tag}-run' + +- job-template: + name: '{repo}-functest-benchmarking-{tag}-run' + project-type: multijob + builders: + - multijob: + name: smoke + projects: + - name: '{repo}-functest-benchmarking-vmtp-{tag}-run' + - name: '{repo}-functest-benchmarking-shaker-{tag}-run' + +- project: + name: '{repo}-functest-benchmarking-{tag}-run' + jobs: + - '{repo}-functest-benchmarking-{tag}-run' + +- project: name: '{repo}-functest-components-{test}-{tag}-run' container: components test: diff --git a/docker/benchmarking/Dockerfile b/docker/benchmarking/Dockerfile new file mode 100644 index 000000000..3afa5c483 --- /dev/null +++ b/docker/benchmarking/Dockerfile @@ -0,0 +1,24 @@ +FROM opnfv/functest-core + +ARG BRANCH=master +ARG OPENSTACK_TAG=stable/queens +ARG VMTP_TAG=99b261ccccc2f8a08ee2d8fca9f54ef9d69899d7 + +RUN apk --no-cache add --update libxml2 libxslt && \ + apk --no-cache add --virtual .build-deps --update \ + python-dev build-base linux-headers libffi-dev \ + openssl-dev libjpeg-turbo-dev libxml2-dev libxslt-dev && \ + wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ + sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest@\\1#egg=tempest/ \ + > upper-constraints.txt && \ + wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ + sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ + git clone https://git.openstack.org/openstack/vmtp.git /src/vmtp && \ + (cd /src/vmtp && git fetch origin --tags $VMTP_TAG && git checkout FETCH_HEAD) && \ + update-requirements -s --source /src/openstack-requirements /src/vmtp/ && \ + pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ + /src/vmtp && \ + rm -r upper-constraints.txt upper-constraints.opnfv.txt src/vmtp && \ + apk del .build-deps +COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml +CMD ["run_tests", "-t", "all"] diff --git a/docker/benchmarking/hooks/post_checkout b/docker/benchmarking/hooks/post_checkout new file mode 100644 index 000000000..8d0e98124 --- /dev/null +++ b/docker/benchmarking/hooks/post_checkout @@ -0,0 +1,6 @@ +#!/bin/bash + +from="${DOCKER_REPO%/*}/functest-core:${DOCKER_TAG}" +sed -i "s|^FROM.*$|FROM ${from}|" Dockerfile + +exit $? diff --git a/docker/benchmarking/testcases.yaml b/docker/benchmarking/testcases.yaml new file mode 100644 index 000000000..0c4372679 --- /dev/null +++ b/docker/benchmarking/testcases.yaml @@ -0,0 +1,35 @@ +--- +tiers: + - + name: benchmarking + order: 2 + description: >- + Run several OpenStack performance tools + https://docs.openstack.org/performance-docs/latest/methodologies/tools.html + testcases: + - + case_name: vmtp + project_name: functest + criteria: 100 + blocking: false + description: >- + VMTP is a small python application that will automatically + perform ping connectivity, round trip time measurement + (latency) and TCP/UDP throughput + dependencies: + - POD_ARCH: '^(?!aarch64$)' + run: + name: vmtp + + - + case_name: shaker + project_name: functest + criteria: 100 + blocking: false + description: >- + Shaker wraps around popular system network testing tools + like iperf, iperf3 and netperf (with help of flent). Shaker + is able to deploy OpenStack instances and networks in + different topologies. + run: + name: shaker diff --git a/docker/components/testcases.yaml b/docker/components/testcases.yaml index 6292d339c..73a88aeb7 100644 --- a/docker/components/testcases.yaml +++ b/docker/components/testcases.yaml @@ -2,7 +2,7 @@ tiers: - name: components - order: 3 + order: 4 description: >- Extensive testing of OpenStack API. testcases: @@ -17,8 +17,7 @@ tiers: the OpenStack deployment. https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83 run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' option: @@ -35,8 +34,7 @@ tiers: the OpenStack deployment. https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84 run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' option: @@ -51,5 +49,4 @@ tiers: This test case runs the full suite of scenarios of the OpenStack Rally suite using several threads and iterations. run: - module: 'functest.opnfv_tests.openstack.rally.rally' - class: 'RallyFull' + name: rally_full diff --git a/docker/core/Dockerfile b/docker/core/Dockerfile index 97e4b17e8..8f744db67 100644 --- a/docker/core/Dockerfile +++ b/docker/core/Dockerfile @@ -15,8 +15,12 @@ RUN apk --no-cache add --update \ pip install --no-cache-dir --src /src -cupper-constraints.txt \ -chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG \ pip==$PIP_TAG && \ + pip install --no-cache-dir --src /src -cupper-constraints.txt \ + -chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG \ + -e git+https://git.openstack.org/openstack/requirements@$OPENSTACK_TAG#egg=openstack_requirements && \ git clone --depth 1 https://gerrit.opnfv.org/gerrit/functest /src/functest && \ (cd /src/functest && git fetch --depth 1 --tags origin $BRANCH && git checkout FETCH_HEAD) && \ + update-requirements -s --source /src/openstack-requirements /src/functest && \ pip install --no-cache-dir --src /src -cupper-constraints.txt \ -chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG \ /src/functest && \ diff --git a/docker/features/hooks/post_checkout b/docker/features/hooks/post_checkout index 8d0e98124..3e5670b4a 100644 --- a/docker/features/hooks/post_checkout +++ b/docker/features/hooks/post_checkout @@ -1,6 +1,6 @@ #!/bin/bash -from="${DOCKER_REPO%/*}/functest-core:${DOCKER_TAG}" +from="${DOCKER_REPO%/*}/functest-tempest:${DOCKER_TAG}" sed -i "s|^FROM.*$|FROM ${from}|" Dockerfile exit $? diff --git a/docker/features/testcases.yaml b/docker/features/testcases.yaml index 138451174..a2f71f8c1 100644 --- a/docker/features/testcases.yaml +++ b/docker/features/testcases.yaml @@ -2,7 +2,7 @@ tiers: - name: features - order: 2 + order: 3 description: >- Test suites from feature projects integrated in functest @@ -18,8 +18,7 @@ tiers: - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)' - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$' run: - module: 'xtesting.core.feature' - class: 'BashFeature' + name: bashfeature args: cmd: 'doctor-test' @@ -33,8 +32,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'bgpvpn' run: - module: 'sdnvpn.test.functest.run_sdnvpn_tests' - class: 'SdnvpnFunctest' + name: bgpvpn - case_name: functest-odl-sfc @@ -47,8 +45,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'odl.*sfc' run: - module: 'sfc.tests.functest.run_sfc_tests' - class: 'SfcFunctest' + name: functest-odl-sfc - case_name: barometercollectd @@ -62,8 +59,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'bar' run: - module: 'baro_tests.barometer' - class: 'BarometerCollectd' + name: barometercollectd - case_name: fds @@ -77,8 +73,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'odl.*-fdio' run: - module: 'functest.opnfv_tests.sdn.odl.odl' - class: 'ODLTests' + name: odl args: suites: - /src/fds/testing/robot @@ -92,8 +87,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'vgpu' run: - module: 'functest.opnfv_tests.openstack.vgpu.vgpu' - class: 'vGPU' + name: vgpu - case_name: stor4nfv_os @@ -106,5 +100,4 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'stor4nfv' run: - module: 'stor4nfv_tests.stor4nfv_os' - class: 'OsStor4nfvTest' + name: stor4nfv_os diff --git a/docker/healthcheck/testcases.yaml b/docker/healthcheck/testcases.yaml index 84135678b..cdb13e53b 100644 --- a/docker/healthcheck/testcases.yaml +++ b/docker/healthcheck/testcases.yaml @@ -17,9 +17,7 @@ tiers: Keystone, Glance, Neutron and Nova and may perform some simple queries. run: - module: - 'functest.opnfv_tests.openstack.api.connection_check' - class: 'ConnectionCheck' + name: connection_check - case_name: tenantnetwork1 @@ -31,8 +29,7 @@ tiers: required by advanced testcases (subnet, network and router). run: - module: 'functest.core.tenantnetwork' - class: 'TenantNetwork1' + name: tenantnetwork1 - case_name: tenantnetwork2 @@ -44,8 +41,7 @@ tiers: all tenant network ressources required by a testcase (subnet, network and router). run: - module: 'functest.core.tenantnetwork' - class: 'TenantNetwork2' + name: tenantnetwork2 - case_name: vmready1 @@ -57,8 +53,7 @@ tiers: resources and prepares a future VM attached to that network. run: - module: 'functest.core.singlevm' - class: 'VmReady1' + name: vmready1 - case_name: vmready2 @@ -70,8 +65,7 @@ tiers: all tenant network ressources, flavors, images, etc. required by advanced testcases. run: - module: 'functest.core.singlevm' - class: 'VmReady2' + name: vmready2 - case_name: singlevm1 @@ -83,8 +77,7 @@ tiers: resources and completes it by booting a VM attached to that network. run: - module: 'functest.core.singlevm' - class: 'SingleVm1' + name: singlevm1 - case_name: singlevm2 @@ -96,8 +89,7 @@ tiers: all tenant network ressources and vms required by advanced testcases. run: - module: 'functest.core.singlevm' - class: 'SingleVm2' + name: singlevm2 - case_name: vping_ssh @@ -109,8 +101,7 @@ tiers: floating IPs over the public network. 2) Connectivity between 2 instances over a private network. run: - module: 'functest.opnfv_tests.openstack.vping.vping_ssh' - class: 'VPingSSH' + name: vping_ssh - case_name: vping_userdata @@ -121,9 +112,7 @@ tiers: This test case verifies: 1) Boot a VM with given userdata. 2) Connectivity between 2 instances over a private network. run: - module: - 'functest.opnfv_tests.openstack.vping.vping_userdata' - class: 'VPingUserdata' + name: vping_userdata - case_name: cinder_test @@ -136,8 +125,7 @@ tiers: from instance 1, attach it on instance 2 3) Read volume data run: - module: 'functest.opnfv_tests.openstack.cinder.cinder_test' - class: 'CinderCheck' + name: cinder_test - case_name: odl @@ -151,8 +139,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'odl' run: - module: 'functest.opnfv_tests.sdn.odl.odl' - class: 'ODLTests' + name: odl args: suites: - /src/odl_test/csit/suites/integration/basic @@ -170,8 +157,7 @@ tiers: snaps.use_keystone is True, functest must have access to the cloud's private network. run: - module: 'functest.opnfv_tests.openstack.snaps.api_check' - class: 'ApiCheck' + name: api_check - case_name: snaps_health_check @@ -184,5 +170,4 @@ tiers: and Cirros VM instance and observes the console output to validate the single port obtains the correct IP address. run: - module: 'functest.opnfv_tests.openstack.snaps.health_check' - class: 'HealthCheck' + name: snaps_health_check diff --git a/docker/smoke/Dockerfile b/docker/smoke/Dockerfile index 7e4939a41..58d096108 100644 --- a/docker/smoke/Dockerfile +++ b/docker/smoke/Dockerfile @@ -4,14 +4,12 @@ ARG BRANCH=master ARG OPENSTACK_TAG=stable/queens ARG REFSTACK_TARGET=2018.02 ARG PATROLE_TAG=0.3.0 -ARG VMTP_TAG=99b261ccccc2f8a08ee2d8fca9f54ef9d69899d7 -ARG NEUTRON_TAG=d307f5635d4015e7842a4a177f66c76ba4bd818a -ARG BARBICAN_TAG=b8bf147bdcdd33f3ad276ca8815fd253ec9b24af +ARG NEUTRON_TAG=0.2.0 +ARG BARBICAN_TAG=0.1.0 -RUN apk --no-cache add --update libxml2 libxslt && \ - apk --no-cache add --virtual .build-deps --update \ +RUN apk --no-cache add --virtual .build-deps --update \ python-dev build-base linux-headers libffi-dev \ - openssl-dev libjpeg-turbo-dev libxml2-dev libxslt-dev && \ + openssl-dev libjpeg-turbo-dev && \ wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest@\\1#egg=tempest/ \ > upper-constraints.txt && \ @@ -20,9 +18,6 @@ RUN apk --no-cache add --update libxml2 libxslt && \ git clone --depth 1 https://git.openstack.org/openstack/patrole.git /src/patrole && \ (cd /src/patrole && git fetch --tags origin $PATROLE_TAG && git checkout FETCH_HEAD) && \ update-requirements -s --source /src/openstack-requirements /src/patrole/ && \ - git clone https://git.openstack.org/openstack/vmtp.git /src/vmtp && \ - (cd /src/vmtp && git fetch origin --tags $VMTP_TAG && git checkout FETCH_HEAD) && \ - update-requirements -s --source /src/openstack-requirements /src/vmtp/ && \ git clone https://git.openstack.org/openstack/neutron-tempest-plugin.git /src/neutron-tempest-plugin && \ (cd /src/neutron-tempest-plugin && git fetch --tags origin $NEUTRON_TAG && git checkout FETCH_HEAD) && \ update-requirements -s --source /src/openstack-requirements /src/neutron-tempest-plugin && \ @@ -31,9 +26,9 @@ RUN apk --no-cache add --update libxml2 libxslt && \ update-requirements -s --source /src/openstack-requirements /src/barbican-tempest-plugin/ && \ pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin \ - networking-bgpvpn networking-sfc /src/vmtp && \ + networking-bgpvpn networking-sfc && \ rm -r upper-constraints.txt upper-constraints.opnfv.txt \ - /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin /src/vmtp && \ + /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin && \ mkdir -p /home/opnfv/functest/data/refstack && \ wget "https://refstack.openstack.org/api/v1/guidelines/${REFSTACK_TARGET}/tests?target=compute&type=required&alias=true&flag=false" \ -O /home/opnfv/functest/data/refstack/defcore.txt && \ diff --git a/docker/smoke/hooks/post_checkout b/docker/smoke/hooks/post_checkout index 8d0e98124..3e5670b4a 100644 --- a/docker/smoke/hooks/post_checkout +++ b/docker/smoke/hooks/post_checkout @@ -1,6 +1,6 @@ #!/bin/bash -from="${DOCKER_REPO%/*}/functest-core:${DOCKER_TAG}" +from="${DOCKER_REPO%/*}/functest-tempest:${DOCKER_TAG}" sed -i "s|^FROM.*$|FROM ${from}|" Dockerfile exit $? diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml index 7c22e838c..2897e1045 100644 --- a/docker/smoke/testcases.yaml +++ b/docker/smoke/testcases.yaml @@ -18,8 +18,7 @@ tiers: the OpenStack deplopyment. https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114 run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)' option: @@ -36,8 +35,7 @@ tiers: Tempest automatically and depends on the parameters of the OpenStack deployment. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '^neutron_tempest_plugin\.api' option: @@ -52,8 +50,7 @@ tiers: This test case runs a sub group of tests of the OpenStack Rally suite in smoke mode. run: - module: 'functest.opnfv_tests.openstack.rally.rally' - class: 'RallySanity' + name: rally_sanity - case_name: refstack_defcore @@ -64,9 +61,7 @@ tiers: This test case runs a sub group of tests of the OpenStack Defcore testcases. run: - module: - 'functest.opnfv_tests.openstack.refstack.refstack' - class: 'Refstack' + name: refstack_defcore args: option: - '--concurrency=4' @@ -79,8 +74,7 @@ tiers: description: >- Test suite from Patrole project. run: - module: 'functest.opnfv_tests.openstack.patrole.patrole' - class: 'Patrole' + name: patrole args: role: 'admin' includes: @@ -101,9 +95,7 @@ tiers: dependencies: - POD_ARCH: '^(?!aarch64$)' run: - module: - 'functest.opnfv_tests.openstack.vmtp.vmtp' - class: 'Vmtp' + name: vmtp - case_name: shaker @@ -116,9 +108,7 @@ tiers: is able to deploy OpenStack instances and networks in different topologies. run: - module: - 'functest.opnfv_tests.openstack.shaker.shaker' - class: 'Shaker' + name: shaker - case_name: snaps_smoke @@ -134,8 +124,7 @@ tiers: snaps.use_keystone is True, functest must have access to the cloud's private network. run: - module: 'functest.opnfv_tests.openstack.snaps.smoke' - class: 'SnapsSmoke' + name: snaps_smoke - case_name: neutron_trunk @@ -148,8 +137,7 @@ tiers: generated by Tempest having as input the relevant testcase list file. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: 'neutron_tempest_plugin.(api|scenario).test_trunk' neutron_extensions: @@ -166,8 +154,7 @@ tiers: suite. The list of test cases is generated by Tempest having as input the relevant regex. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '^networking_bgpvpn_tempest\.' neutron_extensions: @@ -185,8 +172,7 @@ tiers: suite. The list of test cases is generated by Tempest having as input the relevant regex. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$' @@ -204,8 +190,7 @@ tiers: It leverages on the tempest plugin containing tests used to verify the functionality of a barbican installation. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: 'barbican_tempest_plugin.tests.(api|scenario).(?!test_image_signing)' diff --git a/docker/tempest/Dockerfile b/docker/tempest/Dockerfile index 3c30bae5a..d53457913 100644 --- a/docker/tempest/Dockerfile +++ b/docker/tempest/Dockerfile @@ -21,8 +21,6 @@ RUN apk --no-cache add --virtual .build-deps --update \ esac && \ wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ - pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ - -e git+https://git.openstack.org/openstack/requirements@$OPENSTACK_TAG#egg=openstack_requirements && \ git clone --depth 1 https://git.openstack.org/openstack/os-faults.git /src/os-faults && \ (cd /src/os-faults && git fetch --depth 1 --tags origin $OS_FAULTS_TAG && git checkout FETCH_HEAD) && \ update-requirements -s --source /src/openstack-requirements /src/os-faults/ && \ diff --git a/docker/vnf/testcases.yaml b/docker/vnf/testcases.yaml index 1f0817708..170eed0e5 100644 --- a/docker/vnf/testcases.yaml +++ b/docker/vnf/testcases.yaml @@ -2,7 +2,7 @@ tiers: - name: vnf - order: 4 + order: 5 description: >- Collection of VNF test cases. testcases: @@ -16,8 +16,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.core.cloudify' - class: 'Cloudify' + name: cloudify - case_name: cloudify_ims project_name: functest @@ -30,8 +29,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.opnfv_tests.vnf.ims.cloudify_ims' - class: 'CloudifyIms' + name: cloudify_ims - case_name: heat_ims @@ -45,8 +43,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.opnfv_tests.vnf.ims.heat_ims' - class: 'HeatIms' + name: heat_ims - case_name: vyos_vrouter @@ -58,8 +55,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.opnfv_tests.vnf.router.cloudify_vrouter' - class: 'CloudifyVrouter' + name: vyos_vrouter - case_name: juju_epc @@ -72,5 +68,4 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.opnfv_tests.vnf.epc.juju_epc' - class: 'JujuEpc' + name: juju_epc diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 4b73a8418..20dc865d1 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -17,9 +17,7 @@ tiers: Keystone, Glance, Neutron and Nova and may perform some simple queries. run: - module: - 'functest.opnfv_tests.openstack.api.connection_check' - class: 'ConnectionCheck' + name: connection_check - case_name: tenantnetwork1 @@ -31,8 +29,7 @@ tiers: required by advanced testcases (subnet, network and router). run: - module: 'functest.core.tenantnetwork' - class: 'TenantNetwork1' + name: tenantnetwork1 - case_name: tenantnetwork2 @@ -44,8 +41,7 @@ tiers: all tenant network ressources required by a testcase (subnet, network and router). run: - module: 'functest.core.tenantnetwork' - class: 'TenantNetwork2' + name: tenantnetwork2 - case_name: vmready1 @@ -57,8 +53,7 @@ tiers: resources and prepares a future VM attached to that network. run: - module: 'functest.core.singlevm' - class: 'VmReady1' + name: vmready1 - case_name: vmready2 @@ -70,8 +65,7 @@ tiers: all tenant network ressources, flavors, images, etc. required by advanced testcases. run: - module: 'functest.core.singlevm' - class: 'VmReady2' + name: vmready2 - case_name: singlevm1 @@ -83,8 +77,7 @@ tiers: resources and completes it by booting a VM attached to that network. run: - module: 'functest.core.singlevm' - class: 'SingleVm1' + name: singlevm1 - case_name: singlevm2 @@ -96,8 +89,7 @@ tiers: all tenant network ressources and vms required by advanced testcases. run: - module: 'functest.core.singlevm' - class: 'SingleVm2' + name: singlevm2 - case_name: vping_ssh @@ -109,8 +101,7 @@ tiers: floating IPs over the public network. 2) Connectivity between 2 instances over a private network. run: - module: 'functest.opnfv_tests.openstack.vping.vping_ssh' - class: 'VPingSSH' + name: vping_ssh - case_name: vping_userdata @@ -121,9 +112,7 @@ tiers: This test case verifies: 1) Boot a VM with given userdata. 2) Connectivity between 2 instances over a private network. run: - module: - 'functest.opnfv_tests.openstack.vping.vping_userdata' - class: 'VPingUserdata' + name: vping_userdata - case_name: cinder_test @@ -136,8 +125,7 @@ tiers: from instance 1, attach it on instance 2 3) Read volume data run: - module: 'functest.opnfv_tests.openstack.cinder.cinder_test' - class: 'CinderCheck' + name: cinder_test - case_name: odl @@ -151,8 +139,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'odl' run: - module: 'functest.opnfv_tests.sdn.odl.odl' - class: 'ODLTests' + name: odl args: suites: - /src/odl_test/csit/suites/integration/basic @@ -170,8 +157,7 @@ tiers: snaps.use_keystone is True, functest must have access to the cloud's private network. run: - module: 'functest.opnfv_tests.openstack.snaps.api_check' - class: 'ApiCheck' + name: api_check - case_name: snaps_health_check @@ -184,8 +170,7 @@ tiers: and Cirros VM instance and observes the console output to validate the single port obtains the correct IP address. run: - module: 'functest.opnfv_tests.openstack.snaps.health_check' - class: 'HealthCheck' + name: snaps_health_check - name: smoke @@ -205,8 +190,7 @@ tiers: the OpenStack deplopyment. https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114 run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)' option: @@ -223,8 +207,7 @@ tiers: Tempest automatically and depends on the parameters of the OpenStack deployment. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '^neutron_tempest_plugin\.api' option: @@ -239,8 +222,7 @@ tiers: This test case runs a sub group of tests of the OpenStack Rally suite in smoke mode. run: - module: 'functest.opnfv_tests.openstack.rally.rally' - class: 'RallySanity' + name: rally_sanity - case_name: refstack_defcore @@ -251,9 +233,7 @@ tiers: This test case runs a sub group of tests of the OpenStack Defcore testcases. run: - module: - 'functest.opnfv_tests.openstack.refstack.refstack' - class: 'Refstack' + name: refstack_defcore args: option: - '--concurrency=4' @@ -266,8 +246,7 @@ tiers: description: >- Test suite from Patrole project. run: - module: 'functest.opnfv_tests.openstack.patrole.patrole' - class: 'Patrole' + name: patrole args: role: 'admin' includes: @@ -277,37 +256,6 @@ tiers: - 'test_networks_multiprovider_rbac' - - case_name: vmtp - project_name: functest - criteria: 100 - blocking: false - description: >- - VMTP is a small python application that will automatically - perform ping connectivity, round trip time measurement - (latency) and TCP/UDP throughput - dependencies: - - POD_ARCH: '^(?!aarch64$)' - run: - module: - 'functest.opnfv_tests.openstack.vmtp.vmtp' - class: 'Vmtp' - - - - case_name: shaker - project_name: functest - criteria: 100 - blocking: false - description: >- - Shaker wraps around popular system network testing tools - like iperf, iperf3 and netperf (with help of flent). Shaker - is able to deploy OpenStack instances and networks in - different topologies. - run: - module: - 'functest.opnfv_tests.openstack.shaker.shaker' - class: 'Shaker' - - - case_name: snaps_smoke project_name: functest criteria: 100 @@ -321,8 +269,7 @@ tiers: snaps.use_keystone is True, functest must have access to the cloud's private network. run: - module: 'functest.opnfv_tests.openstack.snaps.smoke' - class: 'SnapsSmoke' + name: snaps_smoke - case_name: neutron_trunk @@ -335,8 +282,7 @@ tiers: generated by Tempest having as input the relevant testcase list file. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: 'neutron_tempest_plugin.(api|scenario).test_trunk' neutron_extensions: @@ -353,8 +299,7 @@ tiers: suite. The list of test cases is generated by Tempest having as input the relevant regex. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '^networking_bgpvpn_tempest\.' neutron_extensions: @@ -372,8 +317,7 @@ tiers: suite. The list of test cases is generated by Tempest having as input the relevant regex. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '^(?:networking_sfc\.tests\.tempest_plugin.tests.api).*$' @@ -391,8 +335,7 @@ tiers: It leverages on the tempest plugin containing tests used to verify the functionality of a barbican installation. run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: 'barbican_tempest_plugin.tests.(api|scenario).(?!test_image_signing)' @@ -400,9 +343,43 @@ tiers: - barbican - - name: features + name: benchmarking order: 2 description: >- + Run several OpenStack performance tools + https://docs.openstack.org/performance-docs/latest/methodologies/tools.html + testcases: + - + case_name: vmtp + project_name: functest + criteria: 100 + blocking: false + description: >- + VMTP is a small python application that will automatically + perform ping connectivity, round trip time measurement + (latency) and TCP/UDP throughput + dependencies: + - POD_ARCH: '^(?!aarch64$)' + run: + name: vmtp + + - + case_name: shaker + project_name: functest + criteria: 100 + blocking: false + description: >- + Shaker wraps around popular system network testing tools + like iperf, iperf3 and netperf (with help of flent). Shaker + is able to deploy OpenStack instances and networks in + different topologies. + run: + name: shaker + + - + name: features + order: 3 + description: >- Test suites from feature projects integrated in functest testcases: @@ -417,8 +394,7 @@ tiers: - INSTALLER_TYPE: '(apex)|(fuel)|(daisy)' - DEPLOY_SCENARIO: '^((?!fdio|nofeature).)*$' run: - module: 'xtesting.core.feature' - class: 'BashFeature' + name: bashfeature args: cmd: 'doctor-test' @@ -432,8 +408,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'bgpvpn' run: - module: 'sdnvpn.test.functest.run_sdnvpn_tests' - class: 'SdnvpnFunctest' + name: bgpvpn - case_name: functest-odl-sfc @@ -446,8 +421,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'odl.*sfc' run: - module: 'sfc.tests.functest.run_sfc_tests' - class: 'SfcFunctest' + name: functest-odl-sfc - case_name: barometercollectd @@ -461,8 +435,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'bar' run: - module: 'baro_tests.barometer' - class: 'BarometerCollectd' + name: barometercollectd - case_name: fds @@ -476,8 +449,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'odl.*-fdio' run: - module: 'functest.opnfv_tests.sdn.odl.odl' - class: 'ODLTests' + name: odl args: suites: - /src/fds/testing/robot @@ -491,8 +463,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'vgpu' run: - module: 'functest.opnfv_tests.openstack.vgpu.vgpu' - class: 'vGPU' + name: vgpu - case_name: stor4nfv_os @@ -505,12 +476,11 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'stor4nfv' run: - module: 'stor4nfv_tests.stor4nfv_os' - class: 'OsStor4nfvTest' + name: stor4nfv_os - name: components - order: 3 + order: 4 description: >- Extensive testing of OpenStack API. testcases: @@ -525,8 +495,7 @@ tiers: the OpenStack deployment. https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83 run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' option: @@ -543,8 +512,7 @@ tiers: the OpenStack deployment. https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84 run: - module: 'functest.opnfv_tests.openstack.tempest.tempest' - class: 'TempestCommon' + name: tempest_common args: mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' option: @@ -559,12 +527,11 @@ tiers: This test case runs the full suite of scenarios of the OpenStack Rally suite using several threads and iterations. run: - module: 'functest.opnfv_tests.openstack.rally.rally' - class: 'RallyFull' + name: rally_full - name: vnf - order: 4 + order: 5 description: >- Collection of VNF test cases. testcases: @@ -578,8 +545,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.core.cloudify' - class: 'Cloudify' + name: cloudify - case_name: cloudify_ims project_name: functest @@ -592,8 +558,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.opnfv_tests.vnf.ims.cloudify_ims' - class: 'CloudifyIms' + name: cloudify_ims - case_name: heat_ims @@ -607,8 +572,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.opnfv_tests.vnf.ims.heat_ims' - class: 'HeatIms' + name: heat_ims - case_name: vyos_vrouter @@ -620,8 +584,7 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.opnfv_tests.vnf.router.cloudify_vrouter' - class: 'CloudifyVrouter' + name: vyos_vrouter - case_name: juju_epc @@ -634,5 +597,4 @@ tiers: dependencies: - DEPLOY_SCENARIO: 'os-.*-nofeature-.*ha' run: - module: 'functest.opnfv_tests.vnf.epc.juju_epc' - class: 'JujuEpc' + name: juju_epc diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index bd337ce2f..329e6b917 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -306,9 +306,9 @@ class TempestCommon(singlevm.VmReady2): """Set image name as tempest img_name_regex""" rconfig = configparser.RawConfigParser() rconfig.read(rally_conf) - if not rconfig.has_section('tempest'): - rconfig.add_section('tempest') - rconfig.set('tempest', 'img_name_regex', '^{}$'.format( + if not rconfig.has_section('openstack'): + rconfig.add_section('openstack') + rconfig.set('openstack', 'img_name_regex', '^{}$'.format( self.image.name)) with open(rally_conf, 'wb') as config_file: rconfig.write(config_file) @@ -320,9 +320,9 @@ class TempestCommon(singlevm.VmReady2): return rconfig = configparser.RawConfigParser() rconfig.read(rally_conf) - if not rconfig.has_section('tempest'): - rconfig.add_section('tempest') - rconfig.set('tempest', 'swift_operator_role', role.name) + if not rconfig.has_section('openstack'): + rconfig.add_section('openstack') + rconfig.set('openstack', 'swift_operator_role', role.name) with open(rally_conf, 'wb') as config_file: rconfig.write(config_file) @@ -342,10 +342,10 @@ class TempestCommon(singlevm.VmReady2): """Clean Rally config""" rconfig = configparser.RawConfigParser() rconfig.read(rally_conf) - if rconfig.has_option('tempest', 'img_name_regex'): - rconfig.remove_option('tempest', 'img_name_regex') - if rconfig.has_option('tempest', 'swift_operator_role'): - rconfig.remove_option('tempest', 'swift_operator_role') + if rconfig.has_option('openstack', 'img_name_regex'): + rconfig.remove_option('openstack', 'img_name_regex') + if rconfig.has_option('openstack', 'swift_operator_role'): + rconfig.remove_option('openstack', 'swift_operator_role') if rconfig.has_option('DEFAULT', 'log-file'): rconfig.remove_option('DEFAULT', 'log-file') if rconfig.has_option('DEFAULT', 'log_dir'): diff --git a/functest/tests/unit/odl/test_odl.py b/functest/tests/unit/odl/test_odl.py index f65170453..1b2e10b07 100644 --- a/functest/tests/unit/odl/test_odl.py +++ b/functest/tests/unit/odl/test_odl.py @@ -253,7 +253,7 @@ class ODLMainTesting(ODLTesting): mock.patch.object(self.test, 'parse_results'), \ mock.patch.object(self.test, 'generate_report', return_value=1): - self._test_run_suites(testcase.TestCase.EX_RUN_ERROR, *args) + self._test_run_suites(testcase.TestCase.EX_OK, *args) @mock.patch('os.makedirs') @mock.patch('robot.run') @@ -13,6 +13,34 @@ scripts = console_scripts = functest_odl = functest.opnfv_tests.sdn.odl.odl:main check_deployment = functest.ci.check_deployment:main +xtesting.testcase = + connection_check = functest.opnfv_tests.openstack.api.connection_check:ConnectionCheck + tenantnetwork1 = functest.core.tenantnetwork:TenantNetwork1 + tenantnetwork2 = functest.core.tenantnetwork:TenantNetwork2 + vmready1 = functest.core.singlevm:VmReady1 + vmready2 = functest.core.singlevm:VmReady2 + singlevm1 = functest.core.singlevm:SingleVm1 + singlevm2 = functest.core.singlevm:SingleVm2 + vping_ssh = functest.opnfv_tests.openstack.vping.vping_ssh:VPingSSH + vping_userdata = functest.opnfv_tests.openstack.vping.vping_userdata:VPingUserdata + cinder_test = functest.opnfv_tests.openstack.cinder.cinder_test:CinderCheck + odl = functest.opnfv_tests.sdn.odl.odl:ODLTests + api_check = functest.opnfv_tests.openstack.snaps.api_check:ApiCheck + snaps_health_check = functest.opnfv_tests.openstack.snaps.health_check:HealthCheck + tempest_common = functest.opnfv_tests.openstack.tempest.tempest:TempestCommon + rally_sanity = functest.opnfv_tests.openstack.rally.rally:RallySanity + refstack_defcore = functest.opnfv_tests.openstack.refstack.refstack:Refstack + patrole = functest.opnfv_tests.openstack.patrole.patrole:Patrole + vmtp = functest.opnfv_tests.openstack.vmtp.vmtp:Vmtp + shaker = functest.opnfv_tests.openstack.shaker.shaker:Shaker + snaps_smoke = functest.opnfv_tests.openstack.snaps.smoke:SnapsSmoke + vgpu = functest.opnfv_tests.openstack.vgpu.vgpu:vGPU + rally_full = functest.opnfv_tests.openstack.rally.rally:RallyFull + cloudify = functest.core.cloudify:Cloudify + cloudify_ims = functest.opnfv_tests.vnf.ims.cloudify_ims:CloudifyIms + heat_ims = functest.opnfv_tests.vnf.ims.heat_ims:HeatIms + vyos_vrouter = functest.opnfv_tests.vnf.router.cloudify_vrouter:CloudifyVrouter + juju_epc = functest.opnfv_tests.vnf.epc.juju_epc:JujuEpc [build_sphinx] all_files = 1 diff --git a/upper-constraints.txt b/upper-constraints.txt index 2ffb034e7..8ac4fd069 100644 --- a/upper-constraints.txt +++ b/upper-constraints.txt @@ -16,6 +16,6 @@ robotframework-httplibrary===0.4.2 robotframework-requests===0.4.7 robotframework-sshlibrary===2.1.3;python_version=='2.7' ansible===2.3.2.0 -xtesting===0.58.0 +xtesting===0.60.0 networking-bgpvpn===8.0.0 networking-sfc===6.0.0 |