diff options
35 files changed, 646 insertions, 427 deletions
diff --git a/.travis.yml b/.travis.yml index e4060757f..49dd91adf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,11 +62,6 @@ jobs: - script: sudo -E bash build.sh env: - REPO="${DOCKER_USERNAME}" - - amd64_dirs="docker/components" - - arm64_dirs="" - - script: sudo -E bash build.sh - env: - - REPO="${DOCKER_USERNAME}" - amd64_dirs="docker/vnf" - arm64_dirs="" - stage: publish all manifests @@ -88,10 +83,5 @@ jobs: - script: > sudo manifest-tool push from-args \ --platforms linux/amd64 \ - --template ${DOCKER_USERNAME}/functest-components:ARCH-latest \ - --target ${DOCKER_USERNAME}/functest-components:latest - - script: > - sudo manifest-tool push from-args \ - --platforms linux/amd64 \ --template ${DOCKER_USERNAME}/functest-vnf:ARCH-latest \ --target ${DOCKER_USERNAME}/functest-vnf:latest diff --git a/ansible/site.yml b/ansible/site.yml index a6f71522d..38dbfacd5 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -33,9 +33,6 @@ - name: functest-benchmarking ref_arg: BRANCH path: docker/benchmarking - - name: functest-components - ref_arg: - path: docker/components - name: functest-vnf ref_arg: path: docker/vnf @@ -58,13 +55,14 @@ - odl - api_check - snaps_health_check + - tempest_smoke - container: functest-smoke tests: - - tempest_smoke - neutron-tempest-plugin-api - rally_sanity - - rally_jobs - refstack_defcore + - tempest_full + - tempest_scenario - patrole - snaps_smoke - neutron_trunk @@ -73,13 +71,10 @@ - barbican - container: functest-benchmarking tests: + - rally_full + - rally_jobs - vmtp - shaker - - container: functest-components - tests: - - tempest_full - - tempest_scenario - - rally_full - container: functest-vnf tests: - cloudify @@ -10,7 +10,6 @@ docker/healthcheck \ docker/smoke \ docker/benchmarking \ docker/features \ -docker/components \ docker/vnf"} arm64_dirs=${arm64_dirs-${amd64_dirs}} build_opts=(--pull=true --no-cache --force-rm=true) diff --git a/docker/benchmarking/Dockerfile b/docker/benchmarking/Dockerfile index ff87f56d6..cf13da815 100644 --- a/docker/benchmarking/Dockerfile +++ b/docker/benchmarking/Dockerfile @@ -1,4 +1,4 @@ -FROM opnfv/functest-core +FROM opnfv/functest-tempest ARG BRANCH=master ARG OPENSTACK_TAG=master @@ -8,11 +8,10 @@ RUN apk --no-cache add --update libxml2 libxslt && \ apk --no-cache add --virtual .build-deps --update \ python-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev libxml2-dev libxslt-dev && \ - wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ - sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest@\\1#egg=tempest/ \ - > upper-constraints.txt && \ - wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ - sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ + wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \ + sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \ + wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \ + sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \ git init /src/vmtp && \ (cd /src/vmtp && \ git fetch --tags https://git.openstack.org/openstack/vmtp.git $VMTP_TAG && \ @@ -20,7 +19,13 @@ RUN apk --no-cache add --update libxml2 libxslt && \ update-requirements -s --source /src/openstack-requirements /src/vmtp/ && \ pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ /src/vmtp && \ - rm -r upper-constraints.txt upper-constraints.opnfv.txt src/vmtp && \ + mkdir -p /home/opnfv/functest/data/rally/neutron && \ + git init /src/neutron && \ + (cd /src/neutron && \ + git fetch --tags https://git.openstack.org/openstack/neutron.git $OPENSTACK_TAG && \ + git checkout FETCH_HEAD) && \ + cp -r /src/neutron/rally-jobs /home/opnfv/functest/data/rally/neutron/rally-jobs && \ + rm -r upper-constraints.txt upper-constraints.opnfv.txt src/vmtp /src/neutron && \ apk del .build-deps COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml CMD ["run_tests", "-t", "all"] diff --git a/docker/benchmarking/testcases.yaml b/docker/benchmarking/testcases.yaml index 0c4372679..eeb209113 100644 --- a/docker/benchmarking/testcases.yaml +++ b/docker/benchmarking/testcases.yaml @@ -8,6 +8,34 @@ tiers: https://docs.openstack.org/performance-docs/latest/methodologies/tools.html testcases: - + case_name: rally_full + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs the full suite of scenarios of the + OpenStack Rally suite using several threads and iterations. + run: + name: rally_full + args: + optional: + - 'gnocchi' + + - + case_name: rally_jobs + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs a group of Rally jobs used in + OpenStack gating + run: + name: rally_jobs + args: + optional: + - 'gnocchi' + + - case_name: vmtp project_name: functest criteria: 100 diff --git a/docker/components/Dockerfile b/docker/components/Dockerfile deleted file mode 100644 index 1153e9d1a..000000000 --- a/docker/components/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -FROM opnfv/functest-tempest - -COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml -CMD ["run_tests", "-t", "all"] diff --git a/docker/components/hooks/post_checkout b/docker/components/hooks/post_checkout deleted file mode 100644 index 8d0e98124..000000000 --- a/docker/components/hooks/post_checkout +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -from="${DOCKER_REPO%/*}/functest-core:${DOCKER_TAG}" -sed -i "s|^FROM.*$|FROM ${from}|" Dockerfile - -exit $? diff --git a/docker/components/testcases.yaml b/docker/components/testcases.yaml deleted file mode 100644 index d0bfa0afe..000000000 --- a/docker/components/testcases.yaml +++ /dev/null @@ -1,55 +0,0 @@ ---- -tiers: - - - name: components - order: 4 - description: >- - Extensive testing of OpenStack API. - testcases: - - - case_name: tempest_full - project_name: functest - criteria: 100 - blocking: false - description: >- - The list of test cases is generated by - Tempest automatically and depends on the parameters of - the OpenStack deployment. - https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83 - run: - name: tempest_common - args: - mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' - option: - - '--concurrency=4' - - - - case_name: tempest_scenario - project_name: functest - criteria: 100 - blocking: false - description: >- - The list of test cases is generated by - Tempest automatically and depends on the parameters of - the OpenStack deployment. - https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84 - run: - name: tempest_common - args: - mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' - option: - - '--concurrency=1' - - - - case_name: rally_full - project_name: functest - criteria: 100 - blocking: false - description: >- - This test case runs the full suite of scenarios of the - OpenStack Rally suite using several threads and iterations. - run: - name: rally_full - args: - optional: - - 'gnocchi' diff --git a/docker/core/Dockerfile b/docker/core/Dockerfile index eaedf2a29..5e6794c82 100644 --- a/docker/core/Dockerfile +++ b/docker/core/Dockerfile @@ -10,12 +10,12 @@ RUN apk --no-cache add --update \ apk --no-cache add --virtual .build-deps --update \ python-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev && \ - wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ - sed -E /#egg=functest/d > upper-constraints.txt && \ - pip install --no-cache-dir --src /src -cupper-constraints.txt \ + wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \ + sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \ + pip install --no-cache-dir --src /src -cupper-constraints.opnfv.txt \ -chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG \ pip==$PIP_TAG && \ - pip install --no-cache-dir --src /src -cupper-constraints.txt \ + pip install --no-cache-dir --src /src -cupper-constraints.opnfv.txt \ -chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG \ -e git+https://git.openstack.org/openstack/requirements@$OPENSTACK_TAG#egg=openstack_requirements && \ git init /src/functest && \ @@ -23,10 +23,10 @@ RUN apk --no-cache add --update \ git fetch --tags https://gerrit.opnfv.org/gerrit/functest $BRANCH && \ git checkout FETCH_HEAD) && \ update-requirements -s --source /src/openstack-requirements /src/functest && \ - pip install --no-cache-dir --src /src -cupper-constraints.txt \ + pip install --no-cache-dir --src /src -cupper-constraints.opnfv.txt \ -chttps://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG \ /src/functest && \ - rm -r upper-constraints.txt /src/functest && \ + rm -r upper-constraints.opnfv.txt /src/functest && \ cp /usr/lib/python2.7/site-packages/functest/ci/logging.ini /usr/lib/python2.7/site-packages/xtesting/ci/ && \ bash -c "mkdir -p /var/lib/xtesting /home/opnfv" && \ ln -s /var/lib/xtesting /home/opnfv/functest && \ diff --git a/docker/features/Dockerfile b/docker/features/Dockerfile index af940e271..46ee9730c 100644 --- a/docker/features/Dockerfile +++ b/docker/features/Dockerfile @@ -8,11 +8,10 @@ RUN apk --no-cache add --update python3 sshpass && \ apk --no-cache add --virtual .build-deps --update \ python-dev python3-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev && \ - wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ - sed -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ \ - > upper-constraints.txt && \ - wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ - sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ + wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \ + sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \ + wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \ + sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \ pip install --no-cache-dir --src /src -cupper-constraints.txt \ -cupper-constraints.opnfv.txt \ -rthirdparty-requirements.txt && \ diff --git a/docker/healthcheck/Dockerfile b/docker/healthcheck/Dockerfile index 19fd0aa5b..3a854bfb1 100644 --- a/docker/healthcheck/Dockerfile +++ b/docker/healthcheck/Dockerfile @@ -1,4 +1,4 @@ -FROM opnfv/functest-core +FROM opnfv/functest-tempest ARG BRANCH=master ARG OPENSTACK_TAG=master @@ -7,11 +7,10 @@ ARG ODL_TAG=85448c9d97b89989488e675b29b38ac42d8674e4 COPY thirdparty-requirements.txt thirdparty-requirements.txt RUN apk --no-cache add --virtual .build-deps --update \ python-dev build-base linux-headers libffi-dev openssl-dev && \ - wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ - sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest@\\1#egg=tempest/ \ - > upper-constraints.txt && \ - wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ - sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ + wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \ + sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \ + wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \ + sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \ pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ -rthirdparty-requirements.txt && \ git init /src/odl_test && \ diff --git a/docker/healthcheck/testcases.yaml b/docker/healthcheck/testcases.yaml index cdb13e53b..e3eb028c9 100644 --- a/docker/healthcheck/testcases.yaml +++ b/docker/healthcheck/testcases.yaml @@ -171,3 +171,21 @@ tiers: validate the single port obtains the correct IP address. run: name: snaps_health_check + + - + case_name: tempest_smoke + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs the smoke subset of the OpenStack + Tempest suite. The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deplopyment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114 + run: + name: tempest_common + args: + mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)' + option: + - '--concurrency=4' diff --git a/docker/smoke/Dockerfile b/docker/smoke/Dockerfile index 2511c6ad9..370680200 100644 --- a/docker/smoke/Dockerfile +++ b/docker/smoke/Dockerfile @@ -4,34 +4,27 @@ ARG BRANCH=master ARG OPENSTACK_TAG=master ARG REFSTACK_TARGET=2018.02 ARG PATROLE_TAG=master -ARG NEUTRON_TAG=master ARG BARBICAN_TAG=master RUN apk --no-cache add --virtual .build-deps --update \ python-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev libxml2-dev libxslt-dev && \ - wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ - sed -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ \ - > upper-constraints.txt && \ - wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ - sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ + wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \ + sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \ + wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \ + sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \ git init /src/patrole && \ (cd /src/patrole && \ git fetch --tags https://git.openstack.org/openstack/patrole.git $PATROLE_TAG && \ git checkout FETCH_HEAD) && \ update-requirements -s --source /src/openstack-requirements /src/patrole/ && \ - git init /src/neutron-tempest-plugin && \ - (cd /src/neutron-tempest-plugin && \ - git fetch --tags https://git.openstack.org/openstack/neutron-tempest-plugin.git $NEUTRON_TAG && \ - git checkout FETCH_HEAD) && \ - update-requirements -s --source /src/openstack-requirements /src/neutron-tempest-plugin && \ git init /src/barbican-tempest-plugin && \ (cd /src/barbican-tempest-plugin && \ git fetch --tags https://git.openstack.org/openstack/barbican-tempest-plugin.git $BARBICAN_TAG && \ git checkout FETCH_HEAD) && \ update-requirements -s --source /src/openstack-requirements /src/barbican-tempest-plugin/ && \ pip install --no-cache-dir --src /src -cupper-constraints.txt -cupper-constraints.opnfv.txt \ - /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin \ + /src/patrole /src/barbican-tempest-plugin neutron-tempest-plugin \ networking-bgpvpn networking-sfc && \ mkdir -p /home/opnfv/functest/data/refstack && \ wget "https://refstack.openstack.org/api/v1/guidelines/${REFSTACK_TARGET}/tests?target=compute&type=required&alias=true&flag=false" \ @@ -49,10 +42,8 @@ RUN apk --no-cache add --virtual .build-deps --update \ oslopolicy-sample-generator --format json --output-file /etc/neutron/policy.json --namespace neutron && \ oslopolicy-sample-generator --format json --output-file /etc/nova/policy.json --namespace nova && \ deactivate && \ - mkdir -p /home/opnfv/functest/data/rally/neutron && \ - cp -r /src/neutron/rally-jobs /home/opnfv/functest/data/rally/neutron/rally-jobs && \ rm -r oslo upper-constraints.txt upper-constraints.opnfv.txt \ - /src/neutron /src/patrole /src/barbican-tempest-plugin /src/neutron-tempest-plugin && \ + /src/patrole /src/barbican-tempest-plugin && \ apk del .build-deps COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml CMD ["run_tests", "-t", "all"] diff --git a/docker/smoke/testcases.yaml b/docker/smoke/testcases.yaml index a811257a1..878db7059 100644 --- a/docker/smoke/testcases.yaml +++ b/docker/smoke/testcases.yaml @@ -7,24 +7,6 @@ tiers: Set of basic Functional tests to validate the OPNFV scenarios. testcases: - - case_name: tempest_smoke - project_name: functest - criteria: 100 - blocking: false - description: >- - This test case runs the smoke subset of the OpenStack - Tempest suite. The list of test cases is generated by - Tempest automatically and depends on the parameters of - the OpenStack deplopyment. - https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L114 - run: - name: tempest_common - args: - mode: '(?=.*\[.*\bsmoke\b.*\])(^tempest\.api)' - option: - - '--concurrency=4' - - - case_name: neutron-tempest-plugin-api project_name: functest criteria: 100 @@ -56,34 +38,54 @@ tiers: - 'gnocchi' - - case_name: rally_jobs + case_name: refstack_defcore project_name: functest criteria: 100 blocking: false description: >- - This test case runs a group of Rally jobs used in - OpenStack gating + This test case runs a sub group of tests of the OpenStack + Defcore testcases. run: - name: rally_jobs + name: refstack_defcore args: - optional: - - 'gnocchi' + option: + - '--concurrency=4' - - case_name: refstack_defcore + case_name: tempest_full project_name: functest criteria: 100 blocking: false description: >- - This test case runs a sub group of tests of the OpenStack - Defcore testcases. + The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83 run: - name: refstack_defcore + name: tempest_common args: + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' option: - '--concurrency=4' - + case_name: tempest_scenario + project_name: functest + criteria: 100 + blocking: false + description: >- + The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84 + run: + name: tempest_common + args: + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' + option: + - '--concurrency=1' + + - case_name: patrole project_name: functest criteria: 100 @@ -149,7 +151,7 @@ tiers: run: name: tempest_common args: - mode: '^networking_bgpvpn_tempest\.' + mode: '^networking_bgpvpn_tempest.tests.api\.' neutron_extensions: - bgpvpn option: diff --git a/docker/tempest/Dockerfile b/docker/tempest/Dockerfile index 1bb25d5ca..fee6793ad 100644 --- a/docker/tempest/Dockerfile +++ b/docker/tempest/Dockerfile @@ -9,11 +9,10 @@ ARG OS_FAULTS_TAG=0.1.18 RUN apk --no-cache add --virtual .build-deps --update \ python-dev build-base linux-headers libffi-dev \ openssl-dev libjpeg-turbo-dev && \ - wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ - sed -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ \ - > upper-constraints.txt && \ - wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ - sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ + wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \ + sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \ + wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \ + sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \ git init /src/os-faults && \ (cd /src/os-faults && \ git fetch --tags https://git.openstack.org/openstack/os-faults.git $OS_FAULTS_TAG && \ diff --git a/docker/vnf/Dockerfile b/docker/vnf/Dockerfile index e915120ed..aba09ebbf 100644 --- a/docker/vnf/Dockerfile +++ b/docker/vnf/Dockerfile @@ -4,9 +4,10 @@ ARG BRANCH=master ARG OPENSTACK_TAG=master ARG VIMS_TEST_TAG=release-130 ARG QUAFF_TAG=59213d6d8ee29433552bb75f505cdc96b0b18909 -ARG CLOUDIFY_VIMS_TAG=fraser +ARG CLOUDIFY_VIMS_TAG=gambia ARG HEAT_VIMS_TAG=release-130 ARG VROUTER_TAG=fraser +ARG VROUTER_BP_TAG=9b76d46a388d32d4985797620e67c2ed3315b3e4 ARG JUJU_TAG=tags/juju-2.3.9 ARG JUJU_WAIT_TAG=2.6.4 ARG ABOT_CHARM=opnfv-fraser @@ -16,16 +17,16 @@ ENV GOPATH /src/epc-requirements/go ENV GOBIN /src/epc-requirements/go/bin ENV PATH $GOBIN:$PATH +COPY clearwater-heat-add-deps.patch /tmp/clearwater-heat-add-deps.patch RUN apk --no-cache add --update \ ruby ruby-bundler ruby-irb ruby-rdoc dnsmasq \ procps libxslt libxml2 zlib libffi python3 go musl-dev && \ apk --no-cache add --virtual .build-deps --update \ ruby-dev g++ make libxslt-dev libxml2-dev zlib-dev libffi-dev g++ make && \ - wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG | \ - sed -E s/^tempest==+\(.*\)$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest@\\1#egg=tempest/ \ - > upper-constraints.txt && \ - wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH | \ - sed -E /#egg=functest/d > upper-constraints.opnfv.txt && \ + wget -q -O- https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=$OPENSTACK_TAG > upper-constraints.txt && \ + sed -i -E s/^tempest==+.*$/-e\ git+https:\\/\\/git.openstack.org\\/openstack\\/tempest#egg=tempest/ upper-constraints.txt && \ + wget -q -O- https://git.opnfv.org/functest/plain/upper-constraints.txt?h=$BRANCH > upper-constraints.opnfv.txt && \ + sed -i -E /#egg=functest/d upper-constraints.opnfv.txt && \ git clone --depth 1 -b $VIMS_TEST_TAG https://github.com/Metaswitch/clearwater-live-test /src/vims-test && \ sed -i s/unf_ext\ \(.*\)/unf_ext\ \(0.0.7.4\)/g /src/vims-test/Gemfile.lock && \ git init /src/vims-test/quaff && \ @@ -43,10 +44,11 @@ RUN apk --no-cache add --update \ git init /src/heat_vims && \ (cd /src/heat_vims && \ git fetch --tags https://github.com/Metaswitch/clearwater-heat.git $HEAT_VIMS_TAG && \ - git checkout FETCH_HEAD) && \ + git checkout FETCH_HEAD && \ + patch -p1 < /tmp/clearwater-heat-add-deps.patch) && \ git init /src/opnfv-vnf-vyos-blueprint && \ (cd /src/opnfv-vnf-vyos-blueprint && \ - git fetch --tags https://github.com/oolorg/opnfv-vnf-vyos-blueprint.git $VROUTER_TAG && \ + git fetch --tags https://github.com/oolorg/opnfv-vnf-vyos-blueprint.git $VROUTER_BP_TAG && \ git checkout FETCH_HEAD) && \ mkdir -p /home/opnfv/functest/data/router && \ git init /home/opnfv/functest/data/router/opnfv-vnf-data && \ @@ -68,7 +70,8 @@ RUN apk --no-cache add --update \ (cd /src/vims-test && bundle config build.nokogiri --use-system-libraries && bundle install --system) && \ rm -r upper-constraints.txt upper-constraints.opnfv.txt /src/vims-test/.git /src/cloudify_vims/.git /src/heat_vims/.git /src/vims-test/quaff/.git \ /src/vims-test/build-infra/.git /src/opnfv-vnf-vyos-blueprint/.git \ - /src/epc-requirements/abot_charm/.git && \ + /src/epc-requirements/abot_charm/.git /root/.cache/go-build \ + /tmp/clearwater-heat-add-deps.patch && \ apk del .build-deps COPY testcases.yaml /usr/lib/python2.7/site-packages/xtesting/ci/testcases.yaml CMD ["run_tests", "-t", "all"] diff --git a/docker/vnf/clearwater-heat-add-deps.patch b/docker/vnf/clearwater-heat-add-deps.patch new file mode 100644 index 000000000..4c9dd143e --- /dev/null +++ b/docker/vnf/clearwater-heat-add-deps.patch @@ -0,0 +1,126 @@ +diff --git a/clearwater.yaml b/clearwater.yaml +index a155c60..1de2e0f 100644 +--- a/clearwater.yaml ++++ b/clearwater.yaml +@@ -185,6 +185,7 @@ resources: + + ellis: + type: ./ellis.yaml ++ depends_on: dns + properties: + public_mgmt_net_id: { get_param: public_mgmt_net_id } + private_mgmt_net_id: { get_attr: [ mgmt_network, private_net ] } +@@ -202,6 +203,7 @@ resources: + + bono: + type: OS::Heat::ResourceGroup ++ depends_on: ellis + properties: + count: { get_param: bono_cluster_size } + index_var: "__index__" +@@ -229,6 +231,7 @@ resources: + + sprout: + type: OS::Heat::ResourceGroup ++ depends_on: ellis + properties: + count: { get_param: sprout_cluster_size } + index_var: __index__ +@@ -257,6 +260,7 @@ resources: + + homer: + type: OS::Heat::ResourceGroup ++ depends_on: ellis + properties: + count: { get_param: homer_cluster_size } + index_var: __index__ +@@ -285,6 +289,7 @@ resources: + + dime: + type: OS::Heat::ResourceGroup ++ depends_on: ellis + properties: + count: { get_param: dime_cluster_size } + index_var: __index__ +@@ -313,6 +318,7 @@ resources: + + vellum: + type: OS::Heat::ResourceGroup ++ depends_on: ellis + properties: + count: { get_param: vellum_cluster_size } + index_var: __index__ +diff --git a/dns.yaml b/dns.yaml +index 825ede1..3e6c938 100644 +--- a/dns.yaml ++++ b/dns.yaml +@@ -91,6 +91,16 @@ resources: + security_groups: + - { get_param: dns_security_group } + ++ wait_condition: ++ type: OS::Heat::WaitCondition ++ properties: ++ handle: {get_resource: wait_handle} ++ count: 1 ++ timeout: 600 ++ ++ wait_handle: ++ type: OS::Heat::WaitConditionHandle ++ + server: + type: OS::Nova::Server + properties: +@@ -110,6 +120,7 @@ resources: + __public_ip__: { get_attr: [ mgmt_floating_ip, floating_ip_address ] } + __private_sig_ip__: { get_attr: [ sig_port, fixed_ips, 0, ip_address ] } + __private_sig_cidr__: { get_param: private_sig_net_cidr } ++ wc_notify: { get_attr: [wait_handle, curl_cli] } + template: | + #!/bin/bash + +@@ -162,6 +173,7 @@ resources: + + # Now that BIND configuration is correct, kick it to reload. + service bind9 reload ++ wc_notify --data-binary '{"status": "SUCCESS"}' + + outputs: + public_mgmt_ip: +diff --git a/ellis.yaml b/ellis.yaml +index 963352d..d39c235 100644 +--- a/ellis.yaml ++++ b/ellis.yaml +@@ -103,6 +103,16 @@ resources: + floating_network_id: { get_param: public_mgmt_net_id } + port_id: { get_resource: mgmt_port } + ++ wait_condition: ++ type: OS::Heat::WaitCondition ++ properties: ++ handle: {get_resource: wait_handle} ++ count: 1 ++ timeout: 600 ++ ++ wait_handle: ++ type: OS::Heat::WaitConditionHandle ++ + server: + type: OS::Nova::Server + properties: +@@ -126,6 +136,7 @@ resources: + __dnssec_key__: { get_param: dnssec_key } + __etcd_ip__ : { get_param: etcd_ip } + __index__ : { get_param: index } ++ wc_notify: { get_attr: [wait_handle, curl_cli] } + template: | + #!/bin/bash + +@@ -227,6 +238,7 @@ resources: + echo 'nameserver __dns_mgmt_ip__' > /etc/dnsmasq.resolv.conf + echo 'RESOLV_CONF=/etc/dnsmasq.resolv.conf' >> /etc/default/dnsmasq + service dnsmasq force-reload ++ wc_notify --data-binary '{"status": "SUCCESS"}' + + outputs: + public_mgmt_ip: diff --git a/docker/vnf/testcases.yaml b/docker/vnf/testcases.yaml index 9822a515a..7d5548b38 100644 --- a/docker/vnf/testcases.yaml +++ b/docker/vnf/testcases.yaml @@ -2,7 +2,7 @@ tiers: - name: vnf - order: 5 + order: 4 description: >- Collection of VNF test cases. testcases: diff --git a/functest/ci/download_images.sh b/functest/ci/download_images.sh index b4abc532e..9044bfe99 100644 --- a/functest/ci/download_images.sh +++ b/functest/ci/download_images.sh @@ -10,11 +10,11 @@ http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-amd64-disk1.img https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2 https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img -http://repository.cloudifysource.org/cloudify/4.0.1/sp-release/cloudify-manager-premium-4.0.1.qcow2 http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-aarch64-disk.img https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-arm64-uefi1.img http://cloud.centos.org/altarch/7/images/aarch64/CentOS-7-aarch64-GenericCloud.qcow2.xz -https://sourceforge.net/projects/ool-opnfv/files/vyos-1.1.7.img +http://repository.cloudifysource.org/cloudify/19.01.24/community-release/cloudify-docker-manager-community-19.01.24.tar +http://testresults.opnfv.org/functest/vyos-1.1.8-amd64.qcow2 http://testresults.opnfv.org/functest/shaker-image.qcow2 http://testresults.opnfv.org/functest/shaker-image-arm64.qcow2 EOF diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 15acaadd8..3e515bf8d 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -172,12 +172,6 @@ tiers: run: name: snaps_health_check - - - name: smoke - order: 1 - description: >- - Set of basic Functional tests to validate the OPNFV scenarios. - testcases: - case_name: tempest_smoke project_name: functest @@ -196,6 +190,12 @@ tiers: option: - '--concurrency=4' + - + name: smoke + order: 1 + description: >- + Set of basic Functional tests to validate the OPNFV scenarios. + testcases: - case_name: neutron-tempest-plugin-api project_name: functest @@ -223,33 +223,59 @@ tiers: Rally suite in smoke mode. run: name: rally_sanity + args: + optional: + - 'gnocchi' - - case_name: rally_jobs + case_name: refstack_defcore project_name: functest criteria: 100 blocking: false description: >- - This test case runs a group of Rally jobs used in - OpenStack gating + This test case runs a sub group of tests of the OpenStack + Defcore testcases. run: - name: rally_jobs + name: refstack_defcore + args: + option: + - '--concurrency=4' - - case_name: refstack_defcore + case_name: tempest_full project_name: functest criteria: 100 blocking: false description: >- - This test case runs a sub group of tests of the OpenStack - Defcore testcases. + The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83 run: - name: refstack_defcore + name: tempest_common args: + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' option: - '--concurrency=4' - + case_name: tempest_scenario + project_name: functest + criteria: 100 + blocking: false + description: >- + The list of test cases is generated by + Tempest automatically and depends on the parameters of + the OpenStack deployment. + https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84 + run: + name: tempest_common + args: + mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' + option: + - '--concurrency=1' + + - case_name: patrole project_name: functest criteria: 100 @@ -315,7 +341,7 @@ tiers: run: name: tempest_common args: - mode: '^networking_bgpvpn_tempest\.' + mode: '^networking_bgpvpn_tempest.tests.api\.' neutron_extensions: - bgpvpn option: @@ -364,6 +390,34 @@ tiers: https://docs.openstack.org/performance-docs/latest/methodologies/tools.html testcases: - + case_name: rally_full + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs the full suite of scenarios of the + OpenStack Rally suite using several threads and iterations. + run: + name: rally_full + args: + optional: + - 'gnocchi' + + - + case_name: rally_jobs + project_name: functest + criteria: 100 + blocking: false + description: >- + This test case runs a group of Rally jobs used in + OpenStack gating + run: + name: rally_jobs + args: + optional: + - 'gnocchi' + + - case_name: vmtp project_name: functest criteria: 100 @@ -481,59 +535,8 @@ tiers: name: stor4nfv_os - - name: components - order: 4 - description: >- - Extensive testing of OpenStack API. - testcases: - - - case_name: tempest_full - project_name: functest - criteria: 100 - blocking: false - description: >- - The list of test cases is generated by - Tempest automatically and depends on the parameters of - the OpenStack deployment. - https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L83 - run: - name: tempest_common - args: - mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.api)' - option: - - '--concurrency=4' - - - - case_name: tempest_scenario - project_name: functest - criteria: 100 - blocking: false - description: >- - The list of test cases is generated by - Tempest automatically and depends on the parameters of - the OpenStack deployment. - https://github.com/openstack/tempest/blob/18.0.0/tox.ini#L84 - run: - name: tempest_common - args: - mode: '(?!.*\[.*\bslow\b.*\])(^tempest\.scenario)' - option: - - '--concurrency=1' - - - - case_name: rally_full - project_name: functest - criteria: 100 - blocking: false - description: >- - This test case runs the full suite of scenarios of the - OpenStack Rally suite using several threads and iterations. - run: - name: rally_full - - - name: vnf - order: 5 + order: 4 description: >- Collection of VNF test cases. testcases: diff --git a/functest/core/cloudify.py b/functest/core/cloudify.py index 83dd603ad..c5613cab5 100644 --- a/functest/core/cloudify.py +++ b/functest/core/cloudify.py @@ -12,11 +12,13 @@ from __future__ import division import logging +import os import time import traceback from cloudify_rest_client import CloudifyClient from cloudify_rest_client.executions import Execution +import scp from functest.core import singlevm @@ -27,15 +29,19 @@ class Cloudify(singlevm.SingleVm2): __logger = logging.getLogger(__name__) filename = ('/home/opnfv/functest/images/' - 'cloudify-manager-premium-4.0.1.qcow2') + 'ubuntu-16.04-server-cloudimg-amd64-disk1.img') flavor_ram = 4096 flavor_vcpus = 2 flavor_disk = 40 - username = 'centos' + username = 'ubuntu' ssh_connect_loops = 12 create_server_timeout = 600 ports = [80, 443, 5671, 53333] + cloudify_archive = ('/home/opnfv/functest/images/' + 'cloudify-docker-manager-community-19.01.24.tar') + cloudify_container = "docker-cfy-manager:latest" + def __init__(self, **kwargs): """Initialize Cloudify testcase object.""" if "case_name" not in kwargs: @@ -54,14 +60,29 @@ class Cloudify(singlevm.SingleVm2): """ Deploy Cloudify Manager. """ + scpc = scp.SCPClient(self.ssh.get_transport()) + scpc.put(self.cloudify_archive, + remote_path=os.path.basename(self.cloudify_archive)) + (_, stdout, stderr) = self.ssh.exec_command( + "sudo wget https://get.docker.com/ -O script.sh && " + "sudo chmod +x script.sh && " + "sudo ./script.sh && " + "sudo docker load -i ~/{} && " + "sudo docker run --name cfy_manager_local -d " + "--restart unless-stopped -v /sys/fs/cgroup:/sys/fs/cgroup:ro " + "--tmpfs /run --tmpfs /run/lock --security-opt seccomp:unconfined " + "--cap-add SYS_ADMIN --network=host {}".format( + os.path.basename(self.cloudify_archive), + self.cloudify_container)) + self.__logger.debug("output:\n%s", stdout.read()) + self.__logger.debug("error:\n%s", stderr.read()) self.cfy_client = CloudifyClient( host=self.fip.floating_ip_address, - username='admin', password='admin', tenant='default_tenant', - api_version='v3') + username='admin', password='admin', tenant='default_tenant') self.__logger.info("Attemps running status of the Manager") secret_key = "foo" secret_value = "bar" - for loop in range(10): + for loop in range(20): try: self.__logger.debug( "status %s", self.cfy_client.manager.get_status()) diff --git a/functest/core/singlevm.py b/functest/core/singlevm.py index ec7f967ec..16c066ef6 100644 --- a/functest/core/singlevm.py +++ b/functest/core/singlevm.py @@ -353,7 +353,7 @@ class SingleVm1(VmReady1): self.keypair = self.cloud.create_keypair( '{}-kp_{}'.format(self.case_name, self.guid)) self.__logger.debug("keypair: %s", self.keypair) - self.__logger.debug("private_key: %s", self.keypair.private_key) + self.__logger.debug("private_key:\n%s", self.keypair.private_key) with open(self.key_filename, 'w') as private_key_file: private_key_file.write(self.keypair.private_key) self.sec = self.cloud.create_security_group( diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index 7ff6cd2f8..a6ec148cf 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -11,7 +11,9 @@ """Rally testcases implementation.""" from __future__ import division +from __future__ import print_function +import fileinput import json import logging import os @@ -28,7 +30,6 @@ from xtesting.core import testcase import yaml from functest.core import singlevm -from functest.opnfv_tests.openstack.tempest import conf_utils from functest.utils import config from functest.utils import env @@ -38,10 +39,13 @@ LOGGER = logging.getLogger(__name__) class RallyBase(singlevm.VmReady2): """Base class form Rally testcases implementation.""" - # pylint: disable=too-many-instance-attributes + # pylint: disable=too-many-instance-attributes, too-many-public-methods TESTS = ['authenticate', 'glance', 'cinder', 'gnocchi', 'heat', 'keystone', 'neutron', 'nova', 'quotas'] + RALLY_CONF_PATH = "/etc/rally/rally.conf" + RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename( + 'functest', 'ci/rally_aarch64_patch.conf') RALLY_DIR = pkg_resources.resource_filename( 'functest', 'opnfv_tests/openstack/rally') RALLY_SCENARIO_DIR = pkg_resources.resource_filename( @@ -83,7 +87,6 @@ class RallyBase(singlevm.VmReady2): self.summary = [] self.scenario_dir = '' self.smoke = None - self.test_name = None self.start_time = None self.result = None self.details = None @@ -147,6 +150,57 @@ class RallyBase(singlevm.VmReady2): return test_file_name @staticmethod + def get_verifier_deployment_id(): + """ + Returns deployment id for active Rally deployment + """ + cmd = ("rally deployment list | awk '/" + + getattr(config.CONF, 'rally_deployment_name') + + "/ {print $2}'") + proc = subprocess.Popen(cmd, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + deployment_uuid = proc.stdout.readline().rstrip() + return deployment_uuid + + @staticmethod + def create_rally_deployment(environ=None): + """Create new rally deployment""" + # set the architecture to default + pod_arch = env.get("POD_ARCH") + arch_filter = ['aarch64'] + + if pod_arch and pod_arch in arch_filter: + LOGGER.info("Apply aarch64 specific to rally config...") + with open(RallyBase.RALLY_AARCH64_PATCH_PATH, "r") as pfile: + rally_patch_conf = pfile.read() + + for line in fileinput.input(RallyBase.RALLY_CONF_PATH): + print(line, end=' ') + if "cirros|testvm" in line: + print(rally_patch_conf) + + LOGGER.info("Creating Rally environment...") + try: + cmd = ['rally', 'deployment', 'destroy', + '--deployment', + str(getattr(config.CONF, 'rally_deployment_name'))] + output = subprocess.check_output(cmd) + LOGGER.info("%s\n%s", " ".join(cmd), output) + except subprocess.CalledProcessError: + pass + + cmd = ['rally', 'deployment', 'create', '--fromenv', + '--name', str(getattr(config.CONF, 'rally_deployment_name'))] + output = subprocess.check_output(cmd, env=environ) + LOGGER.info("%s\n%s", " ".join(cmd), output) + + cmd = ['rally', 'deployment', 'check'] + output = subprocess.check_output(cmd) + LOGGER.info("%s\n%s", " ".join(cmd), output) + return RallyBase.get_verifier_deployment_id() + + @staticmethod def update_keystone_default_role(rally_conf='/etc/rally/rally.conf'): """Set keystone_default_role in rally.conf""" if env.get("NEW_USER_ROLE").lower() != "member": @@ -349,15 +403,6 @@ class RallyBase(singlevm.VmReady2): output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) LOGGER.info("%s\n%s", " ".join(cmd), output) - # save report as HTML - report_html_name = '{}.html'.format(test_name) - report_html_dir = os.path.join(self.results_dir, report_html_name) - cmd = (["rally", "task", "report", "--html", "--uuid", task_id, - "--out", report_html_dir]) - LOGGER.debug('running command: %s', cmd) - output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) - LOGGER.info("%s\n%s", " ".join(cmd), output) - json_results = open(report_json_dir).read() self._append_summary(json_results, test_name) @@ -388,10 +433,13 @@ class RallyBase(singlevm.VmReady2): nb_tests = 0 nb_success = 0 overall_duration = 0.0 + success = [] + failures = [] rally_report = json.loads(json_raw) for task in rally_report.get('tasks'): for subtask in task.get('subtasks'): + has_errors = False for workload in subtask.get('workloads'): if workload.get('full_duration'): overall_duration += workload.get('full_duration') @@ -402,24 +450,32 @@ class RallyBase(singlevm.VmReady2): for result in workload.get('data'): if not result.get('error'): nb_success += 1 + else: + has_errors = True + + if has_errors: + failures.append(subtask['title']) + else: + success.append(subtask['title']) scenario_summary = {'test_name': test_name, 'overall_duration': overall_duration, 'nb_tests': nb_tests, 'nb_success': nb_success, + 'success': success, + 'failures': failures, 'task_status': self.task_succeed(json_raw)} self.summary.append(scenario_summary) - def prepare_run(self): + def prepare_run(self, **kwargs): """Prepare resources needed by test scenarios.""" assert self.cloud - LOGGER.debug('Validating the test name...') - if self.test_name == 'all': - self.tests = self.TESTS - elif self.test_name in self.TESTS: - self.tests = [self.test_name] - else: - raise Exception("Test name '%s' is invalid" % self.test_name) + LOGGER.debug('Validating run tests...') + for test in kwargs.get('tests', self.TESTS): + if test in self.TESTS: + self.tests.append(test) + else: + raise Exception("Test name '%s' is invalid" % test) if not os.path.exists(self.TASK_DIR): os.makedirs(self.TASK_DIR) @@ -504,7 +560,9 @@ class RallyBase(singlevm.VmReady2): payload.append({'module': item['test_name'], 'details': {'duration': item['overall_duration'], 'nb tests': item['nb_tests'], - 'success': success_str}}) + 'success rate': success_str, + 'success': item['success'], + 'failures': item['failures']}}) total_duration_str = time.strftime("%H:%M:%S", time.gmtime(total_duration)) @@ -527,6 +585,40 @@ class RallyBase(singlevm.VmReady2): 'nb success': success_rate}}) self.details = payload + @staticmethod + def export_task(file_name, export_type="html"): + """Export all task results (e.g. html or xunit report) + + Raises: + subprocess.CalledProcessError: if Rally doesn't return 0 + + Returns: + None + """ + cmd = ["rally", "task", "export", "--type", export_type, + "--deployment", + str(getattr(config.CONF, 'rally_deployment_name')), + "--to", file_name] + LOGGER.debug('running command: %s', cmd) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + LOGGER.info("%s\n%s", " ".join(cmd), output) + + @staticmethod + def verify_report(file_name, uuid, export_type="html"): + """Generate the verifier report (e.g. html or xunit report) + + Raises: + subprocess.CalledProcessError: if Rally doesn't return 0 + + Returns: + None + """ + cmd = ["rally", "verify", "report", "--type", export_type, + "--uuid", uuid, "--to", file_name] + LOGGER.debug('running command: %s', cmd) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + LOGGER.info("%s\n%s", " ".join(cmd), output) + def clean(self): """Cleanup of OpenStack resources. Should be called on completion.""" self.clean_rally_conf() @@ -559,10 +651,15 @@ class RallyBase(singlevm.VmReady2): del environ['OS_TENANT_ID'] except Exception: # pylint: disable=broad-except pass - conf_utils.create_rally_deployment(environ=environ) - self.prepare_run() + self.create_rally_deployment(environ=environ) + self.prepare_run(**kwargs) self.run_tests(**kwargs) self._generate_report() + self.export_task( + "{}/{}.html".format(self.results_dir, self.case_name)) + self.export_task( + "{}/{}.xml".format(self.results_dir, self.case_name), + export_type="junit-xml") res = testcase.TestCase.EX_OK except Exception as exc: # pylint: disable=broad-except LOGGER.error('Error with run: %s', exc) @@ -580,7 +677,6 @@ class RallySanity(RallyBase): if "case_name" not in kwargs: kwargs["case_name"] = "rally_sanity" super(RallySanity, self).__init__(**kwargs) - self.test_name = 'all' self.smoke = True self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'sanity') @@ -593,7 +689,6 @@ class RallyFull(RallyBase): if "case_name" not in kwargs: kwargs["case_name"] = "rally_full" super(RallyFull, self).__init__(**kwargs) - self.test_name = 'all' self.smoke = False self.scenario_dir = os.path.join(self.RALLY_SCENARIO_DIR, 'full') @@ -608,20 +703,20 @@ class RallyJobs(RallyBase): if "case_name" not in kwargs: kwargs["case_name"] = "rally_jobs" super(RallyJobs, self).__init__(**kwargs) - self.test_name = 'all' self.task_file = os.path.join(self.RALLY_DIR, 'rally_jobs.yaml') self.task_yaml = None - def prepare_run(self): + def prepare_run(self, **kwargs): """Create resources needed by test scenarios.""" - super(RallyJobs, self).prepare_run() + super(RallyJobs, self).prepare_run(**kwargs) with open(os.path.join(self.RALLY_DIR, 'rally_jobs.yaml'), 'r') as task_file: self.task_yaml = yaml.safe_load(task_file) - if not all(task in self.task_yaml for task in self.tests): - raise Exception("Test '%s' not in '%s'" % - (self.test_name, self.tests)) + for task in self.task_yaml: + if task not in self.tests: + raise Exception("Test '%s' not in '%s'" % + (task, self.tests)) def apply_blacklist(self, case_file_name, result_file_name): # pylint: disable=too-many-branches diff --git a/functest/opnfv_tests/openstack/tempest/conf_utils.py b/functest/opnfv_tests/openstack/tempest/conf_utils.py index 12671d4c1..d490ab036 100644 --- a/functest/opnfv_tests/openstack/tempest/conf_utils.py +++ b/functest/opnfv_tests/openstack/tempest/conf_utils.py @@ -10,11 +10,8 @@ """Tempest configuration utilities.""" -from __future__ import print_function - import json import logging -import fileinput import os import subprocess @@ -27,9 +24,6 @@ from functest.utils import env from functest.utils import functest_utils -RALLY_CONF_PATH = "/etc/rally/rally.conf" -RALLY_AARCH64_PATCH_PATH = pkg_resources.resource_filename( - 'functest', 'ci/rally_aarch64_patch.conf') GLANCE_IMAGE_PATH = os.path.join( getattr(config.CONF, 'dir_functest_images'), getattr(config.CONF, 'openstack_image_file_name')) @@ -46,44 +40,6 @@ CI_INSTALLER_TYPE = env.get('INSTALLER_TYPE') LOGGER = logging.getLogger(__name__) -def create_rally_deployment(environ=None): - """Create new rally deployment""" - # set the architecture to default - pod_arch = env.get("POD_ARCH") - arch_filter = ['aarch64'] - - if pod_arch and pod_arch in arch_filter: - LOGGER.info("Apply aarch64 specific to rally config...") - with open(RALLY_AARCH64_PATCH_PATH, "r") as pfile: - rally_patch_conf = pfile.read() - - for line in fileinput.input(RALLY_CONF_PATH): - print(line, end=' ') - if "cirros|testvm" in line: - print(rally_patch_conf) - - LOGGER.info("Creating Rally environment...") - - try: - cmd = ['rally', 'deployment', 'destroy', - '--deployment', - str(getattr(config.CONF, 'rally_deployment_name'))] - output = subprocess.check_output(cmd) - LOGGER.info("%s\n%s", " ".join(cmd), output) - except subprocess.CalledProcessError: - pass - - cmd = ['rally', 'deployment', 'create', '--fromenv', - '--name', str(getattr(config.CONF, 'rally_deployment_name'))] - output = subprocess.check_output(cmd, env=environ) - LOGGER.info("%s\n%s", " ".join(cmd), output) - - cmd = ['rally', 'deployment', 'check'] - output = subprocess.check_output(cmd) - LOGGER.info("%s\n%s", " ".join(cmd), output) - return get_verifier_deployment_id() - - def create_verifier(): """Create new verifier""" LOGGER.info("Create verifier from existing repo...") @@ -119,20 +75,6 @@ def get_verifier_id(): return verifier_uuid -def get_verifier_deployment_id(): - """ - Returns deployment id for active Rally deployment - """ - cmd = ("rally deployment list | awk '/" + - getattr(config.CONF, 'rally_deployment_name') + - "/ {print $2}'") - proc = subprocess.Popen(cmd, shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - deployment_uuid = proc.stdout.readline().rstrip() - return deployment_uuid - - def get_verifier_repo_dir(verifier_id): """ Returns installed verifier repo directory for Tempest diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index d7e3f243d..597c711fa 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -24,6 +24,7 @@ from xtesting.core import testcase import yaml from functest.core import singlevm +from functest.opnfv_tests.openstack.rally import rally from functest.opnfv_tests.openstack.tempest import conf_utils from functest.utils import config from functest.utils import env @@ -260,7 +261,7 @@ class TempestCommon(singlevm.VmReady2): return with open(os.path.join(self.res_dir, - "tempest.log"), 'r') as logfile: + "rally.log"), 'r') as logfile: output = logfile.read() success_testcases = [] @@ -272,7 +273,7 @@ class TempestCommon(singlevm.VmReady2): output): failed_testcases.append(match) skipped_testcases = [] - for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip:', + for match in re.findall(r'.*\{\d{1,2}\} (.*?) \.{3} skip(?::| )', output): skipped_testcases.append(match) @@ -289,14 +290,6 @@ class TempestCommon(singlevm.VmReady2): LOGGER.info("Tempest %s success_rate is %s%%", self.case_name, self.result) - def generate_report(self): - """Generate verification report.""" - html_file = os.path.join(self.res_dir, - "tempest-report.html") - cmd = ["rally", "verify", "report", "--type", "html", "--uuid", - self.verification_id, "--to", str(html_file)] - subprocess.check_output(cmd) - def update_rally_regex(self, rally_conf='/etc/rally/rally.conf'): """Set image name as tempest img_name_regex""" rconfig = configparser.RawConfigParser() @@ -420,7 +413,7 @@ class TempestCommon(singlevm.VmReady2): del environ['OS_TENANT_ID'] except Exception: # pylint: disable=broad-except pass - self.deployment_id = conf_utils.create_rally_deployment( + self.deployment_id = rally.RallyBase.create_rally_deployment( environ=environ) if not self.deployment_id: raise Exception("Deployment create failed") @@ -471,7 +464,12 @@ class TempestCommon(singlevm.VmReady2): self.apply_tempest_blacklist() self.run_verifier_tests(**kwargs) self.parse_verifier_result() - self.generate_report() + rally.RallyBase.verify_report( + os.path.join(self.res_dir, "tempest-report.html"), + self.verification_id) + rally.RallyBase.verify_report( + os.path.join(self.res_dir, "tempest-report.xml"), + self.verification_id, "junit-xml") res = testcase.TestCase.EX_OK except Exception: # pylint: disable=broad-except LOGGER.exception('Error with run') diff --git a/functest/opnfv_tests/vnf/epc/juju_epc.py b/functest/opnfv_tests/vnf/epc/juju_epc.py index 3830907a6..102f77d96 100644 --- a/functest/opnfv_tests/vnf/epc/juju_epc.py +++ b/functest/opnfv_tests/vnf/epc/juju_epc.py @@ -79,7 +79,7 @@ class JujuEpc(singlevm.VmReady2): flavor_alt_vcpus = 1 flavor_alt_disk = 10 - juju_timeout = '3600' + juju_timeout = '4800' def __init__(self, **kwargs): if "case_name" not in kwargs: diff --git a/functest/opnfv_tests/vnf/ims/clearwater.py b/functest/opnfv_tests/vnf/ims/clearwater.py index cc4d4be04..605ba8a2a 100644 --- a/functest/opnfv_tests/vnf/ims/clearwater.py +++ b/functest/opnfv_tests/vnf/ims/clearwater.py @@ -83,7 +83,7 @@ class ClearwaterTesting(object): return output_dict def _create_ellis_account(self, account_url, params): - i = 50 + i = 80 for iloop in range(i): try: req = requests.post(account_url, data=params) @@ -98,7 +98,7 @@ class ClearwaterTesting(object): except Exception: # pylint: disable=broad-except self.logger.info( "try %s: cannot create ellis account", iloop + 1) - time.sleep(25) + time.sleep(30) raise Exception( "Unable to create an account {}".format( params.get('full_name'))) diff --git a/functest/opnfv_tests/vnf/ims/cloudify_ims.py b/functest/opnfv_tests/vnf/ims/cloudify_ims.py index bbb19694a..77a4eaa74 100644 --- a/functest/opnfv_tests/vnf/ims/cloudify_ims.py +++ b/functest/opnfv_tests/vnf/ims/cloudify_ims.py @@ -36,14 +36,20 @@ class CloudifyIms(cloudify.Cloudify): filename_alt = ('/home/opnfv/functest/images/' 'ubuntu-14.04-server-cloudimg-amd64-disk1.img') - flavor_alt_ram = 2048 - flavor_alt_vcpus = 2 - flavor_alt_disk = 25 + flavor_alt_ram = 1024 + flavor_alt_vcpus = 1 + flavor_alt_disk = 3 quota_security_group = 20 quota_security_group_rule = 100 quota_port = 50 + cop_yaml = ("https://github.com/cloudify-cosmo/cloudify-openstack-plugin/" + "releases/download/2.14.7/plugin.yaml") + cop_wgn = ("https://github.com/cloudify-cosmo/cloudify-openstack-plugin/" + "releases/download/2.14.7/cloudify_openstack_plugin-2.14.7-py27" + "-none-linux_x86_64-centos-Core.wgn") + def __init__(self, **kwargs): """Initialize CloudifyIms testcase object.""" if "case_name" not in kwargs: @@ -152,10 +158,14 @@ class CloudifyIms(cloudify.Cloudify): scpc = scp.SCPClient(self.ssh.get_transport()) scpc.put(self.key_filename, remote_path='~/cloudify_ims.pem') (_, stdout, stderr) = self.ssh.exec_command( - "sudo cp ~/cloudify_ims.pem /etc/cloudify/ && " - "sudo chmod 444 /etc/cloudify/cloudify_ims.pem && " - "sudo yum install -y gcc python-devel python-cmd2 && " - "cfy status") + "sudo docker exec cfy_manager_local " + "cfy plugins upload -y {} {} && " + "sudo docker cp ~/cloudify_ims.pem " + "cfy_manager_local:/etc/cloudify/ && " + "sudo docker exec cfy_manager_local " + "chmod 444 /etc/cloudify/cloudify_ims.pem && " + "sudo docker exec cfy_manager_local cfy status".format( + self.cop_yaml, self.cop_wgn)) self.__logger.info("output:\n%s", stdout.read()) self.__logger.info("error:\n%s", stderr.read()) @@ -293,6 +303,7 @@ class CloudifyIms(cloudify.Cloudify): self.cfy_client, execution, self.__logger) self.cfy_client.deployments.delete( self.vnf['descriptor'].get('name')) + time.sleep(10) self.cfy_client.blueprints.delete( self.vnf['descriptor'].get('name')) except Exception: # pylint: disable=broad-except diff --git a/functest/opnfv_tests/vnf/ims/heat_ims.py b/functest/opnfv_tests/vnf/ims/heat_ims.py index 7bd4f207e..0caa2b00d 100644 --- a/functest/opnfv_tests/vnf/ims/heat_ims.py +++ b/functest/opnfv_tests/vnf/ims/heat_ims.py @@ -37,14 +37,24 @@ class HeatIms(singlevm.VmReady2): filename = ('/home/opnfv/functest/images/' 'ubuntu-14.04-server-cloudimg-amd64-disk1.img') - flavor_ram = 2048 - flavor_vcpus = 2 - flavor_disk = 25 + flavor_ram = 1024 + flavor_vcpus = 1 + flavor_disk = 3 quota_security_group = 20 quota_security_group_rule = 100 quota_port = 50 + parameters = { + 'private_mgmt_net_cidr': '192.168.100.0/24', + 'private_mgmt_net_gateway': '192.168.100.254', + 'private_mgmt_net_pool_start': '192.168.100.1', + 'private_mgmt_net_pool_end': '192.168.100.253', + 'private_sig_net_cidr': '192.168.101.0/24', + 'private_sig_net_gateway': '192.168.101.254', + 'private_sig_net_pool_start': '192.168.101.1', + 'private_sig_net_pool_end': '192.168.101.253'} + def __init__(self, **kwargs): """Initialize HeatIms testcase object.""" if "case_name" not in kwargs: @@ -101,7 +111,7 @@ class HeatIms(singlevm.VmReady2): domain=self.project.domain.id) self.keypair = self.cloud.create_keypair( '{}-kp_{}'.format(self.case_name, self.guid)) - self.__logger.info("keypair: %s", self.keypair.private_key) + self.__logger.info("keypair:\n%s", self.keypair.private_key) if self.deploy_vnf() and self.test_vnf(): self.result = 100 @@ -148,6 +158,7 @@ class HeatIms(singlevm.VmReady2): parameters['key_name'] = self.keypair.name parameters['external_mgmt_dns_ip'] = env.get('NAMESERVER') parameters['external_sig_dns_ip'] = env.get('NAMESERVER') + parameters.update(self.parameters) self.__logger.info("Create Heat Stack") self.stack = self.cloud.create_stack( diff --git a/functest/opnfv_tests/vnf/router/cloudify_vrouter.py b/functest/opnfv_tests/vnf/router/cloudify_vrouter.py index a0145b801..aaaeba0f7 100644 --- a/functest/opnfv_tests/vnf/router/cloudify_vrouter.py +++ b/functest/opnfv_tests/vnf/router/cloudify_vrouter.py @@ -34,11 +34,17 @@ class CloudifyVrouter(cloudify.Cloudify): __logger = logging.getLogger(__name__) - filename_alt = '/home/opnfv/functest/images/vyos-1.1.7.img' + filename_alt = '/home/opnfv/functest/images/vyos-1.1.8-amd64.qcow2' - flavor_alt_ram = 2048 + flavor_alt_ram = 1024 flavor_alt_vcpus = 1 - flavor_alt_disk = 50 + flavor_alt_disk = 3 + + cop_yaml = ("https://github.com/cloudify-cosmo/cloudify-openstack-plugin/" + "releases/download/2.14.7/plugin.yaml") + cop_wgn = ("https://github.com/cloudify-cosmo/cloudify-openstack-plugin/" + "releases/download/2.14.7/cloudify_openstack_plugin-2.14.7-py27" + "-none-linux_x86_64-centos-Core.wgn") def __init__(self, **kwargs): if "case_name" not in kwargs: @@ -118,10 +124,14 @@ class CloudifyVrouter(cloudify.Cloudify): scpc = scp.SCPClient(self.ssh.get_transport()) scpc.put(self.key_filename, remote_path='~/cloudify_ims.pem') (_, stdout, stderr) = self.ssh.exec_command( - "sudo cp ~/cloudify_ims.pem /etc/cloudify/ && " - "sudo chmod 444 /etc/cloudify/cloudify_ims.pem && " - "sudo yum install -y gcc python-devel python-cmd2 && " - "cfy status") + "sudo docker exec cfy_manager_local " + "cfy plugins upload -y {} {} && " + "sudo docker cp ~/cloudify_ims.pem " + "cfy_manager_local:/etc/cloudify/ && " + "sudo docker exec cfy_manager_local " + "chmod 444 /etc/cloudify/cloudify_ims.pem && " + "sudo docker exec cfy_manager_local cfy status".format( + self.cop_yaml, self.cop_wgn)) self.__logger.info("output:\n%s", stdout.read()) self.__logger.info("error:\n%s", stderr.read()) @@ -205,8 +215,7 @@ class CloudifyVrouter(cloudify.Cloudify): def test_vnf(self): start_time = time.time() - testing = vrouter_base.VrouterOnBoardingBase( - self.case_name, self.util, self.util_info) + testing = vrouter_base.VrouterOnBoardingBase(self.util, self.util_info) result, test_result_data = testing.test_vnf() duration = time.time() - start_time if result: @@ -240,6 +249,7 @@ class CloudifyVrouter(cloudify.Cloudify): self.cfy_client, execution, self.__logger) self.cfy_client.deployments.delete( self.vnf['descriptor'].get('name')) + time.sleep(10) self.cfy_client.blueprints.delete( self.vnf['descriptor'].get('name')) except Exception: # pylint: disable=broad-except diff --git a/functest/opnfv_tests/vnf/router/vrouter_base.py b/functest/opnfv_tests/vnf/router/vrouter_base.py index e8a933ff5..8cfab341e 100644 --- a/functest/opnfv_tests/vnf/router/vrouter_base.py +++ b/functest/opnfv_tests/vnf/router/vrouter_base.py @@ -19,7 +19,6 @@ import time import pkg_resources -from functest.utils import config from functest.opnfv_tests.vnf.router.test_controller import function_test_exec __author__ = "Shuya Nakama <shuya.nakama@okinawaopenlabs.org>" @@ -28,19 +27,14 @@ __author__ = "Shuya Nakama <shuya.nakama@okinawaopenlabs.org>" class VrouterOnBoardingBase(object): """vrouter testing base class""" - def __init__(self, case_name, util, util_info): + def __init__(self, util, util_info): self.logger = logging.getLogger(__name__) self.case_dir = pkg_resources.resource_filename( 'functest', 'opnfv_tests/vnf/router') - self.result_dir = os.path.join( - getattr(config.CONF, 'dir_results'), case_name) self.util = util self.util_info = util_info self.vnf_list = [] - if not os.path.exists(self.result_dir): - os.makedirs(self.result_dir) - def test_vnf(self): """vrouter test execution""" result = False diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py index bd691b8ab..5dc38a20f 100644 --- a/functest/tests/unit/openstack/rally/test_rally.py +++ b/functest/tests/unit/openstack/rally/test_rally.py @@ -10,6 +10,7 @@ import json import logging import os +import subprocess import unittest import mock @@ -17,6 +18,7 @@ import munch from xtesting.core import testcase from functest.opnfv_tests.openstack.rally import rally +from functest.utils import config class OSRallyTesting(unittest.TestCase): @@ -30,7 +32,6 @@ class OSRallyTesting(unittest.TestCase): self.rally_base.image = munch.Munch(name='foo') self.rally_base.flavor = munch.Munch(name='foo') self.rally_base.flavor_alt = munch.Munch(name='bar') - self.rally_base.test_name = 'all' self.assertTrue(mock_get_config.called) self.assertTrue(mock_shade.called) self.assertTrue(mock_new_project.called) @@ -68,6 +69,21 @@ class OSRallyTesting(unittest.TestCase): return True return False + @mock.patch('functest.opnfv_tests.openstack.rally.rally.' + 'RallyBase.get_verifier_deployment_id', return_value='foo') + @mock.patch('subprocess.check_output') + def test_create_rally_deployment(self, mock_exec, mock_get_id): + # pylint: disable=unused-argument + self.assertEqual(rally.RallyBase.create_rally_deployment(), 'foo') + calls = [ + mock.call(['rally', 'deployment', 'destroy', '--deployment', + str(getattr(config.CONF, 'rally_deployment_name'))]), + mock.call(['rally', 'deployment', 'create', '--fromenv', '--name', + str(getattr(config.CONF, 'rally_deployment_name'))], + env=None), + mock.call(['rally', 'deployment', 'check'])] + mock_exec.assert_has_calls(calls) + @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists') @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.makedirs') @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' @@ -262,9 +278,8 @@ class OSRallyTesting(unittest.TestCase): def test_prepare_run_testname_invalid(self): self.rally_base.TESTS = ['test1', 'test2'] - self.rally_base.test_name = 'test' with self.assertRaises(Exception): - self.rally_base.prepare_run() + self.rally_base.prepare_run(tests=['test']) @mock.patch('functest.opnfv_tests.openstack.rally.rally.os.path.exists') @mock.patch('functest.opnfv_tests.openstack.rally.rally.shutil.copyfile') @@ -273,14 +288,13 @@ class OSRallyTesting(unittest.TestCase): def test_prepare_run_flavor_alt_creation_failed(self, *args): # pylint: disable=unused-argument self.rally_base.TESTS = ['test1', 'test2'] - self.rally_base.test_name = 'test1' with mock.patch.object(self.rally_base.cloud, 'list_hypervisors') as mock_list_hyperv, \ mock.patch.object(self.rally_base, 'create_flavor_alt', side_effect=Exception) \ as mock_create_flavor: with self.assertRaises(Exception): - self.rally_base.prepare_run() + self.rally_base.prepare_run(tests=['test1']) mock_list_hyperv.assert_called_once() mock_create_flavor.assert_called_once() @@ -290,7 +304,6 @@ class OSRallyTesting(unittest.TestCase): 'run_task') def test_run_tests_all(self, mock_run_task, mock_prepare_task): self.rally_base.tests = ['test1', 'test2'] - self.rally_base.test_name = 'all' self.rally_base.run_tests() mock_prepare_task.assert_any_call('test1') mock_prepare_task.assert_any_call('test2') @@ -316,7 +329,7 @@ class OSRallyTesting(unittest.TestCase): self.rally_base.clean() self.assertEqual(mock_delete_flavor.call_count, 1) - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'create_rally_deployment') @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'prepare_run') @@ -324,18 +337,20 @@ class OSRallyTesting(unittest.TestCase): 'run_tests') @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' '_generate_report') + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' + 'export_task') def test_run_default(self, *args): self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_OK) for func in args: func.assert_called() - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'create_rally_deployment', side_effect=Exception) def test_run_exception_create_rally_dep(self, mock_create_rally_dep): self.assertEqual(self.rally_base.run(), testcase.TestCase.EX_RUN_ERROR) mock_create_rally_dep.assert_called() - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' + @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'create_rally_deployment', return_value=mock.Mock()) @mock.patch('functest.opnfv_tests.openstack.rally.rally.RallyBase.' 'prepare_run', side_effect=Exception) @@ -345,14 +360,31 @@ class OSRallyTesting(unittest.TestCase): mock_prep_env.assert_called() def test_append_summary(self): - text = '{"tasks": [{"subtasks": [{"workloads": [{"full_duration": ' \ - '1.23,"data": [{"error": []}]}]},{"workloads": ' \ - '[{"full_duration": 2.78, "data": [{"error": ["err"]}]}]}]}]}' - self.rally_base._append_summary(text, "foo_test") + json_dict = { + 'tasks': [{ + 'subtasks': [{ + 'title': 'sub_task', + 'workloads': [{ + 'full_duration': 1.23, + 'data': [{ + 'error': [] + }] + }, { + 'full_duration': 2.78, + 'data': [{ + 'error': ['err'] + }] + }] + }] + }] + } + self.rally_base._append_summary(json.dumps(json_dict), "foo_test") self.assertEqual(self.rally_base.summary[0]['test_name'], "foo_test") self.assertEqual(self.rally_base.summary[0]['overall_duration'], 4.01) self.assertEqual(self.rally_base.summary[0]['nb_tests'], 2) self.assertEqual(self.rally_base.summary[0]['nb_success'], 1) + self.assertEqual(self.rally_base.summary[0]['success'], []) + self.assertEqual(self.rally_base.summary[0]['failures'], ['sub_task']) def test_is_successful_false(self): with mock.patch('six.moves.builtins.super') as mock_super: @@ -370,6 +402,48 @@ class OSRallyTesting(unittest.TestCase): self.assertEqual(self.rally_base.is_successful(), 424) mock_super(rally.RallyBase, self).is_successful.assert_called() + @mock.patch('subprocess.check_output', + side_effect=subprocess.CalledProcessError('', '')) + def test_export_task_ko(self, *args): + file_name = "{}/{}.html".format( + self.rally_base.results_dir, self.rally_base.case_name) + with self.assertRaises(subprocess.CalledProcessError): + self.rally_base.export_task(file_name) + cmd = ["rally", "task", "export", "--type", "html", "--deployment", + str(getattr(config.CONF, 'rally_deployment_name')), + "--to", file_name] + args[0].assert_called_with(cmd, stderr=subprocess.STDOUT) + + @mock.patch('subprocess.check_output', return_value=None) + def test_export_task(self, *args): + file_name = "{}/{}.html".format( + self.rally_base.results_dir, self.rally_base.case_name) + self.assertEqual(self.rally_base.export_task(file_name), None) + cmd = ["rally", "task", "export", "--type", "html", "--deployment", + str(getattr(config.CONF, 'rally_deployment_name')), + "--to", file_name] + args[0].assert_called_with(cmd, stderr=subprocess.STDOUT) + + @mock.patch('subprocess.check_output', + side_effect=subprocess.CalledProcessError('', '')) + def test_verify_report_ko(self, *args): + file_name = "{}/{}.html".format( + self.rally_base.results_dir, self.rally_base.case_name) + with self.assertRaises(subprocess.CalledProcessError): + self.rally_base.verify_report(file_name, "1") + cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1", + "--to", file_name] + args[0].assert_called_with(cmd, stderr=subprocess.STDOUT) + + @mock.patch('subprocess.check_output', return_value=None) + def test_verify_report(self, *args): + file_name = "{}/{}.html".format( + self.rally_base.results_dir, self.rally_base.case_name) + self.assertEqual(self.rally_base.verify_report(file_name, "1"), None) + cmd = ["rally", "verify", "report", "--type", "html", "--uuid", "1", + "--to", file_name] + args[0].assert_called_with(cmd, stderr=subprocess.STDOUT) + if __name__ == "__main__": logging.disable(logging.CRITICAL) diff --git a/functest/tests/unit/openstack/tempest/test_conf_utils.py b/functest/tests/unit/openstack/tempest/test_conf_utils.py index 19b07ba3e..f99b02b50 100644 --- a/functest/tests/unit/openstack/tempest/test_conf_utils.py +++ b/functest/tests/unit/openstack/tempest/test_conf_utils.py @@ -13,6 +13,7 @@ import unittest import mock +from functest.opnfv_tests.openstack.rally import rally from functest.opnfv_tests.openstack.tempest import conf_utils from functest.utils import config @@ -21,21 +22,6 @@ class OSTempestConfUtilsTesting(unittest.TestCase): # pylint: disable=too-many-public-methods @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils' - '.get_verifier_deployment_id', return_value='foo') - @mock.patch('subprocess.check_output') - def test_create_rally_deployment(self, mock_exec, mock_get_id): - # pylint: disable=unused-argument - self.assertEqual(conf_utils.create_rally_deployment(), 'foo') - calls = [ - mock.call(['rally', 'deployment', 'destroy', '--deployment', - str(getattr(config.CONF, 'rally_deployment_name'))]), - mock.call(['rally', 'deployment', 'create', '--fromenv', '--name', - str(getattr(config.CONF, 'rally_deployment_name'))], - env=None), - mock.call(['rally', 'deployment', 'check'])] - mock_exec.assert_has_calls(calls) - - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils' '.LOGGER.debug') def test_create_verifier(self, mock_logger_debug): mock_popen = mock.Mock() @@ -52,8 +38,8 @@ class OSTempestConfUtilsTesting(unittest.TestCase): @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' 'create_verifier', return_value=mock.Mock()) - @mock.patch('functest.opnfv_tests.openstack.tempest.conf_utils.' - 'create_rally_deployment', return_value=mock.Mock()) + @mock.patch('functest.opnfv_tests.openstack.rally.rally.' + 'RallyBase.create_rally_deployment', return_value=mock.Mock()) def test_get_verifier_id_default(self, mock_rally, mock_tempest): # pylint: disable=unused-argument setattr(config.CONF, 'tempest_verifier_name', 'test_verifier_name') @@ -76,7 +62,7 @@ class OSTempestConfUtilsTesting(unittest.TestCase): mock_stdout.configure_mock(**attrs) mock_popen.return_value = mock_stdout - self.assertEqual(conf_utils.get_verifier_deployment_id(), + self.assertEqual(rally.RallyBase.get_verifier_deployment_id(), 'test_deploy_id') def test_get_verif_repo_dir_default(self): diff --git a/functest/tests/unit/openstack/tempest/test_tempest.py b/functest/tests/unit/openstack/tempest/test_tempest.py index c1f245c72..646f88290 100644 --- a/functest/tests/unit/openstack/tempest/test_tempest.py +++ b/functest/tests/unit/openstack/tempest/test_tempest.py @@ -16,7 +16,6 @@ from xtesting.core import testcase from functest.opnfv_tests.openstack.tempest import tempest from functest.opnfv_tests.openstack.tempest import conf_utils -from functest.utils import config class OSTempestTesting(unittest.TestCase): @@ -26,15 +25,15 @@ class OSTempestTesting(unittest.TestCase): with mock.patch('os_client_config.get_config'), \ mock.patch('shade.OpenStackCloud'), \ mock.patch('functest.core.tenantnetwork.NewProject'), \ - mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' - 'conf_utils.create_rally_deployment'), \ + mock.patch('functest.opnfv_tests.openstack.rally.rally.' + 'RallyBase.create_rally_deployment'), \ mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'conf_utils.create_verifier'), \ mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'conf_utils.get_verifier_id', return_value='test_deploy_id'), \ - mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' - 'conf_utils.get_verifier_deployment_id', + mock.patch('functest.opnfv_tests.openstack.rally.rally.' + 'RallyBase.get_verifier_deployment_id', return_value='test_deploy_id'), \ mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'conf_utils.get_verifier_repo_dir', @@ -171,19 +170,6 @@ class OSTempestTesting(unittest.TestCase): mock_logger_info. \ assert_any_call("Starting Tempest test suite: '%s'.", cmd) - @mock.patch('subprocess.check_output') - def test_generate_report(self, mock_popen): - self.tempestcommon.verification_id = "1234" - html_file = os.path.join( - os.path.join( - getattr(config.CONF, 'dir_results'), - self.tempestcommon.case_name), - "tempest-report.html") - cmd = ["rally", "verify", "report", "--type", "html", "--uuid", - "1234", "--to", html_file] - self.tempestcommon.generate_report() - mock_popen.assert_called_once_with(cmd) - @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'os.path.exists', return_value=False) @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.os.makedirs', @@ -260,8 +246,7 @@ class OSTempestTesting(unittest.TestCase): 'apply_tempest_blacklist'), \ mock.patch.object(self.tempestcommon, 'run_verifier_tests'), \ mock.patch.object(self.tempestcommon, - 'parse_verifier_result'), \ - mock.patch.object(self.tempestcommon, 'generate_report'): + 'parse_verifier_result'): self._test_run(testcase.TestCase.EX_OK) args[0].assert_called_once_with() diff --git a/upper-constraints.txt b/upper-constraints.txt index 7683b04f4..493ceb744 100644 --- a/upper-constraints.txt +++ b/upper-constraints.txt @@ -10,7 +10,7 @@ git+https://gerrit.opnfv.org/gerrit/clover#egg=clover git+https://gerrit.opnfv.org/gerrit/parser#egg=nfv-heattranslator&subdirectory=tosca2heat/heat-translator git+https://gerrit.opnfv.org/gerrit/parser#egg=nfv-toscaparser&subdirectory=tosca2heat/tosca-parser -e git+https://gerrit.opnfv.org/gerrit/parser#egg=nfv-parser -cloudify-rest-client===4.3.2 +cloudify-rest-client===4.3.3 robotframework===3.0.2 robotframework-httplibrary===0.4.2 robotframework-requests===0.4.7 |