diff options
| -rw-r--r-- | docker/Dockerfile | 11 | ||||
| -rw-r--r-- | docker/Dockerfile.aarch64 | 11 | ||||
| -rwxr-xr-x | docs/com/pres/Summit/Berlin-2016/summit-Berlin.html | 2 | ||||
| -rwxr-xr-x | docs/com/pres/Summit/Berlin-2016/testapi.html | 28 | ||||
| -rw-r--r-- | docs/release/release-notes/functest-release.rst | 6 | ||||
| -rw-r--r-- | docs/testing/developer/devguide/index.rst | 26 | ||||
| -rw-r--r-- | docs/testing/developer/internship/testapi_evolution/index.rst | 40 | ||||
| -rw-r--r-- | docs/testing/user/configguide/configguide.rst | 1 | ||||
| -rwxr-xr-x | functest/ci/exec_test.sh | 148 | ||||
| -rwxr-xr-x | functest/ci/run_tests.py | 23 | ||||
| -rwxr-xr-x | functest/ci/testcases.yaml | 34 | ||||
| -rwxr-xr-x | functest/ci/tier_builder.py | 1 | ||||
| -rwxr-xr-x | functest/ci/tier_handler.py | 13 |
13 files changed, 114 insertions, 230 deletions
diff --git a/docker/Dockerfile b/docker/Dockerfile index de47e157a..38161d1d1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -108,10 +108,10 @@ RUN pip install -r ${REPOS_DIR}/tempest/requirements.txt RUN cd ${FUNCTEST_REPO_DIR} \ && pip install -r requirements.txt \ - && pip install . + && pip install -e . RUN cd ${RELENG_MODULE_DIR} \ - && pip install . + && pip install -e . RUN cd ${REPOS_DIR}/barometer \ && pip install . @@ -133,12 +133,11 @@ RUN pip install -e ${REPOS_DIR}/snaps/ # SFC integration RUN /bin/bash -c ". ${REPOS_DIR}/sfc/sfc/tests/functest/setup_scripts/tacker_client_install.sh" -RUN cd ${REPOS_DIR}/sfc && pip install . +RUN cd ${REPOS_DIR}/sfc && pip install -e . # SDNVPN integration -RUN cd ${REPOS_DIR}/sdnvpn && pip install . - -RUN cd ${REPOS_DIR}/bgpvpn && pip install . +RUN cd ${REPOS_DIR}/sdnvpn && pip install -e . +RUN cd ${REPOS_DIR}/bgpvpn && pip install -e . # Kingbird integration RUN cd ${REPOS_DIR}/kingbird && pip install -e . diff --git a/docker/Dockerfile.aarch64 b/docker/Dockerfile.aarch64 index a469801f3..82add067b 100644 --- a/docker/Dockerfile.aarch64 +++ b/docker/Dockerfile.aarch64 @@ -105,10 +105,10 @@ RUN pip install -r ${REPOS_DIR}/tempest/requirements.txt RUN cd ${FUNCTEST_REPO_DIR} \ && pip install -r requirements.txt \ - && pip install . + && pip install -e . RUN cd ${RELENG_MODULE_DIR} \ - && pip install . + && pip install -e . RUN find ${FUNCTEST_REPO_DIR} -name "*.py" \ -not -path "*tests/unit*" |xargs grep __main__ |cut -d\: -f 1 |xargs chmod -c 755 \ @@ -127,12 +127,11 @@ RUN pip install -e ${REPOS_DIR}/snaps/ # SFC integration RUN /bin/bash -c ". ${REPOS_DIR}/sfc/sfc/tests/functest/setup_scripts/tacker_client_install.sh" -RUN cd ${REPOS_DIR}/sfc && pip install . +RUN cd ${REPOS_DIR}/sfc && pip install -e . # SDNVPN integration -RUN cd ${REPOS_DIR}/sdnvpn && pip install . - -RUN cd ${REPOS_DIR}/bgpvpn && pip install . +RUN cd ${REPOS_DIR}/sdnvpn && pip install -e . +RUN cd ${REPOS_DIR}/bgpvpn && pip install -e . # Kingbird integration RUN cd ${REPOS_DIR}/kingbird && pip install -e . diff --git a/docs/com/pres/Summit/Berlin-2016/summit-Berlin.html b/docs/com/pres/Summit/Berlin-2016/summit-Berlin.html index 4173a86ac..8369443f7 100755 --- a/docs/com/pres/Summit/Berlin-2016/summit-Berlin.html +++ b/docs/com/pres/Summit/Berlin-2016/summit-Berlin.html @@ -230,7 +230,7 @@ ## What's new? * Slicing of the tests (healthcheck / Smoke / SDN controllers / Features / Components / VNFs) * Better Test duration management - * Refactoring (repo, case management, test API) + * Refactoring (repo, case management, TestAPI) * Automatic reporting * Dashboard evolution to ELK * CLI diff --git a/docs/com/pres/Summit/Berlin-2016/testapi.html b/docs/com/pres/Summit/Berlin-2016/testapi.html index 7035d71df..16f97c44d 100755 --- a/docs/com/pres/Summit/Berlin-2016/testapi.html +++ b/docs/com/pres/Summit/Berlin-2016/testapi.html @@ -6,7 +6,7 @@ <title>OPNFV presentation</title> - <meta name="description" content="Test API"> + <meta name="description" content="TestAPI"> <meta name="author" content="Serena Feng"> <meta name="apple-mobile-web-app-capable" content="yes" /> @@ -42,7 +42,7 @@ <section data-background="../../../img/title-bg-berlin.png" data-background-transition="none"> <br><br><br><br><br> - <h1>Test API</h1> + <h1>TestAPI</h1> <h3>Clean, Easy, Complete</h3> <br> <h4>OPNFV testing community</h4> @@ -52,7 +52,7 @@ <section data-markdown> # Agenda - * testAPI: what for? + * TestAPI: what for? * API overview * API evolution * Roadmap @@ -60,7 +60,7 @@ <section> <section data-markdown> - # test API: what for? + # TestAPI: what for? </section> <section data-markdown> ## Consistant view for @@ -78,12 +78,12 @@ </section> <section data-markdown> <script type='text/template'> - ## Achieve using testAPI - * Uniform API: testAPI + ## Achieve using TestAPI + * Uniform API: TestAPI * Uniform format: JSON * Universal Location: http://testresults.opnfv.org <aside class='notes'> - By using testAPI, we hope to provide a uniform way of collection and saving test results to a universal location + By using TestAPI, we hope to provide a uniform way of collection and saving test results to a universal location </aside> </script> </section> @@ -101,11 +101,11 @@ </section> <section data-markdown> ## Storage structure -  +  </section> <section data-markdown> ## API in Brahmaputra -  +  https://wiki.opnfv.org/display/functest/Collection+Of+Test+Results </section> @@ -137,7 +137,7 @@ <aside class='notes'> So you can discover and understand the capabilities of the service without access to source code, documentation, or through network traffic inspection, - and also you can interact with the testAPI directly through swagger website. + and also you can interact with the TestAPI directly through swagger website. </aside> </ul> <p class="fragment fade-up"><b>All done in Colorado!</b></p> @@ -152,7 +152,7 @@ <section> <h2> URI changes...</h2> <div style="text-align:left""> - <p> testresults.opnfv.org/<span style="color:lightblue">testapi</span> => <br>testresults.opnfv.org/<span style="color:yellow">test/api/v1</span> </p> + <p> testresults.opnfv.org/<span style="color:lightblue">TestAPI</span> => <br>testresults.opnfv.org/<span style="color:yellow">test/api/v1</span> </p> <p> /test/api/v1/<b>pods</b></p> <p> /test/api/v1/<b>projects</b></p> @@ -195,7 +195,7 @@ </section> <section data-markdown> http://testresults.opnfv.org/test/swagger/spec.html -  +  </section> <section> <h2>unit tests</h2> @@ -223,8 +223,8 @@ OK </section> <section data-markdown> ## Roadmap - * Dockerize testAPI - * Automatic update of testAPI + * Dockerize TestAPI + * Automatic update of TestAPI * Command Line Interface * Automatic update for pods/projects/testcases </section> diff --git a/docs/release/release-notes/functest-release.rst b/docs/release/release-notes/functest-release.rst index 07389bb5b..ce03047f6 100644 --- a/docs/release/release-notes/functest-release.rst +++ b/docs/release/release-notes/functest-release.rst @@ -32,7 +32,7 @@ Version history +------------+----------+------------------+------------------------+ OPNFV Colorado Release -========================= +====================== Functest deals with functional testing of the OPNFV solution. It includes test cases developed within the project and test cases developed in @@ -99,7 +99,7 @@ Software - The Functest Docker image: https://hub.docker.com/r/opnfv/functest (tag: colorado.1.0) - - The testapi Docker image: https://hub.docker.com/r/opnfv/testapi (tag:colorado.1.0) + - The TestAPI Docker image: https://hub.docker.com/r/opnfv/testapi (tag:colorado.1.0) Documents @@ -120,7 +120,7 @@ Feature evolution - refactoring of ODL functional tests (with upstream modifications) - - refactoring of testapi (update, swagger documentation, dockerization) + - refactoring of TestAPI (update, swagger documentation, dockerization) - jenkins logs improvement diff --git a/docs/testing/developer/devguide/index.rst b/docs/testing/developer/devguide/index.rst index eee013678..11ca241b2 100644 --- a/docs/testing/developer/devguide/index.rst +++ b/docs/testing/developer/devguide/index.rst @@ -14,7 +14,7 @@ Introduction Functest is a project dealing with functional testing. Functest produces its own internal test cases but can also be considered as a framework to support feature and VNF onboarding project testing. -Functest developed a test API and defined a test collection framework +Functest developed a TestAPI and defined a test collection framework that can be used by any OPNFV project. Therefore there are many ways to contribute to Functest. You can: @@ -260,9 +260,9 @@ The Test result management can be summarized as follows:: | | +----------------------+ -Test API description --------------------- -The Test API is used to declare pods, projects, test cases and test +TestAPI description +------------------- +The TestAPI is used to declare pods, projects, test cases and test results. Pods are the pods used to run the tests. The results pushed in the database are related to pods, projects and cases. If you try to push results of test done on non referenced pod, @@ -500,21 +500,21 @@ Scenarios: The code of the API is hosted in the releng repository `[6]`_. The static documentation of the API can be found at `[17]`_. -The test API has been dockerized and may be installed locally in your +The TestAPI has been dockerized and may be installed locally in your lab. See `[15]`_ for details. -The deployment of the test API has been automated. +The deployment of the TestAPI has been automated. A jenkins job manages: - * the unit tests of the test api + * the unit tests of the TestAPI * the creation of a new docker file - * the deployment of the new test api - * the archive of the old test api + * the deployment of the new TestAPI + * the archive of the old TestAPI * the backup of the Mongo DB -Test API Authorization -~~~~~~~~~~~~~~~~~~~~~~ +TestAPI Authorization +~~~~~~~~~~~~~~~~~~~~~ -PUT/DELETE/POST operations of the testapi now require token based authorization. The token needs +PUT/DELETE/POST operations of the TestAPI now require token based authorization. The token needs to be added in the request using a header 'X-Auth-Token' for access to the database. e.g:: @@ -627,7 +627,7 @@ script can be found in `[16]`_. For next versions, it was decided to integrated bitergia dashboard. Bitergia already provides a dashboard for code and infrastructure. A new Test tab will be added. The dataset will be built by consuming -the test API. +the TestAPI. ======= diff --git a/docs/testing/developer/internship/testapi_evolution/index.rst b/docs/testing/developer/internship/testapi_evolution/index.rst index 6a1cde7df..3038d0ac6 100644 --- a/docs/testing/developer/internship/testapi_evolution/index.rst +++ b/docs/testing/developer/internship/testapi_evolution/index.rst @@ -7,9 +7,9 @@ International License. You should have received a copy of the license along with this. If not, see <http://creativecommons.org/licenses/by/4.0/>. -================== -Test API evolution -================== +================= +TestAPI evolution +================= Author: Sakala Venkata Krishna Rohit Mentors: S. Feng, J.Lausuch, M.Richomme @@ -17,10 +17,10 @@ Mentors: S. Feng, J.Lausuch, M.Richomme Abstract ======== -The testapi is used by all the test opnfv projects to report results. +The TestAPI is used by all the test opnfv projects to report results. It is also used to declare projects, test cases and labs. A major refactoring -has been done in Colorado with the introduction of swagger. The testapi is defined in Functest -developer guide. The purpose of this project is to add more features to the testapi that automate +has been done in Colorado with the introduction of swagger. The TestAPI is defined in Functest +developer guide. The purpose of this project is to add more features to the TestAPI that automate the tasks that are done manually now, though there are tasks other than automation. Version history @@ -48,33 +48,33 @@ documentation is as follows. Problem Statement: ------------------ -The problem statement could be divided into pending features that needed to be added into testapi +The problem statement could be divided into pending features that needed to be added into TestAPI repo. The following were to be accomplished within the internship time frame. -* **Add verification jenkins job for the testapi code** +* **Add verification jenkins job for the TestAPI code** The purpose of this job is to verify whehter the unit tests are successful or not with the inclusion of the patchset submitted. * **Automatic update of opnfv/testapi docker image** - The docker image of testapi is hosted in the opnfv docker hub. To ensure that the testapi image + The docker image of TestAPI is hosted in the opnfv docker hub. To ensure that the TestAPI image is always updated with the repository, automatic updation of the image is necessary and a job is triggered whenever a new patch gets merged. * **Automation deployment of testresults.opnfv.org/test/ website** - In the same manner as the docker image of testapi is updated, the testapi website needs to be + In the same manner as the docker image of TestAPI is updated, the TestAPI website needs to be in sync with the repository code. So, a job has been added to the opnfv jenkins ci for the updation of the testresults website. -* **Generate static documentation of testapi calls** - The purpose of this is to give an static/offline view of testapi. If someone wants to have a - look at the Restful apis of testapi, he/she does't need to go to the website, he can download +* **Generate static documentation of TestAPI calls** + The purpose of this is to give an static/offline view of TestAPI. If someone wants to have a + look at the Restful APIs of TestAPI, he/she does't need to go to the website, he can download a html page and view it anytime. -* **Backup MongoDB of testapi** +* **Backup MongoDB of TestAPI** The mongoDB needs to be backed up every week. Till now it was done manually, but due to this internship, it is now automated using a jenkins job. -* **Add token based authorization to the testapi calls** +* **Add token based authorization to the TestAPI calls** The token based authorization was implemented to ensure that only ci_pods could access the database. Authentication has been added to only delete/put/post requests. @@ -82,7 +82,7 @@ Curation Phase: --------------- The curation phase was the first 3 to 4 weeks of the internship. This phase was to get familiar -with the testapi code and functionality and propose the solutions/tools for the tasks mentioned +with the TestAPI code and functionality and propose the solutions/tools for the tasks mentioned above. Swagger codegen was choosen out of the four tools proposed `[3]`_ for generating static documentaion. @@ -107,7 +107,7 @@ The progress and completion of the tasks is described in the below table. | **Date** | **Comment** | | | | +--------------------------+------------------------------------------+ -| Nov 14th - Dec 31st | Understand Testapi code and the | +| Nov 14th - Dec 31st | Understand TestAPI code and the | | | requirements. | +--------------------------+------------------------------------------+ | Jan 1st - Jan 7th | Add jenkins job to create static | @@ -134,8 +134,8 @@ the internship. This section may help other developers in solving any errors cas code written as a part of this internship. -Test Api --------- +TestAPI +------- What is the difference between defining data_file as "/etc/.." and "etc/.." in setup.cfg ? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -216,7 +216,7 @@ defined. What job style should be used when there is a situation like one build should trigger other builds ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ or when different build scripts need to be run on different machines ? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ MultiJob style should be used as it has phases where each phase can be taken as a build scipt and can have its own parameters by which one can define the SLAVE_LABEL parameter. diff --git a/docs/testing/user/configguide/configguide.rst b/docs/testing/user/configguide/configguide.rst index 08e089c2b..1bb8358a8 100644 --- a/docs/testing/user/configguide/configguide.rst +++ b/docs/testing/user/configguide/configguide.rst @@ -392,7 +392,6 @@ follows:: | |-- check_os.sh | |-- config_functest.yaml | |-- config_patch.yaml - | |-- exec_test.sh | |-- generate_report.py | |-- prepare_env.py | |-- run_tests.py diff --git a/functest/ci/exec_test.sh b/functest/ci/exec_test.sh deleted file mode 100755 index 6f776101d..000000000 --- a/functest/ci/exec_test.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/bash - -# -# Author: Jose Lausuch (jose.lausuch@ericsson.com) -# Morgan Richomme (morgan.richomme@orange.com) -# Installs the Functest framework within the Docker container -# and run the tests automatically -# -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# - -usage="Script to trigger the tests automatically. - -usage: - bash $(basename "$0") [-h|--help] [-t <test_name>] - -where: - -h|--help show this help text - -r|--report push results to database (false by default) - -s|--serial run Tempest tests in one thread - -t|--test run specific test case - <test_name>" - - -report="" -serial=false - -# Get the list of runnable tests -# Check if we are in CI mode -debug="" -if [[ "${CI_DEBUG,,}" == "true" ]];then - debug="--debug" -fi - -FUNCTEST_REPO_DIR=${REPOS_DIR}/functest -FUNCTEST_TEST_DIR=${REPOS_DIR}/functest/functest/opnfv_tests -FUNCTEST_CONF_DIR=/home/opnfv/functest/conf - -export PYTHONUNBUFFERED=1 - -function odl_tests(){ - keystone_ip=$(openstack catalog show identity |grep publicURL| cut -f3 -d"/" | cut -f1 -d":") - neutron_ip=$(openstack catalog show network | grep publicURL | cut -f3 -d"/" | cut -f1 -d":") - odl_ip=${neutron_ip} - odl_port=8080 - odl_restport=8181 - if [ "$INSTALLER_TYPE" == "fuel" ]; then - odl_port=8282 - elif [ "$INSTALLER_TYPE" == "apex" ]; then - odl_ip=$SDN_CONTROLLER_IP - odl_port=8081 - odl_restport=8081 - elif [ "$INSTALLER_TYPE" == "netvirt" ]; then - odl_ip=$SDN_CONTROLLER_IP - odl_port=8081 - odl_restport=8081 - elif [ "$INSTALLER_TYPE" == "joid" ]; then - odl_ip=$SDN_CONTROLLER - elif [ "$INSTALLER_TYPE" == "compass" ]; then - odl_port=8181 - else - odl_ip=$SDN_CONTROLLER_IP - fi -} - - - -function run_test(){ - test_name=$1 - serial_flag="" - if [ $serial == "true" ]; then - serial_flag="-s" - fi - - case $test_name in - "healthcheck") - ${FUNCTEST_TEST_DIR}/openstack/healthcheck/healthcheck.sh - ;; - "odl") - odl_tests - [[ "$report" == "-r" ]] && args=-p - ${FUNCTEST_TEST_DIR}/sdn/odl/odl.py \ - --keystoneip $keystone_ip \ - --neutronip $neutron_ip \ - --odlip $odl_ip \ - --odlrestconfport $odl_restport \ - --odlwebport $odl_port \ - --ospassword ${OS_PASSWORD} \ - --ostenantname ${OS_TENANT_NAME} \ - --osusername ${OS_USERNAME} \ - ${args} - ;; - "ovno") - # suite under rewritting for colorado - # no need to run anything until refactoring done - # ${REPOS_DIR}/ovno/Testcases/RunTests.sh - ;; - *) - echo "The test case '${test_name}' does not exist." - exit 1 - esac - - if [[ $? != 0 ]]; then exit 1 - else exit 0 - fi -} - - -# Parse parameters -while [[ $# > 0 ]] - do - key="$1" - case $key in - -h|--help) - echo "$usage" - exit 0 - shift - ;; - -r|--report) - report="-r" - ;; - -s|--serial) - serial=true - ;; - -t|--test|--tests) - TEST="$2" - shift - ;; - *) - echo "unknown option $1 $2" - exit 1 - ;; - esac - shift # past argument or value -done - - -# Source credentials -echo "Sourcing Credentials ${creds} to run the test.." -source ${creds} - - -# Run test -run_test $TEST diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py index f920e70d8..0e951effe 100755 --- a/functest/ci/run_tests.py +++ b/functest/ci/run_tests.py @@ -32,13 +32,6 @@ from functest.utils.constants import CONST logger = ft_logger.Logger("run_tests").getLogger() -""" global variables """ -EXEC_SCRIPT = ("%s/functest/ci/exec_test.sh" % CONST.dir_repo_functest) - -# This will be the return code of this script. If any of the tests fails, -# this variable will change to Result.EX_ERROR - - class Result(enum.Enum): EX_OK = os.EX_OK EX_ERROR = -1 @@ -119,7 +112,7 @@ def update_test_info(test_name, result, duration): "duration": duration}) -def get_run_dict_if_defined(testname): +def get_run_dict(testname): try: dict = ft_utils.get_dict_by_test(testname) if not dict: @@ -143,7 +136,7 @@ def run_test(test, tier_name, testcases=None): logger.debug("\n%s" % test) source_rc_file() - if GlobalVariables.CLEAN_FLAG: + if test.needs_clean() and GlobalVariables.CLEAN_FLAG: generate_os_snapshot() flags = (" -t %s" % (test_name)) @@ -151,12 +144,13 @@ def run_test(test, tier_name, testcases=None): flags += " -r" result = testcase_base.TestcaseBase.EX_RUN_ERROR - run_dict = get_run_dict_if_defined(test_name) + run_dict = get_run_dict(test_name) if run_dict: try: module = importlib.import_module(run_dict['module']) cls = getattr(module, run_dict['class']) test_case = cls() + try: kwargs = run_dict['args'] result = test_case.run(**kwargs) @@ -173,14 +167,11 @@ def run_test(test, tier_name, testcases=None): logger.exception("Cannot get class {}".format( run_dict['class'])) else: - cmd = ("%s%s" % (EXEC_SCRIPT, flags)) - logger.info("Executing command {} because {} " - "doesn't implement the new framework".format( - cmd, test_name)) - result = ft_utils.execute_command(cmd) + raise Exception("Cannot import the class for the test case.") - if GlobalVariables.CLEAN_FLAG: + if test.needs_clean() and GlobalVariables.CLEAN_FLAG: cleanup() + end = datetime.datetime.now() duration = (end - start).seconds duration_str = ("%02d:%02d" % divmod(duration, 60)) diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 77cd1ae64..1a1d0f4b9 100755 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -11,6 +11,7 @@ tiers: name: snaps_health_check criteria: 'status == "PASS"' blocking: true + clean_flag: false description: >- This test case creates executes the SimpleHealthCheck Python test class which creates an, image, flavor, network, @@ -26,6 +27,7 @@ tiers: name: connection_check criteria: 'status == "PASS"' blocking: true + clean_flag: false description: >- This test case verifies the retrieval of OpenStack clients: Keystone, Glance, Neutron and Nova and may perform some @@ -44,6 +46,7 @@ tiers: name: api_check criteria: 'status == "PASS"' blocking: true + clean_flag: false description: >- This test case verifies the retrieval of OpenStack clients: Keystone, Glance, Neutron and Nova and may perform some @@ -68,6 +71,7 @@ tiers: name: vping_ssh criteria: 'status == "PASS"' blocking: true + clean_flag: true description: >- This test case verifies: 1) SSH to an instance using floating IPs over the public network. 2) Connectivity between 2 instances @@ -83,6 +87,7 @@ tiers: name: vping_userdata criteria: 'status == "PASS"' blocking: true + clean_flag: true description: >- This test case verifies: 1) Boot a VM with given userdata. 2) Connectivity between 2 instances over a private network. @@ -97,6 +102,7 @@ tiers: name: tempest_smoke_serial criteria: 'success_rate == 100%' blocking: false + clean_flag: false description: >- This test case runs the smoke subset of the OpenStack Tempest suite. The list of test cases is generated by @@ -113,6 +119,7 @@ tiers: name: rally_sanity criteria: 'success_rate == 100%' blocking: false + clean_flag: false description: >- This test case runs a sub group of tests of the OpenStack Rally suite in smoke mode. @@ -127,6 +134,7 @@ tiers: name: odl criteria: 'success_rate == 100%' blocking: true + clean_flag: false description: >- Test Suite for the OpenDaylight SDN Controller. It integrates some test suites from upstream using @@ -146,6 +154,7 @@ tiers: name: odl_netvirt criteria: 'success_rate == 100%' blocking: true + clean_flag: false description: >- Test Suite for the OpenDaylight SDN Controller when the NetVirt features are installed. It integrates @@ -167,6 +176,7 @@ tiers: name: onos criteria: 'status == "PASS"' blocking: true + clean_flag: true description: >- Test Suite for the ONOS SDN Controller. It integrates some test suites from upstream using TestON as the test @@ -182,6 +192,7 @@ tiers: name: snaps_smoke criteria: 'status == "PASS"' blocking: false + clean_flag: false description: >- This test case contains tests that setup and destroy environments with VMs with and without Floating IPs @@ -210,6 +221,7 @@ tiers: name: promise criteria: 'success_rate == 100%' blocking: false + clean_flag: true description: >- Test suite from Promise project. dependencies: @@ -223,6 +235,7 @@ tiers: name: doctor criteria: 'status == "PASS"' blocking: false + clean_flag: true description: >- Test suite from Doctor project. dependencies: @@ -236,6 +249,7 @@ tiers: name: bgpvpn criteria: 'status == "PASS"' blocking: false + clean_flag: true description: >- Test suite from SDNVPN project. dependencies: @@ -249,6 +263,7 @@ tiers: name: security_scan criteria: 'status == "PASS"' blocking: false + clean_flag: true description: >- Simple Security Scan dependencies: @@ -261,6 +276,7 @@ tiers: # name: copper # criteria: 'status == "PASS"' # blocking: false +# clean_flag: true # description: >- # Test suite for policy management based on OpenStack Congress # dependencies: @@ -273,6 +289,7 @@ tiers: name: multisite criteria: 'success_rate == 100%' blocking: false + clean_flag: false description: >- Test suite from kingbird dependencies: @@ -285,6 +302,7 @@ tiers: name: odl-sfc criteria: 'status == "PASS"' blocking: false + clean_flag: true description: >- Test suite for odl-sfc to test two chains and two SFs dependencies: @@ -297,6 +315,7 @@ tiers: name: onos_sfc criteria: 'status == "PASS"' blocking: true + clean_flag: true description: >- Test Suite for onos-sfc to test sfc function. dependencies: @@ -309,6 +328,7 @@ tiers: name: parser criteria: 'ret == 0' blocking: false + clean_flag: true description: >- Test suite from Parser project. dependencies: @@ -321,6 +341,7 @@ tiers: name: domino criteria: 'status == "PASS"' blocking: false + clean_flag: true description: >- Test suite from Domino project. dependencies: @@ -333,6 +354,7 @@ tiers: name: orchestra criteria: 'ret == 0' blocking: false + clean_flag: true description: >- Test OpenBaton (Orchestra) stack dependencies: @@ -345,6 +367,7 @@ tiers: name: netready criteria: 'status == "PASS"' blocking: false + clean_flag: true description: >- Test suite from Netready project. dependencies: @@ -357,6 +380,7 @@ tiers: name: barometer criteria: 'status == "PASS"' blocking: false + clean_flag: true description: >- Test suite for the Barometer project. Separate tests verify the proper configuration and functionality of the following @@ -379,6 +403,7 @@ tiers: # name: tempest_full_parallel # criteria: 'success_rate >= 80%' # blocking: false +# clean_flag: false # description: >- # The list of test cases is generated by # Tempest automatically and depends on the parameters of @@ -393,6 +418,7 @@ tiers: name: tempest_defcore criteria: 'success_rate == 100%' blocking: false + clean_flag: false description: >- This is the set of Tempest test cases created by OpenStack Interop Working Group for certification purposes. @@ -406,6 +432,7 @@ tiers: name: tempest_custom criteria: 'success_rate == 100%' blocking: false + clean_flag: false description: >- The test case allows running a customized list of tempest test cases defined in a file under @@ -422,6 +449,7 @@ tiers: # name: rally_full # criteria: 'success_rate >= 90%' # blocking: false +# clean_flag: false # description: >- # This test case runs the full suite of scenarios of the OpenStack # Rally suite using several threads and iterations. @@ -443,6 +471,7 @@ tiers: name: cloudify_ims criteria: 'status == "PASS"' blocking: false + clean_flag: true description: >- This test case deploys an OpenSource vIMS solution from Clearwater using the Cloudify orchestrator. It also runs some signaling traffic. @@ -456,6 +485,7 @@ tiers: name: aaa criteria: 'ret == 0' blocking: false + clean_flag: true description: >- Test suite from Parser project. dependencies: @@ -469,6 +499,7 @@ tiers: name: juju_epc criteria: 'ret == 0' blocking: false + clean_flag: true description: >- Test suite from OAI project, vEPC deployed with Juju. dependencies: @@ -482,6 +513,7 @@ tiers: name: orchestra_ims criteria: 'ret == 0' blocking: false + clean_flag: true description: >- VNF deployment with OpenBaton (Orchestra) dependencies: @@ -495,6 +527,7 @@ tiers: name: opera_ims criteria: 'ret == 0' blocking: false + clean_flag: true description: >- Evolution of vIMS dependencies: @@ -508,6 +541,7 @@ tiers: name: vyos_vrouter criteria: 'status == "PASS"' blocking: false + clean_flag: true description: >- This test case is vRouter testing. dependencies: diff --git a/functest/ci/tier_builder.py b/functest/ci/tier_builder.py index dae7c73e8..f4c6f70fd 100755 --- a/functest/ci/tier_builder.py +++ b/functest/ci/tier_builder.py @@ -50,6 +50,7 @@ class TierBuilder(object): dependency=dep, criteria=dic_testcase['criteria'], blocking=dic_testcase['blocking'], + clean_flag=dic_testcase['clean_flag'], description=dic_testcase['description']) if testcase.is_compatible(self.ci_installer, self.ci_scenario): tier.add_test(testcase) diff --git a/functest/ci/tier_handler.py b/functest/ci/tier_handler.py index 127986bf3..6b4864b5b 100755 --- a/functest/ci/tier_handler.py +++ b/functest/ci/tier_handler.py @@ -104,12 +104,18 @@ class Tier(object): class TestCase(object): - def __init__(self, name, dependency, criteria, blocking, description=""): + def __init__(self, name, + dependency, + criteria, + blocking, + clean_flag, + description=""): self.name = name self.dependency = dependency - self.description = description self.criteria = criteria self.blocking = blocking + self.clean_flag = clean_flag + self.description = description @staticmethod def is_none(item): @@ -138,6 +144,9 @@ class TestCase(object): def is_blocking(self): return self.blocking + def needs_clean(self): + return self.clean_flag + def __str__(self): lines = split_text(self.description, LINE_LENGTH - 6) |