diff options
| -rw-r--r-- | docker/requirements.pip | 8 | ||||
| -rw-r--r-- | docs/configguide/configguide.rst | 119 | ||||
| -rw-r--r-- | docs/configguide/index.rst | 120 | ||||
| -rw-r--r-- | docs/userguide/description.rst | 75 | ||||
| -rw-r--r-- | docs/userguide/index.rst | 247 | ||||
| -rw-r--r-- | docs/userguide/runfunctest.rst | 140 | ||||
| -rw-r--r-- | testcases/Controllers/ONOS/Teston/CI/onosfunctest.py | 41 | ||||
| -rwxr-xr-x | testcases/VIM/OpenStack/CI/libraries/run_rally-cert.py | 47 | ||||
| -rw-r--r-- | testcases/features/promise.py | 10 |
9 files changed, 424 insertions, 383 deletions
diff --git a/docker/requirements.pip b/docker/requirements.pip index e2b67d3bd..0ecb447ca 100644 --- a/docker/requirements.pip +++ b/docker/requirements.pip @@ -9,12 +9,8 @@ # pyyaml==3.10 gitpython==1.0.1 -python-neutronclient==2.6.0 -python-novaclient==2.28.1 -python-glanceclient==1.1.0 -python-cinderclient==1.4.0 +python-openstackclient==2.0.0 python-ceilometerclient==1.5.1 -python-keystoneclient==1.6.0 virtualenv==1.11.4 pexpect==4.0 requests==2.8.0 @@ -25,4 +21,4 @@ configObj==5.0.6 Flask==0.10.1 xmltodict==0.9.2 scp==0.10.2 -paramiko==1.16.0
\ No newline at end of file +paramiko==1.16.0 diff --git a/docs/configguide/configguide.rst b/docs/configguide/configguide.rst new file mode 100644 index 000000000..1908b543e --- /dev/null +++ b/docs/configguide/configguide.rst @@ -0,0 +1,119 @@ +Functional testing Installation +=============================== + +Pull the Functest Docker image from the Docker hub:: + + $ docker pull opnfv/functest:brahmaputra.1.0 + + +Check that the image is available:: + + $ docker images + +Run the docker container giving the environment variables:: + + - INSTALLER_TYPE. Possible values are "apex", "compass", "fuel" or "joid". + - INSTALLER_IP. each installer has its installation strategy. + +Functest may need to know the IP of the installer to retrieve the credentials +(e.g. usually "10.20.0.2" for fuel, not neede for joid...). + +The minimum command to create the Functest docker file can be described as +follows:: + + docker run -it -e "INSTALLER_IP=10.20.0.2" -e "INSTALLER_TYPE=fuel" opnfv/functest:brahmaputra.1.0 /bin/bash + +Optionally, it is possible to precise the container name through the option +--name:: + + docker run --name "CONTAINER_NAME" -it -e "INSTALLER_IP=10.20.0.2" -e "INSTALLER_TYPE=fuel" opnfv/functest:brahmaputra.1.0 /bin/bash + +It is also possible to to indicate the path of the OpenStack creds using -v:: + + docker run -it -e "INSTALLER_IP=10.20.0.2" -e "INSTALLER_TYPE=fuel" -v <path_to_your_local_creds_file>:/home/opnfv/functest/conf/openstack.creds opnfv/functest:brahmaputra.1.0 /bin/bash + +The local file will be mounted in the container under +/home/opnfv/functest/conf/openstack.creds + +After the run command the prompt appears which means that we are inside the +container and ready to run Functest. + +Inside the container, the following directory structure should be in place:: + + `-- home + `-- opnfv + |-- functest + | |-- conf + | |-- data + | `-- results + `-- repos + |-- bgpvpn + |-- functest + |-- odl_integration + |-- rally + |-- releng + `-- vims-test + + +Basically the container includes: + + * Functest directory to store the configuration (the OpenStack creds are paste + in /home/opngb/functest/conf), the data (images neede for test for offline + testing), results (some temporary artifacts may be stored here) + * Repositories: the functest repository will be used to prepare the + environment, run the tests. Other repositories are used for the installation + of the tooling (e.g. rally) and/or the retrieval of feature projects + scenarios (e.g. bgpvpn) + +The arborescence under the functest repo can be described as follow:: + + . + |-- INFO + |-- LICENSE + |-- commons + | |-- ims + | |-- mobile + | `-- traffic-profile-guidelines.rst + |-- docker + | |-- Dockerfile + | |-- common.sh + | |-- prepare_env.sh + | |-- requirements.pip + | `-- run_tests.sh + |-- docs + | |-- configguide + | |-- functest.rst + | |-- images + | `-- userguide + `-- testcases + |-- Controllers + |-- VIM + |-- __init__.py + |-- config_functest.py + |-- config_functest.yaml + |-- functest_utils.py + |-- functest_utils.pyc + |-- vIMS + `-- vPing + +We may distinguish 4 different folders: + + * commons: it is a folder dedicated to store traffic profile or any test + inputs that could be reused by any test project + * docker: this folder includes the scripts that will be used to setup the + environment and run the tests + * docs: this folder includes the user and installation/configuration guide + * testcases: this folder includes the scripts required by Functest internal + test cases + + +Firstly run the script to install functest environment:: + + $ ${repos_dir}/functest/docker/prepare_env.sh + +NOTE: ${repos_dir} is a default environment variable inside the docker +container, which points to /home/opnfv/repos + +Run the script to start the tests:: + + $ ${repos_dir}/functest/docker/run_tests.sh diff --git a/docs/configguide/index.rst b/docs/configguide/index.rst index f23b11d41..7519bb85b 100644 --- a/docs/configguide/index.rst +++ b/docs/configguide/index.rst @@ -169,125 +169,7 @@ Please note that the Functest container has been designed for OPNFV, however, it would be possible to adapt it to any VIM+controller environment as most of the testcases are upstream testcases. -Manual Installation -=================== - -Pull the Functest Docker image from the Docker hub:: - - $ docker pull opnfv/functest:brahmaputra.1.0 - - -Check that the image is available:: - - $ docker images - -Run the docker container giving the environment variables:: - - - INSTALLER_TYPE. Possible values are "apex", "compass", "fuel" or "joid". - - INSTALLER_IP. each installer has its installation strategy. - -Functest may need to know the IP of the installer to retrieve the credentials -(e.g. usually "10.20.0.2" for fuel, not neede for joid...). - -The minimum command to create the Functest docker file can be described as -follows:: - - docker run -it -e "INSTALLER_IP=10.20.0.2" -e "INSTALLER_TYPE=fuel" opnfv/functest:brahmaputra.1.0 /bin/bash - -Optionally, it is possible to precise the container name through the option ---name:: - - docker run --name "CONTAINER_NAME" -it -e "INSTALLER_IP=10.20.0.2" -e "INSTALLER_TYPE=fuel" opnfv/functest:brahmaputra.1.0 /bin/bash - -It is also possible to to indicate the path of the OpenStack creds using -v:: - - docker run -it -e "INSTALLER_IP=10.20.0.2" -e "INSTALLER_TYPE=fuel" -v <path_to_your_local_creds_file>:/home/opnfv/functest/conf/openstack.creds opnfv/functest:brahmaputra.1.0 /bin/bash - -The local file will be mounted in the container under -/home/opnfv/functest/conf/openstack.creds - -After the run command the prompt appears which means that we are inside the -container and ready to run Functest. - -Inside the container, you must have the following directory structure:: - - `-- home - `-- opnfv - |-- functest - | |-- conf - | |-- data - | `-- results - `-- repos - |-- bgpvpn - |-- functest - |-- odl_integration - |-- rally - |-- releng - `-- vims-test - - -Basically the container includes: - - * Functest directory to store the configuration (the OpenStack creds are paste - in /home/opngb/functest/conf), the data (images neede for test for offline - testing), results (some temporary artifacts may be stored here) - * Repositories: the functest repository will be used to prepare the - environment, run the tests. Other repositories are used for the installation - of the tooling (e.g. rally) and/or the retrieval of feature projects - scenarios (e.g. bgpvpn) - -The arborescence under the functest repo can be described as follow:: - - . - |-- INFO - |-- LICENSE - |-- commons - | |-- ims - | |-- mobile - | `-- traffic-profile-guidelines.rst - |-- docker - | |-- Dockerfile - | |-- common.sh - | |-- prepare_env.sh - | |-- requirements.pip - | `-- run_tests.sh - |-- docs - | |-- configguide - | |-- functest.rst - | |-- images - | `-- userguide - `-- testcases - |-- Controllers - |-- VIM - |-- __init__.py - |-- config_functest.py - |-- config_functest.yaml - |-- functest_utils.py - |-- functest_utils.pyc - |-- vIMS - `-- vPing - -We may distinguish 4 different folders: - - * commons: it is a folder dedicated to store traffic profile or any test - inputs that could be reused by any test project - * docker: this folder includes the scripts that will be used to setup the - environment and run the tests - * docs: this folder includes the user and installation/configuration guide - * testcases: this folder includes the scripts required by Functest internal - test cases - - -Firstly run the script to install functest environment:: - - $ ${repos_dir}/functest/docker/prepare_env.sh - -NOTE: ${repos_dir} is a default environment variable inside the docker -container, which points to /home/opnfv/repos - -Run the script to start the tests:: - - $ ${repos_dir}/functest/docker/run_tests.sh +.. include:: ./configguide.rst NOTE: This will run ALL the tests by default, see `[2]`_ for details diff --git a/docs/userguide/description.rst b/docs/userguide/description.rst new file mode 100644 index 000000000..a8fdcd0f2 --- /dev/null +++ b/docs/userguide/description.rst @@ -0,0 +1,75 @@ +Description of the test cases +============================= + +Functest is an OPNFV project dedicated to functional testing. +In the continuous integration, it is launched after an OPNFV fresh installation. +The Functest target is to verify the basic functions of the infrastructure. + +Functest includes different test suites which several test cases within. +Test cases are developed in Functest and in feature projects. + +The current list of test suites can be distributed in 3 main domains:: + + +----------------+----------------+--------------------------------------------+ + | Domain | Test suite | Comments | + +================+================+============================================+ + | | vPing | NFV "Hello World" | + | +----------------+--------------------------------------------+ + | VIM | vPing_userdata | Ping using userdata and cloud-init | + | | | mechanism | + | +----------------+--------------------------------------------+ + |(Virtualised | Tempest | OpenStack reference test suite `[2]`_ | + | Infrastructure +----------------+--------------------------------------------+ + | Manager) | Rally scenario | OpenStack testing tool testing OpenStack | + | | | modules `[3]`_ | + +----------------+----------------+--------------------------------------------+ + | | OpenDaylight | Opendaylight Test suite | + | +----------------+--------------------------------------------+ + | Controllers | ONOS | Test suite of ONOS L2 and L3 functions | + | +----------------+--------------------------------------------+ + | | OpenContrail | | + +----------------+----------------+--------------------------------------------+ + | Features | vIMS | Show the capability to deploy a real NFV | + | | | test cases. | + | | | The IP Multimedia Subsytem is a typical | + | | | Telco test case, referenced by ETSI. | + | | | It provides a fully functional VoIP System.| + | +----------------+--------------------------------------------+ + | | Promise | Resource reservation and management project| + | | | to identify NFV related requirements and | + | | | realize resource reservation for future | + | | | usage by capacity management of resource | + | | | pools regarding compute, network and | + | | | storage. | + | +----------------+--------------------------------------------+ + | | SDNVPN | | + +----------------+----------------+--------------------------------------------+ + + +Most of the test suites are developed upstream. +For example, `Tempest <http://docs.openstack.org/developer/tempest/overview.html>`_ is the +OpenStack integration test suite. +Functest is in charge of the integration of different functional test suites. + +The Tempest suite has been customized but no new test cases have been created. +Some OPNFV feature projects (e.g. SDNVPN) have created Tempest tests cases and +pushed to upstream. + +The tests run from CI are pushed into a database. +The goal is to populate the database with results and to show them on a Test +Dashboard. + +There is no real notion of Test domain or Test coverage yet. +Basic components (VIM, controllers) are tested through their own suites. +Feature projects also provide their own test suites. + +vIMS test case was integrated to demonstrate the capability to deploy a +relatively complex NFV scenario on top of the OPNFV infrastructure. + +Functest considers OPNFV as a black box. +OPNFV, since Brahmaputra, offers lots of possible combinations: + + * 3 controllers (OpenDayligh, ONOS, OpenContrail) + * 4 installers (Apex, Compass, Fuel, Joid) + +However most of the tests shall be runnable on any configuration. diff --git a/docs/userguide/index.rst b/docs/userguide/index.rst index ba20507b1..cf36d9e80 100644 --- a/docs/userguide/index.rst +++ b/docs/userguide/index.rst @@ -17,81 +17,7 @@ A presentation has been created for the first OPNFV Summit `[4]`_. It is assumed that Functest container has been properly installed `[1]`_. - -Description of the test cases -============================= - -Functest is an OPNFV project dedicated to functional testing. -In the continuous integration, it is launched after an OPNFV fresh installation. -The Functest target is to verify the basic functions of the infrastructure. - -Functest includes different test suites which several test cases within. -Test cases are developed in Functest and in feature projects. - -The current list of test suites can be distributed in 3 main domains:: - - +----------------+----------------+--------------------------------------------+ - | Method | Test suite | Comments | - +================+================+============================================+ - | | vPing | NFV "Hello World" | - | +----------------+--------------------------------------------+ - | VIM | vPing_userdata | Ping using userdata and cloud-init | - | | | mechanism | - | +----------------+--------------------------------------------+ - |(Virtualised | Tempest | OpenStack reference test suite `[2]`_ | - | Infrastructure +----------------+--------------------------------------------+ - | Manager) | Rally scenario | OpenStack testing tool testing OpenStack | - | | | modules `[3]`_ | - +----------------+----------------+--------------------------------------------+ - | | OpenDaylight | Opendaylight Test suite | - | +----------------+--------------------------------------------+ - | Controllers | ONOS | Test suite of ONOS L2 and L3 functions | - | +----------------+--------------------------------------------+ - | | OpenContrail | | - +----------------+----------------+--------------------------------------------+ - | Features | vIMS | Show the capability to deploy a real NFV | - | | | test cases. | - | | | The IP Multimedia Subsytem is a typical | - | | | Telco test case, referenced by ETSI. | - | | | It provides a fully functional VoIP System.| - | +----------------+--------------------------------------------+ - | | Promise | Resource reservation and management project| - | | | to identify NFV related requirements and | - | | | realize resource reservation for future | - | | | usage by capacity management of resource | - | | | pools regarding compute, network and | - | | | storage. | - | +----------------+--------------------------------------------+ - | | SDNVPN | | - +----------------+----------------+--------------------------------------------+ - - -Most of the test suites are developed upstream. -For example, Tempest `[2]`_ is the OpenStack integration test suite. -Functest is in charge of the integration of different functional test suites. - -The Tempest suite has been customized but no new test cases have been created. -Some OPNFV feature projects (.e.g. SDNVPN) have created Tempest tests cases and -pushed to upstream. - -The tests run from CI are pushed into a database. -The goal is to populate the database with results and to show them on a Test -Dashboard. - -There is no real notion of Test domain or Test coverage yet. -Basic components (VIM, controllers) are tested through their own suites. -Feature projects also provide their own test suites. - -vIMS test case was integrated to demonstrate the capability to deploy a -relatively complex NFV scenario on top of the OPNFV infrastructure. - -Functest considers OPNFV as a black box. -OPNFV, since Brahmaputra, offers lots of possible combinations: - - * 3 controllers (OpenDayligh, ONOS, OpenContrail) - * 4 installers (Apex, Compass, Fuel, Joid) - -However most of the tests shall be runnable on any configuration. +.. include:: ./description.rst The different scenarios are described in the section hereafter. @@ -177,7 +103,7 @@ Tempest has batteries of tests for: * OpenStack API validation * Scenarios - * other specific tests useful in validating an OpenStack deployment + * Other specific tests useful in validating an OpenStack deployment We use Rally `[3]`_ to run Tempest suite. Rally generates automatically tempest.conf configuration file. @@ -187,10 +113,10 @@ When the Tempest suite is run, each test duration is measured. The full console output is stored in the tempest.log file. As an addition of Arno, Brahmaputra runs a customized set of Tempest test cases. -The list is specificed through --tests-file when running Rally. +The list is specificed through *--tests-file* when running Rally. This option has been introduced in Rally in version 0.1.2. -The customized test list is available in the Functest repo `[4]`_ +The customized test list is available in the Functest repo `[4]`_. This list contains more than 200 Tempest test cases. The list can be divied into two main parts: @@ -416,145 +342,7 @@ include:: flavor_ram: 512 flavor_disk: 0 - -Manual testing -============== - -Once the Functest docker container is running and Functest environment ready -(through /home/opnfv/repos/functest/docker/prepare_env.sh script), the system is -ready to run the tests. - -The script run_tests.sh is located in $repos_dir/functest/docker and it has -several options:: - - ./run_tests.sh -h - Script to trigger the tests automatically. - - usage: - bash run_tests.sh [--offline] [-h|--help] [-t <test_name>] - - where: - -h|--help show this help text - -r|--report push results to database (false by default) - -n|--no-clean do not clean up OpenStack resources after test run - -t|--test run specific set of tests - <test_name> one or more of the following: vping,vping_userdata,odl,rally,tempest,vims,onos,promise. Separated by comma. - - examples: - run_tests.sh - run_tests.sh --test vping,odl - run_tests.sh -t tempest,rally --no-clean - -The -r option is used by the Continuous Integration in order to push the test -results into a test collection database, see in next section for details. -In manual mode, you must not use it, your try will be anyway probably rejected -as your POD must be declared in the database to collect the data. - -The -n option is used for preserving all the existing OpenStack resources after -execution test cases. - -The -t option can be used to specify the list of test you want to launch, by -default Functest will try to launch all its test suites in the following order -vPing, odl, Tempest, vIMS, Rally. -You may launch only one single test by using -t <the test you want to launch> - -Within Tempest test suite you can define which test cases you want to execute in -your environment by editing test_list.txt file before executing run_tests.sh -script. - -Please note that Functest includes cleaning mechanism in order to remove -everything except what was present after a fresh install. -If you create your own VMs, tenants, networks etc. and then launch Functest, -they all will be deleted after executing the tests. Use --no-clean option with -run_test.sh in order to preserve all the existing resources. -However, be aware that Tempest and Rally create of lot of resources (users, -tenants, networks, volumes etc.) that are not always properly cleaned, so this -cleaning function has been set to keep the system as clean as possible after a -full Functest run. - -You may also add you own test by adding a section into the function run_test() - - -Automated testing -================= - -As mentioned in `[1]`, the prepare-env.sh and run_test.sh can be executed within -the container from jenkins. -2 jobs have been created, one to run all the test and one that allows testing -test suite by test suite. -You thus just have to launch the acurate jenkins job on the target lab, all the -tests shall be automatically run. - -When the tests are automatically started from CI, a basic algorithm has been -created in order to detect whether the test is runnable or not on the given -scenario. -In fact, one of the most challenging task in Brahmaputra consists in dealing -with lots of scenario and installers. -Functest test suites cannot be systematically run (e.g. run the ODL suite on an -ONOS scenario). - -CI provides several information: - - * The installer (apex|compass|fuel|joid) - * The scenario [controller]-[feature]-[mode] with - - * controller = (odl|onos|ocl|nosdn) - * feature = (ovs(dpdk)|kvm) - * mode = (ha|noha) - -Constraints per test case are defined in the Functest configuration file -/home/opnfv/functest/config/config_functest.yaml:: - - test-dependencies: - functest: - vims: - scenario: '(ocl)|(odl)|(nosdn)' - vping: - vping_userdata: - scenario: '(ocl)|(odl)|(nosdn)' - tempest: - rally: - odl: - scenario: 'odl' - onos: - scenario: 'onos' - .... - -At the end of the Functest environment creation (prepare_env.sh see `[1]`_), a -file (/home/opnfv/functest/conf/testcase-list.txt) is created with the list of -all the runnable tests. -We consider the static constraints as regex and compare them with the scenario. -For instance, odl can be run only on scenario including odl in its name. - -The order of execution is also described in the Functest configuration file:: - - test_exec_priority: - - 1: vping - 2: vping_userdata - 3: tempest - 4: odl - 5: onos - 6: ovno - #7: doctor - 8: promise - 9: odl-vpnservice - 10: bgpvpn - #11: openstack-neutron-bgpvpn-api-extension-tests - 12: vims - 13: rally - -The tests are executed as follow: - - * Basic scenario (vPing, vPing_userdata, Tempest) - * Controller suites: ODL or ONOS or OpenContrail - * Feature projects - * vIMS - * Rally (benchmark scenario) - -At the end of an automated execution, everything is cleaned. -We keep only the users/networks that have been statically declared in '[9]'_ - +.. include:: ./runfunctest.rst Test results ============ @@ -733,7 +521,7 @@ The results of ODL tests can be seen in the console:: * log.html * report.html - ODL result page +**ODL result page** .. figure:: ../images/functestODL.png :width: 170mm @@ -744,8 +532,8 @@ The results of ODL tests can be seen in the console:: ONOS ^^^^ -The ONOS test logs can be found in OnosSystemTest/TestON/logs -(ONOSCI_PATH to be added),and also can be seen in the console:: +The ONOS test logs can be found in OnosSystemTest/, and TestON/, and logs/ +(ONOSCI_PATH to be added), and can also be seen in the console:: ****************************** Result summary for Testcase4 @@ -902,7 +690,7 @@ steps: * INFO: environment prepared successfully => environment OK * INFO - Cloudify-manager server is UP ! => orchestrator deployed - * INFO The deployment of clearwater-opnfv is ended => VNF deployed + * INFO - The deployment of clearwater-opnfv is ended => VNF deployed * Multiple Identities (UDP) - (6505550771, 6505550675) Passed => tests run * DEBUG - Pushing results to DB.... => tests saved @@ -1035,7 +823,7 @@ But all were not ready for Brahmaputra. The API can described as follow: -**Version:**:: +Version: +--------+--------------------------+------------------------------------------+ | Method | Path | Description | @@ -1044,7 +832,7 @@ The API can described as follow: +--------+--------------------------+------------------------------------------+ -**Pods:**:: +Pods: +--------+--------------------------+------------------------------------------+ | Method | Path | Description | @@ -1059,7 +847,7 @@ The API can described as follow: | | | } | +--------+--------------------------+------------------------------------------+ -**Projects:**:: +Projects: +--------+--------------------------+------------------------------------------+ | Method | Path | Description | @@ -1087,7 +875,7 @@ The API can described as follow: +--------+--------------------------+------------------------------------------+ -**Test cases:**:: +Test cases: +--------+--------------------------+------------------------------------------+ | Method | Path | Description | @@ -1105,7 +893,7 @@ The API can described as follow: | | | } | +--------+--------------------------+------------------------------------------+ | PUT | /test_projects/{project}?| Modify a test case of {project} | - | | case_name={case} | | + | | case_name={case} | | | | | Content-Type: application/json | | | | { | | | | <the field(s) you want to modify> | @@ -1113,9 +901,9 @@ The API can described as follow: +--------+--------------------------+------------------------------------------+ | DELETE | /test_projects/{project}/| Delete a test case | | | case_name={case} | | - +----------------+----------------+--------------------------------------------+ + +--------+--------------------------+------------------------------------------+ -**Test Results:**:: +Test Results: +--------+--------------------------+------------------------------------------+ | Method | Path | Description | @@ -1155,7 +943,7 @@ The API can described as follow: +--------+--------------------------+------------------------------------------+ -**Dashboard:**:: +Dashboard: +--------+--------------------------+------------------------------------------+ | Method | Path | Description | @@ -1292,4 +1080,3 @@ IRC support chan: #opnfv-testperf .. _`Rally installation procedure`: https://rally.readthedocs.org/en/latest/tutorial/step_0_installation.html .. _`config_test.py` : https://git.opnfv.org/cgit/functest/tree/testcases/config_functest.py .. _`config_functest.yaml` : https://git.opnfv.org/cgit/functest/tree/testcases/config_functest.yaml - diff --git a/docs/userguide/runfunctest.rst b/docs/userguide/runfunctest.rst new file mode 100644 index 000000000..8125e8149 --- /dev/null +++ b/docs/userguide/runfunctest.rst @@ -0,0 +1,140 @@ +Executing the functest suites +============================= + +Manual testing +-------------- + +Once the Functest docker container is running and Functest environment ready +(through /home/opnfv/repos/functest/docker/prepare_env.sh script), the system is +ready to run the tests. + +The script *run_tests.sh* is located in $repos_dir/functest/docker and it has +several options:: + + ./run_tests.sh -h + Script to trigger the tests automatically. + + usage: + bash run_tests.sh [--offline] [-h|--help] [-t <test_name>] + + where: + -h|--help show this help text + -r|--report push results to database (false by default) + -n|--no-clean do not clean up OpenStack resources after test run + -t|--test run specific set of tests + <test_name> one or more of the following: vping,vping_userdata,odl,rally,tempest,vims,onos,promise. Separated by comma. + + examples: + run_tests.sh + run_tests.sh --test vping,odl + run_tests.sh -t tempest,rally --no-clean + +The *-r* option is used by the Continuous Integration in order to push the test +results into a test collection database, see in next section for details. +In manual mode, you must not use it, your try will be anyway probably rejected +as your POD must be declared in the database to collect the data. + +The *-n* option is used for preserving all the existing OpenStack resources after +execution test cases. + +The *-t* option can be used to specify the list of test you want to launch, by +default Functest will try to launch all its test suites in the following order +vPing, odl, Tempest, vIMS, Rally. +You may launch only one single test by using *-t <the test you want to launch>*. + +Within Tempest test suite you can define which test cases you want to execute in +your environment by editing test_list.txt file before executing *run_tests.sh* +script. + +Please note that Functest includes cleaning mechanism in order to remove +everything except what was present after a fresh install. +If you create your own VMs, tenants, networks etc. and then launch Functest, +they all will be deleted after executing the tests. Use the *--no-clean* option with +run_test.sh in order to preserve all the existing resources. +However, be aware that Tempest and Rally create of lot of resources (users, +tenants, networks, volumes etc.) that are not always properly cleaned, so this +cleaning function has been set to keep the system as clean as possible after a +full Functest run. + +You may also add you own test by adding a section into the function run_test(). + + +Automated testing +----------------- + +As mentioned in `[1]`, the *prepare-env.sh* and *run_test.sh* can be executed within +the container from jenkins. +2 jobs have been created, one to run all the test and one that allows testing +test suite by test suite. +You thus just have to launch the acurate jenkins job on the target lab, all the +tests shall be automatically run. + +When the tests are automatically started from CI, a basic algorithm has been +created in order to detect whether the test is runnable or not on the given +scenario. +In fact, one of the most challenging task in Brahmaputra consists in dealing +with lots of scenario and installers. +Functest test suites cannot be systematically run (e.g. run the ODL suite on an +ONOS scenario). + +CI provides several information: + + * The installer (apex|compass|fuel|joid) + * The scenario [controller]-[feature]-[mode] with + + * controller = (odl|onos|ocl|nosdn) + * feature = (ovs(dpdk)|kvm) + * mode = (ha|noha) + +Constraints per test case are defined in the Functest configuration file +/home/opnfv/functest/config/config_functest.yaml:: + + test-dependencies: + functest: + vims: + scenario: '(ocl)|(odl)|(nosdn)' + vping: + vping_userdata: + scenario: '(ocl)|(odl)|(nosdn)' + tempest: + rally: + odl: + scenario: 'odl' + onos: + scenario: 'onos' + .... + +At the end of the Functest environment creation (prepare_env.sh see `[1]`_), a +file (/home/opnfv/functest/conf/testcase-list.txt) is created with the list of +all the runnable tests. +We consider the static constraints as regex and compare them with the scenario. +For instance, odl can be run only on scenario including odl in its name. + +The order of execution is also described in the Functest configuration file:: + + test_exec_priority: + + 1: vping + 2: vping_userdata + 3: tempest + 4: odl + 5: onos + 6: ovno + 7: doctor + 8: promise + 9: odl-vpnservice + 10: bgpvpn + 11: openstack-neutron-bgpvpn-api-extension-tests + 12: vims + 13: rally + +The tests are executed as follow: + + * Basic scenario (vPing, vPing_userdata, Tempest) + * Controller suites: ODL or ONOS or OpenContrail + * Feature projects + * vIMS + * Rally (benchmark scenario) + +At the end of an automated execution, everything is cleaned. +We keep only the users/networks that have been statically declared in 'https://git.opnfv.org/cgit/functest/tree/testcases/VIM/OpenStack/CI/libraries/os_defaults.yaml'_ diff --git a/testcases/Controllers/ONOS/Teston/CI/onosfunctest.py b/testcases/Controllers/ONOS/Teston/CI/onosfunctest.py index 2d22baec9..4b3779410 100644 --- a/testcases/Controllers/ONOS/Teston/CI/onosfunctest.py +++ b/testcases/Controllers/ONOS/Teston/CI/onosfunctest.py @@ -98,11 +98,11 @@ def GetResult(): # 'status': Result} cmd = "grep -rnh 'Execution Time' " + LOGPATH Resultbuffer = os.popen(cmd).read() - time1 = Resultbuffer[114:128] - time2 = Resultbuffer[28:42] + time1 = Resultbuffer[114:128] + time2 = Resultbuffer[28:42] cmd = "grep -rnh 'Success Percentage' " + LOGPATH + "/FUNCvirNetNB_*" Resultbuffer = os.popen(cmd).read() - if Resultbuffer.find('100%') >= 0: + if Resultbuffer.find('100%') >= 0: result1='Success' else: result1='Failed' @@ -112,10 +112,33 @@ def GetResult(): result2='Success' else: result2='Failed' - payload={'FUNCvirNet':{'duration': time1, - 'result': result1}, + status1 = [] + status2 = [] + cmd = "grep -rnh 'h3' " + LOGPATH + "/FUNCvirNetNB_*" + Resultbuffer = os.popen(cmd).read() + pattern = re.compile("<h3>([^-]+) - ([^-]+) - (\S*)</h3>") + #res = pattern.search(Resultbuffer).groups() + res = pattern.findall(Resultbuffer) + i = 0 + for index in range(len(res)): + status1.append({'Case name:':res[i][0] + res[i][1],'Case result':res[i][2]}) + i=i+1 + cmd = "grep -rnh 'h3' " + LOGPATH + "/FUNCvirNetNBL3*" + Resultbuffer = os.popen(cmd).read() + pattern = re.compile("<h3>([^-]+) - ([^-]+) - (\S*)</h3>") + #res = pattern.search(Resultbuffer).groups() + res = pattern.findall(Resultbuffer) + i = 0 + for index in range(len(res)): + status2.append({'Case name:':res[i][0] + res[i][1],'Case result':res[i][2]}) + i=i+1 + payload={'timestart': str(starttime), + 'FUNCvirNet':{'duration': time1, + 'result': result1, + 'status': status1}, 'FUNCvirNetL3':{'duration': time2, - 'result': result2}} + 'result': result2, + 'status': status2}} return payload def SetOnosIp(): @@ -155,12 +178,12 @@ def main(): try: logger.debug("Push result into DB") # TODO check path result for the file - git_version = functest_utils.get_git_branch(REPO_PATH) + scenario = functest_utils.get_scenario(logger) pod_name = functest_utils.get_pod_name(logger) result = GetResult() functest_utils.push_results_to_db(TEST_DB, "ONOS", - logger, pod_name, git_version, + logger, pod_name, scenario, payload=result) except: logger.error("Error pushing results into Database") @@ -169,4 +192,4 @@ def main(): if __name__ == '__main__': - main()
\ No newline at end of file + main() diff --git a/testcases/VIM/OpenStack/CI/libraries/run_rally-cert.py b/testcases/VIM/OpenStack/CI/libraries/run_rally-cert.py index efdde6718..71bf743fb 100755 --- a/testcases/VIM/OpenStack/CI/libraries/run_rally-cert.py +++ b/testcases/VIM/OpenStack/CI/libraries/run_rally-cert.py @@ -192,6 +192,32 @@ def build_task_args(test_file_name): return task_args +def get_output(proc): + result = "" + if args.verbose: + while proc.poll() is None: + line = proc.stdout.readline() + print line.replace('\n', '') + result += line + else: + while proc.poll() is None: + line = proc.stdout.readline() + if "Load duration" in line or \ + "started" in line or \ + "finished" in line or \ + " Preparing" in line or \ + "+-" in line or \ + "|" in line: + result += line + elif "test scenario" in line: + result += "\n" + line + elif "Full duration" in line: + result += line + "\n\n" + logger.info("\n" + result) + + return result + + def run_task(test_name): # # the "main" function of the script who launch rally for a task @@ -219,25 +245,8 @@ def run_task(test_name): logger.debug('running command line : {}'.format(cmd_line)) p = subprocess.Popen(cmd_line, stdout=subprocess.PIPE, stderr=RALLY_STDERR, shell=True) - result = "" - while p.poll() is None: - #l = p.stdout.readline() - #line = l.replace('\n', '') - line = p.stdout.readline() - if "Load duration" in line or \ - "started" in line or \ - "finished" in line or \ - " Preparing" in line or \ - "+-" in line or \ - "|" in line: - result += line - elif "test scenario" in line: - result += "\n" + line - elif "Full duration" in line: - result += line + "\n\n" - - logger.info("\n" + result) - task_id = get_task_id(result) + output = get_output(p) + task_id = get_task_id(output) logger.debug('task_id : {}'.format(task_id)) if task_id is None: diff --git a/testcases/features/promise.py b/testcases/features/promise.py index 0dc755b57..22d28fb99 100644 --- a/testcases/features/promise.py +++ b/testcases/features/promise.py @@ -20,6 +20,7 @@ import yaml import keystoneclient.v2_0.client as ksclient import glanceclient.client as glclient import novaclient.client as nvclient +from neutronclient.v2_0 import client as ntclient parser = argparse.ArgumentParser() @@ -172,6 +173,13 @@ def main(): exit(-1) logger.debug("Flavor '%s' with ID '%s' created successfully." % (FLAVOR_NAME, flavor_id)) + neutron = ntclient.Client(**nt_creds) + private_net=functest_utils.get_private_net(neutron) + if private_net == None: + logger.error("There is no private network in the deployment. Aborting...") + exit(-1) + logger.debug("Using private network '%s' (%s)." % (private_net['name'], + private_net['id'])) logger.info("Exporting environment variables...") os.environ["NODE_ENV"] = "functest" @@ -180,6 +188,8 @@ def main(): os.environ["OS_PASSWORD"] = USER_PWD os.environ["OS_TEST_IMAGE"] = image_id os.environ["OS_TEST_FLAVOR"] = flavor_id + os.environ["OS_TEST_NETWORK"] = private_net['id'] + os.chdir(PROMISE_REPO) results_file=open('promise-results.json','w+') |