diff options
| -rw-r--r-- | ci/config_functest.yaml | 11 | ||||
| -rw-r--r-- | docker/Dockerfile | 1 | ||||
| -rw-r--r-- | docker/requirements.pip | 3 | ||||
| -rw-r--r-- | docs/images/FunctestDashboard.png | bin | 0 -> 160735 bytes | |||
| -rw-r--r-- | docs/images/concepts_mapping_final.png | bin | 0 -> 203419 bytes | |||
| -rw-r--r-- | docs/userguide/index.rst | 200 | ||||
| -rw-r--r-- | docs/userguide/introduction.rst | 223 | ||||
| -rw-r--r-- | docs/userguide/runfunctest.rst | 360 | ||||
| -rw-r--r-- | docs/userguide/troubleshooting.rst | 246 | ||||
| -rw-r--r-- | testcases/OpenStack/examples/create_instance_and_ip.py | 136 | ||||
| -rw-r--r-- | utils/openstack_utils.py | 103 |
11 files changed, 931 insertions, 352 deletions
diff --git a/ci/config_functest.yaml b/ci/config_functest.yaml index ea502dacb..fae331d63 100644 --- a/ci/config_functest.yaml +++ b/ci/config_functest.yaml @@ -156,5 +156,16 @@ promise: subnet_cidr: 192.168.121.0/24 router_name: promise-router +example: + example_vm_name: example-vm + example_flavor: m1.small + example_image_name: functest-example-vm + example_private_net_name: example-net + example_private_subnet_name: example-subnet + example_private_subnet_cidr: 192.168.170.0/24 + example_router_name: example-router + example_sg_name: example-sg + example_sg_descr: Example Security group + results: test_db_url: http://testresults.opnfv.org/test/api/v1 diff --git a/docker/Dockerfile b/docker/Dockerfile index ae1343c2d..f8fe31dfe 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -78,6 +78,7 @@ RUN git clone https://github.com/openstack/networking-bgpvpn ${repos_dir}/bgpvpn RUN git clone https://github.com/wuwenbin2/OnosSystemTest.git ${repos_dir}/onos RUN git clone https://github.com/opnfv/promise ${repos_dir}/promise RUN git clone https://gerrit.opnfv.org/gerrit/ovno ${repos_dir}/ovno +RUN git clone https://gerrit.opnfv.org/gerrit/copper ${repos_dir}/copper RUN pip install -r ${repos_dir}/functest/docker/requirements.pip RUN pip install -r ${repos_dir}/rally/requirements.txt diff --git a/docker/requirements.pip b/docker/requirements.pip index 07334db48..88fa74de0 100644 --- a/docker/requirements.pip +++ b/docker/requirements.pip @@ -21,4 +21,5 @@ configObj==5.0.6 Flask==0.10.1 xmltodict==0.9.2 scp==0.10.2 -paramiko==1.16.0
\ No newline at end of file +paramiko==1.16.0 +python-congressclient==1.3.0
\ No newline at end of file diff --git a/docs/images/FunctestDashboard.png b/docs/images/FunctestDashboard.png Binary files differnew file mode 100644 index 000000000..a26533aea --- /dev/null +++ b/docs/images/FunctestDashboard.png diff --git a/docs/images/concepts_mapping_final.png b/docs/images/concepts_mapping_final.png Binary files differnew file mode 100644 index 000000000..7a51f1117 --- /dev/null +++ b/docs/images/concepts_mapping_final.png diff --git a/docs/userguide/index.rst b/docs/userguide/index.rst index 60bbc5b1a..ea2679764 100644 --- a/docs/userguide/index.rst +++ b/docs/userguide/index.rst @@ -12,24 +12,45 @@ OPNFV FUNCTEST user guide Introduction ============ -The goal of this document is to describe the Functest test cases as well as -provide a procedure to execute them. +The goal of this document is to describe the OPNFV Functest test cases and to +provide a procedure to execute them. In the OPNFV Colorado system release, +a Functest CLI utility is introduced for easier execution of test procedures. -A presentation has been created for the first OPNFV Summit `[4]`_. +A overview presentation has been created for the first OPNFV Summit `[4]`_. -This document is a continuation of the Functest Configuration Guide`[1]`_ and it -is assumed that the Functest Docker container is properly deployed. +This document is a continuation of the OPNFV Functest Configuration Guide `[1]`_. -**IMPORTANT**: All the instructions described in this guide must be performed -inside the container. +**IMPORTANT**: It is assumed here that the Functest Docker container is already +properly deployed and that all instructions described in this guide are to be +performed from *inside* the deployed Functest Docker container. .. include:: ./introduction.rst -The different test cases are described in the section hereafter. +The different test cases are described in the remaining sections of this document. VIM (Virtualized Infrastructure Manager) ---------------------------------------- +Healthcheck +^^^^^^^^^^^ +In Colorado release a new Tier 'healthcheck' with one testcase 'healthcheck' +is introduced. The healthcheck testcase verifies that some basic IP connectivity +and essential operations of OpenStack functionality over the command line are +working correctly. + +In particular, the following verifications are performed: + + * DHCP agent functionality for IP address allocation + * Openstack Authentication management functionality via the Keystone API + * OpenStack Image management functionality via the Glance API + * OpenStack Block Storage management functionality via the Cinder API + * OpenStack Networking management functionality via the Neutron API + * Openstack Compute management functionality via the NOVA API + +Self-obviously, successful completion of the 'healthcheck' testcase is a +necessary pre-requisite for the execution of all other test Tiers. + + vPing_ssh ^^^^^^^^^ @@ -50,7 +71,8 @@ Given the script **ping.sh**:: The goal of this test is to establish an SSH connection using a floating IP -on the public network and verify that 2 instances can talk on a private network:: +on the Public/External network and verify that 2 instances can talk over a Private +Tenant network:: vPing_ssh test case +-------------+ +-------------+ @@ -88,15 +110,16 @@ on the public network and verify that 2 instances can talk on a private network: +-------------+ +-------------+ This test can be considered as an "Hello World" example. -It is the first basic use case which shall work on any deployment. +It is the first basic use case which **must** work on any deployment. vPing_userdata ^^^^^^^^^^^^^^ -This test case is similar to vPing_ssh but without the use of floating ips -and the public network. It only checks that 2 instances can talk to each other -on a private network but it also verifies that the Nova metadata service is -properly working:: +This test case is similar to vPing_ssh but without the use of Floating IPs +and the Public/External network to transfer the ping script. +Instead, it uses Nova metadata service to pass it to the instance at booting time. +As vPing_ssh, it checks that 2 instances can talk to +each other on a Private Tenant network:: vPing_userdata test case +-------------+ +-------------+ @@ -123,7 +146,7 @@ properly working:: +-------------+ +-------------+ When the second VM boots it will execute the script passed as userdata -automatically and the ping will be detected capturing periodically the output +automatically. The ping will be detected by periodically capturing the output in the console-log of the second VM. @@ -132,7 +155,7 @@ Tempest Tempest `[2]`_ is the reference OpenStack Integration test suite. It is a set of integration tests to be run against a live OpenStack cluster. -Tempest has batteries of tests for: +Tempest has suites of tests for: * OpenStack API validation * Scenarios @@ -143,22 +166,25 @@ Rally generates automatically the Tempest configuration file **tempest.conf**. Before running the actual test cases, Functest creates the needed resources (user, tenant) and updates the appropriate parameters into the configuration file. + When the Tempest suite is executed, each test duration is measured and the full -console output is stored in a *log* file for further analysis. +console output is stored to a *log* file for further analysis. -As an addition of Arno, Brahmaputra runs a customized set of Tempest test cases. -The list is specificed through *--tests-file* when executing the Rally command. -This option has been introduced in the version 0.1.2 of the Rally framework. +The Tempest testcases are distributed accross two +Tiers: -This customized list contains more than 200 Tempest test cases and can be divided -into two main sections: + * Smoke Tier - Test Case 'tempest_smoke_serial' + * Openstack Tier - Test case 'tempest_full_parallel' - 1) Set of Tempest smoke test cases - 2) Set of test cases from DefCore list `[8]`_ +NOTE: Test case 'tempest_smoke_serial' executes a defined set of tempest smoke +tests with a single thread (i.e. serial mode). Test case 'tempest_full_parallel' +executes all defined Tempest tests using several concurrent threads +(i.e. parallel mode). The number of threads activated corresponds to the number +of available logical CPUs. The goal of the Tempest test suite is to check the basic functionalities of the -different OpenStack components on an OPNFV fresh installation using -the corresponding REST API interfaces. +different OpenStack components on an OPNFV fresh installation, using the +corresponding REST API interfaces. Rally bench test suites @@ -183,13 +209,20 @@ The OPNFV Rally scenarios are based on the collection of the actual Rally scenar * quotas * requests -A basic SLA (stop test on errors) have been implemented. +A basic SLA (stop test on errors) has been implemented. + +The Rally testcases are distributed accross two Tiers: + + * Smoke Tier - Test Case 'rally_sanity' + * Openstack Tier - Test case 'rally_full' + +NOTE: Test case 'rally_sanity' executes a limited number of Rally smoke test +cases. Test case 'rally_full' executes the full defined set of Rally tests. SDN Controllers --------------- -Brahmaputra introduces new SDN controllers. There are currently 2 available controllers: * OpenDaylight (ODL) @@ -271,56 +304,42 @@ The test cases are described as follows: * FUNCvirNetNBL3 - * Create Router: Post dataes for create Router and check it in ONOS + * Create Router: Post data for create Router and check it in ONOS * Update Router: Update the Router and compare it in ONOS - * Delete Router: Delete the Router dataes and check it in ONOS - * Create RouterInterface: Post RouterInterface data to an exist Router + * Delete Router: Delete the Router data and check it in ONOS + * Create RouterInterface: Post Router Interface data to an existing Router and check it in ONOS * Delete RouterInterface: Delete the RouterInterface and check the Router - * Create FloatingIp: Post dataes for create FloatingIp and check it in - ONOS + * Create FloatingIp: Post data for create FloatingIp and check it in ONOS * Update FloatingIp: Update the FloatingIp and compare it in ONOS - * Delete FloatingIp: Delete the FloatingIp and check if it's NULL in - ONOS or not - * Create External Gateway: Post dataes for create External Gateway to an - exit Router and check it - * Update External Gateway: Update the External Gateway and compare it - * Delete External Gateway: Delete the External Gateway and check if it's - NULL in ONOS or not - + * Delete FloatingIp: Delete the FloatingIp and check that it is 'NULL' in + ONOS + * Create External Gateway: Post data to create an External Gateway for an + existing Router and check it in ONOS + * Update External Gateway: Update the External Gateway and compare the change + * Delete External Gateway: Delete the External Gateway and check that it is + 'NULL' in ONOS -Features --------- +Open Contrail +^^^^^^^^^^^^^ +**TODO:** -vIMS -^^^^ -The IP Multimedia Subsystem or IP Multimedia Core Network Subsystem (IMS) is an -architectural framework for delivering IP multimedia services. -vIMS has been integrated in Functest to demonstrate the capability to deploy a -relatively complex NFV scenario on the OPNFV platform. The deployment of a complete -functional VNF allows the test of most of the -essential functions needed for a NFV platform. -The goal of this test suite consists of: +Features +-------- - * deploy a VNF orchestrator (Cloudify) - * deploy a Clearwater vIMS (IP Multimedia Subsystem) VNF from this - orchestrator based on a TOSCA blueprint defined in `[5]`_ - * run suite of signaling tests on top of this VNF +Doctor +^^^^^^ -The Clearwater architecture is described as follows: - -.. figure:: ../images/clearwater-architecture.png - :align: center - :alt: vIMS architecture +**TODO:** Promise ^^^^^^^ -Promise provides a basic set of test cases as part of Brahmaputra. +Promise provides a basic set of test cases as part of the deliverable. The available 33 test cases can be grouped into 7 test suites: @@ -350,7 +369,7 @@ The available 33 test cases can be grouped into 7 test suites: #. Cleanup test allocations: Destroys all allocations in OpenStack. -SDNVPN +bgpvpn ^^^^^^ Many telecom network functions are relying on layer-3 infrastructure services, within a VNF between components, or towards existing external networks. @@ -359,24 +378,58 @@ existing service provider wide-area-networks (WAN). This proven technology provides a good mechanism for inter-operation of a NFV Infrastructure (NFVI) and WAN. -The SDNVPN project defined a bgpvpn suite. -This bgpvpn suite deals with 2 Tempest cases dedicated to the test of the -OpenStack bgpvpn API: +The SDNVPN project defined a 'bgpvpn' test suite. +This 'bgpvpn' test suite deals with 2 Tempest cases dedicated to the test of +the OpenStack 'bgpvpn' API: * test_create_bgpvpn * test_create_bgpvpn_as_non_admin_fail +security_scan +^^^^^^^^^^^^^ + +**TODO:** + + + +VNF +--- + + +vIMS +^^^^ +The IP Multimedia Subsystem or IP Multimedia Core Network Subsystem (IMS) is an +architectural framework for delivering IP multimedia services. + +vIMS has been integrated in Functest to demonstrate the capability to deploy a +relatively complex NFV scenario on the OPNFV platform. The deployment of a complete +functional VNF allows the test of most of the +essential functions needed for a NFV platform. + +The goal of this test suite consists of: + + * deploy a VNF orchestrator (Cloudify) + * deploy a Clearwater vIMS (IP Multimedia Subsystem) VNF from this + orchestrator based on a TOSCA blueprint defined in `[5]`_ + * run suite of signaling tests on top of this VNF + +The Clearwater architecture is described as follows: + +.. figure:: ../images/clearwater-architecture.png + :align: center + :alt: vIMS architecture + + .. include:: ./runfunctest.rst + Test results ============ -For Brahmaputra test results, see the functest results document at `[12]`_ - Note that the results are documented per scenario basis. Although most of the test cases might show the same output, some of them are not supported by -certain scenario. Please select the appropriate scenario and compare the results -to the referenced in the documentation. +certain scenarios. Please select the appropriate scenario and compare the results +to that referenced in the documentation. Test Dashboard @@ -391,18 +444,13 @@ Based on results collected in CI, a test dashboard is dynamically generated. References ========== -.. _`[1]`: http://artifacts.opnfv.org/functest/docs/configguide/#functional-testing-installation +.. _`[1]`: http://artifacts.opnfv.org/functest/docs/configguide/# .. _`[2]`: http://docs.openstack.org/developer/tempest/overview.html .. _`[3]`: https://rally.readthedocs.org/en/latest/index.html .. _`[4]`: http://events.linuxfoundation.org/sites/events/files/slides/Functest%20in%20Depth_0.pdf .. _`[5]`: https://github.com/Orange-OpenSource/opnfv-cloudify-clearwater/blob/master/openstack-blueprint.yaml -.. _`[6]`: https://wiki.opnfv.org/opnfv_test_dashboard -.. _`[7]`: http://testresults.opnfv.org/testapi/test_projects/functest/cases -.. _`[8]`: https://wiki.openstack.org/wiki/Governance/DefCoreCommittee .. _`[9]`: https://git.opnfv.org/cgit/functest/tree/testcases/VIM/OpenStack/CI/libraries/os_defaults.yaml -.. _`[10]`: https://git.opnfv.org/cgit/functest/tree/testcases/VIM/OpenStack/CI/rally_cert/task.yaml .. _`[11]`: http://robotframework.org/ -.. _`[12]`: http://artifacts.opnfv.org/functest/brahmaputra/docs/results/index.html OPNFV main site: opnfvmain_. diff --git a/docs/userguide/introduction.rst b/docs/userguide/introduction.rst index 65f8dc48b..8624bee83 100644 --- a/docs/userguide/introduction.rst +++ b/docs/userguide/introduction.rst @@ -1,94 +1,181 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -Overview of the functest suites +Overview of the Functest suites =============================== Functest is the OPNFV project primarily targeting function testing. In the Continuous Integration pipeline, it is launched after an OPNFV fresh -installation to validate and verify the basic functions of the infrastructure. - -The current list of test suites can be distributed in 3 main domains: VIM -(Virtualised Infrastructure Manager), Controllers and Features. - -+----------------+----------------+-------------------------------------------+ -| Domain | Test suite | Comments | -+================+================+===========================================+ -| VIM | vPing | NFV "Hello World" using SSH connection | -| | | and floatting IP | -| +----------------+-------------------------------------------+ -| | vPing_userdata | Ping using userdata and cloud-init | -| | | mechanism | -| +----------------+-------------------------------------------+ -| | Tempest | OpenStack reference test suite `[2]`_ | -| +----------------+-------------------------------------------+ -| | Rally bench | OpenStack testing tool benchmarking | -| | | OpenStack modules `[3]`_ | -+----------------+----------------+-------------------------------------------+ -| | OpenDaylight | Opendaylight Test suite | -| +----------------+-------------------------------------------+ -| Controllers | ONOS | Test suite of ONOS L2 and L3 functions | -| | | See `ONOSFW User Guide`_ for details | -+----------------+----------------+-------------------------------------------+ -| Features | vIMS | Example of a real VNF deployment to show | -| | | the NFV capabilities of the platform. | -| | | The IP Multimedia Subsytem is a typical | -| | | Telco test case, referenced by ETSI. | -| | | It provides a fully functional VoIP System| -| +----------------+-------------------------------------------+ -| | Promise | Resource reservation and management | -| | | project to identify NFV related | -| | | requirements and realize resource | -| | | reservation for future usage by capacity | -| | | management of resource pools regarding | -| | | compute, network and storage. | -| | | See `Promise User Guide`_ for details | -| +----------------+-------------------------------------------+ -| | Doctor | Doctor platform, as of Brahmaputra release| -| | | , provides the two features: | -| | | * Immediate Notification | -| | | * Consistent resource state awareness | -| | | (compute), see `Doctor User Guide`_ for | -| | | details | -| +----------------+-------------------------------------------+ -| | SDNVPN | Implementation of the OpenStack bgpvpn API| -| | | from the SDNVPN feature project. | -| | | It allowing the cration of BGP VPNs | -| | | see SDNVPN User Guide for | -+----------------+----------------+-------------------------------------------+ - - -Functest includes different test suites with several test cases within. Some -of the tests are developed by Functest team members whereas others are -integrated from upstream communities or other OPNFV projects. For example, +installation to validate and verify the basic functions of the +infrastructure. + +The current list of test suites can be distributed over 4 main domains: VIM +(Virtualised Infrastructure Manager), Controllers (i.e. SDN Controllers), +Features and VNF (Virtual Network Functions). + ++-------------+---------------+----------------+----------------------------------+ +| Domain | Tier | Test case | Comments | ++=============+===============+================+==================================+ +| VIM | healthcheck | healthcheck | Verify basic operation in VIM | +| +---------------+----------------+----------------------------------+ +| | smoke | vPing_SSH | NFV "Hello World" using an SSH | +| | | | connection to a destination VM | +| | | | over a created floating IP | +| | | | address on the SUT Public / | +| | | | External network. Using the SSH | +| | | | connection a test script is then | +| | | | copied to the destination | +| | | | VM and then executed via SSH. | +| | | | The script will ping another | +| | | | VM on a specified IP address over| +| | | | the SUT Private Tenant network. | +| | +----------------+----------------------------------+ +| | | vPing_userdata | Uses Ping with given userdata | +| | | | to test intra-VM connectivity | +| | | | over the SUT Private Tenant | +| | | | network. The correct operation | +| | | | of the NOVA Metadata service is | +| | | | also verified in this test. | +| | +----------------+----------------------------------+ +| | | tempest_smoke | Generate and run a relevant | +| | | \_serial | Tempest Test Suite in smoke mode.| +| | | | The generated test set is | +| | | | dependent on the OpenStack | +| | | | deployment environment. | +| | +----------------+----------------------------------+ +| | | rally_sanity | Run a subset of the OpenStack | +| | | | Rally Test Suite in smoke mode | +| +---------------+----------------+----------------------------------+ +| | openstack | tempest_full | Generate and run a full set of | +| | | \_parallel | the OpenStack Tempest Test Suite.| +| | | | See the OpenStack reference test | +| | | | suite `[2]`_. The generated | +| | | | test set is dependent on the | +| | | | OpenStack deployment environment.| +| | +----------------+----------------------------------+ +| | | rally_full | Run the OpenStack testing tool | +| | | | benchmarking OpenStack modules | +| | | | See the Rally documents `[3]`_. | ++-------------+---------------+----------------+----------------------------------+ +| Controllers | sdn_suites | odl | Opendaylight Test suite | +| | | | TODO: Find a document reference! | +| | +----------------+----------------------------------+ +| | | onos | Test suite of ONOS L2 and L3 | +| | | | functions. | +| | | | See `ONOSFW User Guide`_ for | +| | | | details. | ++-------------+---------------+----------------+----------------------------------+ +| Features | features | Promise | Resource reservation and | +| | | | management project to identify | +| | | | NFV related requirements and | +| | | | realize resource reservation for | +| | | | future usage by capacity | +| | | | management of resource pools | +| | | | regarding compute, network and | +| | | | storage. | +| | | | See `Promise User Guide`_ for | +| | | | details. | +| | +----------------+----------------------------------+ +| | | Doctor | Doctor platform, as of Colorado | +| | | | release, provides the two | +| | | | features: | +| | | | * Immediate Notification | +| | | | * Consistent resource state | +| | | | awareness (compute). See the | +| | | | See `Doctor User Guide`_ for | +| | | | details | +| | +----------------+----------------------------------+ +| | | bgpvpn | Implementation of the OpenStack | +| | | | bgpvpn API from the SDNVPN | +| | | | feature project. It allows for | +| | | | the creation of BGP VPNs. | +| | | | See `SDNVPN User Guide`_ for | +| | | | details | +| | +----------------+----------------------------------+ +| | | security_scan | Implementation of a simple | +| | | | security scan. (Currently | +| | | | available only for the Apex | +| | | | installer environment) | +| | | | TODO: Add document link from | +| | | | Luke Hinds; when received. | ++-------------+---------------+----------------+----------------------------------+ +| VNF | vnf | vims | Example of a real VNF deployment | +| | | | to show the NFV capabilities of | +| | | | the platform. The IP Multimedia | +| | | | Subsytem is a typical Telco test | +| | | | case, referenced by ETSI. | +| | | | It provides a fully functional | +| | | | VoIP System, | ++-------------+---------------+----------------+----------------------------------+ + + +As shown in the above table, Functest is structured into different 'domains', +'tiers' and 'test cases'. Each 'test case' usually represents an actual +'Test Suite' comprised -in turn- of several test cases internally. + +Test cases also have an implicit execution order. For example, if the early +'healthcheck' Tier testcase fails, or if there are any failures in the 'smoke' +Tier testcases, there is little point to launch a full testcase execution round. + +An overview of the Functest Structural Concept is depicted graphically below: + +.. figure:: ../images/concepts_mapping_final.png + :align: center + :alt: Functest Concepts Structure + +Some of the test cases are developed by Functest team members, whereas others +are integrated from upstream communities or other OPNFV projects. For example, `Tempest <http://docs.openstack.org/developer/tempest/overview.html>`_ is the OpenStack integration test suite and Functest is in charge of the selection, -integration and automation of the tests that fit in OPNFV. +integration and automation of those tests that fit suitably to OPNFV. + +The Tempest test suite has been customized but no new test cases have been +created in OPNFV Functest. + +The results produced by the tests run from CI are pushed and collected into a +NoSQL database. The goal is to populate the database with results from different +sources and scenarios and to show them on a `Functest Dashboard`_. A screenshot +of a live Functest Dashboard is shown below: -The Tempest suite has been customized but no new test cases have been created. +.. figure:: ../images/FunctestDashboard.png + :align: center + :alt: Functest Dashboard -The results produced by the tests run from CI are pushed and collected in a NoSQL -database. The goal is to populate the database with results from different sources -and scenarios and to show them on a Dashboard. There is no real notion of Test domain or Test coverage. Basic components -(VIM, controllers) are tested through their own suites. Feature projects also -provide their own test suites with different ways of running their tests. +(VIM, SDN controllers) are tested through their own suites. Feature projects +also provide their own test suites with different ways of running their tests. vIMS test case was integrated to demonstrate the capability to deploy a relatively complex NFV scenario on top of the OPNFV infrastructure. -Functest considers OPNFV as a black box. -OPNFV, since the Brahmaputra release, offers lots of potential combinations: +Functest considers OPNFV as a black box. As of Colorado release the OPNFV +offers a lot of potential combinations: - * 2 controllers (OpenDayligh, ONOS) + * 3 controllers (OpenDaylight, ONOS, OpenContrail) * 4 installers (Apex, Compass, Fuel, Joid) -Most of the tests are runnable on any combination, but some others might have -restrictions imposed by the installers or the available deployed features. +Most of the tests are runnable by any combination, but some tests might have +restrictions imposed by the utilized installers or due to the available +deployed features. The system uses the environment variables (INSTALLER_IP and +DEPLOY_SCENARIO) to automatically determine the valid test cases; for each given +environment. + +In the Colorado OPNFV System release a convenience Functest CLI utility is also +introduced to simplify setting up the Functest evironment, management of the +OpenStack environment (e.g. resource clean-up) and for executing tests. +The Functest CLI organised the testcase into logical Tiers, which contain in +turn one or more testcases. The CLI allow execution of a single specified +testcase, all test cases in a specified Tier, or the special case of execution +of **ALL** testcases. The Functest CLI is introduced in more detail in the +section `Executing the functest suites`_ of this document. .. _`[2]`: http://docs.openstack.org/developer/tempest/overview.html .. _`[3]`: https://rally.readthedocs.org/en/latest/index.html .. _`Doctor User Guide`: http://artifacts.opnfv.org/opnfvdocs/brahmaputra/docs/userguide/featureusage-doctor.html .. _`Promise User Guide`: http://artifacts.opnfv.org/promise/brahmaputra/docs/userguide/index.html .. _`ONOSFW User Guide`: http://artifacts.opnfv.org/onosfw/brahmaputra/docs/userguide/index.html +.. _`SDNVPN User Guide`: http://artifacts.opnfv.org/sdnvpn/brahmaputra/docs/userguide/featureusage.html +.. _`Functest Dashboard`: http://testresults.opnfv.org/dashboard/ + + diff --git a/docs/userguide/runfunctest.rst b/docs/userguide/runfunctest.rst index 580fb9635..4c2e3f54b 100644 --- a/docs/userguide/runfunctest.rst +++ b/docs/userguide/runfunctest.rst @@ -10,107 +10,301 @@ Manual testing This section assumes the following: * The Functest Docker container is running * The docker prompt is shown - * The Functest environment is ready (prepare_env.sh has been executed) + * The Functest environment is ready (Functest CLI command 'functest env prepare' + has been executed) -If any of the above steps is missing please refer to the Functest Config Guide +If any of the above steps are missing please refer to the Functest Config Guide as they are a prerequisite and all the commands explained in this section **must** be performed **inside the container**. -The script **run_tests.sh** launches the test in an automated way. -Although it is possible to execute the different tests manually, it is -recommended to use the previous shell script which makes the call -to the actual scripts with the appropriate parameters. +Note: In Colorado release, the scripts **run_tests.sh** is now replaced with a +new Functest CLI. One difference, is that tests run through the Functest CLI +will always clean-up OpenStack resources. See the `Troubleshooting`_ section of this +document, where this difference is discussed. + +The Functest CLI offers two commands (functest tier ...) and (functest testcase ... ) +for the execution of Test Tiers or Test Cases:: + + root@22e436918db0:~/repos/functest/ci# functest tier --help + Usage: functest tier [OPTIONS] COMMAND [ARGS]... + + Options: + -h, --help Show this message and exit. + + Commands: + get-tests Prints the tests in a tier. + list Lists the available tiers. + run Executes all the tests within a tier. + show Shows information about a tier. + root@22e436918db0:~/repos/functest/ci# functest testcase --help + + and + + Usage: functest testcase [OPTIONS] COMMAND [ARGS]... + + Options: + -h, --help Show this message and exit. + + Commands: + list Lists the available testcases. + run Executes a test case. + show Shows information about a test case. + +More details on the existing Tiers and Test Cases can be seen with the 'list' +command:: + + root@22e436918db0:~/repos/functest/ci# functest tier list + - 0. healthcheck: + ['healthcheck'] + - 1. smoke: + ['vping_ssh', 'vping_userdata', 'tempest_smoke_serial', 'rally_sanity'] + - 2. sdn_suites: + ['odl'] + - 3. features: + ['doctor', 'security_scan'] + - 4. openstack: + ['tempest_full_parallel', 'rally_full'] + - 5. vnf: + ['vims'] + + and + + root@22e436918db0:~/repos/functest/ci# functest testcase list + healthcheck + vping_ssh + vping_userdata + tempest_smoke_serial + rally_sanity + odl + doctor + security_scan + tempest_full_parallel + rally_full + vims + +More specific details on specific Tiers or Test Cases can be seen wih the +'show' command:: + + root@22e436918db0:~/repos/functest/ci# functest tier show smoke + +======================================================================+ + | Tier: smoke | + +======================================================================+ + | Order: 1 | + | CI Loop: (daily)|(weekly) | + | Description: | + | Set of basic Functional tests to validate the OpenStack | + | deployment. | + | Test cases: | + | - vping_ssh | + | - vping_userdata | + | - tempest_smoke_serial | + | - rally_sanity | + | | + +----------------------------------------------------------------------+ + + and + + root@22e436918db0:~/repos/functest/ci# functest testcase show tempest_smoke_serial + +======================================================================+ + | Testcase: tempest_smoke_serial | + +======================================================================+ + | Description: | + | This test case runs the smoke subset of the OpenStack Tempest | + | suite. The list of test cases is generated by Tempest | + | automatically and depends on the parameters of the OpenStack | + | deplopyment. | + | Dependencies: | + | - Installer: | + | - Scenario : | + | | + +----------------------------------------------------------------------+ + + +To execute a Test Tier or Test Case, the 'run' command is used:: + + root@22e436918db0:~/repos/functest/ci# functest tier run healthcheck + Executing command: 'python /home/opnfv/repos/functest/ci/run_tests.py -t healthcheck' + 2016-06-30 11:44:56,933 - run_tests - INFO - Sourcing the OpenStack RC file... + 2016-06-30 11:44:56,937 - run_tests - INFO - ############################################ + 2016-06-30 11:44:56,938 - run_tests - INFO - Running tier 'healthcheck' + 2016-06-30 11:44:56,938 - run_tests - INFO - ############################################ + 2016-06-30 11:44:56,938 - run_tests - INFO - ============================================ + 2016-06-30 11:44:56,938 - run_tests - INFO - Running test case 'healthcheck'... + 2016-06-30 11:44:56,938 - run_tests - INFO - ============================================ + 2016-06-30 11:44:56,953 - healtcheck - INFO - Testing Keystone API... + 2016-06-30 11:45:05,351 - healtcheck - INFO - ...Keystone OK! + 2016-06-30 11:45:05,354 - healtcheck - INFO - Testing Glance API... + 2016-06-30 11:45:29,746 - healtcheck - INFO - ... Glance OK! + 2016-06-30 11:45:29,749 - healtcheck - INFO - Testing Cinder API... + 2016-06-30 11:45:37,502 - healtcheck - INFO - ...Cinder OK! + 2016-06-30 11:45:37,505 - healtcheck - INFO - Testing Neutron API... + 2016-06-30 11:45:39,664 - healtcheck - INFO - External network found. ccd98ad6-d34a-4768-b03c-e28ecfcd51ca + 2016-06-30 11:45:39,667 - healtcheck - INFO - 1. Create Networks... + 2016-06-30 11:45:44,227 - healtcheck - INFO - 2. Create subnets... + 2016-06-30 11:45:46,805 - healtcheck - INFO - 4. Create Routers... + 2016-06-30 11:45:54,261 - healtcheck - INFO - ...Neutron OK! + 2016-06-30 11:45:54,264 - healtcheck - INFO - Testing Nova API... + 2016-06-30 11:47:12,272 - healtcheck - INFO - ...Nova OK! + 2016-06-30 11:47:12,274 - healtcheck - INFO - Checking if instances get an IP from DHCP... + : + : + 2016-06-30 11:48:17,832 - healtcheck - INFO - ...DHCP OK! + 2016-06-30 11:48:17,835 - healtcheck - INFO - Health check passed! + 2016-06-30 11:48:17,837 - clean_openstack - INFO - +++++++++++++++++++++++++++++++ + 2016-06-30 11:48:17,837 - clean_openstack - INFO - Cleaning OpenStack resources... + 2016-06-30 11:48:17,837 - clean_openstack - INFO - +++++++++++++++++++++++++++++++ + Version 1 is deprecated, use alternative version 2 instead. + WARNING:cinderclient.api_versions:Version 1 is deprecated, use alternative version 2 instead. + 2016-06-30 11:48:18,272 - clean_openstack - INFO - Removing Nova instances... + 2016-06-30 11:48:24,439 - clean_openstack - INFO - ------------------------------------------- + 2016-06-30 11:48:24,440 - clean_openstack - INFO - Removing Glance images... + 2016-06-30 11:48:35,853 - clean_openstack - INFO - ------------------------------------------- + 2016-06-30 11:48:35,854 - clean_openstack - INFO - Removing Cinder volumes... + 2016-06-30 11:48:37,344 - clean_openstack - INFO - ------------------------------------------- + 2016-06-30 11:48:37,344 - clean_openstack - INFO - Removing floating IPs... + 2016-06-30 11:48:37,467 - clean_openstack - INFO - ------------------------------------------- + 2016-06-30 11:48:37,467 - clean_openstack - INFO - Removing Neutron objects + 2016-06-30 11:48:53,633 - clean_openstack - INFO - ------------------------------------------- + 2016-06-30 11:48:53,633 - clean_openstack - INFO - Removing Security groups... + 2016-06-30 11:48:53,689 - clean_openstack - INFO - ------------------------------------------- + 2016-06-30 11:48:53,689 - clean_openstack - INFO - Removing Users... + 2016-06-30 11:48:54,444 - clean_openstack - INFO - ------------------------------------------- + 2016-06-30 11:48:54,444 - clean_openstack - INFO - Removing Tenants... + 2016-06-30 11:48:54,711 - clean_openstack - INFO - ------------------------------------------- + + and + + root@22e436918db0:~/repos/functest/ci# functest testcase run vping_ssh + Executing command: 'python /home/opnfv/repos/functest/ci/run_tests.py -t vping_ssh' + 2016-06-30 11:50:31,861 - run_tests - INFO - Sourcing the OpenStack RC file... + 2016-06-30 11:50:31,865 - run_tests - INFO - ============================================ + 2016-06-30 11:50:31,865 - run_tests - INFO - Running test case 'vping_ssh'... + 2016-06-30 11:50:31,865 - run_tests - INFO - ============================================ + 2016-06-30 11:50:32,977 - vping_ssh - INFO - Creating image 'functest-vping' from '/home/opnfv/functest/data/cirros-0.3.4-x86_64-disk.img'... + 2016-06-30 11:50:45,470 - vping_ssh - INFO - Creating neutron network vping-net... + 2016-06-30 11:50:47,645 - vping_ssh - INFO - Creating security group 'vPing-sg'... + 2016-06-30 11:50:48,843 - vping_ssh - INFO - Using existing Flavor 'm1.small'... + 2016-06-30 11:50:48,927 - vping_ssh - INFO - vPing Start Time:'2016-06-30 11:50:48' + 2016-06-30 11:50:48,927 - vping_ssh - INFO - Creating instance 'opnfv-vping-1'... + 2016-06-30 11:51:34,664 - vping_ssh - INFO - Instance 'opnfv-vping-1' is ACTIVE. + 2016-06-30 11:51:34,818 - vping_ssh - INFO - Adding 'opnfv-vping-1' to security group 'vPing-sg'... + 2016-06-30 11:51:35,209 - vping_ssh - INFO - Creating instance 'opnfv-vping-2'... + 2016-06-30 11:52:01,439 - vping_ssh - INFO - Instance 'opnfv-vping-2' is ACTIVE. + 2016-06-30 11:52:01,439 - vping_ssh - INFO - Adding 'opnfv-vping-2' to security group 'vPing-sg'... + 2016-06-30 11:52:01,754 - vping_ssh - INFO - Creating floating IP for VM 'opnfv-vping-2'... + 2016-06-30 11:52:01,969 - vping_ssh - INFO - Floating IP created: '10.17.94.140' + 2016-06-30 11:52:01,969 - vping_ssh - INFO - Associating floating ip: '10.17.94.140' to VM 'opnfv-vping-2' + 2016-06-30 11:52:02,792 - vping_ssh - INFO - Trying to establish SSH connection to 10.17.94.140... + 2016-06-30 11:52:19,915 - vping_ssh - INFO - Waiting for ping... + 2016-06-30 11:52:21,108 - vping_ssh - INFO - vPing detected! + 2016-06-30 11:52:21,108 - vping_ssh - INFO - vPing duration:'92.2' s. + 2016-06-30 11:52:21,109 - vping_ssh - INFO - vPing OK + 2016-06-30 11:52:21,153 - clean_openstack - INFO - +++++++++++++++++++++++++++++++ + 2016-06-30 11:52:21,153 - clean_openstack - INFO - Cleaning OpenStack resources... + 2016-06-30 11:52:21,153 - clean_openstack - INFO - +++++++++++++++++++++++++++++++ + Version 1 is deprecated, use alternative version 2 instead. + : + : + etc. + +To list the test cases which are part of a specific Test Tier, the 'get-tests' +command is used with 'functest tier':: + + root@22e436918db0:~/repos/functest/ci# functest tier get-tests sdn_suites + Test cases in tier 'sdn_suites': + ['odl'] -It is located in *$repos_dir/functest/docker* and it has -several options:: - - ./run_tests.sh -h - Script to trigger the tests automatically. - - usage: - bash run_tests.sh [-h|--help] [-r|--report] [-n|--no-clean] [-t|--test <test_name>] - - where: - -h|--help show this help text - -r|--report push results to database (false by default) - -n|--no-clean do not clean up OpenStack resources after test run - -s|--serial run tests in one thread - -t|--test run specific set of tests - <test_name> one or more of the following separated by comma: - vping_ssh,vping_userdata,odl,onos,tempest,rally,vims,promise,doctor,bgpvpn - - examples: - run_tests.sh - run_tests.sh --test vping_ssh,odl - run_tests.sh -t tempest,rally --no-clean - -The *-r* option is used by the OPNFV Continuous Integration automation mechanisms -in order to push the test results into the NoSQL results collection database. -This database is read only for a regular user given that it needs special rights -and special conditions to push data. - -The *-t* option can be used to specify the list of a desired test to be launched, -by default Functest will launch all the test suites in the following order: - - 1) vPing test cases - 2) Tempest suite - 3) SDN controller suites - 4) Feature project tests cases (Promise, Doctor, SDNVPN) - 5) vIMS suite - 6) Rally suite Please note that for some scenarios some test cases might not be launched. -Functest calculates automatically which test can be executed and which cannot given -the environment variable **DEPLOY_SCENARIO** to the docker container. - -A single or set of test may be launched at once using *-t <test_name>* specifying -the test name or names separated by commas in the following list: -*[vping_ssh,vping_userdata,odl,onos,rally,tempest,vims,promise,doctor]*. - -Functest includes cleaning mechanism in order to remove -all the OpenStack resources except what was present before running any test. The script -*$repos_dir/functest/utils/generate_defaults.py* -is called once by *prepare_env.sh* when setting up the Functest environment -to snapshot all the OpenStack resources (images, networks, volumes, security groups, -tenants, users) so that an eventual cleanup does not remove any of this defaults. - -The *-n* option is used for preserving all the possible OpenStack resources created -by the tests after their execution. - -The *-s* option forces execution of test cases in a single thread. Currently this -option affects Tempest test cases only and can be used e.g. for troubleshooting -concurrency problems. +For example, the last example displayed only the 'odl' testcase for the given +environment. In this system the deployment does not support the 'onos' SDN +Controller Test Case; for example. + +**Important** If you use the command 'functest tier run <tier_name>', then the +Functest CLI utility will call **all valid Test Cases**, which belong to the +specified Test Tier, as relevant to scenarios deployed to the SUT environment. +Thus, the Functest CLI utility calculates automatically which tests can be +executed and which cannot, given the environment variable **DEPLOY_SCENARIO**, +which is passed in to the Functest docker container. + +Currently, the Functest CLI command 'functest testcase run <testcase_name>', supports +two possibilities:: + + * Run a single Test Case, specified by a valid choice of <testcase_name> + * Run ALL test Test Cases (for all Tiers) by specifying <testcase_name> = 'all' + +Example:: + + root@22e436918db0:~/repos/functest/ci# functest testcase run all + Executing command: 'python /home/opnfv/repos/functest/ci/run_tests.py -t all' + 2016-06-30 12:03:28,628 - run_tests - INFO - Sourcing the OpenStack RC file... + 2016-06-30 12:03:28,634 - run_tests - INFO - Tiers to be executed: + - 0. healthcheck: + ['healthcheck'] + - 1. smoke: + ['vping_ssh', 'vping_userdata', 'tempest_smoke_serial', 'rally_sanity'] + - 2. sdn_suites: + ['odl'] + - 3. features: + ['doctor', 'security_scan'] + - 4. openstack: + ['tempest_full_parallel', 'rally_full'] + - 5. vnf: + ['vims'] + 2016-06-30 12:03:28,634 - run_tests - INFO - ############################################ + 2016-06-30 12:03:28,635 - run_tests - INFO - Running tier 'healthcheck' + 2016-06-30 12:03:28,635 - run_tests - INFO - ############################################ + 2016-06-30 12:03:28,635 - run_tests - INFO - ============================================ + 2016-06-30 12:03:28,635 - run_tests - INFO - Running test case 'healthcheck'... + 2016-06-30 12:03:28,635 - run_tests - INFO - ============================================ + 2016-06-30 12:03:28,651 - healtcheck - INFO - Testing Keystone API... + 2016-06-30 12:03:36,676 - healtcheck - INFO - ...Keystone OK! + 2016-06-30 12:03:36,679 - healtcheck - INFO - Testing Glance API... + : + : + etc. + +Functest includes cleaning mechanism in order to remove all the OpenStack +resources except what was present before running any test. The script +*$repos_dir/functest/utils/generate_defaults.py* is called once when setting up +the Functest environment (i.e. CLI command 'functest env prepare') to snapshot +all the OpenStack resources (images, networks, volumes, security groups, tenants, +users) so that an eventual cleanup does not remove any of this defaults. The script **clean_openstack.py** which is located in -*$repos_dir/functest/testcases/VIM/OpenStack/CI/libraries/* -is normally called after a test execution if the *-n* is not specified. It -is in charge of cleaning the OpenStack resources that are not specified -in the defaults file generated previously which is stored in -*/home/opnfv/functest/conf/os_defaults.yaml* in the docker -container. +*$repos_dir/functest/utils/* is normally called after a test execution. It is +in charge of cleaning the OpenStack resources that are not specified in the +defaults file generated previously which is stored in +*/home/opnfv/functest/conf/os_defaults.yaml* in the Functest docker container. It is important to mention that if there are new OpenStack resources created -manually after preparing the Functest environment, they will be removed if this -flag is not specified in the *run_tests.sh* command. +manually after preparing the Functest environment, they will be removed, unless +you use the special method of invoking the test case with specific suppression +of clean up. (See the `Troubleshooting`_ section). + The reason to include this cleanup meachanism in Functest is because some test suites such as Tempest or Rally create a lot of resources (users, tenants, networks, volumes etc.) that are not always properly cleaned, so this function has been set to keep the system as clean as it was before a full Functest execution. -Although **run_tests.sh** provides an easy way to run any test, it is possible to +Although the Functest CLI provides an easy way to run any test, it is possible to do a direct call to the desired test script. For example: - python $repos_dir/functest/testcases/vPing/vPing_ssh.py -d + python $repos_dir/functest/testcases/OpenStack/vPing/vPing_ssh.py -d + Automated testing ----------------- -As mentioned previously, the **prepare-env.sh** and **run_test.sh** can be called within -the container from Jenkins. There are 2 jobs that automate all the manual steps -explained in the previous section. One job runs all the tests and the other one allows testing -test suite by test suite specifying the test name. The user might use one or -the other job to execute the desired test suites. +**TODO** Jose, the next section has not yet been revised! + +As mentioned previously, the Functest Docker container preparation as well as +invocation of Test Cases can be called within the container from the Jenkins CI +system. There are 2 jobs that automate the whole process. The first job runs all +the tests and the second job allows testing test suite by test suite specifying +the test suite name. The user may also use either of these Jenkins jobs to execute +the desired test suites. One of the most challenging task in the Brahmaputra release consists in dealing with lots of scenarios and installers. Thus, when the tests are @@ -126,9 +320,9 @@ variables: * Installer IP of the engine or VM running the actual deployment, stored in INSTALLER_IP * The scenario [controller]-[feature]-[mode], stored in DEPLOY_SCENARIO with - * controller = (odl|onos|ocl|nosdn) - * feature = (ovs(dpdk)|kvm|sfc|bgpvpn) - * mode = (ha|noha) + * controller = (odl|onos|ocl|nosdn) + * feature = (ovs(dpdk)|kvm|sfc|bgpvpn) + * mode = (ha|noha) The constraints per test case are defined in the Functest configuration file */home/opnfv/functest/config/config_functest.yaml*:: @@ -184,3 +378,7 @@ The tests are executed in the following order: As explained before, at the end of an automated execution, the OpenStack resources might be eventually removed. + +**END of TODO** + + diff --git a/docs/userguide/troubleshooting.rst b/docs/userguide/troubleshooting.rst index 99b92977b..cd79bafcf 100644 --- a/docs/userguide/troubleshooting.rst +++ b/docs/userguide/troubleshooting.rst @@ -8,7 +8,8 @@ This section gives some guidelines about how to troubleshoot the test cases owned by Functest. **IMPORTANT**: As in the previous section, the steps defined below must be -executed inside the Functest Docker container and after sourcing the OpenStack credentials:: +executed inside the Functest Docker container and after sourcing the OpenStack +credentials:: . $creds @@ -19,7 +20,9 @@ or:: VIM --- -This section covers the test cases related to the VIM (vPing, Tempest, Rally). +This section covers the test cases related to the VIM (healthcheck, vping_ssh, +vping_userdata, tempest_smoke_serial, tempest_full_parallel, rally_sanity, +rally_full). vPing common ^^^^^^^^^^^^ @@ -29,29 +32,45 @@ similar: * Create Glance image * Create Network * Create Security Group - * Create instances + * Create Instances -After these actions, the test cases differ and will be explained in their respective section. +After these actions, the test cases differ and will be explained in their +respective section. -These test cases can be run inside the container as follows:: +These test cases can be run inside the container, using new Functest CLI as follows:: - $repos_dir/functest/docker/run_tests.sh -t vping_ssh - $repos_dir/functest/docker/run_tests.sh -t vping_userdata + $ functest testcase run vping_ssh + $ functest testcase run vping_userdata -The **run_tests.sh** script is basically calling internally the corresponding -vPing scripts, located in +The Functest CLI is designed to route a call to the corresponding internal +python scripts, located in paths: *$repos_dir/functest/testcases/vPing/CI/libraries/vPing_ssh.py* and -*$repos_dir/functest/testcases/vPing/CI/libraries/vPing_userdata.py* with the -appropriate flags. +*$repos_dir/functest/testcases/vPing/CI/libraries/vPing_userdata.py* -After finishing the test execution, the corresponding script will remove all -created resources in OpenStack (image, instances, network and security group). -When troubleshooting, it is advisable sometimes to keep those resources in case the -test fails and a manual testing is needed. This can be achieved by adding the flag *-n*:: +Notes: + #. In this Colorado Funtest Userguide, the use of the Functest CLI is + emphasized. The Functest CLI replaces the earlier Bash shell script + *run_tests.sh*. - $repos_dir/functest/docker/run_tests.sh -n -t vping_ssh - $repos_dir/functest/docker/run_tests.sh -n -t vping_userdata + #. There is one difference, between the Functest CLI based test case + execution compared to the earlier used Bash shell script, which is + relevant to point out in troubleshooting scenarios: + The Functest CLI does **not yet** support the option to suppress + clean-up of the generated OpenStack resources, following the execution + of a test case. + + Explanation: After finishing the test execution, the corresponding + script will remove, by default, all created resources in OpenStack + (image, instances, network and security group). When troubleshooting, + it is advisable sometimes to keep those resources in case the test + fails and a manual testing is needed. + + It is actually still possible to invoke test execution, with suppression + of OpenStack resource cleanup, however this requires invocation of a + **specific Python script:** '/home/opnfv/repos/functest/ci/run_test.py'. + The `OPNFV Functest Developer Guide`_ provides guidance on the use of that + Python script in such troubleshooting cases. Some of the common errors that can appear in this test case are:: @@ -63,7 +82,8 @@ The debug messages will also help to see when it failed (subnet and router creat Example of Neutron commands (using 10.6.0.0/24 range for example):: neutron net-create net-test - neutron subnet-create --name subnet-test --allocation-pool start=10.6.0.2,end=10.6.0.100 --gateway 10.6.0.254 net-test 10.6.0.0/24 + neutron subnet-create --name subnet-test --allocation-pool start=10.6.0.2,end=10.6.0.100 \ + --gateway 10.6.0.254 net-test 10.6.0.0/24 neutron router-create test_router neutron router-interface-add <ROUTER_ID> test_subnet neutron router-gateway-set <ROUTER_ID> <EXT_NET_NAME> @@ -75,9 +95,12 @@ Another related error can occur while creating the Security Groups for the insta In this case, proceed to create it manually. These are some hints:: neutron security-group-create sg-test - neutron security-group-rule-create sg-test --direction ingress --protocol icmp --remote-ip-prefix 0.0.0.0/0 - neutron security-group-rule-create sg-test --direction ingress --ethertype IPv4 --protocol tcp --port-range-min 80 --port-range-max 80 --remote-ip-prefix 0.0.0.0/0 - neutron security-group-rule-create sg-test --direction egress --ethertype IPv4 --protocol tcp --port-range-min 80 --port-range-max 80 --remote-ip-prefix 0.0.0.0/0 + neutron security-group-rule-create sg-test --direction ingress --protocol icmp \ + --remote-ip-prefix 0.0.0.0/0 + neutron security-group-rule-create sg-test --direction ingress --ethertype IPv4 \ + --protocol tcp --port-range-min 80 --port-range-max 80 --remote-ip-prefix 0.0.0.0/0 + neutron security-group-rule-create sg-test --direction egress --ethertype IPv4 \ + --protocol tcp --port-range-min 80 --port-range-max 80 --remote-ip-prefix 0.0.0.0/0 The next step is to create the instances. The image used is located in */home/opnfv/functest/data/cirros-0.3.4-x86_64-disk.img* and a Glance image is created @@ -100,23 +123,24 @@ vPing_SSH ^^^^^^^^^ This test case creates a floating IP on the external network and assigns it to the second instance **opnfv-vping-2**. The purpose of this is to establish -a SSH connection to that instance and SCP a script that will ping the first instance. -This script is located in the repository under -*$repos_dir/functest/testcases/vPing/CI/libraries/ping.sh* and takes an IP as +a SSH connection to that instance and SCP a script that will ping the first +instance. This script is located in the repository under +*$repos_dir/functest/testcases/OpenStack/vPing/ping.sh* and takes an IP as a parameter. When the SCP is completed, the test will do an SSH call to that script inside the second instance. Some problems can happen here:: vPing_ssh- ERROR - Cannot establish connection to IP xxx.xxx.xxx.xxx. Aborting -If this is displayed, stop the test or wait for it to finish (if you have used the flag -*-n* in **run_tests.sh** explained previously) so that the test does not clean -the OpenStack resources. It means that the Container can not reach the public -IP assigned to the instance **opnfv-vping-2**. There are many possible reasons, and -they really depend on the chosen scenario. For most of the ODL-L3 and ONOS scenarios -this has been noticed and it is a known limitation. +If this is displayed, stop the test or wait for it to finish, if you have used +the special method of test invocation with specific supression of OpenStack +resource clean-up, as explained earler. It means that the Container can not +reach the Public/External IP assigned to the instance **opnfv-vping-2**. There +are many possible reasons, and they really depend on the chosen scenario. For +most of the ODL-L3 and ONOS scenarios this has been noticed and it is a known +limitation. -First, make sure that the instance **opnfv-vping-2** succeeded to get an IP from -the DHCP agent. It can be checked by doing:: +First, make sure that the instance **opnfv-vping-2** succeeded to get an IP +from the DHCP agent. It can be checked by doing:: nova console-log opnfv-vping-2 @@ -132,10 +156,12 @@ If the instance got an IP properly, try to ping manually the VM from the contain If the ping does not return anything, try to ping from the Host where the Docker container is running. If that solves the problem, check the iptable rules because -there might be some rules rejecting ICMP or TCP traffic coming/going from/to the container. +there might be some rules rejecting ICMP or TCP traffic coming/going from/to the +container. At this point, if the ping does not work either, try to reproduce the test -manually with the steps described above in the vPing common section with the addition:: +manually with the steps described above in the vPing common section with the +addition:: neutron floatingip-create <EXT_NET_NAME> nova floating-ip-associate nova-test <FLOATING_IP> @@ -187,8 +213,8 @@ If this text or similar is shown:: it means that the instance failed to read from the metadata service. Contact the Functest or installer teams for more information. -NOTE: Cloud-init in not supported on scenario dealing with ONOS and the tests have been -excluded from CI in those scenarios. +NOTE: Cloud-init in not supported on scenarios dealing with ONOS and the tests +have been excluded from CI in those scenarios. Tempest @@ -198,65 +224,65 @@ In the upstream OpenStack CI all the Tempest test cases are supposed to pass. If some test cases fail in an OPNFV deployment, the reason is very probably one of the following -+-----------------------------+------------------------------------------------+ -| Error | Details | -+=============================+================================================+ -| Resources required for test | Such resources could be e.g. an external | -| case execution are missing | network and access to the management subnet | -| | (adminURL) from the Functest docker container. | -+-----------------------------+------------------------------------------------+ -| OpenStack components or | Check running services in the controller and | -| services are missing or not | compute nodes (e.g. with "systemctl" or | -| configured properly | "service" commands). Configuration parameters | -| | can be verified from related .conf files | -| | located under /etc/<component> directories. | -+-----------------------------+------------------------------------------------+ -| Some resources required for | The tempest.conf file, automatically generated | -| execution test cases are | by Rally in Functest, does not contain all the | -| missing | needed parameters or some parameters are not | -| | set properly. | -| | The tempest.conf file is located in /home/opnfv| -| | /.rally/tempest/for-deployment-<UUID> in | -| | Functest container | -| | Use "rally deployment list" command in order to| -| | check UUID of current deployment. | -+-----------------------------+------------------------------------------------+ - - -When some Tempest test case fails, captured traceback and possibly also related -REST API requests/responses are output to the console. -More detailed debug information can be found from tempest.log file stored into -related Rally deployment folder. ++-----------------------------+-----------------------------------------------------+ +| Error | Details | ++=============================+=====================================================+ +| Resources required for test | Such resources could be e.g. an external network | +| case execution are missing | and access to the management subnet (adminURL) from | +| | the Functest docker container. | ++-----------------------------+-----------------------------------------------------+ +| OpenStack components or | Check running services in the controller and compute| +| services are missing or not | nodes (e.g. with "systemctl" or "service" commands).| +| configured properly | Configuration parameters can be verified from the | +| | related .conf files located under '/etc/<component>'| +| | directories. | ++-----------------------------+-----------------------------------------------------+ +| Some resources required for | The tempest.conf file, automatically generated by | +| execution test cases are | Rally in Functest, does not contain all the needed | +| missing | parameters or some parameters are not set properly. | +| | The tempest.conf file is located in directory | +| | '/home/opnfv/.rally/tempest/for-deployment-<UUID>' | +| | in the Functest Docker container. Use the "rally | +| | deployment list" command in order to check the UUID | +| | the UUID of the current deployment. | ++-----------------------------+-----------------------------------------------------+ + + +When some Tempest test case fails, captured traceback and possibly also the +related REST API requests/responses are output to the console. More detailed debug +information can be found from tempest.log file stored into related Rally deployment +folder. Rally ^^^^^ -Same error causes than for Tempest mentioned above may lead to errors in Rally. +The same error causes which were mentioned above for Tempest test cases, may also +lead to errors in Rally as well. It is possible to run only one Rally scenario, instead of the whole suite. -To do that, call the python script (instead of *run_tests.sh*) as follows:: +To do that, call the alternative python script as follows:: - python $repos_dir/functest/testcases/VIM/OpenStack/CI/libraries/run_rally-cert.py -h - usage: run_rally-cert.py [-h] [-d] [-r] [-s] [-v] [-n] test_name + python $repos_dir/functest/testcases/OpenStack/rally/run_rally-cert.py -h + usage: run_rally-cert.py [-h] [-d] [-r] [-s] [-v] [-n] test_name - positional arguments: - test_name Module name to be tested. Possible values are : [ - authenticate | glance | cinder | heat | keystone | neutron | - nova | quotas | requests | vm | all ] The 'all' value - performs all possible test scenarios + positional arguments: + test_name Module name to be tested. Possible values are : [ + authenticate | glance | cinder | heat | keystone | neutron | + nova | quotas | requests | vm | all ] The 'all' value + performs all possible test scenarios - optional arguments: - -h, --help show this help message and exit - -d, --debug Debug mode - -r, --report Create json result file - -s, --smoke Smoke test mode - -v, --verbose Print verbose info about the progress - -n, --noclean Don't clean the created resources for this test. + optional arguments: + -h, --help show this help message and exit + -d, --debug Debug mode + -r, --report Create json result file + -s, --smoke Smoke test mode + -v, --verbose Print verbose info about the progress + -n, --noclean Don't clean the created resources for this test. For example, to run the Glance scenario with debug information:: - python $repos_dir/functest/testcases/VIM/OpenStack/CI/libraries/run_rally-cert.py -d glance + python $repos_dir/functest/testcases/OpenStack/rally/run_rally-cert.py -d glance Possible scenarios are: * authenticate @@ -270,16 +296,17 @@ Possible scenarios are: * requests * vm -To know more about what those scenarios are doing, they are defined in: -*$repos_dir/functest/testcases/VIM/OpenStack/CI/rally_cert/scenario*. For more info about -Rally scenario definition please refer to the Rally official documentation. +To know more about what those scenarios are doing, they are defined in directory: +*$repos_dir/functest/testcases/OpenStack/rally/scenario* +For more info about Rally scenario definition please refer to the Rally official +documentation. `[3]`_ If the flag *all* is specified, it will run all the scenarios one by one. Please -note that this might take some time (~1,5hr), taking around 1 hour to complete the -Nova scenario. +note that this might take some time (~1,5hr), taking around 1 hour alone to +complete the Nova scenario. -To check any possible problems with rally, the logs are stored under -*/home/opnfv/functest/results/rally/* in the Functest container. +To check any possible problems with Rally, the logs are stored under +*/home/opnfv/functest/results/rally/* in the Functest Docker container. Controllers @@ -287,7 +314,7 @@ Controllers ODL ^^^ -2 versions are supported in Brahmaputra depending on the scenario: +2 versions are supported in Colorado, depending on the scenario: * Lithium * Berylium @@ -298,10 +325,30 @@ due to wrong return code. ONOS ^^^^ -Please refer to the ONOS documentation. +Please refer to the ONOS documentation. `ONOSFW User Guide`_ . + + +Features +-------- + + +Doctor +^^^^^^ +Please refer to the Doctor documentation. `Doctor User Guide`_ + + +Promise +^^^^^^^ +Please refer to the Promise documentation. `Promise User Guide`_ + -Feature -------- +bgpvpn +^^^^^^ +Please refer to the SNVPN documentation. `SDNVPN User Guide`_ + + +NFV +--- vIMS ^^^^ @@ -326,7 +373,7 @@ described in the following table: | Impossible to create a volume | VNF cannot be deployed | +-----------------------------------+------------------------------------+ | SSH connection issue between the | if vPing test fails, vIMS test will| -| Test container and the VM | fail... | +| Test Docker container and the VM | fail... | +-----------------------------------+------------------------------------+ | No Internet access from the VM | the VMs of the VNF must have an | | | external access to Internet | @@ -336,13 +383,4 @@ described in the following table: +-----------------------------------+------------------------------------+ -Promise -^^^^^^^ - -Please refer to the Promise documentation. - - -SDNVPN -^^^^^^^ - -Please refer to the SNVPN documentation. +.. _`OPNFV Functest Developer Guide`: http://artifacts.opnfv.org/functest/docs/devguide/# diff --git a/testcases/OpenStack/examples/create_instance_and_ip.py b/testcases/OpenStack/examples/create_instance_and_ip.py new file mode 100644 index 000000000..41f8a9c8a --- /dev/null +++ b/testcases/OpenStack/examples/create_instance_and_ip.py @@ -0,0 +1,136 @@ +#!/usr/bin/python +# +# Copyright (c) 2015 All rights reserved +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# This script boots an instance and assigns a floating ip +# + +import argparse +import os +import sys +import functest.utils.functest_logger as ft_logger +import functest.utils.functest_utils as ft_utils +import functest.utils.openstack_utils as os_utils + +parser = argparse.ArgumentParser() + +parser.add_argument("-r", "--report", + help="Create json result file", + action="store_true") + +args = parser.parse_args() + +""" logging configuration """ +logger = ft_logger.Logger("create_instance_and_ip").getLogger() + +REPO_PATH = os.environ['repos_dir'] + '/functest/' +HOME = os.environ['HOME'] + "/" + +VM_BOOT_TIMEOUT = 180 + +INSTANCE_NAME = ft_utils.get_parameter_from_yaml("example.example_vm_name") +FLAVOR = ft_utils.get_parameter_from_yaml("example.example_flavor") +IMAGE_NAME = ft_utils.get_parameter_from_yaml("example.example_image_name") +IMAGE_FILENAME = ft_utils.get_parameter_from_yaml( + "general.openstack.image_file_name") +IMAGE_FORMAT = ft_utils.get_parameter_from_yaml( + "general.openstack.image_disk_format") +IMAGE_PATH = ft_utils.get_parameter_from_yaml( + "general.directories.dir_functest_data") + "/" + IMAGE_FILENAME + +# NEUTRON Private Network parameters + +NET_NAME = ft_utils.get_parameter_from_yaml( + "example.example_private_net_name") +SUBNET_NAME = ft_utils.get_parameter_from_yaml( + "example.example_private_subnet_name") +SUBNET_CIDR = ft_utils.get_parameter_from_yaml( + "example.example_private_subnet_cidr") +ROUTER_NAME = ft_utils.get_parameter_from_yaml( + "example.example_router_name") + +SECGROUP_NAME = ft_utils.get_parameter_from_yaml( + "example.example_sg_name") +SECGROUP_DESCR = ft_utils.get_parameter_from_yaml( + "example.example_sg_descr") + +TEST_DB = ft_utils.get_parameter_from_yaml("results.test_db_url") + + +def main(): + + nova_client = os_utils.get_nova_client() + neutron_client = os_utils.get_neutron_client() + glance_client = os_utils.get_glance_client() + + image_id = os_utils.create_glance_image(glance_client, + IMAGE_NAME, + IMAGE_PATH, + disk=IMAGE_FORMAT, + container="bare", + public=True, + logger=logger) + + network_dic = os_utils.create_network_full(logger, + neutron_client, + NET_NAME, + SUBNET_NAME, + ROUTER_NAME, + SUBNET_CIDR) + if not network_dic: + logger.error( + "There has been a problem when creating the neutron network") + sys.exit(-1) + + network_id = network_dic["net_id"] + + sg_id = os_utils.create_security_group_full(logger, neutron_client, + SECGROUP_NAME, SECGROUP_DESCR) + + # boot INTANCE + logger.info("Creating instance '%s'..." % INSTANCE_NAME) + logger.debug( + "Configuration:\n name=%s \n flavor=%s \n image=%s \n " + "network=%s \n" % (INSTANCE_NAME, FLAVOR, image_id, network_id)) + instance = os_utils.create_instance_and_wait_for_active(FLAVOR, + image_id, + network_id, + INSTANCE_NAME) + + if instance is None: + logger.error("Error while booting instance.") + sys.exit(-1) + # Retrieve IP of INSTANCE + instance_ip = instance.networks.get(NET_NAME)[0] + logger.debug("Instance '%s' got private ip '%s'." % + (INSTANCE_NAME, instance_ip)) + + logger.info("Adding '%s' to security group '%s'..." + % (INSTANCE_NAME, SECGROUP_NAME)) + os_utils.add_secgroup_to_instance(nova_client, instance.id, sg_id) + + logger.info("Creating floating IP for VM '%s'..." % INSTANCE_NAME) + floatip_dic = os_utils.create_floating_ip(neutron_client) + floatip = floatip_dic['fip_addr'] + # floatip_id = floatip_dic['fip_id'] + + if floatip is None: + logger.error("Cannot create floating IP.") + sys.exit(-1) + logger.info("Floating IP created: '%s'" % floatip) + + logger.info("Associating floating ip: '%s' to VM '%s' " + % (floatip, INSTANCE_NAME)) + if not os_utils.add_floating_ip(nova_client, instance.id, floatip): + logger.error("Cannot associate floating IP to VM.") + sys.exit(-1) + + sys.exit(0) + +if __name__ == '__main__': + main() diff --git a/utils/openstack_utils.py b/utils/openstack_utils.py index 9640fb9b4..1379d782d 100644 --- a/utils/openstack_utils.py +++ b/utils/openstack_utils.py @@ -191,9 +191,9 @@ def create_flavor(nova_client, flavor_name, ram, disk, vcpus): def create_instance(flavor_name, image_id, network_id, - instance_name="", - config_drive=False, - userdata=""): + instance_name="functest-vm", + confdrive=True, + userdata=None): nova_client = get_nova_client() try: flavor = nova_client.flavors.find(name=flavor_name) @@ -202,14 +202,23 @@ def create_instance(flavor_name, flavor_name) print(nova_client.flavor.list()) return -1 - - return nova_client.servers.create( - name=instance_name, - flavor=flavor, - image=image_id, - config_drive=config_drive, - nics=[{"net-id": network_id}] - ) + if userdata is None: + instance = nova_client.servers.create( + name=instance_name, + flavor=flavor, + image=image_id, + nics=[{"net-id": network_id}] + ) + else: + instance = nova_client.servers.create( + name=instance_name, + flavor=flavor, + image=image_id, + nics=[{"net-id": network_id}], + config_drive=confdrive, + userdata=userdata + ) + return instance def create_instance_and_wait_for_active(flavor_name, @@ -224,9 +233,9 @@ def create_instance_and_wait_for_active(flavor_name, instance = create_instance(flavor_name, image_id, network_id, - instance_name="", - config_drive=False, - userdata="") + instance_name, + config_drive, + userdata) count = VM_BOOT_TIMEOUT / SLEEP for n in range(count, -1, -1): @@ -653,6 +662,46 @@ def create_secgroup_rule(neutron_client, sg_id, direction, protocol, return False +def create_security_group_full(logger, neutron_client, + sg_name, sg_description): + sg_id = get_security_group_id(neutron_client, sg_name) + if sg_id != '': + logger.info("Using existing security group '%s'..." % sg_name) + else: + logger.info("Creating security group '%s'..." % sg_name) + SECGROUP = create_security_group(neutron_client, + sg_name, + sg_description) + if not SECGROUP: + logger.error("Failed to create the security group...") + return False + + sg_id = SECGROUP['id'] + + logger.debug("Security group '%s' with ID=%s created successfully." + % (SECGROUP['name'], sg_id)) + + logger.debug("Adding ICMP rules in security group '%s'..." + % sg_name) + if not create_secgroup_rule(neutron_client, sg_id, + 'ingress', 'icmp'): + logger.error("Failed to create the security group rule...") + return False + + logger.debug("Adding SSH rules in security group '%s'..." + % sg_name) + if not create_secgroup_rule( + neutron_client, sg_id, 'ingress', 'tcp', '22', '22'): + logger.error("Failed to create the security group rule...") + return False + + if not create_secgroup_rule( + neutron_client, sg_id, 'egress', 'tcp', '22', '22'): + logger.error("Failed to create the security group rule...") + return False + return sg_id + + def add_secgroup_to_instance(nova_client, instance_id, secgroup_id): try: nova_client.servers.add_security_group(instance_id, secgroup_id) @@ -711,18 +760,28 @@ def get_image_id(glance_client, image_name): return id -def create_glance_image(glance_client, image_name, file_path, public=True): +def create_glance_image(glance_client, image_name, file_path, disk="qcow2", + container="bare", public=True, logger=None): if not os.path.isfile(file_path): print "Error: file " + file_path + " does not exist." return False try: - with open(file_path) as fimage: - image = glance_client.images.create(name=image_name, - is_public=public, - disk_format="qcow2", - container_format="bare", - data=fimage) - return image.id + image_id = get_image_id(glance_client, image_name) + if image_id != '': + if logger: + logger.info("Image %s already exists." % image_name) + else: + if logger: + logger.info("Creating image '%s' from '%s'..." % (image_name, + file_path)) + with open(file_path) as fimage: + image = glance_client.images.create(name=image_name, + is_public=public, + disk_format=disk, + container_format=container, + data=fimage) + image_id = image.id + return image_id except Exception, e: print ("Error [create_glance_image(glance_client, '%s', '%s', " "'%s')]:" % (image_name, file_path, str(public))), e |
