diff options
-rw-r--r-- | docker/Dockerfile | 8 | ||||
-rw-r--r-- | docs/images/dovetail_offline_mode.png | bin | 0 -> 128341 bytes | |||
-rw-r--r-- | docs/images/dovetail_online_mode.png | bin | 0 -> 128509 bytes | |||
-rw-r--r-- | docs/testing/user/testspecification/ipv6/index.rst | 231 | ||||
-rw-r--r-- | docs/testing/user/userguide/cli_reference.rst | 9 | ||||
-rw-r--r-- | docs/testing/user/userguide/index.rst | 477 | ||||
-rw-r--r-- | docs/testing/user/userguide/testing_guide.rst | 483 | ||||
-rw-r--r-- | dovetail/conf/cmd_config.yml | 2 | ||||
-rw-r--r-- | dovetail/conf/dovetail_config.yml | 2 | ||||
-rw-r--r-- | dovetail/conf/yardstick_config.yml | 5 | ||||
-rw-r--r-- | dovetail/container.py | 31 | ||||
-rwxr-xr-x | dovetail/run.py | 2 | ||||
-rw-r--r-- | dovetail/utils/dovetail_utils.py | 9 | ||||
-rwxr-xr-x | dovetail/utils/local_db/launch_db.sh | 4 | ||||
-rwxr-xr-x | dovetail/utils/local_db/restart_db.sh | 2 | ||||
-rw-r--r-- | dovetail/utils/offline/config.yaml | 6 | ||||
-rw-r--r-- | setup.cfg | 2 |
17 files changed, 667 insertions, 606 deletions
diff --git a/docker/Dockerfile b/docker/Dockerfile index fbe48495..eb85ea49 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,13 @@ RUN \ && \ apt-get update -RUN wget -qO- https://get.docker.com/ | sed 's/ docker-engine/ docker-engine=1.12.3-0~trusty/' | sh +RUN wget -qO- https://get.docker.com/ \ +| \ + sed 's/-q docker-ce/-q docker-ce=17.03.0~ce-0~ubuntu-trusty/' \ +| \ + sed 's/edge/stable/' \ +| \ + sh ENV HOME /home/opnfv ENV REPOS_DIR ${HOME}/dovetail diff --git a/docs/images/dovetail_offline_mode.png b/docs/images/dovetail_offline_mode.png Binary files differnew file mode 100644 index 00000000..520108cc --- /dev/null +++ b/docs/images/dovetail_offline_mode.png diff --git a/docs/images/dovetail_online_mode.png b/docs/images/dovetail_online_mode.png Binary files differnew file mode 100644 index 00000000..e150d9ff --- /dev/null +++ b/docs/images/dovetail_online_mode.png diff --git a/docs/testing/user/testspecification/ipv6/index.rst b/docs/testing/user/testspecification/ipv6/index.rst index 674c69d8..c3dc844b 100644 --- a/docs/testing/user/testspecification/ipv6/index.rst +++ b/docs/testing/user/testspecification/ipv6/index.rst @@ -1180,9 +1180,9 @@ Use case specification This test case evaluates IPv6 address assignment in ipv6_ra_mode 'dhcpv6_stateless' and ipv6_address_mode 'dhcpv6_stateless'. In this case, guest instance obtains IPv6 address from OpenStack managed radvd -using SLAAC and optional info from dnsmasq using DHCPv6. This test case then -verifies the ping6 available VM can ping all of the v4 addresses and other's -v6 addresses as well as the router's in the same network, the reference is +using SLAAC and optional info from dnsmasq using DHCPv6 stateless. This test case then +verifies the ping6 available VM can ping the other VM's v4 and v6 addresses +as well as the v6 subnet's gateway ip in the same network, the reference is tempest.scenario.test_network_v6.TestGettingAddress.test_dhcp6_stateless_from_os @@ -1208,31 +1208,31 @@ Test execution storing the "id" parameter returned in the response * Test action 6: Connect the IPv6 subnet to the router, using the stored IPv6 subnet id * Test action 7: Boot two VMs on this network, storing the "id" parameters returned in the response -* **Test assertion 1:** The vNICs of all VMs get all v6 addresses actually assigned +* **Test assertion 1:** The vNIC of each VM gets one v4 address and one v6 address actually assigned * **Test assertion 2:** Each VM can ping the other's v4 private address -* **Test assertion 3:** The ping6 available VM can ping all of the other's v6 addresses - as well as the router's +* **Test assertion 3:** The ping6 available VM can ping the other's v6 address + as well as the v6 subnet's gateway ip * Test action 8: Delete the 2 VMs created in test action 7, using the stored ids * Test action 9: List all VMs, verifying the ids are no longer present * **Test assertion 4:** The two "id" parameters are not present in the VM list -* Test action 10: Delete the IPv4 subnet created in test action 2, using the stored ids -* Test action 11: Delete the IPv6 subnet created in test action 5, using the stored ids +* Test action 10: Delete the IPv4 subnet created in test action 2, using the stored id +* Test action 11: Delete the IPv6 subnet created in test action 5, using the stored id * Test action 12: List all subnets, verifying the ids are no longer present * **Test assertion 5:** The "id" parameters of IPv4 and IPv6 are not present in the list -* Test action 13: Delete the network created in test action 1, using the stored ids +* Test action 13: Delete the network created in test action 1, using the stored id * Test action 14: List all networks, verifying the id is no longer present * **Test assertion 6:** The "id" parameter is not present in the network list Pass / fail criteria ''''''''''''''''''''' -This test evaluates the ability to assign IPv6 address in ipv6_ra_mode +This test evaluates the ability to assign IPv6 addresses in ipv6_ra_mode 'dhcpv6_stateless' and ipv6_address_mode 'dhcpv6_stateless', -and verify the ping6 available VM can ping all of the v4 addresses and other's -v6 addresses as well as the router's in the same network. Specifically it verifies that: +and verify the ping6 available VM can ping the other VM's v4 and v6 addresses as well as +the v6 subnet's gateway ip in the same network. Specifically it verifies that: * The IPv6 addresses in mode 'dhcpv6_stateless' assigned successfully -* The VM can ping the IPv4 and IPv6 private addresses +* The VM can ping the other VM's IPv4 and IPv6 private addresses as well as the v6 subnet's gateway ip * All items created using create commands are able to be removed using the returned identifiers Post conditions @@ -1255,9 +1255,9 @@ Use case specification This test case evaluates IPv6 address assignment in ipv6_ra_mode 'dhcpv6_stateless' and ipv6_address_mode 'dhcpv6_stateless'. In this case, guest instance obtains IPv6 address from OpenStack managed radvd -using SLAAC and optional info from dnsmasq using DHCPv6. This test case then -verifies the ping6 available VM can ping all of the v4 addresses in another network -and other's v6 addresses as well as the router's in the same network, the reference is +using SLAAC and optional info from dnsmasq using DHCPv6 stateless. This test case then +verifies the ping6 available VM can ping the other VM's v4 address in one network +and v6 address in another network as well as the v6 subnet's gateway ip, the reference is tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_dhcp6_stateless_from_os @@ -1278,17 +1278,18 @@ Test execution * Test action 3: If there exists a public router, use it as the router. Otherwise, use the public network to create a router * Test action 4: Connect the IPv4 subnet to the router, using the stored IPv4 subnet id -* Test action 5: Create another network, storing the "id" parameters returned in the response +* Test action 5: Create another network, storing the "id" parameter returned in the response * Test action 6: Create one IPv6 subnet of network created in test action 5 in ipv6_ra_mode 'dhcpv6_stateless' and ipv6_address_mode 'dhcpv6_stateless', storing the "id" parameter returned in the response * Test action 7: Connect the IPv6 subnet to the router, using the stored IPv6 subnet id * Test action 8: Boot two VMs on these two networks, storing the "id" parameters returned in the response -* Test action 9: Turn on 2nd NIC of each VM -* **Test assertion 1:** The vNICs of all VMs get all v6 addresses actually assigned +* Test action 9: Turn on 2nd NIC of each VM for the network created in test action 5 +* **Test assertion 1:** The 1st vNIC of each VM gets one v4 address assigned and + the 2nd vNIC of each VM gets one v6 address actually assigned * **Test assertion 2:** Each VM can ping the other's v4 private address -* **Test assertion 3:** The ping6 available VM can ping all of the other's v6 addresses - as well as the router's +* **Test assertion 3:** The ping6 available VM can ping the other's v6 address + as well as the v6 subnet's gateway ip * Test action 10: Delete the 2 VMs created in test action 8, using the stored ids * Test action 11: List all VMs, verifying the ids are no longer present * **Test assertion 4:** The two "id" parameters are not present in the VM list @@ -1303,13 +1304,14 @@ Test execution Pass / fail criteria '''''''''''''''''''' -This test evaluates the ability to assign IPv6 address in ipv6_ra_mode 'dhcpv6_stateless' -and ipv6_address_mode 'dhcpv6_stateless', -and verify the ping6 available VM can ping all of the v4 addresses and other's -v6 addresses as well as the router's in another network. Specifically it verifies that: +This test evaluates the ability to assign IPv6 addresses in ipv6_ra_mode 'dhcpv6_stateless' +and ipv6_address_mode 'dhcpv6_stateless', and verify the ping6 available VM can ping +the other VM's v4 address in one network and v6 address in another network as well as +the v6 subnet's gateway ip. Specifically it verifies that: * The IPv6 addresses in mode 'dhcpv6_stateless' assigned successfully -* The VM can ping the IPv4 and IPv6 private addresses +* The VM can ping the other VM's IPv4 address in one network and IPv6 address in another + network as well as the v6 subnet's gateway ip * All items created using create commands are able to be removed using the returned identifiers Post conditions @@ -1331,10 +1333,11 @@ Use case specification This test case evaluates IPv6 address assignment in ipv6_ra_mode 'dhcpv6_stateless' and ipv6_address_mode 'dhcpv6_stateless'. -In this case, guest instance obtains IPv6 address from OpenStack managed radvd -using SLAAC and optional info from dnsmasq using DHCPv6. This test case then -verifies the ping6 available VM can ping all of the v4 addresses and other's -v6 addresses as well as the router's in the same network, the reference is +In this case, guest instance obtains IPv6 addresses from OpenStack managed radvd +using SLAAC and optional info from dnsmasq using DHCPv6 stateless. This test case then +verifies the ping6 available VM can ping the other VM's one v4 address and two v6 +addresses with different prefixes as well as the v6 subnets' gateway ips in the +same network, the reference is tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_dhcpv6_stateless @@ -1358,34 +1361,35 @@ Test execution * Test action 5: Create two IPv6 subnets of the network created in test action 1 in ipv6_ra_mode 'dhcpv6_stateless' and ipv6_address_mode 'dhcpv6_stateless', storing the "id" parameters returned in the response -* Test action 6: Connect the two IPv6 subnets to the router, using the stored IPv6 subnet id +* Test action 6: Connect the two IPv6 subnets to the router, using the stored IPv6 subnet ids * Test action 7: Boot two VMs on this network, storing the "id" parameters returned in the response -* **Test assertion 1:** The vNICs of all VMs get all v6 addresses actually assigned +* **Test assertion 1:** The vNIC of each VM gets one v4 address and two v6 addresses with + different prefixes actually assigned * **Test assertion 2:** Each VM can ping the other's v4 private address -* **Test assertion 3:** The ping6 available VM can ping all of the other's v6 addresses - as well as the router's +* **Test assertion 3:** The ping6 available VM can ping the other's v6 addresses + as well as the v6 subnets' gateway ips * Test action 8: Delete the 2 VMs created in test action 7, using the stored ids * Test action 9: List all VMs, verifying the ids are no longer present * **Test assertion 4:** The two "id" parameters are not present in the VM list -* Test action 10: Delete the IPv4 subnet created in test action 2, using the stored ids +* Test action 10: Delete the IPv4 subnet created in test action 2, using the stored id * Test action 11: Delete two IPv6 subnets created in test action 5, using the stored ids * Test action 12: List all subnets, verifying the ids are no longer present * **Test assertion 5:** The "id" parameters of IPv4 and IPv6 are not present in the list -* Test action 13: Delete the network created in test action 1, using the stored ids +* Test action 13: Delete the network created in test action 1, using the stored id * Test action 14: List all networks, verifying the id is no longer present * **Test assertion 6:** The "id" parameter is not present in the network list Pass / fail criteria ''''''''''''''''''''' -This test evaluates the ability to assign IPv6 address in ipv6_ra_mode 'dhcpv6_stateless' +This test evaluates the ability to assign IPv6 addresses in ipv6_ra_mode 'dhcpv6_stateless' and ipv6_address_mode 'dhcpv6_stateless', -and verify the ping6 available VM can ping all of the v4 addresses and other's -v6 addresses with different prefixes as well as the router's in the same network. +and verify the ping6 available VM can ping the other VM's v4 address and two +v6 addresses with different prefixes as well as the v6 subnets' gateway ips in the same network. Specifically it verifies that: * The different prefixes IPv6 addresses in mode 'dhcpv6_stateless' assigned successfully -* The VM can ping the IPv4 and IPv6 private addresses +* The VM can ping the other VM's IPv4 and IPv6 private addresses as well as the v6 subnets' gateway ips * All items created using create commands are able to be removed using the returned identifiers Post conditions @@ -1407,10 +1411,11 @@ Use case specification This test case evaluates IPv6 address assignment in ipv6_ra_mode 'dhcpv6_stateless' and ipv6_address_mode 'dhcpv6_stateless'. -In this case, guest instance obtains IPv6 address from OpenStack managed radvd -using SLAAC and optional info from dnsmasq using DHCPv6. This test case then -verifies the ping6 available VM can ping all of the v4 addresses in another network -and other's v6 addresses as well as the router's in the same network, the reference is +In this case, guest instance obtains IPv6 addresses from OpenStack managed radvd +using SLAAC and optional info from dnsmasq using DHCPv6 stateless. This test case then +verifies the ping6 available VM can ping the other VM's v4 address in one network +and two v6 addresses with different prefixes in another network as well as the +v6 subnets' gateway ips, the reference is tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_dhcpv6_stateless @@ -1431,21 +1436,22 @@ Test execution * Test action 3: If there exists a public router, use it as the router. Otherwise, use the public network to create a router * Test action 4: Connect the IPv4 subnet to the router, using the stored IPv4 subnet id -* Test action 5: Create another network, storing the "id" parameters returned in the response +* Test action 5: Create another network, storing the "id" parameter returned in the response * Test action 6: Create two IPv6 subnets of network created in test action 5 in ipv6_ra_mode 'dhcpv6_stateless' and ipv6_address_mode 'dhcpv6_stateless', storing the "id" parameters returned in the response * Test action 7: Connect the two IPv6 subnets to the router, using the stored IPv6 subnet ids * Test action 8: Boot two VMs on these two networks, storing the "id" parameters returned in the response -* Test action 9: Turn on 2nd NIC of each VM -* **Test assertion 1:** The vNICs of all VMs get all v6 addresses actually assigned +* Test action 9: Turn on 2nd NIC of each VM for the network created in test action 5 +* **Test assertion 1:** The vNIC of each VM gets one v4 address and two v6 addresses + with different prefixes actually assigned * **Test assertion 2:** Each VM can ping the other's v4 private address -* **Test assertion 3:** The ping6 available VM can ping all of the other's v6 addresses - as well as the router's +* **Test assertion 3:** The ping6 available VM can ping the other's v6 addresses + as well as the v6 subnets' gateway ips * Test action 10: Delete the 2 VMs created in test action 8, using the stored ids * Test action 11: List all VMs, verifying the ids are no longer present * **Test assertion 4:** The two "id" parameters are not present in the VM list -* Test action 12: Delete the IPv4 subnet created in test action 2, using the stored ids +* Test action 12: Delete the IPv4 subnet created in test action 2, using the stored id * Test action 13: Delete two IPv6 subnets created in test action 6, using the stored ids * Test action 14: List all subnets, verifying the ids are no longer present * **Test assertion 5:** The "id" parameters of IPv4 and IPv6 are not present in the list @@ -1456,14 +1462,14 @@ Test execution Pass / fail criteria ''''''''''''''''''''' -This test evaluates the ability to assign IPv6 address in ipv6_ra_mode 'dhcpv6_stateless' +This test evaluates the ability to assign IPv6 addresses in ipv6_ra_mode 'dhcpv6_stateless' and ipv6_address_mode 'dhcpv6_stateless', -and verify the ping6 available VM can ping all of the v4 addresses and other's -v6 addresses with different prefixes as well as the router's in another network. -Specifically it verifies that: +and verify the ping6 available VM can ping the other VM's v4 address in one network and two +v6 addresses with different prefixes in another network as well as the v6 subnets' +gateway ips. Specifically it verifies that: * The IPv6 addresses in mode 'dhcpv6_stateless' assigned successfully -* The VM can ping the IPv4 and IPv6 private addresses +* The VM can ping the other VM's IPv4 and IPv6 private addresses as well as the v6 subnets' gateway ips * All items created using create commands are able to be removed using the returned identifiers Post conditions @@ -1486,8 +1492,8 @@ Use case specification This test case evaluates IPv6 address assignment in ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac'. In this case, guest instance obtains IPv6 address from OpenStack managed radvd -using SLAAC. This test case then verifies the ping6 available VM can ping all -of the v4 addresses and other's v6 addresses as well as the router's in the +using SLAAC. This test case then verifies the ping6 available VM can ping the other +VM's v4 and v6 addresses as well as the v6 subnet's gateway ip in the same network, the reference is tempest.scenario.test_network_v6.TestGettingAddress.test_slaac_from_os @@ -1513,31 +1519,31 @@ Test execution ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac', storing the "id" parameter returned in the response * Test action 6: Connect the IPv6 subnet to the router, using the stored IPv6 subnet id * Test action 7: Boot two VMs on this network, storing the "id" parameters returned in the response -* **Test assertion 1:** The vNICs of all VMs get all v6 addresses actually assigned +* **Test assertion 1:** The vNIC of each VM gets one v4 address and one v6 address actually assigned * **Test assertion 2:** Each VM can ping the other's v4 private address -* **Test assertion 3:** The ping6 available VM can ping all of the other's v6 addresses - as well as the router's +* **Test assertion 3:** The ping6 available VM can ping the other's v6 address + as well as the v6 subnet's gateway ip * Test action 8: Delete the 2 VMs created in test action 7, using the stored ids * Test action 9: List all VMs, verifying the ids are no longer present * **Test assertion 4:** The two "id" parameters are not present in the VM list -* Test action 10: Delete the IPv4 subnet created in test action 2, using the stored ids -* Test action 11: Delete the IPv6 subnet created in test action 5, using the stored ids +* Test action 10: Delete the IPv4 subnet created in test action 2, using the stored id +* Test action 11: Delete the IPv6 subnet created in test action 5, using the stored id * Test action 12: List all subnets, verifying the ids are no longer present * **Test assertion 5:** The "id" parameters of IPv4 and IPv6 are not present in the list -* Test action 13: Delete the network created in test action 1, using the stored ids +* Test action 13: Delete the network created in test action 1, using the stored id * Test action 14: List all networks, verifying the id is no longer present * **Test assertion 6:** The "id" parameter is not present in the network list Pass / fail criteria ''''''''''''''''''''' -This test evaluates the ability to assign IPv6 address in ipv6_ra_mode 'slaac' +This test evaluates the ability to assign IPv6 addresses in ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac', -and verify the ping6 available VM can ping all of the v4 addresses and other's -v6 addresses as well as the router's in the same network. Specifically it verifies that: +and verify the ping6 available VM can ping the other VM's v4 and v6 addresses as well as +the v6 subnet's gateway ip in the same network. Specifically it verifies that: * The IPv6 addresses in mode 'slaac' assigned successfully -* The VM can ping the IPv4 and IPv6 private addresses +* The VM can ping the other VM's IPv4 and IPv6 private addresses as well as the v6 subnet's gateway ip * All items created using create commands are able to be removed using the returned identifiers Post conditions @@ -1560,9 +1566,9 @@ Use case specification This test case evaluates IPv6 address assignment in ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac'. In this case, guest instance obtains IPv6 address from OpenStack managed radvd -using SLAAC. This test case then verifies the ping6 available VM can ping all -of the v4 addresses in another network and other's v6 addresses as well as the -router's in the same network, the reference is +using SLAAC. This test case then verifies the ping6 available VM can ping the other +VM's v4 address in one network and v6 address in another network as well as the +v6 subnet's gateway ip, the reference is tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_slaac_from_os @@ -1588,16 +1594,17 @@ Test execution ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac', storing the "id" parameter returned in the response * Test action 7: Connect the IPv6 subnet to the router, using the stored IPv6 subnet id * Test action 8: Boot two VMs on these two networks, storing the "id" parameters returned in the response -* Test action 9: Turn on 2nd NIC of each VM -* **Test assertion 1:** The vNICs of all VMs get all v6 addresses actually assigned +* Test action 9: Turn on 2nd NIC of each VM for the network created in test action 5 +* **Test assertion 1:** The 1st vNIC of each VM gets one v4 address assigned and + the 2nd vNIC of each VM gets one v6 address actually assigned * **Test assertion 2:** Each VM can ping the other's v4 private address -* **Test assertion 3:** The ping6 available VM can ping all of the other's v6 addresses - as well as the router's +* **Test assertion 3:** The ping6 available VM can ping the other's v6 address + as well as the v6 subnet's gateway ip * Test action 10: Delete the 2 VMs created in test action 8, using the stored ids * Test action 11: List all VMs, verifying the ids are no longer present * **Test assertion 4:** The two "id" parameters are not present in the VM list -* Test action 12: Delete the IPv4 subnet created in test action 2, using the stored ids -* Test action 13: Delete the IPv6 subnet created in test action 6, using the stored ids +* Test action 12: Delete the IPv4 subnet created in test action 2, using the stored id +* Test action 13: Delete the IPv6 subnet created in test action 6, using the stored id * Test action 14: List all subnets, verifying the ids are no longer present * **Test assertion 5:** The "id" parameters of IPv4 and IPv6 are not present in the list * Test action 15: Delete the 2 networks created in test action 1 and 5, using the stored ids @@ -1607,13 +1614,14 @@ Test execution Pass / fail criteria ''''''''''''''''''''' -This test evaluates the ability to assign IPv6 address in ipv6_ra_mode 'slaac' +This test evaluates the ability to assign IPv6 addresses in ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac', -and verify the ping6 available VM can ping all of the v4 addresses and other's -v6 addresses as well as the router's in another network. Specifically it verifies that: +and verify the ping6 available VM can ping the other VM's v4 address in one network and +v6 address in another network as well as the v6 subnet's gateway ip. Specifically it verifies that: * The IPv6 addresses in mode 'slaac' assigned successfully -* The VM can ping the IPv4 and IPv6 private addresses +* The VM can ping the other VM's IPv4 address in one network and IPv6 address + in another network as well as the v6 subnet's gateway ip * All items created using create commands are able to be removed using the returned identifiers Post conditions @@ -1635,10 +1643,10 @@ Use case specification This test case evaluates IPv6 address assignment in ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac'. -In this case, guest instance obtains IPv6 address from OpenStack managed radvd -using SLAAC. This test case then verifies the ping6 available VM can ping all -of the v4 addresses and other's v6 addresses as well as the router's in the -same network, the reference is +In this case, guest instance obtains IPv6 addresses from OpenStack managed radvd +using SLAAC. This test case then verifies the ping6 available VM can ping the other +VM's one v4 address and two v6 addresses with different prefixes as well as the v6 +subnets' gateway ips in the same network, the reference is tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_slaac @@ -1655,40 +1663,41 @@ Test execution * Test action 1: Create one network, storing the "id" parameter returned in the response * Test action 2: Create one IPv4 subnet of the created network, storing the "id - parameters returned in the response + parameter returned in the response * Test action 3: If there exists a public router, use it as the router. Otherwise, use the public network to create a router * Test action 4: Connect the IPv4 subnet to the router, using the stored IPv4 subnet id * Test action 5: Create two IPv6 subnets of the network created in test action 1 in ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac', storing the "id" parameters returned in the response -* Test action 6: Connect the two IPv6 subnets to the router, using the stored IPv6 subnet id +* Test action 6: Connect the two IPv6 subnets to the router, using the stored IPv6 subnet ids * Test action 7: Boot two VMs on this network, storing the "id" parameters returned in the response -* **Test assertion 1:** The vNICs of all VMs get all v6 addresses actually assigned +* **Test assertion 1:** The vNIC of each VM gets one v4 address and two v6 addresses with + different prefixes actually assigned * **Test assertion 2:** Each VM can ping the other's v4 private address -* **Test assertion 3:** The ping6 available VM can ping all of the other's v6 addresses - as well as the router's +* **Test assertion 3:** The ping6 available VM can ping the other's v6 addresses + as well as the v6 subnets' gateway ips * Test action 8: Delete the 2 VMs created in test action 7, using the stored ids * Test action 9: List all VMs, verifying the ids are no longer present * **Test assertion 4:** The two "id" parameters are not present in the VM list -* Test action 10: Delete the IPv4 subnet created in test action 2, using the stored ids +* Test action 10: Delete the IPv4 subnet created in test action 2, using the stored id * Test action 11: Delete two IPv6 subnets created in test action 5, using the stored ids * Test action 12: List all subnets, verifying the ids are no longer present * **Test assertion 5:** The "id" parameters of IPv4 and IPv6 are not present in the list -* Test action 13: Delete the network created in test action 1, using the stored ids +* Test action 13: Delete the network created in test action 1, using the stored id * Test action 14: List all networks, verifying the id is no longer present * **Test assertion 6:** The "id" parameter is not present in the network list Pass / fail criteria ''''''''''''''''''''' -This test evaluates the ability to assign IPv6 address in ipv6_ra_mode 'slaac' +This test evaluates the ability to assign IPv6 addresses in ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac', -and verify the ping6 available VM can ping all of the v4 addresses and other's -v6 addresses with different prefixes as well as the router's in the same network. +and verify the ping6 available VM can ping the other VM's v4 address and two +v6 addresses with different prefixes as well as the v6 subnets' gateway ips in the same network. Specifically it verifies that: * The different prefixes IPv6 addresses in mode 'slaac' assigned successfully -* The VM can ping the IPv4 and IPv6 private addresses +* The VM can ping the other VM's IPv4 and IPv6 private addresses as well as the v6 subnets' gateway ips * All items created using create commands are able to be removed using the returned identifiers Post conditions @@ -1710,10 +1719,10 @@ Use case specification This test case evaluates IPv6 address assignment in ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac'. -In this case, guest instance obtains IPv6 address from OpenStack managed radvd -using SLAAC. This test case then verifies the ping6 available VM can ping all -of the v4 addresses in another network and other's v6 addresses as well as the -router's in the same network, the reference is +In this case, guest instance obtains IPv6 addresses from OpenStack managed radvd +using SLAAC. This test case then verifies the ping6 available VM can ping the other +VM's v4 address in one network and two v6 addresses with different prefixes in another +network as well as the v6 subnets' gateway ips, the reference is tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_slaac @@ -1730,24 +1739,25 @@ Test execution * Test action 1: Create one network, storing the "id" parameter returned in the response * Test action 2: Create one IPv4 subnet of the created network, storing the "id" - parameters returned in the response + parameter returned in the response * Test action 3: If there exists a public router, use it as the router. Otherwise, use the public network to create a router * Test action 4: Connect the IPv4 subnet to the router, using the stored IPv4 subnet id -* Test action 5: Create another network, storing the "id" parameters returned in the response +* Test action 5: Create another network, storing the "id" parameter returned in the response * Test action 6: Create two IPv6 subnets of network created in test action 5 in ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac', storing the "id" parameters returned in the response * Test action 7: Connect the two IPv6 subnets to the router, using the stored IPv6 subnet ids * Test action 8: Boot two VMs on these two networks, storing the "id" parameters returned in the response -* Test action 9: Turn on 2nd NIC of each VM -* **Test assertion 1:** The vNICs of all VMs get all v6 addresses actually assigned +* Test action 9: Turn on 2nd NIC of each VM for the network created in test action 5 +* **Test assertion 1:** The vNIC of each VM gets one v4 address and two v6 addresses + with different prefixes actually assigned * **Test assertion 2:** Each VM can ping the other's v4 private address -* **Test assertion 3:** The ping6 available VM can ping all of the other's v6 addresses - as well as the router's +* **Test assertion 3:** The ping6 available VM can ping the other's v6 addresses + as well as the v6 subnets' gateway ips * Test action 10: Delete the 2 VMs created in test action 8, using the stored ids * Test action 11: List all VMs, verifying the ids are no longer present * **Test assertion 4:** The two "id" parameters are not present in the VM list -* Test action 12: Delete the IPv4 subnet created in test action 2, using the stored ids +* Test action 12: Delete the IPv4 subnet created in test action 2, using the stored id * Test action 13: Delete two IPv6 subnets created in test action 6, using the stored ids * Test action 14: List all subnets, verifying the ids are no longer present * **Test assertion 5:** The "id" parameters of IPv4 and IPv6 are not present in the list @@ -1758,14 +1768,14 @@ Test execution Pass / fail criteria ''''''''''''''''''''' -This test evaluates the ability to assign IPv6 address in ipv6_ra_mode 'slaac' +This test evaluates the ability to assign IPv6 addresses in ipv6_ra_mode 'slaac' and ipv6_address_mode 'slaac', -and verify the ping6 available VM can ping all of the v4 addresses and other's -v6 addresses with different prefixes as well as the router's in another network. +and verify the ping6 available VM can ping the other VM's v4 address in one network and two +v6 addresses with different prefixes in another network as well as the v6 subnets' gateway ips. Specifically it verifies that: * The IPv6 addresses in mode 'slaac' assigned successfully -* The VM can ping the IPv4 and IPv6 private addresses +* The VM can ping the other VM's IPv4 and IPv6 private addresses as well as the v6 subnets' gateway ips * All items created using create commands are able to be removed using the returned identifiers Post conditions @@ -1774,3 +1784,4 @@ Post conditions None + diff --git a/docs/testing/user/userguide/cli_reference.rst b/docs/testing/user/userguide/cli_reference.rst new file mode 100644 index 00000000..719a991f --- /dev/null +++ b/docs/testing/user/userguide/cli_reference.rst @@ -0,0 +1,9 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV + +========================================= +Dovetail Command Line Interface Reference +========================================= + + diff --git a/docs/testing/user/userguide/index.rst b/docs/testing/user/userguide/index.rst index d8eb124b..aec3e861 100644 --- a/docs/testing/user/userguide/index.rst +++ b/docs/testing/user/userguide/index.rst @@ -1,479 +1,12 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -.. (c) Ericsson AB +.. (c) OPNFV -============================================== -Compliance and Verification program user guide -============================================== +******************************************************** +Compliance Verification Program Testing User Guide +******************************************************** .. toctree:: :maxdepth: 2 -Version history -=============== - -+------------+----------+------------------+----------------------------------+ -| **Date** | **Ver.** | **Author** | **Comment** | -| | | | | -+------------+----------+------------------+----------------------------------+ -| 2017-03-15 | 0.0.1 | Chris Price | Draft version | -| | | | | -+------------+----------+------------------+----------------------------------+ - - -Dovetail CVP Testing Overview -============================= - -The Dovetail testing framework consists of two major parts: the testing client that executes -all test cases in a vendor lab (self-testing) or a third party lab, and the server system that -is under the OPNFV's administration to store and view test results based on OPNFV Test API. The -following diagram illustrates this overall framework. - -/* here is a draft diagram that needs to be revised when exact information is known and fixed */ - -This section mainly focuses on helping the testers in the vendor's domain attempting to run the -CVP tests. - -Dovetail client tool (or just Dovetail tool or Dovetail for short) can be installed in the -jumphost either directly as Python software, or as a Docker(r) container. Comments of pros -and cons of the two options TBD. - -The section 'Installing the test tool'_ describes the steps the tester needs to take to install -Dovetail directly from the source. In 2.3, we describe steps needed for installing Dovetail -Docker(r) container. Once installed, and properly configured, the remaining test process is mostly -identical for the two options. In 2.4, we go over the steps of actually running the test suite. -In 2.5, we discuss how to view test results and make sense of them, for example, what the tester -may do in case of unexpected test failures. Section 2.6 describes additional Dovetail features -that are not absolutely necessary in CVP testing but users may find useful for other purposes. -One example is to run Dovetail for in-house testing as preparation before official CVP testing; -another example is to run Dovetail experimental test suites other than the CVP test suite. -Experimental tests may be made available by the community for experimenting less mature test -cases or functionalities for the purpose of getting feedbacks for improvement. - -Installing the test tool -======================== - -Before taking this step, testers should check the hardware and networking requirements of -the POD, and the jumphost in particular, to make sure they are compliant. - -In this section, we describe the procedure to install Dovetail client tool that runs the CVP -test suite from the jumphost. The jumphost must have network access to both the public Internet -and to the O&M (Operation and Management) network with access rights to all VIM APIs being tested. - -------------------------------- -Checking the Jumphost Readiness -------------------------------- - -While Dovetail does not have hard requirement on a specific operating system type or version, -these have been validated by the community through some level of exercise in OPNFV labs or PlugFests. - -Ubuntu 16.04.2 LTS (Xenial) for x86_64 -Ubuntu 14.04 LTS (Trusty) for x86_64 -CentOS-7-1611 for x86_64 -Red Hat Enterprise Linux 7.3 for x86_64 -Fedora 24 Server for x86_64 -Fedora 25 Server for x86_64 - ------------------------------------- -Configuring the Jumphost Environment ------------------------------------- - -/* First, openstack env variables to be passed to Functest */ - -The jumphost needs to have the right environmental variable setting to enable access to the -Openstack API. This is usually done through the Openstack credential file. - -Sample Openstack credential file environment_config.sh: - -/*Project-level authentication scope (name or ID), recommend admin project.*/ - -export OS_PROJECT_NAME=admin - -/* Authentication username, belongs to the project above, recommend admin user.*/ - -export OS_USERNAME=admin - - -/* Authentication password.*/ - -export OS_PASSWORD=secret - - -/* Authentication URL, one of the endpoints of keystone service. If this is v3 version, there need some extra variables as follows.*/ - -export OS_AUTH_URL='http://xxx.xxx.xxx.xxx:5000/v3' - - -/* Default is 2.0. If use keystone v3 API, this should be set as 3.*/ - -export OS_IDENTITY_API_VERSION=3 - - -/* Domain name or ID containing the user above. Command to check the domain: openstack -user show <OS_USERNAME>*/ - -export OS_USER_DOMAIN_NAME=default - - -/* Domain name or ID containing the project above. Command to check the domain: openstack -project show <OS_PROJECT_NAME>*/ - -export OS_PROJECT_DOMAIN_NAME=default - - -/* home directory for dovetail, if install Dovetail Docker container, DOVETAIL_HOME can -just be /home/opnfv*/ - -export DOVETAIL_HOME=$HOME/cvp - -Export all these variables into environment by, - -% source <OpenStack-credential-file-path> - - -The tester should validate that the Openstack environmental settings are correct by, -% openstack service list - ------------------------------------ -Installing Prerequisite on Jumphost ------------------------------------ - -1. Dovetail requires Python 2.7 and later - -Use the following steps to check if the right version of python is already installed, -and if not, install it. - -% python --version - -2. Dovetail requires Docker 1.8.0 and later - -Use the following steps to check if the right version of Docker is already installed, -and if not, install it. - -% docker --version - -As the docker installation process is much complex, you can refer to the official -document: https://docs.docker.com/engine/installation/linux/ - -------------------------------------- -2.2.4 Installing Dovetail on Jumphost -------------------------------------- - -A tester can choose one of the following two methods for installing and running Dovetail. -In part1, we explain the steps to install Dovetail from the source. In part2, an alternative -using a Docker image with preinstalled Dovetail is introduced. part1. Installing Dovetail directly - -Update and install packages - -a) Ubuntu - -sudo apt-get update - -sudo apt-get -y install gcc git vim python-dev python-pip --no-install-recommends - -b) centos and redhat - -sudo yum -y update - -sudo yum -y install epel-release - -sudo yum -y install gcc git vim-enhanced python-devel python-pip - -c) fedora - -sudo dnf -y update - -sudo dnf -y install gcc git vim-enhanced python-devel python-pip redhat-rpm-config - -p.s When testing SUT's https service, there need some extra packages, such as -apt-transport-https. This still remains to be verified. - - -Installing Dovetail - -Now we are ready to install Dovetail. - -/* Version of dovetail is not specified yet? we are still using the latest in the master -- this needs to be fixed before launch. */ - -First change directory to $DOVETAIL_HOME, - -% cd $DOVETAIL_HOME - -% sudo git clone https://git.opnfv.org/dovetail - -% cd $DOVETAIL_HOME/dovetail - -% sudo pip install -e ./ - -/* test dovetail install is successful */ - -% dovetail -h -part2. Installing Dovetail Docker Container - -The Dovetail project also maintains a Docker image that has Dovetail test tools preinstalled. - -Running CVP Test Suite -====================== - ------------------- -Running Test Suite ------------------- - -The Dovetail client CLI allows the tester to specify which test suite to run. -By default the results are stored in a local file $DOVETAIL_HOME/dovetail/results. - -% dovetail run --testsuite <test suite name> --openrc <path-to-openrc-file> /*?? */ - -Multiple test suites may be available, testsuites named "debug" and "proposed_tests" are just provided for testing. But for the purpose of running CVP test suite, the test suite name follows the following format, - -CVP.<major>.<minor>.<patch> /* test if this format works */ - -For example, CVP_1_0_0 - -% dovetail run --testsuite CVP_1_0_0 - -When the SUT's VIM (Virtual Infrastructure Manager) is Openstack, its configuration is commonly defined in the openrc file. In that case, you can specify the openrc file in the command line, - -% dovetail run --testsuite CVP_1_0_0 --openrc <path-to-openrc-file> - -In order to report official results to OPNFV, run the CVP test suite and report to OPNFV official URL, - -% dovetail run --testsuite <test suite name> --openrc <path-to-openrc-file> --report https://www.opnfv.org/cvp - -The official server https://www.opnfv.org/cvp is still under development, there is a temporal server to use http://205.177.226.237:9997/api/v1/results - --------------------------------- -Making Sense of CVP Test Results --------------------------------- - -When a tester is performing trial runs, Dovetail stores results in a local file by default. - -% cd $DOVETAIL_HOME/dovetail/results - - - -1. local file - -a) Log file: dovetail.log - -/* review the dovetail.log to see if all important information has been captured - in default mode without DEBUG */ - -/* the end of the log file has a summary of all test case test results */ - -Additional log files may be of interests: refstack.log, opnfv_yardstick_tcXXX.out ... - -b) Example: Openstack refstack test case example - -can see the log details in refstack.log, which has the passed/skipped/failed test cases result, the failed test cases have rich debug information - -for the users to see why this test case fails. - -c) Example: OPNFV Yardstick test case example - -for yardstick tool, its log is stored in yardstick.log - -for each test case result in Yardstick, the logs are stored in opnfv_yardstick_tcXXX.out, respectively. - - - -2. OPNFV web interface - -wait for the complement of LF, test community, etc. -2.3.3 Updating Dovetail or Test Suite - -% cd $DOVETAIL_HOME/dovetail - -% sudo git pull - -% sudo pip install -e ./ - -This step is necessary if dovetail software or the CVP test suite have updates. - - -Other Dovetail Usage -==================== - ------------------------- -Running Dovetail Locally ------------------------- - -/*DB*/ - ---------------------------------------------- -Running Dovetail with Experimental Test Cases ---------------------------------------------- - - --------------------------------------------------- -Running Individual Test Cases or for Special Cases --------------------------------------------------- - -1. Refstack client to run Defcore testcases - -a) By default, for Defcore test cases run by Refstack-client, which are consumed by -DoveTail, are run followed with automatically generated configuration file, i.e., -refstack_tempest.conf. - -In some circumstances, the automatic configuration file may not quite satisfied with -the SUT, DoveTail provide a way for users to set its configuration file according -to its own SUT manually, - -besides, the users should define Defcore testcase file, i.e., defcore.txt, at the -same time. The steps are shown as, - -when "Installing Dovetail Docker Container" method is used, - - -% sudo mkdir /home/opnfv/dovetail/userconfig - -% cd /home/opnfv/dovetail/userconfig - -% touch refstack_tempest.conf defcore.txt - -% vim refstack_tempest.conf - -% vim defcore.txt - - -the recommend way to set refstack_tempest.conf is shown in -https://aptira.com/testing-openstack-tempest-part-1/ - -the recommended way to edit defcore.txt is to open -https://refstack.openstack.org/api/v1/guidelines/2016.08/tests?target=compute&type=required&alias=true&flag=false -and copy all the test cases into defcore.txt. - -Then use “docker run” to create a container, - - -% sudo docker run --privileged=true -it -v <openrc_path>:<openrc_path> \ - --v /home/opnfv/dovetail/results:/home/opnfv/dovetail/results \ - --v /home/opnfv/dovetail/userconfig:/home/opnfv/dovetail/userconfig \ - --v /var/run/docker.sock:/var/run/docker.sock \ - ---name <DoveTail_Container_Name> (optional) \ - -opnfv/dovetail:<Tag> /bin/bash - - - -there is a need to adjust the CVP_1_0_0 testsuite, for dovetail, -defcore.tc001.yml and defcore.tc002.yml are used for automatic and -manual running method, respectively. - -Inside the dovetail container, - - -% cd /home/opnfv/dovetail/compliance - -% vim CVP_1_0_0.yml - - -to add defcore.tc002 and annotate defcore.tc001. - - -b) when "Installing Dovetail Directly" method is used, before to run -the dovetail commands, there is a need to set configuration file and -defcore test cases file - - -% cd $DOVETAIL_HOME/dovetail - -% mkdir userconfig - -% cd userconfig - -% touch refstack_tempest.conf defcore.txt - -% vim refstack_tempest.conf - -% vim defcore.txt - -recommended way to set refstack_tempest.conf and defcore.txt is -same as above in "Installing Dovetail Docker Container" method section. - - - -For Defcore test cases manually running method, there is a need to adjust -the compliance_set test suite, - -for dovetail, defcore.tc001.yml and defcore.tc002.yml are used for automatic -and manual running method, respectively. - - - -% cd $DOVETAIL_HOME/dovetail/compliance - -% vim CVP_1_0_0.yml - - -to add defcore.tc002 and annotate defcore.tc001 - -3 Dovetail Client CLI Manual - -This section contains a brief manual for all the features available through the Dovetail client command line interface (CLI). -3.1 Check dovetail commands - -% dovetail -h - -dovetail.PNG - -Dovetail has three commands: list, run and show. -6.2 List -6.2.1 List help - -% dovetail list -h - -list-help.PNG -6.2.2 List a test suite - -List command will list all test cases belong to the given test suite. - -% dovetail list compliance_set - -list-compliance.PNG - -% dovetail list debug - -list-debug.PNG - -The ipv6, example and nfvi are test areas. If no <TESTSUITE> is given, it will list all testsuites. -6.3 Show - -Show command will give the detailed info of one certain test case. -6.3.1 Show help - -% dovetail show -h - -show-help.PNG -6.3.2 Show test case - -show-ipv6.PNG -6.4 Run - -Dovetail supports running a named test suite, or one named test area of a test suite. -6.4.1 Run help - -% dovetail run -h - -run-help.PNGThere are some options: - -func_tag: set FuncTest’s Docker tag, for example stable,latest and danube.1.0 - -openrc: give the path of OpenStack credential file - -yard_tag: set Yardstick’s Docker tag - -testarea: set a certain testarea within a certain testsuite - -offline: run without pull the docker images, and it requires the jumphost to have these images locally. This will ensure DoveTail run in an offline environment. - -report: push results to DB or store with files - -testsuite: set the testsuite to be tested - -debug: flag to show the debug log messages - + testing_guide.rst diff --git a/docs/testing/user/userguide/testing_guide.rst b/docs/testing/user/userguide/testing_guide.rst new file mode 100644 index 00000000..9e46b561 --- /dev/null +++ b/docs/testing/user/userguide/testing_guide.rst @@ -0,0 +1,483 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) OPNFV, Huawei Technologies Co.,Ltd and others. + +========================================== +Conducting CVP Testing with Dovetail +========================================== + +Overview +------------------------------ + +The Dovetail testing framework for CVP consists of two major parts: the testing client that +executes all test cases in a lab (vendor self-testing or a third party lab), +and the server system that is hosted by the CVP administrator to store and +view test results based on a web API. The following diagram illustrates +this overall framework. + +.. image:: ../../../images/dovetail_online_mode.png + :align: center + :scale: 50% + +Within the tester's lab, the Test Host is the machine where Dovetail executes all +automated test cases. As it hosts the test harness, the Test Host must not be part of +the System Under Test (SUT) itself. +The above diagram assumes that the tester's Test Host is situated in a DMZ which +has internal network access to the SUT and external access to the OPNFV server +via the public Internet. +This arrangement may not be supported in some labs. +Dovetail also supports an offline mode of testing that is +illustrated in the next diagram. + +.. image:: ../../../images/dovetail_offline_mode.png + :align: center + :scale: 50% + +In the offline mode, the Test Host only needs to have access to the SUT +via the internal network, but does not need to connect to the public Internet. This +user guide will highlight differences between the online and offline modes of +the Test Host. While it is possible to run the Test Host as a virtual machine, +this user guide assumes it is a physical machine for simplicity. + +The rest of this guide will describe how to install the Dovetail tool as a +Docker container image, go over the steps of running the CVP test suite, and +then discuss how to view test results and make sense of them. + +Readers interested +in using Dovetail for its functionalities beyond CVP testing, e.g. for in-house +or extended testing, should consult the Dovetail developer's guide for additional +information. + +Installing Dovetail +-------------------- + +In this section, we describe the procedure to install Dovetail client tool on the Test Host. +The Test Host must have network access to the management network with access rights to +the Virtual Infrastructure Manager's API. + +Checking the Test Host Readiness +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The Test Host must have network access to the Virtual Infrastructure Manager's API +hosted in the SUT so that the Dovetail tool can exercise the API from the Test Host. +It must also have ``ssh`` access to the Linux operating system +of the compute nodes in the SUT. The ``ssh`` mechanism is used by some test cases +to generate test events in the compute nodes. You can find out which test cases +use this mechanism in the test specification document. + +We have tested the Dovetail tool on the following host operating systems. Other versions +or distribution of Linux may also work, but community support may be more available on +these versions. + +- Ubuntu 16.04.2 LTS (Xenial) or 14.04 LTS (Trusty) +- CentOS-7-1611 +- Red Hat Enterprise Linux 7.3 +- Fedora 24 or 25 Server + +Non-Linux operating systems, such as Windows, Mac OS, have not been tested +and are not supported. + +If online mode is used, the tester should also validate that the Test Host can reach +the public Internet. For example, + +.. code-block:: bash + + $ ping www.opnfv.org + PING www.opnfv.org (50.56.49.117): 56 data bytes + 64 bytes from 50.56.49.117: icmp_seq=0 ttl=48 time=52.952 ms + 64 bytes from 50.56.49.117: icmp_seq=1 ttl=48 time=53.805 ms + 64 bytes from 50.56.49.117: icmp_seq=2 ttl=48 time=53.349 ms + ... + + +Or, if the lab environment does not allow ping, try validating it using HTTPS instead. + +.. code-block:: bash + + $ curl https://www.opnfv.org + <!doctype html> + + + <html lang="en-US" class="no-js"> + <head> + ... + + +Configuring the Test Host Environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The Test Host needs a few environment variables set correctly in order to access the +Openstack API required to drive the Dovetail tests. For convenience and as a convention, +we will also create a home directory for storing all Dovetail related config files and +results files: + +.. code-block:: bash + + $ mkdir -p /home/dovetail + $ export DOVETAIL_HOME=/home/dovetail + +Here we set dovetail home directory to be ``/home/dovetail`` for an example. +Then create a directory named ``pre_config`` in this directory to store all +Dovetail related config files: + +.. code-block:: bash + + $ mkdir -p ${DOVETAIL_HOME}/pre_config + +At this point, you will need to consult your SUT (Openstack) administrator to correctly set +the configurations in a file named ``env_config.sh``. +The Openstack settings need to be configured such that the Dovetail client has all the necessary +credentials and privileges to execute all test operations. If the SUT uses terms +somewhat differently from the standard Openstack naming, you will need to adjust +this file accordingly. + +In our example, we will use the file '${DOVETAIL_HOME}/pre_config/env_config.sh'. Create and edit +the file so that all parameters are set correctly to match your SUT. Here is an example of what +this file should contain. + +.. code-block:: bash + + $ cat ${DOVETAIL_HOME}/pre_config/env_config.sh + + # Project-level authentication scope (name or ID), recommend admin project. + export OS_PROJECT_NAME=admin + + # For identity v2, it uses OS_TENANT_NAME rather than OS_PROJECT_NAME. + export OS_TENANT_NAME=admin + + # Authentication username, belongs to the project above, recommend admin user. + export OS_USERNAME=admin + + # Authentication password. Use your own password + export OS_PASSWORD=xxxxxxxx + + # Authentication URL, one of the endpoints of keystone service. If this is v3 version, + # there need some extra variables as follows. + export OS_AUTH_URL='http://xxx.xxx.xxx.xxx:5000/v3' + + # Default is 2.0. If use keystone v3 API, this should be set as 3. + export OS_IDENTITY_API_VERSION=3 + + # Domain name or ID containing the user above. + # Command to check the domain: openstack user show <OS_USERNAME> + export OS_USER_DOMAIN_NAME=default + + # Domain name or ID containing the project above. + # Command to check the domain: openstack project show <OS_PROJECT_NAME> + export OS_PROJECT_DOMAIN_NAME=default + + # Home directory for dovetail that you have created before. + export DOVETAIL_HOME=/home/dovetail + +Export all these variables into environment by, + +.. code-block:: bash + + $ source ${DOVETAIL_HOME}/pre_config/env_config.sh + +If OpenStack client is installed, you can validate that the OpenStack environmental +settings are correct by, + +.. code-block:: bash + + $ openstack service list + + +Installing Prerequisite on the Test Host +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The main prerequisite software for Dovetail are Python and Docker. + +In the CVP test suite for the Danube release, Dovetail requires Python 2.7. Python 3.x +is not supported at this time. + +Use the following steps to check if the right version of python is already installed, +and if not, install it. + +.. code-block:: bash + + $ python --version + Python 2.7.6 + +If your Test Host does not have Python installed, or the version is not 2.7, you +should consult Python installation guides corresponding to the operating system +in your Test Host on how to install Python 2.7. + +Dovetail does not work with Docker versions prior to 1.12.3. We have validated +Dovetail with Docker 17.03 CE. Other versions of Docker later than 1.12.3 may +also work, but community support may be more available on Docker 17.03 CE. + +.. code-block:: bash + + $ sudo docker version + Client: + Version: 17.03.1-ce + API version: 1.27 + Go version: go1.7.5 + Git commit: c6d412e + Built: Mon Mar 27 17:10:36 2017 + OS/Arch: linux/amd64 + + Server: + Version: 17.03.1-ce + API version: 1.27 (minimum version 1.12) + Go version: go1.7.5 + Git commit: c6d412e + Built: Mon Mar 27 17:10:36 2017 + OS/Arch: linux/amd64 + Experimental: false + +If your Test Host does not have Docker installed, or Docker is older than 1.12.3, +or you have Docker version other than 17.03 CE and wish to change, +you will need to install, upgrade, or re-install in order to run Dovetail. +The Docker installation process +can be more complex, you should refer to the official +Docker installation guide that is relevant to your Test Host's operating system. + +The above installation steps assume that the Test Host is in the online mode. For offline +testing, use the following offline installation steps instead. + +In order to install or upgrade Python offline, you may download packaged Python 2.7 +for your Test Host's operating system on a connected host, copy the packge to +the Test Host, then install from that local copy. + +In order to install Docker offline, download Docker static binaries and copy the +tar file to the Test Host, such as for Ubuntu14.04, you may follow the following link +to install, + +.. code-block:: bash + + https://github.com/meetyg/docker-offline-install + + +Installing Dovetail on the Test Host +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The Dovetail project maintains a Docker image that has Dovetail test tools preinstalled. +This Docker image is tagged with versions. Before pulling the Dovetail image, check the +OPNFV's CVP web page first to determine the right tag for CVP testing. + +If the Test Host is online, you can directly pull. + +.. code-block:: bash + + $ sudo docker pull opnfv/dovetail:cvp.0.3.0 + cvp.0.3.0: Pulling from opnfv/dovetail + 30d541b48fc0: Pull complete + 8ecd7f80d390: Pull complete + 46ec9927bb81: Pull complete + 2e67a4d67b44: Pull complete + 7d9dd9155488: Pull complete + cc79be29f08e: Pull complete + e102eed9bf6a: Pull complete + 952b8a9d2150: Pull complete + bfbb639d1f38: Pull complete + bf7c644692de: Pull complete + cdc345e3f363: Pull complete + Digest: sha256:d571b1073b2fdada79562e8cc67f63018e8d89268ff7faabee3380202c05edee + Status: Downloaded newer image for opnfv/dovetail:cvp.0.3.0 + +An example of the <tag> is *cvp.0.3.0*. + +If the Test Host is offline, you will need to first pull the Dovetail Docker image, and all the +dependent images that Dovetail uses, to a host that is online. The reason that you need +to pull all dependent images is because Dovetail normally does dependency checking at run-time +and automatically pull images as needed, if the Test Host is online. If the Test Host is +offline, then all these dependencies will also need to be manually copied. + +.. code-block:: bash + + $ sudo docker pull opnfv/dovetail:cvp.0.3.0 + $ sudo docker pull opnfv/functest:cvp.0.2.0 + $ sudo docker pull opnfv/yardstick:danube.3.1 + $ sudo wget -nc http://artifacts.opnfv.org/sdnvpn/ubuntu-16.04-server-cloudimg-amd64-disk1.img -P {ANY_DIR} + +Once all these images are pulled, save the images, copy to the Test Host, and then load +the Dovetail and all dependent images at the Test Host. + +At the online host, save images. + +.. code-block:: bash + + $ sudo docker save -o dovetail.tar opnfv/dovetail:cvp.0.3.0 opnfv/functest:cvp.0.2.0 opnfv/yardstick:danube.3.1 + +Copy dovetail.tar file to the Test Host, and then load the images on the Test Host. + +.. code-block:: bash + + $ sudo docker load --input dovetail.tar + +Copy sdnvpn test area image ubuntu-16.04-server-cloudimg-amd64-disk1.img to ${DOVETAIL_HOME}/pre_config/. + +Now check to see that the Dovetail image has been pulled or loaded properly. + +.. code-block:: bash + + $ sudo docker images + REPOSITORY TAG IMAGE ID CREATED SIZE + opnfv/functest cvp.0.2.0 9eaeaea5f203 8 days ago 1.53GB + opnfv/dovetail cvp.0.3.0 5d25b289451c 8 days ago 516MB + opnfv/yardstick danube.3.1 574596b6ea12 8 days ago 1.2GB + +Regardless of whether you pulled down the Dovetail image directly online, or loaded from +a static image tar file, you are ready to run Dovetail. + +.. code-block:: bash + + $ sudo docker run --privileged=true -it \ + -e DOVETAIL_HOME=$DOVETAIL_HOME \ + -v $DOVETAIL_HOME:$DOVETAIL_HOME \ + -v /var/run/docker.sock:/var/run/docker.sock \ + opnfv/dovetail:<tag> /bin/bash + +The ``-e`` options set the env variables in the container and the ``-v`` options map files +in the host to files in the container. + +Running the CVP Test Suite +---------------------------- + +Now you should be in the Dovetail container's prompt and ready to execute +test suites. + +The Dovetail client CLI allows the tester to specify which test suite to run. +By default the results are stored in a local file +``$DOVETAIL_HOME/results``. + +.. code-block:: bash + + $ dovetail run --testsuite <test-suite-name> + +Multiple test suites may be available. For the purpose of running +CVP test suite, the test suite name follows the following format, +``CVP_<major>_<minor>_<patch>`` +For example, CVP_1_0_0. + +.. code-block:: bash + + $ dovetail run --testsuite CVP_1_0_0 + +If you are not running the entire test suite, you can choose to run an +individual test area instead. + +.. code-block:: bash + + $ dovetail run --testsuite CVP_1_0_0 --testarea ipv6 + +Until the official test suite is approved and released, you can use +the *proposed_tests* for your trial runs, like this. + +.. code-block:: bash + + $ dovetail run --testsuite proposed_tests --testarea ipv6 + 2017-05-23 05:01:49,488 - run - INFO - ================================================ + 2017-05-23 05:01:49,488 - run - INFO - Dovetail compliance: proposed_tests! + 2017-05-23 05:01:49,488 - run - INFO - ================================================ + 2017-05-23 05:01:49,488 - run - INFO - Build tag: daily-master-4bdde6b8-afa6-40bb-8fc9-5d568d74c8d7 + 2017-05-23 05:01:49,536 - run - INFO - + 2017-05-23 05:01:49,710 - run - INFO - >>[testcase]: dovetail.ipv6.tc001 + 2017-05-23 05:08:22,532 - run - INFO - Results have been stored with file /home/dovetail/results/functest_results.txt. + 2017-05-23 05:08:22,538 - run - INFO - >>[testcase]: dovetail.ipv6.tc002 + ... + +Special Configuration for Running HA Test Cases +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +HA test cases need to know the info of a controller node of the OpenStack. +It should include the node's name, role, ip, as well as the user and key_filename +or password to login the node. Users should create file ${DOVETAIL_HOME}/pre_config/pod.yaml +to store the info. + +There is a sample file for users. + +.. code-block:: bash + + nodes: + - + # This can not be changed and must be node1. + name: node1 + + # This must be controller. + role: Controller + + # This is the install IP of a controller node. + ip: xx.xx.xx.xx + + # User name of this node. This user must have sudo privileges. + user: root + + # Password of the user. + password: root + +Besides the 'password', user could also provide 'key_filename' to login the node. +Users need to create file $DOVETAIL_HOME/pre_config/id_rsa to store the private key. + +.. code-block:: bash + + name: node1 + role: Controller + ip: 10.1.0.50 + user: root + + # Private key of this node. It must be /root/.ssh/id_rsa + # Dovetail will move the key file from $DOVETAIL_HOME/pre_config/id_rsa + # to /root/.ssh/id_rsa of Yardstick container + key_filename: /root/.ssh/id_rsa + + +Making Sense of CVP Test Results +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When a tester is performing trial runs, Dovetail stores results in a local file by default. + +.. code-block:: bash + + cd $DOVETAIL_HOME/results + +#. Local file + + * Log file: dovetail.log + + * Review the dovetail.log to see if all important information has been captured + - in default mode without DEBUG. + + * The end of the log file has a summary of all test case test results. + + * Additional log files may be of interests: refstack.log, dovetail_ha_tcXXX.out ... + + * Example: Openstack refstack test case example + + * Can see the log details in refstack.log, which has the passed/skipped/failed + test cases result, the failed test cases have rich debug information for the + users to see why this test case fails. + + * Example: OPNFV Functest test case example + + * For Functest tool, its log is stored in functest.log + + * For each test case result in Functest, the logs are stored in functest_results.txt. + + * Example: OPNFV Yardstick test case example + + * For Yardstick tool, its log is stored in yardstick.log + + * For each test case result in Yardstick, the logs are stored in dovetail_ha_tcXXX.out, respectively. + +#. OPNFV web interface + + Wait for the complement of LF, test community, etc. + +Updating Dovetail or a Test Suite +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Follow the instructions in section `Installing Dovetail on the Test Host`_ and +`Running the CVP Test Suite`_ by replacing the docker images with new_tags, + +.. code-block:: bash + + sudo docker pull opnfv/dovetail:<dovetail_new_tag> + sudo docker pull opnfv/functest:<functest_new_tag> + sudo docker pull opnfv/yardstick:<yardstick_new_tag> + +This step is necessary if dovetail software or the CVP test suite have updates. + + diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml index aa27f293..a5b262d1 100644 --- a/dovetail/conf/cmd_config.yml +++ b/dovetail/conf/cmd_config.yml @@ -50,7 +50,7 @@ cli: flags: - '--report' - '-r' - help: 'push results to DB (e.g. --report http://192.168.135.2:8000/api/v1)' + help: 'push results to DB (e.g. --report http://192.168.135.2:8000/api/v1/results)' offline: flags: - '--offline' diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml index 793c7051..fecc0ae1 100644 --- a/dovetail/conf/dovetail_config.yml +++ b/dovetail/conf/dovetail_config.yml @@ -79,5 +79,7 @@ validate_input: - 'danube.1.0' - 'danube.2.0' - 'danube.3.0' + - 'danube.3.1' - 'cvp.0.1.0' - 'cvp.0.2.0' + - 'cvp.0.3.0' diff --git a/dovetail/conf/yardstick_config.yml b/dovetail/conf/yardstick_config.yml index 86a768d5..fd93d61a 100644 --- a/dovetail/conf/yardstick_config.yml +++ b/dovetail/conf/yardstick_config.yml @@ -1,7 +1,7 @@ --- yardstick: image_name: opnfv/yardstick - docker_tag: danube.3.0 + docker_tag: danube.3.1 opts: '-id --privileged=true' config: dir: '/home/opnfv/userconfig' @@ -25,3 +25,6 @@ yardstick: file_path: 'yardstick.log' key_path: '/root/.ssh/id_rsa' openrc: '/etc/yardstick/openstack.creds' + yard_conf: + src_file: '/home/opnfv/repos/yardstick/etc/yardstick/yardstick.conf.sample' + dest_file: '/etc/yardstick/yardstick.conf' diff --git a/dovetail/container.py b/dovetail/container.py index 2108cdc1..1a5867a7 100644 --- a/dovetail/container.py +++ b/dovetail/container.py @@ -76,8 +76,7 @@ class Container(object): if 'sdnvpn' in testcase_name: ins_type = "-e INSTALLER_TYPE=netvirt" scenario = " -e DEPLOY_SCENARIO=bgpvpn" - node = " -e NODE_NAME=master" - envs = "%s %s %s" % (ins_type, scenario, node) + envs = "%s %s" % (ins_type, scenario) dovetail_config = dt_cfg.dovetail_config if dovetail_config['report_dest'].startswith("http"): @@ -103,11 +102,6 @@ class Container(object): cls.logger.error("Can't find any external network.") return None - if dovetail_config['report_dest'].startswith("http"): - envs = envs + " -e DISPATCHER=http" - envs = envs + " -e TARGET=%s" % dovetail_config['report_dest'] - envs = envs + " -e NODE_NAME=master" - log_vol = '-v %s:%s ' % (dovetail_config['result_dir'], dovetail_config["yardstick"]['result']['log']) @@ -141,7 +135,7 @@ class Container(object): # CI_DEBUG is used for showing the debug logs of the upstream projects # BUILD_TAG is the unique id for this test - envs = ' -e CI_DEBUG=true' + envs = ' -e CI_DEBUG=true -e NODE_NAME=master' envs = envs + ' -e BUILD_TAG=%s-%s' % (dovetail_config['build_tag'], testcase_name) @@ -177,7 +171,7 @@ class Container(object): cacert_volume = "" https_enabled = dt_utils.check_https_enabled(cls.logger) cacert = os.getenv('OS_CACERT',) - if https_enabled == 0: + if https_enabled: cls.logger.info("https enabled...") if cacert is not None: if not os.path.isfile(cacert): @@ -212,6 +206,10 @@ class Container(object): src_path = os.path.join(prefix_path, 'pre_config', file_name) dest_path = '/home/opnfv/functest/images' Container.pre_copy(container_id, src_path, dest_path) + + if type.lower() == 'yardstick': + cls.set_yardstick_conf_file(container_id) + return container_id @classmethod @@ -306,3 +304,18 @@ class Container(object): return (1, 'src_path or dest_path is empty') cmd = 'cp %s %s' % (src_path, dest_path) return cls.exec_cmd(container_id, cmd, exit_on_error) + + @classmethod + def set_yardstick_conf_file(cls, container_id): + valid_type = 'yardstick' + src = dt_cfg.dovetail_config[valid_type]['yard_conf']['src_file'] + dest = dt_cfg.dovetail_config[valid_type]['yard_conf']['dest_file'] + cls.pre_copy(container_id, src, dest) + url = dt_cfg.dovetail_config['report_dest'] + if url.startswith("http"): + cmd = ("sed -i '16s#http://127.0.0.1:8000/results#{}#g' {}" + .format(url, dest)) + cls.exec_cmd(container_id, cmd) + if url.lower() == 'file': + cmd = ("sed -i '12s/http/file/g' {}".format(dest)) + cls.exec_cmd(container_id, cmd) diff --git a/dovetail/run.py b/dovetail/run.py index 5c0b2ddb..f89671f0 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -220,7 +220,7 @@ def env_init(logger): def main(*args, **kwargs): """Dovetail compliance test entry!""" - build_tag = "daily-master-%s" % str(uuid.uuid4()) + build_tag = "daily-master-%s" % str(uuid.uuid1()) dt_cfg.dovetail_config['build_tag'] = build_tag if not get_result_path(): return diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py index 8112d28d..f74da3a2 100644 --- a/dovetail/utils/dovetail_utils.py +++ b/dovetail/utils/dovetail_utils.py @@ -120,17 +120,18 @@ def source_env(env_file): def check_https_enabled(logger=None): logger.debug("Checking if https enabled or not...") - cmd = "openstack catalog show identity |awk '/public/ {print $4}'| \ - grep 'https'" + cmd = ("openstack catalog show identity |awk '/public/ {print $4}'") ret, msg = exec_cmd(cmd, logger) - return ret + if ret == 0 and "https://" in msg: + return True + return False def get_ext_net_name(env_file, logger=None): https_enabled = check_https_enabled(logger) insecure_option = '' insecure = os.getenv('OS_INSECURE',) - if https_enabled == 0: + if https_enabled: logger.info("https enabled...") if insecure.lower() == "true": insecure_option = ' --insecure ' diff --git a/dovetail/utils/local_db/launch_db.sh b/dovetail/utils/local_db/launch_db.sh index e31f47ce..722dc227 100755 --- a/dovetail/utils/local_db/launch_db.sh +++ b/dovetail/utils/local_db/launch_db.sh @@ -59,8 +59,8 @@ echo "Create the testapi service." echo "==========================" set +e -# pull image opnfv/testapi:cvp.0.2.0 -testapi_img="opnfv/testapi:cvp.0.2.0" +# pull image opnfv/testapi:cvp.0.3.0 +testapi_img="opnfv/testapi:cvp.0.3.0" echo "Step1: pull the image $testapi_img." sudo docker pull $testapi_img set -e diff --git a/dovetail/utils/local_db/restart_db.sh b/dovetail/utils/local_db/restart_db.sh index 5d6b9b68..35096b99 100755 --- a/dovetail/utils/local_db/restart_db.sh +++ b/dovetail/utils/local_db/restart_db.sh @@ -26,4 +26,4 @@ export db_host_ip=${db_host_ip:-"$1"} sudo docker rm -f testapi sudo docker run -itd -p $testapi_port:8000 --name testapi \ - -e mongodb_url=mongodb://$db_host_ip:$mongodb_port/ opnfv/testapi:cvp.0.2.0 + -e mongodb_url=mongodb://$db_host_ip:$mongodb_port/ opnfv/testapi:cvp.0.3.0 diff --git a/dovetail/utils/offline/config.yaml b/dovetail/utils/offline/config.yaml index edb8a5b8..11134dd1 100644 --- a/dovetail/utils/offline/config.yaml +++ b/dovetail/utils/offline/config.yaml @@ -2,7 +2,7 @@ docker_images: dovetail: domain: opnfv - tag: cvp.0.2.0 + tag: cvp.0.3.0 store_name: image_dovetail.docker functest: domain: opnfv @@ -10,11 +10,11 @@ docker_images: store_name: image_functest.docker yardstick: domain: opnfv - tag: danube.3.0 + tag: danube.3.1 store_name: image_yardstick.docker testapi: domain: opnfv - tag: cvp.0.2.0 + tag: cvp.0.3.0 store_name: image_testapi.docker mongo: tag: 3.2.1 @@ -1,6 +1,6 @@ [metadata] name = dovetail -version = 0.2.0 +version = 0.3.0 home-page = https://wiki.opnfv.org/display/dovetail [entry_points] |