From c8f970e5b5968952f9c1fea5c7c63bddb12a7b73 Mon Sep 17 00:00:00 2001 From: "dimitris.tsiolakis" Date: Tue, 12 Jun 2018 14:12:25 +0300 Subject: Fix so that testcases run in order. A new key "order" was added in config.py file to prevent the testcases executed in random order. JIRA: SDNVPN-216 Change-Id: I4ee72cb98351e35bf50ecf9c6dd1506a59e34f6e Signed-off-by: dimitris.tsiolakis --- sdnvpn/test/functest/config.yaml | 397 ++++++++++++++++--------------- sdnvpn/test/functest/run_sdnvpn_tests.py | 7 +- 2 files changed, 209 insertions(+), 195 deletions(-) diff --git a/sdnvpn/test/functest/config.yaml b/sdnvpn/test/functest/config.yaml index d719453..777db86 100644 --- a/sdnvpn/test/functest/config.yaml +++ b/sdnvpn/test/functest/config.yaml @@ -3,218 +3,229 @@ defaults: testcases: sdnvpn.test.functest.run_tempest: - enabled: true - description: Neutron BGPVPN tests in tempest + enabled: true + order: 0 + description: Neutron BGPVPN tests in tempest sdnvpn.test.functest.testcase_1: - enabled: true - description: VPN provides connectivity between subnets - instance_1_name: sdnvpn-1-1 - instance_2_name: sdnvpn-1-2 - instance_3_name: sdnvpn-1-3 - instance_4_name: sdnvpn-1-4 - instance_5_name: sdnvpn-1-5 - image_name: sdnvpn-image - net_1_name: sdnvpn-1-1-net - subnet_1_name: sdnvpn-1-1-subnet - subnet_1_cidr: 10.10.10.0/24 - router_1_name: sdnvpn-1-1-router - net_2_name: sdnvpn-1-2-net - subnet_2_name: sdnvpn-1-2-subnet - subnet_2_cidr: 10.10.11.0/24 - router_2_name: sdnvpn-1-2-router - secgroup_name: sdnvpn-sg - secgroup_descr: Security group for SDNVPN test cases - targets1: '88:88' - targets2: '55:55' - route_distinguishers: '11:11' + enabled: true + order: 1 + description: VPN provides connectivity between subnets + instance_1_name: sdnvpn-1-1 + instance_2_name: sdnvpn-1-2 + instance_3_name: sdnvpn-1-3 + instance_4_name: sdnvpn-1-4 + instance_5_name: sdnvpn-1-5 + image_name: sdnvpn-image + net_1_name: sdnvpn-1-1-net + subnet_1_name: sdnvpn-1-1-subnet + subnet_1_cidr: 10.10.10.0/24 + router_1_name: sdnvpn-1-1-router + net_2_name: sdnvpn-1-2-net + subnet_2_name: sdnvpn-1-2-subnet + subnet_2_cidr: 10.10.11.0/24 + router_2_name: sdnvpn-1-2-router + secgroup_name: sdnvpn-sg + secgroup_descr: Security group for SDNVPN test cases + targets1: '88:88' + targets2: '55:55' + route_distinguishers: '11:11' sdnvpn.test.functest.testcase_2: - enabled: true - description: Tenant separation - instance_1_name: sdnvpn-2-1 - instance_2_name: sdnvpn-2-2 - instance_3_name: sdnvpn-2-3 - instance_4_name: sdnvpn-2-4 - instance_5_name: sdnvpn-2-5 - instance_1_ip: 10.10.10.11 - instance_2_ip: 10.10.10.12 - instance_3_ip: 10.10.11.13 - instance_4_ip: 10.10.10.12 - instance_5_ip: 10.10.11.13 - image_name: sdnvpn-image - net_1_name: sdnvpn-2-1-net - subnet_1a_name: sdnvpn-2-1a-subnet - subnet_1a_cidr: 10.10.10.0/24 - subnet_1b_name: sdnvpn-2-1b-subnet - subnet_1b_cidr: 10.10.11.0/24 - router_1_name: sdnvpn-2-1-router - net_2_name: sdnvpn-2-2-net - subnet_2a_name: sdnvpn-2-2a-subnet - subnet_2a_cidr: 10.10.11.0/24 - subnet_2b_name: sdnvpn-2-2b-subnet - subnet_2b_cidr: 10.10.10.0/24 - router_2_name: sdnvpn-2-2-router - secgroup_name: sdnvpn-sg - secgroup_descr: Security group for SDNVPN test cases - targets1: '88:88' - targets2: '55:55' - route_distinguishers1: '111:111' - route_distinguishers2: '222:222' + enabled: true + order: 2 + description: Tenant separation + instance_1_name: sdnvpn-2-1 + instance_2_name: sdnvpn-2-2 + instance_3_name: sdnvpn-2-3 + instance_4_name: sdnvpn-2-4 + instance_5_name: sdnvpn-2-5 + instance_1_ip: 10.10.10.11 + instance_2_ip: 10.10.10.12 + instance_3_ip: 10.10.11.13 + instance_4_ip: 10.10.10.12 + instance_5_ip: 10.10.11.13 + image_name: sdnvpn-image + net_1_name: sdnvpn-2-1-net + subnet_1a_name: sdnvpn-2-1a-subnet + subnet_1a_cidr: 10.10.10.0/24 + subnet_1b_name: sdnvpn-2-1b-subnet + subnet_1b_cidr: 10.10.11.0/24 + router_1_name: sdnvpn-2-1-router + net_2_name: sdnvpn-2-2-net + subnet_2a_name: sdnvpn-2-2a-subnet + subnet_2a_cidr: 10.10.11.0/24 + subnet_2b_name: sdnvpn-2-2b-subnet + subnet_2b_cidr: 10.10.10.0/24 + router_2_name: sdnvpn-2-2-router + secgroup_name: sdnvpn-sg + secgroup_descr: Security group for SDNVPN test cases + targets1: '88:88' + targets2: '55:55' + route_distinguishers1: '111:111' + route_distinguishers2: '222:222' sdnvpn.test.functest.testcase_3: - enabled: true - description: Data center gateway integration - secgroup_name: sdnvpn-sg - secgroup_descr: Security group for SDNVPN test cases - image_name: sdnvpn-image - ubuntu_image_name: sdnvpn-ubuntu-image - net_1_name: sdnvpn-3-1-net - subnet_1_name: sdnvpn-3-1-subnet - subnet_1_cidr: 10.10.10.0/24 - router_1_name: sdnvpn-3-1-router - quagga_net_name: sdnvpn-3-2-quagga-net - quagga_subnet_name: sdnvpn-3-2-quagga-subnet - quagga_subnet_cidr: 10.10.11.0/24 - quagga_router_name: sdnvpn-3-2-quagga-router - quagga_instance_name: sdnvpn-3-2-quagga - quagga_instance_ip: 10.10.11.5 - instance_1_name: sdnvpn-3-1 - instance_1_ip: 10.10.10.5 - import_targets: '31:31' - export_targets: '32:32' - + enabled: true + order: 3 + description: Data center gateway integration + secgroup_name: sdnvpn-sg + secgroup_descr: Security group for SDNVPN test cases + image_name: sdnvpn-image + ubuntu_image_name: sdnvpn-ubuntu-image + net_1_name: sdnvpn-3-1-net + subnet_1_name: sdnvpn-3-1-subnet + subnet_1_cidr: 10.10.10.0/24 + router_1_name: sdnvpn-3-1-router + quagga_net_name: sdnvpn-3-2-quagga-net + quagga_subnet_name: sdnvpn-3-2-quagga-subnet + quagga_subnet_cidr: 10.10.11.0/24 + quagga_router_name: sdnvpn-3-2-quagga-router + quagga_instance_name: sdnvpn-3-2-quagga + quagga_instance_ip: 10.10.11.5 + instance_1_name: sdnvpn-3-1 + instance_1_ip: 10.10.10.5 + import_targets: '31:31' + export_targets: '32:32' sdnvpn.test.functest.testcase_4: - enabled: true - description: VPN provides connectivity between subnets using router association - instance_1_name: sdnvpn-4-1 - instance_2_name: sdnvpn-4-2 - instance_3_name: sdnvpn-4-3 - instance_4_name: sdnvpn-4-4 - instance_5_name: sdnvpn-4-5 - image_name: sdnvpn-image - net_1_name: sdnvpn-4-1-net - subnet_1_name: sdnvpn-4-1-subnet - subnet_1_cidr: 10.10.10.0/24 - router_1_name: sdnvpn-4-1-router - net_2_name: sdnvpn-4-2-net - subnet_2_name: sdnvpn-4-2-subnet - subnet_2_cidr: 10.10.11.0/24 - router_2_name: sdnvpn-4-2-router - secgroup_name: sdnvpn-sg - secgroup_descr: Security group for SDNVPN test cases - targets1: '88:88' - targets2: '55:55' - route_distinguishers: '12:12' + enabled: true + order: 4 + description: VPN provides connectivity between subnets using router association + instance_1_name: sdnvpn-4-1 + instance_2_name: sdnvpn-4-2 + instance_3_name: sdnvpn-4-3 + instance_4_name: sdnvpn-4-4 + instance_5_name: sdnvpn-4-5 + image_name: sdnvpn-image + net_1_name: sdnvpn-4-1-net + subnet_1_name: sdnvpn-4-1-subnet + subnet_1_cidr: 10.10.10.0/24 + router_1_name: sdnvpn-4-1-router + net_2_name: sdnvpn-4-2-net + subnet_2_name: sdnvpn-4-2-subnet + subnet_2_cidr: 10.10.11.0/24 + router_2_name: sdnvpn-4-2-router + secgroup_name: sdnvpn-sg + secgroup_descr: Security group for SDNVPN test cases + targets1: '88:88' + targets2: '55:55' + route_distinguishers: '12:12' sdnvpn.test.functest.testcase_7: - enabled: false - description: Network associate VPNs with routers attached (ODL Bug 6962) - image_name: sdnvpn-image - instance_1_name: sdnvpn-7-1 - instance_2_name: sdnvpn-7-2 - net_1_name: sdnvpn-7-1 - subnet_1_name: sdnvpn-7-1-subnet - subnet_1_cidr: 10.10.10.0/24 - router_1_name: sdnvpn-7-1-router - net_2_name: sdnvpn-7-2 - subnet_2_name: sdnvpn-7-2-subnet - subnet_2_cidr: 10.10.20.0/24 - router_2_name: sdnvpn-7-2-router - secgroup_name: sdnvpn-sg - secgroup_descr: Security group for SDNVPN test cases - targets: '77:77' - route_distinguishers: '11:11' + enabled: false + order: 7 + description: Network associate VPNs with routers attached (ODL Bug 6962) + image_name: sdnvpn-image + instance_1_name: sdnvpn-7-1 + instance_2_name: sdnvpn-7-2 + net_1_name: sdnvpn-7-1 + subnet_1_name: sdnvpn-7-1-subnet + subnet_1_cidr: 10.10.10.0/24 + router_1_name: sdnvpn-7-1-router + net_2_name: sdnvpn-7-2 + subnet_2_name: sdnvpn-7-2-subnet + subnet_2_cidr: 10.10.20.0/24 + router_2_name: sdnvpn-7-2-router + secgroup_name: sdnvpn-sg + secgroup_descr: Security group for SDNVPN test cases + targets: '77:77' + route_distinguishers: '11:11' sdnvpn.test.functest.testcase_8: - enabled: true - description: Test floating IP and router assoc coexistence - image_name: sdnvpn-image - instance_1_name: sdnvpn-8-1 - instance_2_name: sdnvpn-8-2 - net_1_name: sdnvpn-8-1 - subnet_1_name: sdnvpn-8-1-subnet - subnet_1_cidr: 10.10.10.0/24 - router_1_name: sdnvpn-8-1-router - net_2_name: sdnvpn-8-2 - subnet_2_name: sdnvpn-8-2-subnet - subnet_2_cidr: 10.10.20.0/24 - router_2_name: sdnvpn-8-2-router - secgroup_name: sdnvpn-sg - secgroup_descr: Security group for SDNVPN test cases - targets: '88:88' - route_distinguishers: '18:18' + enabled: true + order: 8 + description: Test floating IP and router assoc coexistence + image_name: sdnvpn-image + instance_1_name: sdnvpn-8-1 + instance_2_name: sdnvpn-8-2 + net_1_name: sdnvpn-8-1 + subnet_1_name: sdnvpn-8-1-subnet + subnet_1_cidr: 10.10.10.0/24 + router_1_name: sdnvpn-8-1-router + net_2_name: sdnvpn-8-2 + subnet_2_name: sdnvpn-8-2-subnet + subnet_2_cidr: 10.10.20.0/24 + router_2_name: sdnvpn-8-2-router + secgroup_name: sdnvpn-sg + secgroup_descr: Security group for SDNVPN test cases + targets: '88:88' + route_distinguishers: '18:18' sdnvpn.test.functest.testcase_9: - enabled: true - description: Verify that all OpenStack nodes OVS br-int have fail_mode set to secure. + enabled: true + order: 9 + description: Verify that all OpenStack nodes OVS br-int have fail_mode set to secure. sdnvpn.test.functest.testcase_10: - enabled: true - description: Test if interupts occure during ping, when removing and adding instances - instance_1_name: sdnvpn-10-1 - instance_2_name: sdnvpn-10-2 - instance_3_name: sdnvpn-10-3 - instance_4_name: sdnvpn-10-4 - image_name: sdnvpn-image - net_1_name: sdnvpn-10-1-net - subnet_1_name: sdnvpn-10-1-subnet - subnet_1_cidr: 10.10.10.0/24 - router_1_name: sdnvpn-10-1-router - secgroup_name: sdnvpn-sg - secgroup_descr: Security group for SDNVPN test cases + enabled: true + order: 10 + description: Test if interupts occure during ping, when removing and adding instances + instance_1_name: sdnvpn-10-1 + instance_2_name: sdnvpn-10-2 + instance_3_name: sdnvpn-10-3 + instance_4_name: sdnvpn-10-4 + image_name: sdnvpn-image + net_1_name: sdnvpn-10-1-net + subnet_1_name: sdnvpn-10-1-subnet + subnet_1_cidr: 10.10.10.0/24 + router_1_name: sdnvpn-10-1-router + secgroup_name: sdnvpn-sg + secgroup_descr: Security group for SDNVPN test cases sdnvpn.test.functest.testcase_11: - enabled: true - description: Check relevant OVS groups are removed upon deletion of OpenStack topology - instance_1_name: sdnvpn-11-1 - instance_2_name: sdnvpn-11-2 - image_name: sdnvpn-image - net_1_name: sdnvpn-11-1-net - subnet_1_name: sdnvpn-11-1-subnet - subnet_1_cidr: 10.10.10.0/24 - router_1_name: sdnvpn-11-1-router - secgroup_name: sdnvpn-sg - secgroup_descr: Security group for SDNVPN test cases + enabled: true + order: 11 + description: Check relevant OVS groups are removed upon deletion of OpenStack topology + instance_1_name: sdnvpn-11-1 + instance_2_name: sdnvpn-11-2 + image_name: sdnvpn-image + net_1_name: sdnvpn-11-1-net + subnet_1_name: sdnvpn-11-1-subnet + subnet_1_cidr: 10.10.10.0/24 + router_1_name: sdnvpn-11-1-router + secgroup_name: sdnvpn-sg + secgroup_descr: Security group for SDNVPN test cases sdnvpn.test.functest.testcase_12: - enabled: true - description: Check Flows and Groups are reprogrammed after OVS reconnect - instance_1_name: sdnvpn-12-1 - instance_2_name: sdnvpn-12-2 - image_name: sdnvpn-image - net_1_name: sdnvpn-12-1-net - subnet_1_name: sdnvpn-12-1-subnet - subnet_1_cidr: 10.10.10.0/24 - secgroup_name: sdnvpn-sg - secgroup_descr: Security group for SDNVPN test cases + enabled: true + order: 12 + description: Check Flows and Groups are reprogrammed after OVS reconnect + instance_1_name: sdnvpn-12-1 + instance_2_name: sdnvpn-12-2 + image_name: sdnvpn-image + net_1_name: sdnvpn-12-1-net + subnet_1_name: sdnvpn-12-1-subnet + subnet_1_cidr: 10.10.10.0/24 + secgroup_name: sdnvpn-sg + secgroup_descr: Security group for SDNVPN test cases sdnvpn.test.functest.testcase_13: - enabled: true - description: Testing extra route ECMP for intra-data center scenario - instance_1_name: sdnvpn-13-1 - instance_2_name: sdnvpn-13-2 - instance_3_name: sdnvpn-13-3 - image_name: sdnvpn-image - net_1_name: sdnvpn-13-1-net - subnet_1_name: sdnvpn-13-1-subnet - subnet_1_cidr: 10.10.10.0/24 - router_1_name: sdnvpn-13-1-router - net_2_name: sdnvpn-13-2-net - subnet_2_name: sdnvpn-13-2-subnet - subnet_2_cidr: 10.10.11.0/24 - router_2_name: sdnvpn-13-2-router - interface_name: lo - interface_number: 1 - extra_route_cidr: 179.24.1.12/32 - extra_route_ip: 179.24.1.12 - extra_route_subnet_mask: 255.255.255.255 - extra_route_name: sdnvpn_extra_route_13 - secgroup_name: sdnvpn-sg - secgroup_descr: Security group for SDNVPN test cases - targets1: '88:88' - targets2: '88:88' - route_distinguishers: - - '12:12' - - '13:13' + enabled: true + order: 13 + description: Testing extra route ECMP for intra-data center scenario + instance_1_name: sdnvpn-13-1 + instance_2_name: sdnvpn-13-2 + instance_3_name: sdnvpn-13-3 + image_name: sdnvpn-image + net_1_name: sdnvpn-13-1-net + subnet_1_name: sdnvpn-13-1-subnet + subnet_1_cidr: 10.10.10.0/24 + router_1_name: sdnvpn-13-1-router + net_2_name: sdnvpn-13-2-net + subnet_2_name: sdnvpn-13-2-subnet + subnet_2_cidr: 10.10.11.0/24 + router_2_name: sdnvpn-13-2-router + interface_name: lo + interface_number: 1 + extra_route_cidr: 179.24.1.12/32 + extra_route_ip: 179.24.1.12 + extra_route_subnet_mask: 255.255.255.255 + extra_route_name: sdnvpn_extra_route_13 + secgroup_name: sdnvpn-sg + secgroup_descr: Security group for SDNVPN test cases + targets1: '88:88' + targets2: '88:88' + route_distinguishers: + - '12:12' + - '13:13' diff --git a/sdnvpn/test/functest/run_sdnvpn_tests.py b/sdnvpn/test/functest/run_sdnvpn_tests.py index e4d55f1..23a62e6 100644 --- a/sdnvpn/test/functest/run_sdnvpn_tests.py +++ b/sdnvpn/test/functest/run_sdnvpn_tests.py @@ -15,6 +15,7 @@ import sys import traceback import yaml +from collections import OrderedDict from xtesting.core import feature from sdnvpn.lib import config as sdnvpn_config from sdnvpn.lib import openstack_utils as os_utils @@ -74,9 +75,11 @@ class SdnvpnFunctest(feature.Feature): config_yaml = yaml.safe_load(f) testcases = config_yaml.get("testcases") + testcases_ordered = OrderedDict(sorted(testcases.items(), + key=lambda x: x[1]['order'])) overall_status = "PASS" - for tc in testcases: - if testcases[tc]['enabled']: + for tc, test_sdnvpn in testcases_ordered.items(): + if test_sdnvpn['enabled']: test_name = tc test_descr = testcases[tc]['description'] title = ("Running '%s - %s'" % -- cgit 1.2.3-korg