Intra-Data Center ECMP testcase

This testcase add a scenario in which one extra route can be reachable
through two VMs which are in a same network attached to a router which
is later associated to a BGPVPN instance.
This extra route ip address is different from subnet of VMs.

After the extra route is configured with VM ips as nexthops, then when
third VM is trying to reach extra route, traffic would split between two
VMs. The traffic split can happen only in heavy loaded scenario, so this
testcase would test only reachability.

JIRA: SDNVPN-23

Change-Id: I4bb4e94f5c98c0f3c22bff7a6998af2a7a919f16
Signed-off-by: Periyasamy Palanisamy <periyasamy.palanisamy@ericsson.com>

5 files changed, 382 insertions, 9 deletions

diff --git a/docs/development/overview/index.rst b/docs/development/overview/index.rst
index 8747170..e932f9a 100644
--- a/docs/development/overview/index.rst
+++ b/docs/development/overview/index.rst
@@ -244,3 +244,20 @@ https://wiki.opnfv.org/display/sdnvpn/SDNVPN+Testing
     Reconnect the OVS by adding ip tables drop rule and then remove it
         The flows and groups are still intact and none of the flows/groups
         are removed
+
+    Testcase 13: Test ECMP (Equal-cost multi-path routing) for the extra route
+    This testcase validates spraying behavior in OvS when an extra route is
+    configured such that it can be reached from two nova VMs in the
+    same network.
+
+   Setup procedure:
+   Create and start VM1 and VM2 configured with sub interface set to same ip
+   address in both VMs, connected to a common network/router.
+   Update the VM1 and VM2's Neutron ports with allowed address pairs for sub
+   interface ip/mac addresses.
+   Create BGPVPN with two route distinguishers.
+   Associate router with BGPVPN.
+   Update the router with above sub-interface ip address with nexthops set to
+   VMs ip addresses.
+   Create VM3 and connected to the same network.
+   Ping sub-interface IP address from VM3.
diff --git a/sdnvpn/lib/results.py b/sdnvpn/lib/results.py
index 679284e..790a916 100644
--- a/sdnvpn/lib/results.py
+++ b/sdnvpn/lib/results.py
@@ -29,11 +29,17 @@ class Results(object):
                         vm_source,
                         vm_target,
                         expected="PASS", timeout=30):
+        ip_target = vm_target.networks.itervalues().next()[0]
+        self.get_ping_status_target_ip(vm_source, vm_target.name,
+                                       ip_target, expected, timeout)
+
+    def get_ping_status_target_ip(self,
+                                  vm_source,
+                                  target_name,
+                                  ip_target,
+                                  expected="PASS", timeout=30):
         console_log = vm_source.get_console_output()
-
         ip_source = vm_source.networks.itervalues().next()[0]
-        ip_target = vm_target.networks.itervalues().next()[0]
-
         if "request failed" in console_log:
             # Normally, cirros displays this message when userdata fails
             logger.debug("It seems userdata is not supported in "
@@ -46,11 +52,11 @@ class Results(object):
             test_case_name = ("'%s' %s '%s'" %
                               (vm_source.name,
                                expected_result,
-                               vm_target.name))
+                               target_name))
             logger.debug("%sPing\n%sfrom '%s' (%s)\n%sto '%s' (%s).\n"
                          "%s-->Expected result: %s.\n"
                          % (tab, tab, vm_source.name, ip_source,
-                            tab, vm_target.name, ip_target,
+                            tab, target_name, ip_target,
                             tab, expected_result))
             while True:
                 console_log = vm_source.get_console_output()
@@ -60,7 +66,7 @@ class Results(object):
                 last_n_lines = lines[-5:]
                 if ("ping %s OK" % ip_target) in last_n_lines:
                     msg = ("'%s' can ping '%s'"
-                           % (vm_source.name, vm_target.name))
+                           % (vm_source.name, target_name))
                     if expected == "PASS":
                         logger.debug("[PASS] %s" % msg)
                         self.add_success(test_case_name)
@@ -71,7 +77,7 @@ class Results(object):
                     break
                 elif ("ping %s KO" % ip_target) in last_n_lines:
                     msg = ("'%s' cannot ping '%s'" %
-                           (vm_source.name, vm_target.name))
+                           (vm_source.name, target_name))
                     if expected == "FAIL":
                         logger.debug("[PASS] %s" % msg)
                         self.add_success(test_case_name)
diff --git a/sdnvpn/lib/utils.py b/sdnvpn/lib/utils.py
index 78493be..0ab8b84 100644
--- a/sdnvpn/lib/utils.py
+++ b/sdnvpn/lib/utils.py
@@ -14,6 +14,7 @@ import time
 import requests
 import re
 import subprocess
+from concurrent.futures import ThreadPoolExecutor
 
 import functest.utils.openstack_utils as os_utils
 from opnfv.deployment.factory import Factory as DeploymentFactory
@@ -27,6 +28,26 @@ common_config = sdnvpn_config.CommonConfig()
 ODL_USER = 'admin'
 ODL_PASS = 'admin'
 
+executor = ThreadPoolExecutor(5)
+
+
+class ExtraRoute(object):
+    """
+    Class to represent extra route for a router
+    """
+    def __init__(self, destination, nexthop):
+        self.destination = destination
+        self.nexthop = nexthop
+
+
+class AllowedAddressPair(object):
+    """
+    Class to represent allowed address pair for a neutron port
+    """
+    def __init__(self, ipaddress, macaddress):
+        self.ipaddress = ipaddress
+        self.macaddress = macaddress
+
 
 def create_custom_flavor():
     return os_utils.get_or_create_flavor(common_config.custom_flavor_name,
@@ -89,6 +110,37 @@ def create_network(neutron_client, net, subnet1, cidr1,
     return net_id, subnet_id, router_id
 
 
+def get_port(neutron_client, instance_id):
+    ports = os_utils.get_port_list(neutron_client)
+    if ports is not None:
+        for port in ports:
+            if port['device_id'] == instance_id:
+                return port
+    return None
+
+
+def update_port_allowed_address_pairs(neutron_client, port_id, address_pairs):
+    if len(address_pairs) <= 0:
+        return
+    allowed_address_pairs = []
+    for address_pair in address_pairs:
+        address_pair_dict = {'ip_address': address_pair.ipaddress,
+                             'mac_address': address_pair.macaddress}
+        allowed_address_pairs.append(address_pair_dict)
+    json_body = {'port': {
+        "allowed_address_pairs": allowed_address_pairs
+    }}
+
+    try:
+        port = neutron_client.update_port(port=port_id,
+                                          body=json_body)
+        return port['port']['id']
+    except Exception as e:
+        logger.error("Error [update_neutron_port(neutron_client, '%s', '%s')]:"
+                     " %s" % (port_id, address_pairs, e))
+        return None
+
+
 def create_instance(nova_client,
                     name,
                     image_id,
@@ -208,6 +260,19 @@ def generate_userdata_with_ssh(ips_array):
     return (u1 + u2)
 
 
+def generate_userdata_interface_create(interface_name, interface_number,
+                                       ip_Address, net_mask):
+    return ("#!/bin/sh\n"
+            "set -xe\n"
+            "sudo useradd -m sdnvpn\n"
+            "sudo adduser sdnvpn sudo\n"
+            "sudo echo sdnvpn:opnfv | chpasswd\n"
+            "sleep 20\n"
+            "sudo ifconfig %s:%s %s netmask %s up\n"
+            % (interface_name, interface_number,
+               ip_Address, net_mask))
+
+
 def get_installerHandler():
     installer_type = str(os.environ['INSTALLER_TYPE'].lower())
     installer_ip = get_installer_ip()
@@ -247,9 +312,8 @@ def get_instance_ip(instance):
     return instance_ip
 
 
-def wait_for_instance(instance, pattern=".* login:"):
+def wait_for_instance(instance, pattern=".* login:", tries=40):
     logger.info("Waiting for instance %s to boot up" % instance.id)
-    tries = 40
     sleep_time = 2
     expected_regex = re.compile(pattern)
     console_log = ""
@@ -276,6 +340,23 @@ def wait_for_instances_get_dhcp(*instances):
     return all(check)
 
 
+def async_Wait_for_instances(instances, tries=40):
+    if len(instances) <= 0:
+        return
+    futures = []
+    for instance in instances:
+        future = executor.submit(wait_for_instance,
+                                 instance,
+                                 ".* login:",
+                                 tries)
+        futures.append(future)
+    results = []
+    for future in futures:
+        results.append(future.result())
+    if False in results:
+        logger.error("one or more instances is not yet booted up")
+
+
 def wait_for_bgp_net_assoc(neutron_client, bgpvpn_id, net_id):
     tries = 30
     sleep_time = 1
@@ -741,6 +822,39 @@ def get_nova_instances_quota(nova_client):
         raise
 
 
+def update_router_extra_route(neutron_client, router_id, extra_routes):
+    if len(extra_routes) <= 0:
+        return
+    routes_list = []
+    for extra_route in extra_routes:
+        route_dict = {'destination': extra_route.destination,
+                      'nexthop': extra_route.nexthop}
+        routes_list.append(route_dict)
+    json_body = {'router': {
+        "routes": routes_list
+    }}
+
+    try:
+        neutron_client.update_router(router_id, body=json_body)
+        return True
+    except Exception as e:
+        logger.error("Error in updating router with extra route: %s" % e)
+        raise
+
+
+def update_router_no_extra_route(neutron_client, router_ids):
+    json_body = {'router': {
+        "routes": [
+        ]}}
+
+    for router_id in router_ids:
+        try:
+            neutron_client.update_router(router_id, body=json_body)
+            return True
+        except Exception as e:
+            logger.error("Error in clearing extra route: %s" % e)
+
+
 def get_ovs_groups(compute_node_list, ovs_br_list, of_protocol="OpenFlow13"):
     """
     Gets, as input, a list of compute nodes and a list of OVS bridges
diff --git a/sdnvpn/test/functest/config.yaml b/sdnvpn/test/functest/config.yaml
index c50c619..a5f4782 100644
--- a/sdnvpn/test/functest/config.yaml
+++ b/sdnvpn/test/functest/config.yaml
@@ -189,3 +189,32 @@ testcases:
       subnet_1_cidr: 10.10.10.0/24
       secgroup_name: sdnvpn-sg
       secgroup_descr: Security group for SDNVPN test cases
+
+  sdnvpn.test.functest.testcase_13:
+      enabled: true
+      description: Testing extra route ECMP for intra-data center scenario
+      instance_1_name: sdnvpn-13-1
+      instance_2_name: sdnvpn-13-2
+      instance_3_name: sdnvpn-13-3
+      image_name: sdnvpn-image
+      net_1_name: sdnvpn-13-1-net
+      subnet_1_name: sdnvpn-13-1-subnet
+      subnet_1_cidr: 10.10.10.0/24
+      router_1_name: sdnvpn-13-1-router
+      net_2_name: sdnvpn-13-2-net
+      subnet_2_name: sdnvpn-13-2-subnet
+      subnet_2_cidr: 10.10.11.0/24
+      router_2_name: sdnvpn-13-2-router
+      interface_name: lo
+      interface_number: 1
+      extra_route_cidr: 179.24.1.12/32
+      extra_route_ip: 179.24.1.12
+      extra_route_subnet_mask: 255.255.255.255
+      extra_route_name: sdnvpn_extra_route_13
+      secgroup_name: sdnvpn-sg
+      secgroup_descr: Security group for SDNVPN test cases
+      targets1: '88:88'
+      targets2: '88:88'
+      route_distinguishers:
+          - '12:12'
+          - '13:13'
diff --git a/sdnvpn/test/functest/testcase_13.py b/sdnvpn/test/functest/testcase_13.py
new file mode 100644
index 0000000..ec0459d
--- /dev/null
+++ b/sdnvpn/test/functest/testcase_13.py
@@ -0,0 +1,207 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2017 All rights reserved
+# This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+import logging
+import sys
+
+from functest.utils import openstack_utils as os_utils
+from random import randint
+from sdnvpn.lib import config as sdnvpn_config
+from sdnvpn.lib import utils as test_utils
+from sdnvpn.lib.results import Results
+
+logger = logging.getLogger(__name__)
+
+COMMON_CONFIG = sdnvpn_config.CommonConfig()
+TESTCASE_CONFIG = sdnvpn_config.TestcaseConfig(
+    'sdnvpn.test.functest.testcase_13')
+
+
+def main():
+    results = Results(COMMON_CONFIG.line_length)
+
+    results.add_to_summary(0, "=")
+    results.add_to_summary(2, "STATUS", "SUBTEST")
+    results.add_to_summary(0, "=")
+
+    nova_client = os_utils.get_nova_client()
+    neutron_client = os_utils.get_neutron_client()
+    glance_client = os_utils.get_glance_client()
+
+    (floatingip_ids, instance_ids, router_ids, network_ids, image_ids,
+     subnet_ids, interfaces, bgpvpn_ids, flavor_ids) = ([] for i in range(9))
+
+    try:
+        image_id = os_utils.create_glance_image(
+            glance_client,
+            COMMON_CONFIG.ubuntu_image_name,
+            COMMON_CONFIG.ubuntu_image_path,
+            disk="qcow2",
+            container="bare",
+            public="public")
+        image_ids.append(image_id)
+
+        _, flavor_id = test_utils.create_custom_flavor()
+        flavor_ids.append(flavor_id)
+
+        network_1_id, subnet_1_id, router_1_id = test_utils.create_network(
+            neutron_client,
+            TESTCASE_CONFIG.net_1_name,
+            TESTCASE_CONFIG.subnet_1_name,
+            TESTCASE_CONFIG.subnet_1_cidr,
+            TESTCASE_CONFIG.router_1_name)
+
+        interfaces.append(tuple((router_1_id, subnet_1_id)))
+        network_ids.extend([network_1_id])
+        subnet_ids.extend([subnet_1_id])
+        router_ids.extend([router_1_id])
+
+        sg_id = os_utils.create_security_group_full(
+            neutron_client, TESTCASE_CONFIG.secgroup_name,
+            TESTCASE_CONFIG.secgroup_descr)
+
+        compute_nodes = test_utils.assert_and_get_compute_nodes(nova_client)
+
+        av_zone_1 = "nova:" + compute_nodes[0]
+        av_zone_2 = "nova:" + compute_nodes[1]
+
+        u1 = test_utils.generate_userdata_interface_create(
+            TESTCASE_CONFIG.interface_name,
+            TESTCASE_CONFIG.interface_number,
+            TESTCASE_CONFIG.extra_route_ip,
+            TESTCASE_CONFIG.extra_route_subnet_mask)
+        # boot INTANCES
+        vm_1 = test_utils.create_instance(
+            nova_client,
+            TESTCASE_CONFIG.instance_1_name,
+            image_id,
+            network_1_id,
+            sg_id,
+            flavor=COMMON_CONFIG.custom_flavor_name,
+            secgroup_name=TESTCASE_CONFIG.secgroup_name,
+            compute_node=av_zone_1,
+            userdata=u1)
+        vm_1_ip = test_utils.get_instance_ip(vm_1)
+
+        vm1_port = test_utils.get_port(neutron_client, vm_1.id)
+        test_utils.update_port_allowed_address_pairs(
+            neutron_client,
+            vm1_port['id'],
+            [test_utils.AllowedAddressPair(
+                TESTCASE_CONFIG.extra_route_cidr,
+                vm1_port['mac_address'])])
+
+        vm_2 = test_utils.create_instance(
+            nova_client,
+            TESTCASE_CONFIG.instance_2_name,
+            image_id,
+            network_1_id,
+            sg_id,
+            flavor=COMMON_CONFIG.custom_flavor_name,
+            secgroup_name=TESTCASE_CONFIG.secgroup_name,
+            compute_node=av_zone_1,
+            userdata=u1)
+        vm_2_ip = test_utils.get_instance_ip(vm_2)
+
+        vm2_port = test_utils.get_port(neutron_client, vm_2.id)
+        test_utils.update_port_allowed_address_pairs(
+            neutron_client,
+            vm2_port['id'],
+            [test_utils.AllowedAddressPair(
+                TESTCASE_CONFIG.extra_route_cidr,
+                vm2_port['mac_address'])])
+
+        test_utils.async_Wait_for_instances([vm_1, vm_2])
+
+        msg = ("Create VPN with multiple RDs")
+        results.record_action(msg)
+        vpn_name = "sdnvpn-" + str(randint(100000, 999999))
+        kwargs = {
+            "import_targets": TESTCASE_CONFIG.targets1,
+            "export_targets": TESTCASE_CONFIG.targets2,
+            "route_distinguishers": TESTCASE_CONFIG.route_distinguishers,
+            "name": vpn_name
+        }
+        bgpvpn = test_utils.create_bgpvpn(neutron_client, **kwargs)
+        bgpvpn_id = bgpvpn['bgpvpn']['id']
+        logger.debug("VPN created details: %s" % bgpvpn)
+        bgpvpn_ids.append(bgpvpn_id)
+
+        msg = ("Associate router '%s' to the VPN." %
+               TESTCASE_CONFIG.router_1_name)
+        results.record_action(msg)
+        results.add_to_summary(0, "-")
+
+        test_utils.create_router_association(
+            neutron_client, bgpvpn_id, router_1_id)
+
+        test_utils.update_router_extra_route(
+            neutron_client, router_1_id,
+            [test_utils.ExtraRoute(TESTCASE_CONFIG.extra_route_cidr,
+                                   vm_1_ip),
+             test_utils.ExtraRoute(TESTCASE_CONFIG.extra_route_cidr,
+                                   vm_2_ip)])
+
+        image_2_id = os_utils.create_glance_image(
+            glance_client, TESTCASE_CONFIG.image_name,
+            COMMON_CONFIG.image_path, disk=COMMON_CONFIG.image_format,
+            container="bare", public='public')
+        image_ids.append(image_2_id)
+
+        logger.info("Waiting for the VMs to connect to each other using the"
+                    " updated network configuration")
+        test_utils.wait_before_subtest()
+
+        u3 = test_utils.generate_ping_userdata(
+            [TESTCASE_CONFIG.extra_route_ip])
+        vm_3 = test_utils.create_instance(
+            nova_client,
+            TESTCASE_CONFIG.instance_3_name,
+            image_2_id,
+            network_1_id,
+            sg_id,
+            flavor=COMMON_CONFIG.custom_flavor_name,
+            secgroup_name=TESTCASE_CONFIG.secgroup_name,
+            compute_node=av_zone_2,
+            userdata=u3)
+
+        instance_ids.extend([vm_1.id, vm_2.id, vm_3.id])
+
+        instance_dhcp_up = test_utils.wait_for_instances_get_dhcp(vm_3)
+
+        if (not instance_dhcp_up):
+            logger.error("vm_3 instance is down")
+
+        results.get_ping_status_target_ip(vm_3,
+                                          TESTCASE_CONFIG.extra_route_name,
+                                          TESTCASE_CONFIG.extra_route_ip,
+                                          expected="PASS",
+                                          timeout=300)
+
+        results.add_to_summary(0, "=")
+        logger.info("\n%s" % results.summary)
+
+    except Exception as e:
+        logger.error("exception occurred while executing testcase_13: %s", e)
+        raise
+    finally:
+        test_utils.update_router_no_extra_route(neutron_client, router_ids)
+        test_utils.cleanup_nova(nova_client, instance_ids)
+        test_utils.cleanup_glance(glance_client, image_ids)
+        test_utils.cleanup_neutron(neutron_client, floatingip_ids,
+                                   bgpvpn_ids, interfaces, subnet_ids,
+                                   router_ids, network_ids)
+
+    return results.compile_summary()
+
+
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
+    sys.exit(main())