Fix most end-to-end issues with testcase 3 and reenable it

- Make the test work end-to-end on Fuel
- Use the instance log to verify quagga started
- Don't attempt to check for peering, as that is for
a future release
- Use the external IP of the controller for peering
- Fix not getting the output of OpenDaylight commands

Change-Id: Ia7bb533ab65e0fca6b7f48bb64133f6d8f3ff8ae
JIRA: SDNVPN-114
Signed-off-by: Romanos Skiadas <rski@intracom-telecom.com>

5 files changed, 122 insertions, 118 deletions

diff --git a/sdnvpn/artifacts/quagga_setup.sh b/sdnvpn/artifacts/quagga_setup.sh
index 8a90198..1785f14 100644
--- a/sdnvpn/artifacts/quagga_setup.sh
+++ b/sdnvpn/artifacts/quagga_setup.sh
@@ -68,6 +68,6 @@ log file ${BGPD_LOG_FILE}
 end
 CATEOF
 chown quagga:quagga $BGPD_CONFIG_LOCATION
-
+service quagga restart
 pgrep bgpd
 pgrep zebra
diff --git a/sdnvpn/lib/quagga.py b/sdnvpn/lib/quagga.py
index 9f8a4cd..dc80657 100644
--- a/sdnvpn/lib/quagga.py
+++ b/sdnvpn/lib/quagga.py
@@ -13,10 +13,14 @@ logger = ft_logger.Logger("sdnvpn-quagga").getLogger()
 COMMON_CONFIG = config.CommonConfig()
 
 
-def odl_add_neighbor(neighbor_ip, controller):
+def odl_add_neighbor(neighbor_ip, controller_ip, controller):
+    # Explicitly pass controller_ip because controller.ip
+    # Might not be accessible from the Quagga instance
     command = 'configure-bgp -op add-neighbor --as-num 200'
-    command += ' --ip %s --use-source-ip %s' % (neighbor_ip, controller.ip)
+    command += ' --ip %s --use-source-ip %s' % (neighbor_ip, controller_ip)
     success = run_odl_cmd(controller, command)
+    # The run_cmd api is really whimsical
+    logger.info("Maybe stdout of %s: %s", command, success)
     return success
 
 
@@ -41,7 +45,7 @@ def gen_quagga_setup_script(controller_ip,
 
 
 def check_for_peering(controller):
-    cmd = 'show-bgp --cmd "ip bgp neighbors"'
+    cmd = 'show-bgp --cmd \\"ip bgp neighbors\\"'
     tries = 20
     neighbors = None
     bgp_state_regex = re.compile("(BGP state =.*)")
diff --git a/sdnvpn/lib/utils.py b/sdnvpn/lib/utils.py
index 8c4aebf..d085111 100644
--- a/sdnvpn/lib/utils.py
+++ b/sdnvpn/lib/utils.py
@@ -386,12 +386,42 @@ def check_odl_fib(ip, controller_ip):
 
 
 def run_odl_cmd(odl_node, cmd):
-    '''
-    Run a command in the OpenDaylight Karaf shell
+    '''Run a command in the OpenDaylight Karaf shell
 
     This is a bit flimsy because of shell quote escaping, make sure that
     the cmd passed does not have any top level double quotes or this
     function will break.
+
+    The /dev/null is used because client works, but outputs something
+    that contains "ERROR" and run_cmd doesn't like that.
+
     '''
-    karaf_cmd = '/opt/opendaylight/bin/client "%s" ' % cmd
+    karaf_cmd = '/opt/opendaylight/bin/client "%s" 2>/dev/null' % cmd
     return odl_node.run_cmd(karaf_cmd)
+
+
+def wait_for_cloud_init(instance):
+    success = True
+    # ubuntu images take a long time to start
+    tries = 20
+    sleep_time = 30
+    while tries > 0:
+        instance_log = instance.get_console_output()
+        if "Failed to run module" in instance_log:
+            success = False
+            logger.error("Cloud init failed to run. Reason: %s",
+                         instance_log)
+            break
+        if re.search(r"Cloud-init v. .+ finished at" in instance_log):
+            success = True
+            break
+        time.sleep(sleep_time)
+        tries = tries - 1
+
+    if tries == 0:
+        logger.error("Cloud init timed out"
+                     ". Reason: %s",
+                     instance_log)
+        success = False
+
+    return success
diff --git a/sdnvpn/test/functest/config.yaml b/sdnvpn/test/functest/config.yaml
index dd46dde..4359f2f 100644
--- a/sdnvpn/test/functest/config.yaml
+++ b/sdnvpn/test/functest/config.yaml
@@ -67,7 +67,7 @@ testcases:
       route_distinguishers2: '222:222'
 
   testcase_3:
-      enabled: false
+      enabled: true
       description: Data center gateway integration
       testname_db: functest_testcase_3
       secgroup_name: sdnvpn-sg
diff --git a/sdnvpn/test/functest/testcase_3.py b/sdnvpn/test/functest/testcase_3.py
index 0253444..8a8aa30 100644
--- a/sdnvpn/test/functest/testcase_3.py
+++ b/sdnvpn/test/functest/testcase_3.py
@@ -11,11 +11,7 @@
 #   - Set up a Quagga instance in the functest container
 #   - Start a BGP router with OpenDaylight
 #   - Add the functest Quagga as a neighbor
-#   - Verify that the OpenDaylight and functest Quaggas peer
-# - Exchange routing information with Quagga:
-#   - Create a network, instance and BGPVPN in OpenStack
-#   - Verify the route to the instance is present in the OpenDaylight FIB
-#   - Verify that the functest Quagga also learns these routes
+#   - Verify that the OpenDaylight and gateway Quagga peer
 import os
 import argparse
 
@@ -67,69 +63,71 @@ def main():
         logger.info(msg)
         results.add_success(msg)
 
-    for controller in controllers:
-        logger.info("Starting bgp speaker of controller at IP %s "
-                    % controller.ip)
-        logger.info("Checking if zrpcd is "
-                    "running on the controller node")
-
-        cmd = "systemctl status zrpcd"
-        output = controller.run_cmd(cmd)
-        msg = ("zrpcd is running")
-
-        if not output:
-            logger.info("zrpcd is not running on the controller node")
-            results.add_failure(msg)
-        else:
-            logger.info("zrpcd is running on the controller node")
-            results.add_success(msg)
-
-        results.add_to_summary(0, "-")
-
-        start_quagga = "odl:configure-bgp -op start-bgp-server " \
-                       "--as-num 100 --router-id {0}".format(controller.ip)
-        test_utils.run_odl_cmd(controller, start_quagga)
-
-        logger.info("Checking if bgpd is running"
-                    " on the controller node")
-
-        # Check if there is a non-zombie bgpd process
-        output_bgpd = controller.run_cmd("ps --no-headers -C "
-                                         "bgpd -o state")
-        states = output_bgpd.split()
-        running = any([s != 'Z' for s in states])
-
-        msg = ("bgpd is running")
-        if not running:
-            logger.info("bgpd is not running on the controller node")
-            results.add_failure(msg)
-        else:
-            logger.info("bgpd is running on the controller node")
-            results.add_success(msg)
-
-        results.add_to_summary(0, "-")
-
-        stop_quagga = 'odl:configure-bgp -op stop-bgp-server'
-
-        test_utils.run_odl_cmd(controller, stop_quagga)
-
-        # disabled because of buggy upstream
-        # https://github.com/6WIND/zrpcd/issues/15
-        # logger.info("Checking if bgpd is still running"
-        #             " on the controller node")
-
-        # output_bgpd = controller.run_cmd("ps --no-headers -C " \
-        #                                  "bgpd -o state")
-        # states = output_bgpd.split()
-        # running = any([s != 'Z' for s in states])
-
-        # msg = ("bgpd is stopped")
-        # if not running:
-        #     logger.info("bgpd is not running on the controller node")
-        #     results.add_success(msg)
-        # else:
-        #     logger.info("bgpd is still running on the controller node")
-        #     results.add_failure(msg)
+    controller = controllers[0]  # We don't handle HA well
+    get_ext_ip_cmd = "ip a | grep br-ex | grep inet | awk '{print $2}'"
+    controller_ext_ip = controller.run_cmd(get_ext_ip_cmd).split("/")[0]
+    logger.info("Starting bgp speaker of controller at IP %s "
+                % controller_ext_ip)
+    logger.info("Checking if zrpcd is "
+                "running on the controller node")
+
+    cmd = "systemctl status zrpcd"
+    output = controller.run_cmd(cmd)
+    msg = ("zrpcd is running")
+
+    if not output:
+        logger.info("zrpcd is not running on the controller node")
+        results.add_failure(msg)
+    else:
+        logger.info("zrpcd is running on the controller node")
+        results.add_success(msg)
+
+    results.add_to_summary(0, "-")
+
+    start_quagga = "odl:configure-bgp -op start-bgp-server " \
+                   "--as-num 100 --router-id {0}".format(controller_ext_ip)
+    test_utils.run_odl_cmd(controller, start_quagga)
+
+    logger.info("Checking if bgpd is running"
+                " on the controller node")
+
+    # Check if there is a non-zombie bgpd process
+    output_bgpd = controller.run_cmd("ps --no-headers -C "
+                                     "bgpd -o state")
+    states = output_bgpd.split()
+    running = any([s != 'Z' for s in states])
+
+    msg = ("bgpd is running")
+    if not running:
+        logger.info("bgpd is not running on the controller node")
+        results.add_failure(msg)
+    else:
+        logger.info("bgpd is running on the controller node")
+        results.add_success(msg)
+
+    results.add_to_summary(0, "-")
+
+    # We should be able to restart the speaker
+    # but the test is disabled because of buggy upstream
+    # https://github.com/6WIND/zrpcd/issues/15
+    # stop_quagga = 'odl:configure-bgp -op stop-bgp-server'
+    # test_utils.run_odl_cmd(controller, stop_quagga)
+
+    # logger.info("Checking if bgpd is still running"
+    #             " on the controller node")
+
+    # output_bgpd = controller.run_cmd("ps --no-headers -C " \
+    #                                  "bgpd -o state")
+    # states = output_bgpd.split()
+    # running = any([s != 'Z' for s in states])
+
+    # msg = ("bgpd is stopped")
+    # if not running:
+    #     logger.info("bgpd is not running on the controller node")
+    #     results.add_success(msg)
+    # else:
+    #     logger.info("bgpd is still running on the controller node")
+    #     results.add_failure(msg)
 
     # Taken from the sfc tests
     if not os.path.isfile(COMMON_CONFIG.ubuntu_image_path):
@@ -195,11 +193,10 @@ def main():
             break
     # Get the mask of ext net of the compute where quagga is running
     # TODO check this works on apex
-    cmd = "ip a | grep br-ex | grep inet | awk '{print $2}'"
-    ext_cidr = compute.run_cmd(cmd).split("/")
+    ext_cidr = compute.run_cmd(get_ext_ip_cmd).split("/")
     ext_net_mask = ext_cidr[1]
     quagga_bootstrap_script = quagga.gen_quagga_setup_script(
-        controllers[0].ip,
+        controller_ext_ip,
         fake_fip['fip_addr'],
         ext_net_mask)
     quagga_vm = test_utils.create_instance(
@@ -223,14 +220,6 @@ def main():
     else:
         results.add_failure(msg)
 
-    testcase = "Bootstrap quagga inside an OpenStack instance"
-    success = False
-    if success:
-        results.add_success(testcase)
-    else:
-        results.add_failure(testcase)
-    results.add_to_summary(0, "=")
-
     # This part works around NAT
     # What we do is attach the instance directly to the OpenStack
     # external network. This way is is directly accessible from the
@@ -240,43 +229,24 @@ def main():
     compute.run_cmd("virsh attach-interface %s"
                     " bridge br-ex" % libvirt_instance_name)
 
+    testcase = "Bootstrap quagga inside an OpenStack instance"
+    cloud_init_success = test_utils.wait_for_cloud_init(quagga_vm)
+    if cloud_init_success:
+        results.add_success(testcase)
+    else:
+        results.add_failure(testcase)
+    results.add_to_summary(0, "=")
+
     results.add_to_summary(0, '-')
     results.add_to_summary(1, "Peer Quagga with OpenDaylight")
     results.add_to_summary(0, '-')
 
-    neighbor = quagga.odl_add_neighbor(fip['fip_addr'], controller)
+    neighbor = quagga.odl_add_neighbor(fake_fip['fip_addr'],
+                                       controller_ext_ip,
+                                       controller)
     peer = quagga.check_for_peering(controller)
 
-    image_id = os_utils.create_glance_image(glance_client,
-                                            TESTCASE_CONFIG.image_name,
-                                            COMMON_CONFIG.image_path,
-                                            disk=COMMON_CONFIG.image_format,
-                                            container="bare",
-                                            public="public")
-
-    instance = test_utils.create_instance(
-        nova_client,
-        TESTCASE_CONFIG.instance_1_name,
-        image_id,
-        net_id,
-        sg_id,
-        fixed_ip=TESTCASE_CONFIG.instance_1_ip,
-        secgroup_name=TESTCASE_CONFIG.secgroup_name)
-
-    kwargs = {"import_targets": TESTCASE_CONFIG.import_targets,
-              "export_targets": TESTCASE_CONFIG.export_targets,
-              "route_targets": TESTCASE_CONFIG.export_targets,
-              "name": "bgpvpn-3-1"}
-
-    bgpvpn = os_utils.create_bgpvpn(neutron_client, **kwargs)
-    bgpvpn_id = bgpvpn['bgpvpn']['id']
-    os_utils.create_network_association(
-        neutron_client, bgpvpn_id, net_id)
-
-    test_utils.wait_for_instance(instance)
-
-    exchange = quagga.check_for_route_exchange(fip['fip_addr'])
-    if neighbor and peer and exchange:
+    if neighbor and peer:
         results.add_success("Peering with quagga")
     else:
         results.add_failure("Peering with quagga")