Add support to push the results to the DB

JIRA: SDNVPN-19

The test scripts now return a json object with
the following format:

{"status": criteria, "details": results}

where
criteria: "PASS"|"FAILED" (result of the test case)
details: free output that will be printed to the console
and pushed to the DB

Change-Id: I9e5e0c3b6cb5f4b060929b71a06f6e4f95f814fb
Signed-off-by: jose.lausuch <jose.lausuch@ericsson.com>

5 files changed, 149 insertions, 53 deletions

diff --git a/test/functest/config.yaml b/test/functest/config.yaml
index 83564c4..521b517 100644
--- a/test/functest/config.yaml
+++ b/test/functest/config.yaml
@@ -2,10 +2,15 @@ testcases:
   tempest:
       enabled: true
       description: Neutron BGPVPN tests in tempest
+      succes_criteria: 100 # all the subtests must pass
+      testname_db: functest_tempest # declared name in the test api
+      # http://testresults.opnfv.org/test/api/v1/projects/sdnvpn/cases
 
   testcase_1:
       enabled: true
       description: VPN provides connectivity between subnets
+      succes_criteria: 75 # we let fail 25% of the subtests
+      testname_db: functest_testcase_1
       ping_timeout:   200
       flavor: m1.tiny # adapt to your environment
       instance_1_name: sdnvpn-1-1
@@ -24,10 +29,14 @@ testcases:
       router_2_name: sdnvpn-1-2-router
       sdnvpn_sg_name: sdnvpn-sg
       sdnvpn_sg_descr: Security group for SDNVPN test cases
+      targets1: '88:88'
+      targets2: '55:55'
 
   testcase_2:
       enabled: true
       description: Tenant separation
+      succes_criteria: 100 # all the subtests must pass
+      testname_db: functest_testcase_2
       ping_timeout:   200
       flavor: m1.tiny # adapt to your environment
       instance_1_name: sdnvpn-2-1
@@ -50,8 +59,12 @@ testcases:
       router_2_name: sdnvpn-2-2-router
       sdnvpn_sg_name: sdnvpn-sg
       sdnvpn_sg_descr: Security group for SDNVPN test cases
+      targets1: '88:88'
+      targets2: '55:55'
 
   testcase_3:
       enabled: false
       description: Data center gateway integration
+      succes_criteria: 100 # all the subtests must pass
+      testname_db: functest_testcase_3
 
diff --git a/test/functest/run_tests.py b/test/functest/run_tests.py
index 0176161..91f0f0a 100644
--- a/test/functest/run_tests.py
+++ b/test/functest/run_tests.py
@@ -7,27 +7,69 @@
 #
 # http://www.apache.org/licenses/LICENSE-2.0
 #
-
+import argparse
 import importlib
+import os
+import time
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils
 import yaml
 
+""" tests configuration """
+parser = argparse.ArgumentParser()
+parser.add_argument("-r", "--report",
+                    help="Create json result file",
+                    action="store_true")
+args = parser.parse_args()
+
+TEST_DB_URL = ft_utils.get_parameter_from_yaml('results.test_db_url')
+logger = ft_logger.Logger("sdnvpn-run-tests").getLogger()
+REPO_PATH = os.environ['repos_dir'] + '/sdnvpn/'
+
+
+def push_results(testname, start_time, end_time, criteria, details):
+    logger.info("Push testcase '%s' results into the DB...\n" % testname)
+    ft_utils.push_results_to_db("sdnvpn",
+                                testname,
+                                logger,
+                                start_time,
+                                end_time,
+                                criteria,
+                                details)
+
+
+def main():
+
+    with open(REPO_PATH + 'test/functest/config.yaml') as f:
+        config_yaml = yaml.safe_load(f)
+
+    testcases = config_yaml.get("testcases")
+    for testcase in testcases:
+        if testcases[testcase]['enabled']:
+            test_name = testcase
+            test_descr = testcases[testcase]['description']
+            test_name_db = testcases[testcase]['testname_db']
+            title = ("Running '%s - %s'" %
+                     (test_name, test_descr))
+            logger.info(title)
+            logger.info("%s\n" % ("=" * len(title)))
+            t = importlib.import_module(testcase, package=None)
+            start_time = time.time()
+            result = t.main()
+            if result < 0:
+                result = {"status": "FAILED", "details": "execution error."}
+            logger.info("Results of test case '%s - %s':\n%s\n" %
+                        (test_name, test_descr, result))
+            end_time = time.time()
+            # duration = end_time - start_time
+            criteria = result.get("status")
+            details = result.get("details")
+            if args.report:
+                push_results(
+                    test_name_db, start_time, end_time, criteria, details)
 
-logger = ft_logger.Logger("sdnvpn").getLogger()
+    exit(0)
 
-with open('config.yaml') as f:
-    config_yaml = yaml.safe_load(f)
 
-testcases = config_yaml.get("testcases")
-for testcase in testcases:
-    title = ("Running '%s - %s'" %
-             (testcase, testcases[testcase]['description']))
-    print(title)
-    print("%s\n" % ("=" * len(title)))
-    if testcases[testcase]['type'] == 'python':
-        t = importlib.import_module(testcase, package=None)
-        t.main()
-    else:
-        cmd = "bash " + testcase + ".sh"
-        result = ft_utils.execute_command(cmd, logger, exit_on_error=False)
+if __name__ == '__main__':
+    main()
diff --git a/test/functest/tempest.py b/test/functest/tempest.py
index 8d4b664..832cb4a 100644
--- a/test/functest/tempest.py
+++ b/test/functest/tempest.py
@@ -15,7 +15,14 @@ import shutil
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils
 
-logger = ft_logger.Logger("bgpvpn").getLogger()
+
+logger = ft_logger.Logger("sdnvpn-tempest").getLogger()
+
+REPO_PATH = os.environ['repos_dir'] + '/sdnvpn/'
+config_file = REPO_PATH + 'test/functest/config.yaml'
+
+SUCCESS_CRITERIA = ft_utils.get_parameter_from_yaml(
+    "testcases.testcase_1.succes_criteria", config_file)
 
 
 def main():
@@ -68,11 +75,13 @@ def main():
         # Look for name of the tests
         testcases = re.findall("\{0\} (.*)", output)
 
-        results = {"test_name": "tempest", "duration": duration,
+        results = {"duration": duration,
                    "num_tests": num_tests, "failed": failed,
                    "tests": testcases}
-        logger.info("Results: %s" % results)
-        return results
+        status = "PASS"
+        if 100 - (100 * int(failed) / int(num_tests)) < int(SUCCESS_CRITERIA):
+            status = "FAILED"
+        return {"status": status, "details": results}
     except:
         logger.error("Problem when parsing the results.")
 
diff --git a/test/functest/testcase_1.py b/test/functest/testcase_1.py
index 0d7eb48..250db20 100644
--- a/test/functest/testcase_1.py
+++ b/test/functest/testcase_1.py
@@ -27,11 +27,9 @@ parser.add_argument("-r", "--report",
 
 args = parser.parse_args()
 
-""" logging configuration """
 logger = ft_logger.Logger("sdnvpn-testcase-1").getLogger()
 
 REPO_PATH = os.environ['repos_dir'] + '/sdnvpn/'
-HOME = os.environ['HOME'] + "/"
 
 VM_BOOT_TIMEOUT = 180
 
@@ -80,12 +78,20 @@ SECGROUP_NAME = ft_utils.get_parameter_from_yaml(
     "testcases.testcase_1.sdnvpn_sg_name", config_file)
 SECGROUP_DESCR = ft_utils.get_parameter_from_yaml(
     "testcases.testcase_1.sdnvpn_sg_descr", config_file)
-
+TARGETS_1 = ft_utils.get_parameter_from_yaml(
+    "testcases.testcase_1.targets1", config_file)
+TARGETS_2 = ft_utils.get_parameter_from_yaml(
+    "testcases.testcase_1.targets2", config_file)
+SUCCESS_CRITERIA = ft_utils.get_parameter_from_yaml(
+    "testcases.testcase_1.succes_criteria", config_file)
 TEST_DB = ft_utils.get_parameter_from_yaml("results.test_db_url")
 
 TEST_RESULT = "PASS"
 SUMMARY = ""
 LINE_LENGTH = 60  # length for the summary table
+DETAILS = []
+NUM_TESTS = 0
+NUM_TESTS_FAILED = 0
 
 
 def create_network(neutron_client, net, subnet, router, cidr):
@@ -177,7 +183,7 @@ def get_ping_status(vm_source, ip_source,
         expected_result = 'can ping' if expected == 'PASS' else 'cannot ping'
         test_case_name = ("'%s' %s '%s'" %
                           (vm_source.name, expected_result, vm_target.name))
-        logger.debug("\n%sPing\n%sfrom '%s' (%s)\n%sto '%s' (%s).\n"
+        logger.debug("%sPing\n%sfrom '%s' (%s)\n%sto '%s' (%s).\n"
                      "%s-->Expected result: %s.\n"
                      % (tab, tab, vm_source.name, ip_source,
                         tab, vm_target.name, ip_target,
@@ -221,7 +227,7 @@ def get_ping_status(vm_source, ip_source,
 
 
 def add_to_summary(num_cols, col1, col2=""):
-    global SUMMARY, LINE_LENGTH
+    global SUMMARY, LINE_LENGTH, DETAILS, NUM_TESTS, NUM_TESTS_FAILED
     if num_cols == 0:
         SUMMARY += ("+%s+\n" % (col1 * (LINE_LENGTH - 2)))
     elif num_cols == 1:
@@ -229,6 +235,11 @@ def add_to_summary(num_cols, col1, col2=""):
     elif num_cols == 2:
         SUMMARY += ("| %s" % col1.ljust(7) + "| ")
         SUMMARY += (col2.ljust(LINE_LENGTH - 12) + "|\n")
+        DETAILS.append({col2: col1})
+        if col1 in ("FAIL", "PASS"):
+            NUM_TESTS += 1
+            if col1 == "FAIL":
+                NUM_TESTS_FAILED += 1
 
 
 def main():
@@ -337,18 +348,18 @@ def main():
     logger.debug("Instance '%s' booted successfully. IP='%s'." %
                  (INSTANCE_1_NAME, vm_1_ip))
     msg = ("Create VPN with eRT<>iRT")
-    logger.info("\n\n--> %s ..." % msg)
+    logger.info(msg)
     add_to_summary(1, msg)
     vpn_name = "sdnvpn-" + str(randint(100000, 999999))
-    kwargs = {"import_targets": "88:88",
-              "export_targets": "55:55",
+    kwargs = {"import_targets": TARGETS_1,
+              "export_targets": TARGETS_2,
               "name": vpn_name}
     bgpvpn = os_utils.create_bgpvpn(neutron_client, **kwargs)
     bgpvpn_id = bgpvpn['bgpvpn']['id']
     logger.debug("VPN created details: %s" % bgpvpn)
 
     msg = ("Associate network '%s' to the VPN." % NET_1_NAME)
-    logger.info("\n\n--> %s..." % msg)
+    logger.info(msg)
     add_to_summary(1, msg)
     add_to_summary(0, "-")
 
@@ -366,7 +377,7 @@ def main():
     get_ping_status(vm_1, vm_1_ip, vm_4, vm_4_ip, expected="FAIL", timeout=30)
 
     msg = ("Associate network '%s' to the VPN." % NET_2_NAME)
-    logger.info("\n\n--> %s..." % msg)
+    logger.info(msg)
     add_to_summary(0, "-")
     add_to_summary(1, msg)
     add_to_summary(0, "-")
@@ -374,7 +385,7 @@ def main():
         neutron_client, bgpvpn_id, network_2_id)
 
     # Wait a bit for this to take effect
-    time.sleep(10)
+    time.sleep(30)
 
     # Ping from VM4 to VM5 should work
     get_ping_status(vm_4, vm_4_ip, vm_5, vm_5_ip, expected="PASS", timeout=30)
@@ -384,16 +395,16 @@ def main():
     get_ping_status(vm_1, vm_1_ip, vm_5, vm_5_ip, expected="FAIL", timeout=30)
 
     msg = ("Update VPN with eRT=iRT ...")
-    logger.info("\n\n--> %s..." % msg)
+    logger.info(msg)
     add_to_summary(0, "-")
     add_to_summary(1, msg)
     add_to_summary(0, "-")
-    kwargs = {"import_targets": "88:88",
-              "export_targets": "88:88",
+    kwargs = {"import_targets": TARGETS_1,
+              "export_targets": TARGETS_1,
               "name": vpn_name}
     bgpvpn = os_utils.update_bgpvpn(neutron_client, bgpvpn_id, **kwargs)
     # Wait a bit for this to take effect
-    time.sleep(10)
+    time.sleep(30)
 
     # Ping from VM1 to VM4 should work
     get_ping_status(vm_1, vm_1_ip, vm_4, vm_4_ip, expected="PASS", timeout=30)
@@ -408,7 +419,12 @@ def main():
     else:
         logger.info("One or more ping tests have failed.")
 
-    sys.exit(0)
+    status = "PASS"
+    success = 100 - (100 * int(NUM_TESTS_FAILED) / int(NUM_TESTS))
+    if success < int(SUCCESS_CRITERIA):
+        status = "FAILED"
+
+    return {"status": status, "details": DETAILS}
 
 
 if __name__ == '__main__':
diff --git a/test/functest/testcase_2.py b/test/functest/testcase_2.py
index 76757d6..c28e535 100644
--- a/test/functest/testcase_2.py
+++ b/test/functest/testcase_2.py
@@ -27,11 +27,9 @@ parser.add_argument("-r", "--report",
 
 args = parser.parse_args()
 
-""" logging configuration """
-logger = ft_logger.Logger("sdnvpn-testcase-1").getLogger()
+logger = ft_logger.Logger("sdnvpn-testcase-2").getLogger()
 
 REPO_PATH = os.environ['repos_dir'] + '/sdnvpn/'
-HOME = os.environ['HOME'] + "/"
 
 VM_BOOT_TIMEOUT = 180
 
@@ -97,12 +95,20 @@ SECGROUP_NAME = ft_utils.get_parameter_from_yaml(
     "testcases.testcase_2.sdnvpn_sg_name", config_file)
 SECGROUP_DESCR = ft_utils.get_parameter_from_yaml(
     "testcases.testcase_2.sdnvpn_sg_descr", config_file)
-
+TARGETS_1 = ft_utils.get_parameter_from_yaml(
+    "testcases.testcase_2.targets1", config_file)
+TARGETS_2 = ft_utils.get_parameter_from_yaml(
+    "testcases.testcase_2.targets2", config_file)
+SUCCESS_CRITERIA = ft_utils.get_parameter_from_yaml(
+    "testcases.testcase_1.succes_criteria", config_file)
 TEST_DB = ft_utils.get_parameter_from_yaml("results.test_db_url")
 
 TEST_RESULT = "PASS"
 SUMMARY = ""
 LINE_LENGTH = 90  # length for the summary table
+DETAILS = []
+NUM_TESTS = 0
+NUM_TESTS_FAILED = 0
 
 
 def create_network(neutron_client, net, subnet1, cidr1,
@@ -231,7 +237,7 @@ def check_ssh_output(vm_source, ip_source,
         test_case_name = ("[%s] returns 'I am %s' to '%s'[%s]" %
                           (ip_target, expected,
                            vm_source.name, ip_source))
-        logger.debug("\n%sSSH\n%sfrom '%s' (%s)\n%sto '%s' (%s).\n"
+        logger.debug("%sSSH\n%sfrom '%s' (%s)\n%sto '%s' (%s).\n"
                      "%s-->Expected result: %s.\n"
                      % (tab, tab, vm_source.name, ip_source,
                         tab, vm_target.name, ip_target,
@@ -262,7 +268,7 @@ def check_ssh_output(vm_source, ip_source,
 
 
 def add_to_summary(num_cols, col1, col2=""):
-    global SUMMARY, LINE_LENGTH
+    global SUMMARY, LINE_LENGTH, NUM_TESTS, NUM_TESTS_FAILED
     if num_cols == 0:
         SUMMARY += ("+%s+\n" % (col1 * (LINE_LENGTH - 2)))
     elif num_cols == 1:
@@ -270,6 +276,11 @@ def add_to_summary(num_cols, col1, col2=""):
     elif num_cols == 2:
         SUMMARY += ("| %s" % col1.ljust(7) + "| ")
         SUMMARY += (col2.ljust(LINE_LENGTH - 12) + "|\n")
+        DETAILS.append({col2: col1})
+        if col1 in ("FAIL", "PASS"):
+            NUM_TESTS += 1
+            if col1 == "FAIL":
+                NUM_TESTS_FAILED += 1
 
 
 def main():
@@ -402,19 +413,19 @@ def main():
                  (INSTANCE_1_NAME, vm_1_ip))
 
     msg = ("Create VPN1 with eRT=iRT")
-    logger.info("\n\n--> %s ..." % msg)
+    logger.info(msg)
     add_to_summary(1, msg)
     vpn1_name = "sdnvpn-1-" + str(randint(100000, 999999))
-    kwargs = {"import_targets": "55:55",
-              "export_targets": "55:55",
-              "route_targets": "55:55",
+    kwargs = {"import_targets": TARGETS_2,
+              "export_targets": TARGETS_2,
+              "route_targets": TARGETS_2,
               "name": vpn1_name}
     bgpvpn1 = os_utils.create_bgpvpn(neutron_client, **kwargs)
     bgpvpn1_id = bgpvpn1['bgpvpn']['id']
     logger.debug("VPN1 created details: %s" % bgpvpn1)
 
     msg = ("Associate network '%s' to the VPN." % NET_1_NAME)
-    logger.info("\n\n--> %s..." % msg)
+    logger.info(msg)
     add_to_summary(1, msg)
     add_to_summary(0, "-")
 
@@ -433,19 +444,19 @@ def main():
 
     add_to_summary(0, "-")
     msg = ("Create VPN2 with eRT=iRT")
-    logger.info("\n\n--> %s ..." % msg)
+    logger.info(msg)
     add_to_summary(1, msg)
     vpn2_name = "sdnvpn-2-" + str(randint(100000, 999999))
-    kwargs = {"import_targets": "88:88",
-              "export_targets": "88:88",
-              "route_targets": "88:88",
+    kwargs = {"import_targets": TARGETS_1,
+              "export_targets": TARGETS_1,
+              "route_targets": TARGETS_1,
               "name": vpn2_name}
     bgpvpn2 = os_utils.create_bgpvpn(neutron_client, **kwargs)
     bgpvpn2_id = bgpvpn2['bgpvpn']['id']
     logger.debug("VPN created details: %s" % bgpvpn2)
 
     msg = ("Associate network '%s' to the VPN2." % NET_2_NAME)
-    logger.info("\n\n--> %s..." % msg)
+    logger.info(msg)
     add_to_summary(1, msg)
     add_to_summary(0, "-")
 
@@ -471,7 +482,12 @@ def main():
     else:
         logger.info("One or more sub tests have failed.")
 
-    sys.exit(0)
+    status = "PASS"
+    success = 100 - (100 * int(NUM_TESTS_FAILED) / int(NUM_TESTS))
+    if success < int(SUCCESS_CRITERIA):
+        status = "FAILED"
+
+    return {"status": status, "details": DETAILS}
 
 
 if __name__ == '__main__':