Add criteria in testcases.yaml and Test class

JIRA: FUNCTEST-257

Added example of usage in vping_ssh

Change-Id: Iac143b811db163f0f86682b0688948c20843ede5
Signed-off-by: jose.lausuch <jose.lausuch@ericsson.com>

5 files changed, 47 insertions, 6 deletions

diff --git a/ci/testcases.yaml b/ci/testcases.yaml
index 4cbc00add..4281f73ce 100644
--- a/ci/testcases.yaml
+++ b/ci/testcases.yaml
@@ -9,6 +9,7 @@ tiers:
         testcases:
             -
                 name: healthcheck
+                criteria: 'status == "PASS"'
                 description: >-
                     This test case verifies the basic OpenStack services like
                     Keystone, Glance, Cinder, Neutron and Nova.
@@ -26,6 +27,7 @@ tiers:
         testcases:
             -
                 name: vping_ssh
+                criteria: 'status == "PASS"'
                 description: >-
                     This test case verifies: 1) SSH to an instance using floating
                     IPs over the public network. 2) Connectivity between 2 instances
@@ -36,6 +38,7 @@ tiers:
 
             -
                 name: vping_userdata
+                criteria: 'status == "PASS"'
                 description: >-
                     This test case verifies:  1) Boot a VM with given userdata.
                     2) Connectivity between 2 instances over a private network.
@@ -45,6 +48,7 @@ tiers:
 
             -
                 name: tempest_smoke_serial
+                criteria: 'success_rate == 100%'
                 description: >-
                     This test case runs the smoke subset of the OpenStack
                     Tempest suite. The list of test cases is generated by
@@ -56,6 +60,7 @@ tiers:
 
             -
                 name: rally_sanity
+                criteria: 'success_rate == 100%'
                 description: >-
                     This test case runs a sub group of tests of the OpenStack
                     Rally suite in smoke mode.
@@ -73,6 +78,7 @@ tiers:
         testcases:
             -
                 name: odl
+                criteria: 'success_rate == 100%'
                 description: >-
                     Test Suite for the OpenDaylight SDN Controller. It integrates
                     some test suites from upstream using Robot as the test
@@ -83,6 +89,7 @@ tiers:
 
             -
                 name: onos
+                criteria: 'status == "PASS"'
                 description: >-
                     Test Suite for the ONOS SDN Controller. It integrates
                     some test suites from upstream using TestON as the test
@@ -93,6 +100,7 @@ tiers:
 
             -
                 name: ovno
+                criteria: 'status == "PASS"'
                 description: >-
                     Test Suite for the Open Contrail SDN Controller.
                     framework.
@@ -110,6 +118,7 @@ tiers:
         testcases:
             -
                 name: promise
+                criteria: 'success_rate == 100%'
                 description: >-
                     Test suite from Promise project.
                 dependencies:
@@ -118,6 +127,7 @@ tiers:
 
             -
                 name: doctor
+                criteria: 'status == "PASS"'
                 description: >-
                     Test suite from Doctor project.
                 dependencies:
@@ -126,6 +136,7 @@ tiers:
 
             -
                 name: bgpvpn
+                criteria: 'status == "PASS"'
                 description: >-
                     Test suite from SDNVPN project.
                 dependencies:
@@ -134,6 +145,7 @@ tiers:
 
             -
                 name: security_scan
+                criteria: 'status == "PASS"'
                 description: >-
                     Simple security Scan
                 dependencies:
@@ -149,6 +161,7 @@ tiers:
         testcases:
             -
                 name: tempest_full_parallel
+                criteria: 'success_rate >= 80%'
                 description: >-
                     The list of test cases is generated by
                     Tempest automatically and depends on the parameters of
@@ -159,6 +172,7 @@ tiers:
 
             -
                 name: rally_full
+                criteria: 'success_rate >= 90%'
                 description: >-
                     This test case runs the full suite of scenarios of the OpenStack
                     Rally suite using several threads and iterations.
@@ -175,6 +189,7 @@ tiers:
         testcases:
             -
                 name: vims
+                criteria: 'status == "PASS"'
                 description: >-
                     This test case deploys an OpenSource vIMS solution from Clearwater
                     using the Cloudify orchestrator. It also runs some signaling traffic.
diff --git a/ci/tier_builder.py b/ci/tier_builder.py
index 4723bf4fa..b9357bc2d 100644
--- a/ci/tier_builder.py
+++ b/ci/tier_builder.py
@@ -48,6 +48,7 @@ class TierBuilder:
 
                 testcase = th.TestCase(name=dic_testcase['name'],
                                        dependency=dep,
+                                       criteria=dic_testcase['criteria'],
                                        description=dic_testcase['description'])
                 if testcase.is_compatible(self.ci_installer, self.ci_scenario):
                     tier.add_test(testcase)
diff --git a/ci/tier_handler.py b/ci/tier_handler.py
index b1ef52d8f..20a47e997 100644
--- a/ci/tier_handler.py
+++ b/ci/tier_handler.py
@@ -102,10 +102,11 @@ class Tier:
 
 
 class TestCase:
-    def __init__(self, name, dependency, description=""):
+    def __init__(self, name, dependency, criteria, description=""):
         self.name = name
         self.dependency = dependency
         self.description = description
+        self.criteria = criteria
 
     def is_compatible(self, ci_installer, ci_scenario):
         try:
@@ -124,6 +125,9 @@ class TestCase:
     def get_name(self):
         return self.name
 
+    def get_criteria(self):
+        return self.criteria
+
     def __str__(self):
         lines = split_text(self.description, LINE_LENGTH - 6)
 
@@ -134,6 +138,8 @@ class TestCase:
         out += ("| Description:".ljust(LINE_LENGTH - 1) + "|\n")
         for line in lines:
             out += ("|    " + line.ljust(LINE_LENGTH - 7) + " |\n")
+        out += ("| Criteria:  " +
+                self.criteria.ljust(LINE_LENGTH - 14) + "|\n")
         out += ("| Dependencies:".ljust(LINE_LENGTH - 1) + "|\n")
         installer = self.dependency.get_installer()
         scenario = self.dependency.get_scenario()
diff --git a/testcases/OpenStack/vPing/vPing_ssh.py b/testcases/OpenStack/vPing/vPing_ssh.py
index 4f83b874d..11887845c 100644
--- a/testcases/OpenStack/vPing/vPing_ssh.py
+++ b/testcases/OpenStack/vPing/vPing_ssh.py
@@ -176,13 +176,14 @@ def create_security_group(neutron_client):
     return sg_id
 
 
-def push_results(start_time_ts, duration, test_status):
+def push_results(start_time_ts, duration, status):
     try:
         logger.debug("Pushing result into DB...")
         scenario = functest_utils.get_scenario(logger)
         version = functest_utils.get_version(logger)
         criteria = "failed"
-        if test_status == "OK":
+        test_criteria = functest_utils.get_criteria_by_test("vping_ssh")
+        if eval(test_criteria):  # evaluates the regex 'status == "PASS"' 
             criteria = "passed"
         pod_name = functest_utils.get_pod_name(logger)
         build_tag = functest_utils.get_build_tag(logger)
@@ -193,7 +194,7 @@ def push_results(start_time_ts, duration, test_status):
                                           criteria, build_tag,
                                           payload={'timestart': start_time_ts,
                                                    'duration': duration,
-                                                   'status': test_status})
+                                                   'status': status})
     except:
         logger.error("Error pushing results into Database '%s'"
                      % sys.exc_info()[0])
@@ -438,10 +439,10 @@ def main():
         logger.debug("Pinging %s. Waiting for response..." % test_ip)
         sec += 1
 
-    test_status = "NOK"
+    test_status = "FAIL"
     if EXIT_CODE == 0:
         logger.info("vPing OK")
-        test_status = "OK"
+        test_status = "PASS"
     else:
         duration = 0
         logger.error("vPing FAILED")
diff --git a/utils/functest_utils.py b/utils/functest_utils.py
index 23dea7e5a..b43a63cb8 100644
--- a/utils/functest_utils.py
+++ b/utils/functest_utils.py
@@ -20,6 +20,12 @@ import sys
 import urllib2
 import yaml
 from git import Repo
+import functest.ci.tier_builder as tb
+
+
+""" global variables """
+REPOS_DIR = os.getenv('repos_dir')
+FUNCTEST_REPO = ("%s/functest/" % REPOS_DIR)
 
 
 # ----------------------------------------------------------
@@ -261,3 +267,15 @@ def get_deployment_dir(logger=None):
     deployment_dir = (rally_dir + "/tempest/for-deployment-" +
                       deployment_uuid)
     return deployment_dir
+
+
+def get_criteria_by_test(testname):
+    criteria = ""
+    file = FUNCTEST_REPO + "/ci/testcases.yaml"
+    tiers = tb.TierBuilder("", "", file)
+    for tier in tiers.get_tiers():
+        for test in tier.get_tests():
+            if test.get_name() == testname:
+                criteria = test.get_criteria()
+
+    return criteria