diff options
| -rw-r--r-- | ci/testcases.yaml | 15 | ||||
| -rw-r--r-- | ci/tier_builder.py | 1 | ||||
| -rw-r--r-- | ci/tier_handler.py | 8 | ||||
| -rw-r--r-- | testcases/OpenStack/vPing/vPing_ssh.py | 11 | ||||
| -rw-r--r-- | utils/functest_utils.py | 18 |
5 files changed, 47 insertions, 6 deletions
diff --git a/ci/testcases.yaml b/ci/testcases.yaml index 4cbc00add..4281f73ce 100644 --- a/ci/testcases.yaml +++ b/ci/testcases.yaml @@ -9,6 +9,7 @@ tiers: testcases: - name: healthcheck + criteria: 'status == "PASS"' description: >- This test case verifies the basic OpenStack services like Keystone, Glance, Cinder, Neutron and Nova. @@ -26,6 +27,7 @@ tiers: testcases: - name: vping_ssh + criteria: 'status == "PASS"' description: >- This test case verifies: 1) SSH to an instance using floating IPs over the public network. 2) Connectivity between 2 instances @@ -36,6 +38,7 @@ tiers: - name: vping_userdata + criteria: 'status == "PASS"' description: >- This test case verifies: 1) Boot a VM with given userdata. 2) Connectivity between 2 instances over a private network. @@ -45,6 +48,7 @@ tiers: - name: tempest_smoke_serial + criteria: 'success_rate == 100%' description: >- This test case runs the smoke subset of the OpenStack Tempest suite. The list of test cases is generated by @@ -56,6 +60,7 @@ tiers: - name: rally_sanity + criteria: 'success_rate == 100%' description: >- This test case runs a sub group of tests of the OpenStack Rally suite in smoke mode. @@ -73,6 +78,7 @@ tiers: testcases: - name: odl + criteria: 'success_rate == 100%' description: >- Test Suite for the OpenDaylight SDN Controller. It integrates some test suites from upstream using Robot as the test @@ -83,6 +89,7 @@ tiers: - name: onos + criteria: 'status == "PASS"' description: >- Test Suite for the ONOS SDN Controller. It integrates some test suites from upstream using TestON as the test @@ -93,6 +100,7 @@ tiers: - name: ovno + criteria: 'status == "PASS"' description: >- Test Suite for the Open Contrail SDN Controller. framework. @@ -110,6 +118,7 @@ tiers: testcases: - name: promise + criteria: 'success_rate == 100%' description: >- Test suite from Promise project. dependencies: @@ -118,6 +127,7 @@ tiers: - name: doctor + criteria: 'status == "PASS"' description: >- Test suite from Doctor project. dependencies: @@ -126,6 +136,7 @@ tiers: - name: bgpvpn + criteria: 'status == "PASS"' description: >- Test suite from SDNVPN project. dependencies: @@ -134,6 +145,7 @@ tiers: - name: security_scan + criteria: 'status == "PASS"' description: >- Simple security Scan dependencies: @@ -149,6 +161,7 @@ tiers: testcases: - name: tempest_full_parallel + criteria: 'success_rate >= 80%' description: >- The list of test cases is generated by Tempest automatically and depends on the parameters of @@ -159,6 +172,7 @@ tiers: - name: rally_full + criteria: 'success_rate >= 90%' description: >- This test case runs the full suite of scenarios of the OpenStack Rally suite using several threads and iterations. @@ -175,6 +189,7 @@ tiers: testcases: - name: vims + criteria: 'status == "PASS"' description: >- This test case deploys an OpenSource vIMS solution from Clearwater using the Cloudify orchestrator. It also runs some signaling traffic. diff --git a/ci/tier_builder.py b/ci/tier_builder.py index 4723bf4fa..b9357bc2d 100644 --- a/ci/tier_builder.py +++ b/ci/tier_builder.py @@ -48,6 +48,7 @@ class TierBuilder: testcase = th.TestCase(name=dic_testcase['name'], dependency=dep, + criteria=dic_testcase['criteria'], description=dic_testcase['description']) if testcase.is_compatible(self.ci_installer, self.ci_scenario): tier.add_test(testcase) diff --git a/ci/tier_handler.py b/ci/tier_handler.py index b1ef52d8f..20a47e997 100644 --- a/ci/tier_handler.py +++ b/ci/tier_handler.py @@ -102,10 +102,11 @@ class Tier: class TestCase: - def __init__(self, name, dependency, description=""): + def __init__(self, name, dependency, criteria, description=""): self.name = name self.dependency = dependency self.description = description + self.criteria = criteria def is_compatible(self, ci_installer, ci_scenario): try: @@ -124,6 +125,9 @@ class TestCase: def get_name(self): return self.name + def get_criteria(self): + return self.criteria + def __str__(self): lines = split_text(self.description, LINE_LENGTH - 6) @@ -134,6 +138,8 @@ class TestCase: out += ("| Description:".ljust(LINE_LENGTH - 1) + "|\n") for line in lines: out += ("| " + line.ljust(LINE_LENGTH - 7) + " |\n") + out += ("| Criteria: " + + self.criteria.ljust(LINE_LENGTH - 14) + "|\n") out += ("| Dependencies:".ljust(LINE_LENGTH - 1) + "|\n") installer = self.dependency.get_installer() scenario = self.dependency.get_scenario() diff --git a/testcases/OpenStack/vPing/vPing_ssh.py b/testcases/OpenStack/vPing/vPing_ssh.py index 4f83b874d..11887845c 100644 --- a/testcases/OpenStack/vPing/vPing_ssh.py +++ b/testcases/OpenStack/vPing/vPing_ssh.py @@ -176,13 +176,14 @@ def create_security_group(neutron_client): return sg_id -def push_results(start_time_ts, duration, test_status): +def push_results(start_time_ts, duration, status): try: logger.debug("Pushing result into DB...") scenario = functest_utils.get_scenario(logger) version = functest_utils.get_version(logger) criteria = "failed" - if test_status == "OK": + test_criteria = functest_utils.get_criteria_by_test("vping_ssh") + if eval(test_criteria): # evaluates the regex 'status == "PASS"' criteria = "passed" pod_name = functest_utils.get_pod_name(logger) build_tag = functest_utils.get_build_tag(logger) @@ -193,7 +194,7 @@ def push_results(start_time_ts, duration, test_status): criteria, build_tag, payload={'timestart': start_time_ts, 'duration': duration, - 'status': test_status}) + 'status': status}) except: logger.error("Error pushing results into Database '%s'" % sys.exc_info()[0]) @@ -438,10 +439,10 @@ def main(): logger.debug("Pinging %s. Waiting for response..." % test_ip) sec += 1 - test_status = "NOK" + test_status = "FAIL" if EXIT_CODE == 0: logger.info("vPing OK") - test_status = "OK" + test_status = "PASS" else: duration = 0 logger.error("vPing FAILED") diff --git a/utils/functest_utils.py b/utils/functest_utils.py index 23dea7e5a..b43a63cb8 100644 --- a/utils/functest_utils.py +++ b/utils/functest_utils.py @@ -20,6 +20,12 @@ import sys import urllib2 import yaml from git import Repo +import functest.ci.tier_builder as tb + + +""" global variables """ +REPOS_DIR = os.getenv('repos_dir') +FUNCTEST_REPO = ("%s/functest/" % REPOS_DIR) # ---------------------------------------------------------- @@ -261,3 +267,15 @@ def get_deployment_dir(logger=None): deployment_dir = (rally_dir + "/tempest/for-deployment-" + deployment_uuid) return deployment_dir + + +def get_criteria_by_test(testname): + criteria = "" + file = FUNCTEST_REPO + "/ci/testcases.yaml" + tiers = tb.TierBuilder("", "", file) + for tier in tiers.get_tiers(): + for test in tier.get_tests(): + if test.get_name() == testname: + criteria = test.get_criteria() + + return criteria |