Add mechanism to check whether the test can be run or not according to the scenario/controller

JIRA: FUNCTEST-119

Change-Id: Ida658760484140ff59f7198dbbf4d5620450ff27
Signed-off-by: Morgan Richomme <morgan.richomme@orange.com>

3 files changed, 277 insertions, 25 deletions

diff --git a/testcases/config_functest.yaml b/testcases/config_functest.yaml
index 7521da500..bc41350e1 100644
--- a/testcases/config_functest.yaml
+++ b/testcases/config_functest.yaml
@@ -169,36 +169,47 @@ results:
 # - installer (e.g. my test can be run only with installer Compass)
 # possible values: apex, compass, fuel, joid
 #
-# - controller (e.g. my test can be done only with controller OpenContrail)
-# possible values: none, odl, onos, opencontrail
+# - the controller (odl, onos, opencontrail) 
 #
-# - options (ovs, kvm, QEMU, proc) (my test is only possible with OVS 2.3.2)
-# possible values:
-# ovs: ovs2.4, ovsX
-# kvm:
-# qemu:
+# - the scenario (ovs, kvm, QEMU, proc) based on Fuel proposal
+# see https://git.opnfv.org/cgit/fuel/tree/deploy/scenario/scenario.yaml
+#  e.g my test is only possible with OVS 2.3.2 on odl
+# not fully clear as the controller may be included with the scenario
+#
+# In a first step, our main need is to trigger ad hox controller suite
+# In second step we may focus with scenario parameter
+# but no so far
+# - either controller suite
+# or
+# - suites that should be runnable on any conf, any scenario
 #
 #######################################################################
 test-dependencies:
-    #congress:
-    #doctor:
+    doctor:
     functest:
+        vping:
+        vims:
+        tempest:
+        rally:
         odl:
             controller: 'odl'
         onos:
             controller: 'onos'
         onos-ovsdb:
             controller: 'onos'
-    #promise:
+    promise:
     ovno:
         controller: 'opencontrail'
     policy-test:
         controller: 'odl'
     sdnvpn:
-        options: 'ovs2.4'
         opnfv-yardstick-tc026-sdnvpn:
-            controller: 'none'
+            controller: 'nosdn'
+            scenario: 'os_ovh_ha'
         odl-vpn_service-tests:
             controller: 'odl'
+            scenario: 'os_ovh_ha'
         openstack-neutron-bgpvpn-api-extension-tests:
-            controller: 'none'
+            controller: 'nosdn'
+            scenario: 'os_ovh_ha'
+ 
diff --git a/testcases/functest_utils.py b/testcases/functest_utils.py
index 0734ccbde..888c043fd 100644
--- a/testcases/functest_utils.py
+++ b/testcases/functest_utils.py
@@ -25,7 +25,7 @@ def check_credentials():
     """
     Check if the OpenStack credentials (openrc) are sourced
     """
-    env_vars = ['OS_AUTH_URL','OS_USERNAME','OS_PASSWORD','OS_TENANT_NAME']
+    env_vars = ['OS_AUTH_URL', 'OS_USERNAME', 'OS_PASSWORD', 'OS_TENANT_NAME']
     return all(map(lambda v: v in os.environ and os.environ[v], env_vars))
 
 
@@ -69,6 +69,7 @@ def get_instances(nova_client):
     except:
         return None
 
+
 def get_instance_status(nova_client, instance):
     try:
         instance = nova_client.servers.get(instance.id)
@@ -76,6 +77,7 @@ def get_instance_status(nova_client, instance):
     except:
         return None
 
+
 def get_instance_by_name(nova_client, instance_name):
     try:
         instance = nova_client.servers.find(name=instance_name)
@@ -84,7 +86,6 @@ def get_instance_by_name(nova_client, instance_name):
         return None
 
 
-
 def get_flavor_id(nova_client, flavor_name):
     flavors = nova_client.flavors.list(detailed=True)
     id = ''
@@ -121,6 +122,7 @@ def get_floating_ips(nova_client):
     except:
         return None
 
+
 def delete_floating_ip(nova_client, floatingip_id):
     try:
         nova_client.floating_ips.delete(floatingip_id)
@@ -222,6 +224,7 @@ def remove_interface_router(neutron_client, router_id, subnet_id):
         print "Error:", sys.exc_info()[0]
         return False
 
+
 def remove_gateway_router(neutron_client, router_id):
     try:
         neutron_client.remove_gateway_router(router_id)
@@ -301,6 +304,7 @@ def get_router_list(neutron_client):
     else:
         return router_list
 
+
 def get_port_list(neutron_client):
     port_list = neutron_client.list_ports()['ports']
     if len(port_list) == 0:
@@ -309,7 +313,6 @@ def get_port_list(neutron_client):
         return port_list
 
 
-
 def get_external_net(neutron_client):
     for network in neutron_client.list_networks()['networks']:
         if network['router:external']:
@@ -331,30 +334,33 @@ def update_sg_quota(neutron_client, tenant_id, sg_quota, sg_rule_quota):
         print "Error:", sys.exc_info()[0]
         return False
 
-def update_cinder_quota(cinder_client, tenant_id, vols_quota, snapshots_quota,gigabytes_quota):
-    quotas_values = {
-            "volumes": vols_quota,
-            "snapshots": snapshots_quota,
-            "gigabytes": gigabytes_quota
-    }
+
+def update_cinder_quota(cinder_client, tenant_id, vols_quota,
+                        snapshots_quota, gigabytes_quota):
+    quotas_values = {"volumes": vols_quota,
+                     "snapshots": snapshots_quota,
+                     "gigabytes": gigabytes_quota}
 
     try:
-        quotas_default=cinder_client.quotas.update(tenant_id,**quotas_values)
+        quotas_default = cinder_client. quotas.update(tenant_id,
+                                                      **quotas_values)
         return True
     except:
         print "Error:", sys.exc_info()[0]
         return False
 
+
 def get_private_net(neutron_client):
     # Checks if there is an existing shared private network
     networks = neutron_client.list_networks()['networks']
     if len(networks) == 0:
         return None
     for net in networks:
-        if (net['router:external'] == False) and (net['shared'] == True):
+        if (net['router:external'] is False) and (net['shared'] is True):
             return net
     return None
 
+
 # ################ GLANCE #################
 def get_images(nova_client):
     try:
@@ -390,6 +396,7 @@ def create_glance_image(glance_client, image_name, file_path, public=True):
         print "Error:", sys.exc_info()[0]
         return False
 
+
 def delete_glance_image(nova_client, image_id):
     try:
         nova_client.images.delete(image_id)
@@ -398,6 +405,7 @@ def delete_glance_image(nova_client, image_id):
         print "Error:", sys.exc_info()[0]
         return False
 
+
 # ################ CINDER #################
 def get_volumes(cinder_client):
     try:
@@ -406,6 +414,7 @@ def get_volumes(cinder_client):
     except:
         return None
 
+
 def delete_volume(cinder_client, volume_id):
     try:
         cinder_client.volumes.delete(volume_id)
@@ -414,6 +423,7 @@ def delete_volume(cinder_client, volume_id):
         print "Error:", sys.exc_info()[0]
         return False
 
+
 # ################ CINDER #################
 def get_security_groups(neutron_client):
     try:
@@ -422,6 +432,7 @@ def get_security_groups(neutron_client):
     except:
         return None
 
+
 def delete_security_group(neutron_client, secgroup_id):
     try:
         neutron_client.delete_security_group(secgroup_id)
@@ -449,6 +460,7 @@ def get_tenant_id(keystone_client, tenant_name):
             break
     return id
 
+
 def get_users(keystone_client):
     try:
         users = keystone_client.users.list()
@@ -456,6 +468,7 @@ def get_users(keystone_client):
     except:
         return None
 
+
 def get_role_id(keystone_client, role_name):
     roles = keystone_client.roles.list()
     id = ''
@@ -611,7 +624,8 @@ def get_pod_name(logger=None):
         return "unknown-pod"
 
 
-def push_results_to_db(db_url, case_name, logger, pod_name, git_version, payload):
+def push_results_to_db(db_url, case_name, logger, pod_name,
+                       git_version, payload):
     url = db_url + "/results"
     installer = get_installer_type(logger)
     params = {"project_name": "functest", "case_name": case_name,
@@ -626,3 +640,108 @@ def push_results_to_db(db_url, case_name, logger, pod_name, git_version, payload
     except:
         print "Error:", sys.exc_info()[0]
         return False
+
+
+def getTestEnv(test, functest_yaml):
+    # get the config of the testcase based on functest_config.yaml
+    # 2 options
+    # - test = test project e.g; ovno
+    # - test = testcase e.g. functest/odl
+    # look for the / to see if it is a test project or a testcase
+    try:
+        TEST_ENV = functest_yaml.get("test-dependencies")
+
+        if test.find("/") < 0:
+            config_test = TEST_ENV[test]
+        else:
+            test_split = test.split("/")
+            testproject = test_split[0]
+            testcase = test_split[1]
+            config_test = TEST_ENV[testproject][testcase]
+    except KeyError:
+        # if not defined in dependencies => no dependencies
+        config_test = ""
+    except:
+        print "Error getTestEnv:", sys.exc_info()[0]
+
+    return config_test
+
+
+def get_ci_envvars():
+    """
+    Get the CI env variables
+    """
+    ci_env_var = {
+        "installer": os.environ.get('INSTALLER_TYPE'),
+        "controller": os.environ.get('SDN_CONTROLLER'),
+        "options": os.environ.get("OPNFV_FEATURE")}
+    return ci_env_var
+
+
+def isTestRunnable(test, functest_yaml):
+    # check getTestEnv(test) and CI env var
+    # check installer, controller and options
+    # e.g. if test needs onos => do not run odl suite
+    try:
+        # By default we assume that all the tests are always runnable...
+        is_runnable = True
+        # Retrieve CI environment
+        ci_env = get_ci_envvars()
+
+        # Retrieve test environement from config file
+        test_env = getTestEnv(test, functest_yaml)
+
+        # if test_env not empty => dependencies to be checked
+        if test_env is not None and len(test_env) > 0:
+            # possible criteria = ["installer", "controller", "options"]
+            # consider test criteria from config file
+            # compare towards CI env through CI en variable
+            for criteria in test_env:
+                if test_env[criteria] != ci_env[criteria]:
+                    # print "Test "+ test + " cannot be run on the environment"
+                    is_runnable = False
+    except:
+        print "Error isTestRunnable:", sys.exc_info()[0]
+    return is_runnable
+
+
+def generateTestcaseList(functest_yaml):
+    try:
+        test_list = ""
+        # Retrieve CI environment
+        get_ci_envvars()
+
+        # get testcases
+        testcase_list = functest_yaml.get("test-dependencies")
+        projects = testcase_list.keys()
+        for project in projects:
+            testcases = testcase_list[project]
+            # 1 or 2 levels for testcases project[/case]
+            # if only project name without controller or scenario
+            # => shall be runnable on any controller/scenario
+            if testcases is None:
+                test_list += project + " "
+            else:
+                for testcase in testcases:
+                    if testcase == "controller" or testcase == "scenario":
+                        # project (1 level)
+                        if isTestRunnable(project, functest_yaml):
+                            test_list += project + " "
+                    else:
+                        # project/testcase (2 levels)
+                        thetest = project + "/" + testcase
+                        if isTestRunnable(thetest, functest_yaml):
+                            test_list += testcase + " "
+
+        # create a file that could be consumed by run-test.sh
+        file = open("testcase-list.txt", 'w')
+        file.write(test_list)
+        file.close()
+
+        return test_list
+
+        # test for each testcase if it is runnable
+        # towards the declared configuration
+        # generate the test config file
+    except:
+        print "Error generateTestcaseList:", sys.exc_info()[0]
diff --git a/testcases/tests/TestFunctestUtils.py b/testcases/tests/TestFunctestUtils.py
new file mode 100644
index 000000000..337e4ee8a
--- /dev/null
+++ b/testcases/tests/TestFunctestUtils.py
@@ -0,0 +1,122 @@
+import unittest
+import os
+import sys
+import yaml
+
+sys.path.append("../")
+from functest_utils import getTestEnv, isTestRunnable, generateTestcaseList
+
+
+class TestFunctestUtils(unittest.TestCase):
+
+    def setUp(self):
+        os.environ["INSTALLER_TYPE"] = "fuel"
+        os.environ["SDN_CONTROLLER"] = "odl"
+        os.environ["OPNFV_FEATURE"] = "ovs2.4"
+
+        global functest_yaml
+
+        with open("../config_functest.yaml") as f:
+            functest_yaml = yaml.safe_load(f)
+            f.close()
+
+    def test_getTestEnv(self):
+
+        env_test = getTestEnv('ovno', functest_yaml)
+        self.assertEqual(env_test, {'controller': 'opencontrail'})
+
+        env_test = getTestEnv('doctor', functest_yaml)
+        self.assertEqual(env_test, None)
+
+        env_test = getTestEnv('promise', functest_yaml)
+        self.assertEqual(env_test, None)
+
+        env_test = getTestEnv('functest/tempest', functest_yaml)
+        self.assertEqual(env_test, None)
+
+        env_test = getTestEnv('functest/vims', functest_yaml)
+        self.assertEqual(env_test, None)
+
+        env_test = getTestEnv('functest/odl', functest_yaml)
+        self.assertEqual(env_test, {'controller': 'odl'})
+
+        env_test = getTestEnv('functest/onos', functest_yaml)
+        self.assertEqual(env_test, {'controller': 'onos'})
+
+        env_test = getTestEnv('functest/onos-ovsdb', functest_yaml)
+        self.assertEqual(env_test, {'controller': 'onos'})
+
+        env_test = getTestEnv('policy-test', functest_yaml)
+        self.assertEqual(env_test, {'controller': 'odl'})
+
+        env_test = getTestEnv('sdnvpn/odl-vpn_service-tests', functest_yaml)
+        self.assertEqual(env_test,
+                         {'controller': 'odl', 'scenario': 'os_ovh_ha'})
+
+        env_test = getTestEnv('sdnvpn/opnfv-yardstick-tc026-sdnvpn',
+                              functest_yaml)
+        self.assertEqual(env_test,
+                         {'controller': 'nosdn', 'scenario': 'os_ovh_ha'})
+
+        env_test = getTestEnv('sdnvpn/openstack-neutron-bgpvpn-api-extension-tests',
+                              functest_yaml)
+        self.assertEqual(env_test,
+                         {'controller': 'nosdn', 'scenario': 'os_ovh_ha'})
+
+        env_test = getTestEnv('foo', functest_yaml)
+        self.assertEqual(env_test, '')
+
+    def test_isTestRunnable(self):
+
+        test = isTestRunnable('ovno', functest_yaml)
+        self.assertFalse(test)
+
+        test = isTestRunnable('doctor', functest_yaml)
+        self.assertTrue(test)
+
+        test = isTestRunnable('promise', functest_yaml)
+        self.assertTrue(test)
+
+        test = isTestRunnable('functest/onos', functest_yaml)
+        self.assertFalse(test)
+
+        test = isTestRunnable('functest/odl', functest_yaml)
+        self.assertTrue(test)
+
+        test = isTestRunnable('functest/vping', functest_yaml)
+        self.assertTrue(test)
+
+        test = isTestRunnable('functest/tempest', functest_yaml)
+        self.assertTrue(test)
+
+        test = isTestRunnable('functest/rally', functest_yaml)
+        self.assertTrue(test)
+
+        test = isTestRunnable('functest/vims', functest_yaml)
+        self.assertTrue(test)
+
+        test = isTestRunnable('sdnvpn/odl-vpn_service-tests',
+                              functest_yaml)
+        self.assertTrue(test)
+
+        test = isTestRunnable('sdnvpn/opnfv-yardstick-tc026-sdnvpn',
+                              functest_yaml)
+        self.assertFalse(test)
+
+    def test_generateTestcaseList(self):
+
+        test = generateTestcaseList(functest_yaml)
+        test = sorted(test.split(' '))
+        expected_list = "doctor vims odl rally vping tempest promise policy-test odl-vpn_service-tests "
+        expected_list_array = sorted(expected_list.split(' '))
+        
+        self.assertEqual(test, expected_list_array)
+
+    def tearDown(self):
+        os.environ["INSTALLER_TYPE"] = ""
+        os.environ["SDN_CONTROLLER"] = ""
+        os.environ["OPNFV_FEATURE"] = ""
+
+
+if __name__ == '__main__':
+    unittest.main()