3 files changed, 49 insertions, 17 deletions
diff --git a/testcases/OpenStack/rally/run_rally-cert.py b/testcases/OpenStack/rally/run_rally-cert.py
index 2f696ec3..1f1214e0 100755
--- a/testcases/OpenStack/rally/run_rally-cert.py
+++ b/testcases/OpenStack/rally/run_rally-cert.py
@@ -504,23 +504,18 @@ def main():
                                 'nb tests': total_nb_tests,
                                 'nb success': success_rate}})
 
-    # Generate json results for DB
-    # json_results = {"timestart": time_start, "duration": total_duration,
-    #                "tests": int(total_nb_tests),
-    #                "success": int(total_success)}
-    # logger.info("Results: "+str(json_results))
-
-    # Evaluation of the success criteria
-    status = "FAIL"
-    # for Rally we decided that the overall success rate must be above 90%
-    if float(success_rate) >= 90:
-        status = "PASS"
-
     if args.sanity:
         case_name = "rally_sanity"
     else:
         case_name = "rally_full"
 
+    # Evaluation of the success criteria
+    status = functest_utils.check_success_rate(case_name, success_rate)
+
+    exit_code = -1
+    if status == "PASS":
+        exit_code = 0
+
     if args.report:
         logger.debug("Pushing Rally summary into DB...")
         functest_utils.push_results_to_db("functest",
@@ -531,7 +526,7 @@ def main():
                                           status,
                                           payload)
     if args.noclean:
-        exit(0)
+        exit(exit_code)
 
     if not image_exists:
         logger.debug("Deleting image '%s' with ID '%s'..."
@@ -545,6 +540,8 @@ def main():
         if not os_utils.delete_volume_type(cinder_client, volume_type):
             logger.error("Error in deleting volume type...")
 
+    exit(exit_code)
+
 
 if __name__ == '__main__':
     main()
diff --git a/testcases/OpenStack/tempest/custom_tests/blacklist.txt b/testcases/OpenStack/tempest/custom_tests/blacklist.txt
new file mode 100644
index 00000000..49605ca7
--- /dev/null
+++ b/testcases/OpenStack/tempest/custom_tests/blacklist.txt
@@ -0,0 +1,17 @@
+-
+    scenarios:
+        - os-odl_l2-bgpvpn-ha
+        - os-odl_l2-bgpvpn-noha
+    installers:
+        - fuel
+        - apex
+    tests:
+        - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke]
+        - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke]
+        - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke]
+        - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke]
+        - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard[id-2cb1baf6-ac8d-4429-bf0d-ba8a0ba53e32,smoke]
+        - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops[compute,id-f323b3ba-82f8-4db7-8ea6-6a895869ec49,network,smoke]
+        - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops[compute,id-7fff3fb3-91d8-4fd0-bd7d-0204f1f180ba,network,smoke]
+        - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern[compute,id-557cd2c2-4eb8-4dce-98be-f86765ff311b,image,smoke,volume]
+        - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern[compute,id-557cd2c2-4eb8-4dce-98be-f86765ff311b,image,smoke,volume]
diff --git a/testcases/OpenStack/tempest/run_tempest.py b/testcases/OpenStack/tempest/run_tempest.py
index f99678df..8d23b741 100755
--- a/testcases/OpenStack/tempest/run_tempest.py
+++ b/testcases/OpenStack/tempest/run_tempest.py
@@ -21,11 +21,12 @@ import shutil
 import subprocess
 import sys
 import time
+import yaml
 
 import functest.utils.functest_logger as ft_logger
 import functest.utils.functest_utils as ft_utils
 import functest.utils.openstack_utils as os_utils
-import yaml
+
 
 modes = ['full', 'smoke', 'baremetal', 'compute', 'data_processing',
          'identity', 'image', 'network', 'object_storage', 'orchestration',
@@ -330,13 +331,30 @@ def apply_tempest_blacklist():
     logger.debug("Applying tempest blacklist...")
     cases_file = read_file(TEMPEST_RAW_LIST)
     result_file = open(TEMPEST_LIST, 'w')
+    black_tests = []
     try:
-        black_file = read_file(TEMPEST_BLACKLIST)
+        installer_type = os.getenv('INSTALLER_TYPE')
+        deploy_scenario = os.getenv('DEPLOY_SCENARIO')
+        if (bool(installer_type) * bool(deploy_scenario)):
+            # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the file
+            black_list_file = open(TEMPEST_BLACKLIST)
+            black_list_yaml = yaml.safe_load(black_list_file)
+            black_list_file.close()
+            for item in black_list_yaml:
+                scenarios = item['sceanrios']
+                installers = item['installers']
+                if (deploy_scenario in scenarios and
+                        installer_type in installers):
+                    tests = item['tests']
+                    for test in tests:
+                        black_tests.append(test)
+                    break
     except:
-        black_file = ''
+        black_tests = []
         logger.debug("Tempest blacklist file does not exist.")
+
     for line in cases_file:
-        if line not in black_file:
+        if line not in black_tests:
             result_file.write(str(line) + '\n')
     result_file.close()