diff options
Diffstat (limited to 'testcases/OpenStack')
| -rwxr-xr-x | testcases/OpenStack/rally/run_rally-cert.py | 23 | ||||
| -rw-r--r-- | testcases/OpenStack/tempest/custom_tests/blacklist.txt | 17 | ||||
| -rwxr-xr-x | testcases/OpenStack/tempest/run_tempest.py | 26 |
3 files changed, 49 insertions, 17 deletions
diff --git a/testcases/OpenStack/rally/run_rally-cert.py b/testcases/OpenStack/rally/run_rally-cert.py index 2f696ec30..1f1214e03 100755 --- a/testcases/OpenStack/rally/run_rally-cert.py +++ b/testcases/OpenStack/rally/run_rally-cert.py @@ -504,23 +504,18 @@ def main(): 'nb tests': total_nb_tests, 'nb success': success_rate}}) - # Generate json results for DB - # json_results = {"timestart": time_start, "duration": total_duration, - # "tests": int(total_nb_tests), - # "success": int(total_success)} - # logger.info("Results: "+str(json_results)) - - # Evaluation of the success criteria - status = "FAIL" - # for Rally we decided that the overall success rate must be above 90% - if float(success_rate) >= 90: - status = "PASS" - if args.sanity: case_name = "rally_sanity" else: case_name = "rally_full" + # Evaluation of the success criteria + status = functest_utils.check_success_rate(case_name, success_rate) + + exit_code = -1 + if status == "PASS": + exit_code = 0 + if args.report: logger.debug("Pushing Rally summary into DB...") functest_utils.push_results_to_db("functest", @@ -531,7 +526,7 @@ def main(): status, payload) if args.noclean: - exit(0) + exit(exit_code) if not image_exists: logger.debug("Deleting image '%s' with ID '%s'..." @@ -545,6 +540,8 @@ def main(): if not os_utils.delete_volume_type(cinder_client, volume_type): logger.error("Error in deleting volume type...") + exit(exit_code) + if __name__ == '__main__': main() diff --git a/testcases/OpenStack/tempest/custom_tests/blacklist.txt b/testcases/OpenStack/tempest/custom_tests/blacklist.txt new file mode 100644 index 000000000..49605ca7b --- /dev/null +++ b/testcases/OpenStack/tempest/custom_tests/blacklist.txt @@ -0,0 +1,17 @@ +- + scenarios: + - os-odl_l2-bgpvpn-ha + - os-odl_l2-bgpvpn-noha + installers: + - fuel + - apex + tests: + - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke] + - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke] + - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke] + - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke] + - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard[id-2cb1baf6-ac8d-4429-bf0d-ba8a0ba53e32,smoke] + - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops[compute,id-f323b3ba-82f8-4db7-8ea6-6a895869ec49,network,smoke] + - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops[compute,id-7fff3fb3-91d8-4fd0-bd7d-0204f1f180ba,network,smoke] + - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern[compute,id-557cd2c2-4eb8-4dce-98be-f86765ff311b,image,smoke,volume] + - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern[compute,id-557cd2c2-4eb8-4dce-98be-f86765ff311b,image,smoke,volume] diff --git a/testcases/OpenStack/tempest/run_tempest.py b/testcases/OpenStack/tempest/run_tempest.py index f99678df4..8d23b7415 100755 --- a/testcases/OpenStack/tempest/run_tempest.py +++ b/testcases/OpenStack/tempest/run_tempest.py @@ -21,11 +21,12 @@ import shutil import subprocess import sys import time +import yaml import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils import functest.utils.openstack_utils as os_utils -import yaml + modes = ['full', 'smoke', 'baremetal', 'compute', 'data_processing', 'identity', 'image', 'network', 'object_storage', 'orchestration', @@ -330,13 +331,30 @@ def apply_tempest_blacklist(): logger.debug("Applying tempest blacklist...") cases_file = read_file(TEMPEST_RAW_LIST) result_file = open(TEMPEST_LIST, 'w') + black_tests = [] try: - black_file = read_file(TEMPEST_BLACKLIST) + installer_type = os.getenv('INSTALLER_TYPE') + deploy_scenario = os.getenv('DEPLOY_SCENARIO') + if (bool(installer_type) * bool(deploy_scenario)): + # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the file + black_list_file = open(TEMPEST_BLACKLIST) + black_list_yaml = yaml.safe_load(black_list_file) + black_list_file.close() + for item in black_list_yaml: + scenarios = item['sceanrios'] + installers = item['installers'] + if (deploy_scenario in scenarios and + installer_type in installers): + tests = item['tests'] + for test in tests: + black_tests.append(test) + break except: - black_file = '' + black_tests = [] logger.debug("Tempest blacklist file does not exist.") + for line in cases_file: - if line not in black_file: + if line not in black_tests: result_file.write(str(line) + '\n') result_file.close() |