From 0bb64c15f2dbc5ce6a4dafe2fd846fa8d58b17cd Mon Sep 17 00:00:00 2001 From: Juha Kosonen Date: Fri, 14 Oct 2016 20:28:03 +0000 Subject: Unify rally test case exclusion mechanisms In all cases exclusion is now controlled via blacklist file. JIRA: FUNCTEST-511 Change-Id: I482c33b6da4c9b711ebeddc13855b1337ab217eb Signed-off-by: Juha Kosonen --- testcases/OpenStack/rally/blacklist.txt | 26 ++++++--- testcases/OpenStack/rally/run_rally-cert.py | 65 ++++++++++++++++------ .../OpenStack/rally/scenario/full/opnfv-nova.yaml | 4 -- .../rally/scenario/sanity/opnfv-nova.yaml | 4 -- 4 files changed, 65 insertions(+), 34 deletions(-) diff --git a/testcases/OpenStack/rally/blacklist.txt b/testcases/OpenStack/rally/blacklist.txt index 02d85f4d..3a17fa61 100644 --- a/testcases/OpenStack/rally/blacklist.txt +++ b/testcases/OpenStack/rally/blacklist.txt @@ -1,8 +1,18 @@ -- - scenarios: - - os-nosdn-lxd-ha - - os-nosdn-lxd-noha - installers: - - joid - tests: - - NovaServers.boot_server_from_volume_and_delete +scenario: + - + scenarios: + - os-nosdn-lxd-ha + - os-nosdn-lxd-noha + installers: + - joid + tests: + - NovaServers.boot_server_from_volume_and_delete + +functionality: + - + functions: + - no_live_migration + tests: + - NovaServers.boot_and_live_migrate_server + - NovaServers.boot_server_attach_created_volume_and_live_migrate + - NovaServers.boot_server_from_volume_and_live_migrate diff --git a/testcases/OpenStack/rally/run_rally-cert.py b/testcases/OpenStack/rally/run_rally-cert.py index d35639cc..90084a12 100755 --- a/testcases/OpenStack/rally/run_rally-cert.py +++ b/testcases/OpenStack/rally/run_rally-cert.py @@ -187,7 +187,6 @@ def build_task_args(test_file_name): net_id = network_dict['net_id'] task_args['netid'] = str(net_id) - task_args['live_migration'] = live_migration_supported() auth_url = os.getenv('OS_AUTH_URL') if auth_url is not None: @@ -273,30 +272,60 @@ def get_cmd_output(proc): return result -def apply_blacklist(case_file_name, result_file_name): - logger.debug("Applying blacklist...") - cases_file = open(case_file_name, 'r') - result_file = open(result_file_name, 'w') +def excl_scenario(): black_tests = [] try: + with open(BLACKLIST_FILE, 'r') as black_list_file: + black_list_yaml = yaml.safe_load(black_list_file) + installer_type = os.getenv('INSTALLER_TYPE') deploy_scenario = os.getenv('DEPLOY_SCENARIO') if (bool(installer_type) * bool(deploy_scenario)): - # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the file - with open(BLACKLIST_FILE, 'r') as black_list_file: - black_list_yaml = yaml.safe_load(black_list_file) - - for item in black_list_yaml: - scenarios = item['scenarios'] - installers = item['installers'] - if (deploy_scenario in scenarios and - installer_type in installers): - tests = item['tests'] - black_tests.extend(tests) + if 'scenario' in black_list_yaml.keys(): + for item in black_list_yaml['scenario']: + scenarios = item['scenarios'] + installers = item['installers'] + if (deploy_scenario in scenarios and + installer_type in installers): + tests = item['tests'] + black_tests.extend(tests) + except: + logger.debug("Scenario exclusion not applied.") + + return black_tests + + +def excl_func(): + black_tests = [] + func_list = [] + + try: + with open(BLACKLIST_FILE, 'r') as black_list_file: + black_list_yaml = yaml.safe_load(black_list_file) + + if not live_migration_supported(): + func_list.append("no_live_migration") + + if 'functionality' in black_list_yaml.keys(): + for item in black_list_yaml['functionality']: + functions = item['functions'] + for func in func_list: + if func in functions: + tests = item['tests'] + black_tests.extend(tests) except: - black_tests = [] - logger.debug("Blacklisting not applied.") + logger.debug("Functionality exclusion not applied.") + + return black_tests + + +def apply_blacklist(case_file_name, result_file_name): + logger.debug("Applying blacklist...") + cases_file = open(case_file_name, 'r') + result_file = open(result_file_name, 'w') + + black_tests = list(set(excl_func() + excl_scenario())) include = True for cases_line in cases_file: diff --git a/testcases/OpenStack/rally/scenario/full/opnfv-nova.yaml b/testcases/OpenStack/rally/scenario/full/opnfv-nova.yaml index 077850f5..d7622093 100644 --- a/testcases/OpenStack/rally/scenario/full/opnfv-nova.yaml +++ b/testcases/OpenStack/rally/scenario/full/opnfv-nova.yaml @@ -227,8 +227,6 @@ sla: {{ no_failures_sla() }} -{% if live_migration %} - NovaServers.boot_and_live_migrate_server: - args: {{ vm_params(image_name, flavor_name) }} @@ -273,8 +271,6 @@ sla: {{ no_failures_sla() }} -{% endif %} - NovaKeypair.boot_and_delete_server_with_keypair: - args: diff --git a/testcases/OpenStack/rally/scenario/sanity/opnfv-nova.yaml b/testcases/OpenStack/rally/scenario/sanity/opnfv-nova.yaml index 947819dc..e2795cf7 100644 --- a/testcases/OpenStack/rally/scenario/sanity/opnfv-nova.yaml +++ b/testcases/OpenStack/rally/scenario/sanity/opnfv-nova.yaml @@ -1,5 +1,3 @@ -{% if live_migration %} - NovaServers.boot_and_live_migrate_server: - args: {{ vm_params(image_name, flavor_name) }} @@ -44,8 +42,6 @@ sla: {{ no_failures_sla() }} -{% endif %} - NovaKeypair.boot_and_delete_server_with_keypair: - args: -- cgit 1.2.3-korg