diff options
Diffstat (limited to 'testcases/OpenStack')
| -rw-r--r-- | testcases/OpenStack/tempest/custom_tests/blacklist.txt | 34 | ||||
| -rwxr-xr-x | testcases/OpenStack/tempest/run_tempest.py | 96 |
2 files changed, 45 insertions, 85 deletions
diff --git a/testcases/OpenStack/tempest/custom_tests/blacklist.txt b/testcases/OpenStack/tempest/custom_tests/blacklist.txt index 49605ca7..5dd8fe2e 100644 --- a/testcases/OpenStack/tempest/custom_tests/blacklist.txt +++ b/testcases/OpenStack/tempest/custom_tests/blacklist.txt @@ -6,12 +6,28 @@ - fuel - apex tests: - - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke] - - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke] - - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers[id-9a438d88-10c6-4bcd-8b5b-5b6e25e1346f,smoke] - - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details[id-5de47127-9977-400a-936f-abcfbec1218f,smoke] - - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard[id-2cb1baf6-ac8d-4429-bf0d-ba8a0ba53e32,smoke] - - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops[compute,id-f323b3ba-82f8-4db7-8ea6-6a895869ec49,network,smoke] - - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops[compute,id-7fff3fb3-91d8-4fd0-bd7d-0204f1f180ba,network,smoke] - - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern[compute,id-557cd2c2-4eb8-4dce-98be-f86765ff311b,image,smoke,volume] - - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern[compute,id-557cd2c2-4eb8-4dce-98be-f86765ff311b,image,smoke,volume] + - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_list_servers + - tempest.api.compute.servers.test_create_server.ServersTestJSON.test_verify_server_details + - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_list_servers + - tempest.api.compute.servers.test_create_server.ServersTestManualDisk.test_verify_server_details + - tempest.api.compute.servers.test_server_actions.ServerActionsTestJSON.test_reboot_server_hard + - tempest.scenario.test_network_basic_ops.TestNetworkBasicOps.test_network_basic_ops + - tempest.scenario.test_server_basic_ops.TestServerBasicOps.test_server_basic_ops + - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPattern.test_volume_boot_pattern + - tempest.scenario.test_volume_boot_pattern.TestVolumeBootPatternV2.test_volume_boot_pattern + + scenarios: + - os-odl_l2-nofeature-ha + - os-odl_l2-nofeature-noha + - os-nosdn-nofeature-ha + - os-nosdn-nofeature-noha + - os-onos-nofeature-ha + - os-onos-nofeature-noha + - os-onos-sfc-ha + - os-onos-sfc-noha + - os-nosdn-lxd-ha + - os-nosdn-lxd-noha + installers: + - joid + tests: + - tempest.api.object_storage diff --git a/testcases/OpenStack/tempest/run_tempest.py b/testcases/OpenStack/tempest/run_tempest.py index 3ee10b78..53a70b79 100755 --- a/testcases/OpenStack/tempest/run_tempest.py +++ b/testcases/OpenStack/tempest/run_tempest.py @@ -214,8 +214,8 @@ def configure_tempest(deployment_dir): def configure_tempest_feature(deployment_dir, mode): - """ - Add/update needed parameters into tempest.conf file generated by Rally + """Add/update needed parameters into tempest.conf file generated by Rally + """ logger.debug("Finding tempest.conf file...") @@ -231,56 +231,11 @@ def configure_tempest_feature(deployment_dir, mode): if mode == 'feature_multisite': config.set('service_available', 'kingbird', 'true') cmd = "openstack endpoint show kingbird | grep publicurl |\ - awk '{print $4}' | awk -F '/' '{print $4}'" - kingbird_api_version = os.popen(cmd).read() - if os.environ.get("INSTALLER_TYPE") == 'fuel': - # For MOS based setup, the service is accessible - # via bind host - kingbird_conf_path = "/etc/kingbird/kingbird.conf" - installer_type = os.getenv('INSTALLER_TYPE', 'Unknown') - installer_ip = os.getenv('INSTALLER_IP', 'Unknown') - installer_username = ft_utils.get_parameter_from_yaml( - "multisite." + installer_type + - "_environment.installer_username") - installer_password = ft_utils.get_parameter_from_yaml( - "multisite." + installer_type + - "_environment.installer_password") - - ssh_options = "-o UserKnownHostsFile=/dev/null -o \ - StrictHostKeyChecking=no" - - # Get the controller IP from the fuel node - cmd = 'sshpass -p %s ssh 2>/dev/null %s %s@%s \ - \'fuel node --env 1| grep controller | grep "True\| 1" \ - | awk -F\| "{print \$5}"\'' % (installer_password, - ssh_options, - installer_username, - installer_ip) - multisite_controller_ip = \ - "".join(os.popen(cmd).read().split()) - - # Login to controller and get bind host details - cmd = 'sshpass -p %s ssh 2>/dev/null %s %s@%s "ssh %s \\" \ - grep -e "^bind_" %s \\""' % (installer_password, - ssh_options, - installer_username, - installer_ip, - multisite_controller_ip, - kingbird_conf_path) - bind_details = os.popen(cmd).read() - bind_details = "".join(bind_details.split()) - # Extract port number from the bind details - bind_port = re.findall(r"\D(\d{4})", bind_details)[0] - # Extract ip address from the bind details - bind_host = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", - bind_details)[0] - kingbird_endpoint_url = "http://" + bind_host + ":" + bind_port + \ - "/" - else: - cmd = "openstack endpoint show kingbird | grep publicurl |\ - awk '{print $4}' | awk -F '/' '{print $3}'" - kingbird_endpoint_url = os.popen(cmd).read() - + awk '{print $4}'" + kingbird_endpoint_details = "".join(os.popen(cmd).read().split()) + kingbird_endpoint_url = kingbird_endpoint_details.rsplit('/', 1)[0] + \ + '/' + kingbird_api_version = kingbird_endpoint_details.rsplit('/', 1)[1] try: config.add_section("kingbird") except Exception: @@ -353,9 +308,12 @@ def apply_tempest_blacklist(): black_tests = [] logger.debug("Tempest blacklist file does not exist.") - for line in cases_file: - if line not in black_tests: - result_file.write(str(line) + '\n') + for cases_line in cases_file: + for black_tests_line in black_tests: + if black_tests_line in cases_line: + break + else: + result_file.write(str(cases_line) + '\n') result_file.close() @@ -427,34 +385,20 @@ def run_tempest(OPTION): dur_sec_int = dur_sec_int + 60 * dur_min stop_time = time.time() - status = "FAIL" try: diff = (int(num_tests) - int(num_failures)) success_rate = 100 * diff / int(num_tests) except: success_rate = 0 - # For Tempest we assume that the success rate is above 90% - if "smoke" in args.mode: - case_name = "tempest_smoke_serial" - # Note criteria hardcoded...TODO read it from testcases.yaml - success_criteria = 100 - if success_rate >= success_criteria: - status = "PASS" - else: - logger.info("Tempest success rate: %s%%. The success criteria to " - "pass this test is %s%%. Marking the test as FAILED." % - (success_rate, success_criteria)) + if 'smoke' in args.mode: + case_name = 'tempest_smoke_serial' else: - case_name = "tempest_full_parallel" - # Note criteria hardcoded...TODO read it from testcases.yaml - success_criteria = 80 - if success_rate >= success_criteria: - status = "PASS" - else: - logger.info("Tempest success rate: %s%%. The success criteria to " - "pass this test is %s%%. Marking the test as FAILED." % - (success_rate, success_criteria)) + case_name = 'tempest_full_parallel' + + status = ft_utils.check_success_rate(case_name, success_rate) + logger.info("Tempest %s success_rate is %s%%, is marked as %s" + % (case_name, success_rate, status)) # Push results in payload of testcase if args.report: |