diff options
Diffstat (limited to 'functest/opnfv_tests/openstack/tempest/tempest.py')
| -rw-r--r-- | functest/opnfv_tests/openstack/tempest/tempest.py | 304 |
1 files changed, 190 insertions, 114 deletions
diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index ec0ca766..0014b718 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -16,13 +16,12 @@ import time import yaml -import conf_utils -import functest.core.testcase_base as testcase_base +from functest.core import testcase_base +from functest.opnfv_tests.openstack.tempest import conf_utils +from functest.utils.constants import CONST import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils import functest.utils.openstack_utils as os_utils -import functest.utils.functest_constants as ft_constants -import opnfv.utils.constants as releng_constants """ logging configuration """ logger = ft_logger.Logger("Tempest").getLogger() @@ -31,14 +30,80 @@ logger = ft_logger.Logger("Tempest").getLogger() class TempestCommon(testcase_base.TestcaseBase): def __init__(self): - self.case_name = "" + super(TempestCommon, self).__init__() self.MODE = "" self.OPTION = "" self.FLAVOR_ID = None self.IMAGE_ID = None - self.DEPLOYMENT_DIR = ft_utils.get_deployment_dir() - - def read_file(self, filename): + self.VERIFIER_ID = self.get_verifier_id() + self.VERIFIER_REPO_DIR = self.get_verifier_repo_dir() + self.DEPLOYMENT_ID = self.get_verifier_deployment_id() + self.DEPLOYMENT_DIR = self.get_verifier_deployment_dir() + self.VERIFICATION_ID = None + + @staticmethod + def get_verifier_id(): + """ + Returns verifer id for current Tempest + """ + cmd = ("rally verify list-verifiers | awk '/" + + CONST.tempest_deployment_name + + "/ {print $2}'") + p = subprocess.Popen(cmd, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + deployment_uuid = p.stdout.readline().rstrip() + if deployment_uuid == "": + logger.error("Tempest verifier not found.") + raise Exception('Error with command:%s' % cmd) + return deployment_uuid + + @staticmethod + def get_verifier_deployment_id(): + """ + Returns deployment id for active Rally deployment + """ + cmd = ("rally deployment list | awk '/" + + CONST.rally_deployment_name + + "/ {print $2}'") + p = subprocess.Popen(cmd, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + deployment_uuid = p.stdout.readline().rstrip() + if deployment_uuid == "": + logger.error("Rally deployment not found.") + raise Exception('Error with command:%s' % cmd) + return deployment_uuid + + def get_verifier_repo_dir(self): + """ + Returns installed verfier repo directory for Tempest + """ + if not self.VERIFIER_ID: + self.VERIFIER_ID = self.get_verifier_id() + + return os.path.join(CONST.dir_rally_inst, + 'verification', + 'verifier-{}'.format(self.VERIFIER_ID), + 'repo') + + def get_verifier_deployment_dir(self): + """ + Returns Rally deployment directory for current verifier + """ + if not self.VERIFIER_ID: + self.VERIFIER_ID = self.get_verifier_id() + + if not self.DEPLOYMENT_ID: + self.DEPLOYMENT_ID = self.get_verifier_deployment_id() + + return os.path.join(CONST.dir_rally_inst, + 'verification', + 'verifier-{}'.format(self.VERIFIER_ID), + 'for-deployment-{}'.format(self.DEPLOYMENT_ID)) + + @staticmethod + def read_file(filename): with open(filename) as src: return [line.strip() for line in src.readlines()] @@ -48,53 +113,52 @@ class TempestCommon(testcase_base.TestcaseBase): logger.debug("Creating tenant and user for Tempest suite") tenant_id = os_utils.create_tenant( keystone_client, - ft_constants.TEMPEST_TENANT_NAME, - ft_constants.TEMPEST_TENANT_DESCRIPTION) + CONST.tempest_identity_tenant_name, + CONST.tempest_identity_tenant_description) if not tenant_id: logger.error("Error : Failed to create %s tenant" - % ft_constants.TEMPEST_TENANT_NAME) + % CONST.tempest_identity_tenant_name) user_id = os_utils.create_user(keystone_client, - ft_constants.TEMPEST_USER_NAME, - ft_constants.TEMPEST_USER_PASSWORD, + CONST.tempest_identity_user_name, + CONST.tempest_identity_user_password, None, tenant_id) if not user_id: logger.error("Error : Failed to create %s user" % - ft_constants.TEMPEST_USER_NAME) + CONST.tempest_identity_user_name) logger.debug("Creating private network for Tempest suite") - network_dic = \ - os_utils.create_shared_network_full( - ft_constants.TEMPEST_PRIVATE_NET_NAME, - ft_constants.TEMPEST_PRIVATE_SUBNET_NAME, - ft_constants.TEMPEST_ROUTER_NAME, - ft_constants.TEMPEST_PRIVATE_SUBNET_CIDR) + network_dic = os_utils.create_shared_network_full( + CONST.tempest_private_net_name, + CONST.tempest_private_subnet_name, + CONST.tempest_router_name, + CONST.tempest_private_subnet_cidr) if not network_dic: - return releng_constants.EXIT_RUN_ERROR + return testcase_base.TestcaseBase.EX_RUN_ERROR - if ft_constants.TEMPEST_USE_CUSTOM_IMAGES: + if CONST.tempest_use_custom_images: # adding alternative image should be trivial should we need it logger.debug("Creating image for Tempest suite") _, self.IMAGE_ID = os_utils.get_or_create_image( - ft_constants.GLANCE_IMAGE_NAME, conf_utils.GLANCE_IMAGE_PATH, - ft_constants.GLANCE_IMAGE_FORMAT) + CONST.openstack_image_name, conf_utils.GLANCE_IMAGE_PATH, + CONST.openstack_image_disk_format) if not self.IMAGE_ID: - return releng_constants.EXIT_RUN_ERROR + return testcase_base.TestcaseBase.EX_RUN_ERROR - if ft_constants.TEMPEST_USE_CUSTOM_FLAVORS: + if CONST.tempest_use_custom_flavors: # adding alternative flavor should be trivial should we need it logger.debug("Creating flavor for Tempest suite") _, self.FLAVOR_ID = os_utils.get_or_create_flavor( - ft_constants.FLAVOR_NAME, - ft_constants.FLAVOR_RAM, - ft_constants.FLAVOR_DISK, - ft_constants.FLAVOR_VCPUS) + CONST.openstack_flavor_name, + CONST.openstack_flavor_ram, + CONST.openstack_flavor_disk, + CONST.openstack_flavor_vcpus) if not self.FLAVOR_ID: - return releng_constants.EXIT_RUN_ERROR + return testcase_base.TestcaseBase.EX_RUN_ERROR - return releng_constants.EXIT_OK + return testcase_base.TestcaseBase.EX_OK - def generate_test_list(self, DEPLOYMENT_DIR): + def generate_test_list(self, verifier_repo_dir): logger.debug("Generating test case list...") if self.MODE == 'defcore': shutil.copyfile( @@ -106,7 +170,7 @@ class TempestCommon(testcase_base.TestcaseBase): else: logger.error("Tempest test list file %s NOT found." % conf_utils.TEMPEST_CUSTOM) - return releng_constants.EXIT_RUN_ERROR + return testcase_base.TestcaseBase.EX_RUN_ERROR else: if self.MODE == 'smoke': testr_mode = "smoke" @@ -116,11 +180,14 @@ class TempestCommon(testcase_base.TestcaseBase): testr_mode = "" else: testr_mode = 'tempest.api.' + self.MODE - cmd = ("cd " + DEPLOYMENT_DIR + ";" + "testr list-tests " + - testr_mode + ">" + conf_utils.TEMPEST_RAW_LIST + ";cd") + cmd = ("cd {0};" + "testr list-tests {1} > {2};" + "cd -;".format(verifier_repo_dir, + testr_mode, + conf_utils.TEMPEST_RAW_LIST)) ft_utils.execute_command(cmd) - return releng_constants.EXIT_OK + return testcase_base.TestcaseBase.EX_OK def apply_tempest_blacklist(self): logger.debug("Applying tempest blacklist...") @@ -128,8 +195,8 @@ class TempestCommon(testcase_base.TestcaseBase): result_file = open(conf_utils.TEMPEST_LIST, 'w') black_tests = [] try: - installer_type = ft_constants.CI_INSTALLER_TYPE - deploy_scenario = ft_constants.CI_SCENARIO + installer_type = CONST.INSTALLER_TYPE + deploy_scenario = CONST.DEPLOY_SCENARIO if (bool(installer_type) * bool(deploy_scenario)): # if INSTALLER_TYPE and DEPLOY_SCENARIO are set we read the # file @@ -145,7 +212,7 @@ class TempestCommon(testcase_base.TestcaseBase): for test in tests: black_tests.append(test) break - except: + except Exception: black_tests = [] logger.debug("Tempest blacklist file does not exist.") @@ -156,52 +223,35 @@ class TempestCommon(testcase_base.TestcaseBase): else: result_file.write(str(cases_line) + '\n') result_file.close() - return releng_constants.EXIT_OK + return testcase_base.TestcaseBase.EX_OK - def run(self): - if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR): - os.makedirs(conf_utils.TEMPEST_RESULTS_DIR) + def _parse_verification_id(line): + first_pos = line.index("UUID=") + len("UUID=") + last_pos = line.index(") for deployment") + return line[first_pos:last_pos] - # Pre-configuration - res = self.create_tempest_resources() - if res != releng_constants.EXIT_OK: - return res + def run_verifier_tests(self): + self.OPTION += (" --load-list {}".format(conf_utils.TEMPEST_LIST)) - res = conf_utils.configure_tempest(logger, - self.DEPLOYMENT_DIR, - self.IMAGE_ID, - self.FLAVOR_ID) - if res != releng_constants.EXIT_OK: - return res - - res = self.generate_test_list(self.DEPLOYMENT_DIR) - if res != releng_constants.EXIT_OK: - return res - - res = self.apply_tempest_blacklist() - if res != releng_constants.EXIT_OK: - return res - - self.OPTION += (" --tests-file %s " % conf_utils.TEMPEST_LIST) - - cmd_line = "rally verify start " + self.OPTION + " --system-wide" + cmd_line = "rally verify start " + self.OPTION logger.info("Starting Tempest test suite: '%s'." % cmd_line) header = ("Tempest environment:\n" " Installer: %s\n Scenario: %s\n Node: %s\n Date: %s\n" % - (ft_constants.CI_INSTALLER_TYPE, - ft_constants.CI_SCENARIO, - ft_constants.CI_NODE, + (CONST.INSTALLER_TYPE, + CONST.DEPLOY_SCENARIO, + CONST.NODE_NAME, time.strftime("%a %b %d %H:%M:%S %Z %Y"))) - f_stdout = open(conf_utils.TEMPEST_RESULTS_DIR + "/tempest.log", 'w+') + f_stdout = open( + os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'w+') f_stderr = open( - conf_utils.TEMPEST_RESULTS_DIR + "/tempest-error.log", 'w+') - f_env = open(conf_utils.TEMPEST_RESULTS_DIR + "/environment.log", 'w+') + os.path.join(conf_utils.TEMPEST_RESULTS_DIR, + "tempest-error.log"), 'w+') + f_env = open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR, + "environment.log"), 'w+') f_env.write(header) - # subprocess.call(cmd_line, shell=True, - # stdout=f_stdout, stderr=f_stderr) p = subprocess.Popen( cmd_line, shell=True, stdout=subprocess.PIPE, @@ -212,6 +262,12 @@ class TempestCommon(testcase_base.TestcaseBase): for line in iter(p.stdout.readline, b''): if re.search("\} tempest\.", line): logger.info(line.replace('\n', '')) + elif re.search('Starting verification', line): + logger.info(line.replace('\n', '')) + first_pos = line.index("UUID=") + len("UUID=") + last_pos = line.index(") for deployment") + self.VERIFICATION_ID = line[first_pos:last_pos] + logger.debug('Verication UUID: %s' % self.VERIFICATION_ID) f_stdout.write(line) p.wait() @@ -219,56 +275,76 @@ class TempestCommon(testcase_base.TestcaseBase): f_stderr.close() f_env.close() - cmd_line = "rally verify show" - output = "" + def parse_verifier_result(self): + if not self.VERIFICATION_ID: + raise Exception('Verification UUID not found') + + cmd_line = "rally verify show --uuid {}".format(self.VERIFICATION_ID) + logger.info("Showing result for a verification: '%s'." % cmd_line) p = subprocess.Popen(cmd_line, shell=True, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + stderr=subprocess.STDOUT) for line in p.stdout: - if re.search("Tests\:", line): + new_line = line.replace(' ', '').split('|') + if 'Tests' in new_line: break - output += line - logger.info(output) - - cmd_line = "rally verify list" - cmd = os.popen(cmd_line) - output = (((cmd.read()).splitlines()[-2]).replace(" ", "")).split("|") - # Format: - # | UUID | Deployment UUID | smoke | tests | failures | Created at | - # Duration | Status | - num_tests = output[4] - num_failures = output[5] - duration = output[7] - # Compute duration (lets assume it does not take more than 60 min) - dur_min = int(duration.split(':')[1]) - dur_sec_float = float(duration.split(':')[2]) - dur_sec_int = int(round(dur_sec_float, 0)) - dur_sec_int = dur_sec_int + 60 * dur_min + + logger.info(line) + if 'Testscount' in new_line: + num_tests = new_line[2] + elif 'Success' in new_line: + num_success = new_line[2] + elif 'Skipped' in new_line: + num_skipped = new_line[2] try: - diff = (int(num_tests) - int(num_failures)) - success_rate = 100 * diff / int(num_tests) - except: + num_executed = int(num_tests) - int(num_skipped) + success_rate = 100 * int(num_success) / int(num_executed) + except Exception: success_rate = 0 - if 'smoke' in self.MODE: - self.CASE_NAME = 'tempest_smoke_serial' - elif 'feature' in self.MODE: - self.CASE_NAME = self.MODE.replace( - "feature_", "") - else: - self.CASE_NAME = 'tempest_full_parallel' - - status = ft_utils.check_success_rate( - self.CASE_NAME, success_rate) + self.criteria = ft_utils.check_success_rate( + self.case_name, success_rate) logger.info("Tempest %s success_rate is %s%%, is marked as %s" - % (self.CASE_NAME, success_rate, status)) + % (self.case_name, success_rate, self.criteria)) + + def run(self): + + self.start_time = time.time() + + if not os.path.exists(conf_utils.TEMPEST_RESULTS_DIR): + os.makedirs(conf_utils.TEMPEST_RESULTS_DIR) + + # Pre-configuration + res = self.create_tempest_resources() + if res != testcase_base.TestcaseBase.EX_OK: + return res + + res = conf_utils.configure_tempest(logger, + self.DEPLOYMENT_DIR, + self.IMAGE_ID, + self.FLAVOR_ID) + if res != testcase_base.TestcaseBase.EX_OK: + return res + + res = self.generate_test_list(self.VERIFIER_REPO_DIR) + if res != testcase_base.TestcaseBase.EX_OK: + return res + + res = self.apply_tempest_blacklist() + if res != testcase_base.TestcaseBase.EX_OK: + return res + + self.run_verifier_tests() + self.parse_verifier_result() + + self.stop_time = time.time() - if status == "PASS": - return releng_constants.EXIT_OK + if self.criteria == "PASS": + return testcase_base.TestcaseBase.EX_OK else: - return releng_constants.EXIT_RUN_ERROR + return testcase_base.TestcaseBase.EX_TESTCASE_FAILED class TempestSmokeSerial(TempestCommon): @@ -277,7 +353,7 @@ class TempestSmokeSerial(TempestCommon): TempestCommon.__init__(self) self.case_name = "tempest_smoke_serial" self.MODE = "smoke" - self.OPTION = "--concur 1" + self.OPTION = "--concurrency 1" class TempestSmokeParallel(TempestCommon): @@ -303,7 +379,7 @@ class TempestMultisite(TempestCommon): TempestCommon.__init__(self) self.case_name = "multisite" self.MODE = "feature_multisite" - self.OPTION = "--concur 1" + self.OPTION = "--concurrency 1" conf_utils.configure_tempest_multisite(logger, self.DEPLOYMENT_DIR) |