diff options
Diffstat (limited to 'functest/core')
| -rw-r--r-- | functest/core/feature.py | 22 | ||||
| -rw-r--r-- | functest/core/pytest_suite_runner.py | 4 | ||||
| -rw-r--r-- | functest/core/testcase.py | 57 | ||||
| -rw-r--r-- | functest/core/vnf_base.py | 113 |
4 files changed, 97 insertions, 99 deletions
diff --git a/functest/core/feature.py b/functest/core/feature.py index 00c7ec74..d65f5a3c 100644 --- a/functest/core/feature.py +++ b/functest/core/feature.py @@ -25,7 +25,7 @@ __author__ = ("Serena Feng <feng.xiaowei@zte.com.cn>, " class Feature(base.TestCase): - """Parent class of Functest Feature.""" + """Base model for single feature.""" def __init__(self, **kwargs): super(Feature, self).__init__(**kwargs) @@ -34,11 +34,13 @@ class Feature(base.TestCase): self.logger = ft_logger.Logger(self.project_name).getLogger() def execute(self, **kwargs): - """Execute Feature. + """Execute the Python method. The subclasses must override the default implementation which - is false on purpose. The only prerequisite is to return 0 if - success or anything else if failure. + is false on purpose. + + The new implementation must return 0 if success or anything + else if failure. Args: kwargs: Arbitrary keyword arguments. @@ -50,14 +52,14 @@ class Feature(base.TestCase): return -1 def run(self, **kwargs): - """Run Feature. + """Run the feature. It allows executing any Python method by calling execute(). It sets the following attributes required to push the results to DB: - * criteria, + * result, * start_time, * stop_time. @@ -72,15 +74,15 @@ class Feature(base.TestCase): """ self.start_time = time.time() exit_code = base.TestCase.EX_RUN_ERROR - self.criteria = "FAIL" + self.result = 0 try: if self.execute(**kwargs) == 0: exit_code = base.TestCase.EX_OK - self.criteria = 'PASS' + self.result = 100 ft_utils.logger_test_results( self.project_name, self.case_name, - self.criteria, self.details) - self.logger.info("%s %s", self.project_name, self.criteria) + self.result, self.details) + self.logger.info("%s %s", self.project_name, self.result) except Exception: # pylint: disable=broad-except self.logger.exception("%s FAILED", self.project_name) self.logger.info("Test result is stored in '%s'", self.result_file) diff --git a/functest/core/pytest_suite_runner.py b/functest/core/pytest_suite_runner.py index 775f0a66..8b5da05e 100644 --- a/functest/core/pytest_suite_runner.py +++ b/functest/core/pytest_suite_runner.py @@ -48,10 +48,10 @@ class PyTestSuiteRunner(base.TestCase): if ((result.errors and len(result.errors) > 0) or (result.failures and len(result.failures) > 0)): self.logger.info("%s FAILED" % self.case_name) - self.criteria = 'FAIL' + self.result = 'FAIL' else: self.logger.info("%s OK" % self.case_name) - self.criteria = 'PASS' + self.result = 'PASS' self.details = {} return exit_code diff --git a/functest/core/testcase.py b/functest/core/testcase.py index 472d847a..3f191b40 100644 --- a/functest/core/testcase.py +++ b/functest/core/testcase.py @@ -18,19 +18,19 @@ __author__ = "Cedric Ollivier <cedric.ollivier@orange.com>" class TestCase(object): - """Parent class of Functest TestCase.""" + """Base model for single test case.""" EX_OK = os.EX_OK - """Status code returned when everything is OK""" + """everything is OK""" EX_RUN_ERROR = os.EX_SOFTWARE - """Status code returned when run() fails""" + """run() failed""" EX_PUSH_TO_DB_ERROR = os.EX_SOFTWARE - 1 - """Status code returned when push_to_db() fails""" + """push_to_db() failed""" EX_TESTCASE_FAILED = os.EX_SOFTWARE - 2 - """Status code returned when results are false""" + """results are false""" logger = ft_logger.Logger(__name__).getLogger() @@ -38,42 +38,54 @@ class TestCase(object): self.details = {} self.project_name = kwargs.get('project_name', 'functest') self.case_name = kwargs.get('case_name', '') - self.criteria = "" + self.criteria = kwargs.get('criteria', 100) + self.result = "" self.start_time = "" self.stop_time = "" - def check_criteria(self): - """Interpret the results of TestCase. + def check_result(self): + """Interpret the result of the test case. - It allows getting the results of TestCase. It completes run() + It allows getting the result of TestCase. It completes run() which only returns the execution status. - It can be overriden if checking criteria is not suitable. + It can be overriden if checking result is not suitable. Returns: - TestCase.EX_OK if criteria is 'PASS'. + TestCase.EX_OK if result is 'PASS'. TestCase.EX_TESTCASE_FAILED otherwise. """ try: assert self.criteria - if self.criteria == 'PASS': - return TestCase.EX_OK + if isinstance(self.result, int) and isinstance(self.criteria, int): + if self.result >= self.criteria: + return TestCase.EX_OK + else: + # Backward compatibility + # It must be removed as soon as TestCase subclasses + # stop setting result = 'PASS' or 'FAIL'. + # In this case criteria is unread. + self.logger.warning( + "Please update result which must be an int!") + if self.result == 'PASS': + return TestCase.EX_OK except AssertionError: self.logger.error("Please run test before checking the results") return TestCase.EX_TESTCASE_FAILED def run(self, **kwargs): - """Run TestCase. + """Run the test case. It allows running TestCase and getting its execution status. The subclasses must override the default implementation which - is false on purpose. The only prerequisite is to set the - following attributes to push the results to DB: + is false on purpose. - * case_name, - * criteria, + The new implementation must set the following attributes to + push the results to DB: + + * result, * start_time, * stop_time. @@ -88,7 +100,7 @@ class TestCase(object): return TestCase.EX_RUN_ERROR def push_to_db(self): - """Push the results of TestCase to the DB. + """Push the results of the test case to the DB. It allows publishing the results and to check the status. @@ -98,7 +110,7 @@ class TestCase(object): * project_name, * case_name, - * criteria, + * result, * start_time, * stop_time. @@ -109,12 +121,13 @@ class TestCase(object): try: assert self.project_name assert self.case_name - assert self.criteria assert self.start_time assert self.stop_time + pub_result = 'PASS' if self.check_result( + ) == TestCase.EX_OK else 'FAIL' if ft_utils.push_results_to_db( self.project_name, self.case_name, self.start_time, - self.stop_time, self.criteria, self.details): + self.stop_time, pub_result, self.details): self.logger.info("The results were successfully pushed to DB") return TestCase.EX_OK else: diff --git a/functest/core/vnf_base.py b/functest/core/vnf_base.py index 6bd6e431..fe4e427f 100644 --- a/functest/core/vnf_base.py +++ b/functest/core/vnf_base.py @@ -27,6 +27,12 @@ class VnfOnBoardingBase(base.TestCase): self.cmd = kwargs.get('cmd', '') self.details = {} self.result_dir = CONST.dir_results + self.details_step_mapping = dict( + deploy_orchestrator='orchestrator', + deploy_vnf='vnf', + test_vnf='test_vnf', + prepare='prepare_env') + self.details['prepare_env'] = {} self.details['orchestrator'] = {} self.details['vnf'] = {} self.details['test_vnf'] = {} @@ -118,48 +124,45 @@ class VnfOnBoardingBase(base.TestCase): self.logger.info("Prepare OpenStack plateform(create tenant and user)") admin_user_id = os_utils.get_user_id(self.keystone_client, self.creds['username']) - if admin_user_id == '': - self.step_failure("Failed to get id of " + - self.creds['username']) + if not admin_user_id: + self.step_failure("Failed to get id of {0}".format( + self.creds['username'])) - tenant_id = os_utils.create_tenant( - self.keystone_client, self.tenant_name, self.tenant_description) + tenant_id = os_utils.get_tenant_id(self.keystone_client, + self.tenant_name) if not tenant_id: - self.step_failure("Failed to create " + - self.tenant_name + " tenant") - - roles_name = ["admin", "Admin"] - role_id = '' - for role_name in roles_name: - if role_id == '': - role_id = os_utils.get_role_id(self.keystone_client, role_name) - - if role_id == '': - self.logger.error("Failed to get id for %s role" % role_name) - self.step_failure("Failed to get role id of " + role_name) - - if not os_utils.add_role_user(self.keystone_client, admin_user_id, - role_id, tenant_id): - self.logger.error("Failed to add %s on tenant" % - self.creds['username']) - self.step_failure("Failed to add %s on tenant" % - self.creds['username']) - - user_id = os_utils.create_user(self.keystone_client, - self.tenant_name, - self.tenant_name, - None, - tenant_id) + tenant_id = os_utils.create_tenant(self.keystone_client, + self.tenant_name, + self.tenant_description) + if not tenant_id: + self.step_failure("Failed to get or create {0} tenant".format( + self.tenant_name)) + roles_name = ["admin", "Admin"] + role_id = '' + for role_name in roles_name: + if not role_id: + role_id = os_utils.get_role_id(self.keystone_client, + role_name) + + if not role_id: + self.step_failure("Failed to get id for {0} role".format( + role_name)) + + if not os_utils.add_role_user(self.keystone_client, admin_user_id, + role_id, tenant_id): + self.step_failure("Failed to add {0} on tenant".format( + self.creds['username'])) + + user_id = os_utils.get_or_create_user(self.keystone_client, + self.tenant_name, + self.tenant_name, + tenant_id) if not user_id: - self.logger.error("Failed to create %s user" % self.tenant_name) - self.step_failure("Failed to create user ") + self.step_failure("Failed to get or create {0} user".format( + self.tenant_name)) - if not os_utils.add_role_user(self.keystone_client, user_id, - role_id, tenant_id): - self.logger.error("Failed to add %s on tenant" % - self.tenant_name) - self.step_failure("Failed to add %s on tenant" % - self.tenant_name) + os_utils.add_role_user(self.keystone_client, user_id, + role_id, tenant_id) self.logger.info("Update OpenStack creds informations") self.admin_creds = self.creds.copy() @@ -187,53 +190,33 @@ class VnfOnBoardingBase(base.TestCase): self.logger.error("VNF must be tested") raise Exception("VNF not tested") + # clean before openstack clean run def clean(self): self.logger.info("test cleaning") - self.logger.info("Removing %s tenant .." % self.tenant_name) - tenant_id = os_utils.get_tenant_id(self.keystone_client, - self.tenant_name) - if tenant_id == '': - self.logger.error("Error : Failed to get id of %s tenant" % - self.tenant_name) - else: - if not os_utils.delete_tenant(self.keystone_client, tenant_id): - self.logger.error("Error : Failed to remove %s tenant" % - self.tenant_name) - - self.logger.info("Removing %s user .." % self.tenant_name) - user_id = os_utils.get_user_id( - self.keystone_client, self.tenant_name) - if user_id == '': - self.logger.error("Error : Failed to get id of %s user" % - self.tenant_name) - else: - if not os_utils.delete_user(self.keystone_client, user_id): - self.logger.error("Error : Failed to remove %s user" % - self.tenant_name) - def parse_results(self): exit_code = self.EX_OK - self.criteria = "PASS" + self.result = "PASS" self.logger.info(self.details) # The 2 VNF steps must be OK to get a PASS result if (self.details['vnf']['status'] is not "PASS" or self.details['test_vnf']['status'] is not "PASS"): exit_code = self.EX_RUN_ERROR - self.criteria = "FAIL" + self.result = "FAIL" return exit_code def log_results(self): ft_utils.logger_test_results(self.project_name, self.case_name, - self.criteria, + self.result, self.details) def step_failure(self, error_msg): part = inspect.stack()[1][3] - self.logger.error("Step '%s' failed: %s", part, error_msg) + self.logger.error("Step {0} failed: {1}".format(part, error_msg)) try: - part_info = self.details[part] + step_name = self.details_step_mapping[part] + part_info = self.details[step_name] except KeyError: self.details[part] = {} part_info = self.details[part] |