From 75dd45462f46c711f950c96aac783932cb7ec1a4 Mon Sep 17 00:00:00 2001 From: xudan Date: Tue, 8 Nov 2016 11:43:28 +0000 Subject: dovetail tool: package all logger initialization 1. For container.py, parser.py, report.py and testcase.py, they use functions in classes to create loggers. So each class will have its own logger. 2. run.py will create all the loggers by create_logs() function before the use of them. So logs will not be initialized until the call of create_logs(). 3. Move logger in run.py into function main(). JIRA: DOVETAIL-56 Change-Id: I53de090df270cc7a34762ff99cbc9115cfb09465 Signed-off-by: xudan --- dovetail/container.py | 30 ++++++++++++-------- dovetail/parser.py | 19 ++++++++----- dovetail/report.py | 78 ++++++++++++++++++++++++++++++++++----------------- dovetail/run.py | 32 +++++++++++++++------ dovetail/testcase.py | 20 +++++++++---- 5 files changed, 122 insertions(+), 57 deletions(-) diff --git a/dovetail/container.py b/dovetail/container.py index 15ccc800..2716a089 100644 --- a/dovetail/container.py +++ b/dovetail/container.py @@ -11,7 +11,6 @@ import utils.dovetail_logger as dt_logger import utils.dovetail_utils as dt_utils from conf.dovetail_config import DovetailConfig as dt_config -logger = dt_logger.Logger('container.py').getLogger() class Container: @@ -19,12 +18,18 @@ class Container: container_list = {} has_pull_latest_image = {'yardstick': False, 'functest': False} + logger = None + def __init__(cls): pass def __str__(cls): pass + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.Container').getLogger() + @classmethod def get(cls, type): return cls.container_list[type] @@ -46,10 +51,10 @@ class Container: dovetail_config[type]['result']['dir']) cmd = 'sudo docker run %s %s %s %s %s /bin/bash' % \ (opts, envs, sshkey, result_volume, docker_image) - dt_utils.exec_cmd(cmd, logger) + dt_utils.exec_cmd(cmd, cls.logger) ret, container_id = \ dt_utils.exec_cmd("sudo docker ps | grep " + docker_image + - " | awk '{print $1}' | head -1", logger) + " | awk '{print $1}' | head -1", cls.logger) cls.container_list[type] = container_id return container_id @@ -57,20 +62,21 @@ class Container: def pull_image(cls, type): docker_image = cls.get_docker_image(type) if cls.has_pull_latest_image[type] is True: - logger.debug('%s is already the newest version.' % (docker_image)) + cls.logger.debug('%s is already the newest version.' % + (docker_image)) else: cmd = 'sudo docker pull %s' % (docker_image) - dt_utils.exec_cmd(cmd, logger) + dt_utils.exec_cmd(cmd, cls.logger) cls.has_pull_latest_image[type] = True - @staticmethod - def clean(container_id): + @classmethod + def clean(cls, container_id): cmd1 = 'sudo docker stop %s' % (container_id) - dt_utils.exec_cmd(cmd1, logger) + dt_utils.exec_cmd(cmd1, cls.logger) cmd2 = 'sudo docker rm %s' % (container_id) - dt_utils.exec_cmd(cmd2, logger) + dt_utils.exec_cmd(cmd2, cls.logger) - @staticmethod - def exec_cmd(container_id, sub_cmd, exit_on_error=False): + @classmethod + def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False): cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd) - dt_utils.exec_cmd(cmd, logger, exit_on_error) + dt_utils.exec_cmd(cmd, cls.logger, exit_on_error) diff --git a/dovetail/parser.py b/dovetail/parser.py index 1740944a..d8f9fa0a 100644 --- a/dovetail/parser.py +++ b/dovetail/parser.py @@ -14,28 +14,33 @@ import utils.dovetail_logger as dt_logger import utils.dovetail_utils as dt_utils from conf.dovetail_config import DovetailConfig as dt_config -logger = dt_logger.Logger('parser.py').getLogger() - class Parser: '''preprocess configuration files''' - @staticmethod - def parse_cmd(cmd, testcase): + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.Parser').getLogger() + + @classmethod + def parse_cmd(cls, cmd, testcase): cmd_lines = None try: template = jinja2.Template(cmd, undefined=jinja2.StrictUndefined) kwargs = {} for arg in dt_config.dovetail_config['parameters']: path = eval(arg['path']) - logger.debug('name: %s, eval path: %s ' % (arg['name'], path)) + cls.logger.debug('name: %s, eval path: %s ' % + (arg['name'], path)) kwargs[arg['name']] = \ dt_utils.get_obj_by_path(testcase.testcase, path) - logger.debug('kwargs: %s' % kwargs) + cls.logger.debug('kwargs: %s' % kwargs) cmd_lines = template.render(**kwargs) except Exception as e: - logger.error('failed to parse cmd %s, exception:%s' % (cmd, e)) + cls.logger.error('failed to parse cmd %s, exception:%s' % (cmd, e)) return None return cmd_lines diff --git a/dovetail/report.py b/dovetail/report.py index 2e544ad3..52c8ebf9 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -16,8 +16,6 @@ import utils.dovetail_logger as dt_logger from conf.dovetail_config import DovetailConfig as dt_config from testcase import Testcase -logger = dt_logger.Logger('report.py').getLogger() - def get_pass_str(passed): if passed: @@ -30,6 +28,12 @@ class Report: results = {'functest': {}, 'yardstick': {}} + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.Report').getLogger() + @staticmethod def check_result(testcase, db_result): checker = CheckerFactory.create(testcase.script_type()) @@ -66,7 +70,7 @@ class Report: testcase.sub_testcase_passed(sub_test)) }) report_obj['testcases_list'].append(testcase_in_rpt) - logger.info(json.dumps(report_obj)) + cls.logger.info(json.dumps(report_obj)) return report_obj @classmethod @@ -91,21 +95,21 @@ class Report: (sub_test['name'], sub_test['result']) rpt_text += split_line - logger.info(rpt_text) + cls.logger.info(rpt_text) cls.save(rpt_text) return rpt_text # save to disk as default - @staticmethod - def save(report): + @classmethod + def save(cls, report): report_file_name = dt_config.dovetail_config['report_file'] try: with open(os.path.join(dt_config.dovetail_config['result_dir'], report_file_name), 'w') as report_file: report_file.write(report) - logger.info('save report to %s' % report_file_name) + cls.logger.info('save report to %s' % report_file_name) except Exception: - logger.error('Failed to save: %s' % report_file_name) + cls.logger.error('Failed to save: %s' % report_file_name) @classmethod def get_result(cls, testcase): @@ -121,11 +125,12 @@ class Report: if result is not None: cls.results[type][script_testcase] = result testcase.script_result_acquired(True) - logger.debug('testcase: %s -> result acquired' % script_testcase) + cls.logger.debug('testcase: %s -> result acquired' % + script_testcase) else: retry = testcase.increase_retry() - logger.debug('testcase: %s -> result acquired retry:%d' % - (script_testcase, retry)) + cls.logger.debug('testcase: %s -> result acquired retry:%d' % + (script_testcase, retry)) return result @@ -144,9 +149,15 @@ class CrawlerFactory: class FunctestCrawler: + logger = None + def __init__(self): self.type = 'functest' + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.FunctestCrawler').getLogger() + def crawl(self, testcase=None): store_type = \ dt_config.dovetail_config[self.type]['result']['store_type'] @@ -162,7 +173,7 @@ class FunctestCrawler: os.path.join(dovetail_config['result_dir'], dovetail_config[self.type]['result']['file_path']) if not os.path.exists(file_path): - logger.info('result file not found: %s' % file_path) + self.logger.info('result file not found: %s' % file_path) return None try: @@ -184,31 +195,37 @@ class FunctestCrawler: "duration": int(dur_sec_int), "tests": int(num_tests), "failures": failed_num, "errors": error_logs}} - logger.debug('Results: %s' % str(json_results)) + self.logger.debug('Results: %s' % str(json_results)) return json_results except Exception as e: - logger.error('Cannot read content from the file: %s, exception: %s' - % (file_path, e)) + self.logger.error('Cannot read content from the file: %s, ' + 'exception: %s' % (file_path, e)) return None def crawl_from_url(self, testcase=None): url = \ dt_config.dovetail_config[self.type]['result']['db_url'] % testcase - logger.debug("Query to rest api: %s" % url) + self.logger.debug("Query to rest api: %s" % url) try: data = json.load(urllib2.urlopen(url)) return data['results'][0] except Exception as e: - logger.error("Cannot read content from the url: %s, exception: %s" - % (url, e)) + self.logger.error("Cannot read content from the url: %s, " + "exception: %s" % (url, e)) return None class YardstickCrawler: + logger = None + def __init__(self): self.type = 'yardstick' + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.YardstickCrawler').getLogger() + def crawl(self, testcase=None): store_type = \ dt_config.dovetail_config[self.type]['result']['store_type'] @@ -222,18 +239,18 @@ class YardstickCrawler: file_path = os.path.join(dt_config.dovetail_config['result_dir'], testcase+'.out') if not os.path.exists(file_path): - logger.info('result file not found: %s' % file_path) + self.logger.info('result file not found: %s' % file_path) return None try: with open(file_path, 'r') as myfile: myfile.read() criteria = 'PASS' json_results = {'criteria': criteria} - logger.debug('Results: %s' % str(json_results)) + self.logger.debug('Results: %s' % str(json_results)) return json_results except Exception as e: - logger.error('Cannot read content from the file: %s, exception: %s' - % (file_path, e)) + self.logger.error('Cannot read content from the file: %s, ' + 'exception: %s' % (file_path, e)) return None def crawl_from_url(self, testcase=None): @@ -262,8 +279,13 @@ class ResultChecker: class FunctestChecker: - @staticmethod - def check(testcase, db_result): + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.FunctestChecker').getLogger() + + def check(self, testcase, db_result): sub_testcase_list = testcase.sub_testcase() if not db_result: @@ -284,7 +306,7 @@ class FunctestChecker: all_passed = True for sub_testcase in sub_testcase_list: - logger.debug('check sub_testcase:%s' % sub_testcase) + self.logger.debug('check sub_testcase:%s' % sub_testcase) if sub_testcase in db_result['details']['errors']: testcase.sub_testcase_passed(sub_testcase, False) all_passed = False @@ -296,6 +318,12 @@ class FunctestChecker: class YardstickChecker: + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.YardstickChecker').getLogger() + @staticmethod def check(testcase, result): if not result: diff --git a/dovetail/run.py b/dovetail/run.py index fcc9da1a..cc0e7940 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -13,15 +13,17 @@ import sys import utils.dovetail_logger as dt_logger - +from parser import Parser from container import Container from testcase import Testcase from testcase import Testsuite from report import Report +from report import FunctestCrawler +from report import YardstickCrawler +from report import FunctestChecker +from report import YardstickChecker from conf.dovetail_config import DovetailConfig as dt_config -logger = dt_logger.Logger('run.py').getLogger() - def load_testsuite(testsuite): Testsuite.load() @@ -40,7 +42,7 @@ def load_testcase(): Testcase.load() -def run_test(testsuite, testarea): +def run_test(testsuite, testarea, logger): testarea_list = [] for value in testsuite['testcases_list']: if value is not None and (testarea == 'full' or testarea in value): @@ -50,7 +52,7 @@ def run_test(testsuite, testarea): logger.info('>>[testcase]: %s' % (testcase_name)) testcase = Testcase.get(testcase_name) if testcase is None: - logger.error('testcase %s is not defined in testcase folder, \ + logger.error('test case %s is not defined in testcase folder, \ skipping' % (testcase_name)) continue run_testcase = True @@ -87,7 +89,7 @@ def run_test(testsuite, testarea): Report.check_result(testcase, db_result) -def validate_options(input_dict): +def validate_options(input_dict, logger): # for 'tag' option for key, value in input_dict.items(): if key == 'tag' and value is not None: @@ -106,12 +108,26 @@ def filter_env_options(input_dict): return envs_options +def create_logs(): + Container.create_log() + Parser.create_log() + Report.create_log() + FunctestCrawler.create_log() + YardstickCrawler.create_log() + FunctestChecker.create_log() + YardstickChecker.create_log() + Testcase.create_log() + Testsuite.create_log() + + def main(*args, **kwargs): """Dovetail compliance test entry!""" + create_logs() + logger = dt_logger.Logger('run').getLogger() logger.info('================================================') logger.info('Dovetail compliance: %s!' % (kwargs['testsuite'])) logger.info('================================================') - validate_options(kwargs) + validate_options(kwargs, logger) envs_options = filter_env_options(kwargs) dt_config.update_envs(envs_options) logger.info('Your new envs for functest: %s' % @@ -132,7 +148,7 @@ def main(*args, **kwargs): if kwargs['testsuite'] in dt_config.testsuite_supported: testsuite_validation = True if testsuite_validation and testarea_validation: - run_test(testsuite_yaml, testarea) + run_test(testsuite_yaml, testarea, logger) Report.generate(testsuite_yaml, testarea) else: logger.error('invalid input commands, testsuite %s testarea %s' % diff --git a/dovetail/testcase.py b/dovetail/testcase.py index b5d28aa1..60ce2b28 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -15,11 +15,11 @@ import utils.dovetail_logger as dt_logger from parser import Parser from conf.dovetail_config import DovetailConfig as dt_config -logger = dt_logger.Logger('testcase.py').getLogger() - class Testcase: + logger = None + def __init__(self, testcase_yaml): self.testcase = testcase_yaml.values()[0] self.testcase['passed'] = False @@ -28,6 +28,10 @@ class Testcase: self.update_script_testcase(self.script_type(), self.script_testcase()) + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.Testcase').getLogger() + def prepare_cmd(self): script_type = self.script_type() for cmd in dt_config.dovetail_config[script_type]['testcase']['cmds']: @@ -52,7 +56,7 @@ class Testcase: def sub_testcase_passed(self, name, passed=None): if passed is not None: - logger.debug('sub_testcase_passed:%s %s' % (name, passed)) + self.logger.debug('sub_testcase_passed:%s %s' % (name, passed)) self.sub_testcase_status[name] = passed return self.sub_testcase_status[name] @@ -146,7 +150,7 @@ class Testcase: testcase_yaml = yaml.safe_load(f) cls.testcase_list[testcase_yaml.keys()[0]] = \ cls(testcase_yaml) - logger.debug(cls.testcase_list) + cls.logger.debug(cls.testcase_list) @classmethod def get(cls, testcase_name): @@ -157,10 +161,16 @@ class Testcase: class Testsuite: + logger = None + def __init__(self, testsuite): self.testsuite = testsuite self.testcase_list = {} + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.Testsuite').getLogger() + def get_test(self, testcase_name): if testcase_name in self.testcase_list: return self.testcase_list[testcase_name] @@ -176,7 +186,7 @@ class Testsuite: testsuite_yaml = yaml.safe_load(f) cls.testsuite_list.update(testsuite_yaml) - logger.debug(cls.testsuite_list) + cls.logger.debug(cls.testsuite_list) @classmethod def get(cls, testsuite_name): -- cgit 1.2.3-korg