diff options
Diffstat (limited to 'dovetail')
-rw-r--r-- | dovetail/compliance/compliance_set.yml (renamed from dovetail/cert/compliance_set.yml) | 4 | ||||
-rw-r--r-- | dovetail/compliance/debug.yml | 9 | ||||
-rw-r--r-- | dovetail/compliance/proposed_tests.yml (renamed from dovetail/cert/proposed_tests.yml) | 4 | ||||
-rw-r--r-- | dovetail/conf/cmd_config.yml | 18 | ||||
-rw-r--r-- | dovetail/conf/dovetail_config.py | 16 | ||||
-rw-r--r-- | dovetail/conf/dovetail_config.yml | 2 | ||||
-rw-r--r-- | dovetail/container.py | 30 | ||||
-rw-r--r-- | dovetail/parser.py | 19 | ||||
-rw-r--r-- | dovetail/report.py | 221 | ||||
-rwxr-xr-x | dovetail/run.py | 93 | ||||
-rw-r--r-- | dovetail/testcase.py | 44 | ||||
-rw-r--r-- | dovetail/tests/unit/test_parser.py | 3 | ||||
-rw-r--r-- | dovetail/utils/dovetail_logger.py | 14 |
13 files changed, 322 insertions, 155 deletions
diff --git a/dovetail/cert/compliance_set.yml b/dovetail/compliance/compliance_set.yml index b4108b4f..df0d427d 100644 --- a/dovetail/cert/compliance_set.yml +++ b/dovetail/compliance/compliance_set.yml @@ -1,5 +1,5 @@ -certification_compliance_set: - name: certification_compliance_set +compliance_set: + name: compliance_set testcases_list: # Temporarily, one test case kept here as default to run # for use of software development/debug diff --git a/dovetail/compliance/debug.yml b/dovetail/compliance/debug.yml new file mode 100644 index 00000000..a0a2d3ee --- /dev/null +++ b/dovetail/compliance/debug.yml @@ -0,0 +1,9 @@ +# only used for dovetail tool development debug +# nfvi.tc001 tc002 running time is shorter, about 3 minutes +# ipv6.tc001 about 20 minutes +debug: + name: debug + testcases_list: + - dovetail.ipv6.tc001 + - dovetail.nfvi.tc001 + - dovetail.nfvi.tc002 diff --git a/dovetail/cert/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml index 6d6c8d19..fcb7e03a 100644 --- a/dovetail/cert/proposed_tests.yml +++ b/dovetail/compliance/proposed_tests.yml @@ -1,5 +1,5 @@ -certification_proposed_tests: - name: certification_proposed_tests +proposed_tests: + name: proposed_tests testcases_list: # TO DO: will adjust the dovetail tool to support in later patches # run.py --name1 {**/proposed/compliance} --name2 {**/vim/ipv6,etc} diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml index c2108c58..4e3d0110 100644 --- a/dovetail/conf/cmd_config.yml +++ b/dovetail/conf/cmd_config.yml @@ -31,18 +31,22 @@ cli: - '--DEPLOY_TYPE' - '-T' help: 'DEPLOY_TYPE of the system under test (SUT).' - CI_DEBUG: + DEBUG: flags: - - '--CI_DEBUG' + - '--DEBUG' - '-d' - help: 'CI_DEBUG for showing debug log.' + help: 'DEBUG for showing debug log.' non-envs: - scenario: + testsuite: flags: - - '--scenario' - - '-s' + - '--testsuite' default: 'compliance_set' - help: 'certification scenario.' + help: 'compliance testsuite.' + testarea: + flags: + - '--testarea' + default: 'full' + help: 'compliance testarea within testsuite' tag: flags: - '--tag' diff --git a/dovetail/conf/dovetail_config.py b/dovetail/conf/dovetail_config.py index 5ac23c43..e5d0608e 100644 --- a/dovetail/conf/dovetail_config.py +++ b/dovetail/conf/dovetail_config.py @@ -14,9 +14,12 @@ import re class DovetailConfig: - CERT_PATH = './cert/' + COMPLIANCE_PATH = './compliance/' TESTCASE_PATH = './testcase/' - SCENARIO_NAMING_FMT = 'certification_%s' + # testsuite supported tuple, should adjust accordingly + testsuite_supported = ('compliance_set', 'proposed_tests', 'debug') + # testarea supported tuple, should adjust accordingly + testarea_supported = ('vimops', 'nfvi', 'ipv6') curr_path = os.path.dirname(os.path.abspath(__file__)) @@ -44,14 +47,17 @@ class DovetailConfig: @classmethod def update_envs(cls, options): for item in options: - if options[item] is not None: - key = cls.cmd_name_trans(item) - os.environ[key] = options[item] + key = cls.cmd_name_trans(item) + if not options[item] and key in os.environ: + options[item] = os.environ[key] + if options[item]: cls.update_config_envs('functest', key) cls.update_config_envs('yardstick', key) @classmethod def update_config_envs(cls, script_type, key): + if key == 'DEBUG': + os.environ['CI_DEBUG'] = os.environ[key] envs = cls.dovetail_config[script_type]['envs'] old_value = re.findall(r'\s+%s=(.*?)(\s+|$)' % key, envs) if old_value == []: diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml index 1f5de672..c4131a91 100644 --- a/dovetail/conf/dovetail_config.yml +++ b/dovetail/conf/dovetail_config.yml @@ -3,6 +3,8 @@ work_dir: /home/opnfv/dovetail result_dir: /home/opnfv/dovetail/results report_file: 'dovetail_report.txt' cli_file_name: 'cmd_config.yml' +# TO DO: once version scheme settled, adjust accordingly +repo: 'https://github.com/opnfv/dovetail/tree/master/' # used for testcase cmd template in jinja2 format # we have two variables available now diff --git a/dovetail/container.py b/dovetail/container.py index 15ccc800..2716a089 100644 --- a/dovetail/container.py +++ b/dovetail/container.py @@ -11,7 +11,6 @@ import utils.dovetail_logger as dt_logger import utils.dovetail_utils as dt_utils from conf.dovetail_config import DovetailConfig as dt_config -logger = dt_logger.Logger('container.py').getLogger() class Container: @@ -19,6 +18,8 @@ class Container: container_list = {} has_pull_latest_image = {'yardstick': False, 'functest': False} + logger = None + def __init__(cls): pass @@ -26,6 +27,10 @@ class Container: pass @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.Container').getLogger() + + @classmethod def get(cls, type): return cls.container_list[type] @@ -46,10 +51,10 @@ class Container: dovetail_config[type]['result']['dir']) cmd = 'sudo docker run %s %s %s %s %s /bin/bash' % \ (opts, envs, sshkey, result_volume, docker_image) - dt_utils.exec_cmd(cmd, logger) + dt_utils.exec_cmd(cmd, cls.logger) ret, container_id = \ dt_utils.exec_cmd("sudo docker ps | grep " + docker_image + - " | awk '{print $1}' | head -1", logger) + " | awk '{print $1}' | head -1", cls.logger) cls.container_list[type] = container_id return container_id @@ -57,20 +62,21 @@ class Container: def pull_image(cls, type): docker_image = cls.get_docker_image(type) if cls.has_pull_latest_image[type] is True: - logger.debug('%s is already the newest version.' % (docker_image)) + cls.logger.debug('%s is already the newest version.' % + (docker_image)) else: cmd = 'sudo docker pull %s' % (docker_image) - dt_utils.exec_cmd(cmd, logger) + dt_utils.exec_cmd(cmd, cls.logger) cls.has_pull_latest_image[type] = True - @staticmethod - def clean(container_id): + @classmethod + def clean(cls, container_id): cmd1 = 'sudo docker stop %s' % (container_id) - dt_utils.exec_cmd(cmd1, logger) + dt_utils.exec_cmd(cmd1, cls.logger) cmd2 = 'sudo docker rm %s' % (container_id) - dt_utils.exec_cmd(cmd2, logger) + dt_utils.exec_cmd(cmd2, cls.logger) - @staticmethod - def exec_cmd(container_id, sub_cmd, exit_on_error=False): + @classmethod + def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False): cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd) - dt_utils.exec_cmd(cmd, logger, exit_on_error) + dt_utils.exec_cmd(cmd, cls.logger, exit_on_error) diff --git a/dovetail/parser.py b/dovetail/parser.py index 1740944a..d8f9fa0a 100644 --- a/dovetail/parser.py +++ b/dovetail/parser.py @@ -14,28 +14,33 @@ import utils.dovetail_logger as dt_logger import utils.dovetail_utils as dt_utils from conf.dovetail_config import DovetailConfig as dt_config -logger = dt_logger.Logger('parser.py').getLogger() - class Parser: '''preprocess configuration files''' - @staticmethod - def parse_cmd(cmd, testcase): + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.Parser').getLogger() + + @classmethod + def parse_cmd(cls, cmd, testcase): cmd_lines = None try: template = jinja2.Template(cmd, undefined=jinja2.StrictUndefined) kwargs = {} for arg in dt_config.dovetail_config['parameters']: path = eval(arg['path']) - logger.debug('name: %s, eval path: %s ' % (arg['name'], path)) + cls.logger.debug('name: %s, eval path: %s ' % + (arg['name'], path)) kwargs[arg['name']] = \ dt_utils.get_obj_by_path(testcase.testcase, path) - logger.debug('kwargs: %s' % kwargs) + cls.logger.debug('kwargs: %s' % kwargs) cmd_lines = template.render(**kwargs) except Exception as e: - logger.error('failed to parse cmd %s, exception:%s' % (cmd, e)) + cls.logger.error('failed to parse cmd %s, exception:%s' % (cmd, e)) return None return cmd_lines diff --git a/dovetail/report.py b/dovetail/report.py index 7dd8ba2d..7fd4076d 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -6,18 +6,20 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 # +from __future__ import division + import json import urllib2 import re import os +import datetime +import uuid import utils.dovetail_logger as dt_logger -from conf.dovetail_config import DovetailConfig as dt_config +from conf.dovetail_config import DovetailConfig as dt_cfg from testcase import Testcase -logger = dt_logger.Logger('report.py').getLogger() - def get_pass_str(passed): if passed: @@ -30,78 +32,133 @@ class Report: results = {'functest': {}, 'yardstick': {}} + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.Report').getLogger() + @staticmethod def check_result(testcase, db_result): checker = CheckerFactory.create(testcase.script_type()) checker.check(testcase, db_result) @classmethod - def generate_json(cls, scenario_yaml): + def generate_json(cls, testsuite_yaml, testarea, duration): report_obj = {} - report_obj['scenario'] = scenario_yaml['name'] + # TO DO: once version scheme settled, adjust accordingly + report_obj['version'] = '1.0' + report_obj['testsuite'] = testsuite_yaml['name'] + # TO DO: once dashboard url settled, adjust accordingly + report_obj['dashboard'] = None + report_obj['validation_ID'] = str(uuid.uuid4()) + report_obj['upload_date'] =\ + datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC") + report_obj['duration'] = duration + report_obj['testcases_list'] = [] - for testcase_name in scenario_yaml['testcases_list']: + testarea_list = [] + for value in testsuite_yaml['testcases_list']: + if value is not None and (testarea == 'full' or testarea in value): + testarea_list.append(value) + for testcase_name in testarea_list: testcase = Testcase.get(testcase_name) - testcase_in_rpt = {} - testcase_in_rpt['name'] = testcase_name + testcase_inreport = {} + testcase_inreport['name'] = testcase_name if testcase is None: - testcase_in_rpt['result'] = 'Undefined' - testcase_in_rpt['objective'] = '' - testcase_in_rpt['sub_testcase'] = [] - report_obj['testcases_list'].append(testcase_in_rpt) + testcase_inreport['result'] = 'Undefined' + testcase_inreport['objective'] = '' + testcase_inreport['sub_testcase'] = [] + report_obj['testcases_list'].append(testcase_inreport) continue - testcase_in_rpt['result'] = get_pass_str(testcase.passed()) - testcase_in_rpt['objective'] = testcase.objective() - testcase_in_rpt['sub_testcase'] = [] + testcase_inreport['result'] = get_pass_str(testcase.passed()) + testcase_inreport['objective'] = testcase.objective() + testcase_inreport['sub_testcase'] = [] if testcase.sub_testcase() is not None: for sub_test in testcase.sub_testcase(): - testcase_in_rpt['sub_testcase'].append({ + testcase_inreport['sub_testcase'].append({ 'name': sub_test, 'result': get_pass_str( testcase.sub_testcase_passed(sub_test)) }) - report_obj['testcases_list'].append(testcase_in_rpt) - logger.info(json.dumps(report_obj)) + report_obj['testcases_list'].append(testcase_inreport) + cls.logger.info(json.dumps(report_obj)) return report_obj @classmethod - def generate(cls, scenario_yaml): - rpt_data = cls.generate_json(scenario_yaml) - rpt_text = '' - split_line = '+-----------------------------------------------------' - split_line += '---------------------+\n' - - rpt_text += '\n\ -+==========================================================================+\n\ -| report |\n' - rpt_text += split_line - rpt_text += '|scenario: %s\n' % rpt_data['scenario'] - for testcase in rpt_data['testcases_list']: - rpt_text += '| [testcase]: %s\t\t\t\t[%s]\n' % \ - (testcase['name'], testcase['result']) - rpt_text += '| |-objective: %s\n' % testcase['objective'] - if 'sub_testcase' in testcase: - for sub_test in testcase['sub_testcase']: - rpt_text += '| |-%s \t\t [%s]\n' % \ - (sub_test['name'], sub_test['result']) - rpt_text += split_line - - logger.info(rpt_text) - cls.save(rpt_text) - return rpt_text + def generate(cls, testsuite_yaml, testarea, duration): + report_data = cls.generate_json(testsuite_yaml, testarea, duration) + report_txt = '' + report_txt += '\n\nDovetail Report\n' + report_txt += 'Version: %s\n' % report_data['version'] + report_txt += 'TestSuite: %s\n' % report_data['testsuite'] + report_txt += 'Result Dashboard: %s\n' % report_data['dashboard'] + report_txt += 'Validation ID: %s\n' % report_data['validation_ID'] + report_txt += 'Upload Date: %s\n' % report_data['upload_date'] + if report_data['duration'] == 0: + report_txt += 'Duration: %s\n\n' % 'NA' + else: + report_txt += 'Duration: %.2f s\n\n' % report_data['duration'] + + total_num = 0 + pass_num = 0 + sub_report = {} + testcase_num = {} + testcase_passnum = {} + for area in dt_cfg.testarea_supported: + sub_report[area] = '' + testcase_num[area] = 0 + testcase_passnum[area] = 0 + + # TO DO: once version scheme settled, adjust accordingly + spec_link = dt_cfg.dovetail_config['repo'] + 'dovetail/testcase' + for testcase in report_data['testcases_list']: + pattern = re.compile('|'.join(dt_cfg.testarea_supported)) + area = pattern.findall(testcase['name'])[0] + result_dir = dt_cfg.dovetail_config['result_dir'] + sub_report[area] += '- <%s> %s result: <%s>\n' %\ + (spec_link, testcase['name'], result_dir) + testcase_num[area] += 1 + total_num += 1 + if testcase['result'] == 'PASS': + testcase_passnum[area] += 1 + pass_num += 1 + + if total_num != 0: + pass_rate = pass_num/total_num + report_txt += 'Pass Rate: %.2f%% (%s/%s)\n' %\ + (pass_rate*100, pass_num, total_num) + report_txt += 'Assessed test areas:\n' + for key in sub_report: + if testcase_num[key] != 0: + pass_rate = testcase_passnum[key]/testcase_num[key] + # TO DO: once version scheme settled, adjust accordingly + doc_link = dt_cfg.dovetail_config['repo'] +\ + ('docs/testsuites/%s' % key) + report_txt += '- %s results: <%s> pass %.2f%%\n' %\ + (key, doc_link, pass_rate*100) + for key in sub_report: + if testcase_num[key] != 0: + pass_rate = testcase_passnum[key]/testcase_num[key] + report_txt += '%s: pass rate %.2f%%\n' % (key, pass_rate*100) + report_txt += sub_report[key] + + cls.logger.info(report_txt) + cls.save(report_txt) + return report_txt # save to disk as default - @staticmethod - def save(report): - report_file_name = dt_config.dovetail_config['report_file'] + @classmethod + def save(cls, report): + report_file_name = dt_cfg.dovetail_config['report_file'] try: - with open(os.path.join(dt_config.dovetail_config['result_dir'], + with open(os.path.join(dt_cfg.dovetail_config['result_dir'], report_file_name), 'w') as report_file: report_file.write(report) - logger.info('save report to %s' % report_file_name) + cls.logger.info('save report to %s' % report_file_name) except Exception: - logger.error('Failed to save: %s' % report_file_name) + cls.logger.error('Failed to save: %s' % report_file_name) @classmethod def get_result(cls, testcase): @@ -117,11 +174,12 @@ class Report: if result is not None: cls.results[type][script_testcase] = result testcase.script_result_acquired(True) - logger.debug('testcase: %s -> result acquired' % script_testcase) + cls.logger.debug('testcase: %s -> result acquired' % + script_testcase) else: retry = testcase.increase_retry() - logger.debug('testcase: %s -> result acquired retry:%d' % - (script_testcase, retry)) + cls.logger.debug('testcase: %s -> result acquired retry:%d' % + (script_testcase, retry)) return result @@ -140,12 +198,18 @@ class CrawlerFactory: class FunctestCrawler: + logger = None + def __init__(self): self.type = 'functest' + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.FunctestCrawler').getLogger() + def crawl(self, testcase=None): store_type = \ - dt_config.dovetail_config[self.type]['result']['store_type'] + dt_cfg.dovetail_config[self.type]['result']['store_type'] if store_type == 'file': return self.crawl_from_file(testcase) @@ -153,12 +217,12 @@ class FunctestCrawler: return self.crawl_from_url(testcase) def crawl_from_file(self, testcase=None): - dovetail_config = dt_config.dovetail_config + dovetail_config = dt_cfg.dovetail_config file_path = \ os.path.join(dovetail_config['result_dir'], dovetail_config[self.type]['result']['file_path']) if not os.path.exists(file_path): - logger.info('result file not found: %s' % file_path) + self.logger.info('result file not found: %s' % file_path) return None try: @@ -180,34 +244,40 @@ class FunctestCrawler: "duration": int(dur_sec_int), "tests": int(num_tests), "failures": failed_num, "errors": error_logs}} - logger.debug('Results: %s' % str(json_results)) + self.logger.debug('Results: %s' % str(json_results)) return json_results except Exception as e: - logger.error('Cannot read content from the file: %s, exception: %s' - % (file_path, e)) + self.logger.error('Cannot read content from the file: %s, ' + 'exception: %s' % (file_path, e)) return None def crawl_from_url(self, testcase=None): url = \ - dt_config.dovetail_config[self.type]['result']['db_url'] % testcase - logger.debug("Query to rest api: %s" % url) + dt_cfg.dovetail_config[self.type]['result']['db_url'] % testcase + self.logger.debug("Query to rest api: %s" % url) try: data = json.load(urllib2.urlopen(url)) return data['results'][0] except Exception as e: - logger.error("Cannot read content from the url: %s, exception: %s" - % (url, e)) + self.logger.error("Cannot read content from the url: %s, " + "exception: %s" % (url, e)) return None class YardstickCrawler: + logger = None + def __init__(self): self.type = 'yardstick' + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.YardstickCrawler').getLogger() + def crawl(self, testcase=None): store_type = \ - dt_config.dovetail_config[self.type]['result']['store_type'] + dt_cfg.dovetail_config[self.type]['result']['store_type'] if store_type == 'file': return self.crawl_from_file(testcase) @@ -215,21 +285,21 @@ class YardstickCrawler: return self.crawl_from_url(testcase) def crawl_from_file(self, testcase=None): - file_path = os.path.join(dt_config.dovetail_config['result_dir'], + file_path = os.path.join(dt_cfg.dovetail_config['result_dir'], testcase+'.out') if not os.path.exists(file_path): - logger.info('result file not found: %s' % file_path) + self.logger.info('result file not found: %s' % file_path) return None try: with open(file_path, 'r') as myfile: myfile.read() criteria = 'PASS' json_results = {'criteria': criteria} - logger.debug('Results: %s' % str(json_results)) + self.logger.debug('Results: %s' % str(json_results)) return json_results except Exception as e: - logger.error('Cannot read content from the file: %s, exception: %s' - % (file_path, e)) + self.logger.error('Cannot read content from the file: %s, ' + 'exception: %s' % (file_path, e)) return None def crawl_from_url(self, testcase=None): @@ -258,8 +328,13 @@ class ResultChecker: class FunctestChecker: - @staticmethod - def check(testcase, db_result): + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.FunctestChecker').getLogger() + + def check(self, testcase, db_result): sub_testcase_list = testcase.sub_testcase() if not db_result: @@ -280,7 +355,7 @@ class FunctestChecker: all_passed = True for sub_testcase in sub_testcase_list: - logger.debug('check sub_testcase:%s' % sub_testcase) + self.logger.debug('check sub_testcase:%s' % sub_testcase) if sub_testcase in db_result['details']['errors']: testcase.sub_testcase_passed(sub_testcase, False) all_passed = False @@ -292,6 +367,12 @@ class FunctestChecker: class YardstickChecker: + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.YardstickChecker').getLogger() + @staticmethod def check(testcase, result): if not result: diff --git a/dovetail/run.py b/dovetail/run.py index 86194389..d24919b6 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -10,22 +10,25 @@ import click import sys +import os +import time import utils.dovetail_logger as dt_logger +import utils.dovetail_utils as dt_utils - +from parser import Parser from container import Container from testcase import Testcase -from testcase import Scenario +from testcase import Testsuite from report import Report +from report import FunctestCrawler, YardstickCrawler +from report import FunctestChecker, YardstickChecker from conf.dovetail_config import DovetailConfig as dt_config -logger = dt_logger.Logger('run.py').getLogger() - -def load_scenario(scenario): - Scenario.load() - return Scenario.get(dt_config.SCENARIO_NAMING_FMT % scenario) +def load_testsuite(testsuite): + Testsuite.load() + return Testsuite.get(testsuite) def set_container_tags(option_str): @@ -40,12 +43,18 @@ def load_testcase(): Testcase.load() -def run_test(scenario): - for testcase_name in scenario['testcases_list']: +def run_test(testsuite, testarea, logger): + testarea_list = [] + for value in testsuite['testcases_list']: + if value is not None and (testarea == 'full' or testarea in value): + testarea_list.append(value) + + duration = 0 + for testcase_name in testarea_list: logger.info('>>[testcase]: %s' % (testcase_name)) testcase = Testcase.get(testcase_name) if testcase is None: - logger.error('testcase %s is not defined in testcase folder, \ + logger.error('test case %s is not defined in testcase folder, \ skipping' % (testcase_name)) continue run_testcase = True @@ -71,8 +80,11 @@ def run_test(scenario): if not testcase.prepare_cmd(): logger.error('failed to prepare testcase:%s' % testcase.name()) else: + start_time = time.time() for cmd in testcase.cmds: Container.exec_cmd(container_id, cmd) + end_time = time.time() + duration = end_time - start_time # testcase.post_condition() @@ -81,8 +93,10 @@ def run_test(scenario): db_result = Report.get_result(testcase) Report.check_result(testcase, db_result) + return duration -def validate_options(input_dict): + +def validate_options(input_dict, logger): # for 'tag' option for key, value in input_dict.items(): if key == 'tag' and value is not None: @@ -101,24 +115,63 @@ def filter_env_options(input_dict): return envs_options +def create_logs(): + Container.create_log() + Parser.create_log() + Report.create_log() + FunctestCrawler.create_log() + YardstickCrawler.create_log() + FunctestChecker.create_log() + YardstickChecker.create_log() + Testcase.create_log() + Testsuite.create_log() + + +def clean_results_dir(): + result_path = dt_config.dovetail_config['result_dir'] + if os.path.exists(result_path): + if os.path.isdir(result_path): + cmd = 'sudo rm -rf %s/*' % (result_path) + dt_utils.exec_cmd(cmd, exit_on_error=False) + else: + print "result_dir in dovetail_config.yml is not a directory." + sys.exit(-1) + + def main(*args, **kwargs): - """Dovetail certification test entry!""" - logger.info('=======================================') - logger.info('Dovetail certification: %s!' % (kwargs['scenario'])) - logger.info('=======================================') - validate_options(kwargs) + """Dovetail compliance test entry!""" + clean_results_dir() + create_logs() + logger = dt_logger.Logger('run').getLogger() + logger.info('================================================') + logger.info('Dovetail compliance: %s!' % (kwargs['testsuite'])) + logger.info('================================================') + validate_options(kwargs, logger) envs_options = filter_env_options(kwargs) dt_config.update_envs(envs_options) logger.info('Your new envs for functest: %s' % dt_config.dovetail_config['functest']['envs']) logger.info('Your new envs for yardstick: %s' % dt_config.dovetail_config['yardstick']['envs']) - load_testcase() - scenario_yaml = load_scenario(kwargs['scenario']) + if 'tag' in kwargs and kwargs['tag'] is not None: set_container_tags(kwargs['tag']) - run_test(scenario_yaml) - Report.generate(scenario_yaml) + + testarea = kwargs['testarea'] + testsuite_validation = False + testarea_validation = False + if (testarea == 'full') or (testarea in dt_config.testarea_supported): + testarea_validation = True + if kwargs['testsuite'] in dt_config.testsuite_supported: + testsuite_validation = True + if testsuite_validation and testarea_validation: + testsuite_yaml = load_testsuite(kwargs['testsuite']) + load_testcase() + duration = run_test(testsuite_yaml, testarea, logger) + Report.generate(testsuite_yaml, testarea, duration) + else: + logger.error('invalid input commands, testsuite %s testarea %s' % + (kwargs['testsuite'], testarea)) CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) diff --git a/dovetail/testcase.py b/dovetail/testcase.py index 6f0552b6..60ce2b28 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -15,11 +15,11 @@ import utils.dovetail_logger as dt_logger from parser import Parser from conf.dovetail_config import DovetailConfig as dt_config -logger = dt_logger.Logger('testcase.py').getLogger() - class Testcase: + logger = None + def __init__(self, testcase_yaml): self.testcase = testcase_yaml.values()[0] self.testcase['passed'] = False @@ -28,6 +28,10 @@ class Testcase: self.update_script_testcase(self.script_type(), self.script_testcase()) + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.Testcase').getLogger() + def prepare_cmd(self): script_type = self.script_type() for cmd in dt_config.dovetail_config[script_type]['testcase']['cmds']: @@ -52,7 +56,7 @@ class Testcase: def sub_testcase_passed(self, name, passed=None): if passed is not None: - logger.debug('sub_testcase_passed:%s %s' % (name, passed)) + self.logger.debug('sub_testcase_passed:%s %s' % (name, passed)) self.sub_testcase_status[name] = passed return self.sub_testcase_status[name] @@ -146,7 +150,7 @@ class Testcase: testcase_yaml = yaml.safe_load(f) cls.testcase_list[testcase_yaml.keys()[0]] = \ cls(testcase_yaml) - logger.debug(cls.testcase_list) + cls.logger.debug(cls.testcase_list) @classmethod def get(cls, testcase_name): @@ -155,31 +159,37 @@ class Testcase: return None -class Scenario: +class Testsuite: - def __init__(self, scenario): - self.scenario = scenario + logger = None + + def __init__(self, testsuite): + self.testsuite = testsuite self.testcase_list = {} + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__+'.Testsuite').getLogger() + def get_test(self, testcase_name): if testcase_name in self.testcase_list: return self.testcase_list[testcase_name] return None - scenario_list = {} + testsuite_list = {} @classmethod def load(cls): - for root, dirs, files in os.walk(dt_config.CERT_PATH): - for scenario_yaml in files: - with open(os.path.join(root, scenario_yaml)) as f: - scenario_yaml = yaml.safe_load(f) - cls.scenario_list.update(scenario_yaml) + for root, dirs, files in os.walk(dt_config.COMPLIANCE_PATH): + for testsuite_yaml in files: + with open(os.path.join(root, testsuite_yaml)) as f: + testsuite_yaml = yaml.safe_load(f) + cls.testsuite_list.update(testsuite_yaml) - logger.debug(cls.scenario_list) + cls.logger.debug(cls.testsuite_list) @classmethod - def get(cls, scenario_name): - if scenario_name in cls.scenario_list: - return cls.scenario_list[scenario_name] + def get(cls, testsuite_name): + if testsuite_name in cls.testsuite_list: + return cls.testsuite_list[testsuite_name] return None diff --git a/dovetail/tests/unit/test_parser.py b/dovetail/tests/unit/test_parser.py index 5b003d1a..b4331ea1 100644 --- a/dovetail/tests/unit/test_parser.py +++ b/dovetail/tests/unit/test_parser.py @@ -17,7 +17,7 @@ import unittest import yaml -import dovetail.parser as dovetail_parser +import parser as dovetail_parser class TestParser(unittest.TestCase): @@ -27,6 +27,7 @@ class TestParser(unittest.TestCase): def setUp(self): """Test case setup""" logging.disable(logging.CRITICAL) + dovetail_parser.Parser.create_log() def test_parser_cmd(self): """Test whether the command is correctly parsed.""" diff --git a/dovetail/utils/dovetail_logger.py b/dovetail/utils/dovetail_logger.py index 6a2d38d5..8afa08a1 100644 --- a/dovetail/utils/dovetail_logger.py +++ b/dovetail/utils/dovetail_logger.py @@ -25,22 +25,12 @@ import logging import os from conf.dovetail_config import DovetailConfig as dt_config -import dovetail_utils as dt_utils - - -def clean_results_dir(): - result_path = dt_config.dovetail_config['result_dir'] - if os.path.exists(result_path): - cmd = 'sudo rm -rf %s/*' % (result_path) - dt_utils.exec_cmd(cmd, exit_on_error=False) - -clean_results_dir() class Logger: def __init__(self, logger_name): - CI_DEBUG = os.getenv('CI_DEBUG') + DEBUG = os.getenv('DEBUG') self.logger = logging.getLogger(logger_name) self.logger.propagate = 0 @@ -50,7 +40,7 @@ class Logger: formatter = logging.Formatter('%(asctime)s - %(name)s - ' '%(levelname)s - %(message)s') ch.setFormatter(formatter) - if CI_DEBUG is not None and CI_DEBUG.lower() == "true": + if DEBUG is not None and DEBUG.lower() == "true": ch.setLevel(logging.DEBUG) else: ch.setLevel(logging.INFO) |