From 931942e3719dda8a269acb63f320146d7327fdd6 Mon Sep 17 00:00:00 2001 From: MatthewLi Date: Wed, 8 Feb 2017 02:52:27 -0500 Subject: dovetail tool: skip testcase status added JIRA: DOVETAIL-344 1) testcase status, True/False --> PASS/SKIP/FAIL, so skipped testcase in tempest can be figured out details can be seen in https://etherpad.opnfv.org/p/dovetail_report 2) some variable name changed more readable, such as passed-->testcase_passed failed--> prepare_failed 3) "dovetail report" minor amend according to 1) Change-Id: Ibe8e50693fc474d78e7d0fb89257f34703248579 Signed-off-by: MatthewLi --- dovetail/report.py | 98 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 58 insertions(+), 40 deletions(-) (limited to 'dovetail/report.py') diff --git a/dovetail/report.py b/dovetail/report.py index b7b27930..66618b23 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -23,13 +23,6 @@ from utils.dovetail_config import DovetailConfig as dt_cfg from testcase import Testcase -def get_pass_str(passed): - if passed: - return 'PASS' - else: - return 'FAIL' - - class Report(object): results = {'functest': {}, 'yardstick': {}, 'shell': {}} @@ -75,15 +68,14 @@ class Report(object): report_obj['testcases_list'].append(testcase_inreport) continue - testcase_inreport['result'] = get_pass_str(testcase.passed()) + testcase_inreport['result'] = testcase.passed() testcase_inreport['objective'] = testcase.objective() testcase_inreport['sub_testcase'] = [] if testcase.sub_testcase() is not None: for sub_test in testcase.sub_testcase(): testcase_inreport['sub_testcase'].append({ 'name': sub_test, - 'result': get_pass_str( - testcase.sub_testcase_passed(sub_test)) + 'result': testcase.sub_testcase_passed(sub_test) }) report_obj['testcases_list'].append(testcase_inreport) cls.logger.info(json.dumps(report_obj)) @@ -114,38 +106,51 @@ class Report(object): testcase_num[area] = 0 testcase_passnum[area] = 0 - repo_link = dt_cfg.dovetail_config['repo']['path'] + \ - dt_cfg.dovetail_config['repo']['tag'] - + testarea_scope = [] for testcase in report_data['testcases_list']: pattern = re.compile( '|'.join(dt_cfg.dovetail_config['testarea_supported'])) area = pattern.findall(testcase['name'])[0] - result_dir = dt_cfg.dovetail_config['result_dir'] - spec_link = repo_link + '/dovetail/testcase' - sub_report[area] += '- <%s> %s result: <%s>\n' %\ - (spec_link, testcase['name'], result_dir) + testarea_scope.append(area) + sub_report[area] += '-%-25s %s\n' %\ + (testcase['name'], testcase['result']) + if 'sub_testcase' in testcase: + for sub_test in testcase['sub_testcase']: + sub_report[area] += '\t%-110s %s\n' %\ + (sub_test['name'], sub_test['result']) testcase_num[area] += 1 total_num += 1 if testcase['result'] == 'PASS': testcase_passnum[area] += 1 pass_num += 1 + elif testcase['result'] == 'SKIP': + testcase_num[area] -= 1 + total_num -= 1 if total_num != 0: pass_rate = pass_num / total_num report_txt += 'Pass Rate: %.2f%% (%s/%s)\n' %\ (pass_rate * 100, pass_num, total_num) report_txt += 'Assessed test areas:\n' + else: + report_txt += \ + 'no testcase or all testcases are skipped in this testsuite' + for key in sub_report: if testcase_num[key] != 0: pass_rate = testcase_passnum[key] / testcase_num[key] - doc_link = repo_link + ('/docs/testsuites/%s' % key) - report_txt += '- %s results: <%s> pass %.2f%%\n' %\ - (key, doc_link, pass_rate * 100) + report_txt += '-%-25s pass %.2f%%\n' %\ + (key + ' results:', pass_rate * 100) + elif key in testarea_scope: + report_txt += '-%-25s all skipped\n' % key for key in sub_report: if testcase_num[key] != 0: pass_rate = testcase_passnum[key] / testcase_num[key] - report_txt += '%s: pass rate %.2f%%\n' % (key, pass_rate * 100) + report_txt += '%-25s pass rate %.2f%%\n' %\ + (key + ':', pass_rate * 100) + report_txt += sub_report[key] + elif key in testarea_scope: + report_txt += '%-25s all skipped\n' % key report_txt += sub_report[key] cls.logger.info(report_txt) @@ -248,12 +253,24 @@ class FunctestCrawler(object): with open(file_path, 'r') as myfile: output = myfile.read() - error_logs = " ".join(re.findall('(.*?)[. ]*fail ', output)) - skipped = " ".join(re.findall('(.*?)[. ]*skip:', output)) + match = re.findall('(.*?)[. ]*fail ', output) + if match: + error_logs = " ".join(match) + else: + error_logs = "" + match = re.findall('(.*?)[. ]*skip:', output) + if match: + skipped = " ".join(match) + else: + skipped = "" - failed_num = int(re.findall(' - Failures: (\d*)', output)[0]) - if failed_num != 0: - criteria = 'FAIL' + match = re.findall(' - Failures: (\d*)', output) + if match: + failed_num = int(match[0]) + else: + failed_num = 0 + if failed_num == 0: + criteria = 'PASS' match = re.findall('Ran: (\d*) tests in (\d*)\.\d* sec.', output) num_tests, dur_sec_int = match[0] @@ -379,31 +396,32 @@ class FunctestChecker(object): if not db_result: if sub_testcase_list is not None: for sub_testcase in sub_testcase_list: - testcase.sub_testcase_passed(sub_testcase, False) + testcase.sub_testcase_passed(sub_testcase, 'FAIL') return - testcase.passed(db_result['criteria'] == 'PASS') + testcase.passed(db_result['criteria']) if sub_testcase_list is None: return - if testcase.testcase['passed'] is True: - for sub_testcase in sub_testcase_list: - testcase.sub_testcase_passed(sub_testcase, True) - return - - all_passed = True + testcase_passed = 'SKIP' for sub_testcase in sub_testcase_list: self.logger.debug('check sub_testcase:%s', sub_testcase) - # TO DO: should think the test case when skipped, should think - # together with the "dovetail report" if sub_testcase in db_result['details']['errors']: - testcase.sub_testcase_passed(sub_testcase, False) - all_passed = False + testcase.sub_testcase_passed(sub_testcase, 'FAIL') + testcase_passed = 'FAIL' + elif sub_testcase in db_result['details']['skipped']: + testcase.sub_testcase_passed(sub_testcase, 'SKIP') else: - testcase.sub_testcase_passed(sub_testcase, True) + testcase.sub_testcase_passed(sub_testcase, 'PASS') + + if testcase_passed == 'SKIP': + for sub_testcase in sub_testcase_list: + if testcase.sub_testcase_status[sub_testcase] == 'PASS': + testcase_passed = 'PASS' + break - testcase.passed(all_passed) + testcase.passed(testcase_passed) class YardstickChecker(object): -- cgit 1.2.3-korg