diff options
Diffstat (limited to 'dovetail/report.py')
| -rw-r--r-- | dovetail/report.py | 309 |
1 files changed, 226 insertions, 83 deletions
diff --git a/dovetail/report.py b/dovetail/report.py index 26cd6c52..ed3f942b 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -12,6 +12,7 @@ from __future__ import division import collections +import hashlib import json import re import os @@ -19,17 +20,18 @@ import datetime import tarfile import time -import utils.dovetail_logger as dt_logger +import dovetail.utils.dovetail_logger as dt_logger -from utils.dovetail_config import DovetailConfig as dt_cfg -import utils.dovetail_utils as dt_utils -from testcase import Testcase +from dovetail.utils.dovetail_config import DovetailConfig as dt_cfg +import dovetail.utils.dovetail_utils as dt_utils +from dovetail.testcase import Testcase class Report(object): results = {'functest': {}, 'yardstick': {}, 'functest-k8s': {}, - 'bottlenecks': {}, 'shell': {}, 'vnftest': {}} + 'bottlenecks': {}, 'shell': {}, 'onap-vtp': {}, + 'onap-vvp': {}} logger = None @@ -39,25 +41,29 @@ class Report(object): def check_tc_result(self, testcase): result_path = dt_cfg.dovetail_config['result_dir'] - check_results_file = dt_utils.get_value_from_dict( - 'report.check_results_file', testcase.testcase) - if not check_results_file: - self.logger.error("Failed to get 'check_results_file' from config " - "file of test case {}".format(testcase.name())) - self.check_result(testcase) - return None - result_file = os.path.join(result_path, check_results_file) - if os.path.isfile(result_file): - self.logger.info( - 'Results have been stored with file {}.'.format(result_file)) - result = self.get_result(testcase, result_file) - self.check_result(testcase, result) - return result - else: - self.logger.error( - 'Failed to store results with file {}.'.format(result_file)) + check_results_files = dt_utils.get_value_from_dict( + 'report.check_results_files', testcase.testcase) + if not check_results_files: + self.logger.error("Failed to get 'check_results_files' from config" + " file of test case {}".format(testcase.name())) self.check_result(testcase) return None + result_files = [] + for check_results_file in check_results_files: + result_file = os.path.join(result_path, check_results_file) + if not os.path.isfile(result_file): + self.logger.error( + 'Failed to store results with file {}.'. + format(result_file)) + self.check_result(testcase) + return None + else: + result_files.append(result_file) + self.logger.info( + 'Results have been stored with files: {}.'.format(result_files)) + result = self.get_result(testcase, result_files) + self.check_result(testcase, result) + return result @staticmethod def check_result(testcase, db_result=None): @@ -65,16 +71,41 @@ class Report(object): if checker is not None: checker.check(testcase, db_result) + @staticmethod + def get_checksum(vnf_type): + if vnf_type == 'tosca': + path = os.path.join(dt_cfg.dovetail_config['config_dir'], + os.getenv('CSAR_FILE')) + elif vnf_type == 'heat': + path = os.path.join( + dt_cfg.dovetail_config['config_dir'], + '{}.zip'.format(os.getenv('VNF_ARCHIVE_NAME'))) + + checksum = hashlib.sha256() + + if os.path.isfile(path): + with open(path, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b''): + checksum.update(chunk) + + return checksum.hexdigest() + def generate_json(self, testcase_list, duration): report_obj = {} # egeokun: using a hardcoded string instead of pbr version for # versioning the result file. The version of the results.json is # logically independent of the release of Dovetail. - report_obj['version'] = '2018.09' + report_obj['version'] = dt_cfg.dovetail_config.get('version') report_obj['build_tag'] = dt_cfg.dovetail_config['build_tag'] report_obj['test_date'] =\ datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S UTC') report_obj['duration'] = duration + vnf_type = dt_cfg.dovetail_config.get('vnf_type') + if vnf_type: + report_obj['vnf_type'] = vnf_type + report_obj['vnf_checksum'] = self.get_checksum(vnf_type) + else: + report_obj['validation'] = os.getenv('validation') report_obj['testcases_list'] = [] if not testcase_list: @@ -88,12 +119,21 @@ class Report(object): testcase_inreport['objective'] = '' testcase_inreport['sub_testcase'] = [] testcase_inreport['mandatory'] = False + testcase_inreport['portal_key_file'] = '' report_obj['testcases_list'].append(testcase_inreport) continue testcase_inreport['result'] = testcase.passed() testcase_inreport['objective'] = testcase.objective() + try: + vnf_type = testcase.vnf_type() + except Exception: + vnf_type = None + if vnf_type: + report_obj['vnf_type'] = vnf_type + report_obj['vnf_checksum'] = self.get_checksum(vnf_type) testcase_inreport['mandatory'] = testcase.is_mandatory + testcase_inreport['portal_key_file'] = testcase.portal_key_file() testcase_inreport['sub_testcase'] = [] if testcase.sub_testcase() is not None: for sub_test in testcase.sub_testcase(): @@ -102,7 +142,6 @@ class Report(object): 'result': testcase.sub_testcase_passed(sub_test) }) report_obj['testcases_list'].append(testcase_inreport) - self.logger.debug(json.dumps(report_obj)) return report_obj def generate(self, testcase_list, duration): @@ -121,22 +160,15 @@ class Report(object): sub_report = collections.OrderedDict() testcase_num = {} testcase_passnum = {} - for area in dt_cfg.dovetail_config['testarea_supported']: - sub_report[area] = '' - testcase_num[area] = 0 - testcase_passnum[area] = 0 testarea_scope = [] for testcase in report_data['testcases_list']: - supported_areas = dt_cfg.dovetail_config['testarea_supported'] - pattern = re.compile('|'.join(supported_areas)) - area = pattern.findall(testcase['name']) - if not supported_areas or not area: - self.logger.error('Test case {} not in supported testarea.' - .format(testcase['name'])) - return None - area = area[0] - testarea_scope.append(area) + area = testcase['name'].split('.')[1] + if area not in testarea_scope: + testarea_scope.append(area) + sub_report[area] = '' + testcase_num[area] = 0 + testcase_passnum[area] = 0 sub_report[area] += '-%-25s %s\n' %\ (testcase['name'], testcase['result']) if 'sub_testcase' in testcase: @@ -200,7 +232,7 @@ class Report(object): f_out.add(os.path.join('results', f)) os.chdir(cwd) - def get_result(self, testcase, check_results_file): + def get_result(self, testcase, check_results_files): validate_testcase = testcase.validate_testcase() type = testcase.validate_type() crawler = CrawlerFactory.create(type) @@ -208,7 +240,7 @@ class Report(object): self.logger.error('Crawler is None: {}'.format(testcase.name())) return None - result = crawler.crawl(testcase, check_results_file) + result = crawler.crawl(testcase, check_results_files) if result is not None: self.results[type][validate_testcase] = result @@ -238,8 +270,8 @@ class FunctestCrawler(Crawler): cls.logger = \ dt_logger.Logger(__name__ + '.FunctestCrawler').getLogger() - def crawl(self, testcase, file_path): - return self.crawl_from_file(testcase, file_path) + def crawl(self, testcase, file_paths): + return self.crawl_from_file(testcase, file_paths[0]) def crawl_from_file(self, testcase, file_path): dovetail_config = dt_cfg.dovetail_config @@ -269,16 +301,10 @@ class FunctestCrawler(Crawler): duration = dt_utils.get_duration(timestart, timestop, self.logger) if complex_testcase: - tests = data['details']['tests_number'] - failed_num = data['details']['failures_number'] - success_case = data['details']['success'] - error_case = data['details']['failures'] - skipped_case = data['details']['skipped'] - details = {'tests': tests, - 'failures': failed_num, - 'success': success_case, - 'errors': error_case, - 'skipped': skipped_case} + if testcase_name == 'rally_full': + details = self.get_rally_details(data) + else: + details = self.get_details(data) except KeyError as e: self.logger.exception( "Result data don't have key {}.".format(e)) @@ -293,6 +319,37 @@ class FunctestCrawler(Crawler): testcase.set_results(json_results) return json_results + def get_details(self, data): + try: + t_details = data['details'] + details = { + 'tests': t_details['tests_number'], + 'failures': t_details['failures_number'], + 'success': t_details['success'], + 'errors': t_details['failures'], + 'skipped': t_details['skipped'] + } + return details + except Exception as e: + self.logger.exception("Failed to get details, {}.".format(e)) + return None + + def get_rally_details(self, data): + try: + t_details = data['details']['modules'][0]['details'] + tests = len(t_details['success']) + len(t_details['failures']) + details = { + 'tests': tests, + 'failures': len(t_details['failures']), + 'success': t_details['success'], + 'errors': t_details['failures'], + 'skipped': [] + } + return details + except Exception as e: + self.logger.exception("Failed to get details, {}.".format(e)) + return None + class FunctestK8sCrawler(FunctestCrawler): @@ -321,8 +378,8 @@ class YardstickCrawler(Crawler): cls.logger = \ dt_logger.Logger(__name__ + '.YardstickCrawler').getLogger() - def crawl(self, testcase, file_path): - return self.crawl_from_file(testcase, file_path) + def crawl(self, testcase, file_paths): + return self.crawl_from_file(testcase, file_paths[0]) def crawl_from_file(self, testcase, file_path): if not os.path.exists(file_path): @@ -362,8 +419,8 @@ class BottlenecksCrawler(Crawler): cls.logger = \ dt_logger.Logger(__name__ + '.BottlenecksCrawler').getLogger() - def crawl(self, testcase, file_path): - return self.crawl_from_file(testcase, file_path) + def crawl(self, testcase, file_paths): + return self.crawl_from_file(testcase, file_paths[0]) def crawl_from_file(self, testcase, file_path): if not os.path.exists(file_path): @@ -392,8 +449,8 @@ class ShellCrawler(Crawler): def __init__(self): self.type = 'shell' - def crawl(self, testcase, file_path): - return self.crawl_from_file(testcase, file_path) + def crawl(self, testcase, file_paths): + return self.crawl_from_file(testcase, file_paths[0]) def crawl_from_file(self, testcase, file_path): if not os.path.exists(file_path): @@ -406,22 +463,37 @@ class ShellCrawler(Crawler): return None -class VnftestCrawler(Crawler): +class OnapVtpCrawler(Crawler): logger = None def __init__(self): - self.type = 'vnftest' + self.type = 'onap-vtp' self.logger.debug('Create crawler: {}'.format(self.type)) @classmethod def create_log(cls): - cls.logger = \ - dt_logger.Logger(__name__ + '.VnftestCrawler').getLogger() - - def crawl(self, testcase, file_path): - return self.crawl_from_file(testcase, file_path) - + cls.logger = dt_logger.Logger(__name__ + '.OnapVtpCrawler').getLogger() + + def crawl(self, testcase, file_paths): + return self.crawl_from_file(testcase, file_paths[0]) + + # The pass result looks like + # { + # "results": [ + # {"property": "results", "value": "{value=SUCCESS}"}, + # {"property": "build_tag", "value": "test"}, + # {"property": "criteria", "value": "PASS"} + # ] + # } + # The fail result looks like + # { + # "results": [ + # {"property": "results", "value": "{value=file doesn't exists}"}, + # {"property": "build_tag", "value": "test"}, + # {"property": "criteria", "value": "FAILED"} + # ] + # } def crawl_from_file(self, testcase, file_path): if not os.path.exists(file_path): self.logger.error('Result file not found: {}'.format(file_path)) @@ -429,23 +501,70 @@ class VnftestCrawler(Crawler): criteria = 'FAIL' with open(file_path, 'r') as f: for jsonfile in f: - data = json.loads(jsonfile) try: - criteria = data['result']['criteria'] + data = json.loads(jsonfile) + for item in data['results']: + if 'criteria' == item['property']: + if 'PASS' == item['value']: + criteria = 'PASS' + break + else: + self.logger.error('There is no property criteria.') except KeyError as e: self.logger.exception('Pass flag not found {}'.format(e)) + except ValueError: + continue + json_results = {'criteria': criteria} + + testcase.set_results(json_results) + return json_results + + +class OnapVvpCrawler(Crawler): + + logger = None + + def __init__(self): + self.type = 'onap-vvp' + self.logger.debug('Create crawler: {}'.format(self.type)) + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__ + '.OnapVvpCrawler').getLogger() + + def crawl(self, testcase, file_paths): + return self.crawl_from_file(testcase, file_paths[0]) + + def crawl_from_file(self, testcase, file_path): + if not os.path.exists(file_path): + self.logger.error('Result file not found: {}'.format(file_path)) + return None + criteria = 'FAIL' + with open(file_path, 'r') as f: + try: + data = json.load(f) + criteria = data['outcome'] + except KeyError as e: + self.logger.exception('Outcome field not found {}'.format(e)) + except ValueError: + self.logger.exception('Result file has invalid format') json_results = {'criteria': criteria} + + testcase.set_results(json_results) return json_results class CrawlerFactory(object): - CRAWLER_MAP = {'functest': FunctestCrawler, - 'yardstick': YardstickCrawler, - 'bottlenecks': BottlenecksCrawler, - 'vnftest': VnftestCrawler, - 'shell': ShellCrawler, - 'functest-k8s': FunctestK8sCrawler} + CRAWLER_MAP = { + 'functest': FunctestCrawler, + 'yardstick': YardstickCrawler, + 'bottlenecks': BottlenecksCrawler, + 'shell': ShellCrawler, + 'functest-k8s': FunctestK8sCrawler, + 'onap-vtp': OnapVtpCrawler, + 'onap-vvp': OnapVvpCrawler + } @classmethod def create(cls, type): @@ -488,6 +607,12 @@ class FunctestChecker(object): match = find_reg.findall(tc) if match: return True + reg = sub_testcase.rsplit('.', 1)[0] + '$' + find_reg = re.compile(reg) + for tc in result: + match = find_reg.findall(tc) + if match: + return True return False def check(self, testcase, db_result): @@ -506,7 +631,6 @@ class FunctestChecker(object): testcase_passed = 'PASS' for sub_testcase in sub_testcase_list: - self.logger.debug('Check sub_testcase: {}'.format(sub_testcase)) try: if self.get_sub_testcase(sub_testcase, db_result['details']['success']): @@ -581,14 +705,30 @@ class ShellChecker(object): testcase.passed(False) -class VnftestChecker(object): +class OnapVtpChecker(object): logger = None @classmethod def create_log(cls): - cls.logger = \ - dt_logger.Logger(__name__ + '.VnftestCheckers').getLogger() + cls.logger = dt_logger.Logger(__name__ + '.OnapVtpChecker').getLogger() + + @staticmethod + def check(testcase, result): + if not result: + testcase.passed('FAIL') + else: + testcase.passed(result['criteria']) + return + + +class OnapVvpChecker(object): + + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__name__ + '.OnapVvpChecker').getLogger() @staticmethod def check(testcase, result): @@ -601,12 +741,15 @@ class VnftestChecker(object): class CheckerFactory(object): - CHECKER_MAP = {'functest': FunctestChecker, - 'yardstick': YardstickChecker, - 'bottlenecks': BottlenecksChecker, - 'shell': ShellChecker, - 'vnftest': VnftestChecker, - 'functest-k8s': FunctestK8sChecker} + CHECKER_MAP = { + 'functest': FunctestChecker, + 'yardstick': YardstickChecker, + 'bottlenecks': BottlenecksChecker, + 'shell': ShellChecker, + 'functest-k8s': FunctestK8sChecker, + 'onap-vtp': OnapVtpChecker, + 'onap-vvp': OnapVvpChecker + } @classmethod def create(cls, type): |