diff options
Diffstat (limited to 'dovetail/run.py')
-rwxr-xr-x | dovetail/run.py | 360 |
1 files changed, 175 insertions, 185 deletions
diff --git a/dovetail/run.py b/dovetail/run.py index 4778fa51..c5281918 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -1,132 +1,85 @@ #!/usr/bin/env python + +############################################################################## +# Copyright (c) 2017 grakiss.wanglei@huawei.com and others. # -# grakiss.wanglei@huawei.com # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 -# +############################################################################## -import click -import os import copy +from datetime import datetime +import json +import os import time import uuid -import utils.dovetail_logger as dt_logger -import utils.dovetail_utils as dt_utils - -from parser import Parser -from container import Container -from testcase import Testcase -from testcase import Testsuite -from report import Report -from report import FunctestCrawler, YardstickCrawler, BottlenecksCrawler -from report import FunctestChecker, YardstickChecker, BottlenecksChecker -from utils.dovetail_config import DovetailConfig as dt_cfg -from test_runner import DockerRunner, ShellRunner +import click +from dovetail.container import Container +from dovetail import constants +from dovetail.parser import Parser +import dovetail.report as dt_report +import dovetail.test_runner as dt_test_runner +import dovetail.testcase as dt_testcase +from dovetail.utils.dovetail_config import DovetailConfig as dt_cfg +import dovetail.utils.dovetail_logger as dt_logger +import dovetail.utils.dovetail_utils as dt_utils -def load_testsuite(testsuite): - Testsuite.load() - return Testsuite.get(testsuite) +EXIT_RUN_FAILED = 2 -def load_testcase(): - Testcase.load() +def load_testsuite(testsuite): + dt_testcase.Testsuite.load() + return dt_testcase.Testsuite.get(testsuite) -def run_test(testsuite, testarea, logger): - testcase_list = Testcase.get_testcase_list(testsuite, testarea) +def run_test(testcase_list, report_flag, logger): + report = dt_report.Report() duration = 0 + if not testcase_list: + logger.warning('No test case will be executed.') + return + start_time = time.time() for testcase_name in testcase_list: logger.info('>>[testcase]: {}'.format(testcase_name)) - testcase = Testcase.get(testcase_name) - if testcase is None: - logger.error('Test case {} is not defined in testcase folder, ' - 'skipping.'.format(testcase_name)) - continue + testcase = dt_testcase.Testcase.get(testcase_name) run_testcase = True - # if testcase.exceed_max_retry_times(): - # run_testcase = False - - # if testcase.script_result_acquired(): - # run_testcase = False - + tc_start_time = datetime.fromtimestamp( + time.time()).strftime('%Y-%m-%d %H:%M:%S') if run_testcase: testcase.run() - - check_tc_result(testcase, logger) + tc_stop_time = datetime.fromtimestamp( + time.time()).strftime('%Y-%m-%d %H:%M:%S') + + result = report.check_tc_result(testcase) + if os.getenv('OPNFV_CI') == 'true': + dt_utils.push_results_to_db(case_name=testcase_name, + start_date=tc_start_time, + stop_date=tc_stop_time, + details=result, + logger=logger) + if dt_cfg.dovetail_config['stop']: + try: + if (not result or result['criteria'] == 'FAIL'): + logger.info('Stop because {} failed'.format(testcase_name)) + return + except KeyError as e: + logger.error('There is no key {}.'.format(e)) + logger.info('Stop because {} failed'.format(testcase_name)) + return end_time = time.time() duration = end_time - start_time - return duration - - -def check_tc_result(testcase, logger): - result_dir = dt_cfg.dovetail_config['result_dir'] - validate_type = testcase.validate_type() - functest_result = dt_cfg.dovetail_config['functest']['result']['file_path'] - dovetail_result = os.path.join(result_dir, - dt_cfg.dovetail_config['result_file']) - if dt_cfg.dovetail_config['report_dest'].startswith("http"): - if dt_utils.store_db_results(dt_cfg.dovetail_config['report_dest'], - dt_cfg.dovetail_config['build_tag'], - testcase.name(), dovetail_result, - logger): - logger.info("Results have been pushed to database and stored " - "with local file {}.".format(dovetail_result)) - else: - logger.error("Failed to push results to database.") - if dt_cfg.dovetail_config['report_dest'] == "file": - if validate_type.lower() == 'yardstick': - result_file = os.path.join(result_dir, testcase.name() + '.out') - elif validate_type.lower() == 'functest': - result_file = os.path.join(result_dir, functest_result) - elif validate_type.lower() == 'bottlenecks': - result_file = os.path.join(result_dir, testcase.name() + '.out') - else: - logger.error("Don't support {} now.".format(validate_type)) - return - if os.path.isfile(result_file): - logger.info( - "Results have been stored with file {}.".format(result_file)) - else: - logger.error( - "Failed to store results with file {}.".format(result_file)) - result = Report.get_result(testcase) - Report.check_result(testcase, result) - - -def validate_input(input_dict, check_dict, logger): - # for 'func_tag' and 'yard_tag' options - func_tag = input_dict['func_tag'] - yard_tag = input_dict['yard_tag'] - bott_tag = input_dict['bott_tag'] - valid_tag = check_dict['valid_docker_tag'] - if func_tag is not None and func_tag not in valid_tag: - logger.error("The input option 'func_tag' can't be {}, " - "valid values are {}.".format(func_tag, valid_tag)) - raise SystemExit(1) - if yard_tag is not None and yard_tag not in valid_tag: - logger.error("The input option 'yard_tag' can't be {}, " - "valid values are {}.".format(yard_tag, valid_tag)) - raise SystemExit(1) - if bott_tag is not None and bott_tag not in valid_tag: - logger.error("The input option 'bott_tag' can't be {}, " - "valid values are {}.".format(bott_tag, valid_tag)) - raise SystemExit(1) - - # for 'report' option - report = input_dict['report'] - if report: - if not (report.startswith("http") or report == "file"): - logger.error("Report type can't be {}, valid types are 'file' " - "and 'http'.".format(input_dict['report'])) - raise SystemExit(1) + report.generate(testcase_list, duration) + if report_flag: + report.save_logs() + return def filter_config(input_dict, logger): @@ -155,7 +108,7 @@ def filter_config(input_dict, logger): break except KeyError as e: logger.exception('KeyError {}.'.format(e)) - raise SystemExit(1) + raise SystemExit(EXIT_RUN_FAILED) if not configs: return None return configs @@ -164,150 +117,187 @@ def filter_config(input_dict, logger): def create_logs(): Container.create_log() Parser.create_log() - Report.create_log() - FunctestCrawler.create_log() - YardstickCrawler.create_log() - BottlenecksCrawler.create_log() - FunctestChecker.create_log() - YardstickChecker.create_log() - BottlenecksChecker.create_log() - Testcase.create_log() - Testsuite.create_log() - DockerRunner.create_log() - ShellRunner.create_log() + dt_report.Report.create_log() + dt_report.FunctestCrawler.create_log() + dt_report.FunctestK8sCrawler.create_log() + dt_report.YardstickCrawler.create_log() + dt_report.BottlenecksCrawler.create_log() + dt_report.OnapVtpCrawler.create_log() + dt_report.OnapVvpCrawler.create_log() + dt_report.FunctestChecker.create_log() + dt_report.FunctestK8sChecker.create_log() + dt_report.YardstickChecker.create_log() + dt_report.BottlenecksChecker.create_log() + dt_report.OnapVtpChecker.create_log() + dt_report.OnapVvpChecker.create_log() + dt_testcase.Testcase.create_log() + dt_testcase.Testsuite.create_log() + dt_test_runner.DockerRunner.create_log() + dt_test_runner.ShellRunner.create_log() def clean_results_dir(): result_path = dt_cfg.dovetail_config['result_dir'] if os.path.exists(result_path): if os.path.isdir(result_path): - cmd = 'sudo rm -rf %s/*' % (result_path) + cmd = 'rm -rf %s/*' % (result_path) dt_utils.exec_cmd(cmd, exit_on_error=False, exec_msg_on=False) else: - print "result_dir in dovetail_config.yml is not a directory." - raise SystemExit(1) + print('result_dir in dovetail_config.yml is not a directory.') + raise SystemExit(EXIT_RUN_FAILED) def get_result_path(): try: - dovetail_home = os.environ["DOVETAIL_HOME"] + dovetail_home = os.environ['DOVETAIL_HOME'] except Exception: print("ERROR: mandatory env variable 'DOVETAIL_HOME' is not found, " "please set in env_config.sh and source this file before " "running.") return None - result_path = os.path.join(dovetail_home, 'results') - dt_cfg.dovetail_config['result_dir'] = result_path - pre_config_path = os.path.join(dovetail_home, 'pre_config') - patch_set_path = os.path.join(dovetail_home, 'patch') - dt_cfg.dovetail_config['config_dir'] = pre_config_path - dt_cfg.dovetail_config['patch_dir'] = patch_set_path + dt_cfg.dovetail_config['result_dir'] = os.path.join(dovetail_home, + 'results') + dt_cfg.dovetail_config['images_dir'] = os.path.join(dovetail_home, + 'images') + dt_cfg.dovetail_config['config_dir'] = os.path.join(dovetail_home, + 'pre_config') + dt_cfg.dovetail_config['patch_dir'] = os.path.join(dovetail_home, + 'patches') + dt_cfg.dovetail_config['userconfig_dir'] = os.path.join(dovetail_home, + 'userconfig') return dovetail_home def copy_userconfig_files(logger): - dovetail_home = os.path.dirname(os.path.abspath(__file__)) - userconfig_path = os.path.join(dovetail_home, 'userconfig') - pre_config_path = dt_cfg.dovetail_config['config_dir'] - if not os.path.isdir(pre_config_path): - os.makedirs(pre_config_path) - cmd = 'sudo cp -r %s/* %s' % (userconfig_path, pre_config_path) + userconfig_path = dt_cfg.dovetail_config['userconfig_dir'] + if not os.path.isdir(userconfig_path): + os.makedirs(userconfig_path) + cmd = 'cp -r %s/* %s' % (constants.USERCONF_PATH, userconfig_path) dt_utils.exec_cmd(cmd, logger, exit_on_error=False) def copy_patch_files(logger): - dovetail_home = os.path.dirname(os.path.abspath(__file__)) - patch_path = os.path.join(dovetail_home, 'patch') patch_set_path = dt_cfg.dovetail_config['patch_dir'] if not os.path.isdir(patch_set_path): os.makedirs(patch_set_path) - cmd = 'sudo cp -r %s/* %s' % (patch_path, patch_set_path) + cmd = 'cp -a -r %s/* %s' % (constants.PATCH_PATH, patch_set_path) dt_utils.exec_cmd(cmd, logger, exit_on_error=False) -# env_init can source some env variable used in dovetail, such as -# when https+credential used, OS_CACERT -def env_init(logger): - openrc = os.path.join(dt_cfg.dovetail_config['config_dir'], - dt_cfg.dovetail_config['env_file']) - if not os.path.isfile(openrc): - logger.error("File {} does not exist.".format(openrc)) - dt_utils.source_env(openrc) +def update_deploy_scenario(logger, **kwargs): + if 'deploy_scenario' in kwargs and kwargs['deploy_scenario'] is not None: + os.environ['DEPLOY_SCENARIO'] = kwargs['deploy_scenario'] + logger.info('DEPLOY_SCENARIO : %s', os.environ['DEPLOY_SCENARIO']) -def check_hosts_file(logger): - hosts_file = os.path.join(dt_cfg.dovetail_config['config_dir'], - 'hosts.yaml') - if not os.path.isfile(hosts_file): - logger.warn("There is no hosts file {}, may be some issues with " - "domain name resolution.".format(hosts_file)) +def parse_cli(logger=None, **kwargs): + configs = filter_config(kwargs, logger) + if configs is not None: + dt_cfg.update_config(configs) + dt_cfg.dovetail_config['offline'] = kwargs['offline'] + dt_cfg.dovetail_config['noclean'] = kwargs['no_clean'] + dt_cfg.dovetail_config['stop'] = kwargs['stop'] + dt_cfg.dovetail_config['mandatory'] = kwargs['mandatory'] + dt_cfg.dovetail_config['optional'] = kwargs['optional'] + if kwargs['no_api_validation']: + dt_cfg.dovetail_config['no_api_validation'] = True + logger.warning('Strict API response validation DISABLED.') + else: + dt_cfg.dovetail_config['no_api_validation'] = False + + +def check_testcase_list(testcase_list, logger=None): + if testcase_list: + for tc in testcase_list: + if tc not in dt_testcase.Testcase.testcase_list: + logger.error('Test case {} is not defined.'.format(tc)) + return None + return testcase_list + logger.error('There is no test case to be executed.') + return None + + +def get_testcase_list(logger=None, **kwargs): + dt_testcase.Testcase.load() + testcase_list = kwargs['testcase'] + + # If specify 'testcase' on the CLI, ignore 'testsuite' and 'testarea'. In + # this case, all test cases are marked as mandatory=false in the result + # file because there is no testsuite to relate to. + # If 'testcase' is not specified on the CLI, check the combination of + # 'testsuite' and 'testarea' + if testcase_list: + return check_testcase_list(testcase_list, logger) + + testsuite_validation = False + testsuite = kwargs['testsuite'] + if testsuite in dt_cfg.dovetail_config['testsuite_supported']: + testsuite_validation = True + origin_testarea = kwargs['testarea'] + testarea_validation, testarea = dt_testcase.Testcase.check_testarea( + origin_testarea) + + if testsuite_validation and testarea_validation: + testsuite_yaml = load_testsuite(testsuite) + dt_cfg.dovetail_config['version'] = dt_utils.get_value_from_dict( + 'version', testsuite_yaml) + dt_cfg.dovetail_config['vnf_type'] = dt_utils.get_value_from_dict( + 'vnf_type', testsuite_yaml) + testcase_list = dt_testcase.Testcase.get_testcases_for_testsuite( + testsuite_yaml, testarea) + return check_testcase_list(testcase_list, logger) + elif not testsuite_validation: + logger.error('Test suite {} is not defined.'.format(testsuite)) + else: + logger.error('Test area {} is not defined.'.format(origin_testarea)) + return None def main(*args, **kwargs): """Dovetail compliance test entry!""" - build_tag = "daily-master-%s" % str(uuid.uuid1()) + build_tag = 'daily-master-%s' % str(uuid.uuid1()) dt_cfg.dovetail_config['build_tag'] = build_tag if not get_result_path(): return clean_results_dir() - if kwargs['debug']: - os.environ['DEBUG'] = 'true' + os.environ['DEBUG'] = 'true' if kwargs['debug'] else 'false' + os.environ['OPNFV_CI'] = 'true' if kwargs['opnfv_ci'] else 'false' + os.environ['validation'] = 'disabled' \ + if kwargs['no_api_validation'] else 'enabled' create_logs() logger = dt_logger.Logger('run').getLogger() + logger.info('================================================') logger.info('Dovetail compliance: {}!'.format(kwargs['testsuite'])) logger.info('================================================') logger.info('Build tag: {}'.format(dt_cfg.dovetail_config['build_tag'])) - env_init(logger) + parse_cli(logger, **kwargs) + update_deploy_scenario(logger, **kwargs) copy_userconfig_files(logger) copy_patch_files(logger) dt_utils.check_docker_version(logger) - dt_utils.get_openstack_endpoint(logger) - validate_input(kwargs, dt_cfg.dovetail_config['validate_input'], logger) - check_hosts_file(logger) - configs = filter_config(kwargs, logger) - - if configs is not None: - dt_cfg.update_config(configs) - if kwargs['report']: - if(kwargs['report'].endswith('/')): - kwargs['report'] = kwargs['report'][0:kwargs['report'].rfind('/')] - dt_cfg.dovetail_config['report_dest'] = kwargs['report'] - dt_cfg.update_cmds() + testcase_list = get_testcase_list(logger, **kwargs) - if kwargs['offline']: - dt_cfg.dovetail_config['offline'] = True - else: - dt_cfg.dovetail_config['offline'] = False + dovetail_home = os.environ['DOVETAIL_HOME'] + testcases_file = os.path.join(dovetail_home, 'results', 'testcases.json') + with open(testcases_file, "w") as f: + data = {'testsuite': kwargs['testsuite'], 'testcases': testcase_list} + f.write(json.dumps(data) + '\n') - dt_utils.get_hardware_info(logger) + if not testcase_list: + raise SystemExit(EXIT_RUN_FAILED) - origin_testarea = kwargs['testarea'] - testsuite_validation = False - if kwargs['testsuite'] in dt_cfg.dovetail_config['testsuite_supported']: - testsuite_validation = True - testarea_validation, testarea = Testcase.check_testarea(origin_testarea) - if testsuite_validation and testarea_validation: - testsuite_yaml = load_testsuite(kwargs['testsuite']) - load_testcase() - duration = run_test(testsuite_yaml, testarea, logger) - if dt_cfg.dovetail_config['report_dest'] == "file": - Report.generate(testsuite_yaml, testarea, duration) - if dt_cfg.dovetail_config['report_dest'].startswith("http"): - Report.save_logs() - else: - logger.error('Invalid input commands, testsuite {} testarea {}' - .format(kwargs['testsuite'], origin_testarea)) + run_test(testcase_list, kwargs['report'], logger) -dt_cfg.load_config_files() +dt_cfg.load_config_files(constants.CONF_PATH) dovetail_config = copy.deepcopy(dt_cfg.dovetail_config) CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) if dovetail_config['cli']['options'] is not None: for key, value in dovetail_config['cli']['options'].items(): if value is not None: - for k, v in value.items(): + for _, v in value.items(): flags = v['flags'] v.pop('flags') v.pop('path', None) @@ -315,7 +305,7 @@ if dovetail_config['cli']['options'] is not None: if dovetail_config['cli']['arguments'] is not None: for key, value in dovetail_config['cli']['arguments'].items(): if value is not None: - for k, v in value.items(): + for _, v in value.items(): flags = v['flags'] v.pop('flags') v.pop('path', None) |