From 3bf95fd09e2128f1cd762905c5d69cf66b56fd75 Mon Sep 17 00:00:00 2001 From: MatthewLi Date: Thu, 24 Nov 2016 01:56:41 -0500 Subject: dovetail tool: testarea argument added JIRA: DOVETAIL-71 1) argument testarea is added 2) terminology "scenario" change to "testsuite" 3) related documents amended 4) all word "certification" changed to "compliance" Change-Id: I038a9e04bf83cfdac20f0c59adb1841bf5470584 Signed-off-by: MatthewLi --- dovetail/cert/compliance_set.yml | 7 ------ dovetail/cert/proposed_tests.yml | 42 ------------------------------- dovetail/compliance/compliance_set.yml | 7 ++++++ dovetail/compliance/proposed_tests.yml | 42 +++++++++++++++++++++++++++++++ dovetail/conf/cmd_config.yml | 12 ++++++--- dovetail/conf/dovetail_config.py | 7 ++++-- dovetail/report.py | 16 +++++++----- dovetail/run.py | 46 +++++++++++++++++++++++----------- dovetail/testcase.py | 26 +++++++++---------- 9 files changed, 117 insertions(+), 88 deletions(-) delete mode 100644 dovetail/cert/compliance_set.yml delete mode 100644 dovetail/cert/proposed_tests.yml create mode 100644 dovetail/compliance/compliance_set.yml create mode 100644 dovetail/compliance/proposed_tests.yml (limited to 'dovetail') diff --git a/dovetail/cert/compliance_set.yml b/dovetail/cert/compliance_set.yml deleted file mode 100644 index b4108b4f..00000000 --- a/dovetail/cert/compliance_set.yml +++ /dev/null @@ -1,7 +0,0 @@ -certification_compliance_set: - name: certification_compliance_set - testcases_list: - # Temporarily, one test case kept here as default to run - # for use of software development/debug - # TO DO: will amend when compliance set is settled - - dovetail.ipv6.tc001 diff --git a/dovetail/cert/proposed_tests.yml b/dovetail/cert/proposed_tests.yml deleted file mode 100644 index 6d6c8d19..00000000 --- a/dovetail/cert/proposed_tests.yml +++ /dev/null @@ -1,42 +0,0 @@ -certification_proposed_tests: - name: certification_proposed_tests - testcases_list: - # TO DO: will adjust the dovetail tool to support in later patches - # run.py --name1 {**/proposed/compliance} --name2 {**/vim/ipv6,etc} - # vim operations - - dovetail.vimops.tc001 - - dovetail.vimops.tc002 - - dovetail.vimops.tc003 - - dovetail.vimops.tc004 - - dovetail.vimops.tc005 - - dovetail.vimops.tc006 - # ipv6 - - dovetail.ipv6.tc001 - - dovetail.ipv6.tc002 - - dovetail.ipv6.tc003 - - dovetail.ipv6.tc004 - - dovetail.ipv6.tc005 - - dovetail.ipv6.tc006 - - dovetail.ipv6.tc007 - - dovetail.ipv6.tc008 - - dovetail.ipv6.tc009 - - dovetail.ipv6.tc010 - - dovetail.ipv6.tc011 - - dovetail.ipv6.tc012 - - dovetail.ipv6.tc013 - - dovetail.ipv6.tc014 - - dovetail.ipv6.tc015 - - dovetail.ipv6.tc016 - - dovetail.ipv6.tc017 - - dovetail.ipv6.tc018 - - dovetail.ipv6.tc010 - - dovetail.ipv6.tc019 - - dovetail.ipv6.tc020 - - dovetail.ipv6.tc021 - - dovetail.ipv6.tc022 - - dovetail.ipv6.tc023 - - dovetail.ipv6.tc024 - - dovetail.ipv6.tc025 - # nfvi compute/network/storage - - dovetail.nfvi.tc001 - - dovetail.nfvi.tc002 diff --git a/dovetail/compliance/compliance_set.yml b/dovetail/compliance/compliance_set.yml new file mode 100644 index 00000000..df0d427d --- /dev/null +++ b/dovetail/compliance/compliance_set.yml @@ -0,0 +1,7 @@ +compliance_set: + name: compliance_set + testcases_list: + # Temporarily, one test case kept here as default to run + # for use of software development/debug + # TO DO: will amend when compliance set is settled + - dovetail.ipv6.tc001 diff --git a/dovetail/compliance/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml new file mode 100644 index 00000000..fcb7e03a --- /dev/null +++ b/dovetail/compliance/proposed_tests.yml @@ -0,0 +1,42 @@ +proposed_tests: + name: proposed_tests + testcases_list: + # TO DO: will adjust the dovetail tool to support in later patches + # run.py --name1 {**/proposed/compliance} --name2 {**/vim/ipv6,etc} + # vim operations + - dovetail.vimops.tc001 + - dovetail.vimops.tc002 + - dovetail.vimops.tc003 + - dovetail.vimops.tc004 + - dovetail.vimops.tc005 + - dovetail.vimops.tc006 + # ipv6 + - dovetail.ipv6.tc001 + - dovetail.ipv6.tc002 + - dovetail.ipv6.tc003 + - dovetail.ipv6.tc004 + - dovetail.ipv6.tc005 + - dovetail.ipv6.tc006 + - dovetail.ipv6.tc007 + - dovetail.ipv6.tc008 + - dovetail.ipv6.tc009 + - dovetail.ipv6.tc010 + - dovetail.ipv6.tc011 + - dovetail.ipv6.tc012 + - dovetail.ipv6.tc013 + - dovetail.ipv6.tc014 + - dovetail.ipv6.tc015 + - dovetail.ipv6.tc016 + - dovetail.ipv6.tc017 + - dovetail.ipv6.tc018 + - dovetail.ipv6.tc010 + - dovetail.ipv6.tc019 + - dovetail.ipv6.tc020 + - dovetail.ipv6.tc021 + - dovetail.ipv6.tc022 + - dovetail.ipv6.tc023 + - dovetail.ipv6.tc024 + - dovetail.ipv6.tc025 + # nfvi compute/network/storage + - dovetail.nfvi.tc001 + - dovetail.nfvi.tc002 diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml index c2108c58..04001774 100644 --- a/dovetail/conf/cmd_config.yml +++ b/dovetail/conf/cmd_config.yml @@ -37,12 +37,16 @@ cli: - '-d' help: 'CI_DEBUG for showing debug log.' non-envs: - scenario: + testsuite: flags: - - '--scenario' - - '-s' + - '--testsuite' default: 'compliance_set' - help: 'certification scenario.' + help: 'compliance testsuite.' + testarea: + flags: + - '--testarea' + default: 'full' + help: 'compliance testarea within testsuite' tag: flags: - '--tag' diff --git a/dovetail/conf/dovetail_config.py b/dovetail/conf/dovetail_config.py index 5ac23c43..5fe1e7b5 100644 --- a/dovetail/conf/dovetail_config.py +++ b/dovetail/conf/dovetail_config.py @@ -14,9 +14,12 @@ import re class DovetailConfig: - CERT_PATH = './cert/' + COMPLIANCE_PATH = './compliance/' TESTCASE_PATH = './testcase/' - SCENARIO_NAMING_FMT = 'certification_%s' + # testsuite supported tuple, should adjust accordingly + testsuite_supported = ('compliance_set', 'proposed_tests') + # testarea supported tuple, should adjust accordingly + testarea_supported = ('vimops', 'nfvi', 'ipv6') curr_path = os.path.dirname(os.path.abspath(__file__)) diff --git a/dovetail/report.py b/dovetail/report.py index 7dd8ba2d..2e544ad3 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -36,11 +36,15 @@ class Report: checker.check(testcase, db_result) @classmethod - def generate_json(cls, scenario_yaml): + def generate_json(cls, testsuite_yaml, testarea): report_obj = {} - report_obj['scenario'] = scenario_yaml['name'] + report_obj['testsuite'] = testsuite_yaml['name'] report_obj['testcases_list'] = [] - for testcase_name in scenario_yaml['testcases_list']: + testarea_list = [] + for value in testsuite_yaml['testcases_list']: + if value is not None and (testarea == 'full' or testarea in value): + testarea_list.append(value) + for testcase_name in testarea_list: testcase = Testcase.get(testcase_name) testcase_in_rpt = {} testcase_in_rpt['name'] = testcase_name @@ -66,8 +70,8 @@ class Report: return report_obj @classmethod - def generate(cls, scenario_yaml): - rpt_data = cls.generate_json(scenario_yaml) + def generate(cls, testsuite_yaml, testarea): + rpt_data = cls.generate_json(testsuite_yaml, testarea) rpt_text = '' split_line = '+-----------------------------------------------------' split_line += '---------------------+\n' @@ -76,7 +80,7 @@ class Report: +==========================================================================+\n\ | report |\n' rpt_text += split_line - rpt_text += '|scenario: %s\n' % rpt_data['scenario'] + rpt_text += '|testsuite: %s\n' % rpt_data['testsuite'] for testcase in rpt_data['testcases_list']: rpt_text += '| [testcase]: %s\t\t\t\t[%s]\n' % \ (testcase['name'], testcase['result']) diff --git a/dovetail/run.py b/dovetail/run.py index 86194389..fcc9da1a 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -16,16 +16,16 @@ import utils.dovetail_logger as dt_logger from container import Container from testcase import Testcase -from testcase import Scenario +from testcase import Testsuite from report import Report from conf.dovetail_config import DovetailConfig as dt_config logger = dt_logger.Logger('run.py').getLogger() -def load_scenario(scenario): - Scenario.load() - return Scenario.get(dt_config.SCENARIO_NAMING_FMT % scenario) +def load_testsuite(testsuite): + Testsuite.load() + return Testsuite.get(testsuite) def set_container_tags(option_str): @@ -40,8 +40,13 @@ def load_testcase(): Testcase.load() -def run_test(scenario): - for testcase_name in scenario['testcases_list']: +def run_test(testsuite, testarea): + testarea_list = [] + for value in testsuite['testcases_list']: + if value is not None and (testarea == 'full' or testarea in value): + testarea_list.append(value) + + for testcase_name in testarea_list: logger.info('>>[testcase]: %s' % (testcase_name)) testcase = Testcase.get(testcase_name) if testcase is None: @@ -102,10 +107,10 @@ def filter_env_options(input_dict): def main(*args, **kwargs): - """Dovetail certification test entry!""" - logger.info('=======================================') - logger.info('Dovetail certification: %s!' % (kwargs['scenario'])) - logger.info('=======================================') + """Dovetail compliance test entry!""" + logger.info('================================================') + logger.info('Dovetail compliance: %s!' % (kwargs['testsuite'])) + logger.info('================================================') validate_options(kwargs) envs_options = filter_env_options(kwargs) dt_config.update_envs(envs_options) @@ -113,12 +118,25 @@ def main(*args, **kwargs): dt_config.dovetail_config['functest']['envs']) logger.info('Your new envs for yardstick: %s' % dt_config.dovetail_config['yardstick']['envs']) - load_testcase() - scenario_yaml = load_scenario(kwargs['scenario']) + if 'tag' in kwargs and kwargs['tag'] is not None: set_container_tags(kwargs['tag']) - run_test(scenario_yaml) - Report.generate(scenario_yaml) + + load_testcase() + testsuite_yaml = load_testsuite(kwargs['testsuite']) + testarea = kwargs['testarea'] + testsuite_validation = False + testarea_validation = False + if (testarea == 'full') or (testarea in dt_config.testarea_supported): + testarea_validation = True + if kwargs['testsuite'] in dt_config.testsuite_supported: + testsuite_validation = True + if testsuite_validation and testarea_validation: + run_test(testsuite_yaml, testarea) + Report.generate(testsuite_yaml, testarea) + else: + logger.error('invalid input commands, testsuite %s testarea %s' % + (kwargs['testsuite'], testarea)) CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) diff --git a/dovetail/testcase.py b/dovetail/testcase.py index 6f0552b6..b5d28aa1 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -155,10 +155,10 @@ class Testcase: return None -class Scenario: +class Testsuite: - def __init__(self, scenario): - self.scenario = scenario + def __init__(self, testsuite): + self.testsuite = testsuite self.testcase_list = {} def get_test(self, testcase_name): @@ -166,20 +166,20 @@ class Scenario: return self.testcase_list[testcase_name] return None - scenario_list = {} + testsuite_list = {} @classmethod def load(cls): - for root, dirs, files in os.walk(dt_config.CERT_PATH): - for scenario_yaml in files: - with open(os.path.join(root, scenario_yaml)) as f: - scenario_yaml = yaml.safe_load(f) - cls.scenario_list.update(scenario_yaml) + for root, dirs, files in os.walk(dt_config.COMPLIANCE_PATH): + for testsuite_yaml in files: + with open(os.path.join(root, testsuite_yaml)) as f: + testsuite_yaml = yaml.safe_load(f) + cls.testsuite_list.update(testsuite_yaml) - logger.debug(cls.scenario_list) + logger.debug(cls.testsuite_list) @classmethod - def get(cls, scenario_name): - if scenario_name in cls.scenario_list: - return cls.scenario_list[scenario_name] + def get(cls, testsuite_name): + if testsuite_name in cls.testsuite_list: + return cls.testsuite_list[testsuite_name] return None -- cgit 1.2.3-korg