diff options
| -rw-r--r-- | docs/dovetailtool/dovetail.tool.installation.rst | 26 | ||||
| -rw-r--r-- | docs/dovetailtool/dovetail.tool.overview.rst | 2 | ||||
| -rw-r--r-- | dovetail/compliance/compliance_set.yml (renamed from dovetail/cert/compliance_set.yml) | 4 | ||||
| -rw-r--r-- | dovetail/compliance/proposed_tests.yml (renamed from dovetail/cert/proposed_tests.yml) | 4 | ||||
| -rw-r--r-- | dovetail/conf/cmd_config.yml | 12 | ||||
| -rw-r--r-- | dovetail/conf/dovetail_config.py | 7 | ||||
| -rw-r--r-- | dovetail/report.py | 16 | ||||
| -rwxr-xr-x | dovetail/run.py | 46 | ||||
| -rw-r--r-- | dovetail/testcase.py | 26 |
9 files changed, 88 insertions, 55 deletions
diff --git a/docs/dovetailtool/dovetail.tool.installation.rst b/docs/dovetailtool/dovetail.tool.installation.rst index 17290a35..2f544480 100644 --- a/docs/dovetailtool/dovetail.tool.installation.rst +++ b/docs/dovetailtool/dovetail.tool.installation.rst @@ -69,22 +69,19 @@ At this point the environment is now ready for Dovetail execution. Compliance and certification test cases ---------------------------------------- -The compliance and certification test cases can be defined under the ``/dovetail/cert`` +The compliance and certification test cases can be defined under the ``/dovetail/compliance`` directory, which is defined in yaml format. A sample file named ``compliance_set.yml`` is provided as follows: :: - certification_compliance_set: - name: certification_compliance_set + compliance_set: + name: compliance_set testcases_list: - dovetail.ipv6.tc001 The testcase listed here is dovetail.ipv6.tc001, defined within ``dovetail/testcase``. -Note: if a new test case yaml file is created, its name should start with ``certification_``, -in similiar fashion as the sample file ``certification_compliance_set``. - Running Dovetail tool --------------------- @@ -92,10 +89,17 @@ After environment preparation is complete and test cases added, the Dovetail too :: - python run.py --scenario compliance_set + python run.py --testsuite compliance_set + +The value ``compliance_set`` passed to the ``testsuite`` flag can be replaced with the test cases yaml file. +If not argument is given, the compliance_set testsuite will be run as the default. + +Moreover, the testcases in given testarea can be run with ``testarea`` command line argument, such as +testarea ``ipv6`` in ``compliance_set`` + +:: -The value ``compliance_set`` passed to the ``scenario`` flag can be replaced with the test cases yaml file. -If not argument is given, the compliance_set scenario will be run as the default. + python run.py --testsuite compliance_set --testarea ipv6 Running Dovetail in a Docker container ######################################## @@ -145,10 +149,10 @@ Attach to the container by starting it and obtaining a bash prompt with :: Inside the container the following commands can be executed to trigger the testcases :: cd /home/opnfv/dovetail/dovetail - python run.py --scenario compliance_set + python run.py --testsuite compliance_set Results Output ############### The running log is stored in ``/home/opnfv/dovetail/results/dovetail.log``. -The certification report is stored in ``/home/opnfv/dovetail/results/dovetail_report.txt``. +The compliance report is stored in ``/home/opnfv/dovetail/results/dovetail_report.txt``. diff --git a/docs/dovetailtool/dovetail.tool.overview.rst b/docs/dovetailtool/dovetail.tool.overview.rst index a1133f34..c4b7fa22 100644 --- a/docs/dovetailtool/dovetail.tool.overview.rst +++ b/docs/dovetailtool/dovetail.tool.overview.rst @@ -10,7 +10,7 @@ Dovetail Tool What is Dovetail tool ##################### -A toolset for providing an evaluation of a deployed cloud environment for OPNFV certification. +A toolset for providing an evaluation of a deployed cloud environment for OPNFV compliance. Overview ######## diff --git a/dovetail/cert/compliance_set.yml b/dovetail/compliance/compliance_set.yml index b4108b4f..df0d427d 100644 --- a/dovetail/cert/compliance_set.yml +++ b/dovetail/compliance/compliance_set.yml @@ -1,5 +1,5 @@ -certification_compliance_set: - name: certification_compliance_set +compliance_set: + name: compliance_set testcases_list: # Temporarily, one test case kept here as default to run # for use of software development/debug diff --git a/dovetail/cert/proposed_tests.yml b/dovetail/compliance/proposed_tests.yml index 6d6c8d19..fcb7e03a 100644 --- a/dovetail/cert/proposed_tests.yml +++ b/dovetail/compliance/proposed_tests.yml @@ -1,5 +1,5 @@ -certification_proposed_tests: - name: certification_proposed_tests +proposed_tests: + name: proposed_tests testcases_list: # TO DO: will adjust the dovetail tool to support in later patches # run.py --name1 {**/proposed/compliance} --name2 {**/vim/ipv6,etc} diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml index c2108c58..04001774 100644 --- a/dovetail/conf/cmd_config.yml +++ b/dovetail/conf/cmd_config.yml @@ -37,12 +37,16 @@ cli: - '-d' help: 'CI_DEBUG for showing debug log.' non-envs: - scenario: + testsuite: flags: - - '--scenario' - - '-s' + - '--testsuite' default: 'compliance_set' - help: 'certification scenario.' + help: 'compliance testsuite.' + testarea: + flags: + - '--testarea' + default: 'full' + help: 'compliance testarea within testsuite' tag: flags: - '--tag' diff --git a/dovetail/conf/dovetail_config.py b/dovetail/conf/dovetail_config.py index 5ac23c43..5fe1e7b5 100644 --- a/dovetail/conf/dovetail_config.py +++ b/dovetail/conf/dovetail_config.py @@ -14,9 +14,12 @@ import re class DovetailConfig: - CERT_PATH = './cert/' + COMPLIANCE_PATH = './compliance/' TESTCASE_PATH = './testcase/' - SCENARIO_NAMING_FMT = 'certification_%s' + # testsuite supported tuple, should adjust accordingly + testsuite_supported = ('compliance_set', 'proposed_tests') + # testarea supported tuple, should adjust accordingly + testarea_supported = ('vimops', 'nfvi', 'ipv6') curr_path = os.path.dirname(os.path.abspath(__file__)) diff --git a/dovetail/report.py b/dovetail/report.py index 7dd8ba2d..2e544ad3 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -36,11 +36,15 @@ class Report: checker.check(testcase, db_result) @classmethod - def generate_json(cls, scenario_yaml): + def generate_json(cls, testsuite_yaml, testarea): report_obj = {} - report_obj['scenario'] = scenario_yaml['name'] + report_obj['testsuite'] = testsuite_yaml['name'] report_obj['testcases_list'] = [] - for testcase_name in scenario_yaml['testcases_list']: + testarea_list = [] + for value in testsuite_yaml['testcases_list']: + if value is not None and (testarea == 'full' or testarea in value): + testarea_list.append(value) + for testcase_name in testarea_list: testcase = Testcase.get(testcase_name) testcase_in_rpt = {} testcase_in_rpt['name'] = testcase_name @@ -66,8 +70,8 @@ class Report: return report_obj @classmethod - def generate(cls, scenario_yaml): - rpt_data = cls.generate_json(scenario_yaml) + def generate(cls, testsuite_yaml, testarea): + rpt_data = cls.generate_json(testsuite_yaml, testarea) rpt_text = '' split_line = '+-----------------------------------------------------' split_line += '---------------------+\n' @@ -76,7 +80,7 @@ class Report: +==========================================================================+\n\ | report |\n' rpt_text += split_line - rpt_text += '|scenario: %s\n' % rpt_data['scenario'] + rpt_text += '|testsuite: %s\n' % rpt_data['testsuite'] for testcase in rpt_data['testcases_list']: rpt_text += '| [testcase]: %s\t\t\t\t[%s]\n' % \ (testcase['name'], testcase['result']) diff --git a/dovetail/run.py b/dovetail/run.py index 86194389..fcc9da1a 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -16,16 +16,16 @@ import utils.dovetail_logger as dt_logger from container import Container from testcase import Testcase -from testcase import Scenario +from testcase import Testsuite from report import Report from conf.dovetail_config import DovetailConfig as dt_config logger = dt_logger.Logger('run.py').getLogger() -def load_scenario(scenario): - Scenario.load() - return Scenario.get(dt_config.SCENARIO_NAMING_FMT % scenario) +def load_testsuite(testsuite): + Testsuite.load() + return Testsuite.get(testsuite) def set_container_tags(option_str): @@ -40,8 +40,13 @@ def load_testcase(): Testcase.load() -def run_test(scenario): - for testcase_name in scenario['testcases_list']: +def run_test(testsuite, testarea): + testarea_list = [] + for value in testsuite['testcases_list']: + if value is not None and (testarea == 'full' or testarea in value): + testarea_list.append(value) + + for testcase_name in testarea_list: logger.info('>>[testcase]: %s' % (testcase_name)) testcase = Testcase.get(testcase_name) if testcase is None: @@ -102,10 +107,10 @@ def filter_env_options(input_dict): def main(*args, **kwargs): - """Dovetail certification test entry!""" - logger.info('=======================================') - logger.info('Dovetail certification: %s!' % (kwargs['scenario'])) - logger.info('=======================================') + """Dovetail compliance test entry!""" + logger.info('================================================') + logger.info('Dovetail compliance: %s!' % (kwargs['testsuite'])) + logger.info('================================================') validate_options(kwargs) envs_options = filter_env_options(kwargs) dt_config.update_envs(envs_options) @@ -113,12 +118,25 @@ def main(*args, **kwargs): dt_config.dovetail_config['functest']['envs']) logger.info('Your new envs for yardstick: %s' % dt_config.dovetail_config['yardstick']['envs']) - load_testcase() - scenario_yaml = load_scenario(kwargs['scenario']) + if 'tag' in kwargs and kwargs['tag'] is not None: set_container_tags(kwargs['tag']) - run_test(scenario_yaml) - Report.generate(scenario_yaml) + + load_testcase() + testsuite_yaml = load_testsuite(kwargs['testsuite']) + testarea = kwargs['testarea'] + testsuite_validation = False + testarea_validation = False + if (testarea == 'full') or (testarea in dt_config.testarea_supported): + testarea_validation = True + if kwargs['testsuite'] in dt_config.testsuite_supported: + testsuite_validation = True + if testsuite_validation and testarea_validation: + run_test(testsuite_yaml, testarea) + Report.generate(testsuite_yaml, testarea) + else: + logger.error('invalid input commands, testsuite %s testarea %s' % + (kwargs['testsuite'], testarea)) CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) diff --git a/dovetail/testcase.py b/dovetail/testcase.py index 6f0552b6..b5d28aa1 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -155,10 +155,10 @@ class Testcase: return None -class Scenario: +class Testsuite: - def __init__(self, scenario): - self.scenario = scenario + def __init__(self, testsuite): + self.testsuite = testsuite self.testcase_list = {} def get_test(self, testcase_name): @@ -166,20 +166,20 @@ class Scenario: return self.testcase_list[testcase_name] return None - scenario_list = {} + testsuite_list = {} @classmethod def load(cls): - for root, dirs, files in os.walk(dt_config.CERT_PATH): - for scenario_yaml in files: - with open(os.path.join(root, scenario_yaml)) as f: - scenario_yaml = yaml.safe_load(f) - cls.scenario_list.update(scenario_yaml) + for root, dirs, files in os.walk(dt_config.COMPLIANCE_PATH): + for testsuite_yaml in files: + with open(os.path.join(root, testsuite_yaml)) as f: + testsuite_yaml = yaml.safe_load(f) + cls.testsuite_list.update(testsuite_yaml) - logger.debug(cls.scenario_list) + logger.debug(cls.testsuite_list) @classmethod - def get(cls, scenario_name): - if scenario_name in cls.scenario_list: - return cls.scenario_list[scenario_name] + def get(cls, testsuite_name): + if testsuite_name in cls.testsuite_list: + return cls.testsuite_list[testsuite_name] return None |