diff options
72 files changed, 771 insertions, 333 deletions
@@ -2,7 +2,7 @@ # Packages *.egg -*.egg-info +.testrepository dist build eggs diff --git a/docker/Dockerfile b/docker/Dockerfile index 19dc3619..2e919ee2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -13,13 +13,7 @@ RUN \ apt-transport-https \ --no-install-recommends \ && \ - apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D \ -&& \ - mkdir -p /etc/apt/sources.list.d \ -&& \ - echo deb https://apt.dockerproject.org/repo ubuntu-trusty main > /etc/apt/sources.list.d/docker.list \ -&& \ - apt-get update && apt-get install -y docker-engine=1.12.3-0~trusty \ + apt-get update && apt-get -y install docker.io \ && \ pip install pyyaml \ click \ diff --git a/docker/Dockerfile.centos7 b/docker/Dockerfile.centos7 new file mode 100644 index 00000000..0107cb7b --- /dev/null +++ b/docker/Dockerfile.centos7 @@ -0,0 +1,20 @@ +FROM centos:centos7 +MAINTAINER Tomofumi Hayashi <tohayash@redhat.com> +LABEL version="0.1" description="OPNFV Dovetail Docker Container" + +ARG BRANCH=master + +RUN yum update -y && yum install -y sudo iproute epel-release && \ + yum install -y python-pip git docker && \ + sed -ie 's/requiretty/!requiretty/g' /etc/sudoers && \ + pip install pyyaml click jinja2 + +ENV HOME /home/opnfv +ENV REPOS_DIR ${HOME}/dovetail +WORKDIR /home/opnfv + +RUN git config --global http.sslVerify false && \ + git clone --depth 1 -b $BRANCH https://gerrit.opnfv.org/gerrit/dovetail.git ${REPOS_DIR} && \ + mkdir -p ${REPOS_DIR}/results + +WORKDIR ${REPOS_DIR}/dovetail diff --git a/docs/dovetailtool/dovetail.tool.installation.rst b/docs/dovetailtool/dovetail.tool.installation.rst index 2f544480..93871e76 100644 --- a/docs/dovetailtool/dovetail.tool.installation.rst +++ b/docs/dovetailtool/dovetail.tool.installation.rst @@ -101,6 +101,22 @@ testarea ``ipv6`` in ``compliance_set`` python run.py --testsuite compliance_set --testarea ipv6 +Dovetail provides some sets, ``debug``, ``proposed_tests`` and ``compliance_set``, +``debug`` is used for locally and Continuous Integration(CI) developing purpose, +which provides typical testcase examples, feel free to edit it when develops locally, such as +only to run a testcase which only takes minutes. ``proposed_tests`` is the testcase +candidate which mainly comes from the wiki link +https://wiki.opnfv.org/display/dovetail/Dovetail+Test+Areas+and+Test+Cases. +``compliance_set`` is used for compliance. Moreover, dovetail tool can be easily +extended to support more complicated compliance requirements, +such as feature set based or scenario based compliance. + +If you want to run ``debug``, just run with + +:: + + python run.py --testsuite debug + Running Dovetail in a Docker container ######################################## @@ -115,6 +131,17 @@ Pull Dovetail Docker image from public Dockerhub <Tag> here is the version, 'latest' is used for the master branch. +(Optional) Create Docker image from Dockerfile +----------------------------------------------- +Instead of an official Docker image, you can build your own Docker image from +Dockerfile(s) located under the ``docker/`` directory. The ``Dockerfile`` +is based on Ubuntu and the ``Dockerfile.centos7`` file is for a CentOS based +Docker image. + +:: + + sudo docker build -t <your_new_image_name> -f <your_Dockerfile> . + Dovetail Docker container creation ---------------------------------- diff --git a/docs/dovetailtool/index.rst b/docs/dovetailtool/index.rst index f528a4f7..676c72db 100644 --- a/docs/dovetailtool/index.rst +++ b/docs/dovetailtool/index.rst @@ -12,6 +12,6 @@ Dovetail Overview dovetail.tool.overview.rst dovetail.tool.installation.rst - dovetail.tool.configuation.rst + dovetail.tool.configuration.rst dovetail.tool.configtemplate.rst dovetail.tool.cli.rst diff --git a/docs/plugfest/2016.05/test_plan/application/testcases.rst b/docs/plugfest/2016.05/test_plan/application/testcases.rst index 0e49c5c8..bb1bba76 100644 --- a/docs/plugfest/2016.05/test_plan/application/testcases.rst +++ b/docs/plugfest/2016.05/test_plan/application/testcases.rst @@ -16,8 +16,3 @@ files in this directory that are imported below the toctree command below. .. toctree:: :maxdepth: 2 - - ./testcase1.rst - ./testcase2.rst - ./testcase_etc.rst - diff --git a/docs/plugfest/2016.05/test_plan/deployment/testcase1.rst b/docs/plugfest/2016.05/test_plan/deployment/testcase1.rst index 6014bda3..f583729a 100644 --- a/docs/plugfest/2016.05/test_plan/deployment/testcase1.rst +++ b/docs/plugfest/2016.05/test_plan/deployment/testcase1.rst @@ -9,12 +9,12 @@ Purpose This test case is the first and therefore most important test case. As the first you should take care to do all the set-up found in the -`Plugfest setup and configuration`_ section. +`Plugfest setup and configuration` section. References ^^^^^^^^^^ -Somewhere in `ETSI`_ there is a useful link. +Somewhere in [ETSI]_ there is a useful link. Test Setup ^^^^^^^^^^ @@ -36,5 +36,4 @@ Expected Results 2. This results is used to indicate the pass/fail status for the test case executor. 3. To pass a test case, it all individual metrics would be expected to “pass”. -.. _ETSI: http://www.etsi.org/ - +.. [ETSI] http://www.etsi.org/ diff --git a/docs/plugfest/2016.05/test_plan/deployment/testcase2.rst b/docs/plugfest/2016.05/test_plan/deployment/testcase2.rst index 9c906146..579bbe7b 100644 --- a/docs/plugfest/2016.05/test_plan/deployment/testcase2.rst +++ b/docs/plugfest/2016.05/test_plan/deployment/testcase2.rst @@ -8,13 +8,13 @@ Purpose ^^^^^^^ This test case is the second and no one really cares for it. -If you have it in you please refer to the `Plugfest setup and configuration`_ section, +If you have it in you please refer to the `Plugfest setup and configuration` section, it's OK to simply amble on at this stage. *(It's only the second test case)* References ^^^^^^^^^^ -Somewhere in `ETSI`_ there may be a useful link. +Somewhere in [ETSI]_ there may be a useful link. Test Setup ^^^^^^^^^^ @@ -36,5 +36,5 @@ Expected Results 2. This results is used to indicate the pass/fail status for the test case itself. 3. To pass a test case, metrics would be expected to “pass” 1 and not stray to 3. -.. _ETSI: http://www.etsi.org/ +.. [ETSI] http://www.etsi.org/ diff --git a/docs/plugfest/2016.05/test_plan/index.rst b/docs/plugfest/2016.05/test_plan/index.rst index 61de3bd5..e4bf5e91 100644 --- a/docs/plugfest/2016.05/test_plan/index.rst +++ b/docs/plugfest/2016.05/test_plan/index.rst @@ -16,4 +16,5 @@ OPNFV Plugfest - test plan ./deployment/testcases.rst ./integration/testcases.rst ./application/testcases.rst + ./templates/template.rst diff --git a/docs/plugfest/2016.05/test_plan/integration/testcases.rst b/docs/plugfest/2016.05/test_plan/integration/testcases.rst index d0f01156..c7211f96 100644 --- a/docs/plugfest/2016.05/test_plan/integration/testcases.rst +++ b/docs/plugfest/2016.05/test_plan/integration/testcases.rst @@ -18,8 +18,3 @@ files in this directory that are imported below the toctree command below. .. toctree:: :maxdepth: 2 - - ./testcase1.rst - ./testcase2.rst - ./testcase_etc.rst - diff --git a/docs/testsuites/ipv6/index.rst b/docs/testsuites/ipv6/index.rst index 72fd6a89..a806d644 100644 --- a/docs/testsuites/ipv6/index.rst +++ b/docs/testsuites/ipv6/index.rst @@ -10,6 +10,10 @@ OPNFV IPv6 Compliance Test Plan :maxdepth: 2 ./testplan.rst + ./testprocedure.rst + ./testspecification.rst + ./designspecification.rst ./ipv6.tc001.specification.rst ./ipv6.tc026.specification.rst + ./ipv6_all_testcases.rst diff --git a/docs/testsuites/ipv6/ipv6.tc001.specification.rst b/docs/testsuites/ipv6/ipv6.tc001.specification.rst index 350ff73d..5afb2095 100644 --- a/docs/testsuites/ipv6/ipv6.tc001.specification.rst +++ b/docs/testsuites/ipv6/ipv6.tc001.specification.rst @@ -6,8 +6,6 @@ Dovetail IPv6 tc001 specification - Bulk Creation and Deletion of IPv6 Networks, Ports and Subnets ================================================================================================== -.. table:: - :class: longtable +-----------------------+----------------------------------------------------------------------------------------------------+ |test case name |Bulk creation and deletion of IPv6 networks, ports and subnets | @@ -45,17 +43,17 @@ Dovetail IPv6 tc001 specification - Bulk Creation and Deletion of IPv6 Networks, | | | +-----------------------+----------------------------------------------------------------------------------------------------+ |input specifications |The parameters needed to execute Neutron network APIs. | -| |Refer to Neutron Networking API v2.0 [1]_ [2]_ | +| |Refer to Neutron Networking API v2.0 `[1]`_ `[2]`_ | +-----------------------+----------------------------------------------------------------------------------------------------+ |output specifications |The responses after executing Network network APIs. | -| |Refer to Neutron Networking API v2.0 [1]_ [2]_ | +| |Refer to Neutron Networking API v2.0 `[1]`_ `[2]`_ | +-----------------------+----------------------------------------------------------------------------------------------------+ |pass/fail criteria |If normal response code 200 is returned, the test passes. | | |Otherwise, the test fails with various error codes. | -| |Refer to Neutron Networking API v2.0 [1]_ [2]_ | +| |Refer to Neutron Networking API v2.0 `[1]`_ `[2]`_ | +-----------------------+----------------------------------------------------------------------------------------------------+ |test report |TBD | +-----------------------+----------------------------------------------------------------------------------------------------+ -.._[1]: http://developer.openstack.org/api-ref/networking/v2/ -.._[2]: http://wiki.openstack.org/wiki/Neutron/APIv2-specification +.. _`[1]`: http://developer.openstack.org/api-ref/networking/v2/ +.. _`[2]`: http://wiki.openstack.org/wiki/Neutron/APIv2-specification diff --git a/docs/testsuites/ipv6/ipv6.tc026.specification.rst b/docs/testsuites/ipv6/ipv6.tc026.specification.rst index a4bb6586..e7fd82e7 100644 --- a/docs/testsuites/ipv6/ipv6.tc026.specification.rst +++ b/docs/testsuites/ipv6/ipv6.tc026.specification.rst @@ -6,8 +6,6 @@ Dovetail IPv6 tc026 specification - Service VM as IPv6 vRouter ============================================================== -.. table:: - :class: longtable +-----------------------+--------------------------------------------------------------------------+ |test case name |Service VM as IPv6 vRouter | diff --git a/docs/userguide/02-certification_criteria.rst b/docs/userguide/02-certification_criteria.rst index e69de29b..7665f9a0 100644 --- a/docs/userguide/02-certification_criteria.rst +++ b/docs/userguide/02-certification_criteria.rst @@ -0,0 +1,3 @@ +================== +Dovetail Criteria +================== diff --git a/docs/userguide/03-certification_progress.rst b/docs/userguide/03-certification_progress.rst index e69de29b..6ee02012 100644 --- a/docs/userguide/03-certification_progress.rst +++ b/docs/userguide/03-certification_progress.rst @@ -0,0 +1,3 @@ +================== +Dovetail Progress +================== diff --git a/dovetail/compliance/debug.yml b/dovetail/compliance/debug.yml index a0a2d3ee..8cc4b36c 100644 --- a/dovetail/compliance/debug.yml +++ b/dovetail/compliance/debug.yml @@ -4,6 +4,7 @@ debug: name: debug testcases_list: + - dovetail.example.tc002 - dovetail.ipv6.tc001 - dovetail.nfvi.tc001 - dovetail.nfvi.tc002 diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml index 4e3d0110..e2159ca7 100644 --- a/dovetail/conf/cmd_config.yml +++ b/dovetail/conf/cmd_config.yml @@ -1,42 +1,58 @@ cli: arguments: - envs: + config: # This is a simple example of arguments. # Dovetail has no need of this kind of parameters currently. # The arguments must be given orderly at the run-time. # # docker_tag: # flags: 'docker_tag' - non-envs: + # path: + # - 'functest/docker_tag' + # - 'yardstick/docker_tag' + control: options: - envs: + config: SUT_TYPE: flags: - '--SUT_TYPE' - '-t' + path: + - 'functest/envs' + - 'yardstick/envs' help: 'Installer type of the system under test (SUT).' SUT_IP: flags: - '--SUT_IP' - '-i' + path: + - 'functest/envs' + - 'yardstick/envs' help: 'IP of the system under test (SUT).' - DEPLOY_SCENARIO: + CON_DEBUG: flags: - - '--DEPLOY_SCENARIO' - - '-S' - help: 'DEPLOY_SCENARIO of the system under test (SUT).' - DEPLOY_TYPE: + - '--CON_DEBUG' + - '-c' + path: + - 'functest/envs' + - 'yardstick/envs' + help: 'True for showing debug log in functest/yardstick container.' + yard_tag: flags: - - '--DEPLOY_TYPE' - - '-T' - help: 'DEPLOY_TYPE of the system under test (SUT).' - DEBUG: + - '--yard_tag' + - '-y' + path: + - 'yardstick/docker_tag' + help: 'Overwrite tag for yardstick docker container (e.g. stable or latest)' + func_tag: flags: - - '--DEBUG' - - '-d' - help: 'DEBUG for showing debug log.' - non-envs: + - '--func_tag' + - '-f' + path: + - 'functest/docker_tag' + help: 'Overwrite tag for functest docker container (e.g. stable or latest)' + control: testsuite: flags: - '--testsuite' @@ -47,8 +63,8 @@ cli: - '--testarea' default: 'full' help: 'compliance testarea within testsuite' - tag: + debug: flags: - - '--tag' - - '-o' - help: 'Overwrite tags for each docker container (e.g. "functest:stable,yardstick:latest")' + - '--debug' + - '-d' + help: 'True for showing debug log in screen.' diff --git a/dovetail/conf/dovetail_config.py b/dovetail/conf/dovetail_config.py index 14dc59a3..1087bdc5 100644 --- a/dovetail/conf/dovetail_config.py +++ b/dovetail/conf/dovetail_config.py @@ -14,15 +14,14 @@ import re class DovetailConfig: - COMPLIANCE_PATH = './compliance/' - TESTCASE_PATH = './testcase/' - # testsuite supported tuple, should adjust accordingly - testsuite_supported = ('compliance_set', 'proposed_tests', 'debug') - # testarea supported tuple, should adjust accordingly - testarea_supported = ('vimops', 'nfvi', 'ipv6') - dovetail_config = {} + CMD_NAME_TRANS = { + 'SUT_TYPE': 'INSTALLER_TYPE', + 'SUT_IP': 'INSTALLER_IP', + 'CON_DEBUG': 'CI_DEBUG', + } + @classmethod def load_config_files(cls): curr_path = os.path.dirname(os.path.abspath(__file__)) @@ -42,31 +41,61 @@ class DovetailConfig: @classmethod def cmd_name_trans(cls, cmd_name): key = cmd_name.upper() - if key == 'SUT_TYPE': - key = 'INSTALLER_TYPE' - if key == 'SUT_IP': - key = 'INSTALLER_IP' - return key + return cls.CMD_NAME_TRANS.get(key, key) + + # Analyze the kind of the giving path, + # return true for env path, + # return false for non_env path. + @classmethod + def is_env_path(cls, path): + if len(path) == 2: + test_project = cls.dovetail_config['test_project'] + if path[0] in test_project and path[1] == 'envs': + return True + else: + return False + # update dovetail_config dict with the giving path. + # if path is in the dovetail_config dict, its value will be replaced. + # if path is not in the dict, it will be added as a new item of the dict. @classmethod - def update_envs(cls, options): - for item in options: - key = cls.cmd_name_trans(item) - if not options[item] and key in os.environ: - options[item] = os.environ[key] - if options[item]: - cls.update_config_envs('functest', key) - cls.update_config_envs('yardstick', key) + def update_config(cls, config_dict): + for key, value in config_dict.items(): + path_list = [] + for item in value['path']: + path_list.append([(k.strip()) for k in item.split('/')]) + for path in path_list: + if cls.is_env_path(path): + cls.update_envs(key, path, value['value']) + else: + cls.update_non_envs(path, value['value']) @classmethod - def update_config_envs(cls, script_type, key): - if key == 'DEBUG': - os.environ['CI_DEBUG'] = os.environ[key] - envs = cls.dovetail_config[script_type]['envs'] + def update_envs(cls, key, path, value): + key = cls.cmd_name_trans(key) + if not value and key in os.environ: + value = os.environ[key] + if value: + cls.update_config_envs(path[0], key, value) + + @classmethod + def update_config_envs(cls, validate_type, key, value): + envs = cls.dovetail_config[validate_type]['envs'] old_value = re.findall(r'\s+%s=(.*?)(\s+|$)' % key, envs) if old_value == []: - envs += ' -e ' + key + '=' + os.environ[key] + envs += ' -e ' + key + '=' + value else: - envs = envs.replace(old_value[0][0], os.environ[key]) - cls.dovetail_config[script_type]['envs'] = envs + envs = envs.replace(old_value[0][0], value) + cls.dovetail_config[validate_type]['envs'] = envs return envs + + @staticmethod + def set_leaf_dict(dic, path, value): + for key in path[:-1]: + dic = dic.setdefault(key, {}) + dic[path[-1]] = value + + @classmethod + def update_non_envs(cls, path, value): + if value: + cls.set_leaf_dict(cls.dovetail_config, path, value) diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml index c4131a91..c01394ed 100644 --- a/dovetail/conf/dovetail_config.yml +++ b/dovetail/conf/dovetail_config.yml @@ -5,6 +5,19 @@ report_file: 'dovetail_report.txt' cli_file_name: 'cmd_config.yml' # TO DO: once version scheme settled, adjust accordingly repo: 'https://github.com/opnfv/dovetail/tree/master/' +COMPLIANCE_PATH: ./compliance/ +TESTCASE_PATH: ./testcase/ +# testsuite supported, should adjust accordingly +testsuite_supported: + - compliance_set + - proposed_tests + - debug +# testarea supported, should adjust accordingly +testarea_supported: + - vimops + - nfvi + - ipv6 + - example # used for testcase cmd template in jinja2 format # we have two variables available now @@ -13,11 +26,25 @@ repo: 'https://github.com/opnfv/dovetail/tree/master/' parameters: - name: testcase path: '("name",)' - - name: script_testcase - path: '("scripts", "testcase")' + - name: validate_testcase + path: '("validate", "testcase")' include_config: - functest_config.yml - yardstick_config.yml +test_project: + - 'yardstick' + - 'functest' + +validate_input: + valid_sut_type: + - 'compass' + - 'fuel' + - 'joid' + - 'apex' + + valid_docker_tag: + - 'stable' + - 'latest' diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml index d32fe87f..682d19bf 100644 --- a/dovetail/conf/functest_config.yml +++ b/dovetail/conf/functest_config.yml @@ -6,15 +6,12 @@ functest: -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal' opts: '-id --privileged=true' pre_condition: - cmds: - - 'echo test for precondition' - testcase: - cmds: - - 'functest env prepare' - - 'functest testcase run {{script_testcase}}' + - 'echo test for precondition' + cmds: + - 'functest env prepare' + - 'functest testcase run {{validate_testcase}}' post_condition: - cmds: - - '' + - 'echo test for postcondition' result: dir: '/home/opnfv/functest/results' store_type: 'file' diff --git a/dovetail/conf/yardstick_config.yml b/dovetail/conf/yardstick_config.yml index f7f05bcc..d13cf2c6 100644 --- a/dovetail/conf/yardstick_config.yml +++ b/dovetail/conf/yardstick_config.yml @@ -7,21 +7,18 @@ yardstick: -e EXTERNAL_NETWORK=ext-net' opts: '-id --privileged=true' pre_condition: - cmds: - - 'source /home/opnfv/repos/yardstick/tests/ci/prepare_env.sh && + - 'source /home/opnfv/repos/yardstick/tests/ci/prepare_env.sh && source /home/opnfv/repos/yardstick/tests/ci/clean_images.sh && cleanup' - - 'source /home/opnfv/repos/yardstick/tests/ci/prepare_env.sh && + - 'source /home/opnfv/repos/yardstick/tests/ci/prepare_env.sh && cd /home/opnfv/repos/yardstick && source tests/ci/load_images.sh' - testcase: - cmds: - - 'mkdir -p /home/opnfv/yardstick/results/' - - 'cd /home/opnfv/repos/yardstick && source tests/ci/prepare_env.sh && - yardstick task start tests/opnfv/test_cases/{{script_testcase}}.yaml - --output-file /home/opnfv/yardstick/results/{{script_testcase}}.out &> + cmds: + - 'mkdir -p /home/opnfv/yardstick/results/' + - 'cd /home/opnfv/repos/yardstick && source tests/ci/prepare_env.sh && + yardstick task start tests/opnfv/test_cases/{{validate_testcase}}.yaml + --output-file /home/opnfv/yardstick/results/{{validate_testcase}}.out &> /home/opnfv/yardstick/results/yardstick.log' post_condition: - cmds: - - '' + - '' result: dir: '/home/opnfv/yardstick/results' store_type: 'file' diff --git a/dovetail/container.py b/dovetail/container.py index 2716a089..87174727 100644 --- a/dovetail/container.py +++ b/dovetail/container.py @@ -10,7 +10,7 @@ import utils.dovetail_logger as dt_logger import utils.dovetail_utils as dt_utils -from conf.dovetail_config import DovetailConfig as dt_config +from conf.dovetail_config import DovetailConfig as dt_cfg class Container: @@ -28,7 +28,7 @@ class Container: @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__name__+'.Container').getLogger() + cls.logger = dt_logger.Logger(__name__ + '.Container').getLogger() @classmethod def get(cls, type): @@ -36,17 +36,16 @@ class Container: @staticmethod def get_docker_image(type): - return '%s:%s' % (dt_config.dovetail_config[type]['image_name'], - dt_config.dovetail_config[type]['docker_tag']) + return '%s:%s' % (dt_cfg.dovetail_config[type]['image_name'], + dt_cfg.dovetail_config[type]['docker_tag']) @classmethod def create(cls, type): - # sshkey="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa " - dovetail_config = dt_config.dovetail_config + sshkey = "-v /root/.ssh/id_rsa:/root/.ssh/id_rsa " + dovetail_config = dt_cfg.dovetail_config docker_image = cls.get_docker_image(type) envs = dovetail_config[type]['envs'] opts = dovetail_config[type]['opts'] - sshkey = '' result_volume = ' -v %s:%s ' % (dovetail_config['result_dir'], dovetail_config[type]['result']['dir']) cmd = 'sudo docker run %s %s %s %s %s /bin/bash' % \ @@ -78,5 +77,7 @@ class Container: @classmethod def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False): + if sub_cmd == "": + return cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd) dt_utils.exec_cmd(cmd, cls.logger, exit_on_error) diff --git a/dovetail/parser.py b/dovetail/parser.py index d8f9fa0a..ad969bc2 100644 --- a/dovetail/parser.py +++ b/dovetail/parser.py @@ -12,7 +12,7 @@ import jinja2 import utils.dovetail_logger as dt_logger import utils.dovetail_utils as dt_utils -from conf.dovetail_config import DovetailConfig as dt_config +from conf.dovetail_config import DovetailConfig as dt_cfg class Parser: @@ -22,7 +22,7 @@ class Parser: @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__name__+'.Parser').getLogger() + cls.logger = dt_logger.Logger(__name__ + '.Parser').getLogger() @classmethod def parse_cmd(cls, cmd, testcase): @@ -30,7 +30,7 @@ class Parser: try: template = jinja2.Template(cmd, undefined=jinja2.StrictUndefined) kwargs = {} - for arg in dt_config.dovetail_config['parameters']: + for arg in dt_cfg.dovetail_config['parameters']: path = eval(arg['path']) cls.logger.debug('name: %s, eval path: %s ' % (arg['name'], path)) diff --git a/dovetail/report.py b/dovetail/report.py index 7fd4076d..1f970b29 100644 --- a/dovetail/report.py +++ b/dovetail/report.py @@ -30,18 +30,19 @@ def get_pass_str(passed): class Report: - results = {'functest': {}, 'yardstick': {}} + results = {'functest': {}, 'yardstick': {}, 'shell': {}} logger = None @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__name__+'.Report').getLogger() + cls.logger = dt_logger.Logger(__name__ + '.Report').getLogger() @staticmethod def check_result(testcase, db_result): - checker = CheckerFactory.create(testcase.script_type()) - checker.check(testcase, db_result) + checker = CheckerFactory.create(testcase.validate_type()) + if checker is not None: + checker.check(testcase, db_result) @classmethod def generate_json(cls, testsuite_yaml, testarea, duration): @@ -106,7 +107,7 @@ class Report: sub_report = {} testcase_num = {} testcase_passnum = {} - for area in dt_cfg.testarea_supported: + for area in dt_cfg.dovetail_config['testarea_supported']: sub_report[area] = '' testcase_num[area] = 0 testcase_passnum[area] = 0 @@ -114,7 +115,8 @@ class Report: # TO DO: once version scheme settled, adjust accordingly spec_link = dt_cfg.dovetail_config['repo'] + 'dovetail/testcase' for testcase in report_data['testcases_list']: - pattern = re.compile('|'.join(dt_cfg.testarea_supported)) + pattern = re.compile( + '|'.join(dt_cfg.dovetail_config['testarea_supported'])) area = pattern.findall(testcase['name'])[0] result_dir = dt_cfg.dovetail_config['result_dir'] sub_report[area] += '- <%s> %s result: <%s>\n' %\ @@ -126,22 +128,22 @@ class Report: pass_num += 1 if total_num != 0: - pass_rate = pass_num/total_num + pass_rate = pass_num / total_num report_txt += 'Pass Rate: %.2f%% (%s/%s)\n' %\ - (pass_rate*100, pass_num, total_num) + (pass_rate * 100, pass_num, total_num) report_txt += 'Assessed test areas:\n' for key in sub_report: if testcase_num[key] != 0: - pass_rate = testcase_passnum[key]/testcase_num[key] + pass_rate = testcase_passnum[key] / testcase_num[key] # TO DO: once version scheme settled, adjust accordingly doc_link = dt_cfg.dovetail_config['repo'] +\ ('docs/testsuites/%s' % key) report_txt += '- %s results: <%s> pass %.2f%%\n' %\ - (key, doc_link, pass_rate*100) + (key, doc_link, pass_rate * 100) for key in sub_report: if testcase_num[key] != 0: - pass_rate = testcase_passnum[key]/testcase_num[key] - report_txt += '%s: pass rate %.2f%%\n' % (key, pass_rate*100) + pass_rate = testcase_passnum[key] / testcase_num[key] + report_txt += '%s: pass rate %.2f%%\n' % (key, pass_rate * 100) report_txt += sub_report[key] cls.logger.info(report_txt) @@ -162,24 +164,26 @@ class Report: @classmethod def get_result(cls, testcase): - script_testcase = testcase.script_testcase() - type = testcase.script_type() + validate_testcase = testcase.validate_testcase() + type = testcase.validate_type() crawler = CrawlerFactory.create(type) + if crawler is None: + return None - if script_testcase in cls.results[type]: - return cls.results[type][script_testcase] + if validate_testcase in cls.results[type]: + return cls.results[type][validate_testcase] - result = crawler.crawl(script_testcase) + result = crawler.crawl(validate_testcase) if result is not None: - cls.results[type][script_testcase] = result + cls.results[type][validate_testcase] = result testcase.script_result_acquired(True) cls.logger.debug('testcase: %s -> result acquired' % - script_testcase) + validate_testcase) else: retry = testcase.increase_retry() cls.logger.debug('testcase: %s -> result acquired retry:%d' % - (script_testcase, retry)) + (validate_testcase, retry)) return result @@ -205,7 +209,8 @@ class FunctestCrawler: @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__name__+'.FunctestCrawler').getLogger() + cls.logger = \ + dt_logger.Logger(__name__ + '.FunctestCrawler').getLogger() def crawl(self, testcase=None): store_type = \ @@ -273,7 +278,8 @@ class YardstickCrawler: @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__name__+'.YardstickCrawler').getLogger() + cls.logger = \ + dt_logger.Logger(__name__ + '.YardstickCrawler').getLogger() def crawl(self, testcase=None): store_type = \ @@ -286,7 +292,7 @@ class YardstickCrawler: def crawl_from_file(self, testcase=None): file_path = os.path.join(dt_cfg.dovetail_config['result_dir'], - testcase+'.out') + testcase + '.out') if not os.path.exists(file_path): self.logger.info('result file not found: %s' % file_path) return None @@ -332,7 +338,8 @@ class FunctestChecker: @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__name__+'.FunctestChecker').getLogger() + cls.logger = \ + dt_logger.Logger(__name__ + '.FunctestChecker').getLogger() def check(self, testcase, db_result): sub_testcase_list = testcase.sub_testcase() @@ -371,7 +378,8 @@ class YardstickChecker: @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__name__+'.YardstickChecker').getLogger() + cls.logger = \ + dt_logger.Logger(__name__ + '.YardstickChecker').getLogger() @staticmethod def check(testcase, result): diff --git a/dovetail/run.py b/dovetail/run.py index 0c57b4ed..c0cc872c 100755 --- a/dovetail/run.py +++ b/dovetail/run.py @@ -9,9 +9,8 @@ import click -import sys import os -import time +import copy import utils.dovetail_logger as dt_logger import utils.dovetail_utils as dt_utils @@ -23,7 +22,8 @@ from testcase import Testsuite from report import Report from report import FunctestCrawler, YardstickCrawler from report import FunctestChecker, YardstickChecker -from conf.dovetail_config import DovetailConfig as dt_config +from conf.dovetail_config import DovetailConfig as dt_cfg +from test_runner import DockerRunner, ShellRunner def load_testsuite(testsuite): @@ -31,14 +31,6 @@ def load_testsuite(testsuite): return Testsuite.get(testsuite) -def set_container_tags(option_str): - for script_tag_opt in option_str.split(','): - option_str = script_tag_opt.split(':') - script_type = option_str[0].strip() - script_tag = option_str[1].strip() - dt_config.dovetail_config[script_type]['docker_tag'] = script_tag - - def load_testcase(): Testcase.load() @@ -66,29 +58,7 @@ def run_test(testsuite, testarea, logger): run_testcase = False if run_testcase: - Container.pull_image(testcase.script_type()) - container_id = Container.create(testcase.script_type()) - logger.debug('container id:%s' % container_id) - - if not Testcase.prepared(testcase.script_type()): - cmds = testcase.pre_condition()['cmds'] - if cmds: - for cmd in cmds: - Container.exec_cmd(container_id, cmd) - Testcase.prepared(testcase.script_type(), True) - - if not testcase.prepare_cmd(): - logger.error('failed to prepare testcase:%s' % testcase.name()) - else: - start_time = time.time() - for cmd in testcase.cmds: - Container.exec_cmd(container_id, cmd) - end_time = time.time() - duration = end_time - start_time - - # testcase.post_condition() - - Container.clean(container_id) + testcase.run() db_result = Report.get_result(testcase) Report.check_result(testcase, db_result) @@ -96,23 +66,56 @@ def run_test(testsuite, testarea, logger): return duration -def validate_options(input_dict, logger): - # for 'tag' option - for key, value in input_dict.items(): - if key == 'tag' and value is not None: - for tag in value.split(','): - if len(tag.split(':')) != 2: - logger.error('TAGS option must be "<image>:<tag>,..."') - sys.exit(1) - - -def filter_env_options(input_dict): - envs_options = {} - for key, value in input_dict.items(): - key = key.upper() - if key in dt_config.dovetail_config['cli']['options']['envs']: - envs_options[key] = value - return envs_options +def validate_input(input_dict, check_dict, logger): + # for 'func_tag' and 'yard_tag' options + func_tag = input_dict['func_tag'] + yard_tag = input_dict['yard_tag'] + valid_tag = check_dict['valid_docker_tag'] + if func_tag is not None and func_tag not in valid_tag: + logger.error("func_tag can't be %s, valid in %s", func_tag, valid_tag) + raise SystemExit(1) + if yard_tag is not None and yard_tag not in valid_tag: + logger.error("yard_tag can't be %s, valid in %s", yard_tag, valid_tag) + raise SystemExit(1) + + # for 'SUT_TYPE' option + sut_type = input_dict['sut_type'] + valid_type = check_dict['valid_sut_type'] + if sut_type is not None and sut_type not in valid_type: + logger.error("SUT_TYPE can't be %s, valid in %s", sut_type, valid_type) + raise SystemExit(1) + + +def filter_config(input_dict, logger): + cli_dict = dt_cfg.dovetail_config['cli'] + configs = {} + for key in cli_dict: + if not cli_dict[key]: + continue + try: + cli_config = cli_dict[key]['config'] + if cli_config is None: + continue + except KeyError: + continue + for key, value in input_dict.items(): + for config_key, config_value in cli_config.items(): + value_dict = {} + value_dict['value'] = value + try: + value_dict['path'] = config_value['path'] + if key == config_key: + configs[key] = value_dict + break + if key.upper() == config_key: + configs[key.upper()] = value_dict + break + except KeyError as e: + logger.exception('%s lacks subsection %s', config_key, e) + raise SystemExit(1) + if not configs: + return None + return configs def create_logs(): @@ -125,44 +128,48 @@ def create_logs(): YardstickChecker.create_log() Testcase.create_log() Testsuite.create_log() + DockerRunner.create_log() + ShellRunner.create_log() def clean_results_dir(): - result_path = dt_config.dovetail_config['result_dir'] + result_path = dt_cfg.dovetail_config['result_dir'] if os.path.exists(result_path): if os.path.isdir(result_path): cmd = 'sudo rm -rf %s/*' % (result_path) dt_utils.exec_cmd(cmd, exit_on_error=False) else: print "result_dir in dovetail_config.yml is not a directory." - sys.exit(-1) + raise SystemExit(1) def main(*args, **kwargs): """Dovetail compliance test entry!""" clean_results_dir() + if kwargs['debug']: + os.environ['DEBUG'] = kwargs['debug'] create_logs() logger = dt_logger.Logger('run').getLogger() logger.info('================================================') logger.info('Dovetail compliance: %s!' % (kwargs['testsuite'])) logger.info('================================================') - validate_options(kwargs, logger) - envs_options = filter_env_options(kwargs) - dt_config.update_envs(envs_options) + validate_input(kwargs, dt_cfg.dovetail_config['validate_input'], logger) + configs = filter_config(kwargs, logger) + + if configs is not None: + dt_cfg.update_config(configs) logger.info('Your new envs for functest: %s' % - dt_config.dovetail_config['functest']['envs']) + dt_cfg.dovetail_config['functest']['envs']) logger.info('Your new envs for yardstick: %s' % - dt_config.dovetail_config['yardstick']['envs']) - - if 'tag' in kwargs and kwargs['tag'] is not None: - set_container_tags(kwargs['tag']) + dt_cfg.dovetail_config['yardstick']['envs']) testarea = kwargs['testarea'] testsuite_validation = False testarea_validation = False - if (testarea == 'full') or (testarea in dt_config.testarea_supported): + if (testarea == 'full') or \ + (testarea in dt_cfg.dovetail_config['testarea_supported']): testarea_validation = True - if kwargs['testsuite'] in dt_config.testsuite_supported: + if kwargs['testsuite'] in dt_cfg.dovetail_config['testsuite_supported']: testsuite_validation = True if testsuite_validation and testarea_validation: testsuite_yaml = load_testsuite(kwargs['testsuite']) @@ -174,22 +181,24 @@ def main(*args, **kwargs): (kwargs['testsuite'], testarea)) -dt_config.load_config_files() - +dt_cfg.load_config_files() +dovetail_config = copy.deepcopy(dt_cfg.dovetail_config) CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) -if dt_config.dovetail_config['cli']['options'] is not None: - for key, value in dt_config.dovetail_config['cli']['options'].items(): +if dovetail_config['cli']['options'] is not None: + for key, value in dovetail_config['cli']['options'].items(): if value is not None: for k, v in value.items(): flags = v['flags'] - del v['flags'] + v.pop('flags') + v.pop('path', None) main = click.option(*flags, **v)(main) -if dt_config.dovetail_config['cli']['arguments'] is not None: - for key, value in dt_config.dovetail_config['cli']['arguments'].items(): +if dovetail_config['cli']['arguments'] is not None: + for key, value in dovetail_config['cli']['arguments'].items(): if value is not None: for k, v in value.items(): flags = v['flags'] - del v['flags'] + v.pop('flags') + v.pop('path', None) main = click.argument(flags, **v)(main) main = click.command(context_settings=CONTEXT_SETTINGS)(main) diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py new file mode 100644 index 00000000..8a95b1f7 --- /dev/null +++ b/dovetail/test_runner.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# +# grakiss.wanglei@huawei.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +import utils.dovetail_utils as dt_utils +import utils.dovetail_logger as dt_logger + +from container import Container + + +class DockerRunner(object): + + logger = None + + def __init__(self, testcase): + self.testcase = testcase + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__file__).getLogger() + + def run(self): + Container.pull_image(self.testcase.validate_type()) + container_id = Container.create(self.testcase.validate_type()) + self.logger.debug('container id:%s' % container_id) + + if not self.testcase.prepared(): + cmds = self.testcase.pre_condition() + if cmds: + for cmd in cmds: + Container.exec_cmd(container_id, cmd) + self.testcase.prepared(True) + + if not self.testcase.prepare_cmd(): + self.logger.error('failed to prepare testcase:%s', + self.testcase.name) + else: + for cmd in self.testcase.cmds: + Container.exec_cmd(container_id, cmd) + + cmds = self.testcase.post_condition() + if cmds: + for cmd in cmds: + Container.exec_cmd(container_id, cmd) + self.testcase.cleaned(True) + + Container.clean(container_id) + + +class FunctestRunner(DockerRunner): + + def __init__(self, testcase): + super(FunctestRunner, self).__init__(testcase) + self.name = 'functest' + + +class YardstickRunner(DockerRunner): + + def __init__(self, testcase): + super(YardstickRunner, self).__init__(testcase) + self.name = 'yardstick' + + +class ShellRunner(object): + + logger = None + + @classmethod + def create_log(cls): + cls.logger = dt_logger.Logger(__file__).getLogger() + + def __init__(self, testcase): + super(ShellRunner, self).__init__() + self.testcase = testcase + self.name = 'shell' + + def run(self): + for cmd in self.testcase.cmds: + dt_utils.exec_cmd(cmd, self.logger) + + +class TestRunnerFactory(object): + + TEST_RUNNER_MAP = { + "functest": FunctestRunner, + "yardstick": YardstickRunner, + "shell": ShellRunner, + } + + @classmethod + def create(cls, testcase): + try: + return cls.TEST_RUNNER_MAP[testcase.validate_type()](testcase) + except KeyError: + return None diff --git a/dovetail/testcase.py b/dovetail/testcase.py index 60ce2b28..6f2d76de 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -13,10 +13,11 @@ import yaml import utils.dovetail_logger as dt_logger from parser import Parser -from conf.dovetail_config import DovetailConfig as dt_config +from conf.dovetail_config import DovetailConfig as dt_cfg +from test_runner import TestRunnerFactory -class Testcase: +class Testcase(object): logger = None @@ -25,22 +26,27 @@ class Testcase: self.testcase['passed'] = False self.cmds = [] self.sub_testcase_status = {} - self.update_script_testcase(self.script_type(), - self.script_testcase()) + self.update_validate_testcase(self.validate_testcase()) @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__name__+'.Testcase').getLogger() + cls.logger = dt_logger.Logger(__name__ + '.Testcase').getLogger() def prepare_cmd(self): - script_type = self.script_type() - for cmd in dt_config.dovetail_config[script_type]['testcase']['cmds']: - cmd_lines = Parser.parse_cmd(cmd, self) - if not cmd_lines: + try: + for cmd in self.testcase['validate']['cmds']: + cmd_lines = Parser.parse_cmd(cmd, self) + if not cmd_lines: + return False + # self.logger.debug('cmd_lines:%s', cmd_lines) + self.cmds.append(cmd_lines) + self.logger.debug('cmds:%s', self.cmds) + if len(self.cmds) > 0: + return True + else: return False - self.cmds.append(cmd_lines) - - return True + except KeyError: + return False def __str__(self): return self.testcase @@ -52,29 +58,31 @@ class Testcase: return self.testcase['objective'] def sub_testcase(self): - return self.testcase['scripts']['sub_testcase_list'] + try: + return self.testcase['report']['sub_testcase_list'] + except KeyError: + return [] def sub_testcase_passed(self, name, passed=None): if passed is not None: - self.logger.debug('sub_testcase_passed:%s %s' % (name, passed)) + self.logger.debug('sub_testcase_passed:%s %s', name, passed) self.sub_testcase_status[name] = passed return self.sub_testcase_status[name] - def script_type(self): - return self.testcase['scripts']['type'] + def validate_type(self): + return self.testcase['validate']['type'] - def script_testcase(self): - return self.testcase['scripts']['testcase'] + def validate_testcase(self): + return self.testcase['validate']['testcase'] def exceed_max_retry_times(self): # logger.debug('retry times:%d' % self.testcase['retry']) - return self._exceed_max_retry_times(self.script_type(), - self.script_testcase()) + return self._exceed_max_retry_times(self.validate_testcase()) def increase_retry(self): # self.testcase['retry'] = self.testcase['retry'] + 1 # return self.testcase['retry'] - return self._increase_retry(self.script_type(), self.script_testcase()) + return self._increase_retry(self.validate_testcase()) def passed(self, passed=None): if passed is not None: @@ -82,75 +90,87 @@ class Testcase: return self.testcase['passed'] def script_result_acquired(self, acquired=None): - return self._result_acquired(self.script_type(), - self.script_testcase(), acquired) + return self._result_acquired(self.validate_testcase(), acquired) def pre_condition(self): - return self.pre_condition_cls(self.script_type()) + return self.pre_condition_cls(self.validate_type()) def post_condition(self): - return self.post_condition_cls(self.script_type()) + return self.post_condition_cls(self.validate_type()) - # testcase in upstream testing project - script_testcase_list = {'functest': {}, 'yardstick': {}} + def run(self): + runner = TestRunnerFactory.create(self) + try: + runner.run() + except AttributeError: + pass + # testcase in upstream testing project + # validate_testcase_list = {'functest': {}, 'yardstick': {}, 'shell': {}} + validate_testcase_list = {} # testcase in dovetail testcase_list = {} @classmethod - def prepared(cls, script_type, prepared=None): + def prepared(cls, prepared=None): if prepared is not None: - cls.script_testcase_list[script_type]['prepared'] = prepared - return cls.script_testcase_list[script_type]['prepared'] + cls.validate_testcase_list['prepared'] = prepared + return cls.validate_testcase_list['prepared'] @classmethod - def cleaned(cls, script_type, cleaned=None): + def cleaned(cls, cleaned=None): if cleaned is not None: - cls.scrpit_testcase_list[script_type]['cleaned'] = cleaned - return cls.script_testcase_list[script_type]['cleaned'] + cls.validate_testcase_list['cleaned'] = cleaned + return cls.validate_testcase_list['cleaned'] @staticmethod - def pre_condition_cls(script_type): - return dt_config.dovetail_config[script_type]['pre_condition'] + def pre_condition_cls(validate_type): + return dt_cfg.dovetail_config[validate_type]['pre_condition'] @staticmethod - def post_condition_cls(script_type): - return dt_config.dovetail_config[script_type]['post_condition'] + def post_condition_cls(validate_type): + return dt_cfg.dovetail_config[validate_type]['post_condition'] @classmethod - def update_script_testcase(cls, script_type, script_testcase): - if script_testcase not in cls.script_testcase_list[script_type]: - cls.script_testcase_list[script_type][script_testcase] = \ + def update_validate_testcase(cls, testcase_name): + if testcase_name not in cls.validate_testcase_list: + cls.validate_testcase_list[testcase_name] = \ {'retry': 0, 'acquired': False} - cls.script_testcase_list[script_type]['prepared'] = False - cls.script_testcase_list[script_type]['cleaned'] = False + cls.validate_testcase_list['prepared'] = False + cls.validate_testcase_list['cleaned'] = False @classmethod - def _exceed_max_retry_times(cls, script_type, script_testcase): - retry = cls.script_testcase_list[script_type][script_testcase]['retry'] + def _exceed_max_retry_times(cls, validate_testcase): + retry = cls.validate_testcase_list[validate_testcase]['retry'] return retry > 1 @classmethod - def _increase_retry(cls, script_type, script_testcase): - cls.script_testcase_list[script_type][script_testcase]['retry'] += 1 - return cls.script_testcase_list[script_type][script_testcase]['retry'] + def _increase_retry(cls, validate_testcase): + cls.validate_testcase_list[validate_testcase]['retry'] += 1 + return cls.validate_testcase_list[validate_testcase]['retry'] @classmethod - def _result_acquired(cls, script_type, testcase, acquired=None): + def _result_acquired(cls, testcase, acquired=None): if acquired is not None: - cls.script_testcase_list[script_type][testcase]['acquired'] = \ + cls.validate_testcase_list[testcase]['acquired'] = \ acquired - return cls.script_testcase_list[script_type][testcase]['acquired'] + return cls.validate_testcase_list[testcase]['acquired'] @classmethod def load(cls): - for root, dirs, files in os.walk(dt_config.TESTCASE_PATH): + for root, dirs, files in \ + os.walk(dt_cfg.dovetail_config['TESTCASE_PATH']): for testcase_file in files: with open(os.path.join(root, testcase_file)) as f: testcase_yaml = yaml.safe_load(f) - cls.testcase_list[testcase_yaml.keys()[0]] = \ - cls(testcase_yaml) - cls.logger.debug(cls.testcase_list) + case_type = testcase_yaml.values()[0]['validate']['type'] + testcase = TestcaseFactory.create(case_type, testcase_yaml) + if testcase is not None: + cls.testcase_list[next(testcase_yaml.iterkeys())] = \ + testcase + else: + cls.logger.error('failed to create testcase: %s', + testcase_file) @classmethod def get(cls, testcase_name): @@ -159,6 +179,60 @@ class Testcase: return None +class FunctestTestcase(Testcase): + + validate_testcase_list = {} + + def __init__(self, testcase_yaml): + super(FunctestTestcase, self).__init__(testcase_yaml) + self.name = 'functest' + + def prepare_cmd(self): + ret = super(FunctestTestcase, self).prepare_cmd() + if not ret: + for cmd in \ + dt_cfg.dovetail_config[self.name]['cmds']: + cmd_lines = Parser.parse_cmd(cmd, self) + if not cmd_lines: + return False + self.logger.debug('cmd_lines:%s', cmd_lines) + self.cmds.append(cmd_lines) + return True + + +class YardstickTestcase(Testcase): + + validate_testcae_list = {} + + def __init__(self, testcase_yaml): + super(YardstickTestcase, self).__init__(testcase_yaml) + self.name = 'yardstick' + + +class ShellTestcase(Testcase): + + validate_testcase_list = {} + + def __init__(self, testcase_yaml): + super(ShellTestcase, self).__init__(testcase_yaml) + self.name = 'shell' + + +class TestcaseFactory(object): + TESTCASE_TYPE_MAP = { + 'functest': FunctestTestcase, + 'yardstick': YardstickTestcase, + 'shell': ShellTestcase, + } + + @classmethod + def create(cls, testcase_type, testcase_yaml): + try: + return cls.TESTCASE_TYPE_MAP[testcase_type](testcase_yaml) + except KeyError: + return None + + class Testsuite: logger = None @@ -169,7 +243,7 @@ class Testsuite: @classmethod def create_log(cls): - cls.logger = dt_logger.Logger(__name__+'.Testsuite').getLogger() + cls.logger = dt_logger.Logger(__name__ + '.Testsuite').getLogger() def get_test(self, testcase_name): if testcase_name in self.testcase_list: @@ -180,7 +254,8 @@ class Testsuite: @classmethod def load(cls): - for root, dirs, files in os.walk(dt_config.COMPLIANCE_PATH): + for root, dirs, files in \ + os.walk(dt_cfg.dovetail_config['COMPLIANCE_PATH']): for testsuite_yaml in files: with open(os.path.join(root, testsuite_yaml)) as f: testsuite_yaml = yaml.safe_load(f) diff --git a/dovetail/testcase/example.tc001.yml b/dovetail/testcase/example.tc001.yml new file mode 100644 index 00000000..eaf7e754 --- /dev/null +++ b/dovetail/testcase/example.tc001.yml @@ -0,0 +1,18 @@ +dovetail.example.tc001: + name: dovetail.example.tc001 + objective: Bulk creation and deletion of IPv6 networks, ports and subnets + validate: + type: functest + testcase: tempest_smoke_serial + pre_condition: + - 'echo test for precondition' + cmds: + - 'functest env prepare' + - 'functest testcase run {{validate_testcase}}' + post_condition: + - 'echo test for precondition' + report: + sub_testcase_list: + - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network + - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_port + - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_subnet diff --git a/dovetail/testcase/example.tc002.yml b/dovetail/testcase/example.tc002.yml new file mode 100644 index 00000000..89d000c9 --- /dev/null +++ b/dovetail/testcase/example.tc002.yml @@ -0,0 +1,15 @@ +dovetail.example.tc002: + name: dovetail.example.tc002 + objective: VIM ipv6 operations, to create/update/delete an IPv6 network and subnet + validate: + type: shell + testcase: "run shell" + pre_condition: + - "echo pre_condition" + cmds: + - "echo test2" + post_condition: + - "echo post_condition" + report: + sub_testcase_list: + diff --git a/dovetail/testcase/ipv6.tc001.yml b/dovetail/testcase/ipv6.tc001.yml index 1d9a9c38..0bc0baaa 100644 --- a/dovetail/testcase/ipv6.tc001.yml +++ b/dovetail/testcase/ipv6.tc001.yml @@ -1,9 +1,17 @@ dovetail.ipv6.tc001: name: dovetail.ipv6.tc001 objective: Bulk creation and deletion of IPv6 networks, ports and subnets - scripts: + validate: type: functest testcase: tempest_smoke_serial + pre_condition: + - 'echo test for precondition' + cmds: + - 'functest env prepare' + - 'functest testcase run {{validate_testcase}}' + post_condition: + - 'echo test for precondition' + report: sub_testcase_list: - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_port diff --git a/dovetail/testcase/ipv6.tc002.yml b/dovetail/testcase/ipv6.tc002.yml index 86af7300..efdc7dce 100644 --- a/dovetail/testcase/ipv6.tc002.yml +++ b/dovetail/testcase/ipv6.tc002.yml @@ -1,9 +1,11 @@ dovetail.ipv6.tc002: name: dovetail.ipv6.tc002 objective: VIM ipv6 operations, to create/update/delete an IPv6 network and subnet - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_networks.NetworksIpV6Test.test_create_update_delete_network_subnet - tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_create_update_delete_network_subnet + diff --git a/dovetail/testcase/ipv6.tc003.yml b/dovetail/testcase/ipv6.tc003.yml index 1fedf32c..0b7dc9b2 100644 --- a/dovetail/testcase/ipv6.tc003.yml +++ b/dovetail/testcase/ipv6.tc003.yml @@ -1,9 +1,10 @@ dovetail.ipv6.tc003: name: dovetail.ipv6.tc003 objective: VIM ipv6 operations, to check external network visibility - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_networks.NetworksIpV6Test.test_external_network_visibility - tempest.api.network.test_networks.NetworksIpV6TestAttrs.test_external_network_visibility diff --git a/dovetail/testcase/ipv6.tc004.yml b/dovetail/testcase/ipv6.tc004.yml index 53f9f2ed..10a977e1 100644 --- a/dovetail/testcase/ipv6.tc004.yml +++ b/dovetail/testcase/ipv6.tc004.yml @@ -1,9 +1,10 @@ dovetail.ipv6.tc004: name: dovetail.ipv6.tc004 objective: VIM ipv6 operations, to list IPv6 networks and subnets of a tenant - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_networks.NetworksIpV6Test.test_list_networks - tempest.api.network.test_networks.NetworksIpV6Test.test_list_subnets diff --git a/dovetail/testcase/ipv6.tc005.yml b/dovetail/testcase/ipv6.tc005.yml index 737127ca..8d6deece 100644 --- a/dovetail/testcase/ipv6.tc005.yml +++ b/dovetail/testcase/ipv6.tc005.yml @@ -1,9 +1,10 @@ dovetail.ipv6.tc005: name: dovetail.ipv6.tc005 objective: VIM ipv6 operations, to show information of an IPv6 network and subnet - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_networks.NetworksIpV6Test.test_show_network - tempest.api.network.test_networks.NetworksIpV6Test.test_show_subnet diff --git a/dovetail/testcase/ipv6.tc006.yml b/dovetail/testcase/ipv6.tc006.yml index 2aff3bba..96a902c5 100644 --- a/dovetail/testcase/ipv6.tc006.yml +++ b/dovetail/testcase/ipv6.tc006.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc006: name: dovetail.ipv6.tc006 objective: VIM ipv6 operations, to create an IPv6 port in allowed allocation pools - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_port_in_allowed_allocation_pools diff --git a/dovetail/testcase/ipv6.tc007.yml b/dovetail/testcase/ipv6.tc007.yml index 695ae2e6..9e9de499 100644 --- a/dovetail/testcase/ipv6.tc007.yml +++ b/dovetail/testcase/ipv6.tc007.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc007: name: dovetail.ipv6.tc007 objective: VIM ipv6 operations, to create an IPv6 port without security groups - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_port_with_no_securitygroups diff --git a/dovetail/testcase/ipv6.tc008.yml b/dovetail/testcase/ipv6.tc008.yml index f1889446..e057a570 100644 --- a/dovetail/testcase/ipv6.tc008.yml +++ b/dovetail/testcase/ipv6.tc008.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc008: name: dovetail.ipv6.tc008 objective: VIM ipv6 operations, to create/update/delete an IPv6 port - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_ports.PortsIpV6TestJSON.test_create_update_delete_port diff --git a/dovetail/testcase/ipv6.tc009.yml b/dovetail/testcase/ipv6.tc009.yml index 790c0ec7..56e8015b 100644 --- a/dovetail/testcase/ipv6.tc009.yml +++ b/dovetail/testcase/ipv6.tc009.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc009: name: dovetail.ipv6.tc009 objective: VIM ipv6 operations, to list IPv6 ports of a tenant - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_ports.PortsIpV6TestJSON.test_list_ports diff --git a/dovetail/testcase/ipv6.tc010.yml b/dovetail/testcase/ipv6.tc010.yml index 35d8ee91..eb055ec6 100644 --- a/dovetail/testcase/ipv6.tc010.yml +++ b/dovetail/testcase/ipv6.tc010.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc010: name: dovetail.ipv6.tc010 objective: VIM ipv6 operations, to show information of an IPv6 port - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_ports.PortsIpV6TestJSON.test_show_port diff --git a/dovetail/testcase/ipv6.tc011.yml b/dovetail/testcase/ipv6.tc011.yml index db3d155e..caf44cf0 100644 --- a/dovetail/testcase/ipv6.tc011.yml +++ b/dovetail/testcase/ipv6.tc011.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc011: name: dovetail.ipv6.tc011 objective: VIM ipv6 operations, to add multiple interfaces for an IPv6 router - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_routers.RoutersIpV6Test.test_add_multiple_router_interfaces diff --git a/dovetail/testcase/ipv6.tc012.yml b/dovetail/testcase/ipv6.tc012.yml index 2b471342..ab8e7c48 100644 --- a/dovetail/testcase/ipv6.tc012.yml +++ b/dovetail/testcase/ipv6.tc012.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc012: name: dovetail.ipv6.tc012 objective: VIM ipv6 operations, to add and remove an IPv6 router interface with port_id - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_routers.RoutersIpV6Test.test_add_remove_router_interface_with_port_id diff --git a/dovetail/testcase/ipv6.tc013.yml b/dovetail/testcase/ipv6.tc013.yml index ca52a434..58732624 100644 --- a/dovetail/testcase/ipv6.tc013.yml +++ b/dovetail/testcase/ipv6.tc013.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc013: name: dovetail.ipv6.tc013 objective: VIM ipv6 operations, to add and remove an IPv6 router interface with subnet_id - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_routers.RoutersIpV6Test.test_add_remove_router_interface_with_subnet_id diff --git a/dovetail/testcase/ipv6.tc014.yml b/dovetail/testcase/ipv6.tc014.yml index 0982ad09..c8c6de1c 100644 --- a/dovetail/testcase/ipv6.tc014.yml +++ b/dovetail/testcase/ipv6.tc014.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc014: name: dovetail.ipv6.tc014 objective: VIM ipv6 operations, to create, update, delete, list and show an IPv6 router - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_routers.RoutersIpV6Test.test_create_show_list_update_delete_router diff --git a/dovetail/testcase/ipv6.tc015.yml b/dovetail/testcase/ipv6.tc015.yml index f4c38ac4..e3a47261 100644 --- a/dovetail/testcase/ipv6.tc015.yml +++ b/dovetail/testcase/ipv6.tc015.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc015: name: dovetail.ipv6.tc015 objective: VIM ipv6 operations, to create, update, delete, list and show an IPv6 security group - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_list_update_show_delete_security_group diff --git a/dovetail/testcase/ipv6.tc016.yml b/dovetail/testcase/ipv6.tc016.yml index 0bc17c1b..8195e59c 100644 --- a/dovetail/testcase/ipv6.tc016.yml +++ b/dovetail/testcase/ipv6.tc016.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc016: name: dovetail.ipv6.tc016 objective: VIM ipv6 operations, to create, delete and show security group rules - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_security_groups.SecGroupIPv6Test.test_create_show_delete_security_group_rule diff --git a/dovetail/testcase/ipv6.tc017.yml b/dovetail/testcase/ipv6.tc017.yml index 1df7caf4..8b7ae5fc 100644 --- a/dovetail/testcase/ipv6.tc017.yml +++ b/dovetail/testcase/ipv6.tc017.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc017: name: dovetail.ipv6.tc017 objective: VIM ipv6 operations, to list all security groups - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_security_groups.SecGroupIPv6Test.test_list_security_groups diff --git a/dovetail/testcase/ipv6.tc018.yml b/dovetail/testcase/ipv6.tc018.yml index 01c4d3f7..be888b4f 100644 --- a/dovetail/testcase/ipv6.tc018.yml +++ b/dovetail/testcase/ipv6.tc018.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc018: name: dovetail.ipv6.tc018 objective: VIM ipv6 operations, to show information of an IPv6 port - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_dhcp6_stateless_from_os diff --git a/dovetail/testcase/ipv6.tc019.yml b/dovetail/testcase/ipv6.tc019.yml index d44b9309..524d0a4c 100644 --- a/dovetail/testcase/ipv6.tc019.yml +++ b/dovetail/testcase/ipv6.tc019.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc019: name: dovetail.ipv6.tc019 objective: VIM ipv6 operations, to do IPv6 address assignment - dual stack, DHCPv6 stateless - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_dhcp6_stateless_from_os diff --git a/dovetail/testcase/ipv6.tc020.yml b/dovetail/testcase/ipv6.tc020.yml index e974e083..4cc65e1a 100644 --- a/dovetail/testcase/ipv6.tc020.yml +++ b/dovetail/testcase/ipv6.tc020.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc020: name: dovetail.ipv6.tc020 objective: VIM ipv6 operations, to do IPv6 Address Assignment - Multiple Prefixes, DHCPv6 Stateless - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_dhcpv6_stateless diff --git a/dovetail/testcase/ipv6.tc021.yml b/dovetail/testcase/ipv6.tc021.yml index 20544530..199336aa 100644 --- a/dovetail/testcase/ipv6.tc021.yml +++ b/dovetail/testcase/ipv6.tc021.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc021: name: dovetail.ipv6.tc021 objective: VIM ipv6 operations, to do IPv6 Address Assignment - Dual Stack, Multiple Prefixes, DHCPv6 Stateless - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_dhcpv6_stateless diff --git a/dovetail/testcase/ipv6.tc022.yml b/dovetail/testcase/ipv6.tc022.yml index e01c5b6f..1e6cf0bd 100644 --- a/dovetail/testcase/ipv6.tc022.yml +++ b/dovetail/testcase/ipv6.tc022.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc022: name: dovetail.ipv6.tc022 objective: VIM ipv6 operations, to do IPv6 Address Assignment - SLAAC - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_slaac_from_os diff --git a/dovetail/testcase/ipv6.tc023.yml b/dovetail/testcase/ipv6.tc023.yml index cd17501d..1c735f0a 100644 --- a/dovetail/testcase/ipv6.tc023.yml +++ b/dovetail/testcase/ipv6.tc023.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc023: name: dovetail.ipv6.tc023 objective: VIM ipv6 operations, to do IPv6 Address Assignment - Dual Stack, SLAAC - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_dhcp6_stateless_from_os diff --git a/dovetail/testcase/ipv6.tc024.yml b/dovetail/testcase/ipv6.tc024.yml index 1c8a93f8..010008db 100644 --- a/dovetail/testcase/ipv6.tc024.yml +++ b/dovetail/testcase/ipv6.tc024.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc024: name: dovetail.ipv6.tc024 objective: VIM ipv6 operations, to do IPv6 address assignment - multiple prefixes, SLAAC - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_multi_prefix_slaac diff --git a/dovetail/testcase/ipv6.tc025.yml b/dovetail/testcase/ipv6.tc025.yml index 3f9d97b8..d735a60f 100644 --- a/dovetail/testcase/ipv6.tc025.yml +++ b/dovetail/testcase/ipv6.tc025.yml @@ -1,8 +1,9 @@ dovetail.ipv6.tc025: name: dovetail.ipv6.tc025 objective: VIM ipv6 operations, to do IPv6 address assignment - dual stack, multiple prefixes, SLAAC - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.scenario.test_network_v6.TestGettingAddress.test_dualnet_multi_prefix_slaac diff --git a/dovetail/testcase/nfvi.tc001.yml b/dovetail/testcase/nfvi.tc001.yml index 136fd9d1..b796e803 100644 --- a/dovetail/testcase/nfvi.tc001.yml +++ b/dovetail/testcase/nfvi.tc001.yml @@ -1,7 +1,8 @@ dovetail.nfvi.tc001: name: dovetail.nfvi.tc001 objective: testing for vping using ssh - scripts: + validate: type: functest testcase: vping_ssh + report: sub_testcase_list: diff --git a/dovetail/testcase/nfvi.tc002.yml b/dovetail/testcase/nfvi.tc002.yml index f5724c56..d9413477 100644 --- a/dovetail/testcase/nfvi.tc002.yml +++ b/dovetail/testcase/nfvi.tc002.yml @@ -1,7 +1,8 @@ dovetail.nfvi.tc002: name: dovetail.nfvi.tc002 objective: testing for vping using userdata - scripts: + validate: type: functest testcase: vping_userdata + report: sub_testcase_list: diff --git a/dovetail/testcase/vimops.tc001.yml b/dovetail/testcase/vimops.tc001.yml index 3d2ba0c0..5f8da6a1 100644 --- a/dovetail/testcase/vimops.tc001.yml +++ b/dovetail/testcase/vimops.tc001.yml @@ -1,8 +1,9 @@ dovetail.vimops.tc001: name: dovetail.vimops.tc001 objective: Glance images v2 index - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.ListImagesTest.test_list_no_params diff --git a/dovetail/testcase/vimops.tc002.yml b/dovetail/testcase/vimops.tc002.yml index 15f5bf08..90201f56 100644 --- a/dovetail/testcase/vimops.tc002.yml +++ b/dovetail/testcase/vimops.tc002.yml @@ -1,9 +1,10 @@ dovetail.vimops.tc002: name: dovetail.vimops.tc002 objective: Glance Images v2 Delete - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_delete_image - tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_delete_image_null_id diff --git a/dovetail/testcase/vimops.tc003.yml b/dovetail/testcase/vimops.tc003.yml index 418c041c..d63a17c4 100644 --- a/dovetail/testcase/vimops.tc003.yml +++ b/dovetail/testcase/vimops.tc003.yml @@ -1,9 +1,10 @@ dovetail.vimops.tc003: name: dovetail.vimops.tc003 objective: Glance images v2 list - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.ListImagesTest.test_get_image_schema - tempest.api.image.v2.test_images.ListImagesTest.test_get_images_schema diff --git a/dovetail/testcase/vimops.tc004.yml b/dovetail/testcase/vimops.tc004.yml index 3d205a38..3f924a4c 100644 --- a/dovetail/testcase/vimops.tc004.yml +++ b/dovetail/testcase/vimops.tc004.yml @@ -1,9 +1,10 @@ dovetail.vimops.tc004: name: dovetail.vimops.tc004 objective: Glance images v2 list - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_container_format - tempest.api.image.v2.test_images.ListImagesTest.test_list_images_param_disk_format diff --git a/dovetail/testcase/vimops.tc005.yml b/dovetail/testcase/vimops.tc005.yml index d9413849..acaad685 100644 --- a/dovetail/testcase/vimops.tc005.yml +++ b/dovetail/testcase/vimops.tc005.yml @@ -1,9 +1,10 @@ dovetail.vimops.tc005: name: dovetail.vimops.tc005 objective: Glance images v2 import - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_register_upload_get_image_file - tempest.api.image.v2.test_images_negative.ImagesNegativeTest.test_register_with_invalid_container_format diff --git a/dovetail/testcase/vimops.tc006.yml b/dovetail/testcase/vimops.tc006.yml index 0676c317..68e2ca05 100644 --- a/dovetail/testcase/vimops.tc006.yml +++ b/dovetail/testcase/vimops.tc006.yml @@ -1,9 +1,10 @@ dovetail.vimops.tc006: name: dovetail.vimops.tc006 objective: Glance images v2 update - scripts: + validate: type: functest testcase: tempest_full_parallel + report: sub_testcase_list: - tempest.api.image.v2.test_images.BasicOperationsImagesTest.test_update_image - tempest.api.image.v2.test_images_tags.ImagesTagsTest.test_update_delete_tags_for_image diff --git a/dovetail/tests/unit/test_parser.py b/dovetail/tests/unit/test_parser.py index b4331ea1..410a6629 100644 --- a/dovetail/tests/unit/test_parser.py +++ b/dovetail/tests/unit/test_parser.py @@ -14,7 +14,6 @@ Test 'parser' module import logging import os import unittest - import yaml import parser as dovetail_parser @@ -26,12 +25,15 @@ class TestParser(unittest.TestCase): def setUp(self): """Test case setup""" - logging.disable(logging.CRITICAL) + from conf.dovetail_config import DovetailConfig as dt_cfg + dt_cfg.load_config_files() dovetail_parser.Parser.create_log() + logging.disable(logging.CRITICAL) def test_parser_cmd(self): """Test whether the command is correctly parsed.""" - mock_cmd = "python /functest/ci/run_tests.py -t {{script_testcase}} -r" + mock_cmd = "python /functest/ci/run_tests.py "\ + "-t {{validate_testcase}} -r" with open(os.path.join(self.test_path, 'test_testcase.yaml')) as f: mock_testcase_yaml = yaml.safe_load(f) MockTestcase = type('Testcase', (object,), {}) @@ -44,7 +46,8 @@ class TestParser(unittest.TestCase): def test_parser_cmd_fail(self): """Test whether the command is correctly parsed.""" - mock_cmd = "python /functest/ci/run_tests.py -t {{script_testcase}} -r" + mock_cmd = "python /functest/ci/run_tests.py "\ + "-t {{validate_testcase}} -r" mock_testcase_yaml = {} MockTestcase = type('Testcase', (object,), {}) mock_testcase = MockTestcase() @@ -54,5 +57,6 @@ class TestParser(unittest.TestCase): "None -r") self.assertEqual(expected_output, output) + if __name__ == '__main__': unittest.main() diff --git a/dovetail/tests/unit/test_testcase.yaml b/dovetail/tests/unit/test_testcase.yaml index 1b03262f..735219c5 100644 --- a/dovetail/tests/unit/test_testcase.yaml +++ b/dovetail/tests/unit/test_testcase.yaml @@ -1,9 +1,10 @@ dovetail.ipv6.tc001: name: dovetail.ipv6.tc001 objective: VIM ipv6 operations, to create/delete network, port and subnet in bulk operation - scripts: + validate: type: functest testcase: tempest_smoke_serial + report: sub_testcase_list: - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network - tempest.api.network.test_networks.BulkNetworkOpsIpV7Test.test_bulk_create_delete_port diff --git a/dovetail/utils/dovetail_logger.py b/dovetail/utils/dovetail_logger.py index 8afa08a1..bb09a7b1 100644 --- a/dovetail/utils/dovetail_logger.py +++ b/dovetail/utils/dovetail_logger.py @@ -24,7 +24,7 @@ import logging import os -from conf.dovetail_config import DovetailConfig as dt_config +from conf.dovetail_config import DovetailConfig as dt_cfg class Logger: @@ -46,7 +46,7 @@ class Logger: ch.setLevel(logging.INFO) self.logger.addHandler(ch) - result_path = dt_config.dovetail_config['result_dir'] + result_path = dt_cfg.dovetail_config['result_dir'] if not os.path.exists(result_path): os.makedirs(result_path) hdlr = logging.FileHandler(os.path.join(result_path, 'dovetail.log')) diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py index e4f91987..2d5f0dd1 100644 --- a/dovetail/utils/dovetail_utils.py +++ b/dovetail/utils/dovetail_utils.py @@ -10,6 +10,7 @@ # import sys +import time import subprocess from collections import Mapping, Set, Sequence @@ -29,6 +30,15 @@ def exec_cmd(cmd, logger=None, exit_on_error=True, info=False, print(msg_exec) p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + # show progress bar + seconds = 0 + while p.poll() is None: + seconds += 1 + if seconds > 3: + show_progress_bar(seconds) + time.sleep(1) + output = p.communicate() for line in output[0].strip().split('\n'): line = line.replace('\n', '') @@ -89,3 +99,12 @@ def get_obj_by_path(obj, dst_path): for path, obj in objwalk(obj): if path == dst_path: return obj + + +def show_progress_bar(length): + max_len = 50 + length %= max_len + sys.stdout.write('Running ' + ' ' * max_len + '\r') + sys.stdout.flush() + sys.stdout.write('Running ' + '=' * length + '>' + '\r') + sys.stdout.flush() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..a176ad61 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +pbr>=1.6 +coverage>=3.6 +flake8<3.0 +Jinja2>=2.6 +PyYAML>=3.10 +Click diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..c691f066 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,7 @@ +[metadata] +name = dovetail +home-page = https://wiki.opnfv.org/display/dovetail + +[entry_points] +console_scripts = + dovetail = dovetail.main:main @@ -7,32 +7,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from setuptools import setup, find_packages +import setuptools - -setup( - name="dovetail", - version="0.dev0", - packages=find_packages(), - include_package_data=True, - package_data={ - 'dovetail': [ - '*.py', - 'conf/*.py', - 'conf/*.yml', - 'utils/*.py', - ] - }, - url="https://www.opnfv.org", - install_requires=["coverage>=3.6", - "flake8", - "Jinja2>=2.6", - "PyYAML>=3.10", - "Click" - ], - entry_points={ - 'console_scripts': [ - 'dovetail=dovetail.main:main', - ], - } -) +setuptools.setup( + setup_requires=['pbr>=1.8'], + pbr=True) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000..8ea9c2b6 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,6 @@ +pytest +pykwalify +mock +testrepository +testscenarios +testtools diff --git a/tox.ini b/tox.ini new file mode 100644 index 00000000..12cf8e32 --- /dev/null +++ b/tox.ini @@ -0,0 +1,26 @@ +[tox] +minversion = 1.6 +skipsdist = True +envlist = py27,pep8 + +[testenv] +basepython=python2 +usedevelop = True +install_command = pip install -U {opts} {packages} +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + py.test --basetemp={envtmpdir} {posargs} +setenv = + VIRTUAL_ENV={envdir} + HOME = {envtmpdir} + PYTHONPATH = {toxinidir} + +[testenv:pep8] +deps = flake8 +commands = flake8 {toxinidir} + +[flake8] +show-source = True +ignore = E123,E125,H803 +exclude = .tox,dist,docs,.egg,build,.venv,.git diff --git a/unittests/unittest.sh b/unittests/unittest.sh index 1560a90d..215d0ebd 100755 --- a/unittests/unittest.sh +++ b/unittests/unittest.sh @@ -35,4 +35,26 @@ run_flake8() { fi } +run_tests() { + echo "Running unittest ..." + cd dovetail/ + if [ $FILE_OPTION == "f" ]; then + python -m unittest discover -v -s tests/unit > $logfile 2>&1 + else + python -m unittest discover -v -s tests/unit + fi + + if [ $? -ne 0 ]; then + if [ $FILE_OPTION == "f" ]; then + echo "FAILED, results in $logfile" + fi + exit 1 + else + if [ $FILE_OPTION == "f" ]; then + echo "OK, results in $logfile" + fi + fi +} + run_flake8 +run_tests |