diff options
26 files changed, 215 insertions, 162 deletions
diff --git a/docker/Dockerfile b/docker/Dockerfile index 6137cc949..202eb5cb3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -25,6 +25,7 @@ ARG REPOS_DIR=/home/opnfv/repos ARG FUNCTEST_BASE_DIR=/home/opnfv/functest ARG FUNCTEST_CONF_DIR=${FUNCTEST_BASE_DIR}/conf ARG FUNCTEST_DATA_DIR=${FUNCTEST_BASE_DIR}/data +ARG FUNCTEST_IMAGES_DIR=${FUNCTEST_BASE_DIR}/images ARG FUNCTEST_RESULTS_DIR=${FUNCTEST_BASE_DIR}/results ARG FUNCTEST_REPO_DIR=${REPOS_DIR}/functest ARG FUNCTEST_TEST_DIR=${FUNCTEST_REPO_DIR}/functest/opnfv_tests @@ -71,7 +72,9 @@ RUN pip install --upgrade pip RUN mkdir -p ${REPOS_DIR} \ && mkdir -p ${REPOS_VNFS_DIR} \ && mkdir -p ${FUNCTEST_BASE_DIR}/results \ - && mkdir -p ${FUNCTEST_BASE_DIR}/conf \ + && mkdir -p ${FUNCTEST_CONF_DIR} \ + && mkdir -p ${FUNCTEST_DATA_DIR} \ + && mkdir -p ${FUNCTEST_IMAGES_DIR} \ && mkdir -p /root/.ssh \ && chmod 700 /root/.ssh diff --git a/docker/Dockerfile.aarch64 b/docker/Dockerfile.aarch64 index 83df2d0cb..b03c6d978 100644 --- a/docker/Dockerfile.aarch64 +++ b/docker/Dockerfile.aarch64 @@ -24,6 +24,7 @@ ARG REPOS_DIR=/home/opnfv/repos ARG FUNCTEST_BASE_DIR=/home/opnfv/functest ARG FUNCTEST_CONF_DIR=${FUNCTEST_BASE_DIR}/conf ARG FUNCTEST_DATA_DIR=${FUNCTEST_BASE_DIR}/data +ARG FUNCTEST_IMAGES_DIR=${FUNCTEST_BASE_DIR}/images ARG FUNCTEST_RESULTS_DIR=${FUNCTEST_BASE_DIR}/results ARG FUNCTEST_REPO_DIR=${REPOS_DIR}/functest ARG FUNCTEST_TEST_DIR=${FUNCTEST_REPO_DIR}/functest/opnfv_tests @@ -71,6 +72,8 @@ RUN mkdir -p ${REPOS_DIR} \ && mkdir -p ${REPOS_VNFS_DIR} \ && mkdir -p ${FUNCTEST_BASE_DIR}/results \ && mkdir -p ${FUNCTEST_BASE_DIR}/conf \ + && mkdir -p ${FUNCTEST_DATA_DIR} \ + && mkdir -p ${FUNCTEST_IMAGES_DIR} \ && mkdir -p /root/.ssh \ && chmod 700 /root/.ssh diff --git a/docker/add_images.sh b/docker/add_images.sh index af2956c29..919cecd8f 100755 --- a/docker/add_images.sh +++ b/docker/add_images.sh @@ -7,11 +7,10 @@ CIRROS_REPO_URL=http://download.cirros-cloud.net CIRROS_AARCH64_TAG=161201 CIRROS_X86_64_TAG=0.3.5 -wget ${CIRROS_REPO_URL}/${CIRROS_X86_64_TAG}/cirros-${CIRROS_X86_64_TAG}-x86_64-disk.img -P ${FUNCTEST_BASE_DIR}/data/ -wget ${CIRROS_REPO_URL}/${CIRROS_X86_64_TAG}/cirros-${CIRROS_X86_64_TAG}-x86_64-lxc.tar.gz -P ${FUNCTEST_BASE_DIR}/data/ -wget http://205.177.226.237:9999/onosfw/firewall_block_image.img -P ${FUNCTEST_BASE_DIR}/data/ +wget ${CIRROS_REPO_URL}/${CIRROS_X86_64_TAG}/cirros-${CIRROS_X86_64_TAG}-x86_64-disk.img -P ${FUNCTEST_IMAGES_DIR} +wget ${CIRROS_REPO_URL}/${CIRROS_X86_64_TAG}/cirros-${CIRROS_X86_64_TAG}-x86_64-lxc.tar.gz -P ${FUNCTEST_IMAGES_DIR} # Add the 3-part image for aarch64, since functest can be run from an x86 machine to test an aarch64 POD -wget ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-disk.img -P ${FUNCTEST_BASE_DIR}/data/ -wget ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-initramfs -P ${FUNCTEST_BASE_DIR}/data/ -wget ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-kernel -P ${FUNCTEST_BASE_DIR}/data/ +wget ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-disk.img -P ${FUNCTEST_IMAGES_DIR} +wget ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-initramfs -P ${FUNCTEST_IMAGES_DIR} +wget ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-kernel -P ${FUNCTEST_IMAGES_DIR} diff --git a/functest/ci/config_aarch64_patch.yaml b/functest/ci/config_aarch64_patch.yaml index b43b5a76c..cd395ab89 100644 --- a/functest/ci/config_aarch64_patch.yaml +++ b/functest/ci/config_aarch64_patch.yaml @@ -16,5 +16,13 @@ os: vping: image_name: TestVM + odl_sfc: + image_base_url: "http://artifacts.opnfv.org/sfc/demo" + image_name: sfc_nsh_danube + image_file_name: sf_nsh_danube_arm64.img + image_initrd: sf_nsh_danube_arm64-initrd + image_kernel: sf_nsh_danube_arm64-kernel + image_format: ami + os_cmd_line: 'root=LABEL=cloudimg-rootfs ro' doctor: image_name: TestVM diff --git a/functest/ci/config_functest.yaml b/functest/ci/config_functest.yaml index 6d44f3989..9b7960718 100644 --- a/functest/ci/config_functest.yaml +++ b/functest/ci/config_functest.yaml @@ -38,6 +38,7 @@ general: functest_conf: /home/opnfv/functest/conf functest_data: /home/opnfv/functest/data ims_data: /home/opnfv/functest/data/ims/ + functest_images: /home/opnfv/functest/images rally_inst: /home/opnfv/.rally repo_kingbird: /home/opnfv/repos/kingbird refstack_client: /home/opnfv/repos/refstack-client @@ -117,6 +118,12 @@ onos_sfc: image_name: TestSfcVm image_file_name: firewall_block_image.img +odl_sfc: + image_base_url: "http://artifacts.opnfv.org/sfc/images" + image_name: sfc_nsh_danube + image_file_name: sfc_nsh_danube.qcow2 + image_format: qcow2 + tempest: deployment_name: opnfv-tempest identity: diff --git a/functest/ci/download_images.sh b/functest/ci/download_images.sh new file mode 100644 index 000000000..f3fdef2e3 --- /dev/null +++ b/functest/ci/download_images.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +CIRROS_REPO_URL=http://download.cirros-cloud.net +CIRROS_AARCH64_TAG=161201 +CIRROS_X86_64_TAG=0.3.5 + +RED='\033[1;31m' +NC='\033[0m' # No Color + +function usage(){ + echo -e "${RED}USAGE: $script <destination_folder>${NC}" + exit 0 +} + +script=`basename "$0"` +IMAGES_FOLDER_DIR=$1 + +if [[ -z $IMAGES_FOLDER_DIR ]]; then usage; fi; + +set -ex +mkdir -p ${IMAGES_FOLDER_DIR} + +wget -nc ${CIRROS_REPO_URL}/${CIRROS_X86_64_TAG}/cirros-${CIRROS_X86_64_TAG}-x86_64-disk.img -P ${IMAGES_FOLDER_DIR} +wget -nc ${CIRROS_REPO_URL}/${CIRROS_X86_64_TAG}/cirros-${CIRROS_X86_64_TAG}-x86_64-lxc.tar.gz -P ${IMAGES_FOLDER_DIR} +wget -nc http://artifacts.opnfv.org/sdnvpn/ubuntu-16.04-server-cloudimg-amd64-disk1.img -P ${IMAGES_FOLDER_DIR} + +# Add 3rd-party images for aarch64, since Functest can be run on an x86 machine to test an aarch64 POD +wget -nc ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-disk.img -P ${IMAGES_FOLDER_DIR} +wget -nc ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-initramfs -P ${IMAGES_FOLDER_DIR} +wget -nc ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-kernel -P ${IMAGES_FOLDER_DIR} +set +ex
\ No newline at end of file diff --git a/functest/ci/prepare_env.py b/functest/ci/prepare_env.py index 9fd07958e..8e17a4fcc 100755 --- a/functest/ci/prepare_env.py +++ b/functest/ci/prepare_env.py @@ -156,8 +156,8 @@ def create_directories(): logger.info(" %s created." % CONST.__getattribute__('dir_functest_conf')) else: - logger.debug(" %s already exists." - % CONST.__getattribute__('dir_functest_conf')) + logger.debug(" %s already exists." % + CONST.__getattribute__('dir_functest_conf')) if not os.path.exists(CONST.__getattribute__('dir_functest_data')): os.makedirs(CONST.__getattribute__('dir_functest_data')) @@ -166,6 +166,13 @@ def create_directories(): else: logger.debug(" %s already exists." % CONST.__getattribute__('dir_functest_data')) + if not os.path.exists(CONST.__getattribute__('dir_functest_images')): + os.makedirs(CONST.__getattribute__('dir_functest_images')) + logger.info(" %s created." % + CONST.__getattribute__('dir_functest_images')) + else: + logger.debug(" %s already exists." % + CONST.__getattribute__('dir_functest_images')) def source_rc_file(): diff --git a/functest/cli/commands/cli_env.py b/functest/cli/commands/cli_env.py index 14ad01bfc..f5ba4b342 100644 --- a/functest/cli/commands/cli_env.py +++ b/functest/cli/commands/cli_env.py @@ -28,7 +28,7 @@ class CliEnv(object): "it again? [y|n]\n") while True: if answer.lower() in ["y", "yes"]: - os.remove(CONST.env_active) + os.remove(CONST.__getattribute__('env_active')) break elif answer.lower() in ["n", "no"]: return @@ -36,19 +36,19 @@ class CliEnv(object): answer = raw_input("Invalid answer. Please type [y|n]\n") cmd = ("python %s/functest/ci/prepare_env.py start" % - CONST.dir_repo_functest) + CONST.__getattribute__('dir_repo_functest')) ft_utils.execute_command(cmd) def show(self): def _get_value(attr, default='Unknown'): return attr if attr else default - install_type = _get_value(CONST.INSTALLER_TYPE) - installer_ip = _get_value(CONST.INSTALLER_IP) + install_type = _get_value(CONST.__getattribute__('INSTALLER_TYPE')) + installer_ip = _get_value(CONST.__getattribute__('INSTALLER_IP')) installer_info = ("%s, %s" % (install_type, installer_ip)) - scenario = _get_value(CONST.DEPLOY_SCENARIO) - node = _get_value(CONST.NODE_NAME) - repo_h = git.Repo(CONST.dir_repo_functest).head + scenario = _get_value(CONST.__getattribute__('DEPLOY_SCENARIO')) + node = _get_value(CONST.__getattribute__('NODE_NAME')) + repo_h = git.Repo(CONST.__getattribute__('dir_repo_functest')).head if repo_h.is_detached: git_branch = 'detached from FETCH_HEAD' git_hash = repo_h.commit.hexsha @@ -56,8 +56,8 @@ class CliEnv(object): branch = repo_h.reference git_branch = branch.name git_hash = branch.commit.hexsha - is_debug = _get_value(CONST.CI_DEBUG, 'false') - build_tag = CONST.BUILD_TAG + is_debug = _get_value(CONST.__getattribute__('CI_DEBUG'), 'false') + build_tag = CONST.__getattribute__('BUILD_TAG') if build_tag is not None: build_tag = build_tag.lstrip( "jenkins-").lstrip("functest").lstrip("-") @@ -84,7 +84,7 @@ class CliEnv(object): def status(self, verbose=True): ret_val = 0 - if not os.path.isfile(CONST.env_active): + if not os.path.isfile(CONST.__getattribute__('env_active')): if verbose: click.echo("Functest environment is not installed.\n") ret_val = 1 diff --git a/functest/cli/commands/cli_os.py b/functest/cli/commands/cli_os.py index f85f4041f..e54eb423a 100644 --- a/functest/cli/commands/cli_os.py +++ b/functest/cli/commands/cli_os.py @@ -21,11 +21,11 @@ import functest.utils.openstack_snapshot as os_snapshot class CliOpenStack(object): def __init__(self): - self.os_auth_url = CONST.OS_AUTH_URL + self.os_auth_url = CONST.__getattribute__('OS_AUTH_URL') self.endpoint_ip = None self.endpoint_port = None - self.openstack_creds = CONST.openstack_creds - self.snapshot_file = CONST.openstack_snapshot_file + self.openstack_creds = CONST.__getattribute__('openstack_creds') + self.snapshot_file = CONST.__getattribute__('openstack_snapshot_file') if self.os_auth_url: self.endpoint_ip = self.os_auth_url.rsplit("/")[2].rsplit(":")[0] self.endpoint_port = self.os_auth_url.rsplit("/")[2].rsplit(":")[1] @@ -59,16 +59,16 @@ class CliOpenStack(object): else: answer = raw_input("Invalid answer. Please type [y|n]\n") - installer_type = CONST.INSTALLER_TYPE + installer_type = CONST.__getattribute__('INSTALLER_TYPE') if installer_type is None: click.echo("The environment variable 'INSTALLER_TYPE' is not" "defined. Please export it") - installer_ip = CONST.INSTALLER_IP + installer_ip = CONST.__getattribute__('INSTALLER_IP') if installer_ip is None: click.echo("The environment variable 'INSTALLER_IP' is not" "defined. Please export it") cmd = ("%s/releng/utils/fetch_os_creds.sh -d %s -i %s -a %s" - % (CONST.dir_repos, + % (CONST.__getattribute__('dir_repos'), self.openstack_creds, installer_type, installer_ip)) @@ -78,7 +78,8 @@ class CliOpenStack(object): def check(self): self.ping_endpoint() - cmd = CONST.dir_repo_functest + "/functest/ci/check_os.sh" + cmd = os.path.join(CONST.__getattribute__('dir_repo_functest'), + "functest/ci/check_os.sh") ft_utils.execute_command(cmd, verbose=False) def snapshot_create(self): diff --git a/functest/cli/commands/cli_testcase.py b/functest/cli/commands/cli_testcase.py index 6644a0c29..3d3f1cb3e 100644 --- a/functest/cli/commands/cli_testcase.py +++ b/functest/cli/commands/cli_testcase.py @@ -22,9 +22,10 @@ import functest.utils.functest_vacation as vacation class CliTestcase(object): def __init__(self): - self.tiers = tb.TierBuilder(CONST.INSTALLER_TYPE, - CONST.DEPLOY_SCENARIO, - CONST.functest_testcases_yaml) + self.tiers = tb.TierBuilder( + CONST.__getattribute__('INSTALLER_TYPE'), + CONST.__getattribute__('DEPLOY_SCENARIO'), + CONST.__getattribute__('functest_testcases_yaml')) def list(self): summary = "" @@ -52,12 +53,14 @@ class CliTestcase(object): if testname == 'vacation': vacation.main() - elif not os.path.isfile(CONST.env_active): + elif not os.path.isfile(CONST.__getattribute__('env_active')): click.echo("Functest environment is not ready. " "Run first 'functest env prepare'") else: tests = testname.split(",") for test in tests: cmd = ("python %s/functest/ci/run_tests.py " - "%s -t %s" % (CONST.dir_repo_functest, flags, test)) + "%s -t %s" % + (CONST.__getattribute__('dir_repo_functest'), + flags, test)) ft_utils.execute_command(cmd) diff --git a/functest/cli/commands/cli_tier.py b/functest/cli/commands/cli_tier.py index 012b11d0e..531f0ff96 100644 --- a/functest/cli/commands/cli_tier.py +++ b/functest/cli/commands/cli_tier.py @@ -21,9 +21,10 @@ import functest.utils.functest_utils as ft_utils class CliTier(object): def __init__(self): - self.tiers = tb.TierBuilder(CONST.INSTALLER_TYPE, - CONST.DEPLOY_SCENARIO, - CONST.functest_testcases_yaml) + self.tiers = tb.TierBuilder( + CONST.__getattribute__('INSTALLER_TYPE'), + CONST.__getattribute__('DEPLOY_SCENARIO'), + CONST.__getattribute__('functest_testcases_yaml')) def list(self): summary = "" @@ -62,10 +63,12 @@ class CliTier(object): if report: flags += "-r " - if not os.path.isfile(CONST.env_active): + if not os.path.isfile(CONST.__getattribute__('env_active')): click.echo("Functest environment is not ready. " "Run first 'functest env prepare'") else: cmd = ("python %s/functest/ci/run_tests.py " - "%s -t %s" % (CONST.dir_repo_functest, flags, tiername)) + "%s -t %s" % + (CONST.__getattribute__('dir_repo_functest'), + flags, tiername)) ft_utils.execute_command(cmd) diff --git a/functest/core/pytest_suite_runner.py b/functest/core/pytest_suite_runner.py index bcbaa25d3..a6e476600 100644 --- a/functest/core/pytest_suite_runner.py +++ b/functest/core/pytest_suite_runner.py @@ -8,12 +8,12 @@ # pylint: disable=missing-docstring import logging -import unittest import time +import unittest -from functest.core import testcase +import six -logging.basicConfig() +from functest.core import testcase class PyTestSuiteRunner(testcase.TestCase): @@ -21,17 +21,30 @@ class PyTestSuiteRunner(testcase.TestCase): This superclass is designed to execute pre-configured unittest.TestSuite() objects """ + def __init__(self, **kwargs): super(PyTestSuiteRunner, self).__init__(**kwargs) - self.logger = logging.getLogger(self.__class__.__name__) self.suite = None + self.logger = logging.getLogger(__name__) def run(self, **kwargs): """ Starts test execution from the functest framework """ + try: + name = kwargs["name"] + try: + self.suite = unittest.TestLoader().loadTestsFromName(name) + except ImportError: + self.logger.error("Can not import %s", name) + return testcase.TestCase.EX_RUN_ERROR + except KeyError: + pass self.start_time = time.time() - result = unittest.TextTestRunner(verbosity=2).run(self.suite) + stream = six.StringIO() + result = unittest.TextTestRunner( + stream=stream, verbosity=2).run(self.suite) + self.logger.debug("\n\n%s", stream.getvalue()) self.stop_time = time.time() if result.errors: diff --git a/functest/opnfv_tests/openstack/rally/rally.py b/functest/opnfv_tests/openstack/rally/rally.py index f762383af..40f8386c8 100644 --- a/functest/opnfv_tests/openstack/rally/rally.py +++ b/functest/opnfv_tests/openstack/rally/rally.py @@ -8,6 +8,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 # +from __future__ import division + import json import logging import os @@ -20,7 +22,6 @@ import yaml from functest.core import testcase from functest.utils.constants import CONST -import functest.utils.functest_utils as ft_utils import functest.utils.openstack_utils as os_utils logger = logging.getLogger(__name__) @@ -31,7 +32,7 @@ class RallyBase(testcase.TestCase): 'neutron', 'nova', 'quotas', 'requests', 'vm', 'all'] GLANCE_IMAGE_NAME = CONST.openstack_image_name GLANCE_IMAGE_FILENAME = CONST.openstack_image_file_name - GLANCE_IMAGE_PATH = os.path.join(CONST.dir_functest_data, + GLANCE_IMAGE_PATH = os.path.join(CONST.dir_functest_images, GLANCE_IMAGE_FILENAME) GLANCE_IMAGE_FORMAT = CONST.openstack_image_disk_format FLAVOR_NAME = "m1.tiny" @@ -480,11 +481,12 @@ class RallyBase(testcase.TestCase): total_duration_str2 = "{0:<10}".format(total_duration_str) total_nb_tests_str = "{0:<13}".format(total_nb_tests) - if len(self.summary): - success_rate = total_success / len(self.summary) - else: - success_rate = 100 - success_rate = "{:0.2f}".format(success_rate) + try: + self.result = total_success / len(self.summary) + except ZeroDivisionError: + self.result = 100 + + success_rate = "{:0.2f}".format(self.result) success_rate_str = "{0:<10}".format(str(success_rate) + '%') report += ("+===================+============" "+===============+===========+") @@ -500,12 +502,10 @@ class RallyBase(testcase.TestCase): 'nb tests': total_nb_tests, 'nb success': success_rate}}) - self.result = ft_utils.check_success_rate( - self.case_name, success_rate) self.details = payload - logger.info("Rally '%s' success_rate is %s%%, is marked as %s" - % (self.case_name, success_rate, self.result)) + logger.info("Rally '%s' success_rate is %s%%" + % (self.case_name, success_rate)) def _clean_up(self): if self.volume_type: diff --git a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py index ebae4b867..2a2718dd8 100755 --- a/functest/opnfv_tests/openstack/refstack_client/refstack_client.py +++ b/functest/opnfv_tests/openstack/refstack_client/refstack_client.py @@ -5,6 +5,10 @@ # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 + +from __future__ import division + + import argparse import logging import os @@ -123,7 +127,11 @@ class RefstackClient(testcase.TestCase): skipped_testcases += match + ", " num_executed = int(num_tests) - int(num_skipped) - success_rate = 100 * int(num_success) / int(num_executed) + + try: + self.result = 100 * int(num_success) / int(num_executed) + except ZeroDivisionError: + logger.error("No test has been executed") self.details = {"tests": int(num_tests), "failures": int(num_failures), @@ -131,12 +139,10 @@ class RefstackClient(testcase.TestCase): "errors": failed_testcases, "skipped": skipped_testcases} except Exception: - success_rate = 0 + self.result = 0 - self.result = ft_utils.check_success_rate( - self.case_name, success_rate) - logger.info("Testcase %s success_rate is %s%%, is marked as %s" - % (self.case_name, success_rate, self.result)) + logger.info("Testcase %s success_rate is %s%%" + % (self.case_name, self.result)) def run(self): '''used for functest command line, diff --git a/functest/opnfv_tests/openstack/tempest/conf_utils.py b/functest/opnfv_tests/openstack/tempest/conf_utils.py index 54f7428ca..cd6a2a8c1 100644 --- a/functest/opnfv_tests/openstack/tempest/conf_utils.py +++ b/functest/opnfv_tests/openstack/tempest/conf_utils.py @@ -22,7 +22,7 @@ import functest.utils.openstack_utils as os_utils IMAGE_ID_ALT = None FLAVOR_ID_ALT = None REPO_PATH = CONST.dir_repo_functest -GLANCE_IMAGE_PATH = os.path.join(CONST.dir_functest_data, +GLANCE_IMAGE_PATH = os.path.join(CONST.dir_functest_images, CONST.openstack_image_file_name) TEMPEST_TEST_LIST_DIR = CONST.dir_tempest_cases TEMPEST_RESULTS_DIR = os.path.join(CONST.dir_results, diff --git a/functest/opnfv_tests/openstack/tempest/tempest.py b/functest/opnfv_tests/openstack/tempest/tempest.py index 984e2a1b7..a41d07c7e 100644 --- a/functest/opnfv_tests/openstack/tempest/tempest.py +++ b/functest/opnfv_tests/openstack/tempest/tempest.py @@ -8,6 +8,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 # +from __future__ import division + import logging import os import re @@ -181,7 +183,13 @@ class TempestCommon(testcase.TestCase): try: num_executed = int(num_tests) - int(num_skipped) - success_rate = 100 * int(num_success) / int(num_executed) + try: + self.result = 100 * int(num_success) / int(num_executed) + except ZeroDivisionError: + logger.error("No test has been executed") + self.result = 0 + return + with open(os.path.join(conf_utils.TEMPEST_RESULTS_DIR, "tempest.log"), 'r') as logfile: output = logfile.read() @@ -198,12 +206,10 @@ class TempestCommon(testcase.TestCase): "errors": error_logs, "skipped": skipped_testcase} except Exception: - success_rate = 0 + self.result = 0 - self.result = ft_utils.check_success_rate( - self.case_name, success_rate) - logger.info("Tempest %s success_rate is %s%%, is marked as %s" - % (self.case_name, success_rate, self.result)) + logger.info("Tempest %s success_rate is %s%%" + % (self.case_name, self.result)) def run(self): diff --git a/functest/opnfv_tests/sdn/onos/onos.py b/functest/opnfv_tests/sdn/onos/onos.py index d7a2d38ed..5dfff036b 100644 --- a/functest/opnfv_tests/sdn/onos/onos.py +++ b/functest/opnfv_tests/sdn/onos/onos.py @@ -25,7 +25,7 @@ class OnosBase(testcase.TestCase): onos_repo_path = CONST.__getattribute__('dir_repo_onos') onos_sfc_image_name = CONST.__getattribute__('onos_sfc_image_name') onos_sfc_image_path = os.path.join( - CONST.__getattribute__('dir_functest_data'), + CONST.__getattribute__('dir_functest_images'), CONST.__getattribute__('onos_sfc_image_file_name')) onos_sfc_path = os.path.join(CONST.__getattribute__('dir_repo_functest'), CONST.__getattribute__('dir_onos_sfc')) diff --git a/functest/tests/unit/ci/test_prepare_env.py b/functest/tests/unit/ci/test_prepare_env.py index 39a0825c3..fbb596512 100644 --- a/functest/tests/unit/ci/test_prepare_env.py +++ b/functest/tests/unit/ci/test_prepare_env.py @@ -192,12 +192,17 @@ class PrepareEnvTesting(unittest.TestCase): CONST.__getattribute__('dir_functest_conf')) mock_method.assert_any_call( CONST.__getattribute__('dir_functest_data')) + mock_method.assert_any_call( + CONST.__getattribute__('dir_functest_images')) mock_logger_info.assert_any_call(" %s created." % CONST.__getattribute__( 'dir_functest_conf')) mock_logger_info.assert_any_call(" %s created." % CONST.__getattribute__( 'dir_functest_data')) + mock_logger_info.assert_any_call(" %s created." % + CONST.__getattribute__( + 'dir_functest_images')) @mock.patch('functest.ci.prepare_env.logger.info') @mock.patch('functest.ci.prepare_env.logger.debug') @@ -213,6 +218,9 @@ class PrepareEnvTesting(unittest.TestCase): mock_logger_debug.assert_any_call(" %s already exists." % CONST.__getattribute__( 'dir_functest_data')) + mock_logger_debug.assert_any_call(" %s already exists." % + CONST.__getattribute__( + 'dir_functest_images')) def _get_env_cred_dict(self, os_prefix=''): return {'OS_USERNAME': os_prefix + 'username', diff --git a/functest/tests/unit/cli/commands/test_cli_env.py b/functest/tests/unit/cli/commands/test_cli_env.py index 4b6ea57a7..c3d89ea3a 100644 --- a/functest/tests/unit/cli/commands/test_cli_env.py +++ b/functest/tests/unit/cli/commands/test_cli_env.py @@ -28,7 +28,7 @@ class CliEnvTesting(unittest.TestCase): @mock.patch('functest.cli.commands.cli_testcase.ft_utils.execute_command') def test_prepare_default(self, mock_ft_utils, mock_os): cmd = ("python %s/functest/ci/prepare_env.py start" % - CONST.dir_repo_functest) + CONST.__getattribute__('dir_repo_functest')) self.cli_environ.prepare() mock_ft_utils.assert_called_with(cmd) @@ -40,29 +40,30 @@ class CliEnvTesting(unittest.TestCase): mock.patch('functest.cli.commands.cli_testcase.os.remove') \ as mock_os_remove: cmd = ("python %s/functest/ci/prepare_env.py start" % - CONST.dir_repo_functest) + CONST.__getattribute__('dir_repo_functest')) self.cli_environ.prepare() - mock_os_remove.assert_called_once_with(CONST.env_active) + mock_os_remove.assert_called_once_with( + CONST.__getattribute__('env_active')) mock_ft_utils.assert_called_with(cmd) def _test_show_missing_env_var(self, var, *args): if var == 'INSTALLER_TYPE': - CONST.INSTALLER_TYPE = None + CONST.__setattr__('INSTALLER_TYPE', None) reg_string = "| INSTALLER: Unknown, \S+\s*|" elif var == 'INSTALLER_IP': - CONST.INSTALLER_IP = None + CONST.__setattr__('INSTALLER_IP', None) reg_string = "| INSTALLER: \S+, Unknown\s*|" elif var == 'SCENARIO': - CONST.DEPLOY_SCENARIO = None + CONST.__setattr__('DEPLOY_SCENARIO', None) reg_string = "| SCENARIO: Unknown\s*|" elif var == 'NODE': - CONST.NODE_NAME = None + CONST.__setattr__('NODE_NAME', None) reg_string = "| POD: Unknown\s*|" elif var == 'BUILD_TAG': - CONST.BUILD_TAG = None + CONST.__setattr__('BUILD_TAG', None) reg_string = "| BUILD TAG: None|" elif var == 'DEBUG': - CONST.CI_DEBUG = None + CONST.__setattr__('CI_DEBUG', None) reg_string = "| DEBUG FLAG: false\s*|" elif var == 'STATUS': reg_string = "| STATUS: not ready\s*|" @@ -106,7 +107,7 @@ class CliEnvTesting(unittest.TestCase): @mock.patch('functest.cli.commands.cli_env.os.path.exists', return_value=False) def test_show_missing_git_repo_dir(self, *args): - CONST.dir_repo_functest = None + CONST.__setattr__('dir_repo_functest', None) self.assertRaises(NoSuchPathError, lambda: self.cli_environ.show()) @mock.patch('functest.cli.commands.cli_env.click.echo') diff --git a/functest/tests/unit/cli/commands/test_cli_os.py b/functest/tests/unit/cli/commands/test_cli_os.py index f0e58c676..540427690 100644 --- a/functest/tests/unit/cli/commands/test_cli_os.py +++ b/functest/tests/unit/cli/commands/test_cli_os.py @@ -69,10 +69,10 @@ class CliOpenStackTesting(unittest.TestCase): def test_fetch_credentials_default(self, mock_click_echo, mock_os_path, mock_ftutils_execute): - CONST.INSTALLER_TYPE = self.installer_type - CONST.INSTALLER_IP = self.installer_ip + CONST.__setattr__('INSTALLER_TYPE', self.installer_type) + CONST.__setattr__('INSTALLER_IP', self.installer_ip) cmd = ("%s/releng/utils/fetch_os_creds.sh -d %s -i %s -a %s" - % (CONST.dir_repos, + % (CONST.__getattribute__('dir_repos'), self.openstack_creds, self.installer_type, self.installer_ip)) @@ -92,15 +92,13 @@ class CliOpenStackTesting(unittest.TestCase): def test_fetch_credentials_missing_installer_type(self, mock_click_echo, mock_os_path, mock_ftutils_execute): - installer_type = None - installer_ip = self.installer_ip - CONST.INSTALLER_TYPE = installer_type - CONST.INSTALLER_IP = installer_ip + CONST.__setattr__('INSTALLER_TYPE', None) + CONST.__setattr__('INSTALLER_IP', self.installer_ip) cmd = ("%s/releng/utils/fetch_os_creds.sh -d %s -i %s -a %s" - % (CONST.dir_repos, + % (CONST.__getattribute__('dir_repos'), self.openstack_creds, - installer_type, - installer_ip)) + None, + self.installer_ip)) self.cli_os.openstack_creds = self.openstack_creds self.cli_os.fetch_credentials() mock_click_echo.assert_any_call("The environment variable " @@ -109,8 +107,8 @@ class CliOpenStackTesting(unittest.TestCase): mock_click_echo.assert_any_call("Fetching credentials from " "installer node '%s' with " "IP=%s.." % - (installer_type, - installer_ip)) + (None, + self.installer_ip)) mock_ftutils_execute.assert_called_once_with(cmd, verbose=False) @mock.patch('functest.cli.commands.cli_os.ft_utils.execute_command') @@ -122,10 +120,10 @@ class CliOpenStackTesting(unittest.TestCase): mock_ftutils_execute): installer_type = self.installer_type installer_ip = None - CONST.INSTALLER_TYPE = installer_type - CONST.INSTALLER_IP = installer_ip + CONST.__setattr__('INSTALLER_TYPE', installer_type) + CONST.__setattr__('INSTALLER_IP', installer_ip) cmd = ("%s/releng/utils/fetch_os_creds.sh -d %s -i %s -a %s" - % (CONST.dir_repos, + % (CONST.__getattribute__('dir_repos'), self.openstack_creds, installer_type, installer_ip)) @@ -144,8 +142,9 @@ class CliOpenStackTesting(unittest.TestCase): @mock.patch('functest.cli.commands.cli_os.ft_utils.execute_command') def test_check(self, mock_ftutils_execute): with mock.patch.object(self.cli_os, 'ping_endpoint'): - CONST.dir_repo_functest = self.dir_repo_functest - cmd = CONST.dir_repo_functest + "/functest/ci/check_os.sh" + CONST.__setattr__('dir_repo_functest', self.dir_repo_functest) + cmd = os.path.join(CONST.__getattribute__('dir_repo_functest'), + "functest/ci/check_os.sh") self.cli_os.check() mock_ftutils_execute.assert_called_once_with(cmd, verbose=False) diff --git a/functest/tests/unit/cli/commands/test_cli_testcase.py b/functest/tests/unit/cli/commands/test_cli_testcase.py index 39c8139d7..2d09514ec 100644 --- a/functest/tests/unit/cli/commands/test_cli_testcase.py +++ b/functest/tests/unit/cli/commands/test_cli_testcase.py @@ -42,7 +42,9 @@ class CliTestCasesTesting(unittest.TestCase): @mock.patch('functest.cli.commands.cli_testcase.ft_utils.execute_command') def test_run_default(self, mock_ft_utils, mock_os): cmd = ("python %s/functest/ci/run_tests.py " - "%s -t %s" % (CONST.dir_repo_functest, "-n -r ", self.testname)) + "%s -t %s" % + (CONST.__getattribute__('dir_repo_functest'), + "-n -r ", self.testname)) self.cli_tests.run(self.testname, noclean=True, report=True) mock_ft_utils.assert_called_with(cmd) @@ -51,7 +53,9 @@ class CliTestCasesTesting(unittest.TestCase): @mock.patch('functest.cli.commands.cli_testcase.ft_utils.execute_command') def test_run_noclean_missing_report(self, mock_ft_utils, mock_os): cmd = ("python %s/functest/ci/run_tests.py " - "%s -t %s" % (CONST.dir_repo_functest, "-n ", self.testname)) + "%s -t %s" % + (CONST.__getattribute__('dir_repo_functest'), + "-n ", self.testname)) self.cli_tests.run(self.testname, noclean=True, report=False) mock_ft_utils.assert_called_with(cmd) @@ -60,7 +64,9 @@ class CliTestCasesTesting(unittest.TestCase): @mock.patch('functest.cli.commands.cli_testcase.ft_utils.execute_command') def test_run_report_missing_noclean(self, mock_ft_utils, mock_os): cmd = ("python %s/functest/ci/run_tests.py " - "%s -t %s" % (CONST.dir_repo_functest, "-r ", self.testname)) + "%s -t %s" % + (CONST.__getattribute__('dir_repo_functest'), + "-r ", self.testname)) self.cli_tests.run(self.testname, noclean=False, report=True) mock_ft_utils.assert_called_with(cmd) @@ -69,7 +75,9 @@ class CliTestCasesTesting(unittest.TestCase): @mock.patch('functest.cli.commands.cli_testcase.ft_utils.execute_command') def test_run_missing_noclean_report(self, mock_ft_utils, mock_os): cmd = ("python %s/functest/ci/run_tests.py " - "%s -t %s" % (CONST.dir_repo_functest, "", self.testname)) + "%s -t %s" % + (CONST.__getattribute__('dir_repo_functest'), + "", self.testname)) self.cli_tests.run(self.testname, noclean=False, report=False) mock_ft_utils.assert_called_with(cmd) diff --git a/functest/tests/unit/cli/commands/test_cli_tier.py b/functest/tests/unit/cli/commands/test_cli_tier.py index 802359f16..fbc752537 100644 --- a/functest/tests/unit/cli/commands/test_cli_tier.py +++ b/functest/tests/unit/cli/commands/test_cli_tier.py @@ -90,8 +90,9 @@ class CliTierTesting(unittest.TestCase): @mock.patch('functest.cli.commands.cli_tier.ft_utils.execute_command') def test_run_default(self, mock_ft_utils, mock_os): cmd = ("python %s/functest/ci/run_tests.py " - "%s -t %s" % (CONST.dir_repo_functest, "-n -r ", - self.tiername)) + "%s -t %s" % + (CONST.__getattribute__('dir_repo_functest'), + "-n -r ", self.tiername)) self.cli_tier.run(self.tiername, noclean=True, report=True) mock_ft_utils.assert_called_with(cmd) @@ -100,8 +101,9 @@ class CliTierTesting(unittest.TestCase): @mock.patch('functest.cli.commands.cli_tier.ft_utils.execute_command') def test_run_report_missing_noclean(self, mock_ft_utils, mock_os): cmd = ("python %s/functest/ci/run_tests.py " - "%s -t %s" % (CONST.dir_repo_functest, "-r ", - self.tiername)) + "%s -t %s" % + (CONST.__getattribute__('dir_repo_functest'), + "-r ", self.tiername)) self.cli_tier.run(self.tiername, noclean=False, report=True) mock_ft_utils.assert_called_with(cmd) @@ -110,8 +112,9 @@ class CliTierTesting(unittest.TestCase): @mock.patch('functest.cli.commands.cli_tier.ft_utils.execute_command') def test_run_noclean_missing_report(self, mock_ft_utils, mock_os): cmd = ("python %s/functest/ci/run_tests.py " - "%s -t %s" % (CONST.dir_repo_functest, "-n ", - self.tiername)) + "%s -t %s" % + (CONST.__getattribute__('dir_repo_functest'), + "-n ", self.tiername)) self.cli_tier.run(self.tiername, noclean=True, report=False) mock_ft_utils.assert_called_with(cmd) @@ -120,8 +123,9 @@ class CliTierTesting(unittest.TestCase): @mock.patch('functest.cli.commands.cli_tier.ft_utils.execute_command') def test_run_missing_noclean_report(self, mock_ft_utils, mock_os): cmd = ("python %s/functest/ci/run_tests.py " - "%s -t %s" % (CONST.dir_repo_functest, "", - self.tiername)) + "%s -t %s" % + (CONST.__getattribute__('dir_repo_functest'), + "", self.tiername)) self.cli_tier.run(self.tiername, noclean=False, report=False) mock_ft_utils.assert_called_with(cmd) diff --git a/functest/tests/unit/openstack/rally/test_rally.py b/functest/tests/unit/openstack/rally/test_rally.py index fe25dfcf9..c78286186 100644 --- a/functest/tests/unit/openstack/rally/test_rally.py +++ b/functest/tests/unit/openstack/rally/test_rally.py @@ -343,19 +343,6 @@ class OSRallyTesting(unittest.TestCase): self.rally_base._run_tests() self.rally_base._run_task.assert_any_call('test1') - @mock.patch('functest.opnfv_tests.openstack.rally.rally.logger.info') - def test_generate_report(self, mock_logger_info): - summary = [{'test_name': 'test_name', - 'overall_duration': 5, - 'nb_tests': 3, - 'success': 5}] - self.rally_base.summary = summary - with mock.patch('functest.opnfv_tests.openstack.rally.rally.' - 'ft_utils.check_success_rate', - return_value='criteria'): - self.rally_base._generate_report() - self.assertTrue(mock_logger_info.called) - def test_clean_up_default(self): self.rally_base.volume_type = mock.Mock() self.rally_base.cinder_client = mock.Mock() diff --git a/functest/tests/unit/openstack/tempest/test_tempest.py b/functest/tests/unit/openstack/tempest/test_tempest.py index e05e5dfac..bb75c9ed7 100644 --- a/functest/tests/unit/openstack/tempest/test_tempest.py +++ b/functest/tests/unit/openstack/tempest/test_tempest.py @@ -151,24 +151,6 @@ class OSTempestTesting(unittest.TestCase): assert_any_call("Starting Tempest test suite: '%s'." % cmd_line) - @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.logger.info') - def test_parse_verifier_result_default(self, mock_logger_info): - self.tempestcommon.VERIFICATION_ID = 'test_uuid' - self.tempestcommon.case_name = 'test_case_name' - stdout = ['Testscount||2', 'Success||2', 'Skipped||0', 'Failures||0'] - with mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' - 'subprocess.Popen') as mock_popen, \ - mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' - 'ft_utils.check_success_rate') as mock_method, \ - mock.patch('__builtin__.open', mock.mock_open()): - mock_stdout = mock.Mock() - attrs = {'stdout': stdout} - mock_stdout.configure_mock(**attrs) - mock_popen.return_value = mock_stdout - - self.tempestcommon.parse_verifier_result() - mock_method.assert_any_call('test_case_name', 100) - @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.' 'os.path.exists', return_value=False) @mock.patch('functest.opnfv_tests.openstack.tempest.tempest.os.makedirs', diff --git a/functest/tests/unit/utils/test_functest_utils.py b/functest/tests/unit/utils/test_functest_utils.py index d48e06f92..57e0c465d 100644 --- a/functest/tests/unit/utils/test_functest_utils.py +++ b/functest/tests/unit/utils/test_functest_utils.py @@ -563,22 +563,6 @@ class FunctestUtilsTesting(unittest.TestCase): assert_called_once_with(self.parameter, self.config_yaml) - def test_check_success_rate_default(self): - with mock.patch('functest.utils.functest_utils.get_criteria_by_test') \ - as mock_criteria: - mock_criteria.return_value = self.criteria - resp = functest_utils.check_success_rate(self.case_name, - self.result) - self.assertEqual(resp, 100) - - def test_check_success_rate_failed(self): - with mock.patch('functest.utils.functest_utils.get_criteria_by_test') \ - as mock_criteria: - mock_criteria.return_value = self.criteria - resp = functest_utils.check_success_rate(self.case_name, - 0) - self.assertEqual(resp, 0) - # TODO: merge_dicts def test_get_testcases_file_dir(self): diff --git a/functest/utils/functest_utils.py b/functest/utils/functest_utils.py index 744258b36..bf30f56ee 100644 --- a/functest/utils/functest_utils.py +++ b/functest/utils/functest_utils.py @@ -379,16 +379,6 @@ def get_functest_config(parameter): return get_parameter_from_yaml(parameter, yaml_) -def check_success_rate(case_name, result): - # It should be removed as TestCase tests criteria - # and result. - logger.warning('check_success_rate will be removed soon') - criteria = get_criteria_by_test(case_name) - if type(criteria) == int and result >= criteria: - return 100 - return 0 - - def merge_dicts(dict1, dict2): for k in set(dict1.keys()).union(dict2.keys()): if k in dict1 and k in dict2: |