diff options
Diffstat (limited to 'functest/ci')
| -rwxr-xr-x | functest/ci/check_os.sh | 24 | ||||
| -rw-r--r-- | functest/ci/config_functest.yaml | 39 | ||||
| -rw-r--r-- | functest/ci/download_images.sh | 31 | ||||
| -rwxr-xr-x | functest/ci/generate_report.py | 153 | ||||
| -rw-r--r-- | functest/ci/logging.ini | 75 | ||||
| -rw-r--r-- | functest/ci/logging.json | 29 | ||||
| -rwxr-xr-x | functest/ci/prepare_env.py | 162 | ||||
| -rwxr-xr-x | functest/ci/run_tests.py | 101 | ||||
| -rw-r--r-- | functest/ci/testcases.yaml | 59 | ||||
| -rw-r--r-- | functest/ci/tier_builder.py | 11 | ||||
| -rw-r--r-- | functest/ci/tier_handler.py | 5 |
11 files changed, 341 insertions, 348 deletions
diff --git a/functest/ci/check_os.sh b/functest/ci/check_os.sh index 83f9f476..ce0bc20c 100755 --- a/functest/ci/check_os.sh +++ b/functest/ci/check_os.sh @@ -86,30 +86,6 @@ if [ $RETVAL -ne 0 ]; then fi echo " ...OK" -adminURL=$(openstack catalog show identity |awk '/admin/ {print $4}') -if [ -z ${adminURL} ]; then - echo "ERROR: Cannot determine the admin URL." - openstack catalog show identity - exit 1 -fi -adminIP=$(echo $adminURL|sed 's/^.*http.*\:\/\///'|sed 's/.[^:]*$//') -adminPort=$(echo $adminURL|grep -Po '(?<=:)\d+') -https_enabled=$(echo $adminURL | grep 'https') -if [[ -n $https_enabled ]]; then - echo ">>Verifying SSL connectivity to the admin endpoint $adminIP:$adminPort..." - verify_SSL_connectivity $adminIP $adminPort -else - echo ">>Verifying connectivity to the admin endpoint $adminIP:$adminPort..." - verify_connectivity $adminIP $adminPort -fi -RETVAL=$? -if [ $RETVAL -ne 0 ]; then - echo "ERROR: Cannot talk to the admin endpoint $adminIP:$adminPort ." - echo "$adminURL" - exit 1 -fi -echo " ...OK" - echo "Checking Required OpenStack services:" for service in $MANDATORY_SERVICES; do diff --git a/functest/ci/config_functest.yaml b/functest/ci/config_functest.yaml index 96dd6212..9b796071 100644 --- a/functest/ci/config_functest.yaml +++ b/functest/ci/config_functest.yaml @@ -34,10 +34,11 @@ general: functest: /home/opnfv/functest functest_test: /home/opnfv/repos/functest/functest/opnfv_tests results: /home/opnfv/functest/results - functest_logging_cfg: /home/opnfv/repos/functest/functest/ci/logging.json + functest_logging_cfg: /home/opnfv/repos/functest/functest/ci/logging.ini functest_conf: /home/opnfv/functest/conf functest_data: /home/opnfv/functest/data ims_data: /home/opnfv/functest/data/ims/ + functest_images: /home/opnfv/functest/images rally_inst: /home/opnfv/.rally repo_kingbird: /home/opnfv/repos/kingbird refstack_client: /home/opnfv/repos/refstack-client @@ -49,6 +50,8 @@ general: image_name: Cirros-0.3.5 image_name_alt: Cirros-0.3.5-1 image_file_name: cirros-0.3.5-x86_64-disk.img + image_url: http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img + image_user: cirros image_disk_format: qcow2 image_username: cirros image_password: cubswin:) @@ -71,14 +74,23 @@ general: functest: testcases_yaml: /home/opnfv/repos/functest/functest/ci/testcases.yaml -healthcheck: - disk_image: /home/opnfv/functest/data/cirros-0.3.5-x86_64-disk.img - disk_format: qcow2 - wait_time: 60 - snaps: use_keystone: True - use_floating_ips: False + use_floating_ips: True +# images: +# cirros: +# disk_url: http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img + # ARM +# disk_url: http://download.cirros-cloud.net/daily/20161201/cirros-d161201-aarch64-disk.img +# kernel_url: http://download.cirros-cloud.net/daily/20161201/cirros-d161201-aarch64-kernel +# ramdisk_url: http://download.cirros-cloud.net/daily/20161201/cirros-d161201-aarch64-initramfs +# extra_properties: +# os_command_line: root=/dev/vdb1 rw rootwait console=tty0 console=ttyS0 console=ttyAMA0 +# hw_video_model: vga +# ubuntu: +# disk_url: http://uec-images.ubuntu.com/releases/trusty/14.04/ubuntu-14.04-server-cloudimg-amd64-disk1.img +# centos: +# disk_url: http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2 vping: ping_timeout: 200 @@ -92,6 +104,14 @@ vping: router_name: vping-router sg_name: vPing-sg sg_desc: Security group for vPing test case + keypair_name: vPing-keypair + keypair_priv_file: /tmp/vPing-keypair + keypair_pub_file: /tmp/vPing-keypair.pub + vm_boot_timeout: 180 + vm_delete_timeout: 100 + vm_ssh_connect_timeout: 60 + cleanup_objects: True + unique_names: True onos_sfc: image_base_url: http://artifacts.opnfv.org/sfc/demo @@ -210,3 +230,8 @@ results: # you can also set a file (e.g. /home/opnfv/functest/results/dump.txt) to dump results # test_db_url: file:///home/opnfv/functest/results/dump.txt test_db_url: http://testresults.opnfv.org/test/api/v1/results + +energy_recorder: + api_url: http://161.105.253.100:8888/resources + api_user: "" + api_password: "" diff --git a/functest/ci/download_images.sh b/functest/ci/download_images.sh new file mode 100644 index 00000000..f3fdef2e --- /dev/null +++ b/functest/ci/download_images.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +CIRROS_REPO_URL=http://download.cirros-cloud.net +CIRROS_AARCH64_TAG=161201 +CIRROS_X86_64_TAG=0.3.5 + +RED='\033[1;31m' +NC='\033[0m' # No Color + +function usage(){ + echo -e "${RED}USAGE: $script <destination_folder>${NC}" + exit 0 +} + +script=`basename "$0"` +IMAGES_FOLDER_DIR=$1 + +if [[ -z $IMAGES_FOLDER_DIR ]]; then usage; fi; + +set -ex +mkdir -p ${IMAGES_FOLDER_DIR} + +wget -nc ${CIRROS_REPO_URL}/${CIRROS_X86_64_TAG}/cirros-${CIRROS_X86_64_TAG}-x86_64-disk.img -P ${IMAGES_FOLDER_DIR} +wget -nc ${CIRROS_REPO_URL}/${CIRROS_X86_64_TAG}/cirros-${CIRROS_X86_64_TAG}-x86_64-lxc.tar.gz -P ${IMAGES_FOLDER_DIR} +wget -nc http://artifacts.opnfv.org/sdnvpn/ubuntu-16.04-server-cloudimg-amd64-disk1.img -P ${IMAGES_FOLDER_DIR} + +# Add 3rd-party images for aarch64, since Functest can be run on an x86 machine to test an aarch64 POD +wget -nc ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-disk.img -P ${IMAGES_FOLDER_DIR} +wget -nc ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-initramfs -P ${IMAGES_FOLDER_DIR} +wget -nc ${CIRROS_REPO_URL}/daily/20${CIRROS_AARCH64_TAG}/cirros-d${CIRROS_AARCH64_TAG}-aarch64-kernel -P ${IMAGES_FOLDER_DIR} +set +ex
\ No newline at end of file diff --git a/functest/ci/generate_report.py b/functest/ci/generate_report.py deleted file mode 100755 index 3872a07e..00000000 --- a/functest/ci/generate_report.py +++ /dev/null @@ -1,153 +0,0 @@ -#!/usr/bin/env python -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -# -import json -import re -import urllib2 - -import functest.utils.functest_logger as ft_logger -import functest.utils.functest_utils as ft_utils -from functest.utils.constants import CONST - -COL_1_LEN = 25 -COL_2_LEN = 15 -COL_3_LEN = 12 -COL_4_LEN = 15 -COL_5_LEN = 75 - -# If we run from CI (Jenkins) we will push the results to the DB -# and then we can print the url to the specific test result - - -logger = ft_logger.Logger("generate_report").getLogger() - - -def init(tiers_to_run=[]): - test_cases_arr = [] - for tier in tiers_to_run: - for test in tier.get_tests(): - test_cases_arr.append({'test_name': test.get_name(), - 'tier_name': tier.get_name(), - 'result': 'Not executed', - 'duration': '0', - 'url': ''}) - return test_cases_arr - - -def get_results_from_db(): - url = "%s/results?build_tag=%s" % (ft_utils.get_db_url(), - CONST.BUILD_TAG) - logger.debug("Query to rest api: %s" % url) - try: - data = json.load(urllib2.urlopen(url)) - return data['results'] - except: - logger.error("Cannot read content from the url: %s" % url) - return None - - -def get_data(test, results): - test_result = test['result'] - url = '' - for test_db in results: - if test['test_name'] in test_db['case_name']: - id = test_db['_id'] - url = ft_utils.get_db_url() + '/results/' + id - test_result = test_db['criteria'] - - return {"url": url, "result": test_result} - - -def print_line(w1, w2='', w3='', w4='', w5=''): - str = ('| ' + w1.ljust(COL_1_LEN - 1) + - '| ' + w2.ljust(COL_2_LEN - 1) + - '| ' + w3.ljust(COL_3_LEN - 1) + - '| ' + w4.ljust(COL_4_LEN - 1)) - if CONST.IS_CI_RUN: - str += ('| ' + w5.ljust(COL_5_LEN - 1)) - str += '|\n' - return str - - -def print_line_no_columns(str): - TOTAL_LEN = COL_1_LEN + COL_2_LEN + COL_3_LEN + COL_4_LEN + 2 - if CONST.IS_CI_RUN: - TOTAL_LEN += COL_5_LEN + 1 - return ('| ' + str.ljust(TOTAL_LEN) + "|\n") - - -def print_separator(char="=", delimiter="+"): - str = ("+" + char * COL_1_LEN + - delimiter + char * COL_2_LEN + - delimiter + char * COL_3_LEN + - delimiter + char * COL_4_LEN) - if CONST.IS_CI_RUN: - str += (delimiter + char * COL_5_LEN) - str += '+\n' - return str - - -def main(args=[]): - executed_test_cases = args - - if CONST.IS_CI_RUN: - results = get_results_from_db() - if results is not None: - for test in executed_test_cases: - data = get_data(test, results) - test.update({"url": data['url'], - "result": data['result']}) - - TOTAL_LEN = COL_1_LEN + COL_2_LEN + COL_3_LEN + COL_4_LEN - if CONST.IS_CI_RUN: - TOTAL_LEN += COL_5_LEN - MID = TOTAL_LEN / 2 - - if CONST.BUILD_TAG is not None: - if re.search("daily", CONST.BUILD_TAG) is not None: - CONST.CI_LOOP = "daily" - else: - CONST.CI_LOOP = "weekly" - - str = '' - str += print_separator('=', delimiter="=") - str += print_line_no_columns(' ' * (MID - 8) + 'FUNCTEST REPORT') - str += print_separator('=', delimiter="=") - str += print_line_no_columns(' ') - str += print_line_no_columns(" Deployment description:") - str += print_line_no_columns(" INSTALLER: %s" - % CONST.INSTALLER_TYPE) - if CONST.DEPLOY_SCENARIO is not None: - str += print_line_no_columns(" SCENARIO: %s" - % CONST.DEPLOY_SCENARIO) - if CONST.BUILD_TAG is not None: - str += print_line_no_columns(" BUILD TAG: %s" - % CONST.BUILD_TAG) - if CONST.CI_LOOP is not None: - str += print_line_no_columns(" CI LOOP: %s" - % CONST.CI_LOOP) - str += print_line_no_columns(' ') - str += print_separator('=') - if CONST.IS_CI_RUN: - str += print_line('TEST CASE', 'TIER', 'DURATION', 'RESULT', 'URL') - else: - str += print_line('TEST CASE', 'TIER', 'DURATION', 'RESULT') - str += print_separator('=') - for test in executed_test_cases: - str += print_line(test['test_name'], - test['tier_name'], - test['duration'], - test['result'], - test['url']) - str += print_separator('-') - - logger.info("\n\n\n%s" % str) - - -if __name__ == '__main__': - import sys - main(sys.argv[1:]) diff --git a/functest/ci/logging.ini b/functest/ci/logging.ini new file mode 100644 index 00000000..210c8f5f --- /dev/null +++ b/functest/ci/logging.ini @@ -0,0 +1,75 @@ +[loggers] +keys=root,functest,ci,cli,core,energy,opnfv_tests,utils + +[handlers] +keys=console,wconsole,file,null + +[formatters] +keys=standard + +[logger_root] +level=NOTSET +handlers=null + +[logger_functest] +level=NOTSET +handlers=file +qualname=functest + +[logger_ci] +level=NOTSET +handlers=console +qualname=functest.ci + +[logger_cli] +level=NOTSET +handlers=wconsole +qualname=functest.cli + +[logger_core] +level=NOTSET +handlers=console +qualname=functest.core + +[logger_energy] +level=NOTSET +handlers=wconsole +qualname=functest.energy + +[logger_opnfv_tests] +level=NOTSET +handlers=wconsole +qualname=functest.opnfv_tests + +[logger_utils] +level=NOTSET +handlers=wconsole +qualname=functest.utils + +[handler_null] +class=NullHandler +level=NOTSET +formatter=standard +args=() + +[handler_console] +class=StreamHandler +level=INFO +formatter=standard +args=(sys.stdout,) + +[handler_wconsole] +class=StreamHandler +level=WARN +formatter=standard +args=(sys.stdout,) + +[handler_file] +class=FileHandler +level=DEBUG +formatter=standard +args=("/home/opnfv/functest/results/functest.log",) + +[formatter_standard] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt= diff --git a/functest/ci/logging.json b/functest/ci/logging.json deleted file mode 100644 index 2a2399d3..00000000 --- a/functest/ci/logging.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "version": 1, - "disable_existing_loggers": false, - "formatters": { - "standard": { - "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s" - } - }, - "handlers": { - "console": { - "level": "INFO", - "class": "logging.StreamHandler", - "formatter": "standard" - }, - "file": { - "level": "DEBUG", - "class": "logging.FileHandler", - "formatter": "standard", - "filename": "/home/opnfv/functest/results/functest.log" - } - }, - "loggers": { - "": { - "handlers": ["console", "file"], - "level": "DEBUG", - "propagate": "yes" - } - } -} diff --git a/functest/ci/prepare_env.py b/functest/ci/prepare_env.py index e9a470f9..8e17a4fc 100755 --- a/functest/ci/prepare_env.py +++ b/functest/ci/prepare_env.py @@ -8,6 +8,8 @@ import argparse import json +import logging +import logging.config import os import re import subprocess @@ -16,7 +18,6 @@ import fileinput import yaml -import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils import functest.utils.openstack_utils as os_utils from functest.utils.constants import CONST @@ -27,13 +28,13 @@ from opnfv.deployment import factory actions = ['start', 'check'] """ logging configuration """ -logger = ft_logger.Logger("prepare_env").getLogger() +logger = logging.getLogger('functest.ci.prepare_env') handler = None # set the architecture to default pod_arch = None arch_filter = ['aarch64'] -CONFIG_FUNCTEST_PATH = CONST.CONFIG_FUNCTEST_YAML +CONFIG_FUNCTEST_PATH = CONST.__getattribute__('CONFIG_FUNCTEST_YAML') CONFIG_PATCH_PATH = os.path.join(os.path.dirname( CONFIG_FUNCTEST_PATH), "config_patch.yaml") CONFIG_AARCH64_PATCH_PATH = os.path.join(os.path.dirname( @@ -65,63 +66,70 @@ def check_env_variables(): print_separator() logger.info("Checking environment variables...") - if CONST.INSTALLER_TYPE is None: + if CONST.__getattribute__('INSTALLER_TYPE') is None: logger.warning("The env variable 'INSTALLER_TYPE' is not defined.") - CONST.INSTALLER_TYPE = "undefined" + CONST.__setattr__('INSTALLER_TYPE', 'undefined') else: - if CONST.INSTALLER_TYPE not in opnfv_constants.INSTALLERS: + if (CONST.__getattribute__('INSTALLER_TYPE') not in + opnfv_constants.INSTALLERS): logger.warning("INSTALLER_TYPE=%s is not a valid OPNFV installer. " "Available OPNFV Installers are : %s. " "Setting INSTALLER_TYPE=undefined." - % (CONST.INSTALLER_TYPE, + % (CONST.__getattribute__('INSTALLER_TYPE'), opnfv_constants.INSTALLERS)) - CONST.INSTALLER_TYPE = "undefined" + CONST.__setattr__('INSTALLER_TYPE', 'undefined') else: logger.info(" INSTALLER_TYPE=%s" - % CONST.INSTALLER_TYPE) + % CONST.__getattribute__('INSTALLER_TYPE')) - if CONST.INSTALLER_IP is None: + if CONST.__getattribute__('INSTALLER_IP') is None: logger.warning("The env variable 'INSTALLER_IP' is not defined. " "It is needed to fetch the OpenStack credentials. " "If the credentials are not provided to the " "container as a volume, please add this env variable " "to the 'docker run' command.") else: - logger.info(" INSTALLER_IP=%s" % CONST.INSTALLER_IP) + logger.info(" INSTALLER_IP=%s" % + CONST.__getattribute__('INSTALLER_IP')) - if CONST.DEPLOY_SCENARIO is None: + if CONST.__getattribute__('DEPLOY_SCENARIO') is None: logger.warning("The env variable 'DEPLOY_SCENARIO' is not defined. " "Setting CI_SCENARIO=undefined.") - CONST.DEPLOY_SCENARIO = "undefined" + CONST.__setattr__('DEPLOY_SCENARIO', 'undefined') else: - logger.info(" DEPLOY_SCENARIO=%s" % CONST.DEPLOY_SCENARIO) - if CONST.CI_DEBUG: - logger.info(" CI_DEBUG=%s" % CONST.CI_DEBUG) + logger.info(" DEPLOY_SCENARIO=%s" + % CONST.__getattribute__('DEPLOY_SCENARIO')) + if CONST.__getattribute__('CI_DEBUG'): + logger.info(" CI_DEBUG=%s" % CONST.__getattribute__('CI_DEBUG')) - if CONST.NODE_NAME: - logger.info(" NODE_NAME=%s" % CONST.NODE_NAME) + if CONST.__getattribute__('NODE_NAME'): + logger.info(" NODE_NAME=%s" % CONST.__getattribute__('NODE_NAME')) - if CONST.BUILD_TAG: - logger.info(" BUILD_TAG=%s" % CONST.BUILD_TAG) + if CONST.__getattribute__('BUILD_TAG'): + logger.info(" BUILD_TAG=%s" % CONST.__getattribute__('BUILD_TAG')) - if CONST.IS_CI_RUN: - logger.info(" IS_CI_RUN=%s" % CONST.IS_CI_RUN) + if CONST.__getattribute__('IS_CI_RUN'): + logger.info(" IS_CI_RUN=%s" % CONST.__getattribute__('IS_CI_RUN')) def get_deployment_handler(): global handler global pod_arch - installer_params_yaml = os.path.join(CONST.dir_repo_functest, - 'functest/ci/installer_params.yaml') - if (CONST.INSTALLER_IP and CONST.INSTALLER_TYPE and - CONST.INSTALLER_TYPE in opnfv_constants.INSTALLERS): + installer_params_yaml = os.path.join( + CONST.__getattribute__('dir_repo_functest'), + 'functest/ci/installer_params.yaml') + if (CONST.__getattribute__('INSTALLER_IP') and + CONST.__getattribute__('INSTALLER_TYPE') and + CONST.__getattribute__('INSTALLER_TYPE') in + opnfv_constants.INSTALLERS): try: installer_params = ft_utils.get_parameter_from_yaml( - CONST.INSTALLER_TYPE, installer_params_yaml) + CONST.__getattribute__('INSTALLER_TYPE'), + installer_params_yaml) except ValueError as e: logger.debug('Printing deployment info is not supported for %s' % - CONST.INSTALLER_TYPE) + CONST.__getattribute__('INSTALLER_TYPE')) logger.debug(e) else: user = installer_params.get('user', None) @@ -129,8 +137,8 @@ def get_deployment_handler(): pkey = installer_params.get('pkey', None) try: handler = factory.Factory.get_handler( - installer=CONST.INSTALLER_TYPE, - installer_ip=CONST.INSTALLER_IP, + installer=CONST.__getattribute__('INSTALLER_TYPE'), + installer_ip=CONST.__getattribute__('INSTALLER_IP'), installer_user=user, installer_pwd=password, pkey_file=pkey) @@ -143,51 +151,62 @@ def get_deployment_handler(): def create_directories(): print_separator() logger.info("Creating needed directories...") - if not os.path.exists(CONST.dir_functest_conf): - os.makedirs(CONST.dir_functest_conf) - logger.info(" %s created." % CONST.dir_functest_conf) + if not os.path.exists(CONST.__getattribute__('dir_functest_conf')): + os.makedirs(CONST.__getattribute__('dir_functest_conf')) + logger.info(" %s created." % + CONST.__getattribute__('dir_functest_conf')) else: - logger.debug(" %s already exists." - % CONST.dir_functest_conf) + logger.debug(" %s already exists." % + CONST.__getattribute__('dir_functest_conf')) - if not os.path.exists(CONST.dir_functest_data): - os.makedirs(CONST.dir_functest_data) - logger.info(" %s created." % CONST.dir_functest_data) + if not os.path.exists(CONST.__getattribute__('dir_functest_data')): + os.makedirs(CONST.__getattribute__('dir_functest_data')) + logger.info(" %s created." % + CONST.__getattribute__('dir_functest_data')) else: - logger.debug(" %s already exists." - % CONST.dir_functest_data) + logger.debug(" %s already exists." % + CONST.__getattribute__('dir_functest_data')) + if not os.path.exists(CONST.__getattribute__('dir_functest_images')): + os.makedirs(CONST.__getattribute__('dir_functest_images')) + logger.info(" %s created." % + CONST.__getattribute__('dir_functest_images')) + else: + logger.debug(" %s already exists." % + CONST.__getattribute__('dir_functest_images')) def source_rc_file(): print_separator() logger.info("Fetching RC file...") - if CONST.openstack_creds is None: + if CONST.__getattribute__('openstack_creds') is None: logger.warning("The environment variable 'creds' must be set and" "pointing to the local RC file. Using default: " "/home/opnfv/functest/conf/openstack.creds ...") - os.path.join(CONST.dir_functest_conf, 'openstack.creds') + os.path.join( + CONST.__getattribute__('dir_functest_conf'), 'openstack.creds') - if not os.path.isfile(CONST.openstack_creds): + if not os.path.isfile(CONST.__getattribute__('openstack_creds')): logger.info("RC file not provided. " "Fetching it from the installer...") - if CONST.INSTALLER_IP is None: - logger.error("The env variable CI_INSTALLER_IP must be provided in" + if CONST.__getattribute__('INSTALLER_IP')is None: + logger.error("The env variable 'INSTALLER_IP' must be provided in" " order to fetch the credentials from the installer.") raise Exception("Missing CI_INSTALLER_IP.") - if CONST.INSTALLER_TYPE not in opnfv_constants.INSTALLERS: + if (CONST.__getattribute__('INSTALLER_TYPE') not in + opnfv_constants.INSTALLERS): logger.error("Cannot fetch credentials. INSTALLER_TYPE=%s is " "not a valid OPNFV installer. Available " "installers are : %s." % - (CONST.INSTALLER_TYPE, + (CONST.__getattribute__('INSTALLER_TYPE'), opnfv_constants.INSTALLERS)) raise Exception("Wrong INSTALLER_TYPE.") cmd = ("/home/opnfv/repos/releng/utils/fetch_os_creds.sh " "-d %s -i %s -a %s" - % (CONST.openstack_creds, - CONST.INSTALLER_TYPE, - CONST.INSTALLER_IP)) + % (CONST.__getattribute__('openstack_creds'), + CONST.__getattribute__('INSTALLER_TYPE'), + CONST.__getattribute__('INSTALLER_IP'))) logger.debug("Executing command: %s" % cmd) p = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) output = p.communicate()[0] @@ -196,22 +215,23 @@ def source_rc_file(): raise Exception("Failed to fetch credentials from installer.") else: logger.info("RC file provided in %s." - % CONST.openstack_creds) - if os.path.getsize(CONST.openstack_creds) == 0: - raise Exception("The file %s is empty." % CONST.openstack_creds) + % CONST.__getattribute__('openstack_creds')) + if os.path.getsize(CONST.__getattribute__('openstack_creds')) == 0: + raise Exception("The file %s is empty." % + CONST.__getattribute__('openstack_creds')) logger.info("Sourcing the OpenStack RC file...") - os_utils.source_credentials(CONST.openstack_creds) + os_utils.source_credentials(CONST.__getattribute__('openstack_creds')) for key, value in os.environ.iteritems(): if re.search("OS_", key): if key == 'OS_AUTH_URL': - CONST.OS_AUTH_URL = value + CONST.__setattr__('OS_AUTH_URL', value) elif key == 'OS_USERNAME': - CONST.OS_USERNAME = value + CONST.__setattr__('OS_USERNAME', value) elif key == 'OS_TENANT_NAME': - CONST.OS_TENANT_NAME = value + CONST.__setattr__('OS_TENANT_NAME', value) elif key == 'OS_PASSWORD': - CONST.OS_PASSWORD = value + CONST.__setattr__('OS_PASSWORD', value) def patch_config_file(): @@ -228,7 +248,7 @@ def patch_file(patch_file_path): updated = False for key in patch_file: - if key in CONST.DEPLOY_SCENARIO: + if key in CONST.__getattribute__('DEPLOY_SCENARIO'): new_functest_yaml = dict(ft_utils.merge_dicts( ft_utils.get_functest_yaml(), patch_file[key])) updated = True @@ -243,7 +263,8 @@ def patch_file(patch_file_path): def verify_deployment(): print_separator() logger.info("Verifying OpenStack services...") - cmd = ("%s/functest/ci/check_os.sh" % CONST.dir_repo_functest) + cmd = ("%s/functest/ci/check_os.sh" % + CONST.__getattribute__('dir_repo_functest')) logger.debug("Executing command: %s" % cmd) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) @@ -274,7 +295,7 @@ def install_rally(): cmd = "rally deployment destroy opnfv-rally" ft_utils.execute_command(cmd, error_msg=( "Deployment %s does not exist." - % CONST.rally_deployment_name), + % CONST.__getattribute__('rally_deployment_name')), verbose=False) rally_conf = os_utils.get_credentials_for_rally() @@ -282,7 +303,7 @@ def install_rally(): json.dump(rally_conf, fp) cmd = ("rally deployment create " "--file=rally_conf.json --name={0}" - .format(CONST.rally_deployment_name)) + .format(CONST.__getattribute__('rally_deployment_name'))) error_msg = "Problem while creating Rally deployment" ft_utils.execute_command_raise(cmd, error_msg=error_msg) @@ -304,17 +325,18 @@ def install_rally(): def install_tempest(): logger.info("Installing tempest from existing repo...") cmd = ("rally verify list-verifiers | " - "grep '{0}' | wc -l".format(CONST.tempest_deployment_name)) + "grep '{0}' | wc -l".format( + CONST.__getattribute__('tempest_deployment_name'))) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) while p.poll() is None: line = p.stdout.readline().rstrip() if str(line) == '0': logger.debug("Tempest %s does not exist" % - CONST.tempest_deployment_name) + CONST.__getattribute__('tempest_deployment_name')) cmd = ("rally verify create-verifier --source {0} " "--name {1} --type tempest --system-wide" - .format(CONST.dir_repo_tempest, - CONST.tempest_deployment_name)) + .format(CONST.__getattribute__('dir_repo_tempest'), + CONST.__getattribute__('tempest_deployment_name'))) error_msg = "Problem while installing Tempest." ft_utils.execute_command_raise(cmd, error_msg=error_msg) @@ -331,10 +353,10 @@ def create_flavor(): def check_environment(): msg_not_active = "The Functest environment is not installed." - if not os.path.isfile(CONST.env_active): + if not os.path.isfile(CONST.__getattribute__('env_active')): raise Exception(msg_not_active) - with open(CONST.env_active, "r") as env_file: + with open(CONST.__getattribute__('env_active'), "r") as env_file: s = env_file.read() if not re.search("1", s): raise Exception(msg_not_active) @@ -364,7 +386,7 @@ def main(**kwargs): install_rally() install_tempest() create_flavor() - with open(CONST.env_active, "w") as env_file: + with open(CONST.__getattribute__('env_active'), "w") as env_file: env_file.write("1") check_environment() print_deployment_info() @@ -377,6 +399,8 @@ def main(**kwargs): if __name__ == '__main__': + logging.config.fileConfig( + CONST.__getattribute__('dir_functest_logging_cfg')) parser = PrepareEnvParser() args = parser.parse_args(sys.argv[1:]) sys.exit(main(**args)) diff --git a/functest/ci/run_tests.py b/functest/ci/run_tests.py index e68901b8..76760096 100755 --- a/functest/ci/run_tests.py +++ b/functest/ci/run_tests.py @@ -9,26 +9,26 @@ # import argparse -import datetime import enum import importlib +import logging +import logging.config import os import re import sys -import functest.ci.generate_report as generate_report +import prettytable + import functest.ci.tier_builder as tb import functest.core.testcase as testcase -import functest.utils.functest_logger as ft_logger import functest.utils.functest_utils as ft_utils import functest.utils.openstack_clean as os_clean import functest.utils.openstack_snapshot as os_snapshot import functest.utils.openstack_utils as os_utils from functest.utils.constants import CONST - -""" logging configuration """ -logger = ft_logger.Logger("run_tests").getLogger() +# __name__ cannot be used here +logger = logging.getLogger('functest.ci.run_tests') class Result(enum.Enum): @@ -40,6 +40,10 @@ class BlockingTestFailed(Exception): pass +class TestNotEnabled(Exception): + pass + + class RunTestsParser(object): def __init__(self): @@ -75,7 +79,7 @@ def print_separator(str, count=45): def source_rc_file(): - rc_file = CONST.openstack_creds + rc_file = CONST.__getattribute__('openstack_creds') if not os.path.isfile(rc_file): raise Exception("RC file %s does not exist..." % rc_file) logger.debug("Sourcing the OpenStack RC file...") @@ -83,13 +87,13 @@ def source_rc_file(): for key, value in os.environ.iteritems(): if re.search("OS_", key): if key == 'OS_AUTH_URL': - CONST.OS_AUTH_URL = value + CONST.__setattr__('OS_AUTH_URL', value) elif key == 'OS_USERNAME': - CONST.OS_USERNAME = value + CONST.__setattr__('OS_USERNAME', value) elif key == 'OS_TENANT_NAME': - CONST.OS_TENANT_NAME = value + CONST.__setattr__('OS_TENANT_NAME', value) elif key == 'OS_PASSWORD': - CONST.OS_PASSWORD = value + CONST.__setattr__('OS_PASSWORD', value) def generate_os_snapshot(): @@ -100,13 +104,6 @@ def cleanup(): os_clean.main() -def update_test_info(test_name, result, duration): - for test in GlobalVariables.EXECUTED_TEST_CASES: - if test['test_name'] == test_name: - test.update({"result": result, - "duration": duration}) - - def get_run_dict(testname): try: dict = ft_utils.get_dict_by_test(testname) @@ -121,8 +118,9 @@ def get_run_dict(testname): def run_test(test, tier_name, testcases=None): - result_str = "PASS" - start = datetime.datetime.now() + if not test.is_enabled(): + raise TestNotEnabled("The test case {} is not enabled" + .format(test.get_name())) test_name = test.get_name() logger.info("\n") # blank line print_separator("=") @@ -146,7 +144,7 @@ def run_test(test, tier_name, testcases=None): cls = getattr(module, run_dict['class']) test_dict = ft_utils.get_dict_by_test(test_name) test_case = cls(**test_dict) - + GlobalVariables.EXECUTED_TEST_CASES.append(test_case) try: kwargs = run_dict['args'] result = test_case.run(**kwargs) @@ -155,7 +153,8 @@ def run_test(test, tier_name, testcases=None): if result == testcase.TestCase.EX_OK: if GlobalVariables.REPORT_FLAG: test_case.push_to_db() - result = test_case.check_result() + result = test_case.is_successful() + logger.info("Test result:\n\n%s\n", test_case) except ImportError: logger.exception("Cannot import module {}".format( run_dict['module'])) @@ -167,27 +166,13 @@ def run_test(test, tier_name, testcases=None): if test.needs_clean() and GlobalVariables.CLEAN_FLAG: cleanup() - - end = datetime.datetime.now() - duration = (end - start).seconds - duration_str = ("%02d:%02d" % divmod(duration, 60)) - logger.info("Test execution time: %s" % duration_str) - - if result != 0: + if result != testcase.TestCase.EX_OK: logger.error("The test case '%s' failed. " % test_name) GlobalVariables.OVERALL_RESULT = Result.EX_ERROR - result_str = "FAIL" - if test.is_blocking(): - if not testcases or testcases == "all": - # if it is a single test we don't print the whole results table - update_test_info(test_name, result_str, duration_str) - generate_report.main(GlobalVariables.EXECUTED_TEST_CASES) raise BlockingTestFailed("The test case {} failed and is blocking" .format(test.get_name())) - update_test_info(test_name, result_str, duration_str) - def run_tier(tier): tier_name = tier.get_name() @@ -211,27 +196,24 @@ def run_all(tiers): for tier in tiers.get_tiers(): if (len(tier.get_tests()) != 0 and - re.search(CONST.CI_LOOP, tier.get_ci_loop()) is not None): + re.search(CONST.__getattribute__('CI_LOOP'), + tier.get_ci_loop()) is not None): tiers_to_run.append(tier) summary += ("\n - %s:\n\t %s" % (tier.get_name(), tier.get_test_names())) logger.info("Tests to be executed:%s" % summary) - GlobalVariables.EXECUTED_TEST_CASES = generate_report.init(tiers_to_run) for tier in tiers_to_run: run_tier(tier) - generate_report.main(GlobalVariables.EXECUTED_TEST_CASES) - def main(**kwargs): - CI_INSTALLER_TYPE = CONST.INSTALLER_TYPE - CI_SCENARIO = CONST.DEPLOY_SCENARIO - file = CONST.functest_testcases_yaml - _tiers = tb.TierBuilder(CI_INSTALLER_TYPE, CI_SCENARIO, file) + _tiers = tb.TierBuilder(CONST.__getattribute__('INSTALLER_TYPE'), + CONST.__getattribute__('DEPLOY_SCENARIO'), + file) if kwargs['noclean']: GlobalVariables.CLEAN_FLAG = False @@ -243,12 +225,10 @@ def main(**kwargs): if kwargs['test']: source_rc_file() if _tiers.get_tier(kwargs['test']): - GlobalVariables.EXECUTED_TEST_CASES = generate_report.init( - [_tiers.get_tier(kwargs['test'])]) run_tier(_tiers.get_tier(kwargs['test'])) elif _tiers.get_test(kwargs['test']): run_test(_tiers.get_test(kwargs['test']), - _tiers.get_tier(kwargs['test']), + _tiers.get_tier_name(kwargs['test']), kwargs['test']) elif kwargs['test'] == "all": run_all(_tiers) @@ -256,7 +236,8 @@ def main(**kwargs): logger.error("Unknown test case or tier '%s', " "or not supported by " "the given scenario '%s'." - % (kwargs['test'], CI_SCENARIO)) + % (kwargs['test'], + CONST.__getattribute__('DEPLOY_SCENARIO'))) logger.debug("Available tiers are:\n\n%s" % _tiers) return Result.EX_ERROR @@ -265,11 +246,33 @@ def main(**kwargs): except Exception as e: logger.error(e) GlobalVariables.OVERALL_RESULT = Result.EX_ERROR + + msg = prettytable.PrettyTable( + header_style='upper', padding_width=5, + field_names=['env var', 'value']) + for env_var in ['INSTALLER_TYPE', 'DEPLOY_SCENARIO', 'BUILD_TAG', + 'CI_LOOP']: + msg.add_row([env_var, CONST.__getattribute__(env_var)]) + logger.info("Deployment description: \n\n%s\n", msg) + + msg = prettytable.PrettyTable( + header_style='upper', padding_width=5, + field_names=['test case', 'project', 'tier', 'duration', 'result']) + for test_case in GlobalVariables.EXECUTED_TEST_CASES: + result = 'PASS' if(test_case.is_successful( + ) == test_case.EX_OK) else 'FAIL' + msg.add_row([test_case.case_name, test_case.project_name, + _tiers.get_tier_name(test_case.case_name), + test_case.get_duration(), result]) + logger.info("FUNCTEST REPORT: \n\n%s\n", msg) + logger.info("Execution exit value: %s" % GlobalVariables.OVERALL_RESULT) return GlobalVariables.OVERALL_RESULT if __name__ == '__main__': + logging.config.fileConfig( + CONST.__getattribute__('dir_functest_logging_cfg')) parser = RunTestsParser() args = parser.parse_args(sys.argv[1:]) sys.exit(main(**args).value) diff --git a/functest/ci/testcases.yaml b/functest/ci/testcases.yaml index 7009e910..d98a2de2 100644 --- a/functest/ci/testcases.yaml +++ b/functest/ci/testcases.yaml @@ -19,7 +19,6 @@ tiers: simple queries. When the config value of snaps.use_keystone is True, functest must have access to the cloud's private network. - dependencies: installer: '^((?!netvirt).)*$' scenario: '' @@ -39,13 +38,13 @@ tiers: simple queries. When the config value of snaps.use_keystone is True, functest must have access to the cloud's private network. - dependencies: installer: '^((?!netvirt).)*$' scenario: '' run: module: 'functest.opnfv_tests.openstack.snaps.api_check' class: 'ApiCheck' + - case_name: snaps_health_check project_name: functest @@ -63,6 +62,7 @@ tiers: run: module: 'functest.opnfv_tests.openstack.snaps.health_check' class: 'HealthCheck' + - name: smoke order: 1 @@ -266,6 +266,7 @@ tiers: testcases: - case_name: promise + enabled: false project_name: promise criteria: 100 blocking: false @@ -283,6 +284,7 @@ tiers: - case_name: doctor-notification + enabled: false project_name: doctor criteria: 100 blocking: false @@ -300,6 +302,7 @@ tiers: - case_name: bgpvpn + enabled: false project_name: sdnvpn criteria: 100 blocking: false @@ -317,6 +320,7 @@ tiers: - case_name: security_scan + enabled: false project_name: securityscanning criteria: 100 blocking: false @@ -334,6 +338,7 @@ tiers: - case_name: copper + enabled: false project_name: copper criteria: 100 blocking: false @@ -351,6 +356,7 @@ tiers: - case_name: multisite + enabled: false project_name: multisite criteria: 100 blocking: false @@ -363,8 +369,10 @@ tiers: run: module: 'functest.opnfv_tests.openstack.tempest.tempest' class: 'TempestMultisite' + - case_name: functest-odl-sfc + enabled: false project_name: sfc criteria: 100 blocking: false @@ -379,8 +387,10 @@ tiers: class: 'BashFeature' args: cmd: 'cd /home/opnfv/repos/sfc/sfc/tests/functest && python ./run_tests.py' + - case_name: onos_sfc + enabled: false project_name: functest criteria: 100 blocking: true @@ -393,8 +403,10 @@ tiers: run: module: 'functest.opnfv_tests.sdn.onos.onos' class: 'OnosSfc' + - case_name: parser-basics + enabled: false project_name: parser criteria: 100 blocking: false @@ -409,8 +421,10 @@ tiers: class: 'BashFeature' args: cmd: 'cd /home/opnfv/repos/parser/tests && ./functest_run.sh' + - case_name: domino-multinode + enabled: false project_name: domino criteria: 100 blocking: false @@ -425,8 +439,10 @@ tiers: class: 'BashFeature' args: cmd: 'cd /home/opnfv/repos/domino && ./tests/run_multinode.sh' + - case_name: gluon_vping + enabled: false project_name: netready criteria: 100 blocking: false @@ -441,8 +457,10 @@ tiers: class: 'BashFeature' args: cmd: 'cd /home/opnfv/repos/netready/test/functest && python ./gluon-test-suite.py' + - case_name: barometercollectd + enabled: false project_name: barometer criteria: 100 blocking: false @@ -458,6 +476,7 @@ tiers: run: module: 'functest.opnfv_tests.features.barometer' class: 'BarometerCollectd' + - name: components order: 3 @@ -481,6 +500,7 @@ tiers: run: module: 'functest.opnfv_tests.openstack.tempest.tempest' class: 'TempestFullParallel' + - case_name: tempest_custom project_name: functest @@ -499,6 +519,7 @@ tiers: run: module: 'functest.opnfv_tests.openstack.tempest.tempest' class: 'TempestCustom' + - case_name: rally_full project_name: functest @@ -537,22 +558,26 @@ tiers: run: module: 'functest.opnfv_tests.vnf.ims.cloudify_ims' class: 'CloudifyIms' -# - -# case_name: aaa -# project_name: functest -# criteria: 100 -# blocking: false -# clean_flag: true -# description: >- -# Test suite from Parser project. -# dependencies: -# installer: '' -# scenario: '' -# run: -# module: 'functest.opnfv_tests.vnf.aaa.aaa' -# class: 'AaaVnf' + + - + case_name: aaa + enabled: false + project_name: functest + criteria: 100 + blocking: false + clean_flag: true + description: >- + Test suite from Parser project. + dependencies: + installer: '' + scenario: '' + run: + module: 'functest.opnfv_tests.vnf.aaa.aaa' + class: 'AaaVnf' + - case_name: orchestra_ims + enabled: false project_name: functest criteria: 100 blocking: false @@ -568,6 +593,7 @@ tiers: - case_name: opera_vims + enabled: false project_name: opera criteria: 100 blocking: false @@ -583,6 +609,7 @@ tiers: - case_name: vyos_vrouter + enabled: false project_name: functest criteria: 100 blocking: false diff --git a/functest/ci/tier_builder.py b/functest/ci/tier_builder.py index 44b27258..12562f09 100644 --- a/functest/ci/tier_builder.py +++ b/functest/ci/tier_builder.py @@ -47,12 +47,15 @@ class TierBuilder(object): dep = th.Dependency(installer, scenario) testcase = th.TestCase(name=dic_testcase['case_name'], + enabled=dic_testcase.get( + 'enabled', True), dependency=dep, criteria=dic_testcase['criteria'], blocking=dic_testcase['blocking'], clean_flag=dic_testcase['clean_flag'], description=dic_testcase['description']) - if testcase.is_compatible(self.ci_installer, self.ci_scenario): + if (testcase.is_compatible(self.ci_installer, self.ci_scenario) + and testcase.is_enabled()): tier.add_test(testcase) self.tier_objects.append(tier) @@ -72,6 +75,12 @@ class TierBuilder(object): return self.tier_objects[i] return None + def get_tier_name(self, test_name): + for i in range(0, len(self.tier_objects)): + if self.tier_objects[i].is_test(test_name): + return self.tier_objects[i].name + return None + def get_test(self, test_name): for i in range(0, len(self.tier_objects)): if self.tier_objects[i].is_test(test_name): diff --git a/functest/ci/tier_handler.py b/functest/ci/tier_handler.py index fe7372a3..36ce245e 100644 --- a/functest/ci/tier_handler.py +++ b/functest/ci/tier_handler.py @@ -105,12 +105,14 @@ class Tier(object): class TestCase(object): def __init__(self, name, + enabled, dependency, criteria, blocking, clean_flag, description=""): self.name = name + self.enabled = enabled self.dependency = dependency self.criteria = criteria self.blocking = blocking @@ -138,6 +140,9 @@ class TestCase(object): def get_name(self): return self.name + def is_enabled(self): + return self.enabled + def get_criteria(self): return self.criteria |