diff options
| -rw-r--r-- | .gitignore | 41 | ||||
| -rw-r--r-- | docker/Dockerfile | 24 | ||||
| -rw-r--r-- | scripts/__init__.py | 0 | ||||
| -rw-r--r-- | scripts/cert/basic.yml | 4 | ||||
| -rw-r--r-- | scripts/conf/__init__.py | 0 | ||||
| -rw-r--r-- | scripts/conf/dovetail_config.py | 28 | ||||
| -rw-r--r-- | scripts/conf/dovetail_config.yml | 10 | ||||
| -rw-r--r-- | scripts/conf/functest_config.yml | 17 | ||||
| -rw-r--r-- | scripts/conf/yardstick_config.yml | 20 | ||||
| -rw-r--r-- | scripts/container.py | 77 | ||||
| -rw-r--r-- | scripts/parser.py | 9 | ||||
| -rw-r--r-- | scripts/prepare_env.py | 23 | ||||
| -rwxr-xr-x | scripts/prepare_test_yard/build_flavor_image.sh | 146 | ||||
| -rwxr-xr-x | scripts/prepare_test_yard/build_run_test.sh | 23 | ||||
| -rwxr-xr-x | scripts/prepare_test_yard/run_test.sh | 20 | ||||
| -rwxr-xr-x | scripts/prepare_test_yard/source_env.sh | 21 | ||||
| -rw-r--r-- | scripts/report.py | 213 | ||||
| -rwxr-xr-x | scripts/run.py | 90 | ||||
| -rw-r--r-- | scripts/testcase.py | 135 | ||||
| -rw-r--r-- | scripts/testcase/ipv6.tc001.yml | 10 | ||||
| -rw-r--r-- | scripts/utils/__init__.py | 0 | ||||
| -rw-r--r-- | scripts/utils/dovetail_logger.py | 55 | ||||
| -rw-r--r-- | scripts/utils/dovetail_utils.py | 56 |
23 files changed, 1017 insertions, 5 deletions
@@ -1,5 +1,36 @@ -*~ -.*.sw? -/docs_build/ -/docs_output/ -/releng/ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +__pycache__ + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..8d6883c1 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,24 @@ +FROM ubuntu:14.04 +MAINTAINER Leo Wang <grakiss.wanglei@huawei.com> +LABEL version="0.1" description="OPNFV Dovetail Docker Container" + +RUN apt-get update && apt-get install -y \ +docker.io \ +python-pip \ +git \ +--no-install-recommends + +RUN pip install pyyaml +RUN pip install click + +ENV HOME /home/opnfv +ENV REPOS_DIR /home/opnfv/dovetail +WORKDIR /home/opnfv + +RUN git config --global http.sslVerify false +RUN git clone https://gerrit.opnfv.org/gerrit/dovetail.git ${REPOS_DIR} + +RUN mkdir -p ${REPOS_DIR}/results + +WORKDIR /home/opnfv/dovetail/scripts + diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/scripts/__init__.py diff --git a/scripts/cert/basic.yml b/scripts/cert/basic.yml new file mode 100644 index 00000000..25ebc7ae --- /dev/null +++ b/scripts/cert/basic.yml @@ -0,0 +1,4 @@ +certification_basic: + name: certification_basic + testcase_list: + - dovetail.ipv6.tc001 diff --git a/scripts/conf/__init__.py b/scripts/conf/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/scripts/conf/__init__.py diff --git a/scripts/conf/dovetail_config.py b/scripts/conf/dovetail_config.py new file mode 100644 index 00000000..92d1f479 --- /dev/null +++ b/scripts/conf/dovetail_config.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# +# grakiss.wanglei@huawei.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +CERT_PATH = './cert/' +TESTCASE_PATH = './testcase/' +SCENARIO_NAMING_FMT = 'certification_%s' + +import yaml +import os + +with open(os.path.join(os.getcwd(),'conf','dovetail_config.yml')) as f: + dovetail_config = yaml.safe_load(f) + +for extra_config_file in dovetail_config['extra_config']: + with open(os.path.join(os.getcwd(),'conf',extra_config_file)) as f: + extra_config = yaml.safe_load(f) + dovetail_config.update(extra_config) + +container_config = {} + +container_config['functest'] = dovetail_config['functest'] +container_config['yardstick'] = dovetail_config['yardstick'] diff --git a/scripts/conf/dovetail_config.yml b/scripts/conf/dovetail_config.yml new file mode 100644 index 00000000..57d6e894 --- /dev/null +++ b/scripts/conf/dovetail_config.yml @@ -0,0 +1,10 @@ + +work_dir: /home/opnfv/dovetail +result_dir: /home/opnfv/dovetail/results +report_file: 'dovetail_report.txt' + +extra_config: + - functest_config.yml + - yardstick_config.yml + + diff --git a/scripts/conf/functest_config.yml b/scripts/conf/functest_config.yml new file mode 100644 index 00000000..86e6ce75 --- /dev/null +++ b/scripts/conf/functest_config.yml @@ -0,0 +1,17 @@ +functest: + image_name: opnfv/functest + docker_tag: latest + envs: '-e INSTALLER_TYPE=compass -e INSTALLER_IP=192.168.200.2 + -e NODE_NAME=dovetail-pod -e DEPLOY_SCENARIO=ha_nosdn + -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal' + opts: '-id --privileged=true' + result_dir: '/home/opnfv/functest/results' + testcase: + pre_cmd: 'python /home/opnfv/repos/functest/ci/prepare_env.py start' + exec_cmd: 'python /home/opnfv/repos/functest/ci/run_tests.py -t %s -r' + post_cmd: '' + result: + dir: '/home/opnfv/functest/results' + store_type: 'file' + file_path: 'tempest/tempest.log' + db_url: 'http://testresults.opnfv.org/test/api/v1/results?case=%s&last=1' diff --git a/scripts/conf/yardstick_config.yml b/scripts/conf/yardstick_config.yml new file mode 100644 index 00000000..9eda6e54 --- /dev/null +++ b/scripts/conf/yardstick_config.yml @@ -0,0 +1,20 @@ +yardstick: + image_name: opnfv/yardstick + docker_tag: stable + envs: '-e INSTALLER_TYPE=compass -e INSTALLER_IP=192.168.200.2 + -e NODE_NAME=dovetail-pod -e DEPLOY_SCENARIO=ha_nosdn + -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal + -e EXTERNAL_NETWORK=ext-net' + opts: '-id --privileged=true' + result_dir: '/tmp/yardstick/result' + shell_dir: '/tmp/yardstick' + shell_dir_name: 'prepare_test_yard' + testcase: + build_test_cmd: '/tmp/yardstick/build_run_test.sh %s.yaml /tmp/yardstick/result/%s.out' + test_cmd: '/tmp/yardstick/run_test.sh %s.yaml /tmp/yardstick/result/%s.out' + result: + dir: '/tmp/yardstick/result' + store_type: 'file' + db_url: 'http://testresults.opnfv.org/test/api/v1/results?case=%s&last=1' + file_path: '/home/opnfv/dovetail/results/yardstick.log' + diff --git a/scripts/container.py b/scripts/container.py new file mode 100644 index 00000000..678a0929 --- /dev/null +++ b/scripts/container.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# +# grakiss.wanglei@huawei.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +import utils.dovetail_logger as dt_logger +import utils.dovetail_utils as dt_utils +from conf.dovetail_config import * + +logger = dt_logger.Logger('container.py').getLogger() + +class Container: + + container_list = {} + has_pull_latest_image = {'yardstick':False, 'functest':False} + has_build_images = {'yardstick':False, 'functest':False} + + def __init__(cls): + pass + + def __str__(cls): + pass + + @classmethod + def get(cls, type): + return cls.container_list[type] + + @classmethod + def get_docker_image(cls, type): + return '%s:%s' % (dovetail_config[type]['image_name'], dovetail_config[type]['docker_tag']) + + @classmethod + def create(cls, type): + #sshkey="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa " + docker_image = cls.get_docker_image(type) + envs = dovetail_config[type]['envs'] + opts = dovetail_config[type]['opts'] + sshkey = '' + result_volume = ' -v %s:%s ' % (dovetail_config['result_dir'],dovetail_config[type]['result']['dir']) + cmd = 'sudo docker run %s %s %s %s %s /bin/bash' % (opts, envs, sshkey, result_volume, docker_image) + dt_utils.exec_cmd(cmd,logger) + ret, container_id=dt_utils.exec_cmd("sudo docker ps | grep "+ docker_image + " | awk '{print $1}' | head -1",logger) + cls.container_list[type] = container_id + return container_id + + @classmethod + def pull_image(cls, type): + docker_image = cls.get_docker_image(type) + if cls.has_pull_latest_image[type] == True: + logger.debug('%s is already the newest version.' % (docker_image)) + else: + cmd = 'sudo docker pull %s' % (docker_image) + dt_utils.exec_cmd(cmd,logger) + cls.has_pull_latest_image[type] = True + + @classmethod + def clean(cls, container_id): + cmd1 = 'sudo docker stop %s' % (container_id) + dt_utils.exec_cmd(cmd1,logger) + cmd2 = 'sudo docker rm %s' % (container_id) + dt_utils.exec_cmd(cmd2,logger) + + @classmethod + def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False): + cmd = 'sudo docker exec %s %s' % (container_id, sub_cmd) + dt_utils.exec_cmd(cmd,logger,exit_on_error) + + @classmethod + def copy_file(cls, file_dir, container_id, container_dir): + for root, dirs, files in os.walk(file_dir): + for file_name in files: + cmd = 'sudo docker cp %s %s:%s' % (os.path.join(file_dir,file_name), container_id, container_dir) + dt_utils.exec_cmd(cmd, logger, exit_on_error = False) diff --git a/scripts/parser.py b/scripts/parser.py new file mode 100644 index 00000000..c0b18e13 --- /dev/null +++ b/scripts/parser.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python +# +# grakiss.wanglei@huawei.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + diff --git a/scripts/prepare_env.py b/scripts/prepare_env.py new file mode 100644 index 00000000..f3915a95 --- /dev/null +++ b/scripts/prepare_env.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# +# grakiss.wanglei@huawei.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +import os + +import utils.dovetail_logger as dt_logger +import utils.dovetail_utils as dt_utils + + +logger = dt_logger.Logger('prepare_env.py').getLogger() + +cmd = "sudo apt-get -y install docker.io python-pip" +dt_utils.exec_cmd(cmd, logger) + +cmd = "sudo pip install click pyyaml" +dt_utils.exec_cmd(cmd, logger) + diff --git a/scripts/prepare_test_yard/build_flavor_image.sh b/scripts/prepare_test_yard/build_flavor_image.sh new file mode 100755 index 00000000..0802f08d --- /dev/null +++ b/scripts/prepare_test_yard/build_flavor_image.sh @@ -0,0 +1,146 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. + +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -e + +cleanup() +{ + echo + echo "========== Cleanup ==========" + + if ! glance image-list; then + return + fi + + for image in $(glance image-list | grep -e cirros-0.3.3 -e yardstick-trusty-server -e Ubuntu-14.04 \ + -e yardstick-vivid-kernel | awk '{print $2}'); do + echo "Deleting image $image..." + glance image-delete $image || true + done + + nova flavor-delete yardstick-flavor &> /dev/null || true +} + +create_nova_flavor() +{ + if ! nova flavor-list | grep -q yardstick-flavor; then + echo + echo "========== Create nova flavor ==========" + # Create the nova flavor used by some sample test cases + nova flavor-create yardstick-flavor 100 512 3 1 + # DPDK-enabled OVS requires guest memory to be backed by large pages + if [[ "$DEPLOY_SCENARIO" == *"-ovs-"* ]]; then + nova flavor-key yardstick-flavor set hw:mem_page_size=large + fi + fi +} + +load_cirros_image() +{ + echo + echo "========== Loading cirros cloud image ==========" + + local image_file=/home/opnfv/images/cirros-0.3.3-x86_64-disk.img + + output=$(glance image-create \ + --name cirros-0.3.3 \ + --disk-format qcow2 \ + --container-format bare \ + --file $image_file) + echo "$output" + + CIRROS_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}') + if [ -z "$CIRROS_IMAGE_ID" ]; then + echo 'Failed uploading cirros image to cloud'. + exit 1 + fi + + echo "Cirros image id: $CIRROS_IMAGE_ID" +} + +load_ubuntu_image() +{ + echo + echo "========== Loading ubuntu cloud image ==========" + + local ubuntu_image_file=/home/opnfv/images/trusty-server-cloudimg-amd64-disk1.img + + output=$(glance image-create \ + --name Ubuntu-14.04 \ + --disk-format qcow2 \ + --container-format bare \ + --file $ubuntu_image_file) + echo "$output" + + UBUNTU_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}') + + if [ -z "$UBUNTU_IMAGE_ID" ]; then + echo 'Failed uploading UBUNTU image to cloud'. + exit 1 + fi + + echo "Ubuntu image id: $UBUNTU_IMAGE_ID" +} + +QCOW_IMAGE="/tmp/workspace/yardstick/yardstick-trusty-server.img" + +build_yardstick_image() +{ + echo + echo "========== Build yardstick cloud image ==========" + + export YARD_IMG_ARCH="amd64" + sudo echo "Defaults env_keep += \"YARD_IMG_ARCH\"" >> /etc/sudoers + local cmd="sudo $(which yardstick-img-modify) $(pwd)/tools/ubuntu-server-cloudimg-modify.sh" + + # Build the image. Retry once if the build fails. + $cmd || $cmd + + if [ ! -f $QCOW_IMAGE ]; then + echo "Failed building QCOW image" + exit 1 + fi +} + +load_yardstick_image() +{ + echo + echo "========== Loading yardstick cloud image ==========" + + output=$(glance --os-image-api-version 1 image-create \ + --name yardstick-trusty-server \ + --is-public true --disk-format qcow2 \ + --container-format bare \ + --file $QCOW_IMAGE) + echo "$output" + + GLANCE_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}') + + if [ -z "$GLANCE_IMAGE_ID" ]; then + echo 'Failed uploading image to cloud'. + exit 1 + fi + + sudo rm -f $QCOW_IMAGE + + echo "Glance image id: $GLANCE_IMAGE_ID" +} + +main() +{ + cleanup + create_nova_flavor + load_cirros_image + load_ubuntu_image + build_yardstick_image + load_yardstick_image +} + +main diff --git a/scripts/prepare_test_yard/build_run_test.sh b/scripts/prepare_test_yard/build_run_test.sh new file mode 100755 index 00000000..91ac589e --- /dev/null +++ b/scripts/prepare_test_yard/build_run_test.sh @@ -0,0 +1,23 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. + +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -e + +cd /home/opnfv/repos/yardstick + +source /tmp/yardstick/source_env.sh + +echo "========== Build the flavor and load images ==========" +source /tmp/yardstick/build_flavor_image.sh + +echo "========== Run yardstick testcase ==========" +cmd="yardstick task start /home/opnfv/repos/yardstick/tests/opnfv/test_cases/$1 \ + --output-file $2" +${cmd} diff --git a/scripts/prepare_test_yard/run_test.sh b/scripts/prepare_test_yard/run_test.sh new file mode 100755 index 00000000..326c637b --- /dev/null +++ b/scripts/prepare_test_yard/run_test.sh @@ -0,0 +1,20 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. + +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -e + +cd /home/opnfv/repos/yardstick + +source /tmp/yardstick/source_env.sh + +echo "========== Run yardstick testcase ==========" +cmd="yardstick task start /home/opnfv/repos/yardstick/tests/opnfv/test_cases/$1 \ + --output-file $2" +${cmd} diff --git a/scripts/prepare_test_yard/source_env.sh b/scripts/prepare_test_yard/source_env.sh new file mode 100755 index 00000000..80135d70 --- /dev/null +++ b/scripts/prepare_test_yard/source_env.sh @@ -0,0 +1,21 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. + +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +set -e + +: ${YARDSTICK_REPO:='https://gerrit.opnfv.org/gerrit/yardstick'} +: ${YARDSTICK_REPO_DIR:='/home/opnfv/repos/yardstick'} +: ${YARDSTICK_BRANCH:='master'} # branch, tag, sha1 or refspec + +: ${RELENG_REPO:='https://gerrit.opnfv.org/gerrit/releng'} +: ${RELENG_REPO_DIR:='/home/opnfv/repos/releng'} +: ${RELENG_BRANCH:='master'} # branch, tag, sha1 or refspec + +source $YARDSTICK_REPO_DIR/tests/ci/prepare_env.sh diff --git a/scripts/report.py b/scripts/report.py new file mode 100644 index 00000000..d92929f2 --- /dev/null +++ b/scripts/report.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python +# +# grakiss.wanglei@huawei.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# +import json +import urllib2 +import re + +import utils.dovetail_logger as dt_logger +import utils.dovetail_utils as dt_utils + +logger = dt_logger.Logger('report.py').getLogger() + +from conf.dovetail_config import * +from testcase import * + +def get_pass_str(passed): + if passed: + return 'PASS' + else: + return 'FAIL' + +class Report: + + results = {'functest':{},'yardstick':{}} + + @classmethod + def check_result(cls, testcase, db_result): + checker = CheckerFactory.create(testcase.script_type()) + checker.check(testcase, db_result) + + @classmethod + def generate(cls, scenario_yaml): + report = '' + + report += '\n\ ++=============================================================================+\n\ +| report | \n\ ++-----------------------------------------------------------------------------+\n' + report += '|scenario: %s\n' % scenario_yaml['name'] + for testcase_name in scenario_yaml['testcase_list']: + testcase = Testcase.get(testcase_name) + report += '| [testcase]: %s\t\t\t\t[%s]\n' % (testcase_name, get_pass_str(testcase.passed())) + report += '| |-objective: %s\n' % testcase.objective() + if testcase.sub_testcase() is not None: + for subtest in testcase.sub_testcase(): + report += '| |-%s \t\t [%s]\n' % (subtest, get_pass_str(testcase.sub_testcase_passed(subtest))) + report += '+-----------------------------------------------------------------------------+\n' + + logger.info(report) + cls.save(report) + return report + + #save to disk as default + @classmethod + def save(cls, report): + report_file_path = dovetail_config['report_file'] + try: + with open(os.path.join(dovetail_config['result_dir'], report_file_path),'w') as report_file: + report_file.write(report) + logger.info('save report to %s' % report_file_path) + except Exception as e: + logger.error('Failed to save: %s' % report_file_path) + + @classmethod + def get_result(cls, testcase): + script_testcase = testcase.script_testcase() + type = testcase.script_type() + crawler = CrawlerFactory.create(type) + + if script_testcase in cls.results[type]: + return cls.results[type][script_testcase] + + result = crawler.crawl(script_testcase) + + if result is not None: + cls.results[type][script_testcase] = result + testcase.script_result_acquired(True) + logger.debug('testcase: %s -> result acquired' % script_testcase) + else: + retry = testcase.increase_retry() + logger.debug('testcase: %s -> result acquired retry:%d' % (script_testcase, retry)) + return result + +class CrawlerFactory: + + @classmethod + def create(cls, type): + if type == 'functest': + return FunctestCrawler() + + if type == 'yardstick': + return YardstickCrawler() + + return None + +class FunctestCrawler: + + def __init__(self): + self.type = 'functest' + + def crawl(self, testcase=None): + store_type = dovetail_config[self.type]['result']['store_type'] + if store_type == 'file': + return self.crawl_from_file(testcase) + + if store_type == 'url': + return self.crawl_from_url(testcase) + + def crawl_from_file(self, testcase=None): + file_path = os.path.join(dovetail_config['result_dir'],dovetail_config[self.type]['result']['file_path']) + if not os.path.exists(file_path): + logger.info('result file not found: %s' % file_path) + return None + + try: + with open(file_path, 'r') as myfile: + output = myfile.read() + error_logs = "" + + for match in re.findall('(.*?)[. ]*FAILED', output): + error_logs += match + + criteria = 'PASS' + failed_num = int(re.findall(' - Failed: (\d*)', output)[0]) + if failed_num != 0: + criteria = 'FAIL' + + match = re.findall('Ran: (\d*) tests in (\d*)\.\d* sec.', output) + num_tests, dur_sec_int = match[0] + json_results = {'criteria':criteria,'details':{"timestart": '', "duration": int(dur_sec_int), + "tests": int(num_tests), "failures": failed_num, + "errors": error_logs}} + logger.debug('Results: %s' % str(json_results)) + return json_results + except Exception as e: + logger.error('Cannot read content from the file: %s, exception: %s' % (file_path, e)) + return None + + def crawl_from_url(self, testcase=None): + url = dovetail_config[self.type]['result']['db_url'] % testcase + logger.debug("Query to rest api: %s" % url) + try: + data = json.load(urllib2.urlopen(url)) + return data['results'][0] + except Exception as e: + logger.error("Cannot read content from the url: %s, exception: %s" % (url, e)) + return None + +class YardstickCrawler: + + def __init__(self): + self.type = 'yardstick' + + def crawl(self, testcase=None): + return None + +class CheckerFactory: + + @classmethod + def create(cls,type): + if type == 'functest': + return FunctestChecker() + + if type == 'yardstick': + return YardstickChecker() + + return None + +class ResultChecker: + + def check(cls): + return 'PASS' + +class FunctestChecker: + + def check(cls, testcase, db_result): + if not db_result: + for sub_testcase in testcase.sub_testcase(): + testcase.sub_testcase_passed(sub_testcase,False) + return + + testcase.passed(db_result['criteria'] == 'PASS') + + if testcase.sub_testcase() is None: + return + + if testcase.testcase['passed'] == True: + for sub_testcase in testcase.sub_testcase(): + testcase.sub_testcase_passed(sub_testcase, True) + return + + all_passed = True + for sub_testcase in testcase.sub_testcase(): + logger.debug('check sub_testcase:%s' % sub_testcase) + if sub_testcase in db_result['details']['errors']: + testcase.sub_testcase_passed(sub_testcase, False) + all_passed = False + else: + testcase.sub_testcase_passed(sub_testcase, True) + + testcase.passed(all_passed) + +class YardstickChecker: + + def check(cls, testcase, result): + return 'PASS' + + diff --git a/scripts/run.py b/scripts/run.py new file mode 100755 index 00000000..9b59f76f --- /dev/null +++ b/scripts/run.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# +# grakiss.wanglei@huawei.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + + +import click +import yaml +import os +import time + +import utils.dovetail_logger as dt_logger +import utils.dovetail_utils as dt_utils +logger = dt_logger.Logger('run.py').getLogger() + +from container import Container +from testcase import * +from report import * +from conf.dovetail_config import * + +def load_scenario(scenario): + Scenario.load() + return Scenario.get(SCENARIO_NAMING_FMT % scenario) + +def load_testcase(): + Testcase.load() + +def run_functest(testcase, container_id): + sub_cmd = dovetail_config[testcase.script_type()]['testcase']['pre_cmd'] + Container.exec_cmd(container_id, sub_cmd) + sub_cmd = dovetail_config[testcase.script_type()]['testcase']['exec_cmd'] % testcase.script_testcase() + Container.exec_cmd(container_id, sub_cmd) + +def run_yardstick(testcase, container_id): + type = testcase.script_type() + Container.copy_file(os.path.join(os.getcwd(), dovetail_config[type]['shell_dir_name']),\ + container_id, dovetail_config[type]['shell_dir']) + if Container.has_build_images[type] == True: + sub_cmd = dovetail_config[type]['testcase']['test_cmd'] % (testcase.script_testcase(), testcase.name()) + else: + Container.has_build_images[type] = True + sub_cmd = dovetail_config[type]['testcase']['build_test_cmd'] % (testcase.script_testcase(), testcase.name()) + Container.exec_cmd(container_id, sub_cmd) + time.sleep(5) + +def run_test(scenario): + for testcase_name in scenario['testcase_list']: + logger.info('>>[testcase]: %s' % (testcase_name)) + testcase = Testcase.get(testcase_name) + run_test = True + + if testcase.exceed_max_retry_times(): + run_test = False + + if testcase.script_result_acquired(): + run_test = False + + if run_test: + Container.pull_image(testcase.script_type()) + container_id = Container.create(testcase.script_type()) + logger.debug('container id:%s' % container_id) + + if testcase.script_type() == 'functest': + run_functest(testcase, container_id) + else: + run_yardstick(testcase, container_id) + + Container.clean(container_id) + + db_result = Report.get_result(testcase) + Report.check_result(testcase, db_result) + +@click.command() +@click.option('--scenario', default='basic', help='certification scenario') +def main(scenario): + """Dovetail certification test entry!""" + logger.info('=======================================') + logger.info('Dovetail certification: %s!' % scenario) + logger.info('=======================================') + load_testcase() + scenario_yaml = load_scenario(scenario) + run_test(scenario_yaml) + Report.generate(scenario_yaml) + +if __name__ == '__main__': + main() diff --git a/scripts/testcase.py b/scripts/testcase.py new file mode 100644 index 00000000..71ffd7ac --- /dev/null +++ b/scripts/testcase.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# +# grakiss.wanglei@huawei.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +import utils.dovetail_logger as dt_logger +import utils.dovetail_utils as dt_utils + +logger = dt_logger.Logger('testcase.py').getLogger() + +from conf.dovetail_config import * + +class Testcase: + + def __init__(self, testcase_yaml): + self.testcase = testcase_yaml.values()[0] + self.testcase['passed'] = False + self.sub_testcase_status = {} + Testcase.update_script_testcase(self.script_type(), self.script_testcase()) + + def __str__(self): + return self.testcase + + def name(self): + return self.testcase['name'] + + def objective(self): + return self.testcase['objective'] + + def sub_testcase(self): + return self.testcase['scripts']['sub_testcase_list'] + + def sub_testcase_passed(self, name, passed=None): + if passed is not None: + logger.debug('sub_testcase_passed:%s %s' % (name, passed)) + self.sub_testcase_status[name] = passed + return self.sub_testcase_status[name] + + def script_type(self): + return self.testcase['scripts']['type'] + + def script_testcase(self): + return self.testcase['scripts']['testcase'] + + def exceed_max_retry_times(self): + #logger.debug('retry times:%d' % self.testcase['retry']) + return Testcase._exceed_max_retry_times(self.script_type(), self.script_testcase()) + + def increase_retry(self): + #self.testcase['retry'] = self.testcase['retry'] + 1 + #return self.testcase['retry'] + return Testcase._increase_retry(self.script_type(), self.script_testcase()) + + def passed(self, passed = None): + if passed is not None: + self.testcase['passed'] = passed + return self.testcase['passed'] + + def script_result_acquired(self, acquired=None): + return Testcase._result_acquired(self.script_type(), self.script_testcase(), acquired) + + #testcase in upstream testing project + script_testcase_list = {'functest':{}, 'yardstick':{}} + + #testcase in dovetail + testcase_list = {} + + @classmethod + def update_script_testcase(cls,script_type, script_testcase): + if script_testcase not in cls.script_testcase_list[script_type]: + cls.script_testcase_list[script_type][script_testcase] = {'retry':0, 'acquired':False} + + @classmethod + def _exceed_max_retry_times(cls, script_type, script_testcase ): + return cls.script_testcase_list[script_type][script_testcase]['retry'] > 1 + + @classmethod + def _increase_retry(cls, script_type, script_testcase): + cls.script_testcase_list[script_type][script_testcase]['retry'] += 1 + return cls.script_testcase_list[script_type][script_testcase]['retry'] + + @classmethod + def _result_acquired(cls, script_type, script_testcase, acquired=None): + if acquired is not None: + cls.script_testcase_list[script_type][script_testcase]['acquired'] = acquired + return cls.script_testcase_list[script_type][script_testcase]['acquired'] + + @classmethod + def load(cls): + for root, dirs, files in os.walk(TESTCASE_PATH): + for testcase_file in files: + with open(os.path.join(root, testcase_file)) as f: + testcase_yaml = yaml.safe_load(f) + cls.testcase_list[testcase_yaml.keys()[0]] = Testcase(testcase_yaml) + logger.debug( cls.testcase_list ) + + @classmethod + def get(cls, testcase_name): + if testcase_name in cls.testcase_list: + return cls.testcase_list[testcase_name] + return None + + +class Scenario: + + def __init__(self, scenario): + self.scenario = scenario + self.testcase_list = {} + + def get_test(self, testcase_name): + if testcase_name in self.testcase_list: + return self.testcase_list[testcase_name] + return None + + scenario_list = {} + @classmethod + def load(cls): + for root, dirs, files in os.walk(CERT_PATH): + for scenario_yaml in files: + with open(os.path.join(root, scenario_yaml)) as f: + scenario_yaml = yaml.safe_load(f) + cls.scenario_list.update(scenario_yaml) + + logger.debug(cls.scenario_list) + + @classmethod + def get(cls, scenario_name): + if scenario_name in cls.scenario_list: + return cls.scenario_list[scenario_name] + return None + diff --git a/scripts/testcase/ipv6.tc001.yml b/scripts/testcase/ipv6.tc001.yml new file mode 100644 index 00000000..9f11ac76 --- /dev/null +++ b/scripts/testcase/ipv6.tc001.yml @@ -0,0 +1,10 @@ +dovetail.ipv6.tc001: + name: dovetail.ipv6.tc001 + objective: VIM ipv6 operations, to create/delete network, port and subnet in bulk operation + scripts: + type: functest + testcase: tempest_smoke_serial + sub_testcase_list: + - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network + - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_port + - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_subnet diff --git a/scripts/utils/__init__.py b/scripts/utils/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/scripts/utils/__init__.py diff --git a/scripts/utils/dovetail_logger.py b/scripts/utils/dovetail_logger.py new file mode 100644 index 00000000..7335c0ae --- /dev/null +++ b/scripts/utils/dovetail_logger.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# +# jose.lausuch@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Logging levels: +# Level Numeric value +# CRITICAL 50 +# ERROR 40 +# WARNING 30 +# INFO 20 +# DEBUG 10 +# NOTSET 0 +# +# Usage: +# import dovetail_logger as dl +# logger = dl.Logger("script_name").getLogger() +# logger.info("message to be shown with - INFO - ") +# logger.debug("message to be shown with - DEBUG -") + +import logging +import os + +class Logger: + def __init__(self, logger_name): + + CI_DEBUG = os.getenv('CI_DEBUG') + + self.logger = logging.getLogger(logger_name) + self.logger.propagate = 0 + self.logger.setLevel(logging.DEBUG) + + ch = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s - %(name)s - ' + '%(levelname)s - %(message)s') + ch.setFormatter(formatter) + if CI_DEBUG is not None and CI_DEBUG.lower() == "true": + ch.setLevel(logging.DEBUG) + else: + ch.setLevel(logging.INFO) + self.logger.addHandler(ch) + + if not os.path.exists('/home/opnfv/dovetail/results/'): + os.makedirs('/home/opnfv/dovetail/results/') + hdlr = logging.FileHandler('/home/opnfv/dovetail/results/dovetail.log') + hdlr.setFormatter(formatter) + hdlr.setLevel(logging.DEBUG) + self.logger.addHandler(hdlr) + + def getLogger(self): + return self.logger + diff --git a/scripts/utils/dovetail_utils.py b/scripts/utils/dovetail_utils.py new file mode 100644 index 00000000..f79c6fc4 --- /dev/null +++ b/scripts/utils/dovetail_utils.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# +# jose.lausuch@ericsson.com +# valentin.boucher@orange.com +# grakiss.wanglei@huawei.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +# + +import sys +import subprocess + +def exec_cmd(cmd, logger=None, + exit_on_error=True, + info=False, + error_msg="", + verbose=True): + if not error_msg: + error_msg = ("The command '%s' failed." % cmd) + msg_exec = ("Executing command: '%s'" % cmd) + if verbose: + if logger: + if info: + logger.info(msg_exec) + else: + logger.debug(msg_exec) + else: + print(msg_exec) + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = p.communicate() + for line in output[0].strip().split('\n'): + line = line.replace('\n', '') + if logger: + if info: + logger.info(line) + else: + logger.debug(line) + else: + print line + sys.stdout.flush() + + returncode = p.returncode + if returncode != 0: + if verbose: + if logger: + logger.error(error_msg) + else: + print(error_msg) + if exit_on_error: + sys.exit(1) + + return returncode, output[0].strip() + |