Merge "add prototype of dovetail tool"

21 files changed, 919 insertions, 0 deletions

diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/scripts/__init__.py
diff --git a/scripts/cert/basic.yml b/scripts/cert/basic.yml
new file mode 100644
index 00000000..25ebc7ae
--- /dev/null
+++ b/scripts/cert/basic.yml
@@ -0,0 +1,4 @@
+certification_basic:
+  name: certification_basic
+  testcase_list:
+    - dovetail.ipv6.tc001
diff --git a/scripts/conf/__init__.py b/scripts/conf/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/scripts/conf/__init__.py
diff --git a/scripts/conf/dovetail_config.py b/scripts/conf/dovetail_config.py
new file mode 100644
index 00000000..8d60cd97
--- /dev/null
+++ b/scripts/conf/dovetail_config.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#
+# grakiss.wanglei@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+CERT_PATH = './cert/'
+TESTCASE_PATH = './testcase/'
+SCENARIO_NAMING_FMT = 'certification_%s'
+
+import yaml
+import os
+
+with open(os.path.join(os.getcwd(),'conf','dovetail_config.yml')) as f:
+    dovetail_config = yaml.safe_load(f)
+
+for extra_config_file in dovetail_config['extra_config']:
+    with open(os.path.join(os.getcwd(),'conf',extra_config_file)) as f:
+        extra_config = yaml.safe_load(f)
+        dovetail_config.update(extra_config)
+
+container_config = {}
+
+container_config['functest'] = dovetail_config['functest']
+container_config['yardstick'] = dovetail_config['yardstick']
+
+container_config['functest']['has_pull'] = False
+container_config['yardstick']['has_pull'] = False
+container_config['yardstick']['has_build_images'] = False
diff --git a/scripts/conf/dovetail_config.yml b/scripts/conf/dovetail_config.yml
new file mode 100644
index 00000000..b674a190
--- /dev/null
+++ b/scripts/conf/dovetail_config.yml
@@ -0,0 +1,9 @@
+
+work_dir: /home/opnfv/dovetail
+result_dir: /home/opnfv/dovetail/results
+
+extra_config:
+  - functest_config.yml
+  - yardstick_config.yml
+
+
diff --git a/scripts/conf/functest_config.yml b/scripts/conf/functest_config.yml
new file mode 100644
index 00000000..86e6ce75
--- /dev/null
+++ b/scripts/conf/functest_config.yml
@@ -0,0 +1,17 @@
+functest:
+  image_name: opnfv/functest
+  docker_tag: latest
+  envs: '-e INSTALLER_TYPE=compass -e INSTALLER_IP=192.168.200.2
+         -e NODE_NAME=dovetail-pod -e DEPLOY_SCENARIO=ha_nosdn
+         -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal'
+  opts: '-id --privileged=true'
+  result_dir: '/home/opnfv/functest/results'
+  testcase:
+    pre_cmd: 'python /home/opnfv/repos/functest/ci/prepare_env.py start'
+    exec_cmd: 'python /home/opnfv/repos/functest/ci/run_tests.py -t %s -r'
+    post_cmd: ''
+  result:
+    dir: '/home/opnfv/functest/results'
+    store_type: 'file'
+    file_path: 'tempest/tempest.log'
+    db_url: 'http://testresults.opnfv.org/test/api/v1/results?case=%s&last=1'
diff --git a/scripts/conf/yardstick_config.yml b/scripts/conf/yardstick_config.yml
new file mode 100644
index 00000000..5c04a286
--- /dev/null
+++ b/scripts/conf/yardstick_config.yml
@@ -0,0 +1,17 @@
+yardstick:
+  image_name: opnfv/yardstick
+  docker_tag: stable
+  envs: '-e INSTALLER_TYPE=compass -e INSTALLER_IP=192.168.200.2
+         -e NODE_NAME=dovetail-pod -e DEPLOY_SCENARIO=ha_nosdn
+         -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal
+         -e EXTERNAL_NETWORK=ext-net'
+  opts: '-id --privileged=true'
+  result_dir: '/tmp/yardstick/result'
+  shell_dir: '/tmp/yardstick'
+  shell_dir_name: 'prepare_test_yard'
+  result:
+    dir: '/tmp/yardstick/result'
+    store_type: 'file'
+    db_url: 'http://testresults.opnfv.org/test/api/v1/results?case=%s&last=1'
+    file_path: '/tmp/yardstick/result/yardstick.log'
+
diff --git a/scripts/container.py b/scripts/container.py
new file mode 100644
index 00000000..4bbc5e5b
--- /dev/null
+++ b/scripts/container.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+#
+# grakiss.wanglei@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+import utils.dovetail_logger as dt_logger
+import utils.dovetail_utils as dt_utils
+from conf.dovetail_config import *
+
+logger = dt_logger.Logger('container.py').getLogger()
+
+class Container:
+
+    container_list = {}
+
+    def __init__(cls):
+        pass
+
+    def __str__(cls):
+        pass
+
+    @classmethod
+    def get(cls, type):
+        return cls.container_list[type]
+
+    @classmethod
+    def get_docker_image(cls, type):
+        return '%s:%s' % (dovetail_config[type]['image_name'], dovetail_config[type]['docker_tag'])
+
+    @classmethod
+    def create(cls, type):
+        #sshkey="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa "
+        docker_image = cls.get_docker_image(type)
+        envs = dovetail_config[type]['envs']
+        opts = dovetail_config[type]['opts']
+        sshkey = ''
+        result_volume = ' -v %s:%s ' % (dovetail_config['result_dir'],dovetail_config[type]['result']['dir'])
+        cmd = 'sudo docker run %s %s %s %s %s /bin/bash' % (opts, envs, sshkey, result_volume, docker_image)
+        dt_utils.exec_cmd(cmd,logger)
+        ret, container_id=dt_utils.exec_cmd("sudo docker ps | grep "+ docker_image + " | awk '{print $1}' | head -1",logger)
+        cls.container_list[type] = container_id
+        return container_id
+
+    @classmethod
+    def pull_image(cls, type):
+        docker_image = cls.get_docker_image(type)
+        if container_config[type]['has_pull'] == True:
+            logger.debug('%s is already the newest version.' % (docker_image))
+        else:
+            cmd = 'sudo docker pull %s' % (docker_image)
+            dt_utils.exec_cmd(cmd,logger)
+            container_config[type]['has_pull'] = True
+
+    @classmethod
+    def clean(cls, container_id):
+        cmd1 = 'sudo docker stop %s' % (container_id)
+        dt_utils.exec_cmd(cmd1,logger)
+        cmd2 = 'sudo docker rm %s' % (container_id)
+        dt_utils.exec_cmd(cmd2,logger)
+
+    @classmethod
+    def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False):
+        cmd = 'sudo docker exec %s %s' % (container_id, sub_cmd)
+        dt_utils.exec_cmd(cmd,logger,exit_on_error)
+
+
diff --git a/scripts/parser.py b/scripts/parser.py
new file mode 100644
index 00000000..c0b18e13
--- /dev/null
+++ b/scripts/parser.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+#
+# grakiss.wanglei@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
diff --git a/scripts/prepare_env.py b/scripts/prepare_env.py
new file mode 100644
index 00000000..f3915a95
--- /dev/null
+++ b/scripts/prepare_env.py
@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+#
+# grakiss.wanglei@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+import os
+
+import utils.dovetail_logger as dt_logger
+import utils.dovetail_utils as dt_utils
+
+
+logger = dt_logger.Logger('prepare_env.py').getLogger()
+
+cmd = "sudo apt-get -y install docker.io python-pip"
+dt_utils.exec_cmd(cmd, logger)
+
+cmd = "sudo pip install click pyyaml"
+dt_utils.exec_cmd(cmd, logger)
+
diff --git a/scripts/prepare_test_yard/build_flavor_image.sh b/scripts/prepare_test_yard/build_flavor_image.sh
new file mode 100755
index 00000000..0802f08d
--- /dev/null
+++ b/scripts/prepare_test_yard/build_flavor_image.sh
@@ -0,0 +1,146 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -e
+
+cleanup()
+{
+    echo
+    echo "========== Cleanup =========="
+
+    if ! glance image-list; then
+        return
+    fi
+
+    for image in $(glance image-list | grep -e cirros-0.3.3 -e yardstick-trusty-server -e Ubuntu-14.04 \
+        -e yardstick-vivid-kernel | awk '{print $2}'); do
+        echo "Deleting image $image..."
+        glance image-delete $image || true
+    done
+
+    nova flavor-delete yardstick-flavor &> /dev/null || true
+}
+
+create_nova_flavor()
+{
+    if ! nova flavor-list | grep -q yardstick-flavor; then
+        echo
+        echo "========== Create nova flavor =========="
+        # Create the nova flavor used by some sample test cases
+        nova flavor-create yardstick-flavor 100 512 3 1
+        # DPDK-enabled OVS requires guest memory to be backed by large pages
+        if [[ "$DEPLOY_SCENARIO" == *"-ovs-"* ]]; then
+            nova flavor-key yardstick-flavor set hw:mem_page_size=large
+        fi
+    fi
+}
+
+load_cirros_image()
+{
+    echo
+    echo "========== Loading cirros cloud image =========="
+
+    local image_file=/home/opnfv/images/cirros-0.3.3-x86_64-disk.img
+
+    output=$(glance image-create \
+        --name  cirros-0.3.3 \
+        --disk-format qcow2 \
+        --container-format bare \
+        --file $image_file)
+    echo "$output"
+
+    CIRROS_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')
+    if [ -z "$CIRROS_IMAGE_ID" ]; then
+        echo 'Failed uploading cirros image to cloud'.
+        exit 1
+    fi
+
+    echo "Cirros image id: $CIRROS_IMAGE_ID"
+}
+
+load_ubuntu_image()
+{
+    echo
+    echo "========== Loading ubuntu cloud image =========="
+
+    local ubuntu_image_file=/home/opnfv/images/trusty-server-cloudimg-amd64-disk1.img
+
+    output=$(glance image-create \
+        --name Ubuntu-14.04 \
+        --disk-format qcow2 \
+        --container-format bare \
+        --file $ubuntu_image_file)
+    echo "$output"
+
+    UBUNTU_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')
+
+    if [ -z "$UBUNTU_IMAGE_ID" ]; then
+        echo 'Failed uploading UBUNTU image to cloud'.
+        exit 1
+    fi
+
+    echo "Ubuntu image id: $UBUNTU_IMAGE_ID"
+}
+
+QCOW_IMAGE="/tmp/workspace/yardstick/yardstick-trusty-server.img"
+
+build_yardstick_image()
+{
+    echo
+    echo "========== Build yardstick cloud image =========="
+
+    export YARD_IMG_ARCH="amd64"
+    sudo echo "Defaults env_keep += \"YARD_IMG_ARCH\"" >> /etc/sudoers
+    local cmd="sudo $(which yardstick-img-modify) $(pwd)/tools/ubuntu-server-cloudimg-modify.sh"
+
+    # Build the image. Retry once if the build fails.
+    $cmd || $cmd
+
+    if [ ! -f $QCOW_IMAGE ]; then
+        echo "Failed building QCOW image"
+        exit 1
+    fi
+}
+
+load_yardstick_image()
+{
+    echo
+    echo "========== Loading yardstick cloud image =========="
+
+    output=$(glance --os-image-api-version 1 image-create \
+        --name yardstick-trusty-server \
+        --is-public true --disk-format qcow2 \
+        --container-format bare \
+        --file $QCOW_IMAGE)
+    echo "$output"
+
+    GLANCE_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')
+
+    if [ -z "$GLANCE_IMAGE_ID" ]; then
+        echo 'Failed uploading image to cloud'.
+        exit 1
+    fi
+
+    sudo rm -f $QCOW_IMAGE
+
+    echo "Glance image id: $GLANCE_IMAGE_ID"
+}
+
+main()
+{
+    cleanup
+    create_nova_flavor
+    load_cirros_image
+    load_ubuntu_image
+    build_yardstick_image
+    load_yardstick_image
+}
+
+main
diff --git a/scripts/prepare_test_yard/build_run_test.sh b/scripts/prepare_test_yard/build_run_test.sh
new file mode 100755
index 00000000..91ac589e
--- /dev/null
+++ b/scripts/prepare_test_yard/build_run_test.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -e
+
+cd /home/opnfv/repos/yardstick
+
+source /tmp/yardstick/source_env.sh
+
+echo "========== Build the flavor and load images =========="
+source /tmp/yardstick/build_flavor_image.sh
+
+echo "========== Run yardstick testcase =========="
+cmd="yardstick task start /home/opnfv/repos/yardstick/tests/opnfv/test_cases/$1 \
+     --output-file $2"
+${cmd}
diff --git a/scripts/prepare_test_yard/run_test.sh b/scripts/prepare_test_yard/run_test.sh
new file mode 100755
index 00000000..326c637b
--- /dev/null
+++ b/scripts/prepare_test_yard/run_test.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -e
+
+cd /home/opnfv/repos/yardstick
+
+source /tmp/yardstick/source_env.sh
+
+echo "========== Run yardstick testcase =========="
+cmd="yardstick task start /home/opnfv/repos/yardstick/tests/opnfv/test_cases/$1 \
+     --output-file $2"
+${cmd}
diff --git a/scripts/prepare_test_yard/source_env.sh b/scripts/prepare_test_yard/source_env.sh
new file mode 100755
index 00000000..80135d70
--- /dev/null
+++ b/scripts/prepare_test_yard/source_env.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -e
+
+: ${YARDSTICK_REPO:='https://gerrit.opnfv.org/gerrit/yardstick'}
+: ${YARDSTICK_REPO_DIR:='/home/opnfv/repos/yardstick'}
+: ${YARDSTICK_BRANCH:='master'} # branch, tag, sha1 or refspec
+
+: ${RELENG_REPO:='https://gerrit.opnfv.org/gerrit/releng'}
+: ${RELENG_REPO_DIR:='/home/opnfv/repos/releng'}
+: ${RELENG_BRANCH:='master'} # branch, tag, sha1 or refspec
+
+source $YARDSTICK_REPO_DIR/tests/ci/prepare_env.sh
diff --git a/scripts/report.py b/scripts/report.py
new file mode 100644
index 00000000..516ea069
--- /dev/null
+++ b/scripts/report.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+#
+# grakiss.wanglei@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+import json
+import urllib2
+import re
+
+import utils.dovetail_logger as dt_logger
+import utils.dovetail_utils as dt_utils
+
+logger = dt_logger.Logger('report.py').getLogger()
+
+from conf.dovetail_config import *
+from testcase import *
+
+def get_pass_str(passed):
+    if passed:
+        return 'PASS'
+    else:
+        return 'FAIL'
+
+class Report:
+
+    results = {'functest':{},'yardstick':{}}
+
+    @classmethod
+    def check_result(cls, testcase, db_result):
+        checker = CheckerFactory.create(testcase.script_type())
+        checker.check(testcase, db_result)
+
+    @classmethod
+    def generate(cls, scenario_yaml):
+        report = ''
+
+        report += '\n\
++=============================================================================+\n\
+|                                   report                                    | \n\
++-----------------------------------------------------------------------------+\n'
+        report += '|scenario: %s\n' % scenario_yaml['name']
+        for testcase_name in scenario_yaml['testcase_list']:
+            testcase = Testcase.get(testcase_name)
+            report += '|   [testcase]: %s\t\t\t\t[%s]\n' % (testcase_name, get_pass_str(testcase.passed()))
+            report += '|   |-objective: %s\n' % testcase.objective()
+            if testcase.sub_testcase() is not None:
+                for subtest in testcase.sub_testcase():
+                    report += '|       |-%s \t\t [%s]\n' % (subtest, get_pass_str(testcase.sub_testcase_passed(subtest)))
+            report += '+-----------------------------------------------------------------------------+\n'
+
+        logger.info(report)
+        return report
+
+    @classmethod
+    def get_result(cls, testcase):
+        script_testcase = testcase.script_testcase()
+        type = testcase.script_type()
+
+        if script_testcase in cls.results[type]:
+            return cls.results[type][script_testcase]
+
+        if dovetail_config[type]['result']['store_type'] == 'file':
+            result = cls.get_results_from_file(type)
+        else:
+            result = cls.get_results_from_db(type, script_testcase)
+
+        if result is not None:
+            cls.results[type][script_testcase] = result
+            testcase.script_result_acquired(True)
+            logger.debug('testcase: %s -> result acquired' % script_testcase)
+        else:
+            retry = testcase.increase_retry()
+            logger.debug('testcase: %s -> result acquired retry:%d' % (script_testcase, retry))
+        return result
+
+    @classmethod
+    def get_results_from_db(cls, type, testcase):
+        #url = 'http://testresults.opnfv.org/test/api/v1/results?case=%s&last=1' % testcase
+        url = dovetail_config[type]['result']['db_url'] % testcase
+        logger.debug("Query to rest api: %s" % url)
+        try:
+            data = json.load(urllib2.urlopen(url))
+            return data['results'][0]
+        except Exception as e:
+            logger.error("Cannot read content from the url: %s, exception: %s" % (url, e))
+            return None
+
+    @classmethod
+    def get_results_from_file(cls, type, testcase=None):
+        file_path = os.path.join(dovetail_config['result_dir'],dovetail_config[type]['result']['file_path'])
+        if not os.path.exists(file_path):
+            logger.info('result file not found: %s' % file_path)
+            return None
+
+        try:
+            with open(file_path, 'r') as myfile:
+                output = myfile.read()
+            error_logs = ""
+
+            for match in re.findall('(.*?)[. ]*FAILED', output):
+                error_logs += match
+
+            criteria = 'PASS'
+            failed_num = int(re.findall(' - Failed: (\d*)', output)[0])
+            if failed_num != 0:
+                criteria = 'FAIL'
+
+            match =  re.findall('Ran: (\d*) tests in (\d*)\.\d* sec.', output)
+            num_tests, dur_sec_int = match[0]
+            json_results = {'criteria':criteria,'details':{"timestart": '', "duration": int(dur_sec_int),
+                            "tests": int(num_tests), "failures": failed_num,
+                            "errors": error_logs}}
+            logger.debug('Results: %s' % str(json_results))
+            return json_results
+        except Exception as e:
+            logger.error('Cannot read content from the file: %s, exception: %s' % (file_path, e))
+            return None
+
+class CheckerFactory:
+
+    @classmethod
+    def create(cls,type):
+        if type == 'functest':
+            return FunctestChecker()
+
+        if type == 'yardstick':
+            return YardstickChecker()
+
+        return None
+
+class ResultChecker:
+
+    def check(cls):
+        return 'PASS'
+
+class FunctestChecker:
+
+    def check(cls, testcase, db_result):
+        if not db_result:
+            for sub_testcase in testcase.sub_testcase():
+                testcase.sub_testcase_passed(sub_testcase,False)
+            return
+
+        testcase.passed(db_result['criteria'] == 'PASS')
+
+        if testcase.sub_testcase() is None:
+            return
+
+        if testcase.testcase['passed'] == True:
+            for sub_testcase in testcase.sub_testcase():
+                testcase.sub_testcase_passed(sub_testcase, True)
+            return
+
+        all_passed = True
+        for sub_testcase in testcase.sub_testcase():
+            logger.debug('check sub_testcase:%s' % sub_testcase)
+            if sub_testcase in db_result['details']['errors']:
+                testcase.sub_testcase_passed(sub_testcase, False)
+                all_passed = False
+            else:
+                testcase.sub_testcase_passed(sub_testcase, True)
+
+        testcase.passed(all_passed)
+
+class YardstickChecker:
+
+    def check(cls, testcase, result):
+        return 'PASS'
+
+
diff --git a/scripts/run.py b/scripts/run.py
new file mode 100755
index 00000000..a8ef819c
--- /dev/null
+++ b/scripts/run.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+#
+# grakiss.wanglei@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+
+import click
+import yaml
+import os
+import time
+
+import utils.dovetail_logger as dt_logger
+import utils.dovetail_utils as dt_utils
+logger = dt_logger.Logger('run.py').getLogger()
+
+from container import Container
+from testcase import *
+from report import *
+from conf.dovetail_config import *
+
+def load_scenario(scenario):
+    Scenario.load()
+    return Scenario.get(SCENARIO_NAMING_FMT % scenario)
+
+def load_testcase():
+    Testcase.load()
+
+def copy_file(file_dir, container_id, container_dir):
+    for root, dirs, files in os.walk(file_dir):
+        for file_name in files:
+            cmd = 'sudo docker cp %s %s:%s' % (os.path.join(file_dir,file_name), container_id, container_dir)
+            dt_utils.exec_cmd(cmd, logger, exit_on_error = False)
+
+def run_functest(testcase, container_id):
+    sub_cmd = dovetail_config[testcase.script_type()]['testcase']['pre_cmd']
+    Container.exec_cmd(container_id, sub_cmd)
+    sub_cmd = dovetail_config[testcase.script_type()]['testcase']['exec_cmd'] % testcase.script_testcase()
+    Container.exec_cmd(container_id, sub_cmd)
+
+def run_yardstick(testcase, container_id):
+    copy_file(os.path.join(os.getcwd(),container_config[testcase.script_type()]['shell_dir_name']),\
+        container_id,container_config[testcase.script_type()]['shell_dir'])
+    if container_config[testcase.script_type()]['has_build_images'] == True:
+        cmd = 'sudo docker exec %s %s/run_test.sh %s.yaml %s/%s.out' \
+            % (container_id, container_config[testcase.script_type()]['shell_dir'], testcase.script_testcase(),\
+              container_config[testcase.script_type()]['result_dir'], testcase.name())
+    else:
+        container_config[testcase.script_type()]['has_build_images'] = True
+        cmd = 'sudo docker exec %s %s/build_run_test.sh %s.yaml %s/%s.out' \
+            % (container_id, container_config[testcase.script_type()]['shell_dir'], testcase.script_testcase(),\
+              container_config[testcase.script_type()]['result_dir'], testcase.name())
+    dt_utils.exec_cmd(cmd,logger,exit_on_error = False)
+    time.sleep(5)
+
+def run_test(scenario):
+    for testcase_name in scenario['testcase_list']:
+        logger.info('>>[testcase]: %s' % (testcase_name))
+        testcase = Testcase.get(testcase_name)
+        run_test = True
+
+        if testcase.exceed_max_retry_times():
+            run_test = False
+
+        if testcase.script_result_acquired():
+            run_test = False
+
+        if run_test:
+            Container.pull_image(testcase.script_type())
+            container_id = Container.create(testcase.script_type())
+            logger.debug('container id:%s' % container_id)
+
+            if testcase.script_type() == 'functest':
+                run_functest(testcase, container_id)
+            else:
+                run_yardstick(testcase, container_id)
+
+            Container.clean(container_id)
+
+        db_result = Report.get_result(testcase)
+        Report.check_result(testcase, db_result)
+
+@click.command()
+@click.option('--scenario', default='basic', help='certification scenario')
+def main(scenario):
+    """Dovetail certification test entry!"""
+    logger.info('=======================================')
+    logger.info('Dovetail certification: %s!' % scenario)
+    logger.info('=======================================')
+    load_testcase()
+    scenario_yaml = load_scenario(scenario)
+    run_test(scenario_yaml)
+    Report.generate(scenario_yaml)
+
+if __name__ == '__main__':
+    main()
diff --git a/scripts/testcase.py b/scripts/testcase.py
new file mode 100644
index 00000000..71ffd7ac
--- /dev/null
+++ b/scripts/testcase.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+#
+# grakiss.wanglei@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+import utils.dovetail_logger as dt_logger
+import utils.dovetail_utils as dt_utils
+
+logger = dt_logger.Logger('testcase.py').getLogger()
+
+from conf.dovetail_config import *
+
+class Testcase:
+
+    def __init__(self, testcase_yaml):
+        self.testcase = testcase_yaml.values()[0]
+        self.testcase['passed'] = False
+        self.sub_testcase_status = {}
+        Testcase.update_script_testcase(self.script_type(), self.script_testcase())
+
+    def __str__(self):
+        return self.testcase
+
+    def name(self):
+        return self.testcase['name']
+
+    def objective(self):
+        return self.testcase['objective']
+
+    def sub_testcase(self):
+        return self.testcase['scripts']['sub_testcase_list']
+
+    def sub_testcase_passed(self, name, passed=None):
+        if passed is not None:
+            logger.debug('sub_testcase_passed:%s %s' % (name,  passed))
+            self.sub_testcase_status[name] = passed
+        return self.sub_testcase_status[name]
+
+    def script_type(self):
+        return self.testcase['scripts']['type']
+
+    def script_testcase(self):
+        return self.testcase['scripts']['testcase']
+
+    def exceed_max_retry_times(self):
+        #logger.debug('retry times:%d' % self.testcase['retry'])
+        return Testcase._exceed_max_retry_times(self.script_type(), self.script_testcase())
+
+    def increase_retry(self):
+        #self.testcase['retry'] = self.testcase['retry'] + 1
+        #return self.testcase['retry']
+        return Testcase._increase_retry(self.script_type(), self.script_testcase())
+
+    def passed(self, passed = None):
+        if passed is not None:
+            self.testcase['passed'] = passed
+        return self.testcase['passed']
+
+    def script_result_acquired(self, acquired=None):
+        return Testcase._result_acquired(self.script_type(), self.script_testcase(), acquired)
+
+    #testcase in upstream testing project
+    script_testcase_list = {'functest':{}, 'yardstick':{}}
+
+    #testcase in dovetail
+    testcase_list = {}
+
+    @classmethod
+    def update_script_testcase(cls,script_type, script_testcase):
+        if script_testcase not in cls.script_testcase_list[script_type]:
+            cls.script_testcase_list[script_type][script_testcase] = {'retry':0, 'acquired':False}
+
+    @classmethod
+    def _exceed_max_retry_times(cls, script_type, script_testcase ):
+        return cls.script_testcase_list[script_type][script_testcase]['retry'] > 1
+
+    @classmethod
+    def _increase_retry(cls, script_type, script_testcase):
+        cls.script_testcase_list[script_type][script_testcase]['retry'] += 1
+        return cls.script_testcase_list[script_type][script_testcase]['retry']
+
+    @classmethod
+    def _result_acquired(cls, script_type, script_testcase, acquired=None):
+        if acquired is not None:
+            cls.script_testcase_list[script_type][script_testcase]['acquired'] = acquired
+        return cls.script_testcase_list[script_type][script_testcase]['acquired']
+
+    @classmethod
+    def load(cls):
+        for root, dirs, files in os.walk(TESTCASE_PATH):
+            for testcase_file in files:
+                with open(os.path.join(root, testcase_file)) as f:
+                    testcase_yaml = yaml.safe_load(f)
+                    cls.testcase_list[testcase_yaml.keys()[0]] = Testcase(testcase_yaml)
+        logger.debug( cls.testcase_list )
+
+    @classmethod
+    def get(cls, testcase_name):
+        if testcase_name in cls.testcase_list:
+            return cls.testcase_list[testcase_name]
+        return None
+
+
+class Scenario:
+
+    def __init__(self, scenario):
+        self.scenario = scenario
+        self.testcase_list = {}
+
+    def get_test(self, testcase_name):
+        if testcase_name in self.testcase_list:
+            return self.testcase_list[testcase_name]
+        return None
+
+    scenario_list = {}
+    @classmethod
+    def load(cls):
+        for root, dirs, files in os.walk(CERT_PATH):
+            for scenario_yaml in files:
+                with open(os.path.join(root, scenario_yaml)) as f:
+                    scenario_yaml = yaml.safe_load(f)
+                    cls.scenario_list.update(scenario_yaml)
+
+        logger.debug(cls.scenario_list)
+
+    @classmethod
+    def get(cls, scenario_name):
+        if scenario_name in cls.scenario_list:
+            return cls.scenario_list[scenario_name]
+        return None
+
diff --git a/scripts/testcase/ipv6.tc001.yml b/scripts/testcase/ipv6.tc001.yml
new file mode 100644
index 00000000..9f11ac76
--- /dev/null
+++ b/scripts/testcase/ipv6.tc001.yml
@@ -0,0 +1,10 @@
+dovetail.ipv6.tc001:
+  name: dovetail.ipv6.tc001
+  objective: VIM ipv6 operations, to create/delete network, port and subnet in bulk operation
+  scripts:
+    type: functest
+    testcase: tempest_smoke_serial
+    sub_testcase_list:
+      - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_network
+      - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_port
+      - tempest.api.network.test_networks.BulkNetworkOpsIpV6Test.test_bulk_create_delete_subnet
diff --git a/scripts/utils/__init__.py b/scripts/utils/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/scripts/utils/__init__.py
diff --git a/scripts/utils/dovetail_logger.py b/scripts/utils/dovetail_logger.py
new file mode 100644
index 00000000..7335c0ae
--- /dev/null
+++ b/scripts/utils/dovetail_logger.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+#
+# jose.lausuch@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Logging levels:
+#  Level     Numeric value
+#  CRITICAL  50
+#  ERROR     40
+#  WARNING   30
+#  INFO      20
+#  DEBUG     10
+#  NOTSET    0
+#
+# Usage:
+#  import dovetail_logger as dl
+#  logger = dl.Logger("script_name").getLogger()
+#  logger.info("message to be shown with - INFO - ")
+#  logger.debug("message to be shown with - DEBUG -")
+
+import logging
+import os
+
+class Logger:
+    def __init__(self, logger_name):
+
+        CI_DEBUG = os.getenv('CI_DEBUG')
+
+        self.logger = logging.getLogger(logger_name)
+        self.logger.propagate = 0 
+        self.logger.setLevel(logging.DEBUG)
+
+        ch = logging.StreamHandler()
+        formatter = logging.Formatter('%(asctime)s - %(name)s - '
+                                      '%(levelname)s - %(message)s')
+        ch.setFormatter(formatter)
+        if CI_DEBUG is not None and CI_DEBUG.lower() == "true":
+            ch.setLevel(logging.DEBUG)
+        else:
+            ch.setLevel(logging.INFO)
+        self.logger.addHandler(ch)
+
+        if not os.path.exists('/home/opnfv/dovetail/results/'):
+            os.makedirs('/home/opnfv/dovetail/results/')
+        hdlr = logging.FileHandler('/home/opnfv/dovetail/results/dovetail.log')
+        hdlr.setFormatter(formatter)
+        hdlr.setLevel(logging.DEBUG)
+        self.logger.addHandler(hdlr)
+
+    def getLogger(self):
+        return self.logger
+
diff --git a/scripts/utils/dovetail_utils.py b/scripts/utils/dovetail_utils.py
new file mode 100644
index 00000000..f79c6fc4
--- /dev/null
+++ b/scripts/utils/dovetail_utils.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+#
+# jose.lausuch@ericsson.com
+# valentin.boucher@orange.com
+# grakiss.wanglei@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+
+import sys
+import subprocess
+
+def exec_cmd(cmd, logger=None,
+                    exit_on_error=True,
+                    info=False,
+                    error_msg="",
+                    verbose=True):
+    if not error_msg:
+        error_msg = ("The command '%s' failed." % cmd)
+    msg_exec = ("Executing command: '%s'" % cmd)
+    if verbose:
+        if logger:
+            if info:
+                logger.info(msg_exec)
+            else:
+                logger.debug(msg_exec)
+        else:
+            print(msg_exec)
+    p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+                         stderr=subprocess.STDOUT)
+    output = p.communicate()
+    for line in output[0].strip().split('\n'):
+        line = line.replace('\n', '') 
+        if logger:
+            if info:
+                logger.info(line)
+            else:
+                logger.debug(line)
+        else:
+            print line
+            sys.stdout.flush()
+
+    returncode = p.returncode
+    if returncode != 0:
+        if verbose:
+            if logger:
+                logger.error(error_msg)
+            else:
+                print(error_msg)
+        if exit_on_error:
+            sys.exit(1)
+
+    return returncode, output[0].strip()
+