path: root/tests/ci/yardstick-verify

#!/bin/bash
##############################################################################
# Copyright (c) 2015 Ericsson AB and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################

#
# Set up the environment and run yardstick test suites.
#
# Example invocation: yardstick-verify -r 10.4.4.4 suite1.yaml suite2.yaml
#
# Openstack credentials must be set and the script must be run from its
# original location in the yardstick repo.
#
# This script is intended to be used by the CI pipeline but it may also
# be invoked manually.
#

SCRIPT=$0
SCRIPT_ARGS=$@

usage()
{
    cat << EOF
usage: $0 options [TEST_SUITE ...]

If no test suites are given ping.yaml is run.

OPTIONS:
   -h      Show this message
   -r      Http target (example: -r 213.77.62.197/results)
   -i      Influxdb target (example: -i 127.0.0.1:8086)

           Default target is dump to file ($DISPATCHER_FILE_NAME)

EOF
}

DISPATCHER_TYPE=file
DISPATCHER_FILE_NAME="/tmp/yardstick.out"
DISPATCHER_HTTP_TARGET="http://testresults.opnfv.org/test/api/v1/results"
DISPATCHER_INFLUXDB_TARGET=
UCA_HOST="cloud-images.ubuntu.com"
if [ $YARD_IMG_ARCH = "arm64" ]; then
    export VIVID_IMG_URL="http://${UCA_HOST}/vivid/current/vivid-server-cloudimg-arm64.tar.gz"
    if ! grep -q "Defaults env_keep += \"VIVID_IMG_URL\"" "/etc/sudoers"; then
        sudo echo "Defaults env_keep += \"VIVID_IMG_URL\"" >> /etc/sudoers
    fi
fi
while getopts "r:i:h" OPTION; do
    case $OPTION in
        h)
            usage
            exit 0
            ;;
        r)
            DISPATCHER_TYPE=http
            DISPATCHER_HTTP_TARGET=http://${OPTARG}
            DISPATCHER_FILE_NAME=
            ;;
        i)
            DISPATCHER_TYPE=influxdb
            DISPATCHER_INFLUXDB_TARGET=http://${OPTARG}
            DISPATCHER_FILE_NAME=
            ;;
        *)
            echo "${OPTION} is not a valid argument"
            exit 1
            ;;
    esac
done

shift $[OPTIND - 1]
TEST_SUITES=$@

cleanup()
{
    echo
    echo "========== Cleanup =========="

    if ! glance image-list; then
        return
    fi

    for image in $(glance image-list | grep -e cirros-0.3.3 -e yardstick-trusty-server -e Ubuntu-14.04 \
        -e yardstick-vivid-kernel | awk '{print $2}'); do
        echo "Deleting image $image..."
        glance image-delete $image || true
    done

    nova flavor-delete yardstick-flavor &> /dev/null || true
}

exitcode=""

error_exit()
{
    local rc=$?

    if [ -z "$exitcode" ]; then
        # In case of recursive traps (!?)
        exitcode=$rc
    fi

    cleanup

    echo "Exiting with RC=$exitcode"

    exit $exitcode
}

set -o errexit
set -o pipefail

install_yardstick()
{
    echo
    echo "========== Installing yardstick =========="

    # uninstall previous version
    pip uninstall -y yardstick || true

    # Install yardstick
    pip install .
}

install_storperf()
{
    # Install Storper on huawei-pod1
    if [ "$NODE_NAME" == "huawei-pod1" ]; then
        echo
        echo "========== Installing storperf =========="

        if ! yardstick -d plugin install plugin/CI/storperf.yaml; then
            echo "Install storperf plugin FAILED";
            exit 1
        fi

    fi
}

remove_storperf()
{
    # remove Storper from huawei-pod1
    if [ "$NODE_NAME" == "huawei-pod1" ]; then
        echo
        echo "========== Removing storperf =========="

        if ! yardstick -d plugin remove plugin/CI/storperf.yaml; then
            echo "Remove storperf plugin FAILED";
            exit 1
        fi

    fi
}

build_yardstick_image()
{
    echo
    echo "========== Build yardstick cloud image =========="

    local cmd="sudo $(which yardstick-img-modify) $(pwd)/tools/ubuntu-server-cloudimg-modify.sh"

    # Build the image. Retry once if the build fails.
    $cmd || $cmd

    if [ ! -f $QCOW_IMAGE ]; then
        echo "Failed building QCOW image"
        exit 1
    fi
}

create_nova_flavor()
{
    if ! nova flavor-list | grep -q yardstick-flavor; then
        echo
        echo "========== Create nova flavor =========="
        # Create the nova flavor used by some sample test cases
        nova flavor-create yardstick-flavor 100 512 3 1
        # DPDK-enabled OVS requires guest memory to be backed by large pages
        if [[ "$DEPLOY_SCENARIO" == *"-ovs-"* ]]; then
            nova flavor-key yardstick-flavor set hw:mem_page_size=large
        fi
        # VPP requires guest memory to be backed by large pages
        if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then
            nova flavor-key yardstick-flavor set hw:mem_page_size=large
        fi
    fi
}

load_cirros_image()
{
    echo
    echo "========== Loading cirros cloud image =========="

    local image_file=/home/opnfv/images/cirros-0.3.3-x86_64-disk.img

    EXTRA_PARAMS=""
    # VPP requires guest memory to be backed by large pages
    if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then
        EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
    fi

    output=$(glance image-create \
        --name  cirros-0.3.3 \
        --disk-format qcow2 \
        --container-format bare \
        $EXTRA_PARAMS \
        --file $image_file)
    echo "$output"

    CIRROS_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')
    if [ -z "$CIRROS_IMAGE_ID" ]; then
        echo 'Failed uploading cirros image to cloud'.
        exit 1
    fi

    echo "Cirros image id: $CIRROS_IMAGE_ID"
}

load_ubuntu_image()
{
    echo
    echo "========== Loading ubuntu cloud image =========="

    local ubuntu_image_file=/home/opnfv/images/trusty-server-cloudimg-amd64-disk1.img

    EXTRA_PARAMS=""
    # VPP requires guest memory to be backed by large pages
    if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then
        EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
    fi

    output=$(glance image-create \
        --name Ubuntu-14.04 \
        --disk-format qcow2 \
        --container-format bare \
        $EXTRA_PARAMS \
        --file $ubuntu_image_file)
    echo "$output"

    UBUNTU_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')

    if [ -z "$UBUNTU_IMAGE_ID" ]; then
        echo 'Failed uploading UBUNTU image to cloud'.
        exit 1
    fi

    echo "Ubuntu image id: $UBUNTU_IMAGE_ID"
}

load_yardstick_image()
{
    echo
    echo "========== Loading yardstick cloud image =========="
    EXTRA_PARAMS=""
    if [ $YARD_IMG_ARCH = "arm64" ]; then
        VIVID_IMAGE="/tmp/vivid-server-cloudimg-arm64.tar.gz"
        VIVID_KERNEL="/tmp/vivid-server-cloudimg-arm64-vmlinuz-generic"
        cd /tmp
        if [ ! -f $VIVID_IMAGE ]; then
            wget $VIVID_IMG_URL
        fi
        if [ ! -f $VIVID_KERNEL ]; then
            tar zxf $VIVID_IMAGE $(basename $VIVID_KERNEL)
        fi
        create_vivid_kernel=$(glance --os-image-api-version 1 image-create \
                --name yardstick-vivid-kernel \
                --is-public true --disk-format qcow2 \
                --container-format bare \
                --file $VIVID_KERNEL)

        GLANCE_KERNEL_ID=$(echo "$create_vivid_kernel" | grep " id " | awk '{print $(NF-1)}')
        if [ -z "$GLANCE_KERNEL_ID" ]; then
            echo 'Failed uploading kernel to cloud'.
            exit 1
        fi

        command_line="root=/dev/vdb1 console=tty0 console=ttyS0 console=ttyAMA0 rw"

        EXTRA_PARAMS="--property kernel_id=$GLANCE_KERNEL_ID --property os_command_line=\"$command_line\""

        rm -f $VIVID_KERNEL $VIVID_IMAGE
        cd $YARDSTICK_REPO_DIR
    fi

    if [ "$DEPLOY_SCENARIO" == "os-nosdn-lxd-ha" -o "$DEPLOY_SCENARIO" == "os-nosdn-lxd-noha" ]; then
        DISK_FORMAT="raw"
    else
        DISK_FORMAT="qcow2"
    fi

    # VPP requires guest memory to be backed by large pages
    if [[ "$DEPLOY_SCENARIO" == *"-fdio-"* ]]; then
        EXTRA_PARAMS=$EXTRA_PARAMS" --property hw_mem_page_size=large"
    fi

    output=$(eval glance --os-image-api-version 1 image-create \
        --name yardstick-trusty-server \
        --is-public true --disk-format $DISK_FORMAT \
        --container-format bare \
        $EXTRA_PARAMS \
        --file $QCOW_IMAGE)

    echo "$output"

    GLANCE_IMAGE_ID=$(echo "$output" | grep " id " | awk '{print $(NF-1)}')

    if [ -z "$GLANCE_IMAGE_ID" ]; then
        echo 'Failed uploading image to cloud'.
        exit 1
    fi

    sudo rm -f $QCOW_IMAGE

    echo "Glance image id: $GLANCE_IMAGE_ID"
}

report(){

    echo
    echo "========== Reporting Status =========="
    curl -i -H 'content-type: application/json' -X POST -d \
        "{\"project_name\": \"yardstick\",
          \"case_name\": \"scenario_status\",
          \"pod_name\":\"${NODE_NAME}\",
          \"installer\":\"${INSTALLER_TYPE}\",
          \"version\":\"$(basename ${YARDSTICK_BRANCH})\",
          \"scenario\":\"${DEPLOY_SCENARIO}\",
          \"description\": \"yardstick ci scenario status\",
          \"criteria\":\"$1\",
          \"start_date\":\"$2\",
          \"stop_date\":\"$3\",
          \"details\":\"\"}" \
          ${DISPATCHER_HTTP_TARGET}
}

run_test()
{
    echo
    echo "========== Running yardstick test suites =========="

    mkdir -p /etc/yardstick

    cat << EOF >> /etc/yardstick/yardstick.conf
[DEFAULT]
debug = True
dispatcher = ${DISPATCHER_TYPE}

[dispatcher_file]
file_name = ${DISPATCHER_FILE_NAME}

[dispatcher_http]
timeout = 5
target = ${DISPATCHER_HTTP_TARGET}

[dispatcher_influxdb]
timeout = 5
target = ${DISPATCHER_INFLUXDB_TARGET}
db_name = yardstick
username = opnfv
password = 0pnfv2015
EOF

    local failed=0
    local start_date
    local stop_date

    if [ ${#SUITE_FILES[@]} -gt 0 ]; then

        start_date=$(date '+%Y-%m-%d %H:%M:%S')
        for suite in ${SUITE_FILES[*]}; do

            echo "---------------------------"
            echo "Running test suite: $suite"
            echo "---------------------------"
            if ! yardstick task start --suite $suite; then
                 echo "test suite $suite FAILED";

                # Mark the test suite failed but continue
                # running the remaining test suites.
                (( ++failed ))
            fi
            if [ ${DISPATCHER_TYPE} = file ]; then
                echo "---------------------------"
                echo "Dump test suite $suite result"
                echo "---------------------------"
                if [ -f ${DISPATCHER_FILE_NAME} ]; then
                    cat ${DISPATCHER_FILE_NAME}
                else
                    echo "Test result file ${DISPATCHER_FILE_NAME} is not exist"
                fi
            fi
        done
        stop_date=$(date '+%Y-%m-%d %H:%M:%S')



        local scenario_status="SUCCESS"

        if [ $failed -gt 0 ]; then
            scenario_status="FAILED"
        fi

        report $scenario_status $start_date $stop_date

        if [ $failed -gt 0 ]; then
            echo "---------------------------"
            echo "$failed out of ${SUITE_FILES[*]} test suites FAILED"
            echo "---------------------------"
            exit 1
        fi

    else

        echo "---------------------------"
        echo "Running samples/ping.yaml  "
        echo "---------------------------"

        if ! yardstick task start samples/ping.yaml; then
            echo "Yardstick test FAILED"
            exit 1
        fi

        if [ ${DISPATCHER_TYPE} = file ]; then
            echo "---------------------------"
            echo "Dump samples/ping.yaml test result"
            echo "---------------------------"
            if [ -f ${DISPATCHER_FILE_NAME} ]; then
                cat ${DISPATCHER_FILE_NAME}
            else
                echo "Test result file ${DISPATCHER_FILE_NAME} is not exist"
            fi
        fi

    fi

}

main()
{
    GITROOT=$(cd $(dirname $0) && git rev-parse --show-toplevel)

    cd $GITROOT

    export YARDSTICK_VERSION=$(git rev-parse HEAD)

    SUITE_FILES=()

    # find the test suite files
    for suite in $TEST_SUITES; do
        if [ -f $suite ]; then
            SUITE_FILES+=($suite)
        else
            tsdir=$GITROOT/tests/opnfv/test_suites
            if [ ! -f $tsdir/$suite ]; then
                echo "Test suite \"$suite\" does not exist"
                exit 1
            fi
            SUITE_FILES+=($tsdir/$suite)
        fi
    done

    echo
    echo "========== Running Yardstick CI with following parameters =========="
    echo "Script options: ${SCRIPT} $SCRIPT_ARGS"
    echo "Dispatcher: ${DISPATCHER_TYPE} ${DISPATCHER_FILE_NAME}"
    echo "YARDSTICK_VERSION: ${YARDSTICK_VERSION}"
    echo "Number of test suites: ${#SUITE_FILES[@]}"
    for suite in ${SUITE_FILES[*]}; do
        echo "     $suite"
    done
    echo

    # check if some necessary variables is set
    if [ -z "$OS_AUTH_URL" ]; then
        echo "OS_AUTH_URL is unset or empty"
        exit 1
    fi

    echo "OS_AUTH_URL is $OS_AUTH_URL"
    echo

    # check OpenStack services
    echo "Checking OpenStack services:"
    for cmd in "glance image-list" "nova list" "heat stack-list"; do
        echo "  checking ${cmd/%\ */} ..."
        if ! $cmd >/dev/null; then
            echo "error: command \"$cmd\" failed"
            exit 1
        fi
    done

    echo
    echo "Checking for External network:"
    for net in $(neutron net-list --router:external True -c name -f value); do
        echo "  external network: $net"
    done

    # install yardstick
    install_yardstick

    cleanup

    trap "error_exit" EXIT SIGTERM

    QCOW_IMAGE="/tmp/workspace/yardstick/yardstick-trusty-server.img"

    build_yardstick_image
    load_yardstick_image
    if [ $YARD_IMG_ARCH = "arm64" ]; then
        sed -i 's/image: cirros-0.3.3/image: TestVM/g' tests/opnfv/test_cases/opnfv_yardstick_tc002.yaml \
        samples/ping.yaml
        #We have overlapping IP with the real network
        for filename in tests/opnfv/test_cases/*; do
            sed -i "s/cidr: '10.0.1.0\/24'/cidr: '10.3.1.0\/24'/g" $filename
        done
    else
        load_cirros_image
        load_ubuntu_image
    fi
    create_nova_flavor
    install_storperf
    run_test
    remove_storperf
}

main