ci: Enhance verify and merge jobs

Verify and merge jobs will execute real vsperf TCs,
which won't require neither physical NICs nor traffic
generator. Script ./build-vsperf.sh will take care about
creation, update and activation of python virtual environment.
It will also install required system packages at nodes
running merge and verify jobs.
Additional improvements:
* set of simple sanity checks was added
* two new integration TCs were added; However test with
  VNF is not used now, because it requires a VM image.
* detailed logs are shown in case of failure

JIRA: VSPERF-391

Change-Id: I33e75e762fdc62656de92dc34f0ad7a484623603
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Bill Michalowski <bmichalo@redhat.com>
Reviewed-by: Fatih Degirmenci <fatih.degirmenci@ericsson.com>

1 files changed, 109 insertions, 22 deletions

diff --git a/ci/build-vsperf.sh b/ci/build-vsperf.sh
index a02de009..637bc474 100755
--- a/ci/build-vsperf.sh
+++ b/ci/build-vsperf.sh
@@ -26,6 +26,7 @@
 
 EXIT=0
 EXIT_TC_FAILED=1
+EXIT_SANITY_TC_FAILED=2
 EXIT_NO_RESULTS=10
 EXIT_NO_TEST_REPORT_LOG_DIR=11
 
@@ -37,14 +38,15 @@ VSPERF_BIN='./vsperf'
 LOG_FILE_PREFIX="/tmp/vsperf_build"
 DATE=$(date -u +"%Y-%m-%d_%H-%M-%S")
 BRANCH=${GIT_BRANCH##*/}
+VSPERFENV_DIR="$HOME/vsperfenv"
 
 # CI job specific configuration
 # VERIFY - run basic set of TCs with default settings
-TESTCASES_VERIFY="phy2phy_tput pvp_tput"
-TESTPARAM_VERIFY=""
+TESTCASES_VERIFY="vswitch_add_del_bridge vswitch_add_del_bridges vswitch_add_del_vport vswitch_add_del_vports vswitch_vports_add_del_flow"
+TESTPARAM_VERIFY="--integration"
 # MERGE - run selected TCs with default settings
-TESTCASES_MERGE="phy2phy_tput back2back phy2phy_cont pvp_tput pvvp_tput"
-TESTPARAM_MERGE=""
+TESTCASES_MERGE="vswitch_add_del_bridge vswitch_add_del_bridges vswitch_add_del_vport vswitch_add_del_vports vswitch_vports_add_del_flow"
+TESTPARAM_MERGE="--integration"
 # DAILY - run selected TCs for defined packet sizes
 TESTCASES_DAILY='phy2phy_tput back2back phy2phy_tput_mod_vlan phy2phy_scalability pvp_tput pvp_back2back pvvp_tput pvvp_back2back'
 TESTPARAM_DAILY='--test-params pkt_sizes=64,128,512,1024,1518'
@@ -109,19 +111,17 @@ function terminate_vsperf() {
 #   $1 - directory with results
 function print_results() {
     for i in $TESTCASES ; do
-        if [[ $i == *"pvp"* ]]; then
-            DEPLOYMENT="pvp"
-        elif [[ $i == *"pvvp"* ]]; then
-            DEPLOYMENT="pvvp"
-        else
-            DEPLOYMENT="p2p"
-        fi
-        RES_FILE="result_${i}_${DEPLOYMENT}.csv"
+        RES_FILE=`ls -1 $1 | egrep "result_${i}_[0-9a-zA-Z\-]+.csv"`
 
-        if [ -e "${1}/${RES_FILE}" ]; then
-            printf "    %-70s %-6s\n" $RES_FILE "OK"
+        if [ "x$RES_FILE" != "x" -a -e "${1}/${RES_FILE}" ]; then
+            if grep ^FAILED "${1}/${RES_FILE}" &> /dev/null ; then
+                printf "    %-70s %-6s\n" "result_${i}" "FAILED"
+                EXIT=$EXIT_TC_FAILED
+            else
+                printf "    %-70s %-6s\n" "result_${i}" "OK"
+            fi
         else
-            printf "    %-70s %-6s\n" $RES_FILE "FAILED"
+            printf "    %-70s %-6s\n" "result_${i}" "FAILED"
             EXIT=$EXIT_TC_FAILED
         fi
     done
@@ -132,6 +132,7 @@ function print_results() {
 #   $1 - vswitch and vnf combination, one of OVS_vanilla, OVS_with_DPDK_and_vHost_User
 #   $2 - CI job type, one of verify, merge, daily
 function execute_vsperf() {
+    OPNFVPOD=""
     # figure out list of TCs and execution parameters
     case $2 in
         "verify")
@@ -143,9 +144,10 @@ function execute_vsperf() {
             TESTCASES=$TESTCASES_MERGE
             ;;
         *)
-            # by default use daily build
+            # by default use daily build and upload results to the OPNFV databse
             TESTPARAM=$TESTPARAM_DAILY
             TESTCASES=$TESTCASES_DAILY
+            OPNFVPOD="--opnfvpod=$NODE_NAME"
             ;;
     esac
 
@@ -160,16 +162,16 @@ function execute_vsperf() {
             LOG_SUBDIR="OvsVanilla"
             LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
 
-            echo "$VSPERF_BIN --opnfvpod="$NODE_NAME" --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE"
-            $VSPERF_BIN --opnfvpod="$NODE_NAME" --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE
+            echo "    $VSPERF_BIN $OPNFVPOD --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE"
+            $VSPERF_BIN $OPNFVPOD --vswitch OvsVanilla --vnf QemuVirtioNet $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE
             ;;
         *)
             # figure out log file name
             LOG_SUBDIR="OvsDpdkVhost"
             LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
 
-            echo "$VSPERF_BIN --opnfvpod="$NODE_NAME" --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES > $LOG_FILE"
-            $VSPERF_BIN --opnfvpod="$NODE_NAME" --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE
+            echo "    $VSPERF_BIN $OPNFVPOD --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES > $LOG_FILE"
+            $VSPERF_BIN $OPNFVPOD --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES &> $LOG_FILE
             ;;
     esac
 
@@ -178,9 +180,17 @@ function execute_vsperf() {
     RES_DIR="/$(grep "Creating result directory" $LOG_FILE | cut -d'/' -f2-)"
     if [[ "/" == "${RES_DIR}" ]] ; then
         echo "FAILURE: Results are not available."
+        echo "-------------------------------------------------------------------"
+        cat $LOG_FILE
+        echo "-------------------------------------------------------------------"
         exit $EXIT_NO_RESULTS
     else
         print_results "${RES_DIR}"
+        if [ "$EXIT" -eq "$EXIT_TC_FAILED" ] ; then
+            echo "-------------------------------------------------------------------"
+            cat $LOG_FILE
+            echo "-------------------------------------------------------------------"
+        fi
     fi
 
     # show detailed result figures
@@ -277,35 +287,112 @@ function initialize_logdir() {
     fi
 }
 
+# verify basic vsperf functionality
+function execute_vsperf_sanity() {
+    DATE_SUFFIX=$(date -u +"%Y-%m-%d_%H-%M-%S")
+    LOG_FILE="${LOG_FILE_PREFIX}_sanity_${DATE_SUFFIX}.log"
+    echo "Execution of VSPERF sanity checks:"
+    for PARAM in '--version' '--help' '--list-trafficgens' '--list-collectors' '--list-vswitches' '--list-fwdapps' '--list-vnfs' '--list-settings' '--list' '--integration --list'; do
+        echo -e "-------------------------------------------------------------------" >> $LOG_FILE
+        echo "$VSPERF_BIN $PARAM $CONF_FILE" >> $LOG_FILE
+        echo -e "-------------------------------------------------------------------" >> $LOG_FILE
+        $VSPERF_BIN $PARAM $CONF_FILE &>> $LOG_FILE
+        if $VSPERF_BIN $PARAM $CONF_FILE &>> $LOG_FILE ; then
+            printf "    %-70s %-6s\n" "$VSPERF_BIN $PARAM" "OK"
+        else
+            printf "    %-70s %-6s\n" "$VSPERF_BIN $PARAM" "FAILED"
+            EXIT=$EXIT_SANITY_TC_FAILED
+        fi
+        echo >> $LOG_FILE
+    done
+    echo "Sanity log file $LOG_FILE"
+    if [ "$EXIT" -ne "0" ] ; then
+        echo "-------------------------------------------------------------------"
+        cat $LOG_FILE
+        echo "-------------------------------------------------------------------"
+    fi
+}
+
+# check and install required packages at nodes running VERIFY and MERGE jobs
+function dependencies_check() {
+    . /etc/os-release
+    if [ $ID == "ubuntu" ] ; then
+        echo "Dependencies check"
+        echo "=================="
+        for PACKAGE in "python3-tk" ; do
+            if dpkg -s $PACKAGE &> /dev/null ; then
+                printf "    %-70s %-6s\n" $PACKAGE "OK"
+            else
+                printf "    %-70s %-6s\n" $PACKAGE "missing"
+                sudo apt-get install -y $PACKAGE
+            fi
+        done
+        echo
+    fi
+}
+
 #
 # main
 #
 
+echo
+
 # enter workspace dir
 cd $WORKSPACE
 
+# create virtualenv if needed
+if [ ! -e $VSPERFENV_DIR ] ; then
+    echo "Create VSPERF environment"
+    echo "========================="
+    virtualenv --python=python3 "$VSPERFENV_DIR"
+    echo
+fi
+
+# acivate and update virtualenv
+echo "Update VSPERF environment"
+echo "========================="
+source "$VSPERFENV_DIR"/bin/activate
+pip install -r ./requirements.txt
+echo
+
+# VERFIY&MERGE job specific - check if required packages are installed
+dependencies_check
+
 # initialization
 initialize_logdir
 
 # execute job based on passed parameter
 case $1 in
     "verify")
+        echo "================="
         echo "VSPERF verify job"
         echo "================="
 
-        #execute_vsperf OVS_with_DPDK_and_vHost_User $1
+        terminate_vsperf
+        execute_vsperf_sanity
+        terminate_vsperf
+        execute_vsperf OVS_with_DPDK_and_vHost_User $1
+        terminate_vsperf
+        execute_vsperf OVS_vanilla $1
 
         exit $EXIT
         ;;
     "merge")
+        echo "================"
         echo "VSPERF merge job"
         echo "================"
 
-        #execute_vsperf OVS_with_DPDK_and_vHost_User $1
+        terminate_vsperf
+        execute_vsperf_sanity
+        terminate_vsperf
+        execute_vsperf OVS_with_DPDK_and_vHost_User $1
+        terminate_vsperf
+        execute_vsperf OVS_vanilla $1
 
         exit $EXIT
         ;;
     *)
+        echo "================"
         echo "VSPERF daily job"
         echo "================"