This patch includes scripts and configuration files for

executing packet forwarding test cases as a part of
kvmfornfv daily job and printing the results.

Change-Id: I484ce89c4821d03cb997c774f014c59a10dc275e
Signed-off-by: Gundarapu Kalyan Reddy <reddyx.gundarapu@intel.com>

4 files changed, 256 insertions, 15 deletions

diff --git a/ci/cyclicTestTrigger.sh b/ci/cyclicTestTrigger.sh
index 741a865d0..12b2a66de 100755
--- a/ci/cyclicTestTrigger.sh
+++ b/ci/cyclicTestTrigger.sh
@@ -118,6 +118,7 @@ function env_clean {
 #Cleaning the latest kernel changes on host after executing the test.
 function host_clean {
     sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'kernel-${KERNELRPM_VERSION}' | awk '{print \$1}'); rpm -ev \$rpm"
+    sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'kernel-devel-${KERNELRPM_VERSION}' | awk '{print \$1}'); rpm -ev \$rpm"
     sudo ssh root@${HOST_IP} "rm -rf /boot/initramfs-${KERNELRPM_VERSION}*.img"
     sudo ssh root@${HOST_IP} "grub2-mkconfig -o /boot/grub2/grub.cfg"
     sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'qemu-${QEMURPM_VERSION}'| awk '{print \$1}'); rpm -ev \$rpm"
@@ -136,6 +137,31 @@ function cleanup {
    fi
 }
 
+#environment setup for executing packet forwarding test cases
+function setUpPacketForwarding {
+   #copying required files to run packet forwarding test cases
+   ssh root@$HOST_IP "mkdir -p /root/workspace/image"
+   ssh root@$HOST_IP "mkdir -p /root/workspace/rpm"
+   ssh root@$HOST_IP "mkdir -p /root/workspace/scripts"
+   #Copying the host configuration scripts on to host
+   scp -r $WORKSPACE/ci/envs/* root@$HOST_IP:/root/workspace/scripts
+   scp -r $WORKSPACE/tests/vsperf.conf* root@$HOST_IP:/root/workspace/scripts
+   scp -r $WORKSPACE/build_output/kernel-${KERNELRPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
+   scp -r $WORKSPACE/build_output/kernel-devel-${KERNELRPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
+   scp -r $WORKSPACE/build_output/qemu-${QEMURPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
+   #execute host configuration script for installing kvm built kernel.
+   ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-setup0.sh"
+   ssh root@$HOST_IP "cd /root/workspace/rpm ; rpm -ivh kernel-devel-${KERNELRPM_VERSION}*.rpm"
+   ssh root@$HOST_IP "reboot"
+   sleep 10
+}
+
+#executing packet forwarding test cases
+function runPacketForwarding {
+   testType=$1
+   ssh -t -t root@$HOST_IP "cd /root/workspace/scripts ; sudo scl enable python33 'sh packet_forwarding.sh $testType $QEMURPM_VERSION'"
+}
+
 #Creating a docker image with yardstick installed and Verify the results of cyclictest
 function runCyclicTest {
    container_id=`sudo docker ps -a | grep kvmfornfv_${testType} |awk '{print \$1}'|sed -e 's/\r//g'`
@@ -189,4 +215,4 @@ function runCyclicTest {
          return 1
       fi
    fi
-}
\ No newline at end of file
+}
diff --git a/ci/envs/host-config b/ci/envs/host-config
index 727ba1574..e696fcde1 100755
--- a/ci/envs/host-config
+++ b/ci/envs/host-config
@@ -55,3 +55,24 @@ function copyLogs {
    sudo ssh root@${HOST_IP} "cd /root;rm -rf MBWInfo MBWInfo.tar.gz"
 }
 
+function packet_fwd_logs {
+   #Tar and copy logs for uploading to artifacts repository
+   echo "Copying Log files from Node to Jump Server"
+   mkdir -p $WORKSPACE/build_output/log/packet_fwd
+   scp -r root@${HOST_IP}:/tmp/packet_fwd_logs $WORKSPACE/build_output/log/packet_fwd
+   #removing collected logs on the node after copying.
+   sudo ssh root@${HOST_IP} "cd /tmp;rm -rf packet_fwd_logs"
+}
+
+function packet_fwd_exit {
+   exitCode=$1
+   TIMESTAMP=$(date -u +"%Y-%m-%d_%H-%M-%S")
+   cd $WORKSPACE/build_output/
+   if [ -d log ];then
+      tar -czvf log-${TIMESTAMP}.tar.gz log
+      echo "Uploading packet forwarding logs and results"
+      gsutil cp -r log-*.tar.gz gs://artifacts.opnfv.org/kvmfornfv/packet_fwd_${TIMESTAMP} > $WORKSPACE/build_output/gsutil.log 2>&1
+      echo "http://artifacts.opnfv.org/kvmfornfv/packet_fwd_${TIMESTAMP}"
+   fi
+   exit $exitCode
+}
diff --git a/ci/envs/packet_forwarding.sh b/ci/envs/packet_forwarding.sh
new file mode 100644
index 000000000..ec399c6ab
--- /dev/null
+++ b/ci/envs/packet_forwarding.sh
@@ -0,0 +1,174 @@
+#!/bin/bash
+JOB_TYPE=$1
+QEMURPM_VERSION=$2
+HOME='/home/jenkins'
+VSPERF="${HOME}/vswitchperf"
+LOG_FILE_PREFIX="/tmp/vsperf_build"
+VSPERF_BIN='./vsperf'
+DATE=$(date -u +"%Y-%m-%d_%H-%M-%S")
+VSPERFENV_DIR="$HOME/vsperfenv"
+TEST_REPORT_LOG_DIR="/tmp/packet_fwd_logs"
+EXIT=0
+EXIT_TC_FAILED=1
+
+# DAILY - run selected TCs for defined packet sizes
+TESTCASES_DAILY='phy2phy_tput phy2phy_tput_mod_vlan pvp_tput'
+TESTPARAM_DAILY='--test-params TRAFFICGEN_PKT_SIZES=(64,128,512,1024,1518)'
+TESTCASES_SRIOV='pvp_tput'
+TESTPARAM_SRIOV='--test-params TRAFFICGEN_PKT_SIZES=(64,128,512,1024,1518)'
+
+#mounting shared directory for collecting ixia test results.
+shared_dir=$(sudo mount | grep ixia_results)
+if [ -z "$shared_dir" ]; then
+    echo "mounting shared directory for results"
+    sudo mount -t cifs //10.10.100.6/ixia_results/kvm4nfv_ci /mnt/ixia_results/kvm4nfv_ci  -o password=kvm4nfv! -o username=kvm4nfv,file_mode=0777,dir_mode=0777,nounix
+else
+    echo "shared directory is already mounted for results"
+fi
+
+# check if user config file exists if not then we will use default settings
+if [ -f $HOME/vsperf.conf ] ; then
+        CONF_FILE="--conf-file ${HOME}/vsperf.conf"
+    else
+        echo "configuration file not found on the test node"
+        echo "Using configuration file available in kvmfornfv repo"
+        CONF_FILE="--conf-file /root/workspace/scripts/vsperf.conf"
+fi
+
+# check if sriov config file exists if not then we will use default settings
+if [ -f $HOME/vsperf.conf.sriov ] ; then
+        CONF_FILE_SRIOV="--conf-file ${HOME}/vsperf.conf.sriov"
+    else
+        echo "SRIOV configuration file not found on the node"
+        echo "Using SRIOV configuration file available in kvmfornfv repo"
+        CONF_FILE="--conf-file /root/workspace/scripts/vsperf.conf.sriov"
+fi
+
+function install_vsperf() {
+    echo "Installing vsperf....."
+    ( cd $VSPERF/systems ; ./build_base_machine.sh )
+}
+
+#Install kvmfornfv built qemu for launching guest vm.
+function install_qemu() {
+    echo "removing existing qemu packages and installing kvmfornfv built qemu"
+    ( cd /root/workspace/scripts ; ./host-install-qemu.sh )
+}
+
+function print_results() {
+    for i in $TESTCASES ; do
+        RES_FILE=`ls -1 $1 | egrep "result_${i}_[0-9a-zA-Z\-]+.csv"`
+
+        if [ "x$RES_FILE" != "x" -a -e "${1}/${RES_FILE}" ]; then
+            if grep ^FAILED "${1}/${RES_FILE}" &> /dev/null ; then
+                printf "    %-70s %-6s\n" "result_${i}" "FAILED"
+                EXIT=$EXIT_TC_FAILED
+            else
+                echo "--------------------------------------------------------------"
+                printf "    %-50s %-6s\n" "result_${i}" "OK"
+                echo "--------------------------------------------------------------"
+            fi
+        else
+            echo "------------------------------------------------------------------"
+            printf "    %-50s %-6s\n" "result_${i}" "FAILED"
+            echo "------------------------------------------------------------------"
+            EXIT=$EXIT_TC_FAILED
+        fi
+    done
+}
+
+function execute_vsperf() {
+    # figure out list of TCs and execution parameters
+    case $2 in
+        "daily")
+            TESTPARAM=$TESTPARAM_DAILY
+            TESTCASES=$TESTCASES_DAILY
+            ;;
+        *)
+            echo "No vsperf test cases implemented for this job type"
+            ;;
+    esac
+
+    # execute testcases
+    echo -e "\nExecution of VSPERF for $1"
+    DATE_SUFFIX=$(date -u +"%Y-%m-%d_%H-%M-%S")
+    source "$VSPERFENV_DIR"/bin/activate
+    case $1 in
+        "SRIOV")
+            # use SRIOV specific TCs and configuration
+            TESTPARAM=$TESTPARAM_SRIOV
+            TESTCASES=$TESTCASES_SRIOV
+            # figure out log file name
+            LOG_SUBDIR="SRIOV"
+            LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
+            echo "    $VSPERF_BIN --vswitch none --vnf QemuPciPassthrough $CONF_FILE_SRIOV $TESTPARAM $TESTCASES &> $LOG_FILE"
+            $VSPERF_BIN --vswitch none --vnf QemuPciPassthrough $CONF_FILE_SRIOV $TESTPARAM $TESTCASES &> $LOG_FILE
+            ;;
+        *)
+            # figure out log file name
+            LOG_SUBDIR="OvsDpdkVhost"
+            LOG_FILE="${LOG_FILE_PREFIX}_${LOG_SUBDIR}_${DATE_SUFFIX}.log"
+            echo "    $VSPERF_BIN $OPNFVPOD --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES > $LOG_FILE"
+            echo "daily test cases started"
+            cd $HOME/vswitchperf
+            $VSPERF_BIN --list
+            $VSPERF_BIN  --vswitch OvsDpdkVhost --vnf QemuDpdkVhostUser $CONF_FILE $TESTPARAM $TESTCASES &>> $LOG_FILE
+            ;;
+    esac
+    # evaluation of results
+    echo -e "\nResults for $1"
+    RES_DIR="/$(grep "Creating result directory" $LOG_FILE | cut -d'/' -f2-)"
+    if [[ "/" == "${RES_DIR}" ]] ; then
+        echo "FAILURE: Results are not available."
+        echo "-------------------------------------------------------------------"
+        cat $LOG_FILE
+        echo "-------------------------------------------------------------------"
+        exit $EXIT_NO_RESULTS
+    else
+        print_results "${RES_DIR}"
+        if [ $(($EXIT & $EXIT_TC_FAILED)) -gt 0 ] ; then
+            echo "-------------------------------------------------------------------"
+            cat $LOG_FILE
+            echo "-------------------------------------------------------------------"
+        fi
+    fi
+    # show detailed result figures
+    for md_file in $(grep '\.md"$' $LOG_FILE | cut -d'"' -f2); do
+        # TC resut file header
+        echo -e "\n-------------------------------------------------------------------"
+        echo -e " $md_file"
+        echo -e "-------------------------------------------------------------------\n"
+        # TC details
+        sed -n '/^- Test ID/,/Bidirectional/{/Packet size/b;p;/Bidirectional/q};/Results\/Metrics Collected/,/Statistics collected/{/^$/p;/^|/p}' $md_file
+        # TC results
+        sed -n '/Results\/Metrics Collected/,/Statistics collected/{/^$/p;/^|/p}' $md_file | grep -v "Unknown" | cat -s
+    done
+
+    # copy logs into dedicated directory
+    mkdir -p ${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}
+    [ -f "$LOG_FILE" ] && cp -a "${LOG_FILE}" "${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}" &> /dev/null
+    [ -d "$RES_DIR" ] && cp -ar "$RES_DIR" "${TEST_REPORT_LOG_DIR}/${LOG_SUBDIR}" &> /dev/null
+}
+
+#Install vsperf and set up the environment
+install_vsperf
+
+#Install kvmfornfv built qemu rpm
+install_qemu
+
+# execute job based on passed parameter
+case $1 in
+    "daily")
+        echo "================"
+        echo "VSPERF daily job"
+        echo "================"
+        execute_vsperf OVS_with_DPDK_and_vHost_User $1
+        execute_vsperf SRIOV $1
+        exit $EXIT
+        ;;
+    *)
+        echo "test cases not implemented for this job type"
+esac
+
+exit $EXIT
+
diff --git a/ci/test_kvmfornfv.sh b/ci/test_kvmfornfv.sh
index adf0aa3ce..11ea3a5ce 100755
--- a/ci/test_kvmfornfv.sh
+++ b/ci/test_kvmfornfv.sh
@@ -21,13 +21,27 @@ packetforward_result=0 #exit code of packet forward
 source $WORKSPACE/ci/envs/host-config
 
 function packetForward {
-#   source $WORKSPACE/ci/packet_forward_test.sh $HOST_IP
-   echo "Packetforwarding need to be implemented"
-   packetforward_result=$?
-   if [ ${packetforward_result} -ne 0 ];then
-      echo "Packet Forwarding test case execution FAILED"
+   #executing packet forwarding test cases based on the job type.
+   if [ ${test_type} == "verify" ];then
+      echo "packet forwarding test cases are not yet implemented for verify job"
+      packetforward_result=0
+   elif [ ${test_type} == "daily" ];then
+      source $WORKSPACE/ci/cyclicTestTrigger.sh $HOST_IP
+      connect_host
+      #Waiting for ssh to be available for the host machine.
+      sleep 20
+      # copy files and rpms and setup environment required for executing test cases
+      setUpPacketForwarding
+      sleep 1
+      #Verifying whether the test node is up and running
+      connect_host
+      sleep 20
+      #Install and Execute packet forwarding test cases
+      runPacketForwarding $test_type
+      packetforward_result=$?
    else
-      echo "Packet Forwarding test case executed SUCCESSFULLY"
+      echo "Incorrect test type ${test_type}"
+      exit 1
    fi
 }
 
@@ -138,22 +152,28 @@ if [ ${test_type} == "verify" ];then
       #Execution of packet forwarding test cases.
       packetForward
    fi
-      if [ ${cyclictest_result} -ne 0 ] ||  [ ${packetforward_result} -ne 0 ];then
-         echo "Test case FAILED"
-         test_exit 1
-      else
-         test_exit 0
-      fi
+   if [ ${cyclictest_result} -ne 0 ] ||  [ ${packetforward_result} -ne 0 ];then
+      echo "Test case FAILED"
+      test_exit 1
+   else
+      test_exit 0
+   fi
 elif [ ${test_type} == "daily" ];then
    HOST_IP="10.10.100.22"
    test_time=3600000 #1h
    install_pcm
    if [ ${test_name} == "packet_forward" ];then
       packetForward
+      packet_fwd_logs
+      #clean the test environment after the test case execution.
+      sudo ssh root@${HOST_IP} "rm -rf /root/workspace/*"
+      host_clean
       if [ ${packetforward_result} -ne 0 ] ; then
-         test_exit 1
+         echo "Execution of packet forwarding test cases FAILED"
+         packet_fwd_exit 1
       else
-         test_exit 0
+         echo "Executed packet forwarding test cases SUCCESSFULLY"
+         packet_fwd_exit 0
       fi
    elif [ ${test_name} == "cyclictest" ];then
       if [ ${ftrace_enable} -eq '1' ]; then