Execution of Livemigration through Yardstick

This Patch includes the scripts to execute the live migration test
using Yardstick and providing the information of VM downtime,
setuptime,totaltime

Change-Id: Ibe768adde1e5b2289a716524a274081cca97751d
Co-Authored by:RajithaY<rajithax.yerrumsetty@intel.com>
Signed-off-by: Navya <navyax.bathula@intel.com>
(cherry picked from commit 7ea7eea6dab8c1b40c2626eead4ced80e77f87b9)

6 files changed, 187 insertions, 48 deletions

diff --git a/ci/cyclicTestTrigger.sh b/ci/cyclicTestTrigger.sh
index 4bb8fe5d4..f4a516282 100755
--- a/ci/cyclicTestTrigger.sh
+++ b/ci/cyclicTestTrigger.sh
@@ -132,15 +132,35 @@ function cleanup {
    env_clean
    host_clean
    if [ $output != 0 ];then
-      echo "Yardstick Failed.Please check cyclictest.sh"
+      echo "Yardstick Failed.Please check your testcase"
       return 1
    else
       return 0
    fi
 }
 
-function nodeSetup {
-   #copying required files to run kvmfornfv testcases
+#environment setup for executing cyclictest and live migration test cases
+function setUpEnv {
+   test=$1
+   time_stamp=$(date +%Y%m%d%H%M%S)
+   volume=/tmp/kvmtest-${testType}-${time_stamp}
+   mkdir -p $volume/{image,rpm,scripts}
+   #copying required files to run yardstick cyclic testcase
+   cp $WORKSPACE/build_output/kernel-${KERNELRPM_VERSION}*.rpm ${volume}/rpm
+   cp $WORKSPACE/build_output/kernel-devel-${KERNELRPM_VERSION}*.rpm ${volume}/rpm
+   cp $WORKSPACE/build_output/qemu-${QEMURPM_VERSION}*.rpm ${volume}/rpm
+   cp -r $WORKSPACE/ci/envs/* ${volume}/scripts
+   cp -r $WORKSPACE/tests/pod.yaml ${volume}/scripts
+   if [ "$test" == "cyclictest" ];then
+      cp -r $WORKSPACE/tests/kvmfornfv_cyclictest_${testName}.yaml ${volume}
+   else
+      cp -r $WORKSPACE/tests/migrate-node-context.yaml ${volume}
+   fi
+}
+
+#environment setup for executing packet forwarding test cases
+function setUpPacketForwarding {
+   #copying required files to run packet forwarding test cases
    ssh root@$HOST_IP "mkdir -p /root/workspace/image"
    ssh root@$HOST_IP "mkdir -p /root/workspace/rpm"
    ssh root@$HOST_IP "mkdir -p /root/workspace/scripts"
@@ -158,12 +178,6 @@ function nodeSetup {
    sleep 10
 }
 
-#environment setup for executing packet forwarding test cases
-function setUpPacketForwarding {
-   echo "Copying required files to execute packet forwarding test case"
-   nodeSetup
-}
-
 #executing packet forwarding test cases
 function runPacketForwarding {
    testType=$1
@@ -173,6 +187,7 @@ function runPacketForwarding {
 #Creating a docker image with yardstick installed and Verify the results of cyclictest
 function runCyclicTest {
    ftrace_enable=$1
+   variable=$2
    docker_image_dir=$WORKSPACE/docker_image_build
    ( cd ${docker_image_dir}; sudo docker build  -t kvmfornfv:latest --no-cache=true . )
    if [ ${?} -ne 0 ] ; then
@@ -180,15 +195,9 @@ function runCyclicTest {
       id=$(sudo docker ps -a  | head  -2 | tail -1 | awk '{print $1}'); sudo docker rm -f $id
       exit 1
    fi
-   time_stamp=$(date +%Y%m%d%H%M%S)
-   volume=/tmp/kvmtest-${testType}-${time_stamp}
-   mkdir -p $volume/{image,rpm,scripts}
-   #copying required files to run yardstick cyclic testcase
-   cp $WORKSPACE/build_output/kernel-${KERNELRPM_VERSION}*.rpm ${volume}/rpm
-   cp $WORKSPACE/build_output/qemu-${QEMURPM_VERSION}*.rpm ${volume}/rpm
-   cp -r $WORKSPACE/ci/envs/* ${volume}/scripts
-   cp -r $WORKSPACE/tests/kvmfornfv_cyclictest_${testName}.yaml ${volume}
-   cp -r $WORKSPACE/tests/pod.yaml ${volume}/scripts
+
+   #setting up the environment for cyclictest
+   setUpEnv $variable
 
    #Launching ubuntu docker container to run yardstick
    sudo docker run -i -v ${volume}:/opt --net=host --name kvmfornfv_${testType}_${testName} \
@@ -227,23 +236,21 @@ function runCyclicTest {
    fi
 }
 function runLiveMigration {
+   echo "In live migration function"
    test_env=$1
-   if [ ${test_env} == "peer-peer" ];then
-      echo "live migration is not implemented for peer to peer"
+   variable=$2
+   #Setting up the environment for live migration test case
+   setUpEnv $variable
+   #Launching ubuntu docker container to run yardstick
+   sudo docker run -i -v ${volume}:/opt --net=host --name kvmfornfv_lm_${test_env} \
+   kvmfornfv:latest /bin/bash -c "cd /opt/scripts && ls; ./lmtest.sh "
+   lmtest_result=$?
+   #Verifying the results of livemigration
+   if [ ${lmtest_result} -ne 0 ];then
+      env_clean
+      host_clean
+      return 1
    else
-      echo "In runLiveMigration Function"
-      echo "Copying required files to execute live migration"
-      nodeSetup
-      connect_host
-      sleep 15
-      echo " Displaying the number of huge pages on node"
-      ssh root@$HOST_IP "cd /root/workspace/scripts;cat /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages"
-      echo " Displaying the free huge pages on node"
-      ssh root@$HOST_IP "cd /root/workspace/scripts;cat /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages"
-      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-setup1.sh"
-      echo "Setting up ovs-dpdk on the host"
-      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./setup_ovsdpdk.sh"
-      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-install-qemu.sh"
-      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-run-livemigration.sh"
-  fi
+      cleanup $lmtest_result
+   fi
 }
diff --git a/ci/envs/host-run-livemigration.sh b/ci/envs/host-run-livemigration.sh
index a089ad482..a5f285282 100755
--- a/ci/envs/host-run-livemigration.sh
+++ b/ci/envs/host-run-livemigration.sh
@@ -41,7 +41,7 @@ function run_qemusrc() {
         -device virtio-net-pci,netdev=net1,mac=$MACADDRESS1 \
         -chardev socket,id=char2,path=$VHOSTPATH2 \
         -netdev type=vhost-user,id=net2,chardev=char2,vhostforce \
-        -device virtio-net-pci,netdev=net2,mac=$MACADDRESS2 -m 1024 -mem-path /dev/hugepages \
+        -device virtio-net-pci,netdev=net2,mac=$MACADDRESS2 -m 2048 -mem-path /dev/hugepages \
         -mem-prealloc -realtime mlock=on -monitor unix:${qmp_sock_src},server,nowait \
         -balloon virtio -drive file=/root/guest1.qcow2 -vnc :1 &
   if [ ${?} -ne 0 ] ; then
@@ -56,7 +56,7 @@ function run_qemulisten() {
   -device virtio-net-pci,netdev=net1,mac=$MACADDRESS1 \
   -chardev socket,id=char2,path=$VHOSTPATH4 \
   -netdev type=vhost-user,id=net2,chardev=char2,vhostforce \
-  -device virtio-net-pci,netdev=net2,mac=$MACADDRESS2 -m 1024 -mem-path /dev/hugepages \
+  -device virtio-net-pci,netdev=net2,mac=$MACADDRESS2 -m 2048 -mem-path /dev/hugepages \
   -mem-prealloc -realtime mlock=on -monitor unix:${qmp_sock_dst},server,nowait \
   -balloon virtio -drive file=/root/guest1.qcow2 -incoming tcp:${incoming_ip}:${migrate_port} -vnc :3 &
   if [ ${?} -ne 0 ] ; then
@@ -101,8 +101,11 @@ sleep 60
 echo "Running Qemu listen"
 run_qemulisten
 sleep 60
-do_migration $qmp_sock_src $qmp_sock_dst
-if [ ${?} -ne 0 ] ; then
-   echo "Migration Failed"
-   exit 1
-fi
+echo "To check qemu src and dst are created"
+ls -la /tmp/ | grep qmp-sock-*
+echo "End of host-run-livemigration"
+#do_migration $qmp_sock_src $qmp_sock_dst
+#if [ ${?} -ne 0 ] ; then
+   #echo "Migration Failed"
+   #exit 1
+#fi
diff --git a/ci/envs/host-setup0.sh b/ci/envs/host-setup0.sh
index 1bd7f1205..e852726d7 100755
--- a/ci/envs/host-setup0.sh
+++ b/ci/envs/host-setup0.sh
@@ -19,6 +19,7 @@ if [ -z $KERNEL_VERSION ];then
 fi
 rpmdir=${1:-"/root/workspace/rpm/"}
 rpmpat="kernel-${KERNEL_VERSION}*.rpm"
+rpmdev="kernel-devel-${KERNEL_VERSION}*.rpm"
 
 config_grub () {
     key=$1
@@ -31,7 +32,7 @@ config_grub () {
     fi
 }
 
-# The script's caller should passing the rpm directory that is built out from 
+# The script's caller should passing the rpm directory that is built out from
 # build.sh. The default rpmdir is the one used by yardstick scripts.
 install_kernel () {
     # Install the kernel rpm
@@ -46,7 +47,9 @@ install_kernel () {
     	exit 1
     else
     	krpm=`find "${rpmdir}" -name "${rpmpat}"`
+        kdrpm=`find "${rpmdir}" -name "${rpmdev}"`
     	rpm -ihv $krpm
+        rpm -ihv $kdrpm
     fi
 }
 
diff --git a/ci/envs/lmtest.sh b/ci/envs/lmtest.sh
new file mode 100755
index 000000000..9136fb62b
--- /dev/null
+++ b/ci/envs/lmtest.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+###########################################################
+## Invoking this script from ubuntu docker container runs
+## cyclictest through yardstick
+###########################################################
+source utils.sh
+
+HOST_IP=$( getHostIP )
+pod_config='/opt/scripts/pod.yaml'
+lmtest_context_file='/opt/migrate-node-context.yaml'
+yardstick_prefix='/root/yardstick/yardstick/benchmark/scenarios/compute' # yardstick teardown path
+
+if [ ! -f ${pod_config} ] ; then
+    echo "file ${pod_config} not found"
+    exit 1
+fi
+
+if [ ! -f ${lmtest_context_file} ] ; then
+    echo "file ${lmtest_context_file} not found"
+    exit 1
+fi
+
+#Execution of the post-execute script copied requires re-installation of yardstick
+( cd /root/yardstick ; python setup.py install )
+
+#setting up of image for launching guest vm.
+ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+root@$HOST_IP "cp /root/images/guest1.qcow2 /root/"
+
+#Updating the yardstick.conf file for daily
+function updateConfDaily() {
+   DISPATCHER_TYPE=influxdb
+   DISPATCHER_FILE_NAME="/tmp/yardstick.out"
+   # Use the influxDB on the jumping server
+   DISPATCHER_INFLUXDB_TARGET="http://104.197.68.199:8086"
+   mkdir -p /etc/yardstick
+   cat << EOF > /etc/yardstick/yardstick.conf
+[DEFAULT]
+debug = True
+dispatcher = ${DISPATCHER_TYPE}
+
+[dispatcher_file]
+file_name = ${DISPATCHER_FILE_NAME}
+
+[dispatcher_influxdb]
+timeout = 5
+db_name = yardstick
+username = opnfv
+password = 0pnfv2015
+target = ${DISPATCHER_INFLUXDB_TARGET}
+EOF
+}
+
+#Function call to update yardstick conf file based on Job type
+#if [ "$testType" == "daily" ];then
+#   updateConfDaily
+#fi
+
+echo "changing the qemu-migrate.py"
+sed -i "s/\/root\/workspace/\/root\/workspace\//g" ${yardstick_prefix}/qemu_migrate.py
+sed -i "s/host.put_file/host._put_file_shell/g" ${yardstick_prefix}/qemu_migrate.py
+
+#Running livemigration through yardstick
+echo "Executing livemigration through yardstick"
+yardstick -d task start ${lmtest_context_file}
+output=$?
+
+if [ "$testType" == "verify" ];then
+   chmod 777 /tmp/yardstick.out
+   cat /tmp/yardstick.out  > /opt/yardstick.out
+fi
+
+if [ $output != 0 ];then
+   echo "Yardstick Failed !!!"
+   exit 1
+fi
diff --git a/ci/test_kvmfornfv.sh b/ci/test_kvmfornfv.sh
index 874dc9864..09bba8149 100755
--- a/ci/test_kvmfornfv.sh
+++ b/ci/test_kvmfornfv.sh
@@ -19,6 +19,7 @@ cyclictest_env_daily=("idle_idle" "cpustress_idle" "memorystress_idle" "iostress
 cyclictest_result=0 #exit code of cyclictest
 packetforward_result=0 #exit code of packet forward
 lm_env_verify=("peer-peer" "local")
+livemigration_result=0 #exit code of livemigration
 source $WORKSPACE/ci/envs/host-config
 
 #check if any kernel rpms available for testing
@@ -53,20 +54,26 @@ function packetForward {
       exit 1
    fi
 }
+
 function liveMigration {
    #executing live migration test case on the host machine
    test_env=$1
+   test_name=livemigration
    echo "Test Environment ${test_env}"
    if [ ${test_env} == "peer-peer" ];then
       echo "live migration is not implemented for peer to peer"
-      livemigration_result=0
    elif [ ${test_env} == "local" ];then
       source $WORKSPACE/ci/cyclicTestTrigger.sh $HOST_IP
       connect_host
+      env_clean
       #Waiting for ssh to be available for the host machine.
       sleep 20
-      runLiveMigration ${test_env}
-      livemigration_result=$?
+      if runLiveMigration ${test_env} ${test_name};then
+         livemigration_result=`expr ${livemigration_result} + 0`
+      else
+         echo "live migration test case failed"
+         livemigration_result=`expr ${livemigration_result} + 1`
+      fi
    else
       echo "Incorrect test environment for live migration"
       exit 1
@@ -80,6 +87,7 @@ function getTestParams {
 
 function cyclictest {
    test_case=$1
+   test_name=cyclictest
    source $WORKSPACE/ci/cyclicTestTrigger.sh $HOST_IP $test_time $test_type $test_case
    #Verifying whether the test node is up and running
    connect_host
@@ -96,7 +104,7 @@ function cyclictest {
    #Cleaning the environment before running cyclictest through yardstick
    env_clean
    #Creating a docker image with yardstick installed and launching ubuntu docker to run yardstick cyclic testcase
-   if runCyclicTest ${ftrace_enable};then
+   if runCyclicTest ${ftrace_enable} ${test_name};then
       cyclictest_result=`expr ${cyclictest_result} + 0`
    else
       echo "Test case execution FAILED for ${test_case} environment"
@@ -166,7 +174,7 @@ if [ ${test_type} == "verify" ];then
          liveMigration ${envi}
       done
    fi
-   if [ ${cyclictest_result} -ne 0 ] ||  [ ${packetforward_result} -ne 0 ];then
+   if [ ${cyclictest_result} -ne 0 ] || [ ${packetforward_result} -ne 0 ] || [ ${livemigration_result} -ne 0 ];then
       echo "Test case FAILED"
       test_exit 1
    else
diff --git a/tests/migrate-node-context.yaml b/tests/migrate-node-context.yaml
new file mode 100755
index 000000000..9abe793ba
--- /dev/null
+++ b/tests/migrate-node-context.yaml
@@ -0,0 +1,41 @@
+---
+
+schema: "yardstick:task:0.1"
+
+scenarios:
+-
+  type: QemuMigrate
+  options:
+    smp: 2
+    migrate_to_port: 4444
+    incoming_ip: 0
+    qmp_src_path: "/tmp/qmp-sock-src"
+    qmp_dst_path: "/tmp/qmp-sock-dst"
+    max_down_time: "0.10"
+  host: kvm.LF
+  runner:
+    type: Duration
+    duration: 1
+    interval: 1
+  tc: "migrate-node-context"
+  sla:
+    max_totaltime: 10
+    max_downtime: 0.10
+    max_setuptime: 0.50
+    action: monitor
+  setup_options:
+    rpm_dir: "/opt/rpm"
+    script_dir: "/opt/scripts"
+    image_dir: "/opt/image"
+    host_setup_seqs:
+    - "host-setup0.sh"
+    - "reboot"
+    - "host-setup1.sh"
+    - "setup_ovsdpdk.sh"
+    - "host-install-qemu.sh"
+    - "host-run-livemigration.sh"
+
+context:
+  type: Node
+  name: LF
+  file: /opt/scripts/pod.yaml
\ No newline at end of file