summaryrefslogtreecommitdiffstats
path: root/ci/cyclicTestTrigger.sh
blob: 4bb8fe5d4e01b0abacbe7409002918b6bf36015f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
#!/bin/bash

#############################################################
## This script defines the functions present in
## test_kvmfornfv.sh interface.These will launch ubuntu
## docker container runs cyclictest through yardstick
## and verifies the test results.
############################################################

HOST_IP=$1
testTime=$2
testType=$3
testName=$4

source $WORKSPACE/ci/envs/utils.sh
source $WORKSPACE/ci/envs/host-config

checkRPMNames

KERNELRPM_VERSION=$( getKernelVersion )
QEMURPM_VERSION=$( getQemuVersion )

if [ -z ${KERNELRPM_VERSION} ];then
   echo "Kernel RPM not found in build_output Directory"
   exit 1
fi
if [ -z ${QEMURPM_VERSION} ];then
   echo "QEMU RPM not found in build_output Directory"
   exit 1
fi

#calculating and verifying sha512sum of the guestimage.
function verifyGuestImage {
   scp $WORKSPACE/build_output/guest1.sha512 root@${HOST_IP}:/root/images
   checksum=$(sudo ssh root@${HOST_IP} "cd /root/images/ && sha512sum -c guest1.sha512 | awk '{print \$2}'")
   if [ "$checksum" != "OK" ]; then
      echo "Something wrong with the image, please verify"
      return 1
   fi
}

#disabling ftrace and collecting the logs to upload to artifact repository.
function ftrace_disable {
   sudo ssh root@${HOST_IP} "cd /tmp ;  mv trace.txt cyclictest_${env}.txt"
   mkdir -p $WORKSPACE/build_output/log/kernel_trace
   scp root@${HOST_IP}:/tmp/cyclictest_${env}.txt $WORKSPACE/build_output/log/kernel_trace/
   sudo ssh root@${HOST_IP} "cd /tmp ; rm -rf cyclictest_${env}.txt"
}

#Verifying the availability of the host after reboot
function connect_host {
   n=0
   while [ $n -lt 25 ]; do
      host_ping_test="ping -c 1 ${HOST_IP}"
      eval $host_ping_test &> /dev/null
      if [ ${?} -ne 0 ] ; then
         sleep 10
         echo "host machine is still under reboot..trying to connect...."
         n=$(($n+1))
      else
         echo "resuming the execution of test cases....."
         #Waiting for ssh to be available for the host machine.
         sleep 30
         break
      fi
      if [ $n == 24 ];then
         echo "Host machine unable to boot-up!"
         exit 1
      fi
   done
}

#Updating the pod.yaml file with HOST_IP,kvmfornfv_cyclictest_idle_idle.yaml with loops and interval
function updateYaml {
   cd $WORKSPACE/tests/
   sed -ri "s/[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/${HOST_IP}/" pod.yaml
   sed -ri "s/loops: [0-9]*/loops: ${testTime}/"  kvmfornfv_cyclictest_hostenv_guestenv.yaml
   sed -ri "0,/interval: [0-9]*/s//interval: 1000/"  kvmfornfv_cyclictest_hostenv_guestenv.yaml
   cp kvmfornfv_cyclictest_hostenv_guestenv.yaml kvmfornfv_cyclictest_${testName}.yaml
   sed -ri "s/tc: \"kvmfornfv_cyclictest-node-context\"/tc: \"kvmfornfv_cyclictest_${testName}\"/" kvmfornfv_cyclictest_${testName}.yaml
   case $testName in

       idle_idle)
                ;;
       cpustress_idle)
                      sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh cpu\"' kvmfornfv_cyclictest_${testName}.yaml
                      ;;
       memorystress_idle)
                      sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh memory\"' kvmfornfv_cyclictest_${testName}.yaml
                      ;;
       iostress_idle)
                      sed -i '/host-run-qemu.sh/a\    \- \"stress_daily.sh io\"' kvmfornfv_cyclictest_${testName}.yaml
                      ;;
       idle_cpustress)
                      sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh cpu\"' kvmfornfv_cyclictest_${testName}.yaml
                      ;;
       idle_memorystress)
                      sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh memory\"' kvmfornfv_cyclictest_${testName}.yaml
                      ;;
       idle_iostress)
                      sed -i '/guest-setup1.sh/a\    \- \"stress_daily.sh io\"' kvmfornfv_cyclictest_${testName}.yaml
                      ;;
        *)
          echo "Incorrect test environment: $testName"
          exit 1
          ;;
    esac
}

#cleaning the environment after executing the test through yardstick.
function env_clean {
    container_id=`sudo docker ps -a | grep kvmfornfv_${testType} |awk '{print \$1}'|sed -e 's/\r//g'`
    sudo docker stop ${container_id}
    sudo docker rm ${container_id}
    sudo ssh root@${HOST_IP} "rm -rf /root/workspace/*"
    sudo ssh root@${HOST_IP} "pid=\$(ps aux | grep 'qemu' | awk '{print \$2}' | head -1); echo \$pid |xargs kill"
    sudo rm -rf /tmp/kvmtest-${testType}*
}

#Cleaning the latest kernel changes on host after executing the test.
function host_clean {
    sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'kernel-${KERNELRPM_VERSION}' | awk '{print \$1}'); rpm -ev \$rpm"
    sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'kernel-devel-${KERNELRPM_VERSION}' | awk '{print \$1}'); rpm -ev \$rpm"
    sudo ssh root@${HOST_IP} "rm -rf /boot/initramfs-${KERNELRPM_VERSION}*.img"
    sudo ssh root@${HOST_IP} "grub2-mkconfig -o /boot/grub2/grub.cfg"
    sudo ssh root@${HOST_IP} "rpm=\$(rpm -qa | grep 'qemu-${QEMURPM_VERSION}'| awk '{print \$1}'); rpm -ev \$rpm"
    sudo ssh root@${HOST_IP} "reboot"
}

function cleanup {
   output=$1
   env_clean
   host_clean
   if [ $output != 0 ];then
      echo "Yardstick Failed.Please check cyclictest.sh"
      return 1
   else
      return 0
   fi
}

function nodeSetup {
   #copying required files to run kvmfornfv testcases
   ssh root@$HOST_IP "mkdir -p /root/workspace/image"
   ssh root@$HOST_IP "mkdir -p /root/workspace/rpm"
   ssh root@$HOST_IP "mkdir -p /root/workspace/scripts"
   #Copying the host configuration scripts on to host
   scp -r $WORKSPACE/ci/envs/* root@$HOST_IP:/root/workspace/scripts
   scp -r $WORKSPACE/tests/vsperf.conf* root@$HOST_IP:/root/workspace/scripts
   scp -r $WORKSPACE/tests/pod.yaml root@$HOST_IP:/root/workspace/scripts
   scp -r $WORKSPACE/build_output/kernel-${KERNELRPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
   scp -r $WORKSPACE/build_output/kernel-devel-${KERNELRPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
   scp -r $WORKSPACE/build_output/qemu-${QEMURPM_VERSION}*.rpm root@$HOST_IP:/root/workspace/rpm
   #execute host configuration script for installing kvm built kernel.
   ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-setup0.sh"
   ssh root@$HOST_IP "cd /root/workspace/rpm ; rpm -ivh kernel-devel-${KERNELRPM_VERSION}*.rpm"
   ssh root@$HOST_IP "reboot"
   sleep 10
}

#environment setup for executing packet forwarding test cases
function setUpPacketForwarding {
   echo "Copying required files to execute packet forwarding test case"
   nodeSetup
}

#executing packet forwarding test cases
function runPacketForwarding {
   testType=$1
   ssh -t -t root@$HOST_IP "cd /root/workspace/scripts ; sudo scl enable python33 'sh packet_forwarding.sh $testType $QEMURPM_VERSION'"
}

#Creating a docker image with yardstick installed and Verify the results of cyclictest
function runCyclicTest {
   ftrace_enable=$1
   docker_image_dir=$WORKSPACE/docker_image_build
   ( cd ${docker_image_dir}; sudo docker build  -t kvmfornfv:latest --no-cache=true . )
   if [ ${?} -ne 0 ] ; then
      echo  "Docker image build failed"
      id=$(sudo docker ps -a  | head  -2 | tail -1 | awk '{print $1}'); sudo docker rm -f $id
      exit 1
   fi
   time_stamp=$(date +%Y%m%d%H%M%S)
   volume=/tmp/kvmtest-${testType}-${time_stamp}
   mkdir -p $volume/{image,rpm,scripts}
   #copying required files to run yardstick cyclic testcase
   cp $WORKSPACE/build_output/kernel-${KERNELRPM_VERSION}*.rpm ${volume}/rpm
   cp $WORKSPACE/build_output/qemu-${QEMURPM_VERSION}*.rpm ${volume}/rpm
   cp -r $WORKSPACE/ci/envs/* ${volume}/scripts
   cp -r $WORKSPACE/tests/kvmfornfv_cyclictest_${testName}.yaml ${volume}
   cp -r $WORKSPACE/tests/pod.yaml ${volume}/scripts

   #Launching ubuntu docker container to run yardstick
   sudo docker run -i -v ${volume}:/opt --net=host --name kvmfornfv_${testType}_${testName} \
   kvmfornfv:latest /bin/bash -c "cd /opt/scripts && ls; ./cyclictest.sh $testType $testName"
   cyclictest_output=$?

   #Disabling ftrace after completion of executing test cases.
   if [ ${ftrace_enable} -eq '1' ]; then
      ftrace_disable
   fi

   if [ "$testName" == "memorystress_idle" ];then
      copyLogs
   fi

   #Verifying the results of cyclictest
   if [ "$testType" == "verify" ];then
      result=`grep -o '"errors":[^,]*' ${volume}/yardstick.out | awk -F '"' '{print $4}'`

      if [ -z "${result}" ]; then
         echo "####################################################"
         echo " "
         echo `grep -o '"data":[^}]*' ${volume}/yardstick.out | awk -F '{' '{print $2}'`
         echo ""
         echo "####################################################"
         cleanup $cyclictest_output
      else
         echo "Testcase failed"
         echo `grep -o '"errors":[^,]*' ${volume}/yardstick.out | awk -F '"' '{print $4}'`
         env_clean
         host_clean
         return 1
      fi
   else
      cleanup $cyclictest_output
   fi
}
function runLiveMigration {
   test_env=$1
   if [ ${test_env} == "peer-peer" ];then
      echo "live migration is not implemented for peer to peer"
   else
      echo "In runLiveMigration Function"
      echo "Copying required files to execute live migration"
      nodeSetup
      connect_host
      sleep 15
      echo " Displaying the number of huge pages on node"
      ssh root@$HOST_IP "cd /root/workspace/scripts;cat /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages"
      echo " Displaying the free huge pages on node"
      ssh root@$HOST_IP "cd /root/workspace/scripts;cat /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages"
      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-setup1.sh"
      echo "Setting up ovs-dpdk on the host"
      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./setup_ovsdpdk.sh"
      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-install-qemu.sh"
      ssh root@$HOST_IP "cd /root/workspace/scripts ; ./host-run-livemigration.sh"
  fi
}