Enabling ftrace for kernel debugging.

This patch contains updated scripts to enable ftrace for kernel
debugging and storing the log files to artifact repository.

Change-Id: I81b8f987c2c214ca53a6e4502ad953a9adbc6be9
Signed-off-by: Gundarapu Kalyan Reddy <reddyx.gundarapu@intel.com>

5 files changed, 153 insertions, 24 deletions

diff --git a/ci/cyclicTestTrigger.sh b/ci/cyclicTestTrigger.sh
index d222ae15b..b10acddd1 100755
--- a/ci/cyclicTestTrigger.sh
+++ b/ci/cyclicTestTrigger.sh
@@ -13,6 +13,7 @@ testType=$3
 testName=$4
 
 source $WORKSPACE/ci/envs/utils.sh
+source $WORKSPACE/ci/envs/host-config
 KERNELRPM_VERSION=$( getKernelVersion )
 QEMURPM_VERSION=$( getQemuVersion )
 
@@ -110,7 +111,7 @@ function runCyclicTest {
    if [ ${?} -ne 0 ] ; then
       echo  "Docker image build failed"
       id=$(sudo docker ps -a  | head  -2 | tail -1 | awk '{print $1}'); sudo docker rm -f $id
-      exit 1
+      err_exit 1
    fi
    time_stamp=$(date +%Y%m%d%H%M%S)
    volume=/tmp/kvmtest-${testType}-${time_stamp}
diff --git a/ci/envs/disable_trace.sh b/ci/envs/disable_trace.sh
new file mode 100755
index 000000000..1b04e62bb
--- /dev/null
+++ b/ci/envs/disable_trace.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -o xtrace
+curpwd=`pwd`
+TRACE_FILE=$1
+TRACEDIR=/sys/kernel/debug/tracing/
+
+sudo bash -c "echo 0 >$TRACEDIR/tracing_on"
+sleep 1
+sudo bash -c "cat $TRACEDIR/trace > $TRACE_FILE"
+sudo bash -c "echo > $TRACEDIR/set_event"
+sudo bash -c "echo > $TRACEDIR/trace"
+sudo sysctl kernel.ftrace_enabled=0
+sudo bash -c "echo nop > $TRACEDIR/current_tracer"
+
+set +o xtrace
+cd $curpwd
diff --git a/ci/envs/enable-trace.sh b/ci/envs/enable-trace.sh
new file mode 100755
index 000000000..857f53e0d
--- /dev/null
+++ b/ci/envs/enable-trace.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -o xtrace
+EVENT=$1
+curpwd=`pwd`
+TRACEDIR=/sys/kernel/debug/tracing/
+mv /tmp/123.txt /tmp/123.back
+
+function getcpumask {
+        masks=`lscpu | grep "NUMA node1 CPU(s)"| awk -F ':' '{print \$2}' | sed 's/[[:space:]]//g'`
+        first=$(echo ${masks} | cut -f1 -d-)
+        last=$(echo ${masks} | cut -f2 -d-)
+	cpumask=0
+        while [ ${first} -lt ${last} ]; do
+                cputmp=`echo "ibase=10; obase=16; 2^(${c})" | bc`
+                cpumask=`echo "ibase=16; obase=10; ${cputmp}+${cpumask}" |bc`
+                first=`expr $first + 1`
+	done
+	highvalue=`echo "ibase=16;obase=10;$cpumask/(2^20)" |bc `
+	lowvalue=`echo "ibase=16;obase=10;$cpumask%(2^20)" |bc `
+	CPUMASK=`printf '%08x,%08x' 0x$highvalue 0x$lowvalue`
+}
+
+getcpumask
+sudo bash -c "echo $CPUMASK > $TRACEDIR/tracing_cpumask"
+
+#sudo bash -c "echo function > $TRACEDIR/current_tracer"
+#echo :* > set_event
+#echo $EVENT:* > set_event
+
+sudo bash -c "echo 1 > $TRACEDIR/events/irq/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/irq_vectors/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/task/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/syscalls/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/kmem/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/fence/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/context_tracking/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/exceptions/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/irq_vectors/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/nmi/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/kmem/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/migrate/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/sock/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/timer/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/sched/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/rcu/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/kvm/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/workqueue/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/power/enable"
+sudo bash -c "echo 1 > $TRACEDIR/events/signal/enable"
+
+sudo bash -c "echo 1 > events/tlb/enable"
+
+# Clean original log info
+sudo bash -c "echo > $TRACEDIR/trace"
+#sudo bash -c "echo function > $TRACEDIR/current_tracer"
+sudo sysctl kernel.ftrace_enabled=1
+#echo 0 >tracing_on; sleep 1; echo 1 >tracing_on; sleep 20; echo 0 >tracing_on;sleep 1; cat trace >/tmp/123.txt
+sudo bash -c "echo 1 >$TRACEDIR/tracing_on"
+
+cd $curpwd
+#source /home/yjiang5/repo/hostbin/disable_trace.sh
+set +o xtrace
diff --git a/ci/envs/host-config b/ci/envs/host-config
index 0f77b7ac5..4742cc422 100755
--- a/ci/envs/host-config
+++ b/ci/envs/host-config
@@ -33,3 +33,14 @@ done
 #Isolated cpus from host_isolcpus range to run Stress tool
 stress_isolcpus=${first}-${last}
 echo "Stress tool runs on $stress_isolcpus"
+
+#Tar the log files generated during testcase execution.
+function err_exit {
+   exitCode=$1
+   cd $WORKSPACE/build_output/
+   if [ -d log ];then
+      tar -czvf log-$(date -u +"%Y-%m-%d_%H-%M-%S").tar.gz log
+   fi
+   exit $exitCode
+}
+
diff --git a/ci/test_kvmfornfv.sh b/ci/test_kvmfornfv.sh
index a2cee1f95..e87d6eb40 100755
--- a/ci/test_kvmfornfv.sh
+++ b/ci/test_kvmfornfv.sh
@@ -13,10 +13,12 @@
 
 test_type=$1
 test_name=$2
+ftrace_enable=0
 cyclictest_env_verify=("idle_idle" "cpustress_idle" "memorystress_idle" "iostress_idle") #cyclictest environment
 cyclictest_env_daily=("idle_idle" "cpustress_idle" "memorystress_idle" "iostress_idle")
 cyclictest_result=0 #exit code of cyclictest
 packetforward_result=0 #exit code of packet forward
+source $WORKSPACE/ci/envs/host-config
 
 function packetForward {
 #   source $WORKSPACE/ci/packet_forward_test.sh $HOST_IP
@@ -50,48 +52,83 @@ function cyclictest {
    fi
 }
 
+function ftrace_disable {
+   sudo ssh root@${HOST_IP} "sh /root/workspace/scripts/disbale-trace.sh"
+   sudo ssh root@${HOST_IP} "cd /tmp ; a=\$(ls -rt | tail -1) ; echo \$a ; mv \$a cyclictest_${env}.txt"
+   sudo mkdir -p $WORKSPACE/build_output/log/kernel_trace
+   sudo scp root@${HOST_IP}:/tmp/cyclictest_${env}.txt $WORKSPACE/build_output/log/kernel_trace/
+}
+
 #Execution of testcases based on test type and test name from releng.
 if [ ${test_type} == "verify" ];then
    HOST_IP="10.10.100.21"
    test_time=120000 # 2m
-   for env in ${cyclictest_env_verify[@]}
-   do
-      #Executing cyclictest through yardstick.
-      cyclictest ${env}
-      sleep 10
-   done
-   #Execution of packet forwarding test cases.
-   packetForward
-   if [ ${cyclictest_result} -ne 0 ] ||  [ ${packetforward_result} -ne 0 ];then
-      echo "Test case FAILED"
-      exit 1
+   if [ ${ftrace_enable} -eq '1' ]; then
+      for env in ${cyclictest_env_verify[@]}
+      do
+         #Enabling ftrace for kernel debugging.
+         sed -i '/host-setup1.sh/a\    \- \"enable-trace.sh\"' kvmfornfv_cyclictest_hostenv_guestenv.yaml
+         #Executing cyclictest through yardstick.
+         cyclictest ${env}
+         #disabling ftrace and collecting the logs to upload to artifact repository.
+         ftrace_disable
+         sleep 10
+      done
+      #Execution of packet forwarding test cases.
+      packetForward
    else
-      exit 0
+      for env in ${cyclictest_env_verify[@]}
+      do
+         #Executing cyclictest through yardstick.
+         cyclictest ${env}
+         sleep 10
+      done
+      #Execution of packet forwarding test cases.
+      packetForward
    fi
+      if [ ${cyclictest_result} -ne 0 ] ||  [ ${packetforward_result} -ne 0 ];then
+         echo "Test case FAILED"
+         err_exit 1
+      else
+         err_exit 0
+      fi
 elif [ ${test_type} == "daily" ];then
    HOST_IP="10.10.100.22"
    test_time=3600000 #1h
    if [ ${test_name} == "packet_forward" ];then
       packetForward
       if [ ${packetforward_result} -ne 0 ] ; then
-         exit 1
+         err_exit 1
       else
-         exit 0
+         err_exit 0
       fi
    elif [ ${test_name} == "cyclictest" ];then
-      for env in ${cyclictest_env_daily[@]}
-      do
+      if [ ${ftrace_enable} -eq '1' ]; then
+         for env in ${cyclictest_env_daily[@]}
+         do
+            #Enabling ftrace for kernel debugging.
+            sed -i '/host-setup1.sh/a\    \- \"enable-trace.sh\"' kvmfornfv_cyclictest_hostenv_guestenv.yaml
+            #Executing cyclictest through yardstick.
+            cyclictest ${env}
+            #disabling ftrace and collecting the logs to upload to artifact repository. 
+            ftrace_disable
+            sleep 5
+         done
+      else
+         for env in ${cyclictest_env_daily[@]}
+         do
          #Executing cyclictest through yardstick.
          cyclictest ${env}
          sleep 5
-      done
-      if [ ${cyclictest_result} -ne 0 ] ; then
-         echo "Cyclictest case execution FAILED"
-         exit 1
-      else
-         echo "Cyclictest case executed SUCCESSFULLY"
-         exit 0
+         done
       fi
+         if [ ${cyclictest_result} -ne 0 ] ; then
+            echo "Cyclictest case execution FAILED"
+            err_exit 1
+         else
+            echo "Cyclictest case executed SUCCESSFULLY"
+            err_exit 0
+         fi
    fi
 elif [ ${test_type} == "merge" ];then
    echo "Test is not enabled for ${test_type}"