diff options
| -rwxr-xr-x | ci/cyclicTestTrigger.sh | 3 | ||||
| -rwxr-xr-x | ci/envs/disable_trace.sh | 17 | ||||
| -rwxr-xr-x | ci/envs/enable-trace.sh | 63 | ||||
| -rwxr-xr-x | ci/envs/host-config | 11 | ||||
| -rwxr-xr-x | ci/test_kvmfornfv.sh | 83 | ||||
| -rw-r--r-- | fuel-plugin/deployment_scripts/puppet/manifests/kvm-install.pp | 46 | ||||
| -rw-r--r-- | fuel-plugin/deployment_tasks.yaml | 19 | ||||
| -rwxr-xr-x | fuel-plugin/pre_build_hook | 8 |
8 files changed, 198 insertions, 52 deletions
diff --git a/ci/cyclicTestTrigger.sh b/ci/cyclicTestTrigger.sh index d222ae15b..b10acddd1 100755 --- a/ci/cyclicTestTrigger.sh +++ b/ci/cyclicTestTrigger.sh @@ -13,6 +13,7 @@ testType=$3 testName=$4 source $WORKSPACE/ci/envs/utils.sh +source $WORKSPACE/ci/envs/host-config KERNELRPM_VERSION=$( getKernelVersion ) QEMURPM_VERSION=$( getQemuVersion ) @@ -110,7 +111,7 @@ function runCyclicTest { if [ ${?} -ne 0 ] ; then echo "Docker image build failed" id=$(sudo docker ps -a | head -2 | tail -1 | awk '{print $1}'); sudo docker rm -f $id - exit 1 + err_exit 1 fi time_stamp=$(date +%Y%m%d%H%M%S) volume=/tmp/kvmtest-${testType}-${time_stamp} diff --git a/ci/envs/disable_trace.sh b/ci/envs/disable_trace.sh new file mode 100755 index 000000000..1b04e62bb --- /dev/null +++ b/ci/envs/disable_trace.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -o xtrace +curpwd=`pwd` +TRACE_FILE=$1 +TRACEDIR=/sys/kernel/debug/tracing/ + +sudo bash -c "echo 0 >$TRACEDIR/tracing_on" +sleep 1 +sudo bash -c "cat $TRACEDIR/trace > $TRACE_FILE" +sudo bash -c "echo > $TRACEDIR/set_event" +sudo bash -c "echo > $TRACEDIR/trace" +sudo sysctl kernel.ftrace_enabled=0 +sudo bash -c "echo nop > $TRACEDIR/current_tracer" + +set +o xtrace +cd $curpwd diff --git a/ci/envs/enable-trace.sh b/ci/envs/enable-trace.sh new file mode 100755 index 000000000..857f53e0d --- /dev/null +++ b/ci/envs/enable-trace.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +set -o xtrace +EVENT=$1 +curpwd=`pwd` +TRACEDIR=/sys/kernel/debug/tracing/ +mv /tmp/123.txt /tmp/123.back + +function getcpumask { + masks=`lscpu | grep "NUMA node1 CPU(s)"| awk -F ':' '{print \$2}' | sed 's/[[:space:]]//g'` + first=$(echo ${masks} | cut -f1 -d-) + last=$(echo ${masks} | cut -f2 -d-) + cpumask=0 + while [ ${first} -lt ${last} ]; do + cputmp=`echo "ibase=10; obase=16; 2^(${c})" | bc` + cpumask=`echo "ibase=16; obase=10; ${cputmp}+${cpumask}" |bc` + first=`expr $first + 1` + done + highvalue=`echo "ibase=16;obase=10;$cpumask/(2^20)" |bc ` + lowvalue=`echo "ibase=16;obase=10;$cpumask%(2^20)" |bc ` + CPUMASK=`printf '%08x,%08x' 0x$highvalue 0x$lowvalue` +} + +getcpumask +sudo bash -c "echo $CPUMASK > $TRACEDIR/tracing_cpumask" + +#sudo bash -c "echo function > $TRACEDIR/current_tracer" +#echo :* > set_event +#echo $EVENT:* > set_event + +sudo bash -c "echo 1 > $TRACEDIR/events/irq/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/irq_vectors/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/task/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/syscalls/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/kmem/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/fence/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/context_tracking/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/exceptions/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/irq_vectors/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/nmi/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/kmem/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/migrate/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/sock/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/timer/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/sched/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/rcu/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/kvm/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/workqueue/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/power/enable" +sudo bash -c "echo 1 > $TRACEDIR/events/signal/enable" + +sudo bash -c "echo 1 > events/tlb/enable" + +# Clean original log info +sudo bash -c "echo > $TRACEDIR/trace" +#sudo bash -c "echo function > $TRACEDIR/current_tracer" +sudo sysctl kernel.ftrace_enabled=1 +#echo 0 >tracing_on; sleep 1; echo 1 >tracing_on; sleep 20; echo 0 >tracing_on;sleep 1; cat trace >/tmp/123.txt +sudo bash -c "echo 1 >$TRACEDIR/tracing_on" + +cd $curpwd +#source /home/yjiang5/repo/hostbin/disable_trace.sh +set +o xtrace diff --git a/ci/envs/host-config b/ci/envs/host-config index 0f77b7ac5..4742cc422 100755 --- a/ci/envs/host-config +++ b/ci/envs/host-config @@ -33,3 +33,14 @@ done #Isolated cpus from host_isolcpus range to run Stress tool stress_isolcpus=${first}-${last} echo "Stress tool runs on $stress_isolcpus" + +#Tar the log files generated during testcase execution. +function err_exit { + exitCode=$1 + cd $WORKSPACE/build_output/ + if [ -d log ];then + tar -czvf log-$(date -u +"%Y-%m-%d_%H-%M-%S").tar.gz log + fi + exit $exitCode +} + diff --git a/ci/test_kvmfornfv.sh b/ci/test_kvmfornfv.sh index a2cee1f95..e87d6eb40 100755 --- a/ci/test_kvmfornfv.sh +++ b/ci/test_kvmfornfv.sh @@ -13,10 +13,12 @@ test_type=$1 test_name=$2 +ftrace_enable=0 cyclictest_env_verify=("idle_idle" "cpustress_idle" "memorystress_idle" "iostress_idle") #cyclictest environment cyclictest_env_daily=("idle_idle" "cpustress_idle" "memorystress_idle" "iostress_idle") cyclictest_result=0 #exit code of cyclictest packetforward_result=0 #exit code of packet forward +source $WORKSPACE/ci/envs/host-config function packetForward { # source $WORKSPACE/ci/packet_forward_test.sh $HOST_IP @@ -50,48 +52,83 @@ function cyclictest { fi } +function ftrace_disable { + sudo ssh root@${HOST_IP} "sh /root/workspace/scripts/disbale-trace.sh" + sudo ssh root@${HOST_IP} "cd /tmp ; a=\$(ls -rt | tail -1) ; echo \$a ; mv \$a cyclictest_${env}.txt" + sudo mkdir -p $WORKSPACE/build_output/log/kernel_trace + sudo scp root@${HOST_IP}:/tmp/cyclictest_${env}.txt $WORKSPACE/build_output/log/kernel_trace/ +} + #Execution of testcases based on test type and test name from releng. if [ ${test_type} == "verify" ];then HOST_IP="10.10.100.21" test_time=120000 # 2m - for env in ${cyclictest_env_verify[@]} - do - #Executing cyclictest through yardstick. - cyclictest ${env} - sleep 10 - done - #Execution of packet forwarding test cases. - packetForward - if [ ${cyclictest_result} -ne 0 ] || [ ${packetforward_result} -ne 0 ];then - echo "Test case FAILED" - exit 1 + if [ ${ftrace_enable} -eq '1' ]; then + for env in ${cyclictest_env_verify[@]} + do + #Enabling ftrace for kernel debugging. + sed -i '/host-setup1.sh/a\ \- \"enable-trace.sh\"' kvmfornfv_cyclictest_hostenv_guestenv.yaml + #Executing cyclictest through yardstick. + cyclictest ${env} + #disabling ftrace and collecting the logs to upload to artifact repository. + ftrace_disable + sleep 10 + done + #Execution of packet forwarding test cases. + packetForward else - exit 0 + for env in ${cyclictest_env_verify[@]} + do + #Executing cyclictest through yardstick. + cyclictest ${env} + sleep 10 + done + #Execution of packet forwarding test cases. + packetForward fi + if [ ${cyclictest_result} -ne 0 ] || [ ${packetforward_result} -ne 0 ];then + echo "Test case FAILED" + err_exit 1 + else + err_exit 0 + fi elif [ ${test_type} == "daily" ];then HOST_IP="10.10.100.22" test_time=3600000 #1h if [ ${test_name} == "packet_forward" ];then packetForward if [ ${packetforward_result} -ne 0 ] ; then - exit 1 + err_exit 1 else - exit 0 + err_exit 0 fi elif [ ${test_name} == "cyclictest" ];then - for env in ${cyclictest_env_daily[@]} - do + if [ ${ftrace_enable} -eq '1' ]; then + for env in ${cyclictest_env_daily[@]} + do + #Enabling ftrace for kernel debugging. + sed -i '/host-setup1.sh/a\ \- \"enable-trace.sh\"' kvmfornfv_cyclictest_hostenv_guestenv.yaml + #Executing cyclictest through yardstick. + cyclictest ${env} + #disabling ftrace and collecting the logs to upload to artifact repository. + ftrace_disable + sleep 5 + done + else + for env in ${cyclictest_env_daily[@]} + do #Executing cyclictest through yardstick. cyclictest ${env} sleep 5 - done - if [ ${cyclictest_result} -ne 0 ] ; then - echo "Cyclictest case execution FAILED" - exit 1 - else - echo "Cyclictest case executed SUCCESSFULLY" - exit 0 + done fi + if [ ${cyclictest_result} -ne 0 ] ; then + echo "Cyclictest case execution FAILED" + err_exit 1 + else + echo "Cyclictest case executed SUCCESSFULLY" + err_exit 0 + fi fi elif [ ${test_type} == "merge" ];then echo "Test is not enabled for ${test_type}" diff --git a/fuel-plugin/deployment_scripts/puppet/manifests/kvm-install.pp b/fuel-plugin/deployment_scripts/puppet/manifests/kvm-install.pp index 4c04aa0a6..fba3e38ce 100644 --- a/fuel-plugin/deployment_scripts/puppet/manifests/kvm-install.pp +++ b/fuel-plugin/deployment_scripts/puppet/manifests/kvm-install.pp @@ -1,18 +1,32 @@ $kvm_settings = hiera('fuel-plugin-kvm') -if $operatingsystem == 'Ubuntu' { - if $kvm_settings['use_kvm'] { - package { 'linux-headers-4.4.6-rt14nfv': - ensure => "1.0.OPNFV", - notify => Reboot['after_run'], - } -> - package { 'linux-image-4.4.6-rt14nfv': - ensure => "1.0.OPNFV", - notify => Reboot['after_run'], - } - reboot { 'after_run': - apply => finished, - } - } else { - } -} elsif $operatingsystem == 'CentOS' { + +case $::operatingsystem { + 'Ubuntu': { + $version = '4.4.6-rt14nfv' + $kernel_kit = ["linux-headers-${version}", "linux-image-${version}"] + $kernel_src = "/usr/src/linux-headers-${version}" + $kernel_src_link = "/lib/modules/${version}/build" + } + default: { + fail("Unsupported operating system: ${::osfamily}/${::operatingsystem}") + } + +} + +if $kvm_settings['use_kvm'] { + $ensure_pkg = '1.0.OPNFV' + $ensure_link = 'link' +} else { + $ensure_pkg = 'purged' + $ensure_link = 'absent' +} + +package { $kernel_kit: + ensure => $ensure_pkg, + before => File[$kernel_src_link], +} + +file { $kernel_src_link: + ensure => $ensure_link, + target => $kernel_src, } diff --git a/fuel-plugin/deployment_tasks.yaml b/fuel-plugin/deployment_tasks.yaml index 55f91ff21..ab63a9b1f 100644 --- a/fuel-plugin/deployment_tasks.yaml +++ b/fuel-plugin/deployment_tasks.yaml @@ -2,11 +2,22 @@ type: puppet version: 2.1.0 role: [compute] - required_for: [post_deployment_end] - requires: [post_deployment_start] + requires: [pre_deployment_start] + required_for: [pre_deployment_end] condition: - yaql_exp: changed($.get('fuel-plugin-kvm')) + yaql_exp: &kvm changed($.get('fuel-plugin-kvm')) parameters: puppet_manifest: puppet/manifests/kvm-install.pp puppet_modules: puppet/modules:/etc/puppet/modules - timeout: 1200 + timeout: 300 + +- id: node_reboot + type: reboot + version: 2.1.0 + role: [compute] + requires: [kvm_install] + required_for: [pre_deployment_end] + condition: + yaql_exp: *kvm + parameters: + timeout: 900 diff --git a/fuel-plugin/pre_build_hook b/fuel-plugin/pre_build_hook index 259862f8f..a3c883283 100755 --- a/fuel-plugin/pre_build_hook +++ b/fuel-plugin/pre_build_hook @@ -4,10 +4,6 @@ set -eux BUILD_FOR=${BUILD_FOR:-ubuntu} DIR="$(dirname `readlink -f $0`)" -MODULES="${DIR}/deployment_scripts/puppet/modules" - -REBOOT_VER='1.2.1' -REBOOT_URL="https://github.com/puppetlabs/puppetlabs-reboot/archive/${REBOOT_VER}.tar.gz" function build_pkg { case $1 in @@ -29,7 +25,3 @@ for system in $BUILD_FOR do build_pkg $system done - -rm -rf ${MODULES}/reboot -mkdir -p ${MODULES}/reboot -wget -qO- ${REBOOT_URL} | tar -C ${MODULES}/reboot --strip-components=1 -zxvf - |