Hardening FIO interaction

Fixes a problem where FIO does not terminate by scheduling a
second killall if we get a specific message back from FIO
stderr.

Introduces a new flavor for StorPerf that has a little more
memory as larger memory maps for duplicate blocks sometimes
caused out of memory killer to be invoked.

Change-Id: I06856561ad73fef582a81d4136a36a1bea47654a
JIRA: STORPERF-99
Signed-off-by: mbeierl <mark.beierl@dell.com>

6 files changed, 95 insertions, 40 deletions

diff --git a/ci/create_glance_image.sh b/ci/create_glance_image.sh
new file mode 100755
index 0000000..8811897
--- /dev/null
+++ b/ci/create_glance_image.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2017 EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+echo "Checking for Ubuntu 16.04 image in Glance"
+IMAGE=`openstack image list | grep "Ubuntu 16.04 x86_64"`
+if [ -z "$IMAGE" ]
+then
+    wget -q https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img
+    openstack image create "Ubuntu 16.04 x86_64" --disk-format qcow2 --public \
+    --container-format bare --file ubuntu-16.04-server-cloudimg-amd64-disk1.img
+fi
+
+openstack image show "Ubuntu 16.04 x86_64"
diff --git a/ci/create_storperf_flavor.sh b/ci/create_storperf_flavor.sh
new file mode 100755
index 0000000..f25d56d
--- /dev/null
+++ b/ci/create_storperf_flavor.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2017 EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+echo "Checking for StorPerf flavor"
+
+openstack flavor delete storperf
+
+FLAVOUR=`openstack flavor list | grep "storperf"`
+if [ -z "$FLAVOUR" ]
+then
+    openstack flavor create storperf \
+        --id auto \
+        --ram 8192 \
+        --disk 4 \
+        --vcpus 2
+fi
+
+openstack flavor show storperf
diff --git a/ci/daily.sh b/ci/daily.sh
index 1e77d67..c26e8d3 100755
--- a/ci/daily.sh
+++ b/ci/daily.sh
@@ -19,17 +19,17 @@ then
     sudo rm -rf $WORKSPACE/ci/job
 fi
 
-git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng ci/job/releng
+git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/ci/job/releng
 
 virtualenv $WORKSPACE/ci/job/storperf_daily_venv
 source $WORKSPACE/ci/job/storperf_daily_venv/bin/activate
 
-pip install --upgrade setuptools
-pip install functools32
-pip install pytz
-pip install osc_lib
-pip install python-openstackclient
-pip install python-heatclient
+pip install --upgrade setuptools==33.1.1
+pip install functools32==3.2.3.post2
+pip install pytz==2016.10
+pip install osc_lib==1.3.0
+pip install python-openstackclient==3.7.0
+pip install python-heatclient==1.7.0
 
 # This is set by Jenkins, but if we are running manually, just use the
 # current hostname.
@@ -41,45 +41,28 @@ export POD_NAME=$NODE_NAME
 
 sudo find $WORKSPACE/ -name '*.db' -exec rm -fv {} \;
 
-export INSTALLER=`$WORKSPACE/ci/detect_installer.sh`
-
 $WORKSPACE/ci/generate-admin-rc.sh
 $WORKSPACE/ci/generate-environment.sh
 
 . $WORKSPACE/ci/job/environment.rc
-for env in `cat $WORKSPACE/ci/job/admin.rc`
-do
-    export $env
-done
-
-echo "Checking for an existing stack"
-STACK_ID=`openstack stack list | grep StorPerfAgentGroup | awk '{print $2}'`
-if [ ! -z $STACK_ID ]
-then
-    openstack stack delete --yes --wait StorPerfAgentGroup
-fi
 
-echo Checking for Ubuntu 16.04 image in Glance
-IMAGE=`openstack image list | grep "Ubuntu 16.04 x86_64"`
-if [ -z $IMAGE ]
-then
-    wget https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img
-    openstack image create "Ubuntu 16.04 x86_64" --disk-format qcow2 --public \
-    --container-format bare --file ubuntu-16.04-server-cloudimg-amd64-disk1.img
-fi
+while read -r env
+do
+    export "$env"
+done < $WORKSPACE/ci/job/admin.rc
 
 echo "TEST_DB_URL=http://testresults.opnfv.org/test/api/v1" >> $WORKSPACE/ci/job/admin.rc
-echo "INSTALLER_TYPE=${INSTALLER}" >> $WORKSPACE/ci/job/admin.rc
+
+$WORKSPACE/ci/delete_stack.sh
+$WORKSPACE/ci/create_glance_image.sh
+$WORKSPACE/ci/create_storperf_flavor.sh
 $WORKSPACE/ci/launch_docker_container.sh
+$WORKSPACE/ci/create_stack.sh $CINDER_NODES 10 "Ubuntu 16.04 x86_64" $NETWORK
 
-echo "Waiting for StorPerf to become active"
-while [ $(curl -X GET 'http://127.0.0.1:5000/api/v1.0/configurations' > /dev/null 2>&1;echo $?) -ne 0 ]
-do
-    sleep 1
-done
 
-echo Creating 1:1 stack
-$WORKSPACE/ci/create_stack.sh $CINDER_NODES 10 "Ubuntu 16.04 x86_64" $NETWORK
+echo ==========================================================================
+echo Starting warmup
+echo ==========================================================================
 
 export QUEUE_DEPTH=8
 export BLOCK_SIZE=16384
@@ -96,9 +79,14 @@ do
     | awk '/Status/ {print $2}' | sed 's/"//g'`
 done
 
-export QUEUE_DEPTH=1,2,8
-export BLOCK_SIZE=2048,8192,16384
+
+echo ==========================================================================
+echo Starting full matrix run
+echo ==========================================================================
+
 export WORKLOAD=ws,wr,rs,rr,rw
+export BLOCK_SIZE=2048,8192,16384
+export QUEUE_DEPTH=1,2,8
 export SCENARIO_NAME="${CINDER_BACKEND}_${WORKLOAD}"
 
 JOB=`$WORKSPACE/ci/start_job.sh \
diff --git a/ci/delete_stack.sh b/ci/delete_stack.sh
new file mode 100755
index 0000000..a8a3f56
--- /dev/null
+++ b/ci/delete_stack.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2017 EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+echo "Checking for an existing stack"
+STACK_ID=`openstack stack list | grep StorPerfAgentGroup | awk '{print $2}'`
+if [ ! -z $STACK_ID ]
+then
+    openstack stack delete --yes --wait StorPerfAgentGroup
+fi
diff --git a/ci/generate-admin-rc.sh b/ci/generate-admin-rc.sh
index 424d69c..07a0a3e 100755
--- a/ci/generate-admin-rc.sh
+++ b/ci/generate-admin-rc.sh
@@ -42,4 +42,5 @@ then
     echo export OS_PROJECT_NAME=admin >> job/openstack.rc
 fi
 
-sed "s/export //" job/openstack.rc > job/admin.rc
\ No newline at end of file
+sed "s/export //" job/openstack.rc > job/admin.rc
+echo "INSTALLER_TYPE=${INSTALLER}" >> job/admin.rc
diff --git a/ci/launch_docker_container.sh b/ci/launch_docker_container.sh
index 13cfe04..b119946 100755
--- a/ci/launch_docker_container.sh
+++ b/ci/launch_docker_container.sh
@@ -36,5 +36,10 @@ docker run -d --env-file `pwd`/job/admin.rc \
     -p 8000:8000 \
     -v `pwd`/job/carbon:/opt/graphite/storage/whisper \
     --name storperf opnfv/storperf
+#    -v `pwd`/../../storperf:/home/opnfv/repos/storperf \
 
-
+echo "Waiting for StorPerf to become active"
+while [ $(curl -X GET 'http://127.0.0.1:5000/api/v1.0/configurations' > /dev/null 2>&1;echo $?) -ne 0 ]
+do
+    sleep 1
+done