Daily Job

Creation of a daily job script that runs under Jenkins and
kicks off a series of tests using the freshly cloned workspace.

Change-Id: Ibc63c1df954578ad78604321ea410c8fd8c63c41
JIRA: STORPERF-87
Signed-off-by: Mark Beierl <mark.beierl@dell.com>

8 files changed, 271 insertions, 18 deletions

diff --git a/ci/create_stack.sh b/ci/create_stack.sh
new file mode 100755
index 0000000..152f1db
--- /dev/null
+++ b/ci/create_stack.sh
@@ -0,0 +1,23 @@
+#!/bin/bash -x
+##############################################################################
+# Copyright (c) 2015 EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+cat << EOF > body.json
+{
+  "agent_count": $1,
+  "agent_image": "Trusty x86_64",
+  "public_network": "ext-net",
+  "volume_size": $2
+}
+EOF
+
+curl -X POST --header 'Content-Type: application/json' \
+     --header 'Accept: application/json' -d @body.json \
+     'http://127.0.0.1:5000/api/v1.0/configurations'
+
diff --git a/ci/daily.sh b/ci/daily.sh
index 0cc5977..bdfe746 100755
--- a/ci/daily.sh
+++ b/ci/daily.sh
@@ -8,29 +8,103 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-if [ -f ~/jenkins-os.rc ]
+if [ -z $WORKSPACE ]
 then
-    . ~/jenkins-os.rc
+    cd `dirname $0`/..
+    WORKSPACE=`pwd`
 fi
 
-if [ -z $WORKSPACE ]
+# This is set by Jenkins, but if we are running manually, just use the
+# current hostname.
+if [ -z "$NODE_NAME" ]
 then
-    WORKSPACE=`pwd`
+    NODE_NAME=`hostname`
+fi
+export POD_NAME=$NODE_NAME
+
+if [ -d $WORKSPACE/ci/job ]
+then
+    sudo rm -rf $WORKSPACE/ci/job
+fi
+sudo find $WORKSPACE/ -name '*.db' -exec rm -fv {} \;
+
+$WORKSPACE/ci/generate-admin-rc.sh
+$WORKSPACE/ci/generate-environment.sh
+
+. $WORKSPACE/ci/job/environment.rc
+for env in `cat $WORKSPACE/ci/job/admin.rc`
+do
+    export $env
+done
+
+echo "Checking for an existing stack"
+STACK_ID=`heat stack-list | grep StorPerfAgentGroup | awk '{print $2}'`
+if [ ! -z $STACK_ID ]
+then
+    heat stack-delete -y StorPerfAgentGroup
 fi
 
-function stage_base_os_in_glance {
+while [ ! -z $STACK_ID ]
+do
+    STACK_ID=`heat stack-list | grep StorPerfAgentGroup | awk '{print $2}'`
+done
+
+echo "TEST_DB_URL=http://testresults.opnfv.org/test/api/v1" >> $WORKSPACE/ci/job/admin.rc
+echo "INSTALLER_TYPE=${INSTALLER}" >> $WORKSPACE/ci/job/admin.rc
+$WORKSPACE/ci/launch_docker_container.sh
+
+echo "Waiting for StorPerf to become active"
+while [ $(curl -X GET 'http://127.0.0.1:5000/api/v1.0/configurations' > /dev/null 2>&1;echo $?) -ne 0 ]
+do
+    sleep 1
+done
+
+echo Creating 1:1 stack
+$WORKSPACE/ci/create_stack.sh $CINDER_NODES 10
+
+export QUEUE_DEPTH=8
+export BLOCK_SIZE=16384
+export WORKLOAD=ws
+export SCENARIO_NAME="${CINDER_BACKEND}_${WORKLOAD}"
+WARM_UP=`$WORKSPACE/ci/start_job.sh | awk '/job_id/ {print $2}' | sed 's/"//g'`
+
+WARM_UP_STATUS=`curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$WARM_UP&type=status" \
+    | awk '/Status/ {print $2}' | sed 's/"//g'`
+while [ "$WARM_UP_STATUS" != "Completed" ]
+do
+    sleep 10
+    WARM_UP_STATUS=`curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$WARM_UP&type=status" \
+    | awk '/Status/ {print $2}' | sed 's/"//g'`
+done
+
 
-   export OS_IMAGE_API_VERSION=1
+for WORKLOAD in ws wr rs rr rw
+do
+    for BLOCK_SIZE in 2048 8192 16384
+    do
+        for QUEUE_DEPTH in 1 2 8
+        do
+            export QUEUE_DEPTH
+            export BLOCK_SIZE
+            export WORKLOAD
+            export SCENARIO_NAME="${CINDER_BACKEND}_${WORKLOAD}"
 
-    glance image-list | grep "$1"
-    if [ $? -eq 1 ]
-    then
-        curl -s -o $WORKSPACE/$1.qcow2 https://cloud-images.ubuntu.com/releases/15.10/release/ubuntu-15.10-server-cloudimg-amd64-disk1.img
-        glance image-create --name="$1" --disk-format=qcow2 --container-format=bare < $WORKSPACE/$1.qcow2
-    fi
+            JOB=`$WORKSPACE/ci/start_job.sh \
+                | awk '/job_id/ {print $2}' | sed 's/"//g'`
+            JOB_STATUS=`curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$JOB&type=status" \
+                | awk '/Status/ {print $2}' | sed 's/"//g'`
+            while [ "$JOB_STATUS" != "Completed" ]
+            do
+                sleep 10
+                JOB_STATUS=`curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$JOB&type=status" \
+                    | awk '/Status/ {print $2}' | sed 's/"//g'`
+            done
+        done
+    done
+done
 
-}
 
-stage_base_os_in_glance ubuntu-server
+echo "Deleting stack for cleanup"
+curl -X DELETE --header 'Accept: application/json' 'http://127.0.0.1:5000/api/v1.0/configurations'
 
 exit 0
diff --git a/ci/generate-admin-rc.sh b/ci/generate-admin-rc.sh
new file mode 100755
index 0000000..6c5b162
--- /dev/null
+++ b/ci/generate-admin-rc.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+cd `dirname $0`
+
+# TODO: Switch based on installer type, for now this is hard coded to JOID only
+
+if [ ! -d job ]
+then
+    mkdir job
+fi
+
+export OS_AUTH_URL=http://`juju status keystone | grep public | awk '{print $2}'`:5000/v2.0
+export OS_USERNAME=admin
+export OS_PASSWORD=openstack
+export OS_TENANT_ID=`openstack project list|awk '/admin/ {print $2}'`
+
+cat << EOF > job/admin.rc
+OS_AUTH_URL=$OS_AUTH_URL
+OS_USERNAME=$OS_USERNAME
+OS_PASSWORD=$OS_PASSWORD
+OS_TENANT_ID=$OS_TENANT_ID
+OS_TENANT_NAME=admin
+OS_PROJECT_NAME=admin
+OS_REGION_NAME=RegionOne
+EOF
+
diff --git a/ci/generate-environment.sh b/ci/generate-environment.sh
new file mode 100755
index 0000000..6b44313
--- /dev/null
+++ b/ci/generate-environment.sh
@@ -0,0 +1,48 @@
+#!/bin/bash -x
+##############################################################################
+# Copyright (c) 2015 EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+cd `dirname $0`
+if [ ! -d job ]
+then
+    mkdir job
+fi
+
+# TODO: This assumes JOID.  Need to make this programmatic
+
+INSTALLER=JOID
+
+case $INSTALLER in
+    JOID)
+        # Determine Cinder backend
+        if [ ! -z "$(juju status | grep ceph)" ]
+        then
+            CINDER_BACKEND=ceph
+            JUJU_CHARM=ceph-osd
+        fi
+        if [ ! -z "$(juju status | grep scaleio)" ]
+        then
+            CINDER_BACKEND=scaleio
+            JUJU_CHARM=scaleio-sds
+        fi
+            # Determine how many storage blades we have
+            CINDER_NODES=`juju status | grep "$JUJU_CHARM/" | wc -l`
+            # Later, collect info about each node:
+            # juju status | grep hardware: | grep tags | grep -v virtual
+        ;;
+    *)
+        CINDER_BACKEND=ceph
+        CINDER_NODES=4
+esac
+
+cat << EOF > job/environment.rc
+export CINDER_BACKEND=$CINDER_BACKEND
+export CINDER_NODES=$CINDER_NODES
+export INSTALLER=$INSTALLER
+EOF
\ No newline at end of file
diff --git a/ci/launch_docker_container.sh b/ci/launch_docker_container.sh
new file mode 100755
index 0000000..60d1a07
--- /dev/null
+++ b/ci/launch_docker_container.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016 EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+cd `dirname $0`
+
+storperf_container=`docker ps -a -q -f name=storperf`
+
+if [ ! -z $storperf_container ]
+then
+    echo "Stopping any existing StorPerf container"
+    docker rm -fv $storperf_container
+fi
+
+if [ ! -f job/admin.rc ]
+then
+    ./generate-admin-rc.sh
+fi
+
+if [ ! -d job/carbon ]
+then
+    mkdir job/carbon
+    sudo chown 33:33 job/carbon
+fi
+
+docker run -d --env-file `pwd`/job/admin.rc \
+    -p 5000:5000 \
+    -p 8000:8000 \
+    -v `pwd`/job/carbon:/opt/graphite/storage/whisper \
+    -v `pwd`/../../storperf:/home/opnfv/repos/storperf \
+    --name storperf opnfv/storperf
+
+
diff --git a/ci/start_job.sh b/ci/start_job.sh
new file mode 100755
index 0000000..51f35cb
--- /dev/null
+++ b/ci/start_job.sh
@@ -0,0 +1,29 @@
+#!/bin/bash -x
+##############################################################################
+# Copyright (c) 2015 EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+cat << EOF > body.json
+{
+   "block_sizes": "${BLOCK_SIZE}",
+   "nowarm": "string",
+   "nossd": "string",
+   "deadline": 600,
+   "queue_depths": "${QUEUE_DEPTH}",
+   "workload": "${WORKLOAD}",
+    "metadata": {
+       "disk_type": "SSD",
+      "pod_name": "${POD_NAME}",
+      "scenario_name": "${SCENARIO_NAME}",
+      "storage_node_count": ${CINDER_NODES}
+   }
+}
+EOF
+
+curl -s -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' \
+    -d @body.json http://127.0.0.1:5000/api/v1.0/jobs
\ No newline at end of file
diff --git a/storperf/db/test_results_db.py b/storperf/db/test_results_db.py
index 8636e52..4ee7a52 100644
--- a/storperf/db/test_results_db.py
+++ b/storperf/db/test_results_db.py
@@ -8,9 +8,10 @@
 ##############################################################################
 
 import json
-import requests
 import os
 
+import requests
+
 
 def get_installer_type(logger=None):
     """
@@ -26,7 +27,8 @@ def get_installer_type(logger=None):
     return installer
 
 
-def push_results_to_db(db_url, project, case_name, logger, pod_name,
+def push_results_to_db(db_url, project, case_name,
+                       test_start, test_stop, logger, pod_name,
                        version, scenario, criteria, build_tag, payload):
     """
     POST results to the Result target DB
@@ -34,6 +36,7 @@ def push_results_to_db(db_url, project, case_name, logger, pod_name,
     url = db_url + "/results"
     installer = get_installer_type(logger)
     params = {"project_name": project, "case_name": case_name,
+              "start_date": test_start, "stop_date": test_stop,
               "pod_name": pod_name, "installer": installer,
               "version": version, "scenario": scenario, "criteria": criteria,
               "build_tag": build_tag, "details": payload}
diff --git a/storperf/test_executor.py b/storperf/test_executor.py
index c984175..3c456a6 100644
--- a/storperf/test_executor.py
+++ b/storperf/test_executor.py
@@ -242,7 +242,6 @@ class TestExecutor(object):
 
             self.logger.info("Completed workload %s" % (workload_name))
         self.logger.info("Completed job %s" % (self.job_db.job_id))
-        self._terminated = True
 
         end_time = time.time()
         pod_name = dictionary.get_key_from_dict(self.metadata,
@@ -252,7 +251,7 @@ class TestExecutor(object):
                                                'version',
                                                'Unknown')
         scenario = dictionary.get_key_from_dict(self.metadata,
-                                                'scenario',
+                                                'scenario_name',
                                                 'Unknown')
         build_tag = dictionary.get_key_from_dict(self.metadata,
                                                  'build_tag',
@@ -281,6 +280,8 @@ class TestExecutor(object):
                 test_results_db.push_results_to_db(test_db,
                                                    "storperf",
                                                    "Latency Test",
+                                                   start_time,
+                                                   end_time,
                                                    self.logger,
                                                    pod_name,
                                                    version,
@@ -291,6 +292,8 @@ class TestExecutor(object):
             except:
                 self.logger.exception("Error pushing results into Database")
 
+        self._terminated = True
+
     def execute_on_node(self, workload):
 
         invoker = FIOInvoker()