Merge "run storperf test by using these scripts"

9 files changed, 353 insertions, 0 deletions

diff --git a/integration/storperf/README.md b/integration/storperf/README.md
new file mode 100644
index 00000000..137e15be
--- /dev/null
+++ b/integration/storperf/README.md
@@ -0,0 +1,13 @@
+# usage
+
+Please make sure pip, docker and docker-compose are installer on your environment.
+
+```
+storperf.sh -t installer_type -i installer_ip -s stack_json_file -j job_json_file
+
+options:
+    -t : installer type. For now only supports Apex.
+    -i : installer ip address.
+    -s : Stack configuration json file. If not given, default_stack.json will be used.
+    -j : Storperf job configuration json file. If not given, default_job.json will be used.
+```
\ No newline at end of file
diff --git a/integration/storperf/default_job.json b/integration/storperf/default_job.json
new file mode 100644
index 00000000..dd42dcee
--- /dev/null
+++ b/integration/storperf/default_job.json
@@ -0,0 +1,15 @@
+{
+   "block_sizes": "1024",
+   "deadline": 30,
+   "steady_state_samples": 10,
+   "queue_depths": "1",
+   "workload": "wr,rr,rw",
+   "metadata": {
+        "disk_type": "HDD",
+        "scenario_name": "none",
+        "storage_node_count": 2,
+        "version": "",
+        "build_tag": "",
+        "test_case": "snia_steady_state"
+   }
+}
\ No newline at end of file
diff --git a/integration/storperf/default_stack.json b/integration/storperf/default_stack.json
new file mode 100644
index 00000000..79d8fc46
--- /dev/null
+++ b/integration/storperf/default_stack.json
@@ -0,0 +1,6 @@
+{
+  "agent_count": 2,
+  "agent_image": "Ubuntu 16.04 x86_64",
+  "public_network": "external",
+  "volume_size": 2
+}
diff --git a/integration/storperf/openstack.sh b/integration/storperf/openstack.sh
new file mode 100755
index 00000000..011aadb7
--- /dev/null
+++ b/integration/storperf/openstack.sh
@@ -0,0 +1,58 @@
+#! /bin/bash
+##############################################################################
+# Copyright (c) 2017 ZTE and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+delete_storperf_stack()
+{
+    echo "Checking for an existing stack"
+    STACK_ID=`openstack stack list | grep StorPerfAgentGroup | awk '{print $2}'`
+    if [[ ! -z $STACK_ID ]];then
+        openstack stack delete --yes --wait StorPerfAgentGroup
+    fi
+}
+
+load_ubuntu_image()
+{
+    echo "Checking for Ubuntu 16.04 image in Glance"
+    IMAGE=`openstack image list | grep "Ubuntu 16.04 x86_64"`
+    if [[ -z "$IMAGE" ]];then
+        cd $WORKSPACE
+        if [[ ! -f ubuntu-16.04-server-cloudimg-amd64-disk1.img ]];then
+            echo "download Ubuntu 16.04 image"
+            wget https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img
+            wget https://cloud-images.ubuntu.com/releases/16.04/release/MD5SUMS
+            checksum=$(cat ./MD5SUMS |grep ubuntu-16.04-server-cloudimg-amd64-disk1.img | md5sum -c)
+            if [[ $checksum =~ 'FAILED' ]];then
+                echo "Check image md5sum failed. Exit!"
+                exit 1
+            fi
+        fi
+
+        echo "Create openstack image Ubuntu 16.04"
+        openstack image create "Ubuntu 16.04 x86_64" --disk-format qcow2 --public \
+        --container-format bare --file $WORKSPACE/ubuntu-16.04-server-cloudimg-amd64-disk1.img
+    fi
+
+    openstack image show "Ubuntu 16.04 x86_64"
+}
+
+
+create_storperf_flavor()
+{
+    echo "Checking for StorPerf flavor"
+    openstack flavor delete storperf
+    FLAVOR=`openstack flavor list | grep "storperf"`
+    if [[ -z "$FLAVOR" ]];then
+        openstack flavor create storperf \
+            --id auto \
+            --ram 2048 \
+            --disk 4 \
+            --vcpus 2
+    fi
+    openstack flavor show storperf
+}
diff --git a/integration/storperf/start_job.sh b/integration/storperf/start_job.sh
new file mode 100755
index 00000000..6ac95ec0
--- /dev/null
+++ b/integration/storperf/start_job.sh
@@ -0,0 +1,91 @@
+#! /bin/bash
+##############################################################################
+# Copyright (c) 2017 ZTE and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+usage(){
+   echo "usage: $0 -s <stack_json_path> -j <job_json_path>" >&2
+}
+
+#Get options
+while getopts ":s:j:he" optchar; do
+   case "${optchar}" in
+       s) stack_json=${OPTARG} ;;
+       j) job_json=${OPTARG} ;;
+       h) usage
+          exit 0
+          ;;
+       *) echo "Non-option argument: '-${OPTARG}'" >&2
+          usage
+          exit 2
+          ;;
+   esac
+done
+
+if [[ -z $WORKSPACE ]];then
+    WORKSPACE=`pwd`
+fi
+
+echo ==========================================================================
+echo "Start to create storperf stack"
+cat ${stack_json} 1>&2
+echo ==========================================================================
+
+curl -X POST --header 'Content-Type: application/json' \
+     --header 'Accept: application/json' -d @${stack_json} \
+     'http://127.0.0.1:5000/api/v1.0/configurations'
+
+echo
+echo ==========================================================================
+echo "Start to run storperf test"
+cat ${job_json} 1>&2
+echo ==========================================================================
+
+JOB=$(curl -s -X POST --header 'Content-Type: application/json' \
+    --header 'Accept: application/json' \
+    -d @${job_json} 'http://127.0.0.1:5000/api/v1.0/jobs' | \
+    awk '/job_id/ {print $2}' | sed 's/"//g')
+
+echo "JOB ID: $JOB"
+if [[ -z "$JOB" ]]; then
+    echo "Oops, JOB ID is empty!"
+else
+    echo "Loop: check job status"
+    curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$JOB&type=status" \
+        -o $WORKSPACE/status.json
+
+    JOB_STATUS=`cat $WORKSPACE/status.json | awk '/Status/ {print $2}' | cut -d\" -f2`
+    while [ "$JOB_STATUS" != "Completed" ]
+    do
+        sleep 300
+        mv $WORKSPACE/status.json $WORKSPACE/old-status.json
+        curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$JOB&type=status" \
+            -o $WORKSPACE/status.json
+        JOB_STATUS=`cat $WORKSPACE/status.json | awk '/Status/ {print $2}' | cut -d\" -f2`
+        diff $WORKSPACE/status.json $WORKSPACE/old-status.json >/dev/null
+        if [ $? -eq 1 ]
+        then
+            cat $WORKSPACE/status.json
+        fi
+    done
+
+    echo ==========================================================================
+    echo "Storperf test completed!"
+    echo ==========================================================================
+
+    curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$JOB&type=metadata" \
+    -o $WORKSPACE/report.json
+
+    echo ==========================================================================
+    echo Final report
+    echo ==========================================================================
+    cat $WORKSPACE/report.json
+fi
+
+echo "Deleting stack for cleanup"
+curl -s -X DELETE --header 'Accept: application/json' 'http://127.0.0.1:5000/api/v1.0/configurations'
+
diff --git a/integration/storperf/storperf-docker-compose.yaml b/integration/storperf/storperf-docker-compose.yaml
new file mode 100644
index 00000000..ec121802
--- /dev/null
+++ b/integration/storperf/storperf-docker-compose.yaml
@@ -0,0 +1,43 @@
+##############################################################################
+# Copyright (c) 2017 Dell EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+version: '2'
+services:
+
+    storperf-master:
+        container_name: "storperf-master"
+        image: opnfv/storperf-master
+        ports:
+            - "8000:8000"
+        env_file: ${ENV_FILE}
+        volumes:
+            - ${CARBON_DIR}:/opt/graphite/storage/whisper
+            - ./storperf-master/:/home/opnfv/repos/storperf
+
+    storperf-reporting:
+        container_name: "storperf-reporting"
+        image: opnfv/storperf-reporting
+        ports:
+            - "5080:5000"
+        volumes:
+            - ./storperf-reporting/:/home/opnfv/storperf-reporting
+
+    storperf-swaggerui:
+        container_name: "storperf-swaggerui"
+        image: "schickling/swagger-ui"
+
+    storperf-httpfrontend:
+        container_name: "storperf-httpfrontend"
+        image: opnfv/storperf-httpfrontend
+        ports:
+            - "5000:5000"
+        links:
+            - storperf-master
+            - storperf-reporting
+            - storperf-swaggerui
\ No newline at end of file
diff --git a/integration/storperf/storperf.sh b/integration/storperf/storperf.sh
new file mode 100755
index 00000000..7a0253df
--- /dev/null
+++ b/integration/storperf/storperf.sh
@@ -0,0 +1,86 @@
+#! /bin/bash
+##############################################################################
+# Copyright (c) 2017 ZTE and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+usage(){
+   echo "usage: $0 -t <installer_type> -i <installer_ip> -s <stack_json_path> -j <job_json_path> " >&2
+}
+
+#Get options
+while getopts ":t:i:s:j:he" optchar; do
+   case "${optchar}" in
+       t) installer_type=${OPTARG} ;;
+       i) installer_ip=${OPTARG} ;;
+       s) stack_json=${OPTARG} ;;
+       j) job_json=${OPTARG} ;;
+       h) usage
+          exit 0
+          ;;
+       *) echo "Non-option argument: '-${OPTARG}'" >&2
+          usage
+          exit 2
+          ;;
+   esac
+done
+
+if [[ -z $WORKSPACE ]];then
+    WORKSPACE=`pwd`
+fi
+
+#set vars from env if not provided by user as options
+installer_type=${installer_type:-$INSTALLER_TYPE}
+installer_ip=${installer_ip:-$INSTALLER_IP}
+stack_json=${stack_json:-"$WORKSPACE/default_stack.json"}
+job_json=${job_json:-"$WORKSPACE/default_job.json"}
+
+source ./openstack.sh
+source ./storperf_docker.sh
+
+git clone --depth 1 https://gerrit.opnfv.org/gerrit/storperf $WORKSPACE/storperf
+git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng
+
+virtualenv $WORKSPACE/storperf_venv
+source $WORKSPACE/storperf_venv/bin/activate
+
+pip install -r ./storperf_requirements.txt
+
+$WORKSPACE/releng/utils/fetch_os_creds.sh -i ${installer_type} -a ${installer_ip} -d $WORKSPACE/openrc
+source $WORKSPACE/openrc
+
+grep "export" $WORKSPACE/openrc | sed "s/export //"  > $WORKSPACE/admin.rc
+echo "INSTALLER_TYPE=${installer_type}" >> $WORKSPACE/admin.rc
+export ENV_FILE=$WORKSPACE/admin.rc
+
+if [[ ! -d $WORKSPACE/carbon ]];then
+    mkdir -p $WORKSPACE/carbon
+    sudo chown 33:33 $WORKSPACE/carbon
+fi
+export CARBON_DIR=$WORKSPACE/carbon/
+
+delete_storperf_stack
+load_ubuntu_image
+create_storperf_flavor
+
+cd $WORKSPACE/storperf/docker
+cp $WORKSPACE/storperf-docker-compose.yaml ./
+echo "Clean existing storperf containers"
+clean_storperf_container
+echo "Launch new storperf containers"
+launch_storperf_container
+
+cd $WORKSPACE
+./start_job.sh -s $stack_json -j $job_json
+
+echo "Clean up environment"
+cd $WORKSPACE/storperf/docker
+clean_storperf_container
+openstack flavor delete storperf
+openstack image delete "Ubuntu 16.04 x86_64"
+
+echo "Done!"
+
diff --git a/integration/storperf/storperf_docker.sh b/integration/storperf/storperf_docker.sh
new file mode 100755
index 00000000..629a4edb
--- /dev/null
+++ b/integration/storperf/storperf_docker.sh
@@ -0,0 +1,35 @@
+#! /bin/bash
+##############################################################################
+# Copyright (c) 2017 ZTE and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+clean_storperf_container()
+{
+    docker-compose -f storperf-docker-compose.yaml down
+
+    for container_name in storperf storperf-master storperf-swaggerui storperf-httpfrontend storperf-reporting
+    do
+        container=`docker ps -a -q -f name=$container_name`
+        if [[ ! -z $container ]];then
+            echo "Stopping any existing $container_name container"
+            docker rm -fv $container
+        fi
+    done
+}
+
+
+launch_storperf_container()
+{
+    docker-compose -f storperf-docker-compose.yaml up -d
+
+    echo "Waiting for StorPerf to become active"
+
+    while [ $(curl -s -o /dev/null -I -w "%{http_code}" -X GET http://127.0.0.1:5000/api/v1.0/configurations) != "200" ]
+    do
+        sleep 1
+    done
+}
\ No newline at end of file
diff --git a/integration/storperf/storperf_requirements.txt b/integration/storperf/storperf_requirements.txt
new file mode 100644
index 00000000..4f8b07c5
--- /dev/null
+++ b/integration/storperf/storperf_requirements.txt
@@ -0,0 +1,6 @@
+setuptools==33.1.1
+functools32==3.2.3.post2
+pytz==2016.10
+osc_lib==1.3.0
+python-openstackclient==3.7.0
+python-heatclient==1.7.0
\ No newline at end of file