summaryrefslogtreecommitdiffstats
path: root/tests/ci
diff options
context:
space:
mode:
authorzhihui wu <wu.zhihui1@zte.com.cn>2017-08-16 15:56:17 +0800
committerzhihui wu <wu.zhihui1@zte.com.cn>2017-08-18 16:01:39 +0800
commit70fc96ee9d7785a46de6847cd8bbb54a26bff3eb (patch)
treeda9fedcebcffc48a9ea7373deb228f1d6c02f835 /tests/ci
parent84ea7388ecb34034f7c6886886c46b00afc82c2c (diff)
refactor storperf scripts structure
- Run qtip and storperf containers by docker-compose - Cope the current patch changes into qtip container for verification - Run the script run_storage_qpi.sh for storperf test Change-Id: Ic140fb7531d270b9c62bcd72f5dbc13917534daf Signed-off-by: zhihui wu <wu.zhihui1@zte.com.cn>
Diffstat (limited to 'tests/ci')
-rw-r--r--tests/ci/experimental.sh2
-rwxr-xr-xtests/ci/run_storage_qpi.sh14
-rwxr-xr-xtests/ci/storperf/containers.sh49
-rw-r--r--tests/ci/storperf/default_job.json15
-rw-r--r--tests/ci/storperf/default_stack.json6
-rwxr-xr-xtests/ci/storperf/prepare.sh71
-rw-r--r--tests/ci/storperf/qtip-storperf-docker-compose.yaml45
-rwxr-xr-xtests/ci/storperf/start_job.sh110
-rw-r--r--tests/ci/storperf/storperf_requirements.txt5
-rwxr-xr-xtests/ci/verify_storage.sh62
10 files changed, 378 insertions, 1 deletions
diff --git a/tests/ci/experimental.sh b/tests/ci/experimental.sh
index c5572059..379e8602 100644
--- a/tests/ci/experimental.sh
+++ b/tests/ci/experimental.sh
@@ -1,3 +1,3 @@
#!/usr/bin/env bash
-source interation/storperf/run_ci.sh
+source tests/ci/verify_storage.sh
diff --git a/tests/ci/run_storage_qpi.sh b/tests/ci/run_storage_qpi.sh
new file mode 100755
index 00000000..d4859d56
--- /dev/null
+++ b/tests/ci/run_storage_qpi.sh
@@ -0,0 +1,14 @@
+#! /bin/bash
+##############################################################################
+# Copyright (c) 2017 ZTE and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+#TODO: These will be replaced by qtip cli
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+$script_dir/storperf/prepare.sh
+$script_dir/storperf/start_job.sh \ No newline at end of file
diff --git a/tests/ci/storperf/containers.sh b/tests/ci/storperf/containers.sh
new file mode 100755
index 00000000..1c48a62e
--- /dev/null
+++ b/tests/ci/storperf/containers.sh
@@ -0,0 +1,49 @@
+#! /bin/bash
+##############################################################################
+# Copyright (c) 2017 ZTE and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+clean_containers()
+{
+ echo "Cleanup existing qtip and storperf containers"
+ docker-compose -f qtip-storperf-docker-compose.yaml down
+
+ for name in qtip storperf-master storperf-swaggerui storperf-httpfrontend storperf-reporting
+ do
+ container=$(docker ps -a | grep "opnfv/${name}:${DOCKER_TAG}" | awk '{print $1}')
+ if [[ ! -z "$container" ]]; then
+ echo "Removing any existing $name container"
+ docker rm -fv $container
+ fi
+
+ if [[ $(docker images opnfv/${name}:${DOCKER_TAG} | wc -l) -gt 1 ]]; then
+ echo "QTIP: Removing docker image opnfv/$name :$DOCKER_TAG..."
+ docker rmi -f opnfv/${name}:${DOCKER_TAG}
+ fi
+ done
+}
+
+
+launch_containers()
+{
+ echo "Launch new qtip and storperf containers"
+ docker-compose -f qtip-storperf-docker-compose.yaml pull
+ docker-compose -f qtip-storperf-docker-compose.yaml up -d
+
+ echo "Waiting for StorPerf to become active"
+
+ while [ $(curl -s -o /dev/null -I -w "%{http_code}" -X GET http://127.0.0.1:5000/api/v1.0/configurations) != "200" ]
+ do
+ sleep 1
+ done
+
+ container_id=$(docker ps | grep "opnfv/qtip:${DOCKER_TAG}" | awk '{print $1}' | head -1)
+ if [[ -z "$container_id" ]]; then
+ echo "The container opnfv/qtip with ID=${container_id} has not been properly started. Exiting..."
+ exit 1
+ fi
+}
diff --git a/tests/ci/storperf/default_job.json b/tests/ci/storperf/default_job.json
new file mode 100644
index 00000000..4e5e2dfd
--- /dev/null
+++ b/tests/ci/storperf/default_job.json
@@ -0,0 +1,15 @@
+{
+ "block_sizes": "1024",
+ "deadline": 30,
+ "steady_state_samples": 2,
+ "queue_depths": "1",
+ "workload": "rw",
+ "metadata": {
+ "disk_type": "HDD",
+ "scenario_name": "",
+ "storage_node_count": 2,
+ "version": "",
+ "build_tag": "",
+ "test_case": "snia_steady_state"
+ }
+}
diff --git a/tests/ci/storperf/default_stack.json b/tests/ci/storperf/default_stack.json
new file mode 100644
index 00000000..79d8fc46
--- /dev/null
+++ b/tests/ci/storperf/default_stack.json
@@ -0,0 +1,6 @@
+{
+ "agent_count": 2,
+ "agent_image": "Ubuntu 16.04 x86_64",
+ "public_network": "external",
+ "volume_size": 2
+}
diff --git a/tests/ci/storperf/prepare.sh b/tests/ci/storperf/prepare.sh
new file mode 100755
index 00000000..fe0708ad
--- /dev/null
+++ b/tests/ci/storperf/prepare.sh
@@ -0,0 +1,71 @@
+#! /bin/bash
+##############################################################################
+# Copyright (c) 2017 ZTE and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+if [[ -z $WORKSPACE ]];then
+ WORKSPACE=`pwd`
+fi
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+pip install -r $script_dir/storperf_requirements.txt
+
+delete_storperf_stack()
+{
+ echo "Checking for storperf stack"
+ STACK_ID=`openstack stack list | grep StorPerfAgentGroup | awk '{print $2}'`
+ if [[ ! -z $STACK_ID ]];then
+ echo "Deleting any existing storperf stack"
+ openstack stack delete --yes --wait StorPerfAgentGroup
+ fi
+}
+
+load_ubuntu_image()
+{
+ echo "Checking for Ubuntu 16.04 image in Glance"
+ IMAGE=`openstack image list | grep "Ubuntu 16.04 x86_64" | awk '{print $2}'`
+ if [[ -z "$IMAGE" ]];then
+ cd $WORKSPACE
+ if [[ ! -f ubuntu-16.04-server-cloudimg-amd64-disk1.img ]];then
+ echo "Download Ubuntu 16.04 image"
+ wget -q https://cloud-images.ubuntu.com/releases/16.04/release/ubuntu-16.04-server-cloudimg-amd64-disk1.img
+ wget -q https://cloud-images.ubuntu.com/releases/16.04/release/MD5SUMS
+ checksum=$(cat ./MD5SUMS |grep ubuntu-16.04-server-cloudimg-amd64-disk1.img | md5sum -c)
+ if [[ $checksum =~ 'FAILED' ]];then
+ echo "Check image md5sum failed. Exit!"
+ exit 1
+ fi
+ fi
+
+ echo "Creating openstack image Ubuntu 16.04"
+ openstack image create "Ubuntu 16.04 x86_64" --disk-format qcow2 --public \
+ --container-format bare --file $WORKSPACE/ubuntu-16.04-server-cloudimg-amd64-disk1.img
+ else
+ openstack image show "Ubuntu 16.04 x86_64"
+ fi
+}
+
+create_storperf_flavor()
+{
+ echo "Checking for StorPerf flavor"
+ openstack flavor delete storperf
+ FLAVOR=`openstack flavor list | grep "storperf" | awk '{print $2}'`
+ if [[ -z "$FLAVOR" ]];then
+ openstack flavor create storperf \
+ --id auto \
+ --ram 2048 \
+ --disk 4 \
+ --vcpus 2
+ else
+ openstack flavor show storperf
+ fi
+}
+
+delete_storperf_stack
+load_ubuntu_image
+create_storperf_flavor
diff --git a/tests/ci/storperf/qtip-storperf-docker-compose.yaml b/tests/ci/storperf/qtip-storperf-docker-compose.yaml
new file mode 100644
index 00000000..fc837139
--- /dev/null
+++ b/tests/ci/storperf/qtip-storperf-docker-compose.yaml
@@ -0,0 +1,45 @@
+##############################################################################
+# Copyright (c) 2017 Dell EMC, ZTE and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+version: '2'
+services:
+
+ storperf-master:
+ container_name: "storperf-master"
+ image: opnfv/storperf-master:${DOCKER_TAG}
+ ports:
+ - "8000:8000"
+ env_file: ${ENV_FILE}
+
+ storperf-reporting:
+ container_name: "storperf-reporting"
+ image: opnfv/storperf-reporting:${DOCKER_TAG}
+ ports:
+ - "5080:5000"
+
+ storperf-swaggerui:
+ container_name: "storperf-swaggerui"
+ image: "schickling/swagger-ui"
+
+ storperf-httpfrontend:
+ container_name: "storperf-httpfrontend"
+ image: opnfv/storperf-httpfrontend:${DOCKER_TAG}
+ ports:
+ - "5000:5000"
+ links:
+ - storperf-master
+ - storperf-reporting
+ - storperf-swaggerui
+
+ qtip:
+ container_name: qtip
+ image: opnfv/qtip:${DOCKER_TAG}
+ env_file: ${ENV_FILE}
+ links:
+ - storperf-httpfrontend \ No newline at end of file
diff --git a/tests/ci/storperf/start_job.sh b/tests/ci/storperf/start_job.sh
new file mode 100755
index 00000000..15785036
--- /dev/null
+++ b/tests/ci/storperf/start_job.sh
@@ -0,0 +1,110 @@
+#! /bin/bash
+##############################################################################
+# Copyright (c) 2017 ZTE and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+usage(){
+ echo "usage: $0 -s <stack_json_path> -j <job_json_path>" >&2
+}
+
+#Get options
+while getopts ":s:j:he" optchar; do
+ case "${optchar}" in
+ s) stack_json=${OPTARG} ;;
+ j) job_json=${OPTARG} ;;
+ h) usage
+ exit 0
+ ;;
+ *) echo "Non-option argument: '-${OPTARG}'" >&2
+ usage
+ exit 2
+ ;;
+ esac
+done
+
+# See https://stackoverflow.com/questions/59895/getting-the-source-directory-of-a-bash-script-from-within
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+stack_json=${stack_json:-"$script_dir/default_stack.json"}
+job_json=${job_json:-"$script_dir/default_job.json"}
+
+if [[ -z $WORKSPACE ]];then
+ WORKSPACE=`pwd`
+fi
+
+nova_vm_mapping()
+{
+ rm $WORKSPACE/nova_vm.json
+ openstack server list --name storperf-agent -c ID -c Host --long -f json > $WORKSPACE/nova_vm.json
+
+ echo ==========================================================================
+ echo "Vms vs. Compute nodes"
+ cat $WORKSPACE/nova_vm.json
+ echo ==========================================================================
+}
+
+echo ==========================================================================
+echo "Start to create storperf stack"
+cat ${stack_json} 1>&2
+echo ==========================================================================
+
+curl -X POST --header 'Content-Type: application/json' \
+ --header 'Accept: application/json' -d @${stack_json} \
+ 'http://storperf-httpfrontend:5000/api/v1.0/configurations'
+
+nova_vm_mapping
+
+echo
+echo ==========================================================================
+echo "Start to run storperf test"
+cat ${job_json} 1>&2
+echo ==========================================================================
+
+JOB=$(curl -s -X POST --header 'Content-Type: application/json' \
+ --header 'Accept: application/json' \
+ -d @${job_json} 'http://storperf-httpfrontend:5000/api/v1.0/jobs' | \
+ awk '/job_id/ {print $2}' | sed 's/"//g')
+
+echo "JOB ID: $JOB"
+if [[ -z "$JOB" ]]; then
+ echo "Oops, JOB ID is empty!"
+else
+ echo "checking job status..."
+ curl -s -X GET "http://storperf-httpfrontend:5000/api/v1.0/jobs?id=$JOB&type=status" \
+ -o $WORKSPACE/status.json
+
+ cat $WORKSPACE/status.json
+
+ JOB_STATUS=`cat $WORKSPACE/status.json | awk '/Status/ {print $2}' | cut -d\" -f2`
+
+ while [ "$JOB_STATUS" != "Completed" ]
+ do
+ sleep 180
+ mv $WORKSPACE/status.json $WORKSPACE/old-status.json
+ curl -s -X GET "http://storperf-httpfrontend:5000/api/v1.0/jobs?id=$JOB&type=status" \
+ -o $WORKSPACE/status.json
+ JOB_STATUS=`cat $WORKSPACE/status.json | awk '/Status/ {print $2}' | cut -d\" -f2`
+ diff $WORKSPACE/status.json $WORKSPACE/old-status.json >/dev/null
+ if [ $? -eq 1 ]
+ then
+ cat $WORKSPACE/status.json
+ fi
+ done
+
+ echo
+ echo "Storperf test completed!"
+
+ echo ==========================================================================
+ echo Final report
+ echo ==========================================================================
+ curl -s -X GET "http://storperf-httpfrontend:5000/api/v1.0/jobs?id=$JOB&type=metadata" \
+ -o $WORKSPACE/report.json
+ cat $WORKSPACE/report.json
+fi
+
+echo "Deleting stack for cleanup"
+curl -s -X DELETE --header 'Accept: application/json' 'http://storperf-httpfrontend:5000/api/v1.0/configurations' \ No newline at end of file
diff --git a/tests/ci/storperf/storperf_requirements.txt b/tests/ci/storperf/storperf_requirements.txt
new file mode 100644
index 00000000..4a279ae5
--- /dev/null
+++ b/tests/ci/storperf/storperf_requirements.txt
@@ -0,0 +1,5 @@
+functools32==3.2.3.post2
+pytz==2016.10
+osc_lib==1.3.0
+python-openstackclient==3.7.0
+python-heatclient==1.7.0
diff --git a/tests/ci/verify_storage.sh b/tests/ci/verify_storage.sh
new file mode 100755
index 00000000..00120211
--- /dev/null
+++ b/tests/ci/verify_storage.sh
@@ -0,0 +1,62 @@
+#! /bin/bash -x
+##############################################################################
+# Copyright (c) 2017 ZTE and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+WORKSPACE=${WORKSPACE:=`pwd`}
+
+export DOCKER_TAG=${DOCKER_TAG:-latest}
+export ENV_FILE=$WORKSPACE/env_file
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+case $INSTALLER_TYPE in
+ apex)
+ INSTALLER_IP=`sudo virsh domifaddr undercloud | grep ipv4 | awk '{print $4}' | cut -d/ -f1`
+ ;;
+esac
+
+git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng
+$WORKSPACE/releng/utils/fetch_os_creds.sh -i $INSTALLER_TYPE -a $INSTALLER_IP -d $WORKSPACE/openrc
+
+grep "export" $WORKSPACE/openrc | sed "s/export //" > $WORKSPACE/env_file
+echo "INSTALLER_TYPE=$INSTALLER_TYPE" >> $WORKSPACE/env_file
+echo "INSTALLER_IP=$INSTALLER_IP" >> $WORKSPACE/env_file
+echo "NODE_NAME=$NODE_NAME" >> $WORKSPACE/env_file
+echo "SCENARIO=$DEPLOY_SCENARIO" >> $WORKSPACE/env_file
+echo "TESTAPI_URL=$TESTAPI_URL" >> $WORKSPACE/env_file
+echo "DOCKER_TAG=$DOCKER_TAG" >> $WORKSPACE/env_file
+
+echo "--------------------ENV_FILE----------------------------"
+cat $WORKSPACE/env_file
+echo "--------------------------------------------------------"
+
+source $script_dir/storperf/containers.sh
+cd $script_dir/storperf
+clean_containers
+launch_containers
+
+container_id=$(docker ps | grep "opnfv/qtip:${DOCKER_TAG}" | awk '{print $1}' | head -1)
+
+if [[ "$INSTALLER_TYPE" == "apex" ]];then
+ if [ -f /root/.ssh/id_rsa ]; then
+ sudo chmod 600 /root/.ssh/id_rsa
+ sudo docker cp /root/.ssh/id_rsa ${container_id}:/root/.ssh/
+ fi
+fi
+
+QTIP_REPO=/home/opnfv/repos/qtip
+
+echo "QTIP: Copying current submit patch to the container ${container_id}"
+cd $WORKSPACE
+docker cp . ${container_id}:${QTIP_REPO}
+docker exec ${container_id} bash -c "cd ${QTIP_REPO} && pip install -U -e ."
+
+docker exec -t ${container_id} bash -c "bash ${QTIP_REPO}/tests/ci/run_storage_qpi.sh"
+
+echo "Verify storage done!"
+exit 0 \ No newline at end of file