From 6ce8e3db303e3a32c2a57e0638c1de070a49ef06 Mon Sep 17 00:00:00 2001 From: mbeierl Date: Tue, 17 Jan 2017 14:22:18 -0500 Subject: StorPerf Daily Job Helper script that determines what installer was used. Script to generate RC file for container. Script to determine number of Cinder nodes. Daily job to pull new container, create stack based on number of Cinder nodes, issue a warm up, and then run a series of performance tests against the stack. Deletes stack once complete. JIRA: STORPERF-87 Signed-off-by: mbeierl Change-Id: I551f21ffc85f11b0c0724205e952551022a94f5b --- ci/create_stack.sh | 4 ++-- ci/daily.sh | 42 +++++++++++++++++++++++++++++------------- ci/detect_installer.sh | 22 ++++++++++++++++++++++ ci/generate-admin-rc.sh | 37 ++++++++++++++++++++++++------------- ci/generate-environment.sh | 25 +++++++++++++++++-------- ci/launch_docker_container.sh | 4 +++- 6 files changed, 97 insertions(+), 37 deletions(-) create mode 100755 ci/detect_installer.sh diff --git a/ci/create_stack.sh b/ci/create_stack.sh index 152f1db..68c4874 100755 --- a/ci/create_stack.sh +++ b/ci/create_stack.sh @@ -11,8 +11,8 @@ cat << EOF > body.json { "agent_count": $1, - "agent_image": "Trusty x86_64", - "public_network": "ext-net", + "agent_image": "$3", + "public_network": "$4", "volume_size": $2 } EOF diff --git a/ci/daily.sh b/ci/daily.sh index bdfe746..9ee6afb 100755 --- a/ci/daily.sh +++ b/ci/daily.sh @@ -14,6 +14,21 @@ then WORKSPACE=`pwd` fi +if [ -d $WORKSPACE/ci/job ] +then + sudo rm -rf $WORKSPACE/ci/job +fi + +virtualenv $WORKSPACE/ci/job/storperf_daily_venv +source $WORKSPACE/ci/job/storperf_daily_venv/bin/activate + +pip install --upgrade setuptools +pip install functools32 +pip install pytz +pip install osc_lib +pip install python-openstackclient +pip install python-heatclient + # This is set by Jenkins, but if we are running manually, just use the # current hostname. if [ -z "$NODE_NAME" ] @@ -22,12 +37,10 @@ then fi export POD_NAME=$NODE_NAME -if [ -d $WORKSPACE/ci/job ] -then - sudo rm -rf $WORKSPACE/ci/job -fi sudo find $WORKSPACE/ -name '*.db' -exec rm -fv {} \; +export INSTALLER=`./detect_installer.sh` + $WORKSPACE/ci/generate-admin-rc.sh $WORKSPACE/ci/generate-environment.sh @@ -38,16 +51,20 @@ do done echo "Checking for an existing stack" -STACK_ID=`heat stack-list | grep StorPerfAgentGroup | awk '{print $2}'` +STACK_ID=`openstack stack list | grep StorPerfAgentGroup | awk '{print $2}'` if [ ! -z $STACK_ID ] then - heat stack-delete -y StorPerfAgentGroup + openstack stack delete --yes --wait StorPerfAgentGroup fi -while [ ! -z $STACK_ID ] -do - STACK_ID=`heat stack-list | grep StorPerfAgentGroup | awk '{print $2}'` -done +echo Checking for Ubuntu 14.04 image in Glance +IMAGE=`openstack image list | grep "Trusty x86_64"` +if [ -z $IMAGE ] +then + wget https://cloud-images.ubuntu.com/releases/14.04/release/ubuntu-14.04-server-cloudimg-amd64-disk1.img + openstack image create "Trusty x86_64" --disk-format qcow2 --public \ + --container-format bare --file ubuntu-14.04-server-cloudimg-amd64-disk1.img +fi echo "TEST_DB_URL=http://testresults.opnfv.org/test/api/v1" >> $WORKSPACE/ci/job/admin.rc echo "INSTALLER_TYPE=${INSTALLER}" >> $WORKSPACE/ci/job/admin.rc @@ -60,11 +77,11 @@ do done echo Creating 1:1 stack -$WORKSPACE/ci/create_stack.sh $CINDER_NODES 10 +$WORKSPACE/ci/create_stack.sh $CINDER_NODES 10 "Trusty x86_64" $NETWORK export QUEUE_DEPTH=8 export BLOCK_SIZE=16384 -export WORKLOAD=ws +export WORKLOAD=_warm_up export SCENARIO_NAME="${CINDER_BACKEND}_${WORKLOAD}" WARM_UP=`$WORKSPACE/ci/start_job.sh | awk '/job_id/ {print $2}' | sed 's/"//g'` @@ -77,7 +94,6 @@ do | awk '/Status/ {print $2}' | sed 's/"//g'` done - for WORKLOAD in ws wr rs rr rw do for BLOCK_SIZE in 2048 8192 16384 diff --git a/ci/detect_installer.sh b/ci/detect_installer.sh new file mode 100755 index 0000000..035b7e2 --- /dev/null +++ b/ci/detect_installer.sh @@ -0,0 +1,22 @@ +#!/bin/bash -x +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +which juju 2>/dev/null +if [ $? -eq 0 ] +then + INSTALLER=joid +fi +sudo virsh list --all | grep undercloud >/dev/null +if [ $? -eq 0 ] +then + INSTALLER=apex +fi + +echo $INSTALLER \ No newline at end of file diff --git a/ci/generate-admin-rc.sh b/ci/generate-admin-rc.sh index 6c5b162..efe8b94 100755 --- a/ci/generate-admin-rc.sh +++ b/ci/generate-admin-rc.sh @@ -17,18 +17,29 @@ then mkdir job fi -export OS_AUTH_URL=http://`juju status keystone | grep public | awk '{print $2}'`:5000/v2.0 -export OS_USERNAME=admin -export OS_PASSWORD=openstack -export OS_TENANT_ID=`openstack project list|awk '/admin/ {print $2}'` - -cat << EOF > job/admin.rc -OS_AUTH_URL=$OS_AUTH_URL -OS_USERNAME=$OS_USERNAME -OS_PASSWORD=$OS_PASSWORD -OS_TENANT_ID=$OS_TENANT_ID -OS_TENANT_NAME=admin -OS_PROJECT_NAME=admin -OS_REGION_NAME=RegionOne +export INSTALLER=`./detect_installer.sh` +case $INSTALLER in + joid) + export OS_AUTH_URL=http://`juju status keystone | grep public | awk '{print $2}'`:5000/v2.0 + export OS_USERNAME=admin + export OS_PASSWORD=openstack + cat << EOF > job/openstack.rc +export OS_AUTH_URL=$OS_AUTH_URL +export OS_USERNAME=$OS_USERNAME +export OS_PASSWORD=$OS_PASSWORD +export OS_TENANT_NAME=admin +export OS_PROJECT_NAME=admin EOF + ;; + apex) + INSTALLER_IP=`sudo virsh domifaddr undercloud | grep ipv4 | awk '{print $4}' | cut -d/ -f1` + ;; +esac + +if [ ! -z $INSTALLER_IP ] +then + ../../releng/utils/fetch_os_creds.sh -i $INSTALLER -a $INSTALLER_IP -d job/openstack.rc + echo export OS_PROJECT_NAME=admin >> job/openstack.rc +fi +sed "s/export //" job/openstack.rc > job/admin.rc \ No newline at end of file diff --git a/ci/generate-environment.sh b/ci/generate-environment.sh index 6b44313..c65dc17 100755 --- a/ci/generate-environment.sh +++ b/ci/generate-environment.sh @@ -13,13 +13,12 @@ if [ ! -d job ] then mkdir job fi +ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" -# TODO: This assumes JOID. Need to make this programmatic - -INSTALLER=JOID +export INSTALLER=`./detect_installer.sh` case $INSTALLER in - JOID) + joid) # Determine Cinder backend if [ ! -z "$(juju status | grep ceph)" ] then @@ -31,18 +30,28 @@ case $INSTALLER in CINDER_BACKEND=scaleio JUJU_CHARM=scaleio-sds fi - # Determine how many storage blades we have - CINDER_NODES=`juju status | grep "$JUJU_CHARM/" | wc -l` - # Later, collect info about each node: - # juju status | grep hardware: | grep tags | grep -v virtual + # Determine how many storage blades we have + CINDER_NODES=`juju status | grep "$JUJU_CHARM/" | wc -l` + # Later, collect info about each node: + # juju status | grep hardware: | grep tags | grep -v virtual + NETWORK=ext-net + ;; + apex) + INSTALLER_IP=`sudo virsh domifaddr undercloud | grep ipv4 | awk '{print $4}' | cut -d/ -f1` + CINDER_BACKEND=ceph + sudo scp $ssh_options root@$INSTALLER_IP:/home/stack/instackenv.json job/ + CINDER_NODES=`grep capabilities job/instackenv.json | wc -l` + NETWORK=external ;; *) CINDER_BACKEND=ceph CINDER_NODES=4 + NETWORK=external esac cat << EOF > job/environment.rc export CINDER_BACKEND=$CINDER_BACKEND export CINDER_NODES=$CINDER_NODES export INSTALLER=$INSTALLER +export NETWORK=$NETWORK EOF \ No newline at end of file diff --git a/ci/launch_docker_container.sh b/ci/launch_docker_container.sh index 60d1a07..824fade 100755 --- a/ci/launch_docker_container.sh +++ b/ci/launch_docker_container.sh @@ -1,3 +1,4 @@ +emc.)(165 #!/bin/bash ############################################################################## # Copyright (c) 2016 EMC and others. @@ -29,11 +30,12 @@ then sudo chown 33:33 job/carbon fi +docker pull opnfv/storperf:latest + docker run -d --env-file `pwd`/job/admin.rc \ -p 5000:5000 \ -p 8000:8000 \ -v `pwd`/job/carbon:/opt/graphite/storage/whisper \ - -v `pwd`/../../storperf:/home/opnfv/repos/storperf \ --name storperf opnfv/storperf -- cgit 1.2.3-korg