From 2c95b01efa53c7e7d4ea7d73fde0da6099938e50 Mon Sep 17 00:00:00 2001 From: mbeierl Date: Thu, 29 Jun 2017 14:16:39 -0400 Subject: Docker Compose Updates Adds docker-compose directory with content to start container suite using nginx as front and and SwaggerUI in its own container. Updates the documentation to reflect new method of starting StorPerf. Change-Id: I469676e330dcad6c968b90133df8866b69eb5ea1 JIRA: STORPERF-181 Signed-off-by: mbeierl --- ci/create_stack.sh | 1 + ci/daily.sh | 9 +- ci/launch_docker_container.sh | 44 +++---- ci/remove_docker_container.sh | 30 +++++ ci/start_job.sh | 2 + docker-compose/docker-compose.yaml | 25 ++++ docker-compose/nginx.conf | 34 ++++++ docs/testing/user/installation.rst | 228 +++++++++++++++++++++++++++++-------- docs/testing/user/introduction.rst | 153 +++++++++---------------- docs/testing/user/test-usage.rst | 2 + rest_server.py | 3 +- 11 files changed, 350 insertions(+), 181 deletions(-) create mode 100755 ci/remove_docker_container.sh create mode 100644 docker-compose/docker-compose.yaml create mode 100644 docker-compose/nginx.conf diff --git a/ci/create_stack.sh b/ci/create_stack.sh index 68c4874..a0be65c 100755 --- a/ci/create_stack.sh +++ b/ci/create_stack.sh @@ -21,3 +21,4 @@ curl -X POST --header 'Content-Type: application/json' \ --header 'Accept: application/json' -d @body.json \ 'http://127.0.0.1:5000/api/v1.0/configurations' +rm body.json \ No newline at end of file diff --git a/ci/daily.sh b/ci/daily.sh index c42fe64..26d5fea 100755 --- a/ci/daily.sh +++ b/ci/daily.sh @@ -27,8 +27,6 @@ pip install osc_lib==1.3.0 pip install python-openstackclient==3.7.0 pip install python-heatclient==1.7.0 -sudo find $WORKSPACE/ -name '*.db' -exec rm -fv {} \; - $WORKSPACE/ci/generate-admin-rc.sh echo "TEST_DB_URL=http://testresults.opnfv.org/test/api/v1" >> $WORKSPACE/ci/job/admin.rc $WORKSPACE/ci/generate-environment.sh @@ -42,12 +40,12 @@ done < $WORKSPACE/ci/job/admin.rc export AGENT_COUNT=${AGENT_COUNT:-$CINDER_NODES} export BLOCK_SIZES=${BLOCK_SIZES:-1024,16384} +export STEADY_STATE_SAMPLES=${STEADY_STATE_SAMPLES:-10} export DEADLINE=${DEADLINE:-`expr $STEADY_STATE_SAMPLES \* 3`} export DISK_TYPE=${DISK_TYPE:-unspecified} export QUEUE_DEPTHS=${QUEUE_DEPTHS:-1,4} export POD_NAME=${NODE_NAME:-`hostname`} export SCENARIO_NAME=${DEPLOY_SCENARIO:-none} -export STEADY_STATE_SAMPLES=${STEADY_STATE_SAMPLES:-10} export TEST_CASE=${TEST_CASE:-snia_steady_state} export VERSION=`echo ${BUILD_TAG#*daily-} | cut -d- -f1` export VOLUME_SIZE=${VOLUME_SIZE:-2} @@ -58,13 +56,13 @@ echo Environment env | sort echo ========================================================================== +$WORKSPACE/ci/remove_docker_container.sh $WORKSPACE/ci/delete_stack.sh $WORKSPACE/ci/create_glance_image.sh $WORKSPACE/ci/create_storperf_flavor.sh $WORKSPACE/ci/launch_docker_container.sh $WORKSPACE/ci/create_stack.sh $AGENT_COUNT $VOLUME_SIZE "Ubuntu 16.04 x86_64" $NETWORK - export WORKLOAD=_warm_up,$WORKLOADS export BLOCK_SIZE=$BLOCK_SIZES export QUEUE_DEPTH=$QUEUE_DEPTHS @@ -99,7 +97,8 @@ curl -s -X DELETE --header 'Accept: application/json' 'http://127.0.0.1:5000/api curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$JOB&type=metadata" \ -o $WORKSPACE/ci/job/report.json -docker rm -f storperf +$WORKSPACE/ci/remove_docker_container.sh + sudo rm -rf $WORKSPACE/ci/job/carbon echo ========================================================================== diff --git a/ci/launch_docker_container.sh b/ci/launch_docker_container.sh index ec9c4d8..2dfde93 100755 --- a/ci/launch_docker_container.sh +++ b/ci/launch_docker_container.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x ############################################################################## # Copyright (c) 2016 EMC and others. # @@ -9,42 +9,28 @@ ############################################################################## cd `dirname $0` +ci=`pwd` -storperf_container=`docker ps -a -q -f name=storperf` +cd ${ci}/../docker-compose -if [ ! -z $storperf_container ] -then - echo "Stopping any existing StorPerf container" - docker rm -fv $storperf_container -fi - -if [ ! -f job/admin.rc ] -then - ./generate-admin-rc.sh -fi +export TAG=${DOCKER_TAG:-latest} +export ENV_FILE=${ci}/job/admin.rc +export CARBON_DIR=${ci}/job/carbon/ -if [ ! -d job/carbon ] +if [ ! -d ${ci}/job/carbon ] then - mkdir job/carbon - sudo chown 33:33 job/carbon + mkdir ${ci}/job/carbon + sudo chown 33:33 ${ci}/job/carbon fi -if [ -z $DOCKER_TAG ] -then - DOCKER_TAG=latest -fi - -docker pull opnfv/storperf:$DOCKER_TAG - -docker run -d --env-file `pwd`/job/admin.rc \ - -p 5000:5000 \ - -p 8000:8000 \ - -v `pwd`/job/carbon:/opt/graphite/storage/whisper \ - --name storperf opnfv/storperf -# -v `pwd`/../../storperf:/home/opnfv/repos/storperf \ +docker-compose -f ../docker-compose/docker-compose.yaml up -d echo "Waiting for StorPerf to become active" -while [ $(curl -X GET 'http://127.0.0.1:5000/api/v1.0/configurations' > /dev/null 2>&1;echo $?) -ne 0 ] +curl -X GET 'http://127.0.0.1:5000/api/v1.0/configurations' > test.html 2>&1 +while [ `grep 'agent_count' test.html | wc -l` == "0" ] do sleep 1 + curl -X GET 'http://127.0.0.1:5000/api/v1.0/configurations' > test.html 2>&1 done + +rm -f test.html diff --git a/ci/remove_docker_container.sh b/ci/remove_docker_container.sh new file mode 100755 index 0000000..f424110 --- /dev/null +++ b/ci/remove_docker_container.sh @@ -0,0 +1,30 @@ +#!/bin/bash -x +############################################################################## +# Copyright (c) 2016 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +cd `dirname $0` +ci=`pwd` + +cd ${ci}/../docker-compose + +export TAG=${DOCKER_TAG:-latest} +export ENV_FILE=${ci}/job/admin.rc +export CARBON_DIR=${ci}/job/carbon/ + +docker-compose down + +for container_name in storperf swagger-ui http-front-end +do + container=`docker ps -a -q -f name=$container_name` + if [ ! -z $container ] + then + echo "Stopping any existing $container_name container" + docker rm -fv $container + fi +done diff --git a/ci/start_job.sh b/ci/start_job.sh index 487e0c6..dde3dca 100755 --- a/ci/start_job.sh +++ b/ci/start_job.sh @@ -32,3 +32,5 @@ cat body.json 1>&2 curl -s -X POST --header 'Content-Type: application/json' \ --header 'Accept: application/json' \ -d @body.json http://127.0.0.1:5000/api/v1.0/jobs + +rm body.json \ No newline at end of file diff --git a/docker-compose/docker-compose.yaml b/docker-compose/docker-compose.yaml new file mode 100644 index 0000000..dbba825 --- /dev/null +++ b/docker-compose/docker-compose.yaml @@ -0,0 +1,25 @@ +version: '2' +services: + storperf: + container_name: "storperf" + image: "opnfv/storperf:${TAG}" + ports: + - "8000:8000" + env_file: ${ENV_FILE} + volumes: + - ${CARBON_DIR}:/opt/graphite/storage/whisper + + swagger-ui: + container_name: "swagger-ui" + image: "schickling/swagger-ui" + + http-front-end: + container_name: "http-front-end" + image: nginx:stable-alpine + ports: + - "5000:5000" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + links: + - storperf + - swagger-ui \ No newline at end of file diff --git a/docker-compose/nginx.conf b/docker-compose/nginx.conf new file mode 100644 index 0000000..b7933ac --- /dev/null +++ b/docker-compose/nginx.conf @@ -0,0 +1,34 @@ +http { + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + proxy_send_timeout 600; + proxy_read_timeout 600; + send_timeout 600; + + map $args $containsurl { + default 0; + "~(^|&)url=[^&]+($|&)" 1; + } + + server { + listen 5000; + + location /api/ { + proxy_pass http://storperf:5000; + proxy_set_header Host $host:$proxy_port; + } + + location /swagger/ { + if ($containsurl = 0) { + return 302 $scheme://$host:$server_port$uri?url=http://$host:$server_port/api/spec.json$args; + } + proxy_pass http://swagger-ui:80/; + } + } +} + +events { + worker_connections 1024; +} diff --git a/docs/testing/user/installation.rst b/docs/testing/user/installation.rst index 0effb2f..6098360 100755 --- a/docs/testing/user/installation.rst +++ b/docs/testing/user/installation.rst @@ -9,6 +9,11 @@ StorPerf Installation Guide OpenStack Prerequisites =========================== If you do not have an Ubuntu 16.04 image in Glance, you will need to add one. +You also need to create the StorPerf flavor, or choose one that closely +matches. For Ubuntu 16.04, it must have a minimum of a 4 GB disk. It should +also have about 8 GB RAM to support FIO's memory mapping of written data blocks +to ensure 100% coverage of the volume under test. + There are scripts in storperf/ci directory to assist, or you can use the follow code snippets: @@ -26,78 +31,214 @@ code snippets: --disk 4 \ --vcpus 2 +OpenStack Credentials +~~~~~~~~~~~~~~~~~~~~~ + +You must have your OpenStack Controller environment variables defined and passed to +the StorPerf container. The easiest way to do this is to put the rc file contents +into a clean file called admin.rc that looks similar to this for V2 authentication: + +.. code-block:: console + + cat << 'EOF' > admin.rc + OS_AUTH_URL=http://10.13.182.243:5000/v2.0 + OS_TENANT_ID=e8e64985506a4a508957f931d1800aa9 + OS_TENANT_NAME=admin + OS_PROJECT_NAME=admin + OS_USERNAME=admin + OS_PASSWORD=admin + OS_REGION_NAME=RegionOne + EOF + +For V3 authentication, at a minimum, use the following: + +.. code-block:: console + + cat << 'EOF' > admin.rc + OS_AUTH_URL=http://10.10.243.14:5000/v3 + OS_USERNAME=admin + OS_PASSWORD=admin + OS_PROJECT_DOMAIN_NAME=Default + OS_PROJECT_NAME=admin + OS_USER_DOMAIN_NAME=Default + EOF + + +Additionally, if you want your results published to the common OPNFV Test Results +DB, add the following: + +.. code-block:: console + + TEST_DB_URL=http://testresults.opnfv.org/testapi + Planning =========================== -StorPerf is delivered as a `Docker container -`__. There are two possible -methods for installation in your environment: - 1. Run container on Jump Host - 2. Run container in a VM +StorPerf is delivered as a series of Docker containers managed by +docker-compose. There are two possible methods for installation: - -Running StorPerf on Jump Host -============================= +#. Run container on bare metal +#. Run container in a VM Requirements: - * Docker must be installed - * Jump Host must have access to the OpenStack Controller API - * Jump Host must have internet connectivity for downloading docker image - * Enough floating IPs must be available to match your agent count +* Docker and docker-compose must be installed + * (note: sudo_ may be required if user is not part of docker group) +* OpenStack Controller credentials are available +* Host has access to the OpenStack Controller API +* Host must have internet connectivity for downloading docker image +* Enough OpenStack floating IPs must be available to match your agent count -Running StorPerf in a VM -======================== +.. _sudo: https://docs.docker.com/engine/reference/run/#general-form -Requirements: +Ports +===== - * VM has docker installed - * VM has OpenStack Controller credentials and can communicate with the Controller API - * VM has internet connectivity for downloading the docker image - * Enough floating IPs must be available to match your agent count +The following ports are exposed if you use the supplied docker-compose.yaml +file: -VM Creation -~~~~~~~~~~~ +* 5000 for StorPerf ReST API and Swagger UI +* 8000 for StorPerf's Graphite Web Server -The following procedure will create the VM in your environment -.. code-block:: console +Running StorPerf Container +========================== + +**As of Euphrates (development) release (June 2017), StorPerf has +changed to use docker-compose in order to start its services.** - # Create the StorPerf VM itself. Here we use the network ID generated by OPNFV FUEL. - ADMIN_NET_ID=`neutron net-list | grep 'admin_internal_net ' | awk '{print $2}'` +Two files are needed in order +to start StorPerf: - nova boot --nic net-id=$ADMIN_NET_ID --flavor m1.small --key-name=StorPerf --image 'Ubuntu 14.04' 'StorPerf Master' +#. docker-compose.yaml +#. nginx.conf -At this point, you may associate a floating IP with the StorPerf master VM. +Copy and paste the following into a terminal to create the docker-compose.yaml -VM Docker Installation -~~~~~~~~~~~~~~~~~~~~~~ +.. code-block:: console -The following procedure will install Docker on Ubuntu 14.04. + cat << 'EOF' > docker-compose.yaml + version: '2' + services: + storperf: + container_name: "storperf" + image: "opnfv/storperf:${TAG}" + ports: + - "8000:8000" + env_file: ${ENV_FILE} + volumes: + - ${CARBON_DIR}:/opt/graphite/storage/whisper + swagger-ui: + container_name: "swagger-ui" + image: "schickling/swagger-ui" + http-front-end: + container_name: "http-front-end" + image: nginx:stable-alpine + ports: + - "5000:5000" + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + links: + - storperf + - swagger-ui + EOF + +Copy and paste the following into a terminal to create the nginx.conf .. code-block:: console - sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D - cat << EOF | sudo tee /etc/apt/sources.list.d/docker.list - deb https://apt.dockerproject.org/repo ubuntu-trusty main + cat << 'EOF' > nginx.conf + http { + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + proxy_send_timeout 600; + proxy_read_timeout 600; + send_timeout 600; + map $args $containsurl { + default 0; + "~(^|&)url=[^&]+($|&)" 1; + } + server { + listen 5000; + location /api/ { + proxy_pass http://storperf:5000; + proxy_set_header Host $host:$proxy_port; + } + location /swagger/ { + if ($containsurl = 0) { + return 302 $scheme://$host:$server_port$uri?url=http://$host:$server_port/api/spec.json$args; + } + proxy_pass http://swagger-ui:80/; + } + } + } + events { + worker_connections 1024; + } EOF - sudo apt-get update - sudo apt-get install -y docker-engine - sudo usermod -aG docker ubuntu +Local disk used for the Carbon DB storage as the default size of the docker +container is only 10g. Here is an example of how to create a local storage +directory and set its permissions so that StorPerf can write to it: + +.. code-block:: console + + mkdir -p ./carbon + sudo chown 33:33 ./carbon + + +The following command will start all the StorPerf services: + +.. code-block:: console + + TAG=latest ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose pull + TAG=latest ENV_FILE=./admin.rc CARBON_DIR=./carbon/ docker-compose up -d + +You can now view the StorPerf SwaggerUI at: + +``http://127.0.0.1:5000/swagger`` + + +Docker Exec +~~~~~~~~~~~ + +If needed, the container can be entered with docker exec. This is not normally +required. + +.. code-block:: console + + docker exec -it storperf bash + + Pulling StorPerf Container ========================== +Master (Euphrates) +~~~~~~~~~~~~~~~~~~ + +StorPerf has switched to docker-compose in the latest version. The tag for +pulling the latest master Euphrates container is: + +.. code-block:: bash + + docker pull opnfv/storperf:master + +However, by itself, this will no longer provide full functionality. Full +instructions are provided in the Introduction document. + + Danube ~~~~~~ -The tag for the latest stable Danube will be: +The tag for the latest stable Danube is be: .. code-block:: bash - docker pull opnfv/storperf:danube.1.0 + docker pull opnfv/storperf:danube.2.0 Colorado ~~~~~~~~ @@ -116,14 +257,3 @@ The tag for the latest stable Brahmaputra release is: .. code-block:: bash docker pull opnfv/storperf:brahmaputra.1.2 - -Development -~~~~~~~~~~~ - -The tag for the latest development version is: - -.. code-block:: bash - - docker pull opnfv/storperf:master - - diff --git a/docs/testing/user/introduction.rst b/docs/testing/user/introduction.rst index a40750f..49e3220 100644 --- a/docs/testing/user/introduction.rst +++ b/docs/testing/user/introduction.rst @@ -2,100 +2,59 @@ .. http://creativecommons.org/licenses/by/4.0 .. (c) OPNFV, Dell EMC and others. -================================== -StorPerf Container Execution Guide -================================== - -Planning -======== - -There are some ports that the container can expose: - - * 22 for SSHD. Username and password are root/storperf. This is used for CLI access only - * 5000 for StorPerf ReST API. - * 8000 for StorPerf's Graphite Web Server - -OpenStack Credentials -~~~~~~~~~~~~~~~~~~~~~ - -You must have your OpenStack Controller environment variables defined and passed to -the StorPerf container. The easiest way to do this is to put the rc file contents -into a clean file the looks similar to this for V2 authentication: - -.. code-block:: console - - OS_AUTH_URL=http://10.13.182.243:5000/v2.0 - OS_TENANT_ID=e8e64985506a4a508957f931d1800aa9 - OS_TENANT_NAME=admin - OS_PROJECT_NAME=admin - OS_USERNAME=admin - OS_PASSWORD=admin - OS_REGION_NAME=RegionOne - -For V3 authentication, use the following: - -.. code-block:: console - - OS_AUTH_URL=http://10.13.182.243:5000/v3 - OS_PROJECT_ID=32ae78a844bc4f108b359dd7320463e5 - OS_PROJECT_NAME=admin - OS_USER_DOMAIN_NAME=Default - OS_USERNAME=admin - OS_PASSWORD=admin - OS_REGION_NAME=RegionOne - OS_INTERFACE=public - OS_IDENTITY_API_VERSION=3 - -Additionally, if you want your results published to the common OPNFV Test Results - DB, add the following: - -.. code-block:: console - - TEST_DB_URL=http://testresults.opnfv.org/testapi - -Running StorPerf Container -========================== - -You might want to have the local disk used for storage as the default size of the docker -container is only 10g. This is done with the -v option, mounting under -/opt/graphite/storage/whisper - -.. code-block:: console - - mkdir -p ~/carbon - sudo chown 33:33 ~/carbon - -The recommended method of running StorPerf is to expose only the ReST and Graphite -ports. The command line below shows how to run the container with local disk for -the carbon database. - -.. code-block:: console - - docker run -t --env-file admin-rc -p 5000:5000 -p 8000:8000 -v ~/carbon:/opt/graphite/storage/whisper --name storperf opnfv/storperf - - -Docker Exec -~~~~~~~~~~~ - -Instead of exposing port 5022 externally, you can use the exec method in docker. This -provides a slightly more secure method of running StorPerf container without having to -expose port 22. - -If needed, the container can be entered with docker exec. This is not normally required. - -.. code-block:: console - - docker exec -it storperf bash - -Container with SSH -~~~~~~~~~~~~~~~~~~ - -Running the StorPerf Container with all ports open and a local disk for the result -storage. This is not recommended as the SSH port is open. - -.. code-block:: console - - docker run -t --env-file admin-rc -p 5022:22 -p 5000:5000 -p 8000:8000 -v ~/carbon:/opt/graphite/storage/whisper --name storperf opnfv/storperf - -This will then permit ssh to localhost port 5022 for CLI access. - +===================== +StorPerf Introduction +===================== + +The purpose of StorPerf is to provide a tool to measure ephemeral and block +storage performance of OpenStack. + +A key challenge to measuring disk performance is to know when the disk (or, +for OpenStack, the virtual disk or volume) is performing at a consistent and +repeatable level of performance. Initial writes to a volume can perform +poorly due to block allocation, and reads can appear instantaneous when +reading empty blocks. How do we know when the data reported is valid? The +Storage Network Industry Association (SNIA_) has developed methods which enable +manufacturers to set, and customers to compare, the performance specifications +of Solid State Storage devices. StorPerf applies this methodology to OpenStack +Cinder and Glance services to provide a high level of confidence in the +performance metrics in the shortest reasonable time. + +.. _SNIA: http://www.snia.org/sites/default/files/HoEasen_SNIA_Solid_State_Storage_Per_Test_1_0.pdf + +How Does StorPerf Work? +======================= + +Once launched, StorPerf presents you with a ReST interface, along with a +`Swagger UI `_ that makes it easier to +form HTTP ReST requests. Issuing an HTTP POST to the configurations API +causes StorPerf to talk to your OpenStack's heat service to create a new stack +with as many agent VMs and attached Cinder volumes as you specify. + +After the stack is created, you can issue one or more jobs by issuing a POST +to the jobs ReST API. The job is the smallest unit of work that StorPerf +can use to measure the disk's performance. + +While the job is running, StorPerf collects the performance metrics from each +of the disks under test every minute. Once the trend of metrics match the +criteria specified in the SNIA methodology, the job automatically terminates +and the valid set of metrics are available for querying. + +What is the criteria? Simply put, it specifies that when the metrics +measured start to "flat line" and stay within that range for the specified +amount of time, then the metrics are considered to be indicative of a +repeatable level of performance. + +What Data Can I Get? +==================== + +StorPerf provides the following metrics: + +* IOPS +* Bandwidth (number of kilobytes read or written per second) +* Latency + +These metrics are available for every job, and for the specific workloads, +I/O loads and I/O types (read, write) associated with the job. + +As of this time, StorPerf only provides textual reports of the metrics. diff --git a/docs/testing/user/test-usage.rst b/docs/testing/user/test-usage.rst index 2beae69..91733cf 100644 --- a/docs/testing/user/test-usage.rst +++ b/docs/testing/user/test-usage.rst @@ -39,6 +39,7 @@ The following pieces of information are required to prepare the environment: - The number of VMs/Cinder volumes to create - The Glance image that holds the VM operating system to use. StorPerf has only been tested with Ubuntu 16.04 +- The OpenStack flavor to use when creating the VMs - The name of the public network that agents will use - The size, in gigabytes, of the Cinder volumes to create @@ -49,6 +50,7 @@ takes a JSON payload as follows. { "agent_count": int, + "agent_flavor": string "agent_image": string, "public_network": string, "volume_size": int diff --git a/rest_server.py b/rest_server.py index 5bff09c..dcf6be5 100644 --- a/rest_server.py +++ b/rest_server.py @@ -31,6 +31,7 @@ storperf = StorPerfMaster() class ConfigurationRequestModel: resource_fields = { 'agent_count': fields.Integer, + 'agent_flavor': fields.String, 'agent_image': fields.String, 'public_network': fields.String, 'volume_size': fields.Integer @@ -72,7 +73,7 @@ class Configure(Resource): @swagger.operation( notes='''Set the current agent configuration and create a stack in - the controller. Returns once the stack request is submitted.''', + the controller. Returns once the stack create is completed.''', parameters=[ { "name": "configuration", -- cgit 1.2.3-korg