summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authormbeierl <mark.beierl@dell.com>2017-03-29 21:00:50 -0400
committermbeierl <mark.beierl@dell.com>2017-03-29 21:02:33 -0400
commit1bcbe0bac328e26341ba68341d4b809c32b8242f (patch)
treeea28797aae10c2a20c130095ac36ce62a581bd5f
parent5fbde2962c159ace29a7b42bc217f1eed8109808 (diff)
Use tagged version of graphite
Changes the docker build procedure to use a pip install of a specific version of graphite so that it doesn't break when upstream changes occur. Change-Id: I2316912fc04568c441212087665e384075e409a6 JIRA: STORPERF-110 Signed-off-by: mbeierl <mark.beierl@dell.com>
-rwxr-xr-xbuild-dev-docker.sh15
-rwxr-xr-xci/daily.sh5
-rwxr-xr-xci/start_job.sh2
-rw-r--r--docker/Dockerfile35
-rw-r--r--docker/graphite/carbon.conf80
-rw-r--r--docker/graphite/initial_data.json20
-rw-r--r--docker/graphite/local_settings.py42
-rw-r--r--docker/graphite/nginx.conf69
-rw-r--r--docker/graphite/storage-schemas.conf7
-rw-r--r--docker/storperf.pp19
-rw-r--r--docker/supervisord.conf17
11 files changed, 272 insertions, 39 deletions
diff --git a/build-dev-docker.sh b/build-dev-docker.sh
index 131d8ef..3a565f3 100755
--- a/build-dev-docker.sh
+++ b/build-dev-docker.sh
@@ -8,12 +8,17 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
+cd `dirname $0`
+
echo "Creating a docker image from the current working directory..."
-sed "s|RUN git clone https://gerrit.opnfv.org/gerrit/storperf.*$|COPY . \${repos_dir}/storperf|" docker/Dockerfile > Dockerfile
-sed -i "s|COPY storperf.pp|COPY docker/storperf.pp|" Dockerfile
-sed -i "s|COPY supervisord.conf|COPY docker/supervisord.conf|" Dockerfile
+cp docker/Dockerfile Dockerfile.dev
+sed -i "s|COPY |COPY docker\/|" Dockerfile.dev
+sed -i "s|ADD |ADD docker\/|" Dockerfile.dev
+sed -i "s|RUN git clone.*https://gerrit.opnfv.org/gerrit/storperf.*$|COPY . \${repos_dir}/storperf|" Dockerfile.dev
+
+diff docker/Dockerfile Dockerfile.dev
-docker build -t opnfv/storperf:dev .
+docker build -t opnfv/storperf:dev -f Dockerfile.dev .
-rm -f Dockerfile
+rm -f Dockerfile.dev
diff --git a/ci/daily.sh b/ci/daily.sh
index e12964d..b69506a 100755
--- a/ci/daily.sh
+++ b/ci/daily.sh
@@ -122,11 +122,12 @@ done
echo "Deleting stack for cleanup"
curl -s -X DELETE --header 'Accept: application/json' 'http://127.0.0.1:5000/api/v1.0/configurations'
-sudo chmod 777 -R $WORKSPACE/ci/job/carbon
-
curl -s -X GET "http://127.0.0.1:5000/api/v1.0/jobs?id=$JOB&type=metadata" \
-o $WORKSPACE/ci/job/report.json
+docker rm -f storperf
+sudo rm -rf $WORKSPACE/ci/job/carbon
+
echo ==========================================================================
echo Final report
echo ==========================================================================
diff --git a/ci/start_job.sh b/ci/start_job.sh
index 31bfddb..85652d0 100755
--- a/ci/start_job.sh
+++ b/ci/start_job.sh
@@ -11,8 +11,6 @@
cat << EOF > body.json
{
"block_sizes": "${BLOCK_SIZE}",
- "nowarm": "string",
- "nossd": "string",
"deadline": 30,
"queue_depths": "${QUEUE_DEPTH}",
"workload": "${WORKLOAD}",
diff --git a/docker/Dockerfile b/docker/Dockerfile
index ce1f853..96ed58f 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -26,6 +26,7 @@ LABEL version="0.1" description="OPNFV Storperf Docker container"
ARG BRANCH=master
ENV repos_dir /home/opnfv/repos
+ENV DEBIAN_FRONTEND noninteractive
WORKDIR /home/opnfv
@@ -65,6 +66,35 @@ RUN mkdir -p ${repos_dir}
RUN mkdir -p /root/.ssh
RUN chmod 700 /root/.ssh
+# Graphite installation
+RUN echo 'deb http://us.archive.ubuntu.com/ubuntu/ trusty universe' >> /etc/apt/sources.list
+RUN apt-get -y update
+
+# Install required packages
+RUN apt-get -y install python-ldap python-cairo python-django python-twisted python-django-tagging python-simplejson python-memcache python-pysqlite2 python-support python-tz python-pip gunicorn supervisor nginx-light
+RUN pip install whisper==0.9.15
+RUN pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/lib" carbon==0.9.15
+RUN pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/webapp" graphite-web==0.9.15
+
+# Add system service config
+ADD graphite/nginx.conf /etc/nginx/nginx.conf
+ADD graphite/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+# Add graphite config
+ADD graphite/initial_data.json /var/lib/graphite/webapp/graphite/initial_data.json
+ADD graphite/local_settings.py /var/lib/graphite/webapp/graphite/local_settings.py
+ADD graphite/carbon.conf /var/lib/graphite/conf/carbon.conf
+ADD graphite/storage-schemas.conf /var/lib/graphite/conf/storage-schemas.conf
+RUN mkdir -p /opt/graphite/storage
+RUN ln -s /var/lib/graphite/storage/whisper /opt/graphite/storage/whisper
+RUN touch /var/lib/graphite/storage/graphite.db /var/lib/graphite/storage/index
+RUN chown -R www-data /var/lib/graphite/storage
+RUN chmod 0775 /var/lib/graphite/storage /var/lib/graphite/storage/whisper
+RUN chmod 0664 /var/lib/graphite/storage/graphite.db
+RUN cd /var/lib/graphite/webapp/graphite && python manage.py syncdb --noinput
+
+# Git configuration
+
RUN git config --global http.sslVerify false
RUN git clone --depth 1 -b $BRANCH https://gerrit.opnfv.org/gerrit/storperf ${repos_dir}/storperf
RUN git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng ${repos_dir}/releng
@@ -79,17 +109,12 @@ RUN git clone http://git.kernel.dk/fio.git ${repos_dir}/fio
RUN cd ${repos_dir}/fio && git checkout tags/fio-2.2.10
RUN cd ${repos_dir}/fio && make -j 6 install
-RUN puppet module install gdsoperations-graphite
RUN chmod 600 ${repos_dir}/storperf/storperf/resources/ssh/storperf_rsa
RUN pip install --upgrade setuptools==33.1.1
RUN pip install -r ${repos_dir}/storperf/docker/requirements.pip
-COPY storperf.pp /etc/puppet/manifests/storperf.pp
-RUN puppet apply /etc/puppet/manifests/storperf.pp
-
-RUN ln -s /opt/graphite/lib/whisper* /usr/lib/python2.7/
# Open access to SSH if desired
EXPOSE 22
diff --git a/docker/graphite/carbon.conf b/docker/graphite/carbon.conf
new file mode 100644
index 0000000..13088dd
--- /dev/null
+++ b/docker/graphite/carbon.conf
@@ -0,0 +1,80 @@
+[cache]
+LOCAL_DATA_DIR = /var/lib/graphite/storage/whisper/
+
+# Specify the user to drop privileges to
+# If this is blank carbon runs as the user that invokes it
+# This user must have write access to the local data directory
+USER =
+
+# Limit the size of the cache to avoid swapping or becoming CPU bound.
+# Sorts and serving cache queries gets more expensive as the cache grows.
+# Use the value "inf" (infinity) for an unlimited cache size.
+MAX_CACHE_SIZE = inf
+
+# Limits the number of whisper update_many() calls per second, which effectively
+# means the number of write requests sent to the disk. This is intended to
+# prevent over-utilizing the disk and thus starving the rest of the system.
+# When the rate of required updates exceeds this, then carbon's caching will
+# take effect and increase the overall throughput accordingly.
+MAX_UPDATES_PER_SECOND = 1000
+
+# Softly limits the number of whisper files that get created each minute.
+# Setting this value low (like at 50) is a good way to ensure your graphite
+# system will not be adversely impacted when a bunch of new metrics are
+# sent to it. The trade off is that it will take much longer for those metrics'
+# database files to all get created and thus longer until the data becomes usable.
+# Setting this value high (like "inf" for infinity) will cause graphite to create
+# the files quickly but at the risk of slowing I/O down considerably for a while.
+MAX_CREATES_PER_MINUTE = inf
+
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = 2003
+
+ENABLE_UDP_LISTENER = True
+UDP_RECEIVER_INTERFACE = 0.0.0.0
+UDP_RECEIVER_PORT = 2003
+
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2004
+
+CACHE_QUERY_INTERFACE = 0.0.0.0
+CACHE_QUERY_PORT = 7002
+
+LOG_UPDATES = False
+
+# Enable AMQP if you want to receve metrics using an amqp broker
+# ENABLE_AMQP = False
+
+# Verbose means a line will be logged for every metric received
+# useful for testing
+# AMQP_VERBOSE = False
+
+# AMQP_HOST = localhost
+# AMQP_PORT = 5672
+# AMQP_VHOST = /
+# AMQP_USER = guest
+# AMQP_PASSWORD = guest
+# AMQP_EXCHANGE = graphite
+
+# Patterns for all of the metrics this machine will store. Read more at
+# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings
+#
+# Example: store all sales, linux servers, and utilization metrics
+# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization
+#
+# Example: store everything
+# BIND_PATTERNS = #
+
+# NOTE: you cannot run both a cache and a relay on the same server
+# with the default configuration, you have to specify a distinict
+# interfaces and ports for the listeners.
+
+[relay]
+LINE_RECEIVER_INTERFACE = 0.0.0.0
+LINE_RECEIVER_PORT = 2003
+
+PICKLE_RECEIVER_INTERFACE = 0.0.0.0
+PICKLE_RECEIVER_PORT = 2004
+
+CACHE_SERVERS = server1, server2, server3
+MAX_QUEUE_SIZE = 10000
diff --git a/docker/graphite/initial_data.json b/docker/graphite/initial_data.json
new file mode 100644
index 0000000..b3ac9b1
--- /dev/null
+++ b/docker/graphite/initial_data.json
@@ -0,0 +1,20 @@
+[
+ {
+ "pk": 1,
+ "model": "auth.user",
+ "fields": {
+ "username": "admin",
+ "first_name": "",
+ "last_name": "",
+ "is_active": true,
+ "is_superuser": true,
+ "is_staff": true,
+ "last_login": "2011-09-20 17:02:14",
+ "groups": [],
+ "user_permissions": [],
+ "password": "sha1$1b11b$edeb0a67a9622f1f2cfeabf9188a711f5ac7d236",
+ "email": "root@example.com",
+ "date_joined": "2011-09-20 17:02:14"
+ }
+ }
+]
diff --git a/docker/graphite/local_settings.py b/docker/graphite/local_settings.py
new file mode 100644
index 0000000..177d674
--- /dev/null
+++ b/docker/graphite/local_settings.py
@@ -0,0 +1,42 @@
+# Edit this file to override the default graphite settings, do not edit settings.py
+
+# Turn on debugging and restart apache if you ever see an "Internal Server Error" page
+#DEBUG = True
+
+# Set your local timezone (django will try to figure this out automatically)
+TIME_ZONE = 'UTC'
+
+# Setting MEMCACHE_HOSTS to be empty will turn off use of memcached entirely
+#MEMCACHE_HOSTS = ['127.0.0.1:11211']
+
+# Sometimes you need to do a lot of rendering work but cannot share your storage mount
+#REMOTE_RENDERING = True
+#RENDERING_HOSTS = ['fastserver01','fastserver02']
+#LOG_RENDERING_PERFORMANCE = True
+#LOG_CACHE_PERFORMANCE = True
+
+# If you've got more than one backend server they should all be listed here
+#CLUSTER_SERVERS = []
+
+# Override this if you need to provide documentation specific to your graphite deployment
+#DOCUMENTATION_URL = "http://wiki.mycompany.com/graphite"
+
+# Enable email-related features
+#SMTP_SERVER = "mail.mycompany.com"
+
+# LDAP / ActiveDirectory authentication setup
+#USE_LDAP_AUTH = True
+#LDAP_SERVER = "ldap.mycompany.com"
+#LDAP_PORT = 389
+#LDAP_SEARCH_BASE = "OU=users,DC=mycompany,DC=com"
+#LDAP_BASE_USER = "CN=some_readonly_account,DC=mycompany,DC=com"
+#LDAP_BASE_PASS = "readonly_account_password"
+#LDAP_USER_QUERY = "(username=%s)" #For Active Directory use "(sAMAccountName=%s)"
+
+# If sqlite won't cut it, configure your real database here (don't forget to run manage.py syncdb!)
+#DATABASE_ENGINE = 'mysql' # or 'postgres'
+#DATABASE_NAME = 'graphite'
+#DATABASE_USER = 'graphite'
+#DATABASE_PASSWORD = 'graphite-is-awesome'
+#DATABASE_HOST = 'mysql.mycompany.com'
+#DATABASE_PORT = '3306'
diff --git a/docker/graphite/nginx.conf b/docker/graphite/nginx.conf
new file mode 100644
index 0000000..8a11e94
--- /dev/null
+++ b/docker/graphite/nginx.conf
@@ -0,0 +1,69 @@
+daemon off;
+user www-data;
+worker_processes 1;
+pid /var/run/nginx.pid;
+
+events {
+ worker_connections 1024;
+}
+
+http {
+ sendfile on;
+ tcp_nopush on;
+ tcp_nodelay on;
+ keepalive_timeout 65;
+ types_hash_max_size 2048;
+ server_tokens off;
+
+ server_names_hash_bucket_size 32;
+
+ include /etc/nginx/mime.types;
+ default_type application/octet-stream;
+
+ access_log /var/log/nginx/access.log;
+ error_log /var/log/nginx/error.log;
+
+ gzip on;
+ gzip_disable "msie6";
+
+ server {
+ listen 8000 default_server;
+ server_name _;
+
+ open_log_file_cache max=1000 inactive=20s min_uses=2 valid=1m;
+
+ location / {
+ proxy_pass http://127.0.0.1:8080;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header X-Forwarded-Server $host;
+ proxy_set_header X-Forwarded-Host $http_host;
+ proxy_set_header Host $http_host;
+
+ client_max_body_size 10m;
+ client_body_buffer_size 128k;
+
+ proxy_connect_timeout 90;
+ proxy_send_timeout 90;
+ proxy_read_timeout 90;
+
+ proxy_buffer_size 4k;
+ proxy_buffers 4 32k;
+ proxy_busy_buffers_size 64k;
+ proxy_temp_file_write_size 64k;
+ }
+
+ add_header Access-Control-Allow-Origin "*";
+ add_header Access-Control-Allow-Methods "GET, OPTIONS";
+ add_header Access-Control-Allow-Headers "origin, authorization, accept";
+
+ location /content {
+ alias /var/lib/graphite/webapp/content;
+ }
+
+ location /media {
+ alias /usr/share/pyshared/django/contrib/admin/media;
+ }
+ }
+}
diff --git a/docker/graphite/storage-schemas.conf b/docker/graphite/storage-schemas.conf
new file mode 100644
index 0000000..855a9e4
--- /dev/null
+++ b/docker/graphite/storage-schemas.conf
@@ -0,0 +1,7 @@
+[carbon]
+pattern = ^carbon\..*
+retentions = 1m:31d,10m:1y,1h:5y
+
+[default]
+pattern = .*
+retentions = 10s:8d,1m:31d,10m:1y,1h:5y
diff --git a/docker/storperf.pp b/docker/storperf.pp
deleted file mode 100644
index 7de1024..0000000
--- a/docker/storperf.pp
+++ /dev/null
@@ -1,19 +0,0 @@
-##############################################################################
-# Copyright (c) 2015 EMC and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-class { 'python':
- pip => true,
- dev => true,
- virtualenv => true,
-}
-
-class { 'graphite':
- port => 8080,
- bind_address => '0.0.0.0',
-}
diff --git a/docker/supervisord.conf b/docker/supervisord.conf
index 0a78928..834e23c 100644
--- a/docker/supervisord.conf
+++ b/docker/supervisord.conf
@@ -1,19 +1,25 @@
[supervisord]
nodaemon = true
-environment = GRAPHITE_STORAGE_DIR='/opt/graphite/storage',GRAPHITE_CONF_DIR='/opt/graphite/conf'
+environment = GRAPHITE_STORAGE_DIR='/var/lib/graphite/storage',GRAPHITE_CONF_DIR='/var/lib/graphite/conf'
+
+[program:nginx]
+command = /usr/sbin/nginx
+stdout_logfile = /var/log/supervisor/%(program_name)s.log
+stderr_logfile = /var/log/supervisor/%(program_name)s.log
+autorestart = true
[program:carbon-cache]
user = www-data
-command = /opt/graphite/bin/carbon-cache.py --debug start
+command = /var/lib/graphite/bin/carbon-cache.py --debug start
stdout_logfile = /var/log/supervisor/%(program_name)s.log
stderr_logfile = /var/log/supervisor/%(program_name)s.log
autorestart = true
[program:graphite-webapp]
user = www-data
-directory = /opt/graphite/webapp
-environment = PYTHONPATH='/opt/graphite/webapp'
-command = /opt/graphite/bin/gunicorn_django -b0.0.0.0:8000 -w2 graphite/settings.py
+directory = /var/lib/graphite/webapp
+environment = PYTHONPATH='/var/lib/graphite/webapp'
+command = /usr/bin/gunicorn_django -b127.0.0.1:8080 -w2 graphite/settings.py
stdout_logfile = /var/log/supervisor/%(program_name)s.log
stderr_logfile = /var/log/supervisor/%(program_name)s.log
autorestart = true
@@ -32,4 +38,3 @@ command = /usr/bin/python /home/opnfv/repos/storperf/rest_server.py
stdout_logfile = /var/log/supervisor/%(program_name)s.log
stderr_logfile = /var/log/supervisor/%(program_name)s.log
autorestart = true
-