diff options
Diffstat (limited to 'docker/storperf-graphite')
| -rw-r--r-- | docker/storperf-graphite/Dockerfile | 66 | ||||
| -rw-r--r-- | docker/storperf-graphite/conf/carbon.conf | 71 | ||||
| -rw-r--r-- | docker/storperf-graphite/conf/local_settings.py | 13 | ||||
| -rw-r--r-- | docker/storperf-graphite/conf/storage-aggregation.conf | 29 | ||||
| -rw-r--r-- | docker/storperf-graphite/conf/storage-schemas.conf | 3 | ||||
| -rw-r--r-- | docker/storperf-graphite/etc/nginx/conf.d/graphite | 37 | ||||
| -rw-r--r-- | docker/storperf-graphite/etc/nginx/nginx.conf | 20 | ||||
| -rw-r--r-- | docker/storperf-graphite/etc/supervisor.d/carbon.ini | 11 | ||||
| -rw-r--r-- | docker/storperf-graphite/etc/supervisor.d/gunicorn.ini | 13 | ||||
| -rw-r--r-- | docker/storperf-graphite/etc/supervisor.d/nginx.ini | 11 | ||||
| -rw-r--r-- | docker/storperf-graphite/etc/supervisord.conf | 26 | ||||
| -rw-r--r-- | docker/storperf-graphite/run.sh | 72 |
12 files changed, 372 insertions, 0 deletions
diff --git a/docker/storperf-graphite/Dockerfile b/docker/storperf-graphite/Dockerfile new file mode 100644 index 0000000..b566458 --- /dev/null +++ b/docker/storperf-graphite/Dockerfile @@ -0,0 +1,66 @@ +############################################################################## +# Copyright (c) 2017 Dell EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# Docker container for StorPerf HTTP Graphite +# +# Build: +# $ docker build -t opnfv/storperf-graphite:tag . +## + +# From https://github.com/SchweizerischeBundesbahnen/docker-graphite + +ARG ARCH=x86_64 +ARG ALPINE_VERSION=v3.5 +FROM multiarch/alpine:$ARCH-$ALPINE_VERSION + +# Install basic stuff =) +RUN apk add --no-cache \ + bash \ + ca-certificates \ + nginx \ + openssl \ + py2-pip \ + supervisor \ + tini \ + && pip install \ + supervisor-stdout \ + gunicorn + +# Install graphite +ENV GRAPHITE_ROOT /opt/graphite + +RUN apk add --no-cache \ + alpine-sdk \ + fontconfig \ + libffi \ + libffi-dev \ + python-dev \ + py-cairo \ + && export PYTHONPATH="/opt/graphite/lib/:/opt/graphite/webapp/" \ + && pip install https://github.com/graphite-project/whisper/tarball/master \ + && pip install https://github.com/graphite-project/carbon/tarball/master \ + && pip install https://github.com/graphite-project/graphite-web/tarball/master \ + && apk del \ + alpine-sdk \ + python-dev \ + libffi-dev + +EXPOSE 8080 +EXPOSE 2003 +EXPOSE 2004 +EXPOSE 7002 + +VOLUME ["/opt/graphite/conf", "/opt/graphite/storage"] + +COPY run.sh /run.sh +COPY etc/ /etc/ +COPY conf/ /opt/graphite/conf.example/ + +# Enable tiny init +ENTRYPOINT ["/sbin/tini", "--"] +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/storperf-graphite/conf/carbon.conf b/docker/storperf-graphite/conf/carbon.conf new file mode 100644 index 0000000..6463c79 --- /dev/null +++ b/docker/storperf-graphite/conf/carbon.conf @@ -0,0 +1,71 @@ +[cache] +LOCAL_DATA_DIR = /opt/graphite/storage/whisper/ + +# Specify the user to drop privileges to +# If this is blank carbon runs as the user that invokes it +# This user must have write access to the local data directory +USER = + +# Limit the size of the cache to avoid swapping or becoming CPU bound. +# Sorts and serving cache queries gets more expensive as the cache grows. +# Use the value "inf" (infinity) for an unlimited cache size. +MAX_CACHE_SIZE = inf + +# Limits the number of whisper update_many() calls per second, which effectively +# means the number of write requests sent to the disk. This is intended to +# prevent over-utilizing the disk and thus starving the rest of the system. +# When the rate of required updates exceeds this, then carbon's caching will +# take effect and increase the overall throughput accordingly. +MAX_UPDATES_PER_SECOND = 1000 + +# Softly limits the number of whisper files that get created each minute. +# Setting this value low (like at 50) is a good way to ensure your graphite +# system will not be adversely impacted when a bunch of new metrics are +# sent to it. The trade off is that it will take much longer for those metrics' +# database files to all get created and thus longer until the data becomes usable. +# Setting this value high (like "inf" for infinity) will cause graphite to create +# the files quickly but at the risk of slowing I/O down considerably for a while. +MAX_CREATES_PER_MINUTE = inf + +LINE_RECEIVER_INTERFACE = 0.0.0.0 +LINE_RECEIVER_PORT = 2003 + +PICKLE_RECEIVER_INTERFACE = 0.0.0.0 +PICKLE_RECEIVER_PORT = 2004 + +CACHE_QUERY_INTERFACE = 0.0.0.0 +CACHE_QUERY_PORT = 7002 + +# By default, carbon-cache will log every whisper update and cache hit. This can be excessive and +# degrade performance if logging on the same volume as the whisper data is stored. +LOG_UPDATES = False +LOG_CACHE_HITS = False +ENABLE_LOGROTATION = True +LOG_LISTENER_CONNECTIONS = False + +# Enable AMQP if you want to receve metrics using an amqp broker +# ENABLE_AMQP = False + +# Verbose means a line will be logged for every metric received +# useful for testing +# AMQP_VERBOSE = False + +# AMQP_HOST = localhost +# AMQP_PORT = 5672 +# AMQP_VHOST = / +# AMQP_USER = guest +# AMQP_PASSWORD = guest +# AMQP_EXCHANGE = graphite + +# Patterns for all of the metrics this machine will store. Read more at +# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings +# +# Example: store all sales, linux servers, and utilization metrics +# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization +# +# Example: store everything +# BIND_PATTERNS = # + +# NOTE: you cannot run both a cache and a relay on the same server +# with the default configuration, you have to specify a distinict +# interfaces and ports for the listeners. diff --git a/docker/storperf-graphite/conf/local_settings.py b/docker/storperf-graphite/conf/local_settings.py new file mode 100644 index 0000000..88414aa --- /dev/null +++ b/docker/storperf-graphite/conf/local_settings.py @@ -0,0 +1,13 @@ +# flake8: noqa +# Edit this file to override the default graphite settings, do not edit settings.py + +# Turn on debugging and restart apache if you ever see an "Internal Server Error" page +# DEBUG = True + +# Set your local timezone (django will try to figure this out automatically) +TIME_ZONE = 'Europe/Zurich' + +# Secret key for django +SECRET_KEY = '%%SECRET_KEY%%' + +URL_PREFIX = "/graphite/" diff --git a/docker/storperf-graphite/conf/storage-aggregation.conf b/docker/storperf-graphite/conf/storage-aggregation.conf new file mode 100644 index 0000000..bc5e1db --- /dev/null +++ b/docker/storperf-graphite/conf/storage-aggregation.conf @@ -0,0 +1,29 @@ +[min] +pattern = \.lower$ +xFilesFactor = 0.1 +aggregationMethod = min + +[max] +pattern = \.upper(_\d+)?$ +xFilesFactor = 0.1 +aggregationMethod = max + +[sum] +pattern = \.sum$ +xFilesFactor = 0 +aggregationMethod = sum + +[count] +pattern = \.count$ +xFilesFactor = 0 +aggregationMethod = sum + +[count_legacy] +pattern = ^stats_counts.* +xFilesFactor = 0 +aggregationMethod = sum + +[default_average] +pattern = .* +xFilesFactor = 0.3 +aggregationMethod = average diff --git a/docker/storperf-graphite/conf/storage-schemas.conf b/docker/storperf-graphite/conf/storage-schemas.conf new file mode 100644 index 0000000..11a59be --- /dev/null +++ b/docker/storperf-graphite/conf/storage-schemas.conf @@ -0,0 +1,3 @@ +[default] +pattern = .* +retentions = 60s:14d diff --git a/docker/storperf-graphite/etc/nginx/conf.d/graphite b/docker/storperf-graphite/etc/nginx/conf.d/graphite new file mode 100644 index 0000000..e4c405d --- /dev/null +++ b/docker/storperf-graphite/etc/nginx/conf.d/graphite @@ -0,0 +1,37 @@ +server { + listen 8080; + server_name graphite; + charset utf-8; + # Django admin media. + location /graphite/static/admin/ { + alias /usr/lib/python2.7/site-packages/django/contrib/admin/static/admin/; + } + + # Your project's static media. + location /graphite/static/ { + alias /opt/graphite/webapp/content/; + } + + # Finally, send all non-media requests to the Django server. + location / { + proxy_pass http://127.0.0.1:8000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header Host $host; + + client_max_body_size 10m; + client_body_buffer_size 128k; + + proxy_connect_timeout 90; + proxy_send_timeout 90; + proxy_read_timeout 90; + + proxy_buffer_size 4k; + proxy_buffers 4 32k; + proxy_busy_buffers_size 64k; + proxy_temp_file_write_size 64k; + } +} diff --git a/docker/storperf-graphite/etc/nginx/nginx.conf b/docker/storperf-graphite/etc/nginx/nginx.conf new file mode 100644 index 0000000..f2ab7f7 --- /dev/null +++ b/docker/storperf-graphite/etc/nginx/nginx.conf @@ -0,0 +1,20 @@ +worker_processes 1; +pid /var/run/nginx.pid; +daemon off; + +events { + worker_connections 1024; + use epoll; +} + +http { + include mime.types; + default_type application/octet-stream; + + sendfile on; + keepalive_timeout 65; + + gzip on; + + include /etc/nginx/conf.d/*; +} diff --git a/docker/storperf-graphite/etc/supervisor.d/carbon.ini b/docker/storperf-graphite/etc/supervisor.d/carbon.ini new file mode 100644 index 0000000..fb93a95 --- /dev/null +++ b/docker/storperf-graphite/etc/supervisor.d/carbon.ini @@ -0,0 +1,11 @@ +[program:carbon-cache] +autostart = true +autorestart = true +stdout_events_enabled = true +stderr_events_enabled = true +stdout_logfile_maxbytes = 1MB +stdout_logfile_backups = 0 +stderr_logfile_maxbytes = 1MB +stderr_logfile_backups = 0 + +command = /opt/graphite/bin/carbon-cache.py --pidfile /var/run/carbon-cache-a.pid --debug start diff --git a/docker/storperf-graphite/etc/supervisor.d/gunicorn.ini b/docker/storperf-graphite/etc/supervisor.d/gunicorn.ini new file mode 100644 index 0000000..7a94ac8 --- /dev/null +++ b/docker/storperf-graphite/etc/supervisor.d/gunicorn.ini @@ -0,0 +1,13 @@ +[program:graphite-webapp] +autostart = true +autorestart = true +stdout_events_enabled = true +stderr_events_enabled = true +stdout_logfile_maxbytes = 1MB +stdout_logfile_backups = 0 +stderr_logfile_maxbytes = 1MB +stderr_logfile_backups = 0 + +directory = /opt/graphite/webapp +environment = PYTHONPATH='/opt/graphite/webapp' +command = /usr/bin/gunicorn -b127.0.0.1:8000 -w2 graphite.wsgi diff --git a/docker/storperf-graphite/etc/supervisor.d/nginx.ini b/docker/storperf-graphite/etc/supervisor.d/nginx.ini new file mode 100644 index 0000000..be2615c --- /dev/null +++ b/docker/storperf-graphite/etc/supervisor.d/nginx.ini @@ -0,0 +1,11 @@ +[program:nginx] +autostart = true +autorestart = true +stdout_events_enabled = true +stderr_events_enabled = true +stdout_logfile_maxbytes = 1MB +stdout_logfile_backups = 0 +stderr_logfile_maxbytes = 1MB +stderr_logfile_backups = 0 + +command = /usr/sbin/nginx -c /etc/nginx/nginx.conf diff --git a/docker/storperf-graphite/etc/supervisord.conf b/docker/storperf-graphite/etc/supervisord.conf new file mode 100644 index 0000000..01799ab --- /dev/null +++ b/docker/storperf-graphite/etc/supervisord.conf @@ -0,0 +1,26 @@ +[unix_http_server] +file=/run/supervisord.sock + +[supervisord] +user = root +nodaemon = true +logfile_maxbytes = 10MB +logfile_backups = 0 +pidfile = /tmp/supervisord.pid +logfile = /tmp/supervisord.log +environment = GRAPHITE_STORAGE_DIR='/opt/graphite/storage',GRAPHITE_CONF_DIR='/opt/graphite/conf' + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/supervisord.sock + +[eventlistener:stdout] +command = supervisor_stdout +buffer_size = 100 +events = PROCESS_LOG +result_handler = supervisor_stdout:event_handler + +[include] +files = /etc/supervisor.d/*.ini diff --git a/docker/storperf-graphite/run.sh b/docker/storperf-graphite/run.sh new file mode 100644 index 0000000..c9baa93 --- /dev/null +++ b/docker/storperf-graphite/run.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +whisper_dir="/opt/graphite/storage/whisper/" +webapp_dir="/opt/graphite/storage/log/webapp/" + +cd /opt/graphite + +if [ -d $whisper_dir ]; then + echo "Whisper directory already exists" +else + echo "...creating missing whisper dir" + mkdir -p $whisper_dir +fi + +if [ -d $webapp_dir ]; then + echo "Webapp directory already exists" +else + echo "...creating missing webapp dir" + mkdir -p $webapp_dir +fi + +if [ ! -f /opt/graphite/conf/local_settings.py ]; then + echo "Creating default config for graphite-web..." + cp /opt/graphite/conf.example/local_settings.py /opt/graphite/conf/local_settings.py + RANDOM_STRING=$(python -c 'import random; import string; print "".join([random.SystemRandom().choice(string.digits + string.letters) for i in range(100)])') + sed "s/%%SECRET_KEY%%/${RANDOM_STRING}/" -i /opt/graphite/conf/local_settings.py +fi + +if [ ! -L /opt/graphite/webapp/graphite/local_settings.py ]; then + echo "Creating symbolic link for local_settings.py in graphite-web..." + ln -s /opt/graphite/conf/local_settings.py /opt/graphite/webapp/graphite/local_settings.py +fi + +sed "s/%%CLUSTER_SERVERS%%/${CLUSTER_SERVERS}/" -i /opt/graphite/conf/local_settings.py + +if [ ! -f /opt/graphite/conf/carbon.conf ]; then + echo "Creating default config for carbon..." + cp /opt/graphite/conf.example/carbon.conf /opt/graphite/conf/carbon.conf +fi + +if [ ! -f /opt/graphite/conf/storage-schemas.conf ]; then + echo "Creating default storage schema for carbon..." + cp /opt/graphite/conf.example/storage-schemas.conf /opt/graphite/conf/storage-schemas.conf +fi + +if [ ! -f /opt/graphite/conf/storage-aggregation.conf ]; then + echo "Creating default storage schema for carbon..." + cp /opt/graphite/conf.example/storage-aggregation.conf /opt/graphite/conf/storage-aggregation.conf +fi + +if [ ! -f /opt/graphite/storage/graphite.db ]; then + echo "Creating database..." + PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py migrate --settings=graphite.settings --run-syncdb --noinput + chown nginx:nginx /opt/graphite/storage/graphite.db + # Auto-magical create an django user with default login + script="from django.contrib.auth.models import User; + +username = 'admin'; +password = 'admin'; +email = 'admin@example.com'; + +if User.objects.filter(username=username).count()==0: + User.objects.create_superuser(username, email, password); + print('Superuser created.'); +else: + print('Superuser creation skipped.'); + +" + printf "$script" | PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py shell --settings=graphite.settings +fi + +exec supervisord -c /etc/supervisord.conf |