From 9efa75fa08dad32b95ada37bb8e22371c51626c3 Mon Sep 17 00:00:00 2001 From: mbeierl Date: Wed, 12 Jul 2017 21:41:01 -0400 Subject: Creation of Local Docker for Daily Job Moves the dockerfiles for the different submodules/containers into their respective directories. Adds a docker compose file that can build the containers from the local filesystem, and mounts the local filesystem as a volume, making it easier for developer testing as local changes are automatically reflected in the final container. Change-Id: I2376034258cab2a3dc3a0d1ac3f01bbf5d1f59e7 JIRA: STORPERF-189 Signed-off-by: mbeierl --- docker/Dockerfile-httpfrontend | 25 ------ docker/Dockerfile-master | 99 ---------------------- docker/Dockerfile-reporting | 33 -------- docker/graphite/carbon.conf | 80 ----------------- docker/graphite/initial_data.json | 20 ----- docker/graphite/local_settings.py | 42 --------- docker/graphite/nginx.conf | 69 --------------- docker/graphite/storage-schemas.conf | 7 -- docker/local-docker-compose.yaml | 43 ++++++++++ docker/storperf-httpfrontend/Dockerfile | 23 +++++ docker/storperf-master/Dockerfile | 92 ++++++++++++++++++++ docker/storperf-master/graphite/carbon.conf | 80 +++++++++++++++++ docker/storperf-master/graphite/initial_data.json | 20 +++++ docker/storperf-master/graphite/local_settings.py | 1 + docker/storperf-master/graphite/nginx.conf | 69 +++++++++++++++ .../storperf-master/graphite/storage-schemas.conf | 7 ++ docker/storperf-reporting/Dockerfile | 30 +++++++ 17 files changed, 365 insertions(+), 375 deletions(-) delete mode 100644 docker/Dockerfile-httpfrontend delete mode 100644 docker/Dockerfile-master delete mode 100644 docker/Dockerfile-reporting delete mode 100644 docker/graphite/carbon.conf delete mode 100644 docker/graphite/initial_data.json delete mode 100644 docker/graphite/local_settings.py delete mode 100644 docker/graphite/nginx.conf delete mode 100644 docker/graphite/storage-schemas.conf create mode 100644 docker/local-docker-compose.yaml create mode 100644 docker/storperf-httpfrontend/Dockerfile create mode 100644 docker/storperf-master/Dockerfile create mode 100644 docker/storperf-master/graphite/carbon.conf create mode 100644 docker/storperf-master/graphite/initial_data.json create mode 100644 docker/storperf-master/graphite/local_settings.py create mode 100644 docker/storperf-master/graphite/nginx.conf create mode 100644 docker/storperf-master/graphite/storage-schemas.conf create mode 100644 docker/storperf-reporting/Dockerfile (limited to 'docker') diff --git a/docker/Dockerfile-httpfrontend b/docker/Dockerfile-httpfrontend deleted file mode 100644 index dc97b80..0000000 --- a/docker/Dockerfile-httpfrontend +++ /dev/null @@ -1,25 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Dell EMC and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -# Docker container for StorPerf HTTP Front ENd -# -# Build: -# $ docker build -t opnfv/storperf-frontend:tag . -## - - -FROM nginx:stable-alpine -MAINTAINER Mark Beierl -LABEL version="3.1" description="OPNFV Storperf HTTP Front End Container" - -ARG BRANCH=master - -COPY storperf-httpfrontend/nginx.conf /etc/nginx/nginx.conf -COPY storperf-httpfrontend/html /etc/nginx/html - -EXPOSE 5000 diff --git a/docker/Dockerfile-master b/docker/Dockerfile-master deleted file mode 100644 index 38e8265..0000000 --- a/docker/Dockerfile-master +++ /dev/null @@ -1,99 +0,0 @@ -############################################################################## -# Copyright (c) 2015 EMC and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -# Docker container for STORPERF -# -# Purpose: docker image for Storperf project -# -# Maintained by Jose Lausuch, Mark Beierl -# Build: -# $ docker build -t opnfv/storperf-master:tag . -# - -FROM ubuntu:14.04 -MAINTAINER Jose Lausuch -LABEL version="5.0" description="OPNFV Storperf Docker container" - -ARG BRANCH=master - -ENV repos_dir /home/opnfv/repos -ENV DEBIAN_FRONTEND noninteractive - -WORKDIR /home/opnfv - -# Needed for Graphite installation -RUN echo 'deb http://us.archive.ubuntu.com/ubuntu/ trusty universe' >> /etc/apt/sources.list - -# Packaged dependencies -RUN apt-get update && apt-get install -y \ -libaio1 \ -libaio-dev \ -zlib1g-dev \ -supervisor \ -libssl-dev \ -libffi-dev \ -git \ -build-essential \ -python-dev \ -python-pip \ ---no-install-recommends - -RUN mkdir -p /root/.ssh -RUN chmod 700 /root/.ssh - -# Install required packages - -RUN apt-get -y install python-ldap python-cairo python-django python-twisted python-django-tagging python-simplejson python-memcache python-pysqlite2 python-support python-tz python-pip gunicorn supervisor nginx-light -RUN pip install whisper==0.9.15 -RUN pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/lib" carbon==0.9.15 -RUN pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/webapp" graphite-web==0.9.15 - -# Add graphite config - -ADD graphite/nginx.conf /etc/nginx/nginx.conf -ADD graphite/initial_data.json /var/lib/graphite/webapp/graphite/initial_data.json -ADD graphite/local_settings.py /var/lib/graphite/webapp/graphite/local_settings.py -ADD graphite/carbon.conf /var/lib/graphite/conf/carbon.conf -ADD graphite/storage-schemas.conf /var/lib/graphite/conf/storage-schemas.conf -RUN mkdir -p /opt/graphite/storage -RUN ln -s /var/lib/graphite/storage/whisper /opt/graphite/storage/whisper -RUN touch /var/lib/graphite/storage/graphite.db /var/lib/graphite/storage/index -RUN chown -R www-data /var/lib/graphite/storage -RUN chmod 0775 /var/lib/graphite/storage /var/lib/graphite/storage/whisper -RUN chmod 0664 /var/lib/graphite/storage/graphite.db -RUN cd /var/lib/graphite/webapp/graphite && python manage.py syncdb --noinput - - -# Git configuration - -RUN git config --global http.sslVerify false -RUN git clone --depth 1 https://gerrit.opnfv.org/gerrit/releng ${repos_dir}/releng - - -# Third party git fetches - -RUN git clone http://git.kernel.dk/fio.git ${repos_dir}/fio -RUN cd ${repos_dir}/fio && git checkout tags/fio-2.2.10 -#RUN cd ${repos_dir}/fio && make install - - -# Install StorPerf - -COPY storperf-master ${repos_dir}/storperf -RUN chmod 600 ${repos_dir}/storperf/storperf/resources/ssh/storperf_rsa -RUN pip install --upgrade setuptools==33.1.1 -RUN pip install -r ${repos_dir}/storperf/requirements.pip -COPY storperf-master/supervisord.conf /etc/supervisor/conf.d/supervisord.conf - -# Graphite -EXPOSE 8000 - -# ReST API -EXPOSE 5000 - -CMD ["/usr/bin/supervisord"] diff --git a/docker/Dockerfile-reporting b/docker/Dockerfile-reporting deleted file mode 100644 index ad00019..0000000 --- a/docker/Dockerfile-reporting +++ /dev/null @@ -1,33 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Dell EMC and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -# Docker container for StorPerf Graphing and Reporting module -# -# Purpose: self-contained image that can create reports and graphs of data -# -# Build: -# $ docker build -t opnfv/storperf-reporting:tag . -## - - -FROM alpine:3.1 -MAINTAINER Mark Beierl -LABEL version="0.1" description="OPNFV Storperf Reporting Container" - -ARG BRANCH=master - -RUN apk add --update python py-pip - -COPY storperf-reporting /home/opnfv/storperf-reporting -WORKDIR /home/opnfv/storperf-reporting - -RUN pip install -r /home/opnfv/storperf-reporting/requirements.txt -ENTRYPOINT ["python"] -CMD ["/home/opnfv/storperf-reporting/src/app.py"] - -EXPOSE 5000 \ No newline at end of file diff --git a/docker/graphite/carbon.conf b/docker/graphite/carbon.conf deleted file mode 100644 index 13088dd..0000000 --- a/docker/graphite/carbon.conf +++ /dev/null @@ -1,80 +0,0 @@ -[cache] -LOCAL_DATA_DIR = /var/lib/graphite/storage/whisper/ - -# Specify the user to drop privileges to -# If this is blank carbon runs as the user that invokes it -# This user must have write access to the local data directory -USER = - -# Limit the size of the cache to avoid swapping or becoming CPU bound. -# Sorts and serving cache queries gets more expensive as the cache grows. -# Use the value "inf" (infinity) for an unlimited cache size. -MAX_CACHE_SIZE = inf - -# Limits the number of whisper update_many() calls per second, which effectively -# means the number of write requests sent to the disk. This is intended to -# prevent over-utilizing the disk and thus starving the rest of the system. -# When the rate of required updates exceeds this, then carbon's caching will -# take effect and increase the overall throughput accordingly. -MAX_UPDATES_PER_SECOND = 1000 - -# Softly limits the number of whisper files that get created each minute. -# Setting this value low (like at 50) is a good way to ensure your graphite -# system will not be adversely impacted when a bunch of new metrics are -# sent to it. The trade off is that it will take much longer for those metrics' -# database files to all get created and thus longer until the data becomes usable. -# Setting this value high (like "inf" for infinity) will cause graphite to create -# the files quickly but at the risk of slowing I/O down considerably for a while. -MAX_CREATES_PER_MINUTE = inf - -LINE_RECEIVER_INTERFACE = 0.0.0.0 -LINE_RECEIVER_PORT = 2003 - -ENABLE_UDP_LISTENER = True -UDP_RECEIVER_INTERFACE = 0.0.0.0 -UDP_RECEIVER_PORT = 2003 - -PICKLE_RECEIVER_INTERFACE = 0.0.0.0 -PICKLE_RECEIVER_PORT = 2004 - -CACHE_QUERY_INTERFACE = 0.0.0.0 -CACHE_QUERY_PORT = 7002 - -LOG_UPDATES = False - -# Enable AMQP if you want to receve metrics using an amqp broker -# ENABLE_AMQP = False - -# Verbose means a line will be logged for every metric received -# useful for testing -# AMQP_VERBOSE = False - -# AMQP_HOST = localhost -# AMQP_PORT = 5672 -# AMQP_VHOST = / -# AMQP_USER = guest -# AMQP_PASSWORD = guest -# AMQP_EXCHANGE = graphite - -# Patterns for all of the metrics this machine will store. Read more at -# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings -# -# Example: store all sales, linux servers, and utilization metrics -# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization -# -# Example: store everything -# BIND_PATTERNS = # - -# NOTE: you cannot run both a cache and a relay on the same server -# with the default configuration, you have to specify a distinict -# interfaces and ports for the listeners. - -[relay] -LINE_RECEIVER_INTERFACE = 0.0.0.0 -LINE_RECEIVER_PORT = 2003 - -PICKLE_RECEIVER_INTERFACE = 0.0.0.0 -PICKLE_RECEIVER_PORT = 2004 - -CACHE_SERVERS = server1, server2, server3 -MAX_QUEUE_SIZE = 10000 diff --git a/docker/graphite/initial_data.json b/docker/graphite/initial_data.json deleted file mode 100644 index b3ac9b1..0000000 --- a/docker/graphite/initial_data.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "pk": 1, - "model": "auth.user", - "fields": { - "username": "admin", - "first_name": "", - "last_name": "", - "is_active": true, - "is_superuser": true, - "is_staff": true, - "last_login": "2011-09-20 17:02:14", - "groups": [], - "user_permissions": [], - "password": "sha1$1b11b$edeb0a67a9622f1f2cfeabf9188a711f5ac7d236", - "email": "root@example.com", - "date_joined": "2011-09-20 17:02:14" - } - } -] diff --git a/docker/graphite/local_settings.py b/docker/graphite/local_settings.py deleted file mode 100644 index 177d674..0000000 --- a/docker/graphite/local_settings.py +++ /dev/null @@ -1,42 +0,0 @@ -# Edit this file to override the default graphite settings, do not edit settings.py - -# Turn on debugging and restart apache if you ever see an "Internal Server Error" page -#DEBUG = True - -# Set your local timezone (django will try to figure this out automatically) -TIME_ZONE = 'UTC' - -# Setting MEMCACHE_HOSTS to be empty will turn off use of memcached entirely -#MEMCACHE_HOSTS = ['127.0.0.1:11211'] - -# Sometimes you need to do a lot of rendering work but cannot share your storage mount -#REMOTE_RENDERING = True -#RENDERING_HOSTS = ['fastserver01','fastserver02'] -#LOG_RENDERING_PERFORMANCE = True -#LOG_CACHE_PERFORMANCE = True - -# If you've got more than one backend server they should all be listed here -#CLUSTER_SERVERS = [] - -# Override this if you need to provide documentation specific to your graphite deployment -#DOCUMENTATION_URL = "http://wiki.mycompany.com/graphite" - -# Enable email-related features -#SMTP_SERVER = "mail.mycompany.com" - -# LDAP / ActiveDirectory authentication setup -#USE_LDAP_AUTH = True -#LDAP_SERVER = "ldap.mycompany.com" -#LDAP_PORT = 389 -#LDAP_SEARCH_BASE = "OU=users,DC=mycompany,DC=com" -#LDAP_BASE_USER = "CN=some_readonly_account,DC=mycompany,DC=com" -#LDAP_BASE_PASS = "readonly_account_password" -#LDAP_USER_QUERY = "(username=%s)" #For Active Directory use "(sAMAccountName=%s)" - -# If sqlite won't cut it, configure your real database here (don't forget to run manage.py syncdb!) -#DATABASE_ENGINE = 'mysql' # or 'postgres' -#DATABASE_NAME = 'graphite' -#DATABASE_USER = 'graphite' -#DATABASE_PASSWORD = 'graphite-is-awesome' -#DATABASE_HOST = 'mysql.mycompany.com' -#DATABASE_PORT = '3306' diff --git a/docker/graphite/nginx.conf b/docker/graphite/nginx.conf deleted file mode 100644 index 8a11e94..0000000 --- a/docker/graphite/nginx.conf +++ /dev/null @@ -1,69 +0,0 @@ -daemon off; -user www-data; -worker_processes 1; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - server_tokens off; - - server_names_hash_bucket_size 32; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - gzip on; - gzip_disable "msie6"; - - server { - listen 8000 default_server; - server_name _; - - open_log_file_cache max=1000 inactive=20s min_uses=2 valid=1m; - - location / { - proxy_pass http://127.0.0.1:8080; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Server $host; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header Host $http_host; - - client_max_body_size 10m; - client_body_buffer_size 128k; - - proxy_connect_timeout 90; - proxy_send_timeout 90; - proxy_read_timeout 90; - - proxy_buffer_size 4k; - proxy_buffers 4 32k; - proxy_busy_buffers_size 64k; - proxy_temp_file_write_size 64k; - } - - add_header Access-Control-Allow-Origin "*"; - add_header Access-Control-Allow-Methods "GET, OPTIONS"; - add_header Access-Control-Allow-Headers "origin, authorization, accept"; - - location /content { - alias /var/lib/graphite/webapp/content; - } - - location /media { - alias /usr/share/pyshared/django/contrib/admin/media; - } - } -} diff --git a/docker/graphite/storage-schemas.conf b/docker/graphite/storage-schemas.conf deleted file mode 100644 index 855a9e4..0000000 --- a/docker/graphite/storage-schemas.conf +++ /dev/null @@ -1,7 +0,0 @@ -[carbon] -pattern = ^carbon\..* -retentions = 1m:31d,10m:1y,1h:5y - -[default] -pattern = .* -retentions = 10s:8d,1m:31d,10m:1y,1h:5y diff --git a/docker/local-docker-compose.yaml b/docker/local-docker-compose.yaml new file mode 100644 index 0000000..75cf429 --- /dev/null +++ b/docker/local-docker-compose.yaml @@ -0,0 +1,43 @@ +############################################################################## +# Copyright (c) 2017 Dell EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +version: '2' +services: + + storperf-master: + container_name: "storperf-master" + build: storperf-master + ports: + - "8000:8000" + env_file: ${ENV_FILE} + volumes: + - ${CARBON_DIR}:/opt/graphite/storage/whisper + - ./storperf-master/:/home/opnfv/repos/storperf + + storperf-reporting: + container_name: "storperf-reporting" + build: storperf-reporting + ports: + - "5080:5000" + volumes: + - ./storperf-reporting/:/home/opnfv/storperf-reporting + + storperf-swaggerui: + container_name: "storperf-swaggerui" + image: "schickling/swagger-ui" + + storperf-httpfrontend: + container_name: "storperf-httpfrontend" + build: storperf-httpfrontend + ports: + - "5000:5000" + links: + - storperf-master + - storperf-reporting + - storperf-swaggerui \ No newline at end of file diff --git a/docker/storperf-httpfrontend/Dockerfile b/docker/storperf-httpfrontend/Dockerfile new file mode 100644 index 0000000..4069d24 --- /dev/null +++ b/docker/storperf-httpfrontend/Dockerfile @@ -0,0 +1,23 @@ +############################################################################## +# Copyright (c) 2017 Dell EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# Docker container for StorPerf HTTP Front ENd +# +# Build: +# $ docker build -t opnfv/storperf-frontend:tag . +## + + +FROM nginx:stable-alpine +MAINTAINER Mark Beierl +LABEL version="3.1" description="OPNFV Storperf HTTP Front End Container" + +COPY ./nginx.conf /etc/nginx/nginx.conf +COPY ./html /etc/nginx/html + +EXPOSE 5000 diff --git a/docker/storperf-master/Dockerfile b/docker/storperf-master/Dockerfile new file mode 100644 index 0000000..aa52fbb --- /dev/null +++ b/docker/storperf-master/Dockerfile @@ -0,0 +1,92 @@ +############################################################################## +# Copyright (c) 2015 EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# Docker container for STORPERF +# +# Purpose: docker image for Storperf project +# +# Maintained by Jose Lausuch, Mark Beierl +# Build: +# $ docker build -t opnfv/storperf-master:tag . +# + +FROM ubuntu:14.04 +MAINTAINER Jose Lausuch +LABEL version="5.0" description="OPNFV Storperf Docker container" + +ENV repos_dir /home/opnfv/repos +ENV DEBIAN_FRONTEND noninteractive + +WORKDIR /home/opnfv + +# Needed for Graphite installation +RUN echo 'deb http://us.archive.ubuntu.com/ubuntu/ trusty universe' >> /etc/apt/sources.list + +# Packaged dependencies +RUN apt-get update && apt-get install -y \ +libaio1 \ +libaio-dev \ +zlib1g-dev \ +supervisor \ +libssl-dev \ +libffi-dev \ +git \ +build-essential \ +python-dev \ +python-pip \ +--no-install-recommends + +RUN mkdir -p /root/.ssh +RUN chmod 700 /root/.ssh + +# Install required packages + +RUN apt-get -y install python-ldap python-cairo python-django python-twisted python-django-tagging python-simplejson python-memcache python-pysqlite2 python-support python-tz python-pip gunicorn supervisor nginx-light +RUN pip install whisper==0.9.15 +RUN pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/lib" carbon==0.9.15 +RUN pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/webapp" graphite-web==0.9.15 + +# Add graphite config + +ADD graphite/nginx.conf /etc/nginx/nginx.conf +ADD graphite/initial_data.json /var/lib/graphite/webapp/graphite/initial_data.json +ADD graphite/local_settings.py /var/lib/graphite/webapp/graphite/local_settings.py +ADD graphite/carbon.conf /var/lib/graphite/conf/carbon.conf +ADD graphite/storage-schemas.conf /var/lib/graphite/conf/storage-schemas.conf +RUN mkdir -p /opt/graphite/storage +RUN ln -s /var/lib/graphite/storage/whisper /opt/graphite/storage/whisper +RUN touch /var/lib/graphite/storage/graphite.db /var/lib/graphite/storage/index +RUN chown -R www-data /var/lib/graphite/storage +RUN chmod 0775 /var/lib/graphite/storage /var/lib/graphite/storage/whisper +RUN chmod 0664 /var/lib/graphite/storage/graphite.db +RUN cd /var/lib/graphite/webapp/graphite && python manage.py syncdb --noinput + + +# Third party git fetches + +RUN git config --global http.sslVerify false +RUN git clone http://git.kernel.dk/fio.git ${repos_dir}/fio +RUN cd ${repos_dir}/fio && git checkout tags/fio-2.2.10 +#RUN cd ${repos_dir}/fio && make install + + +# Install StorPerf + +COPY . ${repos_dir}/storperf +RUN chmod 600 ${repos_dir}/storperf/storperf/resources/ssh/storperf_rsa +RUN pip install --upgrade setuptools==33.1.1 +RUN pip install -r ${repos_dir}/storperf/requirements.pip +COPY ./supervisord.conf /etc/supervisor/conf.d/supervisord.conf + +# Graphite +EXPOSE 8000 + +# ReST API +EXPOSE 5000 + +CMD ["/usr/bin/supervisord"] diff --git a/docker/storperf-master/graphite/carbon.conf b/docker/storperf-master/graphite/carbon.conf new file mode 100644 index 0000000..13088dd --- /dev/null +++ b/docker/storperf-master/graphite/carbon.conf @@ -0,0 +1,80 @@ +[cache] +LOCAL_DATA_DIR = /var/lib/graphite/storage/whisper/ + +# Specify the user to drop privileges to +# If this is blank carbon runs as the user that invokes it +# This user must have write access to the local data directory +USER = + +# Limit the size of the cache to avoid swapping or becoming CPU bound. +# Sorts and serving cache queries gets more expensive as the cache grows. +# Use the value "inf" (infinity) for an unlimited cache size. +MAX_CACHE_SIZE = inf + +# Limits the number of whisper update_many() calls per second, which effectively +# means the number of write requests sent to the disk. This is intended to +# prevent over-utilizing the disk and thus starving the rest of the system. +# When the rate of required updates exceeds this, then carbon's caching will +# take effect and increase the overall throughput accordingly. +MAX_UPDATES_PER_SECOND = 1000 + +# Softly limits the number of whisper files that get created each minute. +# Setting this value low (like at 50) is a good way to ensure your graphite +# system will not be adversely impacted when a bunch of new metrics are +# sent to it. The trade off is that it will take much longer for those metrics' +# database files to all get created and thus longer until the data becomes usable. +# Setting this value high (like "inf" for infinity) will cause graphite to create +# the files quickly but at the risk of slowing I/O down considerably for a while. +MAX_CREATES_PER_MINUTE = inf + +LINE_RECEIVER_INTERFACE = 0.0.0.0 +LINE_RECEIVER_PORT = 2003 + +ENABLE_UDP_LISTENER = True +UDP_RECEIVER_INTERFACE = 0.0.0.0 +UDP_RECEIVER_PORT = 2003 + +PICKLE_RECEIVER_INTERFACE = 0.0.0.0 +PICKLE_RECEIVER_PORT = 2004 + +CACHE_QUERY_INTERFACE = 0.0.0.0 +CACHE_QUERY_PORT = 7002 + +LOG_UPDATES = False + +# Enable AMQP if you want to receve metrics using an amqp broker +# ENABLE_AMQP = False + +# Verbose means a line will be logged for every metric received +# useful for testing +# AMQP_VERBOSE = False + +# AMQP_HOST = localhost +# AMQP_PORT = 5672 +# AMQP_VHOST = / +# AMQP_USER = guest +# AMQP_PASSWORD = guest +# AMQP_EXCHANGE = graphite + +# Patterns for all of the metrics this machine will store. Read more at +# http://en.wikipedia.org/wiki/Advanced_Message_Queuing_Protocol#Bindings +# +# Example: store all sales, linux servers, and utilization metrics +# BIND_PATTERNS = sales.#, servers.linux.#, #.utilization +# +# Example: store everything +# BIND_PATTERNS = # + +# NOTE: you cannot run both a cache and a relay on the same server +# with the default configuration, you have to specify a distinict +# interfaces and ports for the listeners. + +[relay] +LINE_RECEIVER_INTERFACE = 0.0.0.0 +LINE_RECEIVER_PORT = 2003 + +PICKLE_RECEIVER_INTERFACE = 0.0.0.0 +PICKLE_RECEIVER_PORT = 2004 + +CACHE_SERVERS = server1, server2, server3 +MAX_QUEUE_SIZE = 10000 diff --git a/docker/storperf-master/graphite/initial_data.json b/docker/storperf-master/graphite/initial_data.json new file mode 100644 index 0000000..b3ac9b1 --- /dev/null +++ b/docker/storperf-master/graphite/initial_data.json @@ -0,0 +1,20 @@ +[ + { + "pk": 1, + "model": "auth.user", + "fields": { + "username": "admin", + "first_name": "", + "last_name": "", + "is_active": true, + "is_superuser": true, + "is_staff": true, + "last_login": "2011-09-20 17:02:14", + "groups": [], + "user_permissions": [], + "password": "sha1$1b11b$edeb0a67a9622f1f2cfeabf9188a711f5ac7d236", + "email": "root@example.com", + "date_joined": "2011-09-20 17:02:14" + } + } +] diff --git a/docker/storperf-master/graphite/local_settings.py b/docker/storperf-master/graphite/local_settings.py new file mode 100644 index 0000000..7cff8f7 --- /dev/null +++ b/docker/storperf-master/graphite/local_settings.py @@ -0,0 +1 @@ +TIME_ZONE = 'UTC' diff --git a/docker/storperf-master/graphite/nginx.conf b/docker/storperf-master/graphite/nginx.conf new file mode 100644 index 0000000..8a11e94 --- /dev/null +++ b/docker/storperf-master/graphite/nginx.conf @@ -0,0 +1,69 @@ +daemon off; +user www-data; +worker_processes 1; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + server_tokens off; + + server_names_hash_bucket_size 32; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + gzip on; + gzip_disable "msie6"; + + server { + listen 8000 default_server; + server_name _; + + open_log_file_cache max=1000 inactive=20s min_uses=2 valid=1m; + + location / { + proxy_pass http://127.0.0.1:8080; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header Host $http_host; + + client_max_body_size 10m; + client_body_buffer_size 128k; + + proxy_connect_timeout 90; + proxy_send_timeout 90; + proxy_read_timeout 90; + + proxy_buffer_size 4k; + proxy_buffers 4 32k; + proxy_busy_buffers_size 64k; + proxy_temp_file_write_size 64k; + } + + add_header Access-Control-Allow-Origin "*"; + add_header Access-Control-Allow-Methods "GET, OPTIONS"; + add_header Access-Control-Allow-Headers "origin, authorization, accept"; + + location /content { + alias /var/lib/graphite/webapp/content; + } + + location /media { + alias /usr/share/pyshared/django/contrib/admin/media; + } + } +} diff --git a/docker/storperf-master/graphite/storage-schemas.conf b/docker/storperf-master/graphite/storage-schemas.conf new file mode 100644 index 0000000..855a9e4 --- /dev/null +++ b/docker/storperf-master/graphite/storage-schemas.conf @@ -0,0 +1,7 @@ +[carbon] +pattern = ^carbon\..* +retentions = 1m:31d,10m:1y,1h:5y + +[default] +pattern = .* +retentions = 10s:8d,1m:31d,10m:1y,1h:5y diff --git a/docker/storperf-reporting/Dockerfile b/docker/storperf-reporting/Dockerfile new file mode 100644 index 0000000..606bac0 --- /dev/null +++ b/docker/storperf-reporting/Dockerfile @@ -0,0 +1,30 @@ +############################################################################## +# Copyright (c) 2017 Dell EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# Docker container for StorPerf Graphing and Reporting module +# +# Purpose: self-contained image that can create reports and graphs of data +# +# Build: +# $ docker build -t opnfv/storperf-reporting:tag . +## + + +FROM alpine:3.1 +MAINTAINER Mark Beierl +LABEL version="0.1" description="OPNFV Storperf Reporting Container" + +RUN apk add --update python py-pip + +COPY . /home/opnfv/storperf-reporting +WORKDIR /home/opnfv/storperf-reporting +RUN pip install -r /home/opnfv/storperf-reporting/requirements.txt + +CMD ["python", "app.py"] + +EXPOSE 5000 \ No newline at end of file -- cgit 1.2.3-korg