diff options
29 files changed, 632 insertions, 321 deletions
diff --git a/docker/local-docker-compose.yaml b/docker/local-docker-compose.yaml index 97c77ec..fe83d22 100644 --- a/docker/local-docker-compose.yaml +++ b/docker/local-docker-compose.yaml @@ -17,11 +17,13 @@ services: args: ARCH: ${ARCH} ports: - - "8000:8000" + - "8001:8000" env_file: ${ENV_FILE} volumes: - ${CARBON_DIR}:/opt/graphite/storage/whisper - ./storperf-master/:/storperf + links: + - storperf-graphite storperf-reporting: container_name: "storperf-reporting" @@ -39,6 +41,12 @@ services: args: ARCH: ${ARCH} + storperf-graphite: + container_name: "storperf-graphite" + build: storperf-graphite + ports: + - "8080:8080" + storperf-httpfrontend: container_name: "storperf-httpfrontend" build: @@ -50,4 +58,5 @@ services: links: - storperf-master - storperf-reporting - - storperf-swaggerui
\ No newline at end of file + - storperf-swaggerui + - storperf-graphite diff --git a/docker/storperf-graphite/Dockerfile b/docker/storperf-graphite/Dockerfile new file mode 100644 index 0000000..b566458 --- /dev/null +++ b/docker/storperf-graphite/Dockerfile @@ -0,0 +1,66 @@ +############################################################################## +# Copyright (c) 2017 Dell EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# Docker container for StorPerf HTTP Graphite +# +# Build: +# $ docker build -t opnfv/storperf-graphite:tag . +## + +# From https://github.com/SchweizerischeBundesbahnen/docker-graphite + +ARG ARCH=x86_64 +ARG ALPINE_VERSION=v3.5 +FROM multiarch/alpine:$ARCH-$ALPINE_VERSION + +# Install basic stuff =) +RUN apk add --no-cache \ + bash \ + ca-certificates \ + nginx \ + openssl \ + py2-pip \ + supervisor \ + tini \ + && pip install \ + supervisor-stdout \ + gunicorn + +# Install graphite +ENV GRAPHITE_ROOT /opt/graphite + +RUN apk add --no-cache \ + alpine-sdk \ + fontconfig \ + libffi \ + libffi-dev \ + python-dev \ + py-cairo \ + && export PYTHONPATH="/opt/graphite/lib/:/opt/graphite/webapp/" \ + && pip install https://github.com/graphite-project/whisper/tarball/master \ + && pip install https://github.com/graphite-project/carbon/tarball/master \ + && pip install https://github.com/graphite-project/graphite-web/tarball/master \ + && apk del \ + alpine-sdk \ + python-dev \ + libffi-dev + +EXPOSE 8080 +EXPOSE 2003 +EXPOSE 2004 +EXPOSE 7002 + +VOLUME ["/opt/graphite/conf", "/opt/graphite/storage"] + +COPY run.sh /run.sh +COPY etc/ /etc/ +COPY conf/ /opt/graphite/conf.example/ + +# Enable tiny init +ENTRYPOINT ["/sbin/tini", "--"] +CMD ["/bin/bash", "/run.sh"] diff --git a/docker/storperf-master/graphite/carbon.conf b/docker/storperf-graphite/conf/carbon.conf index 13088dd..6463c79 100644 --- a/docker/storperf-master/graphite/carbon.conf +++ b/docker/storperf-graphite/conf/carbon.conf @@ -1,10 +1,10 @@ [cache] -LOCAL_DATA_DIR = /var/lib/graphite/storage/whisper/ +LOCAL_DATA_DIR = /opt/graphite/storage/whisper/ # Specify the user to drop privileges to # If this is blank carbon runs as the user that invokes it # This user must have write access to the local data directory -USER = +USER = # Limit the size of the cache to avoid swapping or becoming CPU bound. # Sorts and serving cache queries gets more expensive as the cache grows. @@ -30,17 +30,18 @@ MAX_CREATES_PER_MINUTE = inf LINE_RECEIVER_INTERFACE = 0.0.0.0 LINE_RECEIVER_PORT = 2003 -ENABLE_UDP_LISTENER = True -UDP_RECEIVER_INTERFACE = 0.0.0.0 -UDP_RECEIVER_PORT = 2003 - PICKLE_RECEIVER_INTERFACE = 0.0.0.0 PICKLE_RECEIVER_PORT = 2004 CACHE_QUERY_INTERFACE = 0.0.0.0 CACHE_QUERY_PORT = 7002 +# By default, carbon-cache will log every whisper update and cache hit. This can be excessive and +# degrade performance if logging on the same volume as the whisper data is stored. LOG_UPDATES = False +LOG_CACHE_HITS = False +ENABLE_LOGROTATION = True +LOG_LISTENER_CONNECTIONS = False # Enable AMQP if you want to receve metrics using an amqp broker # ENABLE_AMQP = False @@ -68,13 +69,3 @@ LOG_UPDATES = False # NOTE: you cannot run both a cache and a relay on the same server # with the default configuration, you have to specify a distinict # interfaces and ports for the listeners. - -[relay] -LINE_RECEIVER_INTERFACE = 0.0.0.0 -LINE_RECEIVER_PORT = 2003 - -PICKLE_RECEIVER_INTERFACE = 0.0.0.0 -PICKLE_RECEIVER_PORT = 2004 - -CACHE_SERVERS = server1, server2, server3 -MAX_QUEUE_SIZE = 10000 diff --git a/docker/storperf-graphite/conf/local_settings.py b/docker/storperf-graphite/conf/local_settings.py new file mode 100644 index 0000000..88414aa --- /dev/null +++ b/docker/storperf-graphite/conf/local_settings.py @@ -0,0 +1,13 @@ +# flake8: noqa +# Edit this file to override the default graphite settings, do not edit settings.py + +# Turn on debugging and restart apache if you ever see an "Internal Server Error" page +# DEBUG = True + +# Set your local timezone (django will try to figure this out automatically) +TIME_ZONE = 'Europe/Zurich' + +# Secret key for django +SECRET_KEY = '%%SECRET_KEY%%' + +URL_PREFIX = "/graphite/" diff --git a/docker/storperf-graphite/conf/storage-aggregation.conf b/docker/storperf-graphite/conf/storage-aggregation.conf new file mode 100644 index 0000000..bc5e1db --- /dev/null +++ b/docker/storperf-graphite/conf/storage-aggregation.conf @@ -0,0 +1,29 @@ +[min] +pattern = \.lower$ +xFilesFactor = 0.1 +aggregationMethod = min + +[max] +pattern = \.upper(_\d+)?$ +xFilesFactor = 0.1 +aggregationMethod = max + +[sum] +pattern = \.sum$ +xFilesFactor = 0 +aggregationMethod = sum + +[count] +pattern = \.count$ +xFilesFactor = 0 +aggregationMethod = sum + +[count_legacy] +pattern = ^stats_counts.* +xFilesFactor = 0 +aggregationMethod = sum + +[default_average] +pattern = .* +xFilesFactor = 0.3 +aggregationMethod = average diff --git a/docker/storperf-graphite/conf/storage-schemas.conf b/docker/storperf-graphite/conf/storage-schemas.conf new file mode 100644 index 0000000..11a59be --- /dev/null +++ b/docker/storperf-graphite/conf/storage-schemas.conf @@ -0,0 +1,3 @@ +[default] +pattern = .* +retentions = 60s:14d diff --git a/docker/storperf-graphite/etc/nginx/conf.d/graphite b/docker/storperf-graphite/etc/nginx/conf.d/graphite new file mode 100644 index 0000000..e4c405d --- /dev/null +++ b/docker/storperf-graphite/etc/nginx/conf.d/graphite @@ -0,0 +1,37 @@ +server { + listen 8080; + server_name graphite; + charset utf-8; + # Django admin media. + location /graphite/static/admin/ { + alias /usr/lib/python2.7/site-packages/django/contrib/admin/static/admin/; + } + + # Your project's static media. + location /graphite/static/ { + alias /opt/graphite/webapp/content/; + } + + # Finally, send all non-media requests to the Django server. + location / { + proxy_pass http://127.0.0.1:8000; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Forwarded-Host $host; + proxy_set_header Host $host; + + client_max_body_size 10m; + client_body_buffer_size 128k; + + proxy_connect_timeout 90; + proxy_send_timeout 90; + proxy_read_timeout 90; + + proxy_buffer_size 4k; + proxy_buffers 4 32k; + proxy_busy_buffers_size 64k; + proxy_temp_file_write_size 64k; + } +} diff --git a/docker/storperf-graphite/etc/nginx/nginx.conf b/docker/storperf-graphite/etc/nginx/nginx.conf new file mode 100644 index 0000000..f2ab7f7 --- /dev/null +++ b/docker/storperf-graphite/etc/nginx/nginx.conf @@ -0,0 +1,20 @@ +worker_processes 1; +pid /var/run/nginx.pid; +daemon off; + +events { + worker_connections 1024; + use epoll; +} + +http { + include mime.types; + default_type application/octet-stream; + + sendfile on; + keepalive_timeout 65; + + gzip on; + + include /etc/nginx/conf.d/*; +} diff --git a/docker/storperf-graphite/etc/supervisor.d/carbon.ini b/docker/storperf-graphite/etc/supervisor.d/carbon.ini new file mode 100644 index 0000000..fb93a95 --- /dev/null +++ b/docker/storperf-graphite/etc/supervisor.d/carbon.ini @@ -0,0 +1,11 @@ +[program:carbon-cache] +autostart = true +autorestart = true +stdout_events_enabled = true +stderr_events_enabled = true +stdout_logfile_maxbytes = 1MB +stdout_logfile_backups = 0 +stderr_logfile_maxbytes = 1MB +stderr_logfile_backups = 0 + +command = /opt/graphite/bin/carbon-cache.py --pidfile /var/run/carbon-cache-a.pid --debug start diff --git a/docker/storperf-graphite/etc/supervisor.d/gunicorn.ini b/docker/storperf-graphite/etc/supervisor.d/gunicorn.ini new file mode 100644 index 0000000..7a94ac8 --- /dev/null +++ b/docker/storperf-graphite/etc/supervisor.d/gunicorn.ini @@ -0,0 +1,13 @@ +[program:graphite-webapp] +autostart = true +autorestart = true +stdout_events_enabled = true +stderr_events_enabled = true +stdout_logfile_maxbytes = 1MB +stdout_logfile_backups = 0 +stderr_logfile_maxbytes = 1MB +stderr_logfile_backups = 0 + +directory = /opt/graphite/webapp +environment = PYTHONPATH='/opt/graphite/webapp' +command = /usr/bin/gunicorn -b127.0.0.1:8000 -w2 graphite.wsgi diff --git a/docker/storperf-graphite/etc/supervisor.d/nginx.ini b/docker/storperf-graphite/etc/supervisor.d/nginx.ini new file mode 100644 index 0000000..be2615c --- /dev/null +++ b/docker/storperf-graphite/etc/supervisor.d/nginx.ini @@ -0,0 +1,11 @@ +[program:nginx] +autostart = true +autorestart = true +stdout_events_enabled = true +stderr_events_enabled = true +stdout_logfile_maxbytes = 1MB +stdout_logfile_backups = 0 +stderr_logfile_maxbytes = 1MB +stderr_logfile_backups = 0 + +command = /usr/sbin/nginx -c /etc/nginx/nginx.conf diff --git a/docker/storperf-graphite/etc/supervisord.conf b/docker/storperf-graphite/etc/supervisord.conf new file mode 100644 index 0000000..01799ab --- /dev/null +++ b/docker/storperf-graphite/etc/supervisord.conf @@ -0,0 +1,26 @@ +[unix_http_server] +file=/run/supervisord.sock + +[supervisord] +user = root +nodaemon = true +logfile_maxbytes = 10MB +logfile_backups = 0 +pidfile = /tmp/supervisord.pid +logfile = /tmp/supervisord.log +environment = GRAPHITE_STORAGE_DIR='/opt/graphite/storage',GRAPHITE_CONF_DIR='/opt/graphite/conf' + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///run/supervisord.sock + +[eventlistener:stdout] +command = supervisor_stdout +buffer_size = 100 +events = PROCESS_LOG +result_handler = supervisor_stdout:event_handler + +[include] +files = /etc/supervisor.d/*.ini diff --git a/docker/storperf-graphite/run.sh b/docker/storperf-graphite/run.sh new file mode 100644 index 0000000..c9baa93 --- /dev/null +++ b/docker/storperf-graphite/run.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +whisper_dir="/opt/graphite/storage/whisper/" +webapp_dir="/opt/graphite/storage/log/webapp/" + +cd /opt/graphite + +if [ -d $whisper_dir ]; then + echo "Whisper directory already exists" +else + echo "...creating missing whisper dir" + mkdir -p $whisper_dir +fi + +if [ -d $webapp_dir ]; then + echo "Webapp directory already exists" +else + echo "...creating missing webapp dir" + mkdir -p $webapp_dir +fi + +if [ ! -f /opt/graphite/conf/local_settings.py ]; then + echo "Creating default config for graphite-web..." + cp /opt/graphite/conf.example/local_settings.py /opt/graphite/conf/local_settings.py + RANDOM_STRING=$(python -c 'import random; import string; print "".join([random.SystemRandom().choice(string.digits + string.letters) for i in range(100)])') + sed "s/%%SECRET_KEY%%/${RANDOM_STRING}/" -i /opt/graphite/conf/local_settings.py +fi + +if [ ! -L /opt/graphite/webapp/graphite/local_settings.py ]; then + echo "Creating symbolic link for local_settings.py in graphite-web..." + ln -s /opt/graphite/conf/local_settings.py /opt/graphite/webapp/graphite/local_settings.py +fi + +sed "s/%%CLUSTER_SERVERS%%/${CLUSTER_SERVERS}/" -i /opt/graphite/conf/local_settings.py + +if [ ! -f /opt/graphite/conf/carbon.conf ]; then + echo "Creating default config for carbon..." + cp /opt/graphite/conf.example/carbon.conf /opt/graphite/conf/carbon.conf +fi + +if [ ! -f /opt/graphite/conf/storage-schemas.conf ]; then + echo "Creating default storage schema for carbon..." + cp /opt/graphite/conf.example/storage-schemas.conf /opt/graphite/conf/storage-schemas.conf +fi + +if [ ! -f /opt/graphite/conf/storage-aggregation.conf ]; then + echo "Creating default storage schema for carbon..." + cp /opt/graphite/conf.example/storage-aggregation.conf /opt/graphite/conf/storage-aggregation.conf +fi + +if [ ! -f /opt/graphite/storage/graphite.db ]; then + echo "Creating database..." + PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py migrate --settings=graphite.settings --run-syncdb --noinput + chown nginx:nginx /opt/graphite/storage/graphite.db + # Auto-magical create an django user with default login + script="from django.contrib.auth.models import User; + +username = 'admin'; +password = 'admin'; +email = 'admin@example.com'; + +if User.objects.filter(username=username).count()==0: + User.objects.create_superuser(username, email, password); + print('Superuser created.'); +else: + print('Superuser creation skipped.'); + +" + printf "$script" | PYTHONPATH=$GRAPHITE_ROOT/webapp django-admin.py shell --settings=graphite.settings +fi + +exec supervisord -c /etc/supervisord.conf diff --git a/docker/storperf-httpfrontend/nginx.conf b/docker/storperf-httpfrontend/nginx.conf index ef9fe82..7f3ddd7 100644 --- a/docker/storperf-httpfrontend/nginx.conf +++ b/docker/storperf-httpfrontend/nginx.conf @@ -21,7 +21,7 @@ http { } location /graphite/ { - proxy_pass http://storperf-master:8000; + proxy_pass http://storperf-graphite:8080; proxy_set_header Host $host:$proxy_port; } diff --git a/docker/storperf-master/Dockerfile b/docker/storperf-master/Dockerfile index 25eb956..eaaf811 100644 --- a/docker/storperf-master/Dockerfile +++ b/docker/storperf-master/Dockerfile @@ -76,57 +76,5 @@ RUN chmod 600 storperf/resources/ssh/storperf_rsa EXPOSE 5000 -# Install Graphite -# Everything from here down will be removed once Graphite/Carbon gets broken -# out into its own container. - -RUN apk --no-cache add --update \ - python \ - py-pip \ - python-dev \ - alpine-sdk \ - py-tz \ - nginx \ - cairo \ - supervisor - -RUN deluser xfs - -RUN pip install \ - gunicorn==17.5 \ - Django==1.6.11 \ - django-tagging==0.3.1 \ - cairocffi \ - constants \ - zope.interface - -RUN adduser -S -g www-data -u 33 www-data - -RUN pip install whisper==0.9.15 -RUN pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/lib" carbon==0.9.15 -RUN pip install --install-option="--prefix=/var/lib/graphite" --install-option="--install-lib=/var/lib/graphite/webapp" graphite-web==0.9.15 - -ADD graphite/nginx.conf /etc/nginx/nginx.conf -ADD graphite/local_settings.py /var/lib/graphite/webapp/graphite/local_settings.py -ADD graphite/carbon.conf /var/lib/graphite/conf/carbon.conf -ADD graphite/storage-schemas.conf /var/lib/graphite/conf/storage-schemas.conf -RUN mkdir -p /opt/graphite/storage -RUN ln -s /var/lib/graphite/storage/whisper /opt/graphite/storage/whisper -RUN touch /var/lib/graphite/storage/graphite.db /var/lib/graphite/storage/index -RUN chown -R www-data /var/lib/graphite/storage -RUN chmod 0775 /var/lib/graphite/storage /var/lib/graphite/storage/whisper -RUN chmod 0664 /var/lib/graphite/storage/graphite.db - -RUN cd /var/lib/graphite/webapp/graphite && python manage.py syncdb --noinput -ADD graphite/initial_data.json /var/lib/graphite/webapp/graphite/initial_data.json -RUN cd /var/lib/graphite/webapp/graphite && python manage.py syncdb --noinput - -RUN mkdir -p /var/log/supervisor - -COPY ./supervisord.conf /etc/supervisor/conf.d/supervisord.conf - -EXPOSE 8000 - # Entry point - -CMD ["/usr/bin/supervisord"] +CMD [ "python", "./rest_server.py" ] diff --git a/docker/storperf-master/graphite/initial_data.json b/docker/storperf-master/graphite/initial_data.json deleted file mode 100644 index b3ac9b1..0000000 --- a/docker/storperf-master/graphite/initial_data.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - { - "pk": 1, - "model": "auth.user", - "fields": { - "username": "admin", - "first_name": "", - "last_name": "", - "is_active": true, - "is_superuser": true, - "is_staff": true, - "last_login": "2011-09-20 17:02:14", - "groups": [], - "user_permissions": [], - "password": "sha1$1b11b$edeb0a67a9622f1f2cfeabf9188a711f5ac7d236", - "email": "root@example.com", - "date_joined": "2011-09-20 17:02:14" - } - } -] diff --git a/docker/storperf-master/graphite/local_settings.py b/docker/storperf-master/graphite/local_settings.py deleted file mode 100644 index 7cff8f7..0000000 --- a/docker/storperf-master/graphite/local_settings.py +++ /dev/null @@ -1 +0,0 @@ -TIME_ZONE = 'UTC' diff --git a/docker/storperf-master/graphite/nginx.conf b/docker/storperf-master/graphite/nginx.conf deleted file mode 100644 index 8a11e94..0000000 --- a/docker/storperf-master/graphite/nginx.conf +++ /dev/null @@ -1,69 +0,0 @@ -daemon off; -user www-data; -worker_processes 1; -pid /var/run/nginx.pid; - -events { - worker_connections 1024; -} - -http { - sendfile on; - tcp_nopush on; - tcp_nodelay on; - keepalive_timeout 65; - types_hash_max_size 2048; - server_tokens off; - - server_names_hash_bucket_size 32; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - access_log /var/log/nginx/access.log; - error_log /var/log/nginx/error.log; - - gzip on; - gzip_disable "msie6"; - - server { - listen 8000 default_server; - server_name _; - - open_log_file_cache max=1000 inactive=20s min_uses=2 valid=1m; - - location / { - proxy_pass http://127.0.0.1:8080; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Server $host; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header Host $http_host; - - client_max_body_size 10m; - client_body_buffer_size 128k; - - proxy_connect_timeout 90; - proxy_send_timeout 90; - proxy_read_timeout 90; - - proxy_buffer_size 4k; - proxy_buffers 4 32k; - proxy_busy_buffers_size 64k; - proxy_temp_file_write_size 64k; - } - - add_header Access-Control-Allow-Origin "*"; - add_header Access-Control-Allow-Methods "GET, OPTIONS"; - add_header Access-Control-Allow-Headers "origin, authorization, accept"; - - location /content { - alias /var/lib/graphite/webapp/content; - } - - location /media { - alias /usr/share/pyshared/django/contrib/admin/media; - } - } -} diff --git a/docker/storperf-master/graphite/storage-schemas.conf b/docker/storperf-master/graphite/storage-schemas.conf deleted file mode 100644 index 855a9e4..0000000 --- a/docker/storperf-master/graphite/storage-schemas.conf +++ /dev/null @@ -1,7 +0,0 @@ -[carbon] -pattern = ^carbon\..* -retentions = 1m:31d,10m:1y,1h:5y - -[default] -pattern = .* -retentions = 10s:8d,1m:31d,10m:1y,1h:5y diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py index 8d1ad78..bd4b66b 100644 --- a/docker/storperf-master/rest_server.py +++ b/docker/storperf-master/rest_server.py @@ -13,12 +13,13 @@ import os import sys from flask import abort, Flask, request, jsonify +from flask_cors import CORS from flask_restful import Resource, Api, fields from flask_restful_swagger import swagger -from flask_cors import CORS from storperf.storperf_master import StorPerfMaster + app = Flask(__name__, static_url_path="") CORS(app) api = swagger.docs(Api(app), apiVersion='1.0') diff --git a/docker/storperf-master/storperf/carbon/emitter.py b/docker/storperf-master/storperf/carbon/emitter.py index e23dc79..05f6c3c 100644 --- a/docker/storperf-master/storperf/carbon/emitter.py +++ b/docker/storperf-master/storperf/carbon/emitter.py @@ -11,28 +11,75 @@ import logging import socket import time +from storperf.db.graphite_db import GraphiteDB + class CarbonMetricTransmitter(): - carbon_host = '127.0.0.1' - carbon_port = 2003 + carbon_servers = [('127.0.0.1', 2003), + ('storperf-graphite', 2003)] def __init__(self): self.logger = logging.getLogger(__name__) + self.graphite_db = GraphiteDB() + self.commit_markers = {} - def transmit_metrics(self, metrics): - if 'timestamp' in metrics: - metrics.pop('timestamp') + def transmit_metrics(self, metrics, commit_marker): timestamp = str(calendar.timegm(time.gmtime())) + self.commit_markers[commit_marker] = int(timestamp) + + carbon_socket = None + + for host, port in self.carbon_servers: + try: + carbon_socket = socket.socket(socket.AF_INET, + socket.SOCK_STREAM) + carbon_socket.connect((host, port)) + + for key, value in metrics.items(): + try: + float(value) + message = "%s %s %s\n" \ + % (key, value, timestamp) + self.logger.debug("Metric: " + message.strip()) + carbon_socket.send(message) + except ValueError: + self.logger.debug("Ignoring non numeric metric %s %s" + % (key, value)) + + message = "%s.commit-marker %s %s\n" \ + % (commit_marker, timestamp, timestamp) + carbon_socket.send(message) + self.logger.debug("Marker %s" % message.strip()) + self.logger.info("Sent metrics to %s:%s with timestamp %s" + % (host, port, timestamp)) + + except Exception, e: + self.logger.error("While notifying carbon %s:%s %s" + % (host, port, e)) + + if carbon_socket is not None: + carbon_socket.close() + + def confirm_commit(self, commit_marker): + marker_timestamp = self.commit_markers[commit_marker] + request = "%s.commit-marker&from=%s" \ + % (commit_marker, marker_timestamp - 60) + marker_data = self.graphite_db.fetch_item(request) + self.logger.debug("Marker data %s" % marker_data) + fetched_timestamps = self.parse_timestamp(marker_data) - carbon_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - carbon_socket.connect((self.carbon_host, self.carbon_port)) + return marker_timestamp in fetched_timestamps - for key, metric in metrics.items(): - message = key + " " + metric + " " + timestamp - self.logger.debug("Metric: " + message) - carbon_socket.send(message + '\n') + def parse_timestamp(self, marker_data): + timestamps = [] + if (type(marker_data) is list and + len(marker_data) > 0): + datapoints = marker_data[0]['datapoints'] + for datapoint in datapoints: + try: + timestamps.append(int(datapoint[0])) + except Exception: + pass - carbon_socket.close() - self.logger.info("Sent metrics to carbon with timestamp %s" - % timestamp) + return timestamps diff --git a/docker/storperf-master/storperf/db/graphite_db.py b/docker/storperf-master/storperf/db/graphite_db.py index c8a2d35..aa71855 100644 --- a/docker/storperf-master/storperf/db/graphite_db.py +++ b/docker/storperf-master/storperf/db/graphite_db.py @@ -9,33 +9,44 @@ import json import logging - import requests -from storperf.db.job_db import JobDB - class GraphiteDB(object): + graphite_host = "storperf-graphite" + graphite_port = 8080 + def __init__(self): - """ - """ - self._job_db = JobDB() self.logger = logging.getLogger(__name__) + def fetch_item(self, target): + + result = None + request = ("http://%s:%s/graphite/render/?format=json&target=%s" + % (self.graphite_host, self.graphite_port, target)) + self.logger.debug("Calling %s" % (request)) + + response = requests.get(request) + if (response.status_code == 200): + result = json.loads(response.content) + + return result + def fetch_series(self, workload, metric, io_type, time, duration): series = [] end = time start = end - duration - request = ("http://127.0.0.1:8000/render/?target=" + request = ("http://%s:%s/graphite/render/?target=" "averageSeries(%s.*.jobs.1.%s.%s)" "&format=json" "&from=%s" - "&until=%s" % - (workload, io_type, metric, - start, end)) + "&until=%s" + % (self.graphite_host, self.graphite_port, + workload, io_type, metric, + start, end)) self.logger.debug("Calling %s" % (request)) response = requests.get(request) diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py index 7f2c395..3b0af78 100644 --- a/docker/storperf-master/storperf/storperf_master.py +++ b/docker/storperf-master/storperf/storperf_master.py @@ -11,19 +11,20 @@ from datetime import datetime import logging import os import socket -from storperf.db.configuration_db import ConfigurationDB -from storperf.db.job_db import JobDB -from storperf.test_executor import TestExecutor from threading import Thread from time import sleep from cinderclient import client as cinderclient -import heatclient.client as heatclient from keystoneauth1 import loading from keystoneauth1 import session import paramiko from scp import SCPClient +import heatclient.client as heatclient +from storperf.db.configuration_db import ConfigurationDB +from storperf.db.job_db import JobDB +from storperf.test_executor import TestExecutor + class ParameterError(Exception): """ """ @@ -257,7 +258,7 @@ class StorPerfMaster(object): str(self._test_executor.workload_modules)) def get_logs(self, lines=None): - LOG_DIR = '/var/log/supervisor/storperf-webapp.log' + LOG_DIR = './storperf.log' if isinstance(lines, int): logs = [] diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py index b2d5914..dc178d8 100644 --- a/docker/storperf-master/storperf/test_executor.py +++ b/docker/storperf-master/storperf/test_executor.py @@ -15,14 +15,16 @@ from os import listdir import os from os.path import isfile, join import sched +from threading import Thread +from time import sleep +import time + from storperf.carbon.converter import Converter from storperf.carbon.emitter import CarbonMetricTransmitter from storperf.db.job_db import JobDB from storperf.fio.fio_invoker import FIOInvoker from storperf.utilities.data_handler import DataHandler from storperf.utilities.thread_gate import ThreadGate -from threading import Thread -import time class UnknownWorkload(Exception): @@ -98,7 +100,14 @@ class TestExecutor(object): metric, callback_id) - self.metrics_emitter.transmit_metrics(carbon_metrics) + self.metrics_emitter.transmit_metrics(carbon_metrics, callback_id) + + commit_count = 10 + while (commit_count > 0 and + not self.metrics_emitter.confirm_commit(callback_id)): + self.logger.info("Waiting 1 more second for commit") + sleep(1) + commit_count -= 1 if self._thread_gate.report(callback_id): self.broadcast_event() @@ -244,11 +253,11 @@ class TestExecutor(object): if self._terminated: return self.current_workload = ( - "%s.%s.queue-depth.%s.block-size.%s" % - (self.job_db.job_id, - workload_name, - iodepth, - blocksize)) + "%s.%s.queue-depth.%s.block-size.%s" + % (self.job_db.job_id, + workload_name, + iodepth, + blocksize)) self.logger.info("Starting run %s" % self.current_workload) self.workload_status[self.current_workload] = "Running" @@ -287,11 +296,11 @@ class TestExecutor(object): if not scheduler.empty(): try: scheduler.cancel(event) - except: + except ValueError: pass - self.logger.info("Completed run %s" % - self.current_workload) + self.logger.info("Completed run %s" + % self.current_workload) self.workload_status[self.current_workload] = "Completed" self._workload_executors = [] self.current_workload = None diff --git a/docker/storperf-master/storperf/utilities/data_handler.py b/docker/storperf-master/storperf/utilities/data_handler.py index 9d20383..44b1f6b 100644 --- a/docker/storperf-master/storperf/utilities/data_handler.py +++ b/docker/storperf-master/storperf/utilities/data_handler.py @@ -9,6 +9,8 @@ import logging import os +import time + from storperf.db import test_results_db from storperf.db.graphite_db import GraphiteDB from storperf.db.job_db import JobDB @@ -16,8 +18,6 @@ from storperf.utilities import data_treatment as DataTreatment from storperf.utilities import dictionary from storperf.utilities import math as math from storperf.utilities import steady_state as SteadyState -from time import sleep -import time class DataHandler(object): @@ -93,31 +93,12 @@ class DataHandler(object): # A bit of a hack here as Carbon might not be finished storing the # data we just sent to it now = int(time.time()) - backtime = 60 * (executor.steady_state_samples + 2) + backtime = 60 * (executor.steady_state_samples + 1) data_series = graphite_db.fetch_series(workload, metric, io_type, now, backtime) - most_recent_time = now - if len(data_series) > 0: - most_recent_time = data_series[-1][0] - - delta = now - most_recent_time - self.logger.debug("Last update to graphite was %s ago" % delta) - - while (delta < 5 or (delta > 60 and delta < 120)): - sleep(5) - data_series = graphite_db.fetch_series(workload, - metric, - io_type, - now, - backtime) - if len(data_series) > 0: - most_recent_time = data_series[-1][0] - delta = time.time() - most_recent_time - self.logger.debug("Last update to graphite was %s ago" % delta) - return data_series def _convert_timestamps_to_samples(self, executor, series): @@ -201,5 +182,6 @@ class DataHandler(object): build_tag, payload) executor.result_url = response['href'] - except: - self.logger.exception("Error pushing results into Database") + except Exception as e: + self.logger.exception("Error pushing results into Database", + e) diff --git a/docker/storperf-master/storperf/workloads/_base_workload.py b/docker/storperf-master/storperf/workloads/_base_workload.py index 936c839..c045278 100644 --- a/docker/storperf-master/storperf/workloads/_base_workload.py +++ b/docker/storperf-master/storperf/workloads/_base_workload.py @@ -74,9 +74,9 @@ class _base_workload(object): @property def fullname(self): - return ("%s.%s.queue-depth.%s.block-size.%s.%s" % - (str(self.id), - self.__class__.__name__, - str(self.options['iodepth']), - str(self.options['bs']), - str(self.remote_host).replace(".", "-"))) + return ("%s.%s.queue-depth.%s.block-size.%s.%s" + % (str(self.id), + self.__class__.__name__, + str(self.options['iodepth']), + str(self.options['bs']), + str(self.remote_host).replace(".", "-"))) diff --git a/docker/storperf-master/supervisord.conf b/docker/storperf-master/supervisord.conf deleted file mode 100644 index 558f6bf..0000000 --- a/docker/storperf-master/supervisord.conf +++ /dev/null @@ -1,33 +0,0 @@ -[supervisord] -nodaemon = true -environment = GRAPHITE_STORAGE_DIR='/var/lib/graphite/storage',GRAPHITE_CONF_DIR='/var/lib/graphite/conf' - -[program:nginx] -command = /usr/sbin/nginx -stdout_logfile = /var/log/supervisor/%(program_name)s.log -stderr_logfile = /var/log/supervisor/%(program_name)s.log -autorestart = true - -[program:carbon-cache] -user = www-data -command = /var/lib/graphite/bin/carbon-cache.py --debug start -stdout_logfile = /var/log/supervisor/%(program_name)s.log -stderr_logfile = /var/log/supervisor/%(program_name)s.log -autorestart = true - -[program:graphite-webapp] -user = www-data -directory = /var/lib/graphite/webapp -environment = PYTHONPATH='/var/lib/graphite/webapp' -command = /usr/bin/gunicorn_django -b127.0.0.1:8080 -w2 graphite/settings.py -stdout_logfile = /var/log/supervisor/%(program_name)s.log -stderr_logfile = /var/log/supervisor/%(program_name)s.log -autorestart = true - -[program:storperf-webapp] -user = root -directory = /storperf/ -command = /usr/bin/python rest_server.py -stdout_logfile = /var/log/supervisor/%(program_name)s.log -stderr_logfile = /var/log/supervisor/%(program_name)s.log -autorestart = true diff --git a/docker/storperf-master/tests/carbon_tests/emitter_test.py b/docker/storperf-master/tests/carbon_tests/emitter_test.py index 7f61049..f5a78d1 100644 --- a/docker/storperf-master/tests/carbon_tests/emitter_test.py +++ b/docker/storperf-master/tests/carbon_tests/emitter_test.py @@ -7,27 +7,39 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -import SocketServer import json -from storperf.carbon import converter -from storperf.carbon.emitter import CarbonMetricTransmitter -import threading -from time import sleep, strptime +from time import strptime import unittest import mock +from storperf.carbon import converter +from storperf.carbon.emitter import CarbonMetricTransmitter + + +addresses = [] +data = [] +connect_exception = [] +send_exception = [] + + +class MockSocket(object): -class MetricsHandler(SocketServer.BaseRequestHandler): + def __init__(self, *args): + pass - def handle(self): - # Echo the back to the client - CarbonMetricTransmitterTest.response = self.request.recv(1024) - return + def connect(self, address): + if len(connect_exception) != 0: + raise connect_exception[0] + addresses.append(address) + def send(self, datum): + if len(send_exception) != 0: + raise send_exception[0] + data.append(datum) -class MetricsServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): - pass + def close(self): + pass class CarbonMetricTransmitterTest(unittest.TestCase): @@ -35,17 +47,37 @@ class CarbonMetricTransmitterTest(unittest.TestCase): response = None def setUp(self): + del addresses[:] + del data[:] + del connect_exception[:] + del send_exception[:] + + @mock.patch("socket.socket") + @mock.patch("time.gmtime") + def test_transmit_metrics(self, mock_time, mock_socket): + + mock_socket.side_effect = MockSocket + + mock_time.return_value = strptime("30 Nov 00", "%d %b %y") + + testconv = converter.Converter() + json_object = json.loads( + """{"timestamp" : "975542400", "key":123.0 }""") + result = testconv.convert_json_to_flat(json_object, "host.run-name") - address = ('localhost', 0) - server = MetricsServer(address, MetricsHandler) - ip, self.listen_port = server.server_address + emitter = CarbonMetricTransmitter() + emitter.carbon_port = self.listen_port + emitter.transmit_metrics(result, None) - t = threading.Thread(target=server.serve_forever) - t.setDaemon(True) - t.start() + self.assertEqual("host.run-name.key 123.0 975542400\n", + data[1], + data[1]) + @mock.patch("socket.socket") @mock.patch("time.gmtime") - def test_transmit_metrics(self, mock_time): + def test_skip_non_numeric_metrics(self, mock_time, mock_socket): + + mock_socket.side_effect = MockSocket mock_time.return_value = strptime("30 Nov 00", "%d %b %y") @@ -56,17 +88,126 @@ class CarbonMetricTransmitterTest(unittest.TestCase): emitter = CarbonMetricTransmitter() emitter.carbon_port = self.listen_port - emitter.transmit_metrics(result) + emitter.transmit_metrics(result, None) + + self.assertEqual("None.commit-marker 975542400 975542400\n", + data[1], + data[1]) + + @mock.patch("socket.socket") + def test_connect_fails(self, mock_socket): + + mock_socket.side_effect = MockSocket + connect_exception.append(Exception("Mock connection error")) + + testconv = converter.Converter() + json_object = json.loads( + """{"timestamp" : "975542400", "key":"value" }""") + result = testconv.convert_json_to_flat(json_object, "host.run-name") + + emitter = CarbonMetricTransmitter() + emitter.carbon_port = self.listen_port + emitter.transmit_metrics(result, None) + + self.assertEqual(0, + len(data), + len(data)) - count = 0 + @mock.patch("socket.socket") + def test_send_fails(self, mock_socket): - while (CarbonMetricTransmitterTest.response is None and count < 10): - count += 1 - sleep(0.1) + mock_socket.side_effect = MockSocket + send_exception.append(Exception("Mock send error")) + + testconv = converter.Converter() + json_object = json.loads( + """{"timestamp" : "975542400", "key":"value" }""") + result = testconv.convert_json_to_flat(json_object, "host.run-name") + + emitter = CarbonMetricTransmitter() + emitter.carbon_port = self.listen_port + emitter.transmit_metrics(result, None) + + self.assertEqual(0, + len(data), + len(data)) + + @mock.patch("storperf.db.graphite_db.GraphiteDB.fetch_item") + def test_confirm_commit(self, mock_graphite_db): + graphite_return = json.loads("""[ + {"target": + "rw.queue-depth.2.block-size.2048.10-10-243-154.commit-marker", + "datapoints": [[1503078366.0, 1503078370]]}] + """) + mock_graphite_db.return_value = graphite_return + + commit_marker = "commit-marker" + + emitter = CarbonMetricTransmitter() + emitter.commit_markers[commit_marker] = 1503078366 + + committed = emitter.confirm_commit(commit_marker) + self.assertTrue(committed) + + @mock.patch("storperf.db.graphite_db.GraphiteDB.fetch_item") + def test_confirm_multiple_commits(self, mock_graphite_db): + graphite_return = json.loads("""[ + {"target": + "rw.queue-depth.2.block-size.2048.10-10-243-154.commit-marker", + "datapoints": [ + [1503078300.0, 1503078350], + [1503078366.0, 1503078360]]}] + """) + mock_graphite_db.return_value = graphite_return + + commit_marker = "commit-marker" + + emitter = CarbonMetricTransmitter() + emitter.commit_markers[commit_marker] = 1503078366 + + committed = emitter.confirm_commit(commit_marker) + self.assertTrue(committed) + + @mock.patch("storperf.db.graphite_db.GraphiteDB.fetch_item") + def test_empty_commit(self, mock_graphite_db): + graphite_return = json.loads("[]") + mock_graphite_db.return_value = graphite_return + + commit_marker = "commit-marker" + + emitter = CarbonMetricTransmitter() + emitter.commit_markers[commit_marker] = 1503078366 + + committed = emitter.confirm_commit(commit_marker) + self.assertFalse(committed) + + @mock.patch("storperf.db.graphite_db.GraphiteDB.fetch_item") + def test_badtimestamp_commit(self, mock_graphite_db): + graphite_return = json.loads("""[ + {"target": + "rw.queue-depth.2.block-size.2048.10-10-243-154.commit-marker", + "datapoints": [[1234, 1503078370]]}] + """) + mock_graphite_db.return_value = graphite_return + + commit_marker = "commit-marker" + + emitter = CarbonMetricTransmitter() + emitter.commit_markers[commit_marker] = 1503078366 + + committed = emitter.confirm_commit(commit_marker) + self.assertFalse(committed) + + def test_timestamp_parse(self): + emitter = CarbonMetricTransmitter() + result = json.loads("""[ + {"target": + "rw.queue-depth.2.block-size.2048.10-10-243-154.commit-marker", + "datapoints": [[1503078366.0, 1503078370]]}] + """) + timestamps = emitter.parse_timestamp(result) + self.assertEqual(1503078366, timestamps[0], timestamps[0]) - self.assertEqual("host.run-name.key value 975542400\n", - CarbonMetricTransmitterTest.response, - CarbonMetricTransmitterTest.response) if __name__ == '__main__': unittest.main() diff --git a/docker/storperf-master/tests/utilities_tests/data_handler_test.py b/docker/storperf-master/tests/utilities_tests/data_handler_test.py index f514ae7..6010bd3 100644 --- a/docker/storperf-master/tests/utilities_tests/data_handler_test.py +++ b/docker/storperf-master/tests/utilities_tests/data_handler_test.py @@ -8,11 +8,12 @@ ############################################################################## import os -from storperf.utilities.data_handler import DataHandler import unittest import mock +from storperf.utilities.data_handler import DataHandler + class MockGraphiteDB(object): @@ -76,11 +77,11 @@ class DataHandlerTest(unittest.TestCase): mock_graphite_db.return_value = expected mock_time.return_value = expected[-1][0] + 10 - self.current_workload = ("%s.%s.queue-depth.%s.block-size.%s" % - ("job_id", - "rw", - 8, - 8192)) + self.current_workload = ("%s.%s.queue-depth.%s.block-size.%s" + % ("job_id", + "rw", + 8, + 8192)) actual = self.data_handler._lookup_prior_data(self, 'read', 'iops') self.assertEqual(expected, actual) @@ -148,8 +149,9 @@ class DataHandlerTest(unittest.TestCase): @mock.patch("time.time") @mock.patch("storperf.db.test_results_db.push_results_to_db") @mock.patch("storperf.db.graphite_db.GraphiteDB.fetch_series") - @mock.patch("storperf.db.graphite_db.JobDB.fetch_workloads") - def test_non_terminated_report(self, mock_job_db, mock_graphite_db, + @mock.patch("storperf.db.job_db.JobDB.fetch_workloads") + def test_non_terminated_report(self, mock_job_db, + mock_graphite_db, mock_results_db, mock_time): self._terminated = False mock_results_db.side_effect = self.push_results_to_db @@ -169,12 +171,11 @@ class DataHandlerTest(unittest.TestCase): expected_range = 17.78 expected_average = 212.49777777777774 - self.current_workload = ("%s.%s.queue-depth.%s.block-size.%s" % - ("job_id", - "rw", - 8, - 8192)) - + self.current_workload = ("%s.%s.queue-depth.%s.block-size.%s" + % ("job_id", + "rw", + 8, + 8192)) mock_job_db.return_value = [[self.current_workload, 4804559000, None]] self.data_handler.data_event(self) @@ -201,7 +202,7 @@ class DataHandlerTest(unittest.TestCase): @mock.patch("time.time") @mock.patch("storperf.db.test_results_db.push_results_to_db") @mock.patch("storperf.db.graphite_db.GraphiteDB.fetch_series") - @mock.patch("storperf.db.graphite_db.JobDB.fetch_workloads") + @mock.patch("storperf.db.job_db.JobDB.fetch_workloads") def test_report_that_causes_termination(self, mock_job_db, mock_graphite_db, @@ -236,12 +237,11 @@ class DataHandlerTest(unittest.TestCase): expected_range = 17.78 expected_average = 209.2135 - self.current_workload = ("%s.%s.queue-depth.%s.block-size.%s" % - ("job_id", - "rw", - 8, - 8192)) - + self.current_workload = ("%s.%s.queue-depth.%s.block-size.%s" + % ("job_id", + "rw", + 8, + 8192)) mock_job_db.return_value = [[self.current_workload, 4804559000, None]] self.data_handler.data_event(self) |