From da56b4dac702713045aaeeedbab9234e1825ffe0 Mon Sep 17 00:00:00 2001 From: Mark Beierl Date: Wed, 4 May 2016 22:53:07 -0400 Subject: Add Stats report and Swagger UI Add Swagger web ui at /swagger Add ability to fetch read/write latency status via ReST ui Can now delete where stack was removed from OpenStack but not from the storperf DB Change to use Floating IPs instead of private IP Fix delete bug where there was no dependency on resources in the resource group. JIRA: STORPERF-19 JIRA: STORPERF-20 Change-Id: I1d9627d81f3c309b178a9b68cc306a4101c1a231 Signed-off-by: Mark Beierl --- ci/setup.py | 1 + cli.py | 9 +++++- docker/Dockerfile | 9 ++++++ docker/requirements.pip | 2 +- rest_server.py | 80 +++++++++++++++++++++++++++++++++++++++++++--- storperf/db/graphite_db.py | 44 +++++++++++++++++++++++-- storperf/test_executor.py | 3 ++ 7 files changed, 138 insertions(+), 10 deletions(-) diff --git a/ci/setup.py b/ci/setup.py index 293fdda..2a02276 100755 --- a/ci/setup.py +++ b/ci/setup.py @@ -23,6 +23,7 @@ setup( url="https://www.opnfv.org", install_requires=["flask==0.10", "flask-restful==0.3.5", + "flask-restful-swagger==0.19", "html2text==2016.1.8", "python-cinderclient==1.6.0", "python-glanceclient==1.1.0", diff --git a/cli.py b/cli.py index fad275c..85ebcfd 100644 --- a/cli.py +++ b/cli.py @@ -154,7 +154,14 @@ def main(argv=None): raise Usage(content['message']) if (report is not None): - print storperf.fetch_results(report) + print "Fetching report for %s..." % (report,) + response = requests.get( + 'http://127.0.0.1:5000/api/v1.0/job?id=%s' % (report,)) + if (response.status_code == 400): + content = json.loads(response.content) + raise Usage(content['message']) + content = json.loads(response.content) + print content else: print "Calling start..." response = requests.post( diff --git a/docker/Dockerfile b/docker/Dockerfile index fab768f..fb25bfc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -40,6 +40,7 @@ wget \ puppet \ build-essential \ python-dev \ +python-matplotlib \ python-pip \ --no-install-recommends @@ -63,9 +64,17 @@ RUN chmod 700 /root/.ssh RUN git config --global http.sslVerify false RUN git clone https://gerrit.opnfv.org/gerrit/storperf ${repos_dir}/storperf RUN git clone https://gerrit.opnfv.org/gerrit/releng ${repos_dir}/releng + +# Third party git fetches +RUN git clone https://github.com/swagger-api/swagger-ui.git ${repos_dir}/swagger-ui +RUN mkdir -p ${repos_dir}/storperf/storperf/resources/html/swagger +RUN cp -r ${repos_dir}/swagger-ui/dist/* ${repos_dir}/storperf/storperf/resources/html/swagger +RUN sed -i 's|url = "http://petstore.swagger.io/v2/swagger.json";|url = window.location.protocol+"//"+window.location.host+"/api/spec.json";|' ${repos_dir}/storperf/storperf/resources/html/swagger/index.html + RUN git clone http://git.kernel.dk/fio.git ${repos_dir}/fio RUN cd ${repos_dir}/fio && git checkout tags/fio-2.2.10 RUN cd ${repos_dir}/fio && make -j 6 install + RUN puppet module install gdsoperations-graphite RUN chmod 600 ${repos_dir}/storperf/storperf/resources/ssh/storperf_rsa diff --git a/docker/requirements.pip b/docker/requirements.pip index 89f19ae..4c9aaae 100644 --- a/docker/requirements.pip +++ b/docker/requirements.pip @@ -9,4 +9,4 @@ flask==0.10 flask-restful==0.3.5 flask-restful-swagger==0.19 flask-swagger==0.2.12 -html2text==2016.1.8 \ No newline at end of file +html2text==2016.1.8 diff --git a/rest_server.py b/rest_server.py index c5fe99b..f0a817b 100644 --- a/rest_server.py +++ b/rest_server.py @@ -25,15 +25,40 @@ storperf = StorPerfMaster() @app.route('/swagger/') def send_swagger(path): - print "called! storperf/resources/html/swagger/" + path return send_from_directory('storperf/resources/html/swagger', path) +@swagger.model +class ConfigurationRequestModel: + resource_fields = { + 'agent_count': fields.Integer, + 'public_network': fields.String, + 'volume_size': fields.Integer + } + + +@swagger.model +class ConfigurationResponseModel: + resource_fields = { + 'agent_count': fields.Integer, + 'public_network': fields.String, + 'stack_created': fields.Boolean, + 'stack_id': fields.String, + 'volume_size': fields.Integer + } + + class Configure(Resource): + """Configuration API""" + def __init__(self): self.logger = logging.getLogger(__name__) + @swagger.operation( + notes='Fetch the current agent configuration', + type=ConfigurationResponseModel.__name__ + ) def get(self): return jsonify({'agent_count': storperf.agent_count, 'public_network': storperf.public_network, @@ -41,6 +66,23 @@ class Configure(Resource): 'stack_created': storperf.is_stack_created, 'stack_id': storperf.stack_id}) + @swagger.operation( + notes='''Set the current agent configuration and create a stack in + the controller. Returns once the stack request is submitted.''', + parameters=[ + { + "name": "configuration", + "description": '''Configuration to be set. All parameters are + optional, and will retain their previous value if not + specified. Volume size is in GB. + ''', + "required": True, + "type": "ConfigurationRequestModel", + "paramType": "body" + } + ], + type=ConfigurationResponseModel.__name__ + ) def post(self): if not request.json: abort(400, "ERROR: No data specified") @@ -64,6 +106,9 @@ class Configure(Resource): except Exception as e: abort(400, str(e)) + @swagger.operation( + notes='Deletes the agent configuration and the stack' + ) def delete(self): try: storperf.delete_stack() @@ -81,8 +126,17 @@ class WorkloadModel: } +@swagger.model +class WorkloadResponseModel: + resource_fields = { + 'job_id': fields.String + } + + class Job(Resource): + """Job API""" + def __init__(self): self.logger = logging.getLogger(__name__) @@ -131,10 +185,11 @@ class Job(Resource): "paramType": "body" } ], + type=WorkloadResponseModel.__name__, responseMessages=[ { "code": 200, - "message": "Wordload ID found, response in JSON format" + "message": "Job submitted" }, { "code": 400, @@ -181,16 +236,31 @@ class Job(Resource): ) def delete(self): try: - storperf.terminate_workloads() - return True + return jsonify({'Slaves': storperf.terminate_workloads()}) except Exception as e: abort(400, str(e)) +@swagger.model +class QuotaModel: + + resource_fields = { + 'quota': fields.Integer + } + + class Quota(Resource): + """Quota API""" + @swagger.operation( + notes='''Fetch the current Cinder volume quota. This value limits + the number of volumes that can be created, and by extension, defines + the maximum number of agents that can be created for any given test + scenario''', + type=QuotaModel.__name__ + ) def get(self): - quota = storperf.get_volume_quota() + quota = storperf.volume_quota return jsonify({'quota': quota}) diff --git a/storperf/db/graphite_db.py b/storperf/db/graphite_db.py index c62340c..8fef071 100644 --- a/storperf/db/graphite_db.py +++ b/storperf/db/graphite_db.py @@ -1,6 +1,8 @@ from storperf.db.job_db import JobDB +import calendar import json import logging +import time import requests @@ -41,18 +43,54 @@ class GraphiteDB(object): for io_type in ['read', 'write']: for workload_name, times in workload_names.iteritems(): workload_pattern = self.make_fullname_pattern(workload_name) + short_name = '.'.join(workload_name.split('.')[1:6]) + start = times[0] + end = times[1] + + if end is None: + end = str(calendar.timegm(time.gmtime())) + averages[short_name + ".duration"] = \ + (int(end) - int(start)) + + key = short_name + "." + io_type + request = ("http://127.0.0.1:8000/render/?target=" "averageSeries(%s.jobs.1.%s.lat.mean)" "&format=json" "&from=%s" "&until=%s" % - (workload_pattern, io_type, times[0], times[1])) + (workload_pattern, io_type, start, end)) + self.logger.debug("Calling %s" % (request)) + + response = requests.get(request) + if (response.status_code == 200): + averages[key + ".latency"] = \ + self._average_results(json.loads(response.content)) + + request = ("http://127.0.0.1:8000/render/?target=" + "averageSeries(%s.jobs.1.%s.bw)" + "&format=json" + "&from=%s" + "&until=%s" % + (workload_pattern, io_type, start, end)) + self.logger.debug("Calling %s" % (request)) + + response = requests.get(request) + if (response.status_code == 200): + averages[key + ".throughput"] = \ + self._average_results(json.loads(response.content)) + + request = ("http://127.0.0.1:8000/render/?target=" + "averageSeries(%s.jobs.1.%s.iops)" + "&format=json" + "&from=%s" + "&until=%s" % + (workload_pattern, io_type, start, end)) self.logger.debug("Calling %s" % (request)) response = requests.get(request) if (response.status_code == 200): - short_name = '.'.join(workload_name.split('.')[1:6]) - averages[short_name + "." + io_type] = \ + averages[key + ".iops"] = \ self._average_results(json.loads(response.content)) return averages diff --git a/storperf/test_executor.py b/storperf/test_executor.py index c0ea295..309fbcb 100644 --- a/storperf/test_executor.py +++ b/storperf/test_executor.py @@ -122,8 +122,11 @@ class TestExecutor(object): def terminate(self): self._terminated = True + terminated_hosts = [] for workload in self._workload_executors: workload.terminate() + terminated_hosts.append(workload.remote_host) + return terminated_hosts def execute_workloads(self): self._terminated = False -- cgit 1.2.3-korg