diff options
| -rwxr-xr-x | ci/setup.py | 1 | ||||
| -rw-r--r-- | cli.py | 9 | ||||
| -rw-r--r-- | docker/Dockerfile | 9 | ||||
| -rw-r--r-- | docker/requirements.pip | 2 | ||||
| -rw-r--r-- | rest_server.py | 80 | ||||
| -rw-r--r-- | storperf/db/graphite_db.py | 44 | ||||
| -rw-r--r-- | storperf/test_executor.py | 3 |
7 files changed, 138 insertions, 10 deletions
diff --git a/ci/setup.py b/ci/setup.py index 293fdda..2a02276 100755 --- a/ci/setup.py +++ b/ci/setup.py @@ -23,6 +23,7 @@ setup( url="https://www.opnfv.org", install_requires=["flask==0.10", "flask-restful==0.3.5", + "flask-restful-swagger==0.19", "html2text==2016.1.8", "python-cinderclient==1.6.0", "python-glanceclient==1.1.0", @@ -154,7 +154,14 @@ def main(argv=None): raise Usage(content['message']) if (report is not None): - print storperf.fetch_results(report) + print "Fetching report for %s..." % (report,) + response = requests.get( + 'http://127.0.0.1:5000/api/v1.0/job?id=%s' % (report,)) + if (response.status_code == 400): + content = json.loads(response.content) + raise Usage(content['message']) + content = json.loads(response.content) + print content else: print "Calling start..." response = requests.post( diff --git a/docker/Dockerfile b/docker/Dockerfile index fab768f..fb25bfc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -40,6 +40,7 @@ wget \ puppet \ build-essential \ python-dev \ +python-matplotlib \ python-pip \ --no-install-recommends @@ -63,9 +64,17 @@ RUN chmod 700 /root/.ssh RUN git config --global http.sslVerify false RUN git clone https://gerrit.opnfv.org/gerrit/storperf ${repos_dir}/storperf RUN git clone https://gerrit.opnfv.org/gerrit/releng ${repos_dir}/releng + +# Third party git fetches +RUN git clone https://github.com/swagger-api/swagger-ui.git ${repos_dir}/swagger-ui +RUN mkdir -p ${repos_dir}/storperf/storperf/resources/html/swagger +RUN cp -r ${repos_dir}/swagger-ui/dist/* ${repos_dir}/storperf/storperf/resources/html/swagger +RUN sed -i 's|url = "http://petstore.swagger.io/v2/swagger.json";|url = window.location.protocol+"//"+window.location.host+"/api/spec.json";|' ${repos_dir}/storperf/storperf/resources/html/swagger/index.html + RUN git clone http://git.kernel.dk/fio.git ${repos_dir}/fio RUN cd ${repos_dir}/fio && git checkout tags/fio-2.2.10 RUN cd ${repos_dir}/fio && make -j 6 install + RUN puppet module install gdsoperations-graphite RUN chmod 600 ${repos_dir}/storperf/storperf/resources/ssh/storperf_rsa diff --git a/docker/requirements.pip b/docker/requirements.pip index 89f19ae..4c9aaae 100644 --- a/docker/requirements.pip +++ b/docker/requirements.pip @@ -9,4 +9,4 @@ flask==0.10 flask-restful==0.3.5 flask-restful-swagger==0.19 flask-swagger==0.2.12 -html2text==2016.1.8
\ No newline at end of file +html2text==2016.1.8 diff --git a/rest_server.py b/rest_server.py index c5fe99b..f0a817b 100644 --- a/rest_server.py +++ b/rest_server.py @@ -25,15 +25,40 @@ storperf = StorPerfMaster() @app.route('/swagger/<path:path>') def send_swagger(path): - print "called! storperf/resources/html/swagger/" + path return send_from_directory('storperf/resources/html/swagger', path) +@swagger.model +class ConfigurationRequestModel: + resource_fields = { + 'agent_count': fields.Integer, + 'public_network': fields.String, + 'volume_size': fields.Integer + } + + +@swagger.model +class ConfigurationResponseModel: + resource_fields = { + 'agent_count': fields.Integer, + 'public_network': fields.String, + 'stack_created': fields.Boolean, + 'stack_id': fields.String, + 'volume_size': fields.Integer + } + + class Configure(Resource): + """Configuration API""" + def __init__(self): self.logger = logging.getLogger(__name__) + @swagger.operation( + notes='Fetch the current agent configuration', + type=ConfigurationResponseModel.__name__ + ) def get(self): return jsonify({'agent_count': storperf.agent_count, 'public_network': storperf.public_network, @@ -41,6 +66,23 @@ class Configure(Resource): 'stack_created': storperf.is_stack_created, 'stack_id': storperf.stack_id}) + @swagger.operation( + notes='''Set the current agent configuration and create a stack in + the controller. Returns once the stack request is submitted.''', + parameters=[ + { + "name": "configuration", + "description": '''Configuration to be set. All parameters are + optional, and will retain their previous value if not + specified. Volume size is in GB. + ''', + "required": True, + "type": "ConfigurationRequestModel", + "paramType": "body" + } + ], + type=ConfigurationResponseModel.__name__ + ) def post(self): if not request.json: abort(400, "ERROR: No data specified") @@ -64,6 +106,9 @@ class Configure(Resource): except Exception as e: abort(400, str(e)) + @swagger.operation( + notes='Deletes the agent configuration and the stack' + ) def delete(self): try: storperf.delete_stack() @@ -81,8 +126,17 @@ class WorkloadModel: } +@swagger.model +class WorkloadResponseModel: + resource_fields = { + 'job_id': fields.String + } + + class Job(Resource): + """Job API""" + def __init__(self): self.logger = logging.getLogger(__name__) @@ -131,10 +185,11 @@ class Job(Resource): "paramType": "body" } ], + type=WorkloadResponseModel.__name__, responseMessages=[ { "code": 200, - "message": "Wordload ID found, response in JSON format" + "message": "Job submitted" }, { "code": 400, @@ -181,16 +236,31 @@ class Job(Resource): ) def delete(self): try: - storperf.terminate_workloads() - return True + return jsonify({'Slaves': storperf.terminate_workloads()}) except Exception as e: abort(400, str(e)) +@swagger.model +class QuotaModel: + + resource_fields = { + 'quota': fields.Integer + } + + class Quota(Resource): + """Quota API""" + @swagger.operation( + notes='''Fetch the current Cinder volume quota. This value limits + the number of volumes that can be created, and by extension, defines + the maximum number of agents that can be created for any given test + scenario''', + type=QuotaModel.__name__ + ) def get(self): - quota = storperf.get_volume_quota() + quota = storperf.volume_quota return jsonify({'quota': quota}) diff --git a/storperf/db/graphite_db.py b/storperf/db/graphite_db.py index c62340c..8fef071 100644 --- a/storperf/db/graphite_db.py +++ b/storperf/db/graphite_db.py @@ -1,6 +1,8 @@ from storperf.db.job_db import JobDB +import calendar import json import logging +import time import requests @@ -41,18 +43,54 @@ class GraphiteDB(object): for io_type in ['read', 'write']: for workload_name, times in workload_names.iteritems(): workload_pattern = self.make_fullname_pattern(workload_name) + short_name = '.'.join(workload_name.split('.')[1:6]) + start = times[0] + end = times[1] + + if end is None: + end = str(calendar.timegm(time.gmtime())) + averages[short_name + ".duration"] = \ + (int(end) - int(start)) + + key = short_name + "." + io_type + request = ("http://127.0.0.1:8000/render/?target=" "averageSeries(%s.jobs.1.%s.lat.mean)" "&format=json" "&from=%s" "&until=%s" % - (workload_pattern, io_type, times[0], times[1])) + (workload_pattern, io_type, start, end)) + self.logger.debug("Calling %s" % (request)) + + response = requests.get(request) + if (response.status_code == 200): + averages[key + ".latency"] = \ + self._average_results(json.loads(response.content)) + + request = ("http://127.0.0.1:8000/render/?target=" + "averageSeries(%s.jobs.1.%s.bw)" + "&format=json" + "&from=%s" + "&until=%s" % + (workload_pattern, io_type, start, end)) + self.logger.debug("Calling %s" % (request)) + + response = requests.get(request) + if (response.status_code == 200): + averages[key + ".throughput"] = \ + self._average_results(json.loads(response.content)) + + request = ("http://127.0.0.1:8000/render/?target=" + "averageSeries(%s.jobs.1.%s.iops)" + "&format=json" + "&from=%s" + "&until=%s" % + (workload_pattern, io_type, start, end)) self.logger.debug("Calling %s" % (request)) response = requests.get(request) if (response.status_code == 200): - short_name = '.'.join(workload_name.split('.')[1:6]) - averages[short_name + "." + io_type] = \ + averages[key + ".iops"] = \ self._average_results(json.loads(response.content)) return averages diff --git a/storperf/test_executor.py b/storperf/test_executor.py index c0ea295..309fbcb 100644 --- a/storperf/test_executor.py +++ b/storperf/test_executor.py @@ -122,8 +122,11 @@ class TestExecutor(object): def terminate(self): self._terminated = True + terminated_hosts = [] for workload in self._workload_executors: workload.terminate() + terminated_hosts.append(workload.remote_host) + return terminated_hosts def execute_workloads(self): self._terminated = False |