summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMark Beierl <mark.beierl@emc.com>2016-05-04 22:53:07 -0400
committerMark Beierl <mark.beierl@emc.com>2016-05-06 14:13:52 -0400
commitda56b4dac702713045aaeeedbab9234e1825ffe0 (patch)
treeae5594911e25458bada74916f01e82520b2f423c
parent05e863781ce6746fabec176d1fc5f7454f2cdd73 (diff)
Add Stats report and Swagger UI
Add Swagger web ui at /swagger Add ability to fetch read/write latency status via ReST ui Can now delete where stack was removed from OpenStack but not from the storperf DB Change to use Floating IPs instead of private IP Fix delete bug where there was no dependency on resources in the resource group. JIRA: STORPERF-19 JIRA: STORPERF-20 Change-Id: I1d9627d81f3c309b178a9b68cc306a4101c1a231 Signed-off-by: Mark Beierl <mark.beierl@emc.com>
-rwxr-xr-xci/setup.py1
-rw-r--r--cli.py9
-rw-r--r--docker/Dockerfile9
-rw-r--r--docker/requirements.pip2
-rw-r--r--rest_server.py80
-rw-r--r--storperf/db/graphite_db.py44
-rw-r--r--storperf/test_executor.py3
7 files changed, 138 insertions, 10 deletions
diff --git a/ci/setup.py b/ci/setup.py
index 293fdda..2a02276 100755
--- a/ci/setup.py
+++ b/ci/setup.py
@@ -23,6 +23,7 @@ setup(
url="https://www.opnfv.org",
install_requires=["flask==0.10",
"flask-restful==0.3.5",
+ "flask-restful-swagger==0.19",
"html2text==2016.1.8",
"python-cinderclient==1.6.0",
"python-glanceclient==1.1.0",
diff --git a/cli.py b/cli.py
index fad275c..85ebcfd 100644
--- a/cli.py
+++ b/cli.py
@@ -154,7 +154,14 @@ def main(argv=None):
raise Usage(content['message'])
if (report is not None):
- print storperf.fetch_results(report)
+ print "Fetching report for %s..." % (report,)
+ response = requests.get(
+ 'http://127.0.0.1:5000/api/v1.0/job?id=%s' % (report,))
+ if (response.status_code == 400):
+ content = json.loads(response.content)
+ raise Usage(content['message'])
+ content = json.loads(response.content)
+ print content
else:
print "Calling start..."
response = requests.post(
diff --git a/docker/Dockerfile b/docker/Dockerfile
index fab768f..fb25bfc 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -40,6 +40,7 @@ wget \
puppet \
build-essential \
python-dev \
+python-matplotlib \
python-pip \
--no-install-recommends
@@ -63,9 +64,17 @@ RUN chmod 700 /root/.ssh
RUN git config --global http.sslVerify false
RUN git clone https://gerrit.opnfv.org/gerrit/storperf ${repos_dir}/storperf
RUN git clone https://gerrit.opnfv.org/gerrit/releng ${repos_dir}/releng
+
+# Third party git fetches
+RUN git clone https://github.com/swagger-api/swagger-ui.git ${repos_dir}/swagger-ui
+RUN mkdir -p ${repos_dir}/storperf/storperf/resources/html/swagger
+RUN cp -r ${repos_dir}/swagger-ui/dist/* ${repos_dir}/storperf/storperf/resources/html/swagger
+RUN sed -i 's|url = "http://petstore.swagger.io/v2/swagger.json";|url = window.location.protocol+"//"+window.location.host+"/api/spec.json";|' ${repos_dir}/storperf/storperf/resources/html/swagger/index.html
+
RUN git clone http://git.kernel.dk/fio.git ${repos_dir}/fio
RUN cd ${repos_dir}/fio && git checkout tags/fio-2.2.10
RUN cd ${repos_dir}/fio && make -j 6 install
+
RUN puppet module install gdsoperations-graphite
RUN chmod 600 ${repos_dir}/storperf/storperf/resources/ssh/storperf_rsa
diff --git a/docker/requirements.pip b/docker/requirements.pip
index 89f19ae..4c9aaae 100644
--- a/docker/requirements.pip
+++ b/docker/requirements.pip
@@ -9,4 +9,4 @@ flask==0.10
flask-restful==0.3.5
flask-restful-swagger==0.19
flask-swagger==0.2.12
-html2text==2016.1.8 \ No newline at end of file
+html2text==2016.1.8
diff --git a/rest_server.py b/rest_server.py
index c5fe99b..f0a817b 100644
--- a/rest_server.py
+++ b/rest_server.py
@@ -25,15 +25,40 @@ storperf = StorPerfMaster()
@app.route('/swagger/<path:path>')
def send_swagger(path):
- print "called! storperf/resources/html/swagger/" + path
return send_from_directory('storperf/resources/html/swagger', path)
+@swagger.model
+class ConfigurationRequestModel:
+ resource_fields = {
+ 'agent_count': fields.Integer,
+ 'public_network': fields.String,
+ 'volume_size': fields.Integer
+ }
+
+
+@swagger.model
+class ConfigurationResponseModel:
+ resource_fields = {
+ 'agent_count': fields.Integer,
+ 'public_network': fields.String,
+ 'stack_created': fields.Boolean,
+ 'stack_id': fields.String,
+ 'volume_size': fields.Integer
+ }
+
+
class Configure(Resource):
+ """Configuration API"""
+
def __init__(self):
self.logger = logging.getLogger(__name__)
+ @swagger.operation(
+ notes='Fetch the current agent configuration',
+ type=ConfigurationResponseModel.__name__
+ )
def get(self):
return jsonify({'agent_count': storperf.agent_count,
'public_network': storperf.public_network,
@@ -41,6 +66,23 @@ class Configure(Resource):
'stack_created': storperf.is_stack_created,
'stack_id': storperf.stack_id})
+ @swagger.operation(
+ notes='''Set the current agent configuration and create a stack in
+ the controller. Returns once the stack request is submitted.''',
+ parameters=[
+ {
+ "name": "configuration",
+ "description": '''Configuration to be set. All parameters are
+ optional, and will retain their previous value if not
+ specified. Volume size is in GB.
+ ''',
+ "required": True,
+ "type": "ConfigurationRequestModel",
+ "paramType": "body"
+ }
+ ],
+ type=ConfigurationResponseModel.__name__
+ )
def post(self):
if not request.json:
abort(400, "ERROR: No data specified")
@@ -64,6 +106,9 @@ class Configure(Resource):
except Exception as e:
abort(400, str(e))
+ @swagger.operation(
+ notes='Deletes the agent configuration and the stack'
+ )
def delete(self):
try:
storperf.delete_stack()
@@ -81,8 +126,17 @@ class WorkloadModel:
}
+@swagger.model
+class WorkloadResponseModel:
+ resource_fields = {
+ 'job_id': fields.String
+ }
+
+
class Job(Resource):
+ """Job API"""
+
def __init__(self):
self.logger = logging.getLogger(__name__)
@@ -131,10 +185,11 @@ class Job(Resource):
"paramType": "body"
}
],
+ type=WorkloadResponseModel.__name__,
responseMessages=[
{
"code": 200,
- "message": "Wordload ID found, response in JSON format"
+ "message": "Job submitted"
},
{
"code": 400,
@@ -181,16 +236,31 @@ class Job(Resource):
)
def delete(self):
try:
- storperf.terminate_workloads()
- return True
+ return jsonify({'Slaves': storperf.terminate_workloads()})
except Exception as e:
abort(400, str(e))
+@swagger.model
+class QuotaModel:
+
+ resource_fields = {
+ 'quota': fields.Integer
+ }
+
+
class Quota(Resource):
+ """Quota API"""
+ @swagger.operation(
+ notes='''Fetch the current Cinder volume quota. This value limits
+ the number of volumes that can be created, and by extension, defines
+ the maximum number of agents that can be created for any given test
+ scenario''',
+ type=QuotaModel.__name__
+ )
def get(self):
- quota = storperf.get_volume_quota()
+ quota = storperf.volume_quota
return jsonify({'quota': quota})
diff --git a/storperf/db/graphite_db.py b/storperf/db/graphite_db.py
index c62340c..8fef071 100644
--- a/storperf/db/graphite_db.py
+++ b/storperf/db/graphite_db.py
@@ -1,6 +1,8 @@
from storperf.db.job_db import JobDB
+import calendar
import json
import logging
+import time
import requests
@@ -41,18 +43,54 @@ class GraphiteDB(object):
for io_type in ['read', 'write']:
for workload_name, times in workload_names.iteritems():
workload_pattern = self.make_fullname_pattern(workload_name)
+ short_name = '.'.join(workload_name.split('.')[1:6])
+ start = times[0]
+ end = times[1]
+
+ if end is None:
+ end = str(calendar.timegm(time.gmtime()))
+ averages[short_name + ".duration"] = \
+ (int(end) - int(start))
+
+ key = short_name + "." + io_type
+
request = ("http://127.0.0.1:8000/render/?target="
"averageSeries(%s.jobs.1.%s.lat.mean)"
"&format=json"
"&from=%s"
"&until=%s" %
- (workload_pattern, io_type, times[0], times[1]))
+ (workload_pattern, io_type, start, end))
+ self.logger.debug("Calling %s" % (request))
+
+ response = requests.get(request)
+ if (response.status_code == 200):
+ averages[key + ".latency"] = \
+ self._average_results(json.loads(response.content))
+
+ request = ("http://127.0.0.1:8000/render/?target="
+ "averageSeries(%s.jobs.1.%s.bw)"
+ "&format=json"
+ "&from=%s"
+ "&until=%s" %
+ (workload_pattern, io_type, start, end))
+ self.logger.debug("Calling %s" % (request))
+
+ response = requests.get(request)
+ if (response.status_code == 200):
+ averages[key + ".throughput"] = \
+ self._average_results(json.loads(response.content))
+
+ request = ("http://127.0.0.1:8000/render/?target="
+ "averageSeries(%s.jobs.1.%s.iops)"
+ "&format=json"
+ "&from=%s"
+ "&until=%s" %
+ (workload_pattern, io_type, start, end))
self.logger.debug("Calling %s" % (request))
response = requests.get(request)
if (response.status_code == 200):
- short_name = '.'.join(workload_name.split('.')[1:6])
- averages[short_name + "." + io_type] = \
+ averages[key + ".iops"] = \
self._average_results(json.loads(response.content))
return averages
diff --git a/storperf/test_executor.py b/storperf/test_executor.py
index c0ea295..309fbcb 100644
--- a/storperf/test_executor.py
+++ b/storperf/test_executor.py
@@ -122,8 +122,11 @@ class TestExecutor(object):
def terminate(self):
self._terminated = True
+ terminated_hosts = []
for workload in self._workload_executors:
workload.terminate()
+ terminated_hosts.append(workload.remote_host)
+ return terminated_hosts
def execute_workloads(self):
self._terminated = False