1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
from storperf.db.job_db import JobDB
import calendar
import json
import logging
import time
import requests
class GraphiteDB(object):
def __init__(self):
"""
"""
self._job_db = JobDB()
self.logger = logging.getLogger(__name__)
def fetch_averages(self, workload):
workload_executions = self._job_db.fetch_workloads(workload)
# Create a map of job runs
workload_names = {}
for workload_execution in workload_executions:
name = '.'.join(workload_execution[0].split('.')[0:6])
if name in workload_names:
workload_record = workload_names[name]
start = workload_record[0]
end = workload_record[1]
else:
start = None
end = None
if start is None or workload_execution[1] < start:
start = workload_execution[1]
if end is None or workload_execution[2] > end:
end = workload_execution[2]
workload_names[name] = [start, end]
averages = {}
for io_type in ['read', 'write']:
for workload_name, times in workload_names.iteritems():
workload_pattern = self.make_fullname_pattern(workload_name)
short_name = '.'.join(workload_name.split('.')[1:6])
start = times[0]
end = times[1]
if end is None:
end = str(calendar.timegm(time.gmtime()))
averages[short_name + ".duration"] = \
(int(end) - int(start))
key = short_name + "." + io_type
request = ("http://127.0.0.1:8000/render/?target="
"averageSeries(%s.jobs.1.%s.lat.mean)"
"&format=json"
"&from=%s"
"&until=%s" %
(workload_pattern, io_type, start, end))
self.logger.debug("Calling %s" % (request))
response = requests.get(request)
if (response.status_code == 200):
averages[key + ".latency"] = \
self._average_results(json.loads(response.content))
request = ("http://127.0.0.1:8000/render/?target="
"averageSeries(%s.jobs.1.%s.bw)"
"&format=json"
"&from=%s"
"&until=%s" %
(workload_pattern, io_type, start, end))
self.logger.debug("Calling %s" % (request))
response = requests.get(request)
if (response.status_code == 200):
averages[key + ".throughput"] = \
self._average_results(json.loads(response.content))
request = ("http://127.0.0.1:8000/render/?target="
"averageSeries(%s.jobs.1.%s.iops)"
"&format=json"
"&from=%s"
"&until=%s" %
(workload_pattern, io_type, start, end))
self.logger.debug("Calling %s" % (request))
response = requests.get(request)
if (response.status_code == 200):
averages[key + ".iops"] = \
self._average_results(json.loads(response.content))
return averages
def _average_results(self, results):
for item in results:
datapoints = item['datapoints']
total = 0
count = 0
for datapoint in datapoints:
if datapoint[0] is not None:
total += datapoint[0]
count += 1
if count > 0:
average = total / count
else:
average = total
return average
def make_fullname_pattern(self, workload):
parts = workload.split('.')
wildcards_needed = 7 - len(parts)
fullname = workload + (".*" * wildcards_needed)
return fullname
|
