summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorhoujingwen <houjingwen@huawei.com>2015-10-08 09:31:36 +0000
committerhoujingwen <houjingwen@huawei.com>2015-10-13 12:22:51 +0800
commitc7a3775d447e29850463893495c08328960cd3d5 (patch)
tree376bfc60d1ae8f8cead7596d651799a4c8344077
parent89ee821fd10b1de5cd6d319347627ca540ac57c0 (diff)
Fio scenario support sla
JIRA: YARDSTICK-34 Change-Id: I782ba5845f8bd54a19bad078fe7be546400f7524 Signed-off-by: houjingwen <houjingwen@huawei.com>
-rw-r--r--samples/fio.yaml8
-rw-r--r--tests/unit/benchmark/scenarios/storage/fio_read_sample_output.json1
-rw-r--r--tests/unit/benchmark/scenarios/storage/fio_rw_sample_output.json (renamed from tests/unit/benchmark/scenarios/storage/fio_sample_output.json)0
-rw-r--r--tests/unit/benchmark/scenarios/storage/fio_write_sample_output.json1
-rw-r--r--tests/unit/benchmark/scenarios/storage/test_fio.py170
-rw-r--r--yardstick/benchmark/scenarios/storage/fio.py32
6 files changed, 191 insertions, 21 deletions
diff --git a/samples/fio.yaml b/samples/fio.yaml
index 083c57549..6e77f681a 100644
--- a/samples/fio.yaml
+++ b/samples/fio.yaml
@@ -26,6 +26,14 @@ scenarios:
type: Duration
duration: 60
interval: 1
+ sla:
+ read_bw: 6000
+ read_iops: 1500
+ read_lat: 500.1
+ write_bw: 6000
+ write_iops: 1500
+ write_lat: 500.1
+ action: monitor
context:
name: demo
diff --git a/tests/unit/benchmark/scenarios/storage/fio_read_sample_output.json b/tests/unit/benchmark/scenarios/storage/fio_read_sample_output.json
new file mode 100644
index 000000000..e9f642aba
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/storage/fio_read_sample_output.json
@@ -0,0 +1 @@
+{"fioversion": "fio-2.1.3","jobs": [{"jobname": "yardstick-fio","groupid": 0,"error": 0,"read": {"io_bytes": 2166860,"bw": 36113,"iops": 9028,"runtime": 60001,"slat": {"min": 7,"max": 1807,"mean": 10.49,"stddev": 3.00},"clat": {"min": 1,"max": 16902,"mean": 97.84,"stddev": 78.16,"percentile": {"1.000000": 84,"5.000000": 86,"10.000000": 87,"20.000000": 88,"30.000000": 89,"40.000000": 90,"50.000000": 91,"60.000000": 93,"70.000000": 98,"80.000000": 103,"90.000000": 111,"95.000000": 127,"99.000000": 161,"99.500000": 177,"99.900000": 215,"99.950000": 266,"99.990000": 4128,"0.00": 0,"0.00": 0,"0.00": 0}},"lat": {"min": 86,"max": 16912,"mean": 108.70,"stddev": 78.29},"bw_min": 0,"bw_max": 38128,"bw_agg": 35816.54,"bw_mean": 35816.54,"bw_dev": 3579.16},"write": {"io_bytes": 0,"bw": 0,"iops": 0,"runtime": 0,"slat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00},"clat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00,"percentile": {"1.000000": 0,"5.000000": 0,"10.000000": 0,"20.000000": 0,"30.000000": 0,"40.000000": 0,"50.000000": 0,"60.000000": 0,"70.000000": 0,"80.000000": 0,"90.000000": 0,"95.000000": 0,"99.000000": 0,"99.500000": 0,"99.900000": 0,"99.950000": 0,"99.990000": 0,"0.00": 0,"0.00": 0,"0.00": 0}},"lat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00},"bw_min": 0,"bw_max": 0,"bw_agg": 0.00,"bw_mean": 0.00,"bw_dev": 0.00},"trim": {"io_bytes": 0,"bw": 0,"iops": 0,"runtime": 0,"slat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00},"clat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00,"percentile": {"1.000000": 0,"5.000000": 0,"10.000000": 0,"20.000000": 0,"30.000000": 0,"40.000000": 0,"50.000000": 0,"60.000000": 0,"70.000000": 0,"80.000000": 0,"90.000000": 0,"95.000000": 0,"99.000000": 0,"99.500000": 0,"99.900000": 0,"99.950000": 0,"99.990000": 0,"0.00": 0,"0.00": 0,"0.00": 0}},"lat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00},"bw_min": 0,"bw_max": 0,"bw_agg": 0.00,"bw_mean": 0.00,"bw_dev": 0.00},"usr_cpu": 4.86,"sys_cpu": 19.38,"ctx": 632024,"majf": 0,"minf": 30,"iodepth_level": {"1": 116.58,"2": 0.00,"4": 0.00,"8": 0.00,"16": 0.00,"32": 0.00,">=64": 0.00},"latency_us": {"2": 0.01,"4": 0.01,"10": 0.00,"20": 0.00,"50": 0.01,"100": 72.60,"250": 27.34,"500": 0.04,"750": 0.01,"1000": 0.01},"latency_ms": {"2": 0.01,"4": 0.01,"10": 0.01,"20": 0.01,"50": 0.00,"100": 0.00,"250": 0.00,"500": 0.00,"750": 0.00,"1000": 0.00,"2000": 0.00,">=2000": 0.00}}],"disk_util": [{"name": "vda","read_ios": 631084,"write_ios": 212,"read_merges": 0,"write_merges": 232,"read_ticks": 57300,"write_ticks": 324,"in_queue": 57400,"util": 81.55}]}
diff --git a/tests/unit/benchmark/scenarios/storage/fio_sample_output.json b/tests/unit/benchmark/scenarios/storage/fio_rw_sample_output.json
index 4c7501818..4c7501818 100644
--- a/tests/unit/benchmark/scenarios/storage/fio_sample_output.json
+++ b/tests/unit/benchmark/scenarios/storage/fio_rw_sample_output.json
diff --git a/tests/unit/benchmark/scenarios/storage/fio_write_sample_output.json b/tests/unit/benchmark/scenarios/storage/fio_write_sample_output.json
new file mode 100644
index 000000000..7c760e8bc
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/storage/fio_write_sample_output.json
@@ -0,0 +1 @@
+{"fioversion": "fio-2.1.3","jobs": [{"jobname": "yardstick-fio","groupid": 0,"error": 0,"read": {"io_bytes": 0,"bw": 0,"iops": 0,"runtime": 0,"slat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00},"clat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00,"percentile": {"1.000000": 0,"5.000000": 0,"10.000000": 0,"20.000000": 0,"30.000000": 0,"40.000000": 0,"50.000000": 0,"60.000000": 0,"70.000000": 0,"80.000000": 0,"90.000000": 0,"95.000000": 0,"99.000000": 0,"99.500000": 0,"99.900000": 0,"99.950000": 0,"99.990000": 0,"0.00": 0,"0.00": 0,"0.00": 0}},"lat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00},"bw_min": 0,"bw_max": 0,"bw_agg": 0.00,"bw_mean": 0.00,"bw_dev": 0.00},"write": {"io_bytes": 2106508,"bw": 35107,"iops": 8776,"runtime": 60001,"slat": {"min": 8,"max": 5166,"mean": 11.83,"stddev": 7.05},"clat": {"min": 1,"max": 23472,"mean": 99.54,"stddev": 44.23,"percentile": {"1.000000": 85,"5.000000": 87,"10.000000": 88,"20.000000": 89,"30.000000": 90,"40.000000": 91,"50.000000": 93,"60.000000": 99,"70.000000": 104,"80.000000": 107,"90.000000": 113,"95.000000": 127,"99.000000": 161,"99.500000": 179,"99.900000": 231,"99.950000": 286,"99.990000": 628,"0.00": 0,"0.00": 0,"0.00": 0}},"lat": {"min": 87,"max": 23486,"mean": 111.74,"stddev": 45.61},"bw_min": 0,"bw_max": 37288,"bw_agg": 34839.53,"bw_mean": 34839.53,"bw_dev": 3387.37},"trim": {"io_bytes": 0,"bw": 0,"iops": 0,"runtime": 0,"slat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00},"clat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00,"percentile": {"1.000000": 0,"5.000000": 0,"10.000000": 0,"20.000000": 0,"30.000000": 0,"40.000000": 0,"50.000000": 0,"60.000000": 0,"70.000000": 0,"80.000000": 0,"90.000000": 0,"95.000000": 0,"99.000000": 0,"99.500000": 0,"99.900000": 0,"99.950000": 0,"99.990000": 0,"0.00": 0,"0.00": 0,"0.00": 0}},"lat": {"min": 0,"max": 0,"mean": 0.00,"stddev": 0.00},"bw_min": 0,"bw_max": 0,"bw_agg": 0.00,"bw_mean": 0.00,"bw_dev": 0.00},"usr_cpu": 5.25,"sys_cpu": 19.72,"ctx": 616160,"majf": 0,"minf": 27,"iodepth_level": {"1": 116.90,"2": 0.00,"4": 0.00,"8": 0.00,"16": 0.00,"32": 0.00,">=64": 0.00},"latency_us": {"2": 0.01,"4": 0.01,"10": 0.00,"20": 0.00,"50": 0.01,"100": 60.74,"250": 39.18,"500": 0.06,"750": 0.01,"1000": 0.01},"latency_ms": {"2": 0.01,"4": 0.01,"10": 0.01,"20": 0.00,"50": 0.01,"100": 0.00,"250": 0.00,"500": 0.00,"750": 0.00,"1000": 0.00,"2000": 0.00,">=2000": 0.00}}],"disk_util": [{"name": "vda","read_ios": 0,"write_ios": 615418,"read_merges": 0,"write_merges": 231,"read_ticks": 0,"write_ticks": 58284,"in_queue": 58024,"util": 82.45}]}
diff --git a/tests/unit/benchmark/scenarios/storage/test_fio.py b/tests/unit/benchmark/scenarios/storage/test_fio.py
index 45e8a9429..6d38e9c53 100644
--- a/tests/unit/benchmark/scenarios/storage/test_fio.py
+++ b/tests/unit/benchmark/scenarios/storage/test_fio.py
@@ -26,16 +26,21 @@ class FioTestCase(unittest.TestCase):
self.ctx = {
'host': '172.16.0.137',
'user': 'cirros',
- 'key_filename': "mykey.key"
+ 'key_filename': 'mykey.key'
+ }
+ self.sample_output = {
+ 'read': 'fio_read_sample_output.json',
+ 'write': 'fio_write_sample_output.json',
+ 'rw': 'fio_rw_sample_output.json'
}
def test_fio_successful_setup(self, mock_ssh):
p = fio.Fio(self.ctx)
options = {
- 'filename': "/home/ec2-user/data.raw",
- 'bs': "4k",
- 'rw': "rw",
+ 'filename': '/home/ec2-user/data.raw',
+ 'bs': '4k',
+ 'rw': 'rw',
'ramp_time': 10
}
args = {'options': options}
@@ -49,15 +54,134 @@ class FioTestCase(unittest.TestCase):
p = fio.Fio(self.ctx)
options = {
- 'filename': "/home/ec2-user/data.raw",
- 'bs': "4k",
- 'rw': "rw",
+ 'filename': '/home/ec2-user/data.raw',
+ 'bs': '4k',
+ 'rw': 'rw',
+ 'ramp_time': 10
+ }
+ args = {'options': options}
+ p.client = mock_ssh.SSH()
+
+ sample_output = self._read_sample_output(self.sample_output['rw'])
+ mock_ssh.SSH().execute.return_value = (0, sample_output, '')
+
+ result = p.run(args)
+
+ expected_result = '{"read_bw": 83888, "read_iops": 20972,' \
+ '"read_lat": 236.8, "write_bw": 84182, "write_iops": 21045,'\
+ '"write_lat": 233.55}'
+ expected_result = json.loads(expected_result)
+ self.assertEqual(result, expected_result)
+
+ def test_fio_successful_read_no_sla(self, mock_ssh):
+
+ p = fio.Fio(self.ctx)
+ options = {
+ 'filename': '/home/ec2-user/data.raw',
+ 'bs': '4k',
+ 'rw': "read",
+ 'ramp_time': 10
+ }
+ args = {'options': options}
+ p.client = mock_ssh.SSH()
+
+ sample_output = self._read_sample_output(self.sample_output['read'])
+ mock_ssh.SSH().execute.return_value = (0, sample_output, '')
+
+ result = p.run(args)
+
+ expected_result = '{"read_bw": 36113, "read_iops": 9028,' \
+ '"read_lat": 108.7}'
+ expected_result = json.loads(expected_result)
+ self.assertEqual(result, expected_result)
+
+ def test_fio_successful_write_no_sla(self, mock_ssh):
+
+ p = fio.Fio(self.ctx)
+ options = {
+ 'filename': '/home/ec2-user/data.raw',
+ 'bs': '4k',
+ 'rw': 'write',
'ramp_time': 10
}
args = {'options': options}
p.client = mock_ssh.SSH()
- sample_output = self._read_sample_output()
+ sample_output = self._read_sample_output(self.sample_output['write'])
+ mock_ssh.SSH().execute.return_value = (0, sample_output, '')
+
+ result = p.run(args)
+
+ expected_result = '{"write_bw": 35107, "write_iops": 8776,'\
+ '"write_lat": 111.74}'
+ expected_result = json.loads(expected_result)
+ self.assertEqual(result, expected_result)
+
+ def test_fio_successful_lat_sla(self, mock_ssh):
+
+ p = fio.Fio(self.ctx)
+ options = {
+ 'filename': '/home/ec2-user/data.raw',
+ 'bs': '4k',
+ 'rw': 'rw',
+ 'ramp_time': 10
+ }
+ args = {
+ 'options': options,
+ 'sla': {'write_lat': 300.1}
+ }
+
+ p.client = mock_ssh.SSH()
+
+ sample_output = self._read_sample_output(self.sample_output['rw'])
+ mock_ssh.SSH().execute.return_value = (0, sample_output, '')
+
+ result = p.run(args)
+
+ expected_result = '{"read_bw": 83888, "read_iops": 20972,' \
+ '"read_lat": 236.8, "write_bw": 84182, "write_iops": 21045,'\
+ '"write_lat": 233.55}'
+ expected_result = json.loads(expected_result)
+ self.assertEqual(result, expected_result)
+
+
+ def test_fio_unsuccessful_lat_sla(self, mock_ssh):
+
+ p = fio.Fio(self.ctx)
+ options = {
+ 'filename': '/home/ec2-user/data.raw',
+ 'bs': '4k',
+ 'rw': 'rw',
+ 'ramp_time': 10
+ }
+ args = {
+ 'options': options,
+ 'sla': {'write_lat': 200.1}
+ }
+
+ p.client = mock_ssh.SSH()
+
+ sample_output = self._read_sample_output(self.sample_output['rw'])
+ mock_ssh.SSH().execute.return_value = (0, sample_output, '')
+ self.assertRaises(AssertionError, p.run, args)
+
+ def test_fio_successful_bw_iops_sla(self, mock_ssh):
+
+ p = fio.Fio(self.ctx)
+ options = {
+ 'filename': '/home/ec2-user/data.raw',
+ 'bs': '4k',
+ 'rw': 'rw',
+ 'ramp_time': 10
+ }
+ args = {
+ 'options': options,
+ 'sla': {'read_iops': 20000}
+ }
+
+ p.client = mock_ssh.SSH()
+
+ sample_output = self._read_sample_output(self.sample_output['rw'])
mock_ssh.SSH().execute.return_value = (0, sample_output, '')
result = p.run(args)
@@ -68,13 +192,33 @@ class FioTestCase(unittest.TestCase):
expected_result = json.loads(expected_result)
self.assertEqual(result, expected_result)
+ def test_fio_unsuccessful_bw_iops_sla(self, mock_ssh):
+
+ p = fio.Fio(self.ctx)
+ options = {
+ 'filename': '/home/ec2-user/data.raw',
+ 'bs': '4k',
+ 'rw': 'rw',
+ 'ramp_time': 10
+ }
+ args = {
+ 'options': options,
+ 'sla': {'read_iops': 30000}
+ }
+
+ p.client = mock_ssh.SSH()
+
+ sample_output = self._read_sample_output(self.sample_output['rw'])
+ mock_ssh.SSH().execute.return_value = (0, sample_output, '')
+ self.assertRaises(AssertionError, p.run, args)
+
def test_fio_unsuccessful_script_error(self, mock_ssh):
p = fio.Fio(self.ctx)
options = {
- 'filename': "/home/ec2-user/data.raw",
- 'bs': "4k",
- 'rw': "rw",
+ 'filename': '/home/ec2-user/data.raw',
+ 'bs': '4k',
+ 'rw': 'rw',
'ramp_time': 10
}
args = {'options': options}
@@ -83,9 +227,9 @@ class FioTestCase(unittest.TestCase):
mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR')
self.assertRaises(RuntimeError, p.run, args)
- def _read_sample_output(self):
+ def _read_sample_output(self, file_name):
curr_path = os.path.dirname(os.path.abspath(__file__))
- output = os.path.join(curr_path, 'fio_sample_output.json')
+ output = os.path.join(curr_path, file_name)
with open(output) as f:
sample_output = f.read()
return sample_output
diff --git a/yardstick/benchmark/scenarios/storage/fio.py b/yardstick/benchmark/scenarios/storage/fio.py
index 42f159164..1107a8b2c 100644
--- a/yardstick/benchmark/scenarios/storage/fio.py
+++ b/yardstick/benchmark/scenarios/storage/fio.py
@@ -114,14 +114,30 @@ class Fio(base.Scenario):
raw_data = json.loads(stdout)
# The bandwidth unit is KB/s, and latency unit is us
- result["read_bw"] = raw_data["jobs"][0]["read"]["bw"]
- result["read_iops"] = raw_data["jobs"][0]["read"]["iops"]
- result["read_lat"] = raw_data["jobs"][0]["read"]["lat"]["mean"]
- result["write_bw"] = raw_data["jobs"][0]["write"]["bw"]
- result["write_iops"] = raw_data["jobs"][0]["write"]["iops"]
- result["write_lat"] = raw_data["jobs"][0]["write"]["lat"]["mean"]
-
- # TODO: add sla check
+ if rw in ["read", "randread", "rw", "randrw"]:
+ result["read_bw"] = raw_data["jobs"][0]["read"]["bw"]
+ result["read_iops"] = raw_data["jobs"][0]["read"]["iops"]
+ result["read_lat"] = raw_data["jobs"][0]["read"]["lat"]["mean"]
+ if rw in ["write", "randwrite", "rw", "randrw"]:
+ result["write_bw"] = raw_data["jobs"][0]["write"]["bw"]
+ result["write_iops"] = raw_data["jobs"][0]["write"]["iops"]
+ result["write_lat"] = raw_data["jobs"][0]["write"]["lat"]["mean"]
+
+ if "sla" in args:
+ for k, v in result.items():
+ if k not in args['sla']:
+ continue
+
+ if "lat" in k:
+ # For lattency small value is better
+ max_v = float(args['sla'][k])
+ assert v <= max_v, "%s %f > " \
+ "sla:%s(%f)" % (k, v, k, max_v)
+ else:
+ # For bandwidth and iops big value is better
+ min_v = int(args['sla'][k])
+ assert v >= min_v, "%s %d < " \
+ "sla:%s(%d)" % (k, v, k, min_v)
return result