summaryrefslogtreecommitdiffstats
path: root/docker/storperf-master
diff options
context:
space:
mode:
authormbeierl <mark.beierl@dell.com>2018-05-04 14:04:51 -0400
committermbeierl <mark.beierl@dell.com>2018-05-10 10:53:55 -0400
commit1942e374fba718d1d4b6bc5388803c75e71aa197 (patch)
tree045c15e23ef5def85f765df863f9b342cebc713a /docker/storperf-master
parent64c8b3b561ccac6159241cd0485a3259887c6bb0 (diff)
Adds Volume Count and Dynamic Reload
Adds a volume_count parameter that allows the user to change the number of volumes. Test execution still uses each volume specified and appends it to the host ip address in the graphite metrics. Changes the cache policy of the stack from SNAPS-OO so that changes made to the stack are refelcted in StorPerf. Fixes the documentation path and updates the docs for the above changes. Change-Id: I5b9160b469e80d4f7d6906ac270bb3b75411fb5a JIRA: STORPERF-242 Signed-off-by: mbeierl <mark.beierl@dell.com>
Diffstat (limited to 'docker/storperf-master')
-rw-r--r--docker/storperf-master/rest_server.py6
-rw-r--r--docker/storperf-master/storperf/resources/hot/agent-group.yaml7
-rw-r--r--docker/storperf-master/storperf/resources/hot/storperf-agent.yaml24
-rw-r--r--docker/storperf-master/storperf/resources/hot/storperf-volume.yaml33
-rw-r--r--docker/storperf-master/storperf/storperf_master.py40
-rw-r--r--docker/storperf-master/storperf/test_executor.py42
-rw-r--r--docker/storperf-master/storperf/workloads/_base_workload.py4
-rw-r--r--docker/storperf-master/tests/workload_tests/workload_subclass_test.py20
8 files changed, 134 insertions, 42 deletions
diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py
index 7f38ab6..1b66af3 100644
--- a/docker/storperf-master/rest_server.py
+++ b/docker/storperf-master/rest_server.py
@@ -98,6 +98,7 @@ class ConfigurationRequestModel:
'agent_flavor': fields.String,
'agent_image': fields.String,
'public_network': fields.String,
+ 'volume_count': fields.Integer,
'volume_size': fields.Integer,
'availability_zone': fields.String,
'username': fields.String,
@@ -114,6 +115,7 @@ class ConfigurationResponseModel:
'public_network': fields.String,
'stack_created': fields.Boolean,
'stack_id': fields.String,
+ 'volume_count': fields.Integer,
'volume_size': fields.Integer,
'availability_zone': fields.String
}
@@ -135,6 +137,7 @@ class Configure(Resource):
'agent_flavor': storperf.agent_flavor,
'agent_image': storperf.agent_image,
'public_network': storperf.public_network,
+ 'volume_count': storperf.volume_count,
'volume_size': storperf.volume_size,
'stack_created': storperf.is_stack_created,
'availability_zone': storperf.availability_zone,
@@ -170,6 +173,8 @@ class Configure(Resource):
storperf.agent_image = request.json['agent_image']
if ('public_network' in request.json):
storperf.public_network = request.json['public_network']
+ if ('volume_count' in request.json):
+ storperf.volume_count = request.json['volume_count']
if ('volume_size' in request.json):
storperf.volume_size = request.json['volume_size']
if ('availability_zone' in request.json):
@@ -188,6 +193,7 @@ class Configure(Resource):
'agent_image': storperf.agent_image,
'availability_zone': storperf.availability_zone,
'public_network': storperf.public_network,
+ 'volume_count': storperf.volume_count,
'volume_size': storperf.volume_size,
'stack_id': storperf.stack_id})
diff --git a/docker/storperf-master/storperf/resources/hot/agent-group.yaml b/docker/storperf-master/storperf/resources/hot/agent-group.yaml
index 3c02e31..ea7b51f 100644
--- a/docker/storperf-master/storperf/resources/hot/agent-group.yaml
+++ b/docker/storperf-master/storperf/resources/hot/agent-group.yaml
@@ -27,6 +27,12 @@ parameters:
constraints:
- range: { min: 1, max: 1024 }
description: must be between 1 and 1024 Gb.
+ volume_count:
+ type: number
+ default: 0
+ constraints:
+ - range: { min: 0, max: 512 }
+ description: must be between 1 and 512 agents.
agent_count:
type: number
default: 1
@@ -54,6 +60,7 @@ resources:
availability_zone: {get_param: availability_zone},
storperf_open_security_group: {get_resource: storperf_open_security_group},
key_name: {get_resource: storperf_key_pair},
+ volume_count: {get_param: volume_count},
volume_size: {get_param: volume_size}
}
}
diff --git a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml
index 7841e8c..8895c9f 100644
--- a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml
+++ b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml
@@ -24,6 +24,13 @@ parameters:
default: storperf
storperf_open_security_group:
type: string
+ volume_count:
+ type: number
+ description: Number of volumes to be created
+ default: 1
+ constraints:
+ - range: { min: 0, max: 1024 }
+ description: must be between 1 and 1024.
volume_size:
type: number
description: Size of the volume to be created.
@@ -87,15 +94,16 @@ resources:
port_id: { get_resource: storperf_agent_port }
agent_volume:
- type: OS::Cinder::Volume
- properties:
- size: { get_param: volume_size }
-
- agent_volume_att:
- type: OS::Cinder::VolumeAttachment
+ type: OS::Heat::ResourceGroup
properties:
- instance_uuid: { get_resource: storperf_agent }
- volume_id: { get_resource: agent_volume}
+ count: { get_param: volume_count }
+ resource_def: {
+ type: "storperf-volume.yaml",
+ properties: {
+ volume_size: { get_param: volume_size },
+ agent_instance_uuid: { get_resource: storperf_agent }
+ }
+ }
outputs:
storperf_agent_ip:
diff --git a/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml
new file mode 100644
index 0000000..aec3393
--- /dev/null
+++ b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml
@@ -0,0 +1,33 @@
+##############################################################################
+# Copyright (c) 2018 Dell EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+heat_template_version: 2013-05-23
+
+parameters:
+ volume_size:
+ type: number
+ description: Size of the volume to be created.
+ default: 1
+ constraints:
+ - range: { min: 1, max: 1024 }
+ description: must be between 1 and 1024 Gb.
+ agent_instance_uuid:
+ type: string
+
+resources:
+ agent_volume:
+ type: OS::Cinder::Volume
+ properties:
+ size: { get_param: volume_size }
+
+ agent_volume_att:
+ type: OS::Cinder::VolumeAttachment
+ properties:
+ instance_uuid: { get_param: agent_instance_uuid }
+ volume_id: { get_resource: agent_volume}
diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py
index 7244b66..d54fea3 100644
--- a/docker/storperf-master/storperf/storperf_master.py
+++ b/docker/storperf-master/storperf/storperf_master.py
@@ -64,11 +64,24 @@ class StorPerfMaster(object):
self._agent_flavor = "storperf"
self._availability_zone = None
self._public_network = None
+ self._volume_count = 1
self._volume_size = 1
self._cached_stack_id = None
self._last_snaps_check_time = None
@property
+ def volume_count(self):
+ self._get_stack_info()
+ return self._volume_count
+
+ @volume_count.setter
+ def volume_count(self, value):
+ if (self.stack_id is not None):
+ raise ParameterError(
+ "ERROR: Cannot change volume count after stack is created")
+ self._volume_count = value
+
+ @property
def volume_size(self):
self._get_stack_info()
return self._volume_size
@@ -136,14 +149,12 @@ class StorPerfMaster(object):
def _get_stack_info(self):
if self._last_snaps_check_time is not None:
time_since_check = datetime.now() - self._last_snaps_check_time
- if time_since_check.total_seconds() < 30:
+ if time_since_check.total_seconds() < 60:
return self._cached_stack_id
self.heat_stack.initialize()
if self.heat_stack.get_stack() is not None:
self._last_snaps_check_time = datetime.now()
- if self._cached_stack_id == self.heat_stack.get_stack().id:
- return self._cached_stack_id
self._cached_stack_id = self.heat_stack.get_stack().id
cinder_cli = cinder_utils.cinder_client(self.os_creds)
glance_cli = glance_utils.glance_client(self.os_creds)
@@ -162,10 +173,14 @@ class StorPerfMaster(object):
image = glance_utils.get_image_by_id(glance_cli, image_id)
self._agent_image = image.name
- volume_id = server.volume_ids[0]['id']
- volume = cinder_utils.get_volume_by_id(
- cinder_cli, volume_id)
- self._volume_size = volume.size
+ self._volume_count = len(server.volume_ids)
+ if self._volume_count > 0:
+ volume_id = server.volume_ids[0]['id']
+ volume = cinder_utils.get_volume_by_id(
+ cinder_cli, volume_id)
+ self.logger.debug("Volume id %s, size=%s" % (volume.id,
+ volume.size))
+ self._volume_size = volume.size
router_creators = self.heat_stack.get_router_creators()
router1 = router_creators[0]
@@ -265,15 +280,17 @@ class StorPerfMaster(object):
def create_stack(self):
self.stack_settings.resource_files = [
- 'storperf/resources/hot/storperf-agent.yaml']
+ 'storperf/resources/hot/storperf-agent.yaml',
+ 'storperf/resources/hot/storperf-volume.yaml']
self.stack_settings.env_values = self._make_parameters()
try:
self.heat_stack.create()
- except Exception:
+ except Exception as e:
+ self.logger.error("Stack creation failed")
+ self.logger.exception(e)
heat_cli = heat_utils.heat_client(self.os_creds)
res = heat_utils.get_resources(heat_cli,
self.heat_stack.get_stack().id)
- self.logger.error("Stack creation failed")
reason = ""
failed = False
for resource in res:
@@ -325,10 +342,12 @@ class StorPerfMaster(object):
thread.join()
self._test_executor.slaves = slaves
+ self._test_executor.volume_count = self.volume_count
params = metadata
params['agent_count'] = self.agent_count
params['public_network'] = self.public_network
+ params['volume_count'] = self.volume_count
params['volume_size'] = self.volume_size
if self.username and self.password:
params['username'] = self.username
@@ -446,6 +465,7 @@ class StorPerfMaster(object):
heat_parameters = {}
heat_parameters['public_network'] = self.public_network
heat_parameters['agent_count'] = self.agent_count
+ heat_parameters['volume_count'] = self.volume_count
heat_parameters['volume_size'] = self.volume_size
heat_parameters['agent_image'] = self.agent_image
heat_parameters['agent_flavor'] = self.agent_flavor
diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py
index 9ed6386..2ed6a9e 100644
--- a/docker/storperf-master/storperf/test_executor.py
+++ b/docker/storperf-master/storperf/test_executor.py
@@ -53,6 +53,7 @@ class TestExecutor(object):
self.job_db = JobDB()
self._slaves = []
self._terminated = False
+ self._volume_count = 1
self._workload_executors = []
self._workload_thread = None
self._thread_gate = None
@@ -88,6 +89,15 @@ class TestExecutor(object):
self._slaves = slaves
@property
+ def volume_count(self):
+ return self._volume_count
+
+ @volume_count.setter
+ def volume_count(self, volume_count):
+ self.logger.debug("Set volume count to: " + str(volume_count))
+ self._volume_count = volume_count
+
+ @property
def queue_depths(self):
return ','.join(self._queue_depths)
@@ -282,17 +292,24 @@ class TestExecutor(object):
slave_threads = []
for slave in self.slaves:
- slave_workload = copy.copy(current_workload['workload'])
- slave_workload.remote_host = slave
-
- self._workload_executors.append(slave_workload)
-
- t = Thread(target=self.execute_on_node,
- args=(slave_workload,),
- name="%s worker" % slave)
- t.daemon = False
- t.start()
- slave_threads.append(t)
+ volume_number = 0
+ while volume_number < self.volume_count:
+ slave_workload = copy.copy(current_workload['workload'])
+ slave_workload.remote_host = slave
+ last_char_of_filename = chr(ord(
+ slave_workload.filename[-1:]) + volume_number)
+ slave_workload.filename = "%s%s" % \
+ (slave_workload.filename[:-1], last_char_of_filename)
+ self.logger.debug("Device to profile: %s" %
+ slave_workload.filename)
+ self._workload_executors.append(slave_workload)
+ t = Thread(target=self.execute_on_node,
+ args=(slave_workload,),
+ name="%s worker" % slave)
+ t.daemon = False
+ t.start()
+ slave_threads.append(t)
+ volume_number += 1
for slave_thread in slave_threads:
self.logger.debug("Waiting on %s" % slave_thread)
@@ -335,9 +352,6 @@ class TestExecutor(object):
workload.filename = self.filename
workload.id = self.job_db.job_id
- if (self.filename is not None):
- workload.filename = self.filename
-
if (workload_name.startswith("_")):
iodepths = [8, ]
blocksizes = [16384, ]
diff --git a/docker/storperf-master/storperf/workloads/_base_workload.py b/docker/storperf-master/storperf/workloads/_base_workload.py
index d5282d7..c2c7b7b 100644
--- a/docker/storperf-master/storperf/workloads/_base_workload.py
+++ b/docker/storperf-master/storperf/workloads/_base_workload.py
@@ -74,9 +74,11 @@ class _base_workload(object):
@property
def fullname(self):
+ host_file = self.remote_host+"."+self.filename
+ host_file = host_file.replace(".", "-").replace("/", "-")
return ("%s.%s.queue-depth.%s.block-size.%s.%s"
% (str(self.id),
self.__class__.__name__,
str(self.options['iodepth']),
str(self.options['bs']),
- str(self.remote_host).replace(".", "-")))
+ host_file))
diff --git a/docker/storperf-master/tests/workload_tests/workload_subclass_test.py b/docker/storperf-master/tests/workload_tests/workload_subclass_test.py
index e9e47f3..c61fe74 100644
--- a/docker/storperf-master/tests/workload_tests/workload_subclass_test.py
+++ b/docker/storperf-master/tests/workload_tests/workload_subclass_test.py
@@ -22,33 +22,35 @@ class WorkloadSubclassTest(unittest.TestCase):
def test_local_name(self):
workload = rr()
self.assertEqual(workload.fullname,
- "None.rr.queue-depth.1.block-size.64k.None",
+ "None.rr.queue-depth.1.block-size.64k.None--dev-vdb",
workload.fullname)
def test_remote_name(self):
workload = rw()
workload.remote_host = "192.168.0.1"
- self.assertEqual(workload.fullname,
- "None.rw.queue-depth.1.block-size.64k.192-168-0-1",
- workload.fullname)
+ self.assertEqual(
+ workload.fullname,
+ "None.rw.queue-depth.1.block-size.64k.192-168-0-1--dev-vdb",
+ workload.fullname)
def test_blocksize(self):
workload = rs()
workload.options["bs"] = "4k"
self.assertEqual(workload.fullname,
- "None.rs.queue-depth.1.block-size.4k.None",
+ "None.rs.queue-depth.1.block-size.4k.None--dev-vdb",
workload.fullname)
def test_queue_depth(self):
workload = wr()
workload.options["iodepth"] = "8"
self.assertEqual(workload.fullname,
- "None.wr.queue-depth.8.block-size.64k.None",
+ "None.wr.queue-depth.8.block-size.64k.None--dev-vdb",
workload.fullname)
def test_id(self):
workload = ws()
workload.id = "workloadid"
- self.assertEqual(workload.fullname,
- "workloadid.ws.queue-depth.1.block-size.64k.None",
- workload.fullname)
+ self.assertEqual(
+ workload.fullname,
+ "workloadid.ws.queue-depth.1.block-size.64k.None--dev-vdb",
+ workload.fullname)