diff options
Diffstat (limited to 'docker')
8 files changed, 134 insertions, 42 deletions
diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py index 7f38ab6..1b66af3 100644 --- a/docker/storperf-master/rest_server.py +++ b/docker/storperf-master/rest_server.py @@ -98,6 +98,7 @@ class ConfigurationRequestModel: 'agent_flavor': fields.String, 'agent_image': fields.String, 'public_network': fields.String, + 'volume_count': fields.Integer, 'volume_size': fields.Integer, 'availability_zone': fields.String, 'username': fields.String, @@ -114,6 +115,7 @@ class ConfigurationResponseModel: 'public_network': fields.String, 'stack_created': fields.Boolean, 'stack_id': fields.String, + 'volume_count': fields.Integer, 'volume_size': fields.Integer, 'availability_zone': fields.String } @@ -135,6 +137,7 @@ class Configure(Resource): 'agent_flavor': storperf.agent_flavor, 'agent_image': storperf.agent_image, 'public_network': storperf.public_network, + 'volume_count': storperf.volume_count, 'volume_size': storperf.volume_size, 'stack_created': storperf.is_stack_created, 'availability_zone': storperf.availability_zone, @@ -170,6 +173,8 @@ class Configure(Resource): storperf.agent_image = request.json['agent_image'] if ('public_network' in request.json): storperf.public_network = request.json['public_network'] + if ('volume_count' in request.json): + storperf.volume_count = request.json['volume_count'] if ('volume_size' in request.json): storperf.volume_size = request.json['volume_size'] if ('availability_zone' in request.json): @@ -188,6 +193,7 @@ class Configure(Resource): 'agent_image': storperf.agent_image, 'availability_zone': storperf.availability_zone, 'public_network': storperf.public_network, + 'volume_count': storperf.volume_count, 'volume_size': storperf.volume_size, 'stack_id': storperf.stack_id}) diff --git a/docker/storperf-master/storperf/resources/hot/agent-group.yaml b/docker/storperf-master/storperf/resources/hot/agent-group.yaml index 3c02e31..ea7b51f 100644 --- a/docker/storperf-master/storperf/resources/hot/agent-group.yaml +++ b/docker/storperf-master/storperf/resources/hot/agent-group.yaml @@ -27,6 +27,12 @@ parameters: constraints: - range: { min: 1, max: 1024 } description: must be between 1 and 1024 Gb. + volume_count: + type: number + default: 0 + constraints: + - range: { min: 0, max: 512 } + description: must be between 1 and 512 agents. agent_count: type: number default: 1 @@ -54,6 +60,7 @@ resources: availability_zone: {get_param: availability_zone}, storperf_open_security_group: {get_resource: storperf_open_security_group}, key_name: {get_resource: storperf_key_pair}, + volume_count: {get_param: volume_count}, volume_size: {get_param: volume_size} } } diff --git a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml index 7841e8c..8895c9f 100644 --- a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml +++ b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml @@ -24,6 +24,13 @@ parameters: default: storperf storperf_open_security_group: type: string + volume_count: + type: number + description: Number of volumes to be created + default: 1 + constraints: + - range: { min: 0, max: 1024 } + description: must be between 1 and 1024. volume_size: type: number description: Size of the volume to be created. @@ -87,15 +94,16 @@ resources: port_id: { get_resource: storperf_agent_port } agent_volume: - type: OS::Cinder::Volume - properties: - size: { get_param: volume_size } - - agent_volume_att: - type: OS::Cinder::VolumeAttachment + type: OS::Heat::ResourceGroup properties: - instance_uuid: { get_resource: storperf_agent } - volume_id: { get_resource: agent_volume} + count: { get_param: volume_count } + resource_def: { + type: "storperf-volume.yaml", + properties: { + volume_size: { get_param: volume_size }, + agent_instance_uuid: { get_resource: storperf_agent } + } + } outputs: storperf_agent_ip: diff --git a/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml new file mode 100644 index 0000000..aec3393 --- /dev/null +++ b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml @@ -0,0 +1,33 @@ +############################################################################## +# Copyright (c) 2018 Dell EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +heat_template_version: 2013-05-23 + +parameters: + volume_size: + type: number + description: Size of the volume to be created. + default: 1 + constraints: + - range: { min: 1, max: 1024 } + description: must be between 1 and 1024 Gb. + agent_instance_uuid: + type: string + +resources: + agent_volume: + type: OS::Cinder::Volume + properties: + size: { get_param: volume_size } + + agent_volume_att: + type: OS::Cinder::VolumeAttachment + properties: + instance_uuid: { get_param: agent_instance_uuid } + volume_id: { get_resource: agent_volume} diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py index 7244b66..d54fea3 100644 --- a/docker/storperf-master/storperf/storperf_master.py +++ b/docker/storperf-master/storperf/storperf_master.py @@ -64,11 +64,24 @@ class StorPerfMaster(object): self._agent_flavor = "storperf" self._availability_zone = None self._public_network = None + self._volume_count = 1 self._volume_size = 1 self._cached_stack_id = None self._last_snaps_check_time = None @property + def volume_count(self): + self._get_stack_info() + return self._volume_count + + @volume_count.setter + def volume_count(self, value): + if (self.stack_id is not None): + raise ParameterError( + "ERROR: Cannot change volume count after stack is created") + self._volume_count = value + + @property def volume_size(self): self._get_stack_info() return self._volume_size @@ -136,14 +149,12 @@ class StorPerfMaster(object): def _get_stack_info(self): if self._last_snaps_check_time is not None: time_since_check = datetime.now() - self._last_snaps_check_time - if time_since_check.total_seconds() < 30: + if time_since_check.total_seconds() < 60: return self._cached_stack_id self.heat_stack.initialize() if self.heat_stack.get_stack() is not None: self._last_snaps_check_time = datetime.now() - if self._cached_stack_id == self.heat_stack.get_stack().id: - return self._cached_stack_id self._cached_stack_id = self.heat_stack.get_stack().id cinder_cli = cinder_utils.cinder_client(self.os_creds) glance_cli = glance_utils.glance_client(self.os_creds) @@ -162,10 +173,14 @@ class StorPerfMaster(object): image = glance_utils.get_image_by_id(glance_cli, image_id) self._agent_image = image.name - volume_id = server.volume_ids[0]['id'] - volume = cinder_utils.get_volume_by_id( - cinder_cli, volume_id) - self._volume_size = volume.size + self._volume_count = len(server.volume_ids) + if self._volume_count > 0: + volume_id = server.volume_ids[0]['id'] + volume = cinder_utils.get_volume_by_id( + cinder_cli, volume_id) + self.logger.debug("Volume id %s, size=%s" % (volume.id, + volume.size)) + self._volume_size = volume.size router_creators = self.heat_stack.get_router_creators() router1 = router_creators[0] @@ -265,15 +280,17 @@ class StorPerfMaster(object): def create_stack(self): self.stack_settings.resource_files = [ - 'storperf/resources/hot/storperf-agent.yaml'] + 'storperf/resources/hot/storperf-agent.yaml', + 'storperf/resources/hot/storperf-volume.yaml'] self.stack_settings.env_values = self._make_parameters() try: self.heat_stack.create() - except Exception: + except Exception as e: + self.logger.error("Stack creation failed") + self.logger.exception(e) heat_cli = heat_utils.heat_client(self.os_creds) res = heat_utils.get_resources(heat_cli, self.heat_stack.get_stack().id) - self.logger.error("Stack creation failed") reason = "" failed = False for resource in res: @@ -325,10 +342,12 @@ class StorPerfMaster(object): thread.join() self._test_executor.slaves = slaves + self._test_executor.volume_count = self.volume_count params = metadata params['agent_count'] = self.agent_count params['public_network'] = self.public_network + params['volume_count'] = self.volume_count params['volume_size'] = self.volume_size if self.username and self.password: params['username'] = self.username @@ -446,6 +465,7 @@ class StorPerfMaster(object): heat_parameters = {} heat_parameters['public_network'] = self.public_network heat_parameters['agent_count'] = self.agent_count + heat_parameters['volume_count'] = self.volume_count heat_parameters['volume_size'] = self.volume_size heat_parameters['agent_image'] = self.agent_image heat_parameters['agent_flavor'] = self.agent_flavor diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py index 9ed6386..2ed6a9e 100644 --- a/docker/storperf-master/storperf/test_executor.py +++ b/docker/storperf-master/storperf/test_executor.py @@ -53,6 +53,7 @@ class TestExecutor(object): self.job_db = JobDB() self._slaves = [] self._terminated = False + self._volume_count = 1 self._workload_executors = [] self._workload_thread = None self._thread_gate = None @@ -88,6 +89,15 @@ class TestExecutor(object): self._slaves = slaves @property + def volume_count(self): + return self._volume_count + + @volume_count.setter + def volume_count(self, volume_count): + self.logger.debug("Set volume count to: " + str(volume_count)) + self._volume_count = volume_count + + @property def queue_depths(self): return ','.join(self._queue_depths) @@ -282,17 +292,24 @@ class TestExecutor(object): slave_threads = [] for slave in self.slaves: - slave_workload = copy.copy(current_workload['workload']) - slave_workload.remote_host = slave - - self._workload_executors.append(slave_workload) - - t = Thread(target=self.execute_on_node, - args=(slave_workload,), - name="%s worker" % slave) - t.daemon = False - t.start() - slave_threads.append(t) + volume_number = 0 + while volume_number < self.volume_count: + slave_workload = copy.copy(current_workload['workload']) + slave_workload.remote_host = slave + last_char_of_filename = chr(ord( + slave_workload.filename[-1:]) + volume_number) + slave_workload.filename = "%s%s" % \ + (slave_workload.filename[:-1], last_char_of_filename) + self.logger.debug("Device to profile: %s" % + slave_workload.filename) + self._workload_executors.append(slave_workload) + t = Thread(target=self.execute_on_node, + args=(slave_workload,), + name="%s worker" % slave) + t.daemon = False + t.start() + slave_threads.append(t) + volume_number += 1 for slave_thread in slave_threads: self.logger.debug("Waiting on %s" % slave_thread) @@ -335,9 +352,6 @@ class TestExecutor(object): workload.filename = self.filename workload.id = self.job_db.job_id - if (self.filename is not None): - workload.filename = self.filename - if (workload_name.startswith("_")): iodepths = [8, ] blocksizes = [16384, ] diff --git a/docker/storperf-master/storperf/workloads/_base_workload.py b/docker/storperf-master/storperf/workloads/_base_workload.py index d5282d7..c2c7b7b 100644 --- a/docker/storperf-master/storperf/workloads/_base_workload.py +++ b/docker/storperf-master/storperf/workloads/_base_workload.py @@ -74,9 +74,11 @@ class _base_workload(object): @property def fullname(self): + host_file = self.remote_host+"."+self.filename + host_file = host_file.replace(".", "-").replace("/", "-") return ("%s.%s.queue-depth.%s.block-size.%s.%s" % (str(self.id), self.__class__.__name__, str(self.options['iodepth']), str(self.options['bs']), - str(self.remote_host).replace(".", "-"))) + host_file)) diff --git a/docker/storperf-master/tests/workload_tests/workload_subclass_test.py b/docker/storperf-master/tests/workload_tests/workload_subclass_test.py index e9e47f3..c61fe74 100644 --- a/docker/storperf-master/tests/workload_tests/workload_subclass_test.py +++ b/docker/storperf-master/tests/workload_tests/workload_subclass_test.py @@ -22,33 +22,35 @@ class WorkloadSubclassTest(unittest.TestCase): def test_local_name(self): workload = rr() self.assertEqual(workload.fullname, - "None.rr.queue-depth.1.block-size.64k.None", + "None.rr.queue-depth.1.block-size.64k.None--dev-vdb", workload.fullname) def test_remote_name(self): workload = rw() workload.remote_host = "192.168.0.1" - self.assertEqual(workload.fullname, - "None.rw.queue-depth.1.block-size.64k.192-168-0-1", - workload.fullname) + self.assertEqual( + workload.fullname, + "None.rw.queue-depth.1.block-size.64k.192-168-0-1--dev-vdb", + workload.fullname) def test_blocksize(self): workload = rs() workload.options["bs"] = "4k" self.assertEqual(workload.fullname, - "None.rs.queue-depth.1.block-size.4k.None", + "None.rs.queue-depth.1.block-size.4k.None--dev-vdb", workload.fullname) def test_queue_depth(self): workload = wr() workload.options["iodepth"] = "8" self.assertEqual(workload.fullname, - "None.wr.queue-depth.8.block-size.64k.None", + "None.wr.queue-depth.8.block-size.64k.None--dev-vdb", workload.fullname) def test_id(self): workload = ws() workload.id = "workloadid" - self.assertEqual(workload.fullname, - "workloadid.ws.queue-depth.1.block-size.64k.None", - workload.fullname) + self.assertEqual( + workload.fullname, + "workloadid.ws.queue-depth.1.block-size.64k.None--dev-vdb", + workload.fullname) |