summaryrefslogtreecommitdiffstats
path: root/docker
diff options
context:
space:
mode:
authorMark Beierl <mark.beierl@dell.com>2018-05-11 11:54:43 +0000
committerGerrit Code Review <gerrit@opnfv.org>2018-05-11 11:54:43 +0000
commita7d16b3d281e419e73dc729f33c197973cd2f9f5 (patch)
tree82892dd727f0876c04fdfdf94c7921280202b7e7 /docker
parent597e1811fb5761a4899326ebe6ab17bd9c47fdb5 (diff)
parent1942e374fba718d1d4b6bc5388803c75e71aa197 (diff)
Merge "Adds Volume Count and Dynamic Reload"
Diffstat (limited to 'docker')
-rw-r--r--docker/storperf-master/rest_server.py6
-rw-r--r--docker/storperf-master/storperf/resources/hot/agent-group.yaml7
-rw-r--r--docker/storperf-master/storperf/resources/hot/storperf-agent.yaml24
-rw-r--r--docker/storperf-master/storperf/resources/hot/storperf-volume.yaml33
-rw-r--r--docker/storperf-master/storperf/storperf_master.py40
-rw-r--r--docker/storperf-master/storperf/test_executor.py42
-rw-r--r--docker/storperf-master/storperf/workloads/_base_workload.py4
-rw-r--r--docker/storperf-master/tests/workload_tests/workload_subclass_test.py20
8 files changed, 134 insertions, 42 deletions
diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py
index 7f38ab6..1b66af3 100644
--- a/docker/storperf-master/rest_server.py
+++ b/docker/storperf-master/rest_server.py
@@ -98,6 +98,7 @@ class ConfigurationRequestModel:
'agent_flavor': fields.String,
'agent_image': fields.String,
'public_network': fields.String,
+ 'volume_count': fields.Integer,
'volume_size': fields.Integer,
'availability_zone': fields.String,
'username': fields.String,
@@ -114,6 +115,7 @@ class ConfigurationResponseModel:
'public_network': fields.String,
'stack_created': fields.Boolean,
'stack_id': fields.String,
+ 'volume_count': fields.Integer,
'volume_size': fields.Integer,
'availability_zone': fields.String
}
@@ -135,6 +137,7 @@ class Configure(Resource):
'agent_flavor': storperf.agent_flavor,
'agent_image': storperf.agent_image,
'public_network': storperf.public_network,
+ 'volume_count': storperf.volume_count,
'volume_size': storperf.volume_size,
'stack_created': storperf.is_stack_created,
'availability_zone': storperf.availability_zone,
@@ -170,6 +173,8 @@ class Configure(Resource):
storperf.agent_image = request.json['agent_image']
if ('public_network' in request.json):
storperf.public_network = request.json['public_network']
+ if ('volume_count' in request.json):
+ storperf.volume_count = request.json['volume_count']
if ('volume_size' in request.json):
storperf.volume_size = request.json['volume_size']
if ('availability_zone' in request.json):
@@ -188,6 +193,7 @@ class Configure(Resource):
'agent_image': storperf.agent_image,
'availability_zone': storperf.availability_zone,
'public_network': storperf.public_network,
+ 'volume_count': storperf.volume_count,
'volume_size': storperf.volume_size,
'stack_id': storperf.stack_id})
diff --git a/docker/storperf-master/storperf/resources/hot/agent-group.yaml b/docker/storperf-master/storperf/resources/hot/agent-group.yaml
index 3c02e31..ea7b51f 100644
--- a/docker/storperf-master/storperf/resources/hot/agent-group.yaml
+++ b/docker/storperf-master/storperf/resources/hot/agent-group.yaml
@@ -27,6 +27,12 @@ parameters:
constraints:
- range: { min: 1, max: 1024 }
description: must be between 1 and 1024 Gb.
+ volume_count:
+ type: number
+ default: 0
+ constraints:
+ - range: { min: 0, max: 512 }
+ description: must be between 1 and 512 agents.
agent_count:
type: number
default: 1
@@ -54,6 +60,7 @@ resources:
availability_zone: {get_param: availability_zone},
storperf_open_security_group: {get_resource: storperf_open_security_group},
key_name: {get_resource: storperf_key_pair},
+ volume_count: {get_param: volume_count},
volume_size: {get_param: volume_size}
}
}
diff --git a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml
index 7841e8c..8895c9f 100644
--- a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml
+++ b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml
@@ -24,6 +24,13 @@ parameters:
default: storperf
storperf_open_security_group:
type: string
+ volume_count:
+ type: number
+ description: Number of volumes to be created
+ default: 1
+ constraints:
+ - range: { min: 0, max: 1024 }
+ description: must be between 1 and 1024.
volume_size:
type: number
description: Size of the volume to be created.
@@ -87,15 +94,16 @@ resources:
port_id: { get_resource: storperf_agent_port }
agent_volume:
- type: OS::Cinder::Volume
- properties:
- size: { get_param: volume_size }
-
- agent_volume_att:
- type: OS::Cinder::VolumeAttachment
+ type: OS::Heat::ResourceGroup
properties:
- instance_uuid: { get_resource: storperf_agent }
- volume_id: { get_resource: agent_volume}
+ count: { get_param: volume_count }
+ resource_def: {
+ type: "storperf-volume.yaml",
+ properties: {
+ volume_size: { get_param: volume_size },
+ agent_instance_uuid: { get_resource: storperf_agent }
+ }
+ }
outputs:
storperf_agent_ip:
diff --git a/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml
new file mode 100644
index 0000000..aec3393
--- /dev/null
+++ b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml
@@ -0,0 +1,33 @@
+##############################################################################
+# Copyright (c) 2018 Dell EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+heat_template_version: 2013-05-23
+
+parameters:
+ volume_size:
+ type: number
+ description: Size of the volume to be created.
+ default: 1
+ constraints:
+ - range: { min: 1, max: 1024 }
+ description: must be between 1 and 1024 Gb.
+ agent_instance_uuid:
+ type: string
+
+resources:
+ agent_volume:
+ type: OS::Cinder::Volume
+ properties:
+ size: { get_param: volume_size }
+
+ agent_volume_att:
+ type: OS::Cinder::VolumeAttachment
+ properties:
+ instance_uuid: { get_param: agent_instance_uuid }
+ volume_id: { get_resource: agent_volume}
diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py
index 7244b66..d54fea3 100644
--- a/docker/storperf-master/storperf/storperf_master.py
+++ b/docker/storperf-master/storperf/storperf_master.py
@@ -64,11 +64,24 @@ class StorPerfMaster(object):
self._agent_flavor = "storperf"
self._availability_zone = None
self._public_network = None
+ self._volume_count = 1
self._volume_size = 1
self._cached_stack_id = None
self._last_snaps_check_time = None
@property
+ def volume_count(self):
+ self._get_stack_info()
+ return self._volume_count
+
+ @volume_count.setter
+ def volume_count(self, value):
+ if (self.stack_id is not None):
+ raise ParameterError(
+ "ERROR: Cannot change volume count after stack is created")
+ self._volume_count = value
+
+ @property
def volume_size(self):
self._get_stack_info()
return self._volume_size
@@ -136,14 +149,12 @@ class StorPerfMaster(object):
def _get_stack_info(self):
if self._last_snaps_check_time is not None:
time_since_check = datetime.now() - self._last_snaps_check_time
- if time_since_check.total_seconds() < 30:
+ if time_since_check.total_seconds() < 60:
return self._cached_stack_id
self.heat_stack.initialize()
if self.heat_stack.get_stack() is not None:
self._last_snaps_check_time = datetime.now()
- if self._cached_stack_id == self.heat_stack.get_stack().id:
- return self._cached_stack_id
self._cached_stack_id = self.heat_stack.get_stack().id
cinder_cli = cinder_utils.cinder_client(self.os_creds)
glance_cli = glance_utils.glance_client(self.os_creds)
@@ -162,10 +173,14 @@ class StorPerfMaster(object):
image = glance_utils.get_image_by_id(glance_cli, image_id)
self._agent_image = image.name
- volume_id = server.volume_ids[0]['id']
- volume = cinder_utils.get_volume_by_id(
- cinder_cli, volume_id)
- self._volume_size = volume.size
+ self._volume_count = len(server.volume_ids)
+ if self._volume_count > 0:
+ volume_id = server.volume_ids[0]['id']
+ volume = cinder_utils.get_volume_by_id(
+ cinder_cli, volume_id)
+ self.logger.debug("Volume id %s, size=%s" % (volume.id,
+ volume.size))
+ self._volume_size = volume.size
router_creators = self.heat_stack.get_router_creators()
router1 = router_creators[0]
@@ -265,15 +280,17 @@ class StorPerfMaster(object):
def create_stack(self):
self.stack_settings.resource_files = [
- 'storperf/resources/hot/storperf-agent.yaml']
+ 'storperf/resources/hot/storperf-agent.yaml',
+ 'storperf/resources/hot/storperf-volume.yaml']
self.stack_settings.env_values = self._make_parameters()
try:
self.heat_stack.create()
- except Exception:
+ except Exception as e:
+ self.logger.error("Stack creation failed")
+ self.logger.exception(e)
heat_cli = heat_utils.heat_client(self.os_creds)
res = heat_utils.get_resources(heat_cli,
self.heat_stack.get_stack().id)
- self.logger.error("Stack creation failed")
reason = ""
failed = False
for resource in res:
@@ -325,10 +342,12 @@ class StorPerfMaster(object):
thread.join()
self._test_executor.slaves = slaves
+ self._test_executor.volume_count = self.volume_count
params = metadata
params['agent_count'] = self.agent_count
params['public_network'] = self.public_network
+ params['volume_count'] = self.volume_count
params['volume_size'] = self.volume_size
if self.username and self.password:
params['username'] = self.username
@@ -446,6 +465,7 @@ class StorPerfMaster(object):
heat_parameters = {}
heat_parameters['public_network'] = self.public_network
heat_parameters['agent_count'] = self.agent_count
+ heat_parameters['volume_count'] = self.volume_count
heat_parameters['volume_size'] = self.volume_size
heat_parameters['agent_image'] = self.agent_image
heat_parameters['agent_flavor'] = self.agent_flavor
diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py
index 9ed6386..2ed6a9e 100644
--- a/docker/storperf-master/storperf/test_executor.py
+++ b/docker/storperf-master/storperf/test_executor.py
@@ -53,6 +53,7 @@ class TestExecutor(object):
self.job_db = JobDB()
self._slaves = []
self._terminated = False
+ self._volume_count = 1
self._workload_executors = []
self._workload_thread = None
self._thread_gate = None
@@ -88,6 +89,15 @@ class TestExecutor(object):
self._slaves = slaves
@property
+ def volume_count(self):
+ return self._volume_count
+
+ @volume_count.setter
+ def volume_count(self, volume_count):
+ self.logger.debug("Set volume count to: " + str(volume_count))
+ self._volume_count = volume_count
+
+ @property
def queue_depths(self):
return ','.join(self._queue_depths)
@@ -282,17 +292,24 @@ class TestExecutor(object):
slave_threads = []
for slave in self.slaves:
- slave_workload = copy.copy(current_workload['workload'])
- slave_workload.remote_host = slave
-
- self._workload_executors.append(slave_workload)
-
- t = Thread(target=self.execute_on_node,
- args=(slave_workload,),
- name="%s worker" % slave)
- t.daemon = False
- t.start()
- slave_threads.append(t)
+ volume_number = 0
+ while volume_number < self.volume_count:
+ slave_workload = copy.copy(current_workload['workload'])
+ slave_workload.remote_host = slave
+ last_char_of_filename = chr(ord(
+ slave_workload.filename[-1:]) + volume_number)
+ slave_workload.filename = "%s%s" % \
+ (slave_workload.filename[:-1], last_char_of_filename)
+ self.logger.debug("Device to profile: %s" %
+ slave_workload.filename)
+ self._workload_executors.append(slave_workload)
+ t = Thread(target=self.execute_on_node,
+ args=(slave_workload,),
+ name="%s worker" % slave)
+ t.daemon = False
+ t.start()
+ slave_threads.append(t)
+ volume_number += 1
for slave_thread in slave_threads:
self.logger.debug("Waiting on %s" % slave_thread)
@@ -335,9 +352,6 @@ class TestExecutor(object):
workload.filename = self.filename
workload.id = self.job_db.job_id
- if (self.filename is not None):
- workload.filename = self.filename
-
if (workload_name.startswith("_")):
iodepths = [8, ]
blocksizes = [16384, ]
diff --git a/docker/storperf-master/storperf/workloads/_base_workload.py b/docker/storperf-master/storperf/workloads/_base_workload.py
index d5282d7..c2c7b7b 100644
--- a/docker/storperf-master/storperf/workloads/_base_workload.py
+++ b/docker/storperf-master/storperf/workloads/_base_workload.py
@@ -74,9 +74,11 @@ class _base_workload(object):
@property
def fullname(self):
+ host_file = self.remote_host+"."+self.filename
+ host_file = host_file.replace(".", "-").replace("/", "-")
return ("%s.%s.queue-depth.%s.block-size.%s.%s"
% (str(self.id),
self.__class__.__name__,
str(self.options['iodepth']),
str(self.options['bs']),
- str(self.remote_host).replace(".", "-")))
+ host_file))
diff --git a/docker/storperf-master/tests/workload_tests/workload_subclass_test.py b/docker/storperf-master/tests/workload_tests/workload_subclass_test.py
index e9e47f3..c61fe74 100644
--- a/docker/storperf-master/tests/workload_tests/workload_subclass_test.py
+++ b/docker/storperf-master/tests/workload_tests/workload_subclass_test.py
@@ -22,33 +22,35 @@ class WorkloadSubclassTest(unittest.TestCase):
def test_local_name(self):
workload = rr()
self.assertEqual(workload.fullname,
- "None.rr.queue-depth.1.block-size.64k.None",
+ "None.rr.queue-depth.1.block-size.64k.None--dev-vdb",
workload.fullname)
def test_remote_name(self):
workload = rw()
workload.remote_host = "192.168.0.1"
- self.assertEqual(workload.fullname,
- "None.rw.queue-depth.1.block-size.64k.192-168-0-1",
- workload.fullname)
+ self.assertEqual(
+ workload.fullname,
+ "None.rw.queue-depth.1.block-size.64k.192-168-0-1--dev-vdb",
+ workload.fullname)
def test_blocksize(self):
workload = rs()
workload.options["bs"] = "4k"
self.assertEqual(workload.fullname,
- "None.rs.queue-depth.1.block-size.4k.None",
+ "None.rs.queue-depth.1.block-size.4k.None--dev-vdb",
workload.fullname)
def test_queue_depth(self):
workload = wr()
workload.options["iodepth"] = "8"
self.assertEqual(workload.fullname,
- "None.wr.queue-depth.8.block-size.64k.None",
+ "None.wr.queue-depth.8.block-size.64k.None--dev-vdb",
workload.fullname)
def test_id(self):
workload = ws()
workload.id = "workloadid"
- self.assertEqual(workload.fullname,
- "workloadid.ws.queue-depth.1.block-size.64k.None",
- workload.fullname)
+ self.assertEqual(
+ workload.fullname,
+ "workloadid.ws.queue-depth.1.block-size.64k.None--dev-vdb",
+ workload.fullname)