Adds Volume Count and Dynamic Reload

Adds a volume_count parameter that allows the user to
change the number of volumes.  Test execution still
uses each volume specified and appends it to the host
ip address in the graphite metrics.

Changes the cache policy of the stack from SNAPS-OO
so that changes made to the stack are refelcted in
StorPerf.

Fixes the documentation path and updates the docs
for the above changes.

Change-Id: I5b9160b469e80d4f7d6906ac270bb3b75411fb5a
JIRA: STORPERF-242
Signed-off-by: mbeierl <mark.beierl@dell.com>

8 files changed, 134 insertions, 42 deletions

diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py
index 7f38ab6..1b66af3 100644
--- a/docker/storperf-master/rest_server.py
+++ b/docker/storperf-master/rest_server.py
@@ -98,6 +98,7 @@ class ConfigurationRequestModel:
         'agent_flavor': fields.String,
         'agent_image': fields.String,
         'public_network': fields.String,
+        'volume_count': fields.Integer,
         'volume_size': fields.Integer,
         'availability_zone': fields.String,
         'username': fields.String,
@@ -114,6 +115,7 @@ class ConfigurationResponseModel:
         'public_network': fields.String,
         'stack_created': fields.Boolean,
         'stack_id': fields.String,
+        'volume_count': fields.Integer,
         'volume_size': fields.Integer,
         'availability_zone': fields.String
     }
@@ -135,6 +137,7 @@ class Configure(Resource):
                         'agent_flavor': storperf.agent_flavor,
                         'agent_image': storperf.agent_image,
                         'public_network': storperf.public_network,
+                        'volume_count': storperf.volume_count,
                         'volume_size': storperf.volume_size,
                         'stack_created': storperf.is_stack_created,
                         'availability_zone': storperf.availability_zone,
@@ -170,6 +173,8 @@ class Configure(Resource):
                 storperf.agent_image = request.json['agent_image']
             if ('public_network' in request.json):
                 storperf.public_network = request.json['public_network']
+            if ('volume_count' in request.json):
+                storperf.volume_count = request.json['volume_count']
             if ('volume_size' in request.json):
                 storperf.volume_size = request.json['volume_size']
             if ('availability_zone' in request.json):
@@ -188,6 +193,7 @@ class Configure(Resource):
                             'agent_image': storperf.agent_image,
                             'availability_zone': storperf.availability_zone,
                             'public_network': storperf.public_network,
+                            'volume_count': storperf.volume_count,
                             'volume_size': storperf.volume_size,
                             'stack_id': storperf.stack_id})
 
diff --git a/docker/storperf-master/storperf/resources/hot/agent-group.yaml b/docker/storperf-master/storperf/resources/hot/agent-group.yaml
index 3c02e31..ea7b51f 100644
--- a/docker/storperf-master/storperf/resources/hot/agent-group.yaml
+++ b/docker/storperf-master/storperf/resources/hot/agent-group.yaml
@@ -27,6 +27,12 @@ parameters:
     constraints:
       - range: { min: 1, max: 1024 }
         description: must be between 1 and 1024 Gb.
+  volume_count:
+    type: number
+    default: 0
+    constraints:
+      - range: { min: 0, max: 512 }
+        description: must be between 1 and 512 agents.
   agent_count:
     type: number
     default: 1
@@ -54,6 +60,7 @@ resources:
           availability_zone: {get_param: availability_zone},
           storperf_open_security_group: {get_resource: storperf_open_security_group},
           key_name: {get_resource: storperf_key_pair},
+          volume_count: {get_param: volume_count},
           volume_size: {get_param: volume_size}
         }
       }
diff --git a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml
index 7841e8c..8895c9f 100644
--- a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml
+++ b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml
@@ -24,6 +24,13 @@ parameters:
     default: storperf
   storperf_open_security_group:
     type: string
+  volume_count:
+    type: number
+    description: Number of volumes to be created
+    default: 1
+    constraints:
+      - range: { min: 0, max: 1024 }
+        description: must be between 1 and 1024.
   volume_size:
     type: number
     description: Size of the volume to be created.
@@ -87,15 +94,16 @@ resources:
       port_id: { get_resource: storperf_agent_port }
 
   agent_volume:
-    type: OS::Cinder::Volume
-    properties:
-      size: { get_param: volume_size }
-
-  agent_volume_att:
-    type: OS::Cinder::VolumeAttachment
+    type: OS::Heat::ResourceGroup
     properties:
-      instance_uuid: { get_resource: storperf_agent }
-      volume_id: { get_resource: agent_volume}
+      count: { get_param: volume_count }
+      resource_def: {
+        type: "storperf-volume.yaml",
+        properties: {
+          volume_size: { get_param: volume_size },
+          agent_instance_uuid: { get_resource: storperf_agent }
+        }
+      }
 
 outputs:
   storperf_agent_ip:
diff --git a/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml
new file mode 100644
index 0000000..aec3393
--- /dev/null
+++ b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml
@@ -0,0 +1,33 @@
+##############################################################################
+# Copyright (c) 2018 Dell EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+heat_template_version: 2013-05-23
+
+parameters:
+  volume_size:
+    type: number
+    description: Size of the volume to be created.
+    default: 1
+    constraints:
+      - range: { min: 1, max: 1024 }
+        description: must be between 1 and 1024 Gb.
+  agent_instance_uuid:
+    type: string
+
+resources:
+  agent_volume:
+    type: OS::Cinder::Volume
+    properties:
+      size: { get_param: volume_size }
+
+  agent_volume_att:
+    type: OS::Cinder::VolumeAttachment
+    properties:
+      instance_uuid: { get_param: agent_instance_uuid }
+      volume_id: { get_resource: agent_volume}
diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py
index 7244b66..d54fea3 100644
--- a/docker/storperf-master/storperf/storperf_master.py
+++ b/docker/storperf-master/storperf/storperf_master.py
@@ -64,11 +64,24 @@ class StorPerfMaster(object):
         self._agent_flavor = "storperf"
         self._availability_zone = None
         self._public_network = None
+        self._volume_count = 1
         self._volume_size = 1
         self._cached_stack_id = None
         self._last_snaps_check_time = None
 
     @property
+    def volume_count(self):
+        self._get_stack_info()
+        return self._volume_count
+
+    @volume_count.setter
+    def volume_count(self, value):
+        if (self.stack_id is not None):
+            raise ParameterError(
+                "ERROR: Cannot change volume count after stack is created")
+        self._volume_count = value
+
+    @property
     def volume_size(self):
         self._get_stack_info()
         return self._volume_size
@@ -136,14 +149,12 @@ class StorPerfMaster(object):
     def _get_stack_info(self):
         if self._last_snaps_check_time is not None:
             time_since_check = datetime.now() - self._last_snaps_check_time
-            if time_since_check.total_seconds() < 30:
+            if time_since_check.total_seconds() < 60:
                 return self._cached_stack_id
 
         self.heat_stack.initialize()
         if self.heat_stack.get_stack() is not None:
             self._last_snaps_check_time = datetime.now()
-            if self._cached_stack_id == self.heat_stack.get_stack().id:
-                return self._cached_stack_id
             self._cached_stack_id = self.heat_stack.get_stack().id
             cinder_cli = cinder_utils.cinder_client(self.os_creds)
             glance_cli = glance_utils.glance_client(self.os_creds)
@@ -162,10 +173,14 @@ class StorPerfMaster(object):
             image = glance_utils.get_image_by_id(glance_cli, image_id)
             self._agent_image = image.name
 
-            volume_id = server.volume_ids[0]['id']
-            volume = cinder_utils.get_volume_by_id(
-                cinder_cli, volume_id)
-            self._volume_size = volume.size
+            self._volume_count = len(server.volume_ids)
+            if self._volume_count > 0:
+                volume_id = server.volume_ids[0]['id']
+                volume = cinder_utils.get_volume_by_id(
+                    cinder_cli, volume_id)
+                self.logger.debug("Volume id %s, size=%s" % (volume.id,
+                                                             volume.size))
+                self._volume_size = volume.size
             router_creators = self.heat_stack.get_router_creators()
             router1 = router_creators[0]
 
@@ -265,15 +280,17 @@ class StorPerfMaster(object):
 
     def create_stack(self):
         self.stack_settings.resource_files = [
-            'storperf/resources/hot/storperf-agent.yaml']
+            'storperf/resources/hot/storperf-agent.yaml',
+            'storperf/resources/hot/storperf-volume.yaml']
         self.stack_settings.env_values = self._make_parameters()
         try:
             self.heat_stack.create()
-        except Exception:
+        except Exception as e:
+            self.logger.error("Stack creation failed")
+            self.logger.exception(e)
             heat_cli = heat_utils.heat_client(self.os_creds)
             res = heat_utils.get_resources(heat_cli,
                                            self.heat_stack.get_stack().id)
-            self.logger.error("Stack creation failed")
             reason = ""
             failed = False
             for resource in res:
@@ -325,10 +342,12 @@ class StorPerfMaster(object):
             thread.join()
 
         self._test_executor.slaves = slaves
+        self._test_executor.volume_count = self.volume_count
 
         params = metadata
         params['agent_count'] = self.agent_count
         params['public_network'] = self.public_network
+        params['volume_count'] = self.volume_count
         params['volume_size'] = self.volume_size
         if self.username and self.password:
             params['username'] = self.username
@@ -446,6 +465,7 @@ class StorPerfMaster(object):
         heat_parameters = {}
         heat_parameters['public_network'] = self.public_network
         heat_parameters['agent_count'] = self.agent_count
+        heat_parameters['volume_count'] = self.volume_count
         heat_parameters['volume_size'] = self.volume_size
         heat_parameters['agent_image'] = self.agent_image
         heat_parameters['agent_flavor'] = self.agent_flavor
diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py
index 9ed6386..2ed6a9e 100644
--- a/docker/storperf-master/storperf/test_executor.py
+++ b/docker/storperf-master/storperf/test_executor.py
@@ -53,6 +53,7 @@ class TestExecutor(object):
         self.job_db = JobDB()
         self._slaves = []
         self._terminated = False
+        self._volume_count = 1
         self._workload_executors = []
         self._workload_thread = None
         self._thread_gate = None
@@ -88,6 +89,15 @@ class TestExecutor(object):
         self._slaves = slaves
 
     @property
+    def volume_count(self):
+        return self._volume_count
+
+    @volume_count.setter
+    def volume_count(self, volume_count):
+        self.logger.debug("Set volume count to: " + str(volume_count))
+        self._volume_count = volume_count
+
+    @property
     def queue_depths(self):
         return ','.join(self._queue_depths)
 
@@ -282,17 +292,24 @@ class TestExecutor(object):
 
             slave_threads = []
             for slave in self.slaves:
-                slave_workload = copy.copy(current_workload['workload'])
-                slave_workload.remote_host = slave
-
-                self._workload_executors.append(slave_workload)
-
-                t = Thread(target=self.execute_on_node,
-                           args=(slave_workload,),
-                           name="%s worker" % slave)
-                t.daemon = False
-                t.start()
-                slave_threads.append(t)
+                volume_number = 0
+                while volume_number < self.volume_count:
+                    slave_workload = copy.copy(current_workload['workload'])
+                    slave_workload.remote_host = slave
+                    last_char_of_filename = chr(ord(
+                        slave_workload.filename[-1:]) + volume_number)
+                    slave_workload.filename = "%s%s" % \
+                        (slave_workload.filename[:-1], last_char_of_filename)
+                    self.logger.debug("Device to profile: %s" %
+                                      slave_workload.filename)
+                    self._workload_executors.append(slave_workload)
+                    t = Thread(target=self.execute_on_node,
+                               args=(slave_workload,),
+                               name="%s worker" % slave)
+                    t.daemon = False
+                    t.start()
+                    slave_threads.append(t)
+                    volume_number += 1
 
             for slave_thread in slave_threads:
                 self.logger.debug("Waiting on %s" % slave_thread)
@@ -335,9 +352,6 @@ class TestExecutor(object):
                 workload.filename = self.filename
             workload.id = self.job_db.job_id
 
-            if (self.filename is not None):
-                workload.filename = self.filename
-
             if (workload_name.startswith("_")):
                 iodepths = [8, ]
                 blocksizes = [16384, ]
diff --git a/docker/storperf-master/storperf/workloads/_base_workload.py b/docker/storperf-master/storperf/workloads/_base_workload.py
index d5282d7..c2c7b7b 100644
--- a/docker/storperf-master/storperf/workloads/_base_workload.py
+++ b/docker/storperf-master/storperf/workloads/_base_workload.py
@@ -74,9 +74,11 @@ class _base_workload(object):
 
     @property
     def fullname(self):
+        host_file = self.remote_host+"."+self.filename
+        host_file = host_file.replace(".", "-").replace("/", "-")
         return ("%s.%s.queue-depth.%s.block-size.%s.%s"
                 % (str(self.id),
                    self.__class__.__name__,
                    str(self.options['iodepth']),
                    str(self.options['bs']),
-                   str(self.remote_host).replace(".", "-")))
+                   host_file))
diff --git a/docker/storperf-master/tests/workload_tests/workload_subclass_test.py b/docker/storperf-master/tests/workload_tests/workload_subclass_test.py
index e9e47f3..c61fe74 100644
--- a/docker/storperf-master/tests/workload_tests/workload_subclass_test.py
+++ b/docker/storperf-master/tests/workload_tests/workload_subclass_test.py
@@ -22,33 +22,35 @@ class WorkloadSubclassTest(unittest.TestCase):
     def test_local_name(self):
         workload = rr()
         self.assertEqual(workload.fullname,
-                         "None.rr.queue-depth.1.block-size.64k.None",
+                         "None.rr.queue-depth.1.block-size.64k.None--dev-vdb",
                          workload.fullname)
 
     def test_remote_name(self):
         workload = rw()
         workload.remote_host = "192.168.0.1"
-        self.assertEqual(workload.fullname,
-                         "None.rw.queue-depth.1.block-size.64k.192-168-0-1",
-                         workload.fullname)
+        self.assertEqual(
+            workload.fullname,
+            "None.rw.queue-depth.1.block-size.64k.192-168-0-1--dev-vdb",
+            workload.fullname)
 
     def test_blocksize(self):
         workload = rs()
         workload.options["bs"] = "4k"
         self.assertEqual(workload.fullname,
-                         "None.rs.queue-depth.1.block-size.4k.None",
+                         "None.rs.queue-depth.1.block-size.4k.None--dev-vdb",
                          workload.fullname)
 
     def test_queue_depth(self):
         workload = wr()
         workload.options["iodepth"] = "8"
         self.assertEqual(workload.fullname,
-                         "None.wr.queue-depth.8.block-size.64k.None",
+                         "None.wr.queue-depth.8.block-size.64k.None--dev-vdb",
                          workload.fullname)
 
     def test_id(self):
         workload = ws()
         workload.id = "workloadid"
-        self.assertEqual(workload.fullname,
-                         "workloadid.ws.queue-depth.1.block-size.64k.None",
-                         workload.fullname)
+        self.assertEqual(
+            workload.fullname,
+            "workloadid.ws.queue-depth.1.block-size.64k.None--dev-vdb",
+            workload.fullname)