diff options
Diffstat (limited to 'docker/storperf-master')
4 files changed, 117 insertions, 21 deletions
diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py index 1e61693..020517a 100644 --- a/docker/storperf-master/rest_server.py +++ b/docker/storperf-master/rest_server.py @@ -104,6 +104,7 @@ class ConfigurationRequestModel: 'volume_type': fields.String, 'availability_zone': fields.String, 'subnet_CIDR': fields.String, + 'stack_name': fields.String, 'username': fields.String, 'password': fields.String } @@ -123,6 +124,7 @@ class ConfigurationResponseModel: 'volume_type': fields.String, 'availability_zone': fields.String, 'subnet_CIDR': fields.String, + 'stack_name': fields.String, 'slave_addresses': fields.Nested } @@ -136,9 +138,23 @@ class Configure(Resource): @swagger.operation( notes='Fetch the current agent configuration', + parameters=[ + { + "name": "stack_name", + "description": "The name of the stack to use, defaults to" + + "StorPerfAgentGroup or the last stack named", + "required": False, + "type": "string", + "allowMultiple": False, + "paramType": "query" + }], type=ConfigurationResponseModel.__name__ ) def get(self): + stack_name = request.args.get('stack_name') + if stack_name: + storperf.stack_name = stack_name + return jsonify({'agent_count': storperf.agent_count, 'agent_flavor': storperf.agent_flavor, 'agent_image': storperf.agent_image, @@ -149,6 +165,7 @@ class Configure(Resource): 'stack_created': storperf.is_stack_created, 'availability_zone': storperf.availability_zone, 'subnet_CIDR': storperf.subnet_CIDR, + 'stack_name': storperf.stack_name, 'slave_addresses': storperf.slave_addresses, 'stack_id': storperf.stack_id}) @@ -174,6 +191,10 @@ class Configure(Resource): abort(400, "ERROR: No data specified") try: + # Note this must be first in order to be able to create + # more than one stack in the same StorPerf instance. + if ('stack_name' in request.json): + storperf.stack_name = request.json['stack_name'] if ('agent_count' in request.json): storperf.agent_count = request.json['agent_count'] if ('agent_flavor' in request.json): @@ -208,9 +229,22 @@ class Configure(Resource): abort(400, str(e)) @swagger.operation( - notes='Deletes the agent configuration and the stack' + notes='Deletes the agent configuration and the stack', + parameters=[ + { + "name": "stack_name", + "description": "The name of the stack to delete, defaults to" + + "StorPerfAgentGroup or the last stack named", + "required": False, + "type": "string", + "allowMultiple": False, + "paramType": "query" + }] ) def delete(self): + stack_name = request.args.get('stack_name') + if stack_name: + storperf.stack_name = stack_name try: return jsonify({'stack_id': storperf.delete_stack()}) except Exception as e: @@ -226,7 +260,8 @@ class WorkloadModel: "steady_state_samples": fields.Integer, 'workload': fields.String, 'queue_depths': fields.String, - 'block_sizes': fields.String + 'block_sizes': fields.String, + 'stack_name': fields.String } @@ -319,6 +354,9 @@ following parameters: for any single test iteration. "workload":if specified, the workload to run. Defaults to all. + +"stack_name": The target stack to use. Defaults to StorPerfAgentGroup, or +the last stack named. """, "required": True, "type": "WorkloadModel", @@ -344,6 +382,8 @@ for any single test iteration. self.logger.info(request.json) try: + if ('stack_name' in request.json): + storperf.stack_name = request.json['stack_name'] if ('target' in request.json): storperf.filename = request.json['target'] if ('deadline' in request.json): @@ -390,15 +430,36 @@ for any single test iteration. @swagger.model +class WorkloadsBodyModel: + resource_fields = { + "rw": fields.String(default="randrw") + } + required = ['rw'] + + +@swagger.model +@swagger.nested( + name=WorkloadsBodyModel.__name__) +class WorkloadsNameModel: + resource_fields = { + "name": fields.Nested(WorkloadsBodyModel.resource_fields) + } + + +@swagger.model +@swagger.nested( + workloads=WorkloadsNameModel.__name__) class WorkloadV2Model: resource_fields = { 'target': fields.String, 'deadline': fields.Integer, "steady_state_samples": fields.Integer, - 'workloads': fields.Nested, + 'workloads': fields.Nested(WorkloadsNameModel.resource_fields), 'queue_depths': fields.String, - 'block_sizes': fields.String + 'block_sizes': fields.String, + 'stack_name': fields.String } + required = ['workloads'] class Job_v2(Resource): @@ -420,7 +481,10 @@ following parameters: "deadline": if specified, the maximum duration in minutes for any single test iteration. -"workloads":if specified, the workload to run. Defaults to all. +"workloads": A JSON formatted map of workload names and parameters for FIO. + +"stack_name": The target stack to use. Defaults to StorPerfAgentGroup, or +the last stack named. """, "required": True, "type": "WorkloadV2Model", @@ -446,6 +510,8 @@ for any single test iteration. self.logger.info(request.json) try: + if ('stack_name' in request.json): + storperf.stack_name = request.json['stack_name'] if ('target' in request.json): storperf.filename = request.json['target'] if ('deadline' in request.json): @@ -478,6 +544,7 @@ for any single test iteration. @swagger.model class WarmUpModel: resource_fields = { + 'stack_name': fields.String, 'target': fields.String } @@ -496,7 +563,10 @@ class Initialize(Resource): "description": """Fill the target with random data. If no target is specified, it will default to /dev/vdb -"target": The target device or file to fill with random data", +"target": The target device or file to fill with random data. + +"stack_name": The target stack to use. Defaults to StorPerfAgentGroup, or +the last stack named. """, "required": False, "type": "WarmUpModel", @@ -523,10 +593,13 @@ target is specified, it will default to /dev/vdb self.logger.info(request.json) try: - if (request.json and 'target' in request.json): - storperf.filename = request.json['target'] + if request.json: + if 'target' in request.json: + storperf.filename = request.json['target'] + if 'stack_name' in request.json: + storperf.stack_name = request.json['stack_name'] storperf.queue_depths = "8" - storperf.block_sizes = "8192" + storperf.block_sizes = "16k" storperf.workloads = "_warm_up" storperf.custom_workloads = None job_id = storperf.execute_workloads() diff --git a/docker/storperf-master/storperf/resources/hot/agent-group.yaml b/docker/storperf-master/storperf/resources/hot/agent-group.yaml index b4e3b6f..c82ae17 100644 --- a/docker/storperf-master/storperf/resources/hot/agent-group.yaml +++ b/docker/storperf-master/storperf/resources/hot/agent-group.yaml @@ -48,6 +48,9 @@ parameters: subnet_CIDR: type: string default: '172.16.0.0/16' + keypair_name: + type: string + default: storperf_agent_keypair resources: slaves: @@ -74,8 +77,6 @@ resources: storperf_network: type: OS::Neutron::Net - properties: - name: storperf-network storperf_subnet: type: OS::Neutron::Subnet @@ -99,7 +100,7 @@ resources: type: OS::Nova::KeyPair properties: save_private_key: true - name: storperf_agent_keypair + name: { get_param: keypair_name} storperf_open_security_group: type: OS::Neutron::SecurityGroup diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py index a50dcc6..0c7e559 100644 --- a/docker/storperf-master/storperf/storperf_master.py +++ b/docker/storperf-master/storperf/storperf_master.py @@ -25,6 +25,7 @@ from snaps.thread_utils import worker_pool from storperf.db.job_db import JobDB from storperf.test_executor import TestExecutor import json +import uuid class ParameterError(Exception): @@ -37,9 +38,9 @@ class StorPerfMaster(object): self.logger = logging.getLogger(__name__) self.job_db = JobDB() - + self._stack_name = 'StorPerfAgentGroup' self.stack_settings = StackConfig( - name='StorPerfAgentGroup', + name=self.stack_name, template_path='storperf/resources/hot/agent-group.yaml') self.os_creds = OSCreds( @@ -120,6 +121,17 @@ class StorPerfMaster(object): self._volume_type = value @property + def stack_name(self): + return self._stack_name + + @stack_name.setter + def stack_name(self, value): + self._stack_name = value + self.stack_settings.name = self.stack_name + self.stack_id = None + self._last_snaps_check_time = None + + @property def subnet_CIDR(self): return self._subnet_CIDR @@ -411,7 +423,8 @@ class StorPerfMaster(object): self._test_executor.job_id)) if (self.stack_id is None): - raise ParameterError("ERROR: Stack does not exist") + raise ParameterError("ERROR: Stack %s does not exist" % + self.stack_name) self._test_executor = TestExecutor() self._test_executor.register(self.executor_event) @@ -437,16 +450,22 @@ class StorPerfMaster(object): self._test_executor.slaves = slaves self._test_executor.volume_count = self.volume_count - params = metadata params['agent_count'] = self.agent_count + params['agent_flavor'] = self.agent_flavor + params['agent_image'] = self.agent_image + params['agent_info'] = json.dumps(self.slave_info) + params['avaiability_zone'] = self.availability_zone + params['block_sizes'] = self.block_sizes + params['deadline'] = self.deadline params['public_network'] = self.public_network + params['stack_name'] = self.stack_name + params['steady_state_samples'] = self.steady_state_samples + params['subnet_CIDR'] = self.subnet_CIDR + params['target'] = self.filename params['volume_count'] = self.volume_count params['volume_size'] = self.volume_size - params['subnet_CIDR'] = self.subnet_CIDR - params['agent_info'] = json.dumps(self.slave_info) - if self.volume_type is not None: - params['volume_type'] = self.volume_type + params['volume_type'] = self.volume_type if self.username and self.password: params['username'] = self.username params['password'] = self.password @@ -586,11 +605,13 @@ class StorPerfMaster(object): logger.error(line) def _make_parameters(self): + random_str = uuid.uuid4().hex[:6].upper() heat_parameters = {} heat_parameters['public_network'] = self.public_network heat_parameters['agent_count'] = self.agent_count heat_parameters['volume_count'] = self.volume_count heat_parameters['volume_size'] = self.volume_size + heat_parameters['keypair_name'] = 'storperf_agent_keypair' + random_str heat_parameters['subnet_CIDR'] = self.subnet_CIDR if self.volume_type is not None: heat_parameters['volume_type'] = self.volume_type diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py index 0ab5698..9e1736d 100644 --- a/docker/storperf-master/storperf/test_executor.py +++ b/docker/storperf-master/storperf/test_executor.py @@ -222,8 +222,8 @@ class TestExecutor(object): except Exception as e: self.terminate() raise e - self.job_db.record_workload_params(metadata) self._setup_metadata(metadata) + self.job_db.record_workload_params(metadata) self._workload_thread = Thread(target=self.execute_workloads, args=(), name="Workload thread") @@ -263,6 +263,7 @@ class TestExecutor(object): def _execute_workload(self, current_workload, workload, parse_only=False): workload.options['iodepth'] = str(current_workload['queue-depth']) workload.options['bs'] = str(current_workload['blocksize']) + self._workload_executors = [] slave_threads = [] thread_pool = ThreadPool(processes=len(self.slaves) * self.volume_count) |