diff options
Diffstat (limited to 'docker')
28 files changed, 525 insertions, 149 deletions
diff --git a/docker/local-docker-compose.yaml b/docker/local-docker-compose.yaml index 6daa6e2..a4b69b4 100644 --- a/docker/local-docker-compose.yaml +++ b/docker/local-docker-compose.yaml @@ -17,8 +17,10 @@ services: args: ARCH: ${ARCH} env_file: ${ENV_FILE} + user: ${CURRENT_UID} volumes: - ./storperf-master/:/storperf + - ./certs:/etc/ssl/certs/ links: - storperf-graphite @@ -28,6 +30,7 @@ services: context: storperf-reporting args: ARCH: ${ARCH} + user: ${CURRENT_UID} volumes: - ./storperf-reporting/:/home/opnfv/storperf-reporting diff --git a/docker/storperf-httpfrontend/Dockerfile b/docker/storperf-httpfrontend/Dockerfile index 95188b5..6f072b0 100644 --- a/docker/storperf-httpfrontend/Dockerfile +++ b/docker/storperf-httpfrontend/Dockerfile @@ -13,7 +13,7 @@ ## ARG ARCH=x86_64 -ARG ALPINE_VERSION=v3.6 +ARG ALPINE_VERSION=v3.10 FROM nginx:alpine EXPOSE 80 443 diff --git a/docker/storperf-master/Dockerfile b/docker/storperf-master/Dockerfile index 9764a8d..a2e1a1d 100644 --- a/docker/storperf-master/Dockerfile +++ b/docker/storperf-master/Dockerfile @@ -16,12 +16,12 @@ # ARG ARCH=x86_64 -ARG ALPINE_VERSION=v3.6 +ARG ALPINE_VERSION=v3.10 FROM multiarch/alpine:$ARCH-$ALPINE_VERSION as storperf-builder RUN ulimit -n 1024 -LABEL version="7.0" description="OPNFV Storperf Docker container" +LABEL version="8.0" description="OPNFV Storperf Docker container" ARG BRANCH=master @@ -47,28 +47,27 @@ RUN cd ${repos_dir}/fio && EXTFLAGS="-static" make -j $(grep -c ^processor /proc RUN apk --no-cache add --update \ libffi-dev \ libressl-dev \ - python \ - py-pip \ - python-dev \ + python3=3.7.5-r1 \ + python3-dev=3.7.5-r1 \ alpine-sdk \ - linux-headers \ - bash + linux-headers # Install StorPerf COPY requirements.pip /storperf/ -RUN pip install --upgrade setuptools==33.1.1 -RUN pip install -r /storperf/requirements.pip +RUN python3 -m pip install --upgrade setuptools==33.1.1 +RUN python3 -m pip install -r /storperf/requirements.pip # Build stripped down StorPerf image FROM multiarch/alpine:$ARCH-$ALPINE_VERSION as storperf-master RUN apk --no-cache add --update \ - python \ + libressl-dev \ + python3=3.7.5-r1 \ bash -COPY --from=storperf-builder /usr/lib/python2.7/site-packages /usr/lib/python2.7/site-packages +COPY --from=storperf-builder /usr/lib/python3.7/site-packages /usr/lib/python3.7/site-packages COPY --from=storperf-builder /usr/local/bin/fio /usr/local/bin/fio COPY . /storperf @@ -80,4 +79,4 @@ RUN chmod 600 storperf/resources/ssh/storperf_rsa EXPOSE 5000 # Entry point -CMD [ "python", "./rest_server.py" ] +CMD [ "python3", "./rest_server.py" ] diff --git a/docker/storperf-master/rest_server.py b/docker/storperf-master/rest_server.py index 92b6c85..7606eca 100644 --- a/docker/storperf-master/rest_server.py +++ b/docker/storperf-master/rest_server.py @@ -10,7 +10,6 @@ import json import logging.config import os -import sys from flask import abort, Flask, request, jsonify from flask_cors import CORS @@ -18,6 +17,7 @@ from flask_restful import Resource, Api, fields from flask_restful_swagger import swagger from storperf.storperf_master import StorPerfMaster +import flask class ReverseProxied(object): @@ -137,7 +137,9 @@ class Configure(Resource): self.logger = logging.getLogger(__name__) @swagger.operation( - notes='Fetch the current agent configuration', + notes='''Fetch the current agent configuration. + + This API is in sunset until the next OPNFV release.''', parameters=[ { "name": "stack_name", @@ -155,7 +157,7 @@ class Configure(Resource): if stack_name: storperf.stack_name = stack_name - return jsonify({'agent_count': storperf.agent_count, + json = jsonify({'agent_count': storperf.agent_count, 'agent_flavor': storperf.agent_flavor, 'agent_image': storperf.agent_image, 'public_network': storperf.public_network, @@ -168,10 +170,15 @@ class Configure(Resource): 'stack_name': storperf.stack_name, 'slave_addresses': storperf.slave_addresses, 'stack_id': storperf.stack_id}) + response = flask.make_response(json) + response.headers['Sunset'] = "Tue. 31 Mar 2020 23:59:59 GMT" + return response @swagger.operation( notes='''Set the current agent configuration and create a stack in - the controller. Returns once the stack create is completed.''', + the controller. Returns once the stack create is completed. + + This API is in sunset until the next OPNFV release.''', parameters=[ { "name": "configuration", @@ -229,7 +236,9 @@ class Configure(Resource): abort(400, str(e)) @swagger.operation( - notes='Deletes the agent configuration and the stack', + notes='''Deletes the agent configuration and the stack + + This API is in sunset until the next OPNFV release.''', parameters=[ { "name": "stack_name", @@ -246,7 +255,10 @@ class Configure(Resource): if stack_name: storperf.stack_name = stack_name try: - return jsonify({'stack_id': storperf.delete_stack()}) + json = jsonify({'stack_id': storperf.delete_stack()}) + response = flask.make_response(json) + response.headers['Sunset'] = "Tue. 31 Mar 2020 23:59:59 GMT" + return response except Exception as e: self.logger.exception(e) abort(400, str(e)) @@ -355,7 +367,8 @@ for any single test iteration. "workload":if specified, the workload to run. Defaults to all. -"stack_name": The target stack to use. Defaults to StorPerfAgentGroup, or +"stack_name": This field is in sunset until the next OPNVF release. +The target stack to use. Defaults to StorPerfAgentGroup, or the last stack named. """, "required": True, @@ -379,11 +392,13 @@ the last stack named. if not request.json: abort(400, "ERROR: Missing configuration data") + storperf.reset_values() self.logger.info(request.json) try: if ('stack_name' in request.json): storperf.stack_name = request.json['stack_name'] + storperf.stackless = False if ('target' in request.json): storperf.filename = request.json['target'] if ('deadline' in request.json): @@ -422,7 +437,6 @@ the last stack named. ] ) def delete(self): - self.logger.info("Threads: %s" % sys._current_frames()) try: return jsonify({'Slaves': storperf.terminate_workloads()}) except Exception as e: @@ -439,7 +453,7 @@ class WorkloadsBodyModel: @swagger.model @swagger.nested( - name=WorkloadsBodyModel.__name__) + name=WorkloadsBodyModel.__name__) class WorkloadsNameModel: resource_fields = { "name": fields.Nested(WorkloadsBodyModel.resource_fields) @@ -448,7 +462,7 @@ class WorkloadsNameModel: @swagger.model @swagger.nested( - workloads=WorkloadsNameModel.__name__) + workloads=WorkloadsNameModel.__name__) class WorkloadV2Model: resource_fields = { 'target': fields.String, @@ -457,7 +471,11 @@ class WorkloadV2Model: 'workloads': fields.Nested(WorkloadsNameModel.resource_fields), 'queue_depths': fields.String, 'block_sizes': fields.String, - 'stack_name': fields.String + 'stack_name': fields.String, + 'username': fields.String, + 'password': fields.String, + 'ssh_private_key': fields.String, + 'slave_addresses': fields.List } required = ['workloads'] @@ -483,8 +501,21 @@ for any single test iteration. "workloads": A JSON formatted map of workload names and parameters for FIO. -"stack_name": The target stack to use. Defaults to StorPerfAgentGroup, or -the last stack named. +"stack_name": This field is in sunset until the next OPNFV release. +The target stack to use. Defaults to StorPerfAgentGroup, or +the last stack named. Explicitly specifying null will bypass all Heat Stack +operations and go directly against the IP addresses specified. + +"username": if specified, the username to use when logging into the slave. + +"password": if specified, the password to use when logging into the slave. + +"ssh_private_key": if specified, the ssh private key to use when logging +into the slave. + +"slave_addresses": if specified, a list of IP addresses to use instead of +looking all of them up from the stack. + """, "required": True, "type": "WorkloadV2Model", @@ -505,9 +536,10 @@ the last stack named. ) def post(self): if not request.json: - abort(400, "ERROR: Missing configuration data") + abort(400, "ERROR: Missing job data") self.logger.info(request.json) + storperf.reset_values() try: if ('stack_name' in request.json): @@ -534,6 +566,15 @@ the last stack named. else: metadata = {} + if 'username' in request.json: + storperf.username = request.json['username'] + if 'password' in request.json: + storperf.password = request.json['password'] + if 'ssh_private_key' in request.json: + storperf.ssh_key = request.json['ssh_private_key'] + if 'slave_addresses' in request.json: + storperf.slave_addresses = request.json['slave_addresses'] + job_id = storperf.execute_workloads(metadata) return jsonify({'job_id': job_id}) @@ -547,7 +588,16 @@ the last stack named. class WarmUpModel: resource_fields = { 'stack_name': fields.String, - 'target': fields.String + 'target': fields.String, + 'username': fields.String, + 'password': fields.String, + 'ssh_private_key': fields.String, + 'slave_addresses': fields.List, + 'mkfs': fields.String, + 'mount_point': fields.String, + 'file_size': fields.String, + 'nrfiles': fields.String, + 'numjobs': fields.String, } @@ -565,10 +615,36 @@ class Initialize(Resource): "description": """Fill the target with random data. If no target is specified, it will default to /dev/vdb -"target": The target device or file to fill with random data. +"target": The target device to use. -"stack_name": The target stack to use. Defaults to StorPerfAgentGroup, or -the last stack named. +"stack_name": This field is in sunset until the next OPNFV release. +The target stack to use. Defaults to StorPerfAgentGroup, or +the last stack named. Explicitly specifying null will bypass all Heat Stack +operations and go directly against the IP addresses specified. + +"username": if specified, the username to use when logging into the slave. + +"password": if specified, the password to use when logging into the slave. + +"ssh_private_key": if specified, the ssh private key to use when logging +into the slave. + +"slave_addresses": if specified, a list of IP addresses to use instead of +looking all of them up from the stack. + +"mkfs": if specified, the command to execute in order to create a filesystem +on the target device (eg: mkfs.ext4) + +"mount_point": if specified, the directory to use when mounting the device. + +"filesize": if specified, the size of the files to create when profiling +a filesystem. + +"nrfiles": if specified, the number of files to create when profiling +a filesystem + +"numjobs": if specified, the number of jobs for when profiling +a filesystem """, "required": False, "type": "WarmUpModel", @@ -593,17 +669,46 @@ the last stack named. ) def post(self): self.logger.info(request.json) + storperf.reset_values() try: + warm_up_args = { + 'rw': 'randwrite', + 'direct': "1", + 'loops': "1" + } + storperf.queue_depths = "8" + storperf.block_sizes = "16k" + if request.json: if 'target' in request.json: storperf.filename = request.json['target'] if 'stack_name' in request.json: storperf.stack_name = request.json['stack_name'] - storperf.queue_depths = "8" - storperf.block_sizes = "16k" - storperf.workloads = "_warm_up" - storperf.custom_workloads = None + if 'username' in request.json: + storperf.username = request.json['username'] + if 'password' in request.json: + storperf.password = request.json['password'] + if 'ssh_private_key' in request.json: + storperf.ssh_key = request.json['ssh_private_key'] + if 'slave_addresses' in request.json: + storperf.slave_addresses = request.json['slave_addresses'] + if 'mkfs' in request.json: + storperf.mkfs = request.json['mkfs'] + if 'mount_device' in request.json: + storperf.mount_device = request.json['mount_device'] + if 'filesize' in request.json: + warm_up_args['filesize'] = str(request.json['filesize']) + if 'nrfiles' in request.json: + warm_up_args['nrfiles'] = str(request.json['nrfiles']) + if 'numjobs' in request.json: + warm_up_args['numjobs'] = str(request.json['numjobs']) + + storperf.workloads = None + storperf.custom_workloads = { + '_warm_up': warm_up_args + } + self.logger.info(storperf.custom_workloads) job_id = storperf.execute_workloads() return jsonify({'job_id': job_id}) @@ -628,12 +733,18 @@ class Quota(Resource): notes='''Fetch the current Cinder volume quota. This value limits the number of volumes that can be created, and by extension, defines the maximum number of agents that can be created for any given test - scenario''', + scenario + + + This API is in sunset until the next OPNFV release.''', type=QuotaModel.__name__ ) def get(self): - quota = storperf.volume_quota - return jsonify({'quota': quota}) + quota = [] # storperf.volume_quota + # return jsonify({'quota': quota}) + response = flask.make_response(jsonify({'quota': quota})) + response.headers['Sunset'] = "Tue. 31 Mar 2020 23:59:59 GMT" + return response def setup_logging(default_path='logging.json', diff --git a/docker/storperf-master/storperf/carbon/converter.py b/docker/storperf-master/storperf/carbon/converter.py index 623c144..4b5e6aa 100644 --- a/docker/storperf-master/storperf/carbon/converter.py +++ b/docker/storperf-master/storperf/carbon/converter.py @@ -32,12 +32,12 @@ class Converter(object): def resurse_to_flat_dictionary(self, json, prefix=None): if type(json) == dict: - for k, v in json.items(): + for k, v in list(json.items()): if prefix is None: - key = k.decode("utf-8").replace(" ", "_") + key = k.replace(" ", "_") else: - key = prefix + "." + k.decode("utf-8").replace(" ", "_") - if hasattr(v, '__iter__'): + key = prefix + "." + k.replace(" ", "_") + if type(v) is list or type(v) is dict: self.resurse_to_flat_dictionary(v, key) else: self.flat_dictionary[key] = str(v).replace(" ", "_") @@ -45,7 +45,7 @@ class Converter(object): index = 0 for v in json: index += 1 - if hasattr(v, '__iter__'): + if type(v) is list or type(v) is dict: self.resurse_to_flat_dictionary( v, prefix + "." + str(index)) else: diff --git a/docker/storperf-master/storperf/carbon/emitter.py b/docker/storperf-master/storperf/carbon/emitter.py index b196709..13503b2 100644 --- a/docker/storperf-master/storperf/carbon/emitter.py +++ b/docker/storperf-master/storperf/carbon/emitter.py @@ -40,19 +40,19 @@ class CarbonMetricTransmitter(): message = "%s %s %s\n" \ % (key, value, timestamp) self.logger.debug("Metric: " + message.strip()) - carbon_socket.send(message) + carbon_socket.send(message.encode('utf-8')) except ValueError: self.logger.debug("Ignoring non numeric metric %s %s" % (key, value)) message = "%s.commit-marker %s %s\n" \ % (commit_marker, timestamp, timestamp) - carbon_socket.send(message) + carbon_socket.send(message.encode('utf-8')) self.logger.debug("Marker %s" % message.strip()) self.logger.info("Sent metrics to %s:%s with timestamp %s" % (self.host, self.port, timestamp)) - except Exception, e: + except Exception as e: self.logger.error("While notifying carbon %s:%s %s" % (self.host, self.port, e)) diff --git a/docker/storperf-master/storperf/db/graphite_db.py b/docker/storperf-master/storperf/db/graphite_db.py index 8ebd22e..59b9f5d 100644 --- a/docker/storperf-master/storperf/db/graphite_db.py +++ b/docker/storperf-master/storperf/db/graphite_db.py @@ -41,7 +41,7 @@ class GraphiteDB(object): start = end - duration request = ("http://%s:%s/graphite/render/?target=" - "%s(%s.*.jobs.1.%s.%s)" + "%s(%s.*.jobs.*.%s.%s)" "&format=json" "&from=%s" "&until=%s" diff --git a/docker/storperf-master/storperf/db/job_db.py b/docker/storperf-master/storperf/db/job_db.py index b029a35..c3632e4 100644 --- a/docker/storperf-master/storperf/db/job_db.py +++ b/docker/storperf-master/storperf/db/job_db.py @@ -220,7 +220,7 @@ class JobDB(object): db = sqlite3.connect(JobDB.db_name) cursor = db.cursor() - for param, value in params.iteritems(): + for param, value in params.items(): cursor.execute( """insert into job_params (job_id, diff --git a/docker/storperf-master/storperf/fio/fio_invoker.py b/docker/storperf-master/storperf/fio/fio_invoker.py index a361eec..bb81eef 100644 --- a/docker/storperf-master/storperf/fio/fio_invoker.py +++ b/docker/storperf-master/storperf/fio/fio_invoker.py @@ -11,6 +11,7 @@ import json import logging from threading import Thread import paramiko +from storperf.utilities import ip_helper class FIOInvoker(object): @@ -45,6 +46,8 @@ class FIOInvoker(object): self.json_body = "" try: for line in iter(stdout.readline, b''): + if type(line) == bytes: + line = line.decode('utf=8') if line.startswith("fio"): line = "" continue @@ -78,7 +81,8 @@ class FIOInvoker(object): def stderr_handler(self, stderr): self.logger.debug("Started") for line in iter(stderr.readline, b''): - self.logger.error("FIO Error: %s", line.rstrip()) + if len(line) > 0: + self.logger.error("FIO Error: %s", line.rstrip()) self.stderr.append(line.rstrip()) # Sometime, FIO gets stuck and will give us this message: @@ -137,10 +141,12 @@ class FIOInvoker(object): ssh = self._ssh_client() - command = "sudo killall fio" - - self.logger.debug("Executing on %s: %s" % (self.remote_host, command)) - (_, stdout, stderr) = ssh.exec_command(command) + kill_commands = ['sudo killall fio', + 'sudo pkill fio'] + for command in kill_commands: + self.logger.debug("Executing on %s: %s" % + (self.remote_host, command)) + (_, stdout, stderr) = ssh.exec_command(command) for line in stdout.readlines(): self.logger.debug(line.strip()) @@ -153,13 +159,25 @@ class FIOInvoker(object): def _ssh_client(self): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + address, port = ip_helper.parse_address_and_port(self.remote_host) if 'username' in self.metadata and 'password' in self.metadata: - ssh.connect(self.remote_host, + ssh.connect(address, + port=port, + username=self.metadata['username'], + password=self.metadata['password'], + timeout=5) + return ssh + elif 'username' in self.metadata and 'ssh_key' in self.metadata: + ssh.connect(address, + port=port, username=self.metadata['username'], - password=self.metadata['password']) + pkey=self.metadata['ssh_key'], + timeout=5) return ssh else: - ssh.connect(self.remote_host, username='storperf', + ssh.connect(address, + port=port, + username='storperf', key_filename='storperf/resources/ssh/storperf_rsa', - timeout=2) + timeout=5) return ssh diff --git a/docker/storperf-master/storperf/resources/hot/agent-group.yaml b/docker/storperf-master/storperf/resources/hot/agent-group.yaml index c82ae17..f09d95a 100644 --- a/docker/storperf-master/storperf/resources/hot/agent-group.yaml +++ b/docker/storperf-master/storperf/resources/hot/agent-group.yaml @@ -7,7 +7,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -heat_template_version: 2017-09-01 +heat_template_version: newton parameters: public_network: diff --git a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml index 6314514..7a0a9e9 100644 --- a/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml +++ b/docker/storperf-master/storperf/resources/hot/storperf-agent.yaml @@ -7,7 +7,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -heat_template_version: 2017-09-01 +heat_template_version: newton parameters: flavor: diff --git a/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml index cbdd861..d64d0c2 100644 --- a/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml +++ b/docker/storperf-master/storperf/resources/hot/storperf-volume.yaml @@ -7,7 +7,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -heat_template_version: 2017-09-01 +heat_template_version: newton parameters: volume_size: diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py index 0c7e559..73f8f0d 100644 --- a/docker/storperf-master/storperf/storperf_master.py +++ b/docker/storperf-master/storperf/storperf_master.py @@ -7,16 +7,11 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from datetime import datetime -import logging -import os -import socket -from threading import Thread -from time import sleep -import paramiko +from datetime import datetime +from io import StringIO +from multiprocessing.pool import ThreadPool from scp import SCPClient - from snaps.config.stack import StackConfig from snaps.openstack.create_stack import OpenStackHeatStack from snaps.openstack.os_credentials import OSCreds @@ -24,7 +19,13 @@ from snaps.openstack.utils import heat_utils, cinder_utils, glance_utils from snaps.thread_utils import worker_pool from storperf.db.job_db import JobDB from storperf.test_executor import TestExecutor +from storperf.utilities import ip_helper +from time import sleep import json +import logging +import os +import paramiko +import socket import uuid @@ -37,8 +38,9 @@ class StorPerfMaster(object): def __init__(self): self.logger = logging.getLogger(__name__) + self.reset_values() + self.job_db = JobDB() - self._stack_name = 'StorPerfAgentGroup' self.stack_settings = StackConfig( name=self.stack_name, template_path='storperf/resources/hot/agent-group.yaml') @@ -59,21 +61,24 @@ class StorPerfMaster(object): self.heat_stack = OpenStackHeatStack(self.os_creds, self.stack_settings) + + self._snaps_pool = worker_pool(20) + + def reset_values(self): + self._stack_name = 'StorPerfAgentGroup' self.username = None self.password = None + self._ssh_key = None self._test_executor = None self._agent_count = 1 - self._agent_image = "Ubuntu 14.04" - self._agent_flavor = "storperf" + self._agent_image = None + self._agent_flavor = None self._availability_zone = None self._public_network = None self._volume_count = 1 self._volume_size = 1 self._volume_type = None - self._cached_stack_id = None - self._last_snaps_check_time = None self._slave_addresses = [] - self._thread_pool = worker_pool(20) self._filename = None self._deadline = None self._steady_state_samples = 10 @@ -83,6 +88,11 @@ class StorPerfMaster(object): self._custom_workloads = [] self._subnet_CIDR = '172.16.0.0/16' self.slave_info = {} + self.stackless = False + self.mkfs = None + self.mount_device = None + self._last_snaps_check_time = None + self._cached_stack_id = None @property def volume_count(self): @@ -126,10 +136,14 @@ class StorPerfMaster(object): @stack_name.setter def stack_name(self, value): - self._stack_name = value - self.stack_settings.name = self.stack_name - self.stack_id = None - self._last_snaps_check_time = None + if value is None: + self.stackless = True + else: + self.stackless = False + self._stack_name = value + self.stack_settings.name = self.stack_name + self.stack_id = None + self._last_snaps_check_time = None @property def subnet_CIDR(self): @@ -194,6 +208,10 @@ class StorPerfMaster(object): def slave_addresses(self): return self._slave_addresses + @slave_addresses.setter + def slave_addresses(self, value): + self._slave_addresses = value + @property def stack_id(self): self._get_stack_info() @@ -204,6 +222,10 @@ class StorPerfMaster(object): self._cached_stack_id = value def _get_stack_info(self): + if self.stackless: + self._cached_stack_id = None + return None + if self._last_snaps_check_time is not None: time_since_check = datetime.now() - self._last_snaps_check_time if time_since_check.total_seconds() < 60: @@ -216,7 +238,7 @@ class StorPerfMaster(object): cinder_cli = cinder_utils.cinder_client(self.os_creds) glance_cli = glance_utils.glance_client(self.os_creds) - router_worker = self._thread_pool.apply_async( + router_worker = self._snaps_pool.apply_async( self.heat_stack.get_router_creators) vm_inst_creators = self.heat_stack.get_vm_inst_creators() @@ -234,7 +256,7 @@ class StorPerfMaster(object): server = vm1.get_vm_inst() - image_worker = self._thread_pool.apply_async( + image_worker = self._snaps_pool.apply_async( glance_utils.get_image_by_id, (glance_cli, server.image_id)) self._volume_count = len(server.volume_ids) @@ -340,6 +362,19 @@ class StorPerfMaster(object): self._custom_workloads = value @property + def ssh_key(self): + if self._ssh_key is None: + return None + key = StringIO(self._ssh_key) + pkey = paramiko.RSAKey.from_private_key(key) + key.close() + return pkey + + @ssh_key.setter + def ssh_key(self, value): + self._ssh_key = value + + @property def is_stack_created(self): return (self.stack_id is not None and (self.heat_stack.get_status() == u'CREATE_COMPLETE' or @@ -363,6 +398,8 @@ class StorPerfMaster(object): return logs def create_stack(self): + self.stackless = False + self.stack_settings.resource_files = [ 'storperf/resources/hot/storperf-agent.yaml', 'storperf/resources/hot/storperf-volume.yaml'] @@ -422,7 +459,8 @@ class StorPerfMaster(object): raise Exception("ERROR: Job {} is already running".format( self._test_executor.job_id)) - if (self.stack_id is None): + if (not self.stackless and + self.stack_id is None): raise ParameterError("ERROR: Stack %s does not exist" % self.stack_name) @@ -438,20 +476,23 @@ class StorPerfMaster(object): slaves = self._slave_addresses - setup_threads = [] + setup_pool = ThreadPool(processes=len(slaves)) + workers = [] for slave in slaves: - t = Thread(target=self._setup_slave, args=(slave,)) - setup_threads.append(t) - t.start() + worker = setup_pool.apply_async( + self._setup_slave, (slave,)) + workers.append(worker) + + for worker in workers: + worker.get() - for thread in setup_threads: - thread.join() + setup_pool.close() self._test_executor.slaves = slaves self._test_executor.volume_count = self.volume_count params = metadata - params['agent_count'] = self.agent_count + params['agent_count'] = len(slaves) params['agent_flavor'] = self.agent_flavor params['agent_image'] = self.agent_image params['agent_info'] = json.dumps(self.slave_info) @@ -466,9 +507,12 @@ class StorPerfMaster(object): params['volume_count'] = self.volume_count params['volume_size'] = self.volume_size params['volume_type'] = self.volume_type - if self.username and self.password: + if self.username: params['username'] = self.username + if self.password: params['password'] = self.password + if self.ssh_key: + params['ssh_key'] = self.ssh_key job_id = self._test_executor.execute(params) self.slave_info = {} @@ -535,7 +579,8 @@ class StorPerfMaster(object): timer = 10 while not alive: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - result = s.connect_ex((slave, 22)) + host, port = ip_helper.parse_address_and_port(slave) + result = s.connect_ex((host, port)) s.close() if result: @@ -552,13 +597,26 @@ class StorPerfMaster(object): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) if self.username and self.password: - ssh.connect(slave, - username=self.username, - password=self.password) + ssh.connect( + host, + port=port, + username=self.username, + password=self.password, + timeout=2) + elif self.username and self.ssh_key: + ssh.connect( + host, + port=port, + username=self.username, + pkey=self.ssh_key, + timeout=2) else: - ssh.connect(slave, username='storperf', - key_filename='storperf/resources/ssh/storperf_rsa', - timeout=2) + ssh.connect( + slave, + port=port, + username='storperf', + key_filename='storperf/resources/ssh/storperf_rsa', + timeout=2) uname = self._get_uname(ssh) logger.debug("Slave uname is %s" % uname) @@ -582,6 +640,12 @@ class StorPerfMaster(object): logger.debug("Transferring fio to %s" % slave) scp.put('/usr/local/bin/fio', '~/') + if self.mkfs is not None: + self._mkfs(ssh, logger) + + if self.mount_device is not None: + self._mount(ssh, logger) + def _get_uname(self, ssh): (_, stdout, _) = ssh.exec_command("uname -a") return stdout.readline() @@ -594,6 +658,59 @@ class StorPerfMaster(object): available = lines[3] return int(available) + def _mkfs(self, ssh, logger): + command = "sudo umount %s" % (self.mount_device) + logger.info("Attempting %s" % command) + (_, stdout, stderr) = ssh.exec_command(command) + stdout.channel.recv_exit_status() + for line in iter(stdout.readline, b''): + logger.info(line) + for line in iter(stderr.readline, b''): + logger.error(line) + + command = "sudo mkfs.%s %s" % (self.mkfs, self.mount_device) + logger.info("Attempting %s" % command) + (_, stdout, stderr) = ssh.exec_command(command) + rc = stdout.channel.recv_exit_status() + stdout.channel.recv_exit_status() + for line in iter(stdout.readline, b''): + logger.info(line) + error_messages = "" + for line in iter(stderr.readline, b''): + logger.error(line) + error_messages += line.rstrip() + + if rc != 0: + raise Exception( + "Error executing on {0}: {1}".format( + command, error_messages)) + + def _mount(self, ssh, logger): + command = "sudo mkdir -p %s" % (self.filename) + logger.info("Attempting %s" % command) + (_, stdout, stderr) = ssh.exec_command(command) + stdout.channel.recv_exit_status() + for line in iter(stdout.readline, b''): + logger.info(line) + for line in iter(stderr.readline, b''): + logger.error(line) + + command = "sudo mount %s %s" % (self.mount_device, self.filename) + logger.info("Attempting %s" % command) + (_, stdout, stderr) = ssh.exec_command(command) + rc = stdout.channel.recv_exit_status() + for line in iter(stdout.readline, b''): + logger.info(line) + error_messages = "" + for line in iter(stderr.readline, b''): + logger.error(line) + error_messages += line.rstrip() + + if rc != 0: + raise Exception( + "Could not mount {0}: {1}".format( + self.mount_device, error_messages)) + def _resize_root_fs(self, ssh, logger): command = "sudo /usr/sbin/resize2fs /dev/vda1" logger.info("Attempting %s" % command) diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py index f7b577e..cb7e478 100644 --- a/docker/storperf-master/storperf/test_executor.py +++ b/docker/storperf-master/storperf/test_executor.py @@ -217,18 +217,19 @@ class TestExecutor(object): def execute(self, metadata): self.job_db.create_job_id() + self._setup_metadata(metadata) try: self.test_params() except Exception as e: self.terminate() raise e - self._setup_metadata(metadata) - self.job_db.record_workload_params(metadata) + stripped_metadata = metadata.copy() + stripped_metadata.pop('ssh_key', None) + self.job_db.record_workload_params(stripped_metadata) self._workload_thread = Thread(target=self.execute_workloads, args=(), name="Workload thread") self._workload_thread.start() - # seems to be hanging here return self.job_db.job_id def terminate(self): @@ -315,8 +316,9 @@ class TestExecutor(object): continue workload = current_workload['workload'] - self._thread_gate = ThreadGate(len(self.slaves), - workload.options['status-interval']) + self._thread_gate = ThreadGate( + len(self.slaves) * min(1, self.volume_count), + float(workload.options['status-interval'])) self.current_workload = current_workload['name'] @@ -360,20 +362,25 @@ class TestExecutor(object): workloads = [] if self._custom_workloads: - for workload_name in self._custom_workloads.iterkeys(): - if not workload_name.isalnum(): + for workload_name in self._custom_workloads.keys(): + real_name = workload_name + if real_name.startswith('_'): + real_name = real_name.replace('_', '') + self.logger.info("--- real_name: %s" % real_name) + + if not real_name.isalnum(): raise InvalidWorkloadName( "Workload name must be alphanumeric only: %s" % - workload_name) + real_name) workload = _custom_workload() - workload.options['name'] = workload_name + workload.options['name'] = real_name workload.name = workload_name if (self.filename is not None): workload.filename = self.filename workload.id = self.job_db.job_id workload_params = self._custom_workloads[workload_name] - for param, value in workload_params.iteritems(): + for param, value in workload_params.items(): if param == "readwrite": param = "rw" if param in workload.fixed_options: diff --git a/docker/storperf-master/storperf/utilities/data_handler.py b/docker/storperf-master/storperf/utilities/data_handler.py index 6e87781..98ae640 100644 --- a/docker/storperf-master/storperf/utilities/data_handler.py +++ b/docker/storperf-master/storperf/utilities/data_handler.py @@ -157,9 +157,11 @@ class DataHandler(object): test_db = os.environ.get('TEST_DB_URL') if test_db is not None: self.logger.info("Pushing results to %s" % (test_db)) + stripped_metadata = executor.metadata + stripped_metadata.pop("ssh_key", None) response = test_results_db.push_results_to_db( test_db, - executor.metadata, + stripped_metadata, self.logger) if response: self.logger.info("Results reference: %s" % response['href']) diff --git a/docker/storperf-master/storperf/utilities/ip_helper.py b/docker/storperf-master/storperf/utilities/ip_helper.py new file mode 100644 index 0000000..06087b0 --- /dev/null +++ b/docker/storperf-master/storperf/utilities/ip_helper.py @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2019 VMware and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +def parse_address_and_port(address): + port = 22 + if '.' in address: + # this is IPv4 + if ':' in address: + host = address.split(':')[0] + port = int(address.split(':')[1]) + else: + host = address + else: + if ']' in address: + # this is IPv6 + host = address.split(']')[0].split('[')[1] + port = int(address.split(']')[1].split(':')[1]) + else: + host = address + return (host, port) diff --git a/docker/storperf-master/storperf/workloads/_base_workload.py b/docker/storperf-master/storperf/workloads/_base_workload.py index 9b04314..5aa596e 100644 --- a/docker/storperf-master/storperf/workloads/_base_workload.py +++ b/docker/storperf-master/storperf/workloads/_base_workload.py @@ -44,17 +44,24 @@ class _base_workload(object): self.options['size'] = "100%" self.logger.debug( "Profiling a device, using 100% of " + self.filename) + self.options['filename'] = self.filename else: - self.options['size'] = self.default_filesize + if 'size' not in self.options: + self.options['size'] = self.default_filesize self.logger.debug("Profiling a filesystem, using " + - self.default_filesize + " file") - - self.options['filename'] = self.filename + self.options['size'] + " file") + if not self.filename.endswith('/'): + self.filename = self.filename + "/" + self.options['directory'] = self.filename + self.options['filename_format'] = "'storperf.$jobnum.$filenum'" self.setup() - for key, value in self.options.iteritems(): - args.append('--' + key + "=" + value) + for key, value in self.options.items(): + if value is not None: + args.append('--' + key + "=" + str(value)) + else: + args.append('--' + key) if parse_only: args.append('--parse-only') diff --git a/docker/storperf-master/storperf/workloads/_custom_workload.py b/docker/storperf-master/storperf/workloads/_custom_workload.py index 9e0100d..5cd37b3 100644 --- a/docker/storperf-master/storperf/workloads/_custom_workload.py +++ b/docker/storperf-master/storperf/workloads/_custom_workload.py @@ -18,12 +18,12 @@ class _custom_workload(_base_workload._base_workload): self.default_filesize = "1G" self.filename = '/dev/vdb' self.fixed_options = { - 'loops': '200', 'output-format': 'json', 'status-interval': '60' } self.options = { 'ioengine': 'libaio', + 'loops': '200', 'direct': '1', 'numjobs': '1', 'rw': 'read', diff --git a/docker/storperf-master/tests/carbon_tests/emitter_test.py b/docker/storperf-master/tests/carbon_tests/emitter_test.py index f5a78d1..7ea515b 100644 --- a/docker/storperf-master/tests/carbon_tests/emitter_test.py +++ b/docker/storperf-master/tests/carbon_tests/emitter_test.py @@ -11,7 +11,7 @@ import json from time import strptime import unittest -import mock +from unittest import mock from storperf.carbon import converter from storperf.carbon.emitter import CarbonMetricTransmitter @@ -69,9 +69,15 @@ class CarbonMetricTransmitterTest(unittest.TestCase): emitter.carbon_port = self.listen_port emitter.transmit_metrics(result, None) + element = "" + for element in data: + element = element.decode('utf-8') + if element.startswith("host.run-name"): + break + self.assertEqual("host.run-name.key 123.0 975542400\n", - data[1], - data[1]) + element, + data) @mock.patch("socket.socket") @mock.patch("time.gmtime") @@ -90,9 +96,14 @@ class CarbonMetricTransmitterTest(unittest.TestCase): emitter.carbon_port = self.listen_port emitter.transmit_metrics(result, None) + element = "" + for element in data: + element = element.decode('utf-8') + if element.startswith("None.commit-marker"): + break self.assertEqual("None.commit-marker 975542400 975542400\n", - data[1], - data[1]) + element, + data) @mock.patch("socket.socket") def test_connect_fails(self, mock_socket): diff --git a/docker/storperf-master/tests/db_tests/graphite_db_test.py b/docker/storperf-master/tests/db_tests/graphite_db_test.py index d5fbbfc..2fabfd4 100644 --- a/docker/storperf-master/tests/db_tests/graphite_db_test.py +++ b/docker/storperf-master/tests/db_tests/graphite_db_test.py @@ -9,8 +9,7 @@ import unittest -import mock - +from unittest import mock from storperf.db.graphite_db import GraphiteDB diff --git a/docker/storperf-master/tests/db_tests/job_db_test.py b/docker/storperf-master/tests/db_tests/job_db_test.py index 25fda1f..5201963 100644 --- a/docker/storperf-master/tests/db_tests/job_db_test.py +++ b/docker/storperf-master/tests/db_tests/job_db_test.py @@ -11,8 +11,7 @@ import os import sqlite3 import unittest -import mock - +from unittest import mock from storperf.db.job_db import JobDB from storperf.workloads.rr import rr diff --git a/docker/storperf-master/tests/fio_tests/fio_invoker_test.py b/docker/storperf-master/tests/fio_tests/fio_invoker_test.py index 4672651..3a30500 100644 --- a/docker/storperf-master/tests/fio_tests/fio_invoker_test.py +++ b/docker/storperf-master/tests/fio_tests/fio_invoker_test.py @@ -7,11 +7,11 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from StringIO import StringIO import json import unittest from storperf.fio.fio_invoker import FIOInvoker +from io import BytesIO class Test(unittest.TestCase): @@ -34,7 +34,7 @@ class Test(unittest.TestCase): self.fio_invoker.register(self.event) string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True) - output = StringIO(string + "\n") + output = BytesIO((string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(self.simple_dictionary, self.metric) @@ -43,7 +43,7 @@ class Test(unittest.TestCase): self.fio_invoker.register(self.event) string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True) terminating = "fio: terminating on signal 2\n" - output = StringIO(terminating + string + "\n") + output = BytesIO((terminating + string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(self.simple_dictionary, self.metric) @@ -52,7 +52,7 @@ class Test(unittest.TestCase): self.fio_invoker.register(self.event) string = "{'key': 'value'}" - output = StringIO(string + "\n") + output = BytesIO((string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(None, self.metric) @@ -61,7 +61,7 @@ class Test(unittest.TestCase): self.fio_invoker.register(self.event) string = "{'key':\n}" - output = StringIO(string + "\n") + output = BytesIO((string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(None, self.metric) @@ -71,7 +71,7 @@ class Test(unittest.TestCase): string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True) self.fio_invoker.terminated = True - output = StringIO(string + "\n") + output = BytesIO((string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(None, self.metric) @@ -81,7 +81,7 @@ class Test(unittest.TestCase): self.fio_invoker.register(self.event) string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True) - output = StringIO(string + "\n") + output = BytesIO((string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(self.simple_dictionary, self.metric) diff --git a/docker/storperf-master/tests/storperf_master_test.py b/docker/storperf-master/tests/storperf_master_test.py index 03009d1..1edac6d 100644 --- a/docker/storperf-master/tests/storperf_master_test.py +++ b/docker/storperf-master/tests/storperf_master_test.py @@ -9,7 +9,7 @@ import unittest -import mock +from unittest.mock import patch from storperf.storperf_master import StorPerfMaster @@ -17,8 +17,8 @@ from storperf.storperf_master import StorPerfMaster class StorPerfMasterTest(unittest.TestCase): def setUp(self): - with mock.patch("storperf.storperf_master.OSCreds"), \ - mock.patch( + with patch("storperf.storperf_master.OSCreds"), \ + patch( "storperf.storperf_master.OpenStackHeatStack") as oshs: oshs.return_value.get_stack.return_value = None diff --git a/docker/storperf-master/tests/utilities_tests/data_handler_test.py b/docker/storperf-master/tests/utilities_tests/data_handler_test.py index 35150dd..7e8cbcc 100644 --- a/docker/storperf-master/tests/utilities_tests/data_handler_test.py +++ b/docker/storperf-master/tests/utilities_tests/data_handler_test.py @@ -10,7 +10,7 @@ import os import unittest -import mock +from unittest import mock from storperf.utilities.data_handler import DataHandler @@ -311,10 +311,10 @@ class DataHandlerTest(unittest.TestCase): def test_pass_criteria(self): metadata = { "details": { - "steady_state": { - "_warm_up.queue-depth.8.block-size.16384": False, - "rw.queue-depth.4.block-size.16384": True - } + "steady_state": { + "_warm_up.queue-depth.8.block-size.16384": False, + "rw.queue-depth.4.block-size.16384": True + } }, } criteria = self.data_handler._determine_criteria(metadata) @@ -325,11 +325,11 @@ class DataHandlerTest(unittest.TestCase): def test_fail_criteria(self): metadata = { "details": { - "steady_state": { - "_warm_up.queue-depth.8.block-size.16384": False, - "rw.queue-depth.4.block-size.16384": True, - "rw.queue-depth.8.block-size.16384": False - } + "steady_state": { + "_warm_up.queue-depth.8.block-size.16384": False, + "rw.queue-depth.4.block-size.16384": True, + "rw.queue-depth.8.block-size.16384": False + } }, } criteria = self.data_handler._determine_criteria(metadata) diff --git a/docker/storperf-master/tests/utilities_tests/ip_helper_test.py b/docker/storperf-master/tests/utilities_tests/ip_helper_test.py new file mode 100644 index 0000000..f2d662b --- /dev/null +++ b/docker/storperf-master/tests/utilities_tests/ip_helper_test.py @@ -0,0 +1,39 @@ +############################################################################## +# Copyright (c) 2017 Dell EMC and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import unittest + +from storperf.utilities import ip_helper + + +class Test(unittest.TestCase): + + def testNoPortInIPv4(self): + host, port = ip_helper.parse_address_and_port("127.0.0.1") + self.assertEqual("127.0.0.1", host) + self.assertEqual(22, port) + + def testPortInIPv4(self): + host, port = ip_helper.parse_address_and_port("127.0.0.1:2222") + self.assertEqual("127.0.0.1", host) + self.assertEqual(2222, port) + + def testNoPortInIPv6(self): + host, port = ip_helper.parse_address_and_port( + "1fe80::58bb:c8b:f2f2:c888") + self.assertEqual("1fe80::58bb:c8b:f2f2:c888", + host) + self.assertEqual(22, port) + + def testPortInIPv6(self): + host, port = ip_helper.parse_address_and_port( + "[1fe80::58bb:c8b:f2f2:c888]:2222") + self.assertEqual("1fe80::58bb:c8b:f2f2:c888", + host) + self.assertEqual(2222, port) diff --git a/docker/storperf-reporting/Dockerfile b/docker/storperf-reporting/Dockerfile index ff28dd1..6d017ae 100644 --- a/docker/storperf-reporting/Dockerfile +++ b/docker/storperf-reporting/Dockerfile @@ -16,22 +16,22 @@ ARG ARCH=x86_64 -ARG ALPINE_VERSION=v3.6 +ARG ALPINE_VERSION=v3.10 FROM multiarch/alpine:$ARCH-$ALPINE_VERSION MAINTAINER Mark Beierl <mark.beierl@dell.com> -LABEL version="0.1" description="OPNFV Storperf Reporting Container" +LABEL version="8.0" description="OPNFV Storperf Reporting Container" ARG BRANCH=master RUN ulimit -n 1024 -RUN apk add --update python py-pip +RUN apk add --update python3=3.7.5-r1 COPY . /home/opnfv/storperf-reporting WORKDIR /home/opnfv/storperf-reporting/src -RUN pip install -r /home/opnfv/storperf-reporting/requirements.txt +RUN python3 -m pip install -r /home/opnfv/storperf-reporting/requirements.txt -CMD ["python", "app.py"] +CMD ["python3", "app.py"] EXPOSE 5000 diff --git a/docker/storperf-swaggerui/Dockerfile b/docker/storperf-swaggerui/Dockerfile index 5d58a30..9f82890 100644 --- a/docker/storperf-swaggerui/Dockerfile +++ b/docker/storperf-swaggerui/Dockerfile @@ -13,7 +13,7 @@ ## ARG ARCH=x86_64 -ARG ALPINE_VERSION=v3.6 +ARG ALPINE_VERSION=v3.10 FROM node:10-alpine RUN ulimit -n 1024 diff --git a/docker/storperf-workloadagent/Dockerfile b/docker/storperf-workloadagent/Dockerfile new file mode 100644 index 0000000..e6662a9 --- /dev/null +++ b/docker/storperf-workloadagent/Dockerfile @@ -0,0 +1,37 @@ +############################################################################## +# Copyright (c) 2019 VMware and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# Docker container for workload +# +# Purpose: docker image for Storperf to control as a synthetic workload +# +# Maintained by Mark Beierl +# Build: +# $ docker build -t opnfv/storperf-workloadagent:tag . +# + +ARG ARCH=x86_64 +ARG ALPINE_VERSION=v3.10 +FROM multiarch/alpine:$ARCH-$ALPINE_VERSION + +RUN apk add --no-cache --upgrade \ + logrotate \ + openssh-client \ + openssh-server \ + sudo + +RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/g' /etc/ssh/sshd_config +RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config + +RUN echo "root ALL=(ALL) ALL" >> /etc/sudoers +RUN ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa +RUN ssh-keygen -f /etc/ssh/ssh_host_dsa_key -N '' -t dsa + +RUN echo root:password | chpasswd + +CMD /usr/sbin/sshd -D -e
\ No newline at end of file |