From 8d97778672f843cd79ac9862b5fe8c13ef31091f Mon Sep 17 00:00:00 2001 From: Mark Beierl Date: Mon, 16 Dec 2019 16:37:36 -0500 Subject: Update to Python3 Updates to make StorPerf Python3 compatible, including pulling in SNAPS-OO as python3. Change-Id: Ie4e7af684bd5a1b8db086893848b1889da6308ba JIRA: STORPERF-268 Signed-off-by: Mark Beierl --- docker/storperf-master/Dockerfile | 23 +++++++++++----------- .../storperf-master/storperf/carbon/converter.py | 10 +++++----- docker/storperf-master/storperf/carbon/emitter.py | 6 +++--- docker/storperf-master/storperf/db/job_db.py | 2 +- docker/storperf-master/storperf/fio/fio_invoker.py | 15 +++++++++----- docker/storperf-master/storperf/storperf_master.py | 19 +++++++++--------- docker/storperf-master/storperf/test_executor.py | 6 +++--- .../storperf/workloads/_base_workload.py | 2 +- .../tests/carbon_tests/emitter_test.py | 21 +++++++++++++++----- .../tests/db_tests/graphite_db_test.py | 3 +-- .../storperf-master/tests/db_tests/job_db_test.py | 3 +-- .../tests/fio_tests/fio_invoker_test.py | 14 ++++++------- .../storperf-master/tests/storperf_master_test.py | 6 +++--- .../tests/utilities_tests/data_handler_test.py | 20 +++++++++---------- 14 files changed, 81 insertions(+), 69 deletions(-) (limited to 'docker/storperf-master') diff --git a/docker/storperf-master/Dockerfile b/docker/storperf-master/Dockerfile index 9764a8d..a2e1a1d 100644 --- a/docker/storperf-master/Dockerfile +++ b/docker/storperf-master/Dockerfile @@ -16,12 +16,12 @@ # ARG ARCH=x86_64 -ARG ALPINE_VERSION=v3.6 +ARG ALPINE_VERSION=v3.10 FROM multiarch/alpine:$ARCH-$ALPINE_VERSION as storperf-builder RUN ulimit -n 1024 -LABEL version="7.0" description="OPNFV Storperf Docker container" +LABEL version="8.0" description="OPNFV Storperf Docker container" ARG BRANCH=master @@ -47,28 +47,27 @@ RUN cd ${repos_dir}/fio && EXTFLAGS="-static" make -j $(grep -c ^processor /proc RUN apk --no-cache add --update \ libffi-dev \ libressl-dev \ - python \ - py-pip \ - python-dev \ + python3=3.7.5-r1 \ + python3-dev=3.7.5-r1 \ alpine-sdk \ - linux-headers \ - bash + linux-headers # Install StorPerf COPY requirements.pip /storperf/ -RUN pip install --upgrade setuptools==33.1.1 -RUN pip install -r /storperf/requirements.pip +RUN python3 -m pip install --upgrade setuptools==33.1.1 +RUN python3 -m pip install -r /storperf/requirements.pip # Build stripped down StorPerf image FROM multiarch/alpine:$ARCH-$ALPINE_VERSION as storperf-master RUN apk --no-cache add --update \ - python \ + libressl-dev \ + python3=3.7.5-r1 \ bash -COPY --from=storperf-builder /usr/lib/python2.7/site-packages /usr/lib/python2.7/site-packages +COPY --from=storperf-builder /usr/lib/python3.7/site-packages /usr/lib/python3.7/site-packages COPY --from=storperf-builder /usr/local/bin/fio /usr/local/bin/fio COPY . /storperf @@ -80,4 +79,4 @@ RUN chmod 600 storperf/resources/ssh/storperf_rsa EXPOSE 5000 # Entry point -CMD [ "python", "./rest_server.py" ] +CMD [ "python3", "./rest_server.py" ] diff --git a/docker/storperf-master/storperf/carbon/converter.py b/docker/storperf-master/storperf/carbon/converter.py index 623c144..4b5e6aa 100644 --- a/docker/storperf-master/storperf/carbon/converter.py +++ b/docker/storperf-master/storperf/carbon/converter.py @@ -32,12 +32,12 @@ class Converter(object): def resurse_to_flat_dictionary(self, json, prefix=None): if type(json) == dict: - for k, v in json.items(): + for k, v in list(json.items()): if prefix is None: - key = k.decode("utf-8").replace(" ", "_") + key = k.replace(" ", "_") else: - key = prefix + "." + k.decode("utf-8").replace(" ", "_") - if hasattr(v, '__iter__'): + key = prefix + "." + k.replace(" ", "_") + if type(v) is list or type(v) is dict: self.resurse_to_flat_dictionary(v, key) else: self.flat_dictionary[key] = str(v).replace(" ", "_") @@ -45,7 +45,7 @@ class Converter(object): index = 0 for v in json: index += 1 - if hasattr(v, '__iter__'): + if type(v) is list or type(v) is dict: self.resurse_to_flat_dictionary( v, prefix + "." + str(index)) else: diff --git a/docker/storperf-master/storperf/carbon/emitter.py b/docker/storperf-master/storperf/carbon/emitter.py index b196709..13503b2 100644 --- a/docker/storperf-master/storperf/carbon/emitter.py +++ b/docker/storperf-master/storperf/carbon/emitter.py @@ -40,19 +40,19 @@ class CarbonMetricTransmitter(): message = "%s %s %s\n" \ % (key, value, timestamp) self.logger.debug("Metric: " + message.strip()) - carbon_socket.send(message) + carbon_socket.send(message.encode('utf-8')) except ValueError: self.logger.debug("Ignoring non numeric metric %s %s" % (key, value)) message = "%s.commit-marker %s %s\n" \ % (commit_marker, timestamp, timestamp) - carbon_socket.send(message) + carbon_socket.send(message.encode('utf-8')) self.logger.debug("Marker %s" % message.strip()) self.logger.info("Sent metrics to %s:%s with timestamp %s" % (self.host, self.port, timestamp)) - except Exception, e: + except Exception as e: self.logger.error("While notifying carbon %s:%s %s" % (self.host, self.port, e)) diff --git a/docker/storperf-master/storperf/db/job_db.py b/docker/storperf-master/storperf/db/job_db.py index b029a35..c3632e4 100644 --- a/docker/storperf-master/storperf/db/job_db.py +++ b/docker/storperf-master/storperf/db/job_db.py @@ -220,7 +220,7 @@ class JobDB(object): db = sqlite3.connect(JobDB.db_name) cursor = db.cursor() - for param, value in params.iteritems(): + for param, value in params.items(): cursor.execute( """insert into job_params (job_id, diff --git a/docker/storperf-master/storperf/fio/fio_invoker.py b/docker/storperf-master/storperf/fio/fio_invoker.py index c665598..2437763 100644 --- a/docker/storperf-master/storperf/fio/fio_invoker.py +++ b/docker/storperf-master/storperf/fio/fio_invoker.py @@ -45,6 +45,8 @@ class FIOInvoker(object): self.json_body = "" try: for line in iter(stdout.readline, b''): + if type(line) == bytes: + line = line.decode('utf=8') if line.startswith("fio"): line = "" continue @@ -78,7 +80,8 @@ class FIOInvoker(object): def stderr_handler(self, stderr): self.logger.debug("Started") for line in iter(stderr.readline, b''): - self.logger.error("FIO Error: %s", line.rstrip()) + if len(line) > 0: + self.logger.error("FIO Error: %s", line.rstrip()) self.stderr.append(line.rstrip()) # Sometime, FIO gets stuck and will give us this message: @@ -137,10 +140,12 @@ class FIOInvoker(object): ssh = self._ssh_client() - command = "sudo killall fio" - - self.logger.debug("Executing on %s: %s" % (self.remote_host, command)) - (_, stdout, stderr) = ssh.exec_command(command) + kill_commands = ['sudo killall fio', + 'sudo pkill fio'] + for command in kill_commands: + self.logger.debug("Executing on %s: %s" % + (self.remote_host, command)) + (_, stdout, stderr) = ssh.exec_command(command) for line in stdout.readlines(): self.logger.debug(line.strip()) diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py index 3de8478..afcd018 100644 --- a/docker/storperf-master/storperf/storperf_master.py +++ b/docker/storperf-master/storperf/storperf_master.py @@ -7,26 +7,25 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -import StringIO + +from _io import StringIO from datetime import datetime -import json -import logging from multiprocessing.pool import ThreadPool -import os -import socket -from time import sleep -import uuid - -import paramiko from scp import SCPClient from snaps.config.stack import StackConfig from snaps.openstack.create_stack import OpenStackHeatStack from snaps.openstack.os_credentials import OSCreds from snaps.openstack.utils import heat_utils, cinder_utils, glance_utils from snaps.thread_utils import worker_pool - from storperf.db.job_db import JobDB from storperf.test_executor import TestExecutor +from time import sleep +import json +import logging +import os +import paramiko +import socket +import uuid class ParameterError(Exception): diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py index 4b5bbd4..cb7e478 100644 --- a/docker/storperf-master/storperf/test_executor.py +++ b/docker/storperf-master/storperf/test_executor.py @@ -318,7 +318,7 @@ class TestExecutor(object): workload = current_workload['workload'] self._thread_gate = ThreadGate( len(self.slaves) * min(1, self.volume_count), - workload.options['status-interval']) + float(workload.options['status-interval'])) self.current_workload = current_workload['name'] @@ -362,7 +362,7 @@ class TestExecutor(object): workloads = [] if self._custom_workloads: - for workload_name in self._custom_workloads.iterkeys(): + for workload_name in self._custom_workloads.keys(): real_name = workload_name if real_name.startswith('_'): real_name = real_name.replace('_', '') @@ -380,7 +380,7 @@ class TestExecutor(object): workload.id = self.job_db.job_id workload_params = self._custom_workloads[workload_name] - for param, value in workload_params.iteritems(): + for param, value in workload_params.items(): if param == "readwrite": param = "rw" if param in workload.fixed_options: diff --git a/docker/storperf-master/storperf/workloads/_base_workload.py b/docker/storperf-master/storperf/workloads/_base_workload.py index 7468fea..5aa596e 100644 --- a/docker/storperf-master/storperf/workloads/_base_workload.py +++ b/docker/storperf-master/storperf/workloads/_base_workload.py @@ -57,7 +57,7 @@ class _base_workload(object): self.setup() - for key, value in self.options.iteritems(): + for key, value in self.options.items(): if value is not None: args.append('--' + key + "=" + str(value)) else: diff --git a/docker/storperf-master/tests/carbon_tests/emitter_test.py b/docker/storperf-master/tests/carbon_tests/emitter_test.py index f5a78d1..7ea515b 100644 --- a/docker/storperf-master/tests/carbon_tests/emitter_test.py +++ b/docker/storperf-master/tests/carbon_tests/emitter_test.py @@ -11,7 +11,7 @@ import json from time import strptime import unittest -import mock +from unittest import mock from storperf.carbon import converter from storperf.carbon.emitter import CarbonMetricTransmitter @@ -69,9 +69,15 @@ class CarbonMetricTransmitterTest(unittest.TestCase): emitter.carbon_port = self.listen_port emitter.transmit_metrics(result, None) + element = "" + for element in data: + element = element.decode('utf-8') + if element.startswith("host.run-name"): + break + self.assertEqual("host.run-name.key 123.0 975542400\n", - data[1], - data[1]) + element, + data) @mock.patch("socket.socket") @mock.patch("time.gmtime") @@ -90,9 +96,14 @@ class CarbonMetricTransmitterTest(unittest.TestCase): emitter.carbon_port = self.listen_port emitter.transmit_metrics(result, None) + element = "" + for element in data: + element = element.decode('utf-8') + if element.startswith("None.commit-marker"): + break self.assertEqual("None.commit-marker 975542400 975542400\n", - data[1], - data[1]) + element, + data) @mock.patch("socket.socket") def test_connect_fails(self, mock_socket): diff --git a/docker/storperf-master/tests/db_tests/graphite_db_test.py b/docker/storperf-master/tests/db_tests/graphite_db_test.py index d5fbbfc..2fabfd4 100644 --- a/docker/storperf-master/tests/db_tests/graphite_db_test.py +++ b/docker/storperf-master/tests/db_tests/graphite_db_test.py @@ -9,8 +9,7 @@ import unittest -import mock - +from unittest import mock from storperf.db.graphite_db import GraphiteDB diff --git a/docker/storperf-master/tests/db_tests/job_db_test.py b/docker/storperf-master/tests/db_tests/job_db_test.py index 25fda1f..5201963 100644 --- a/docker/storperf-master/tests/db_tests/job_db_test.py +++ b/docker/storperf-master/tests/db_tests/job_db_test.py @@ -11,8 +11,7 @@ import os import sqlite3 import unittest -import mock - +from unittest import mock from storperf.db.job_db import JobDB from storperf.workloads.rr import rr diff --git a/docker/storperf-master/tests/fio_tests/fio_invoker_test.py b/docker/storperf-master/tests/fio_tests/fio_invoker_test.py index 4672651..3a30500 100644 --- a/docker/storperf-master/tests/fio_tests/fio_invoker_test.py +++ b/docker/storperf-master/tests/fio_tests/fio_invoker_test.py @@ -7,11 +7,11 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from StringIO import StringIO import json import unittest from storperf.fio.fio_invoker import FIOInvoker +from io import BytesIO class Test(unittest.TestCase): @@ -34,7 +34,7 @@ class Test(unittest.TestCase): self.fio_invoker.register(self.event) string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True) - output = StringIO(string + "\n") + output = BytesIO((string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(self.simple_dictionary, self.metric) @@ -43,7 +43,7 @@ class Test(unittest.TestCase): self.fio_invoker.register(self.event) string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True) terminating = "fio: terminating on signal 2\n" - output = StringIO(terminating + string + "\n") + output = BytesIO((terminating + string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(self.simple_dictionary, self.metric) @@ -52,7 +52,7 @@ class Test(unittest.TestCase): self.fio_invoker.register(self.event) string = "{'key': 'value'}" - output = StringIO(string + "\n") + output = BytesIO((string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(None, self.metric) @@ -61,7 +61,7 @@ class Test(unittest.TestCase): self.fio_invoker.register(self.event) string = "{'key':\n}" - output = StringIO(string + "\n") + output = BytesIO((string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(None, self.metric) @@ -71,7 +71,7 @@ class Test(unittest.TestCase): string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True) self.fio_invoker.terminated = True - output = StringIO(string + "\n") + output = BytesIO((string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(None, self.metric) @@ -81,7 +81,7 @@ class Test(unittest.TestCase): self.fio_invoker.register(self.event) string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True) - output = StringIO(string + "\n") + output = BytesIO((string + "\n").encode('utf-8')) self.fio_invoker.stdout_handler(output) self.assertEqual(self.simple_dictionary, self.metric) diff --git a/docker/storperf-master/tests/storperf_master_test.py b/docker/storperf-master/tests/storperf_master_test.py index 03009d1..1edac6d 100644 --- a/docker/storperf-master/tests/storperf_master_test.py +++ b/docker/storperf-master/tests/storperf_master_test.py @@ -9,7 +9,7 @@ import unittest -import mock +from unittest.mock import patch from storperf.storperf_master import StorPerfMaster @@ -17,8 +17,8 @@ from storperf.storperf_master import StorPerfMaster class StorPerfMasterTest(unittest.TestCase): def setUp(self): - with mock.patch("storperf.storperf_master.OSCreds"), \ - mock.patch( + with patch("storperf.storperf_master.OSCreds"), \ + patch( "storperf.storperf_master.OpenStackHeatStack") as oshs: oshs.return_value.get_stack.return_value = None diff --git a/docker/storperf-master/tests/utilities_tests/data_handler_test.py b/docker/storperf-master/tests/utilities_tests/data_handler_test.py index 35150dd..7e8cbcc 100644 --- a/docker/storperf-master/tests/utilities_tests/data_handler_test.py +++ b/docker/storperf-master/tests/utilities_tests/data_handler_test.py @@ -10,7 +10,7 @@ import os import unittest -import mock +from unittest import mock from storperf.utilities.data_handler import DataHandler @@ -311,10 +311,10 @@ class DataHandlerTest(unittest.TestCase): def test_pass_criteria(self): metadata = { "details": { - "steady_state": { - "_warm_up.queue-depth.8.block-size.16384": False, - "rw.queue-depth.4.block-size.16384": True - } + "steady_state": { + "_warm_up.queue-depth.8.block-size.16384": False, + "rw.queue-depth.4.block-size.16384": True + } }, } criteria = self.data_handler._determine_criteria(metadata) @@ -325,11 +325,11 @@ class DataHandlerTest(unittest.TestCase): def test_fail_criteria(self): metadata = { "details": { - "steady_state": { - "_warm_up.queue-depth.8.block-size.16384": False, - "rw.queue-depth.4.block-size.16384": True, - "rw.queue-depth.8.block-size.16384": False - } + "steady_state": { + "_warm_up.queue-depth.8.block-size.16384": False, + "rw.queue-depth.4.block-size.16384": True, + "rw.queue-depth.8.block-size.16384": False + } }, } criteria = self.data_handler._determine_criteria(metadata) -- cgit 1.2.3-korg