summaryrefslogtreecommitdiffstats
path: root/docker/storperf-master
diff options
context:
space:
mode:
authorMark Beierl <mbeierl@vmware.com>2019-12-16 16:37:36 -0500
committerMark Beierl <mbeierl@vmware.com>2019-12-17 11:35:36 -0500
commit8d97778672f843cd79ac9862b5fe8c13ef31091f (patch)
treeade1e84445a7341ca72c48e05f343045a0e1b26d /docker/storperf-master
parentddc5a2888edf717ff074c272b01a6310e0d612fa (diff)
Update to Python3
Updates to make StorPerf Python3 compatible, including pulling in SNAPS-OO as python3. Change-Id: Ie4e7af684bd5a1b8db086893848b1889da6308ba JIRA: STORPERF-268 Signed-off-by: Mark Beierl <mbeierl@vmware.com>
Diffstat (limited to 'docker/storperf-master')
-rw-r--r--docker/storperf-master/Dockerfile23
-rw-r--r--docker/storperf-master/storperf/carbon/converter.py10
-rw-r--r--docker/storperf-master/storperf/carbon/emitter.py6
-rw-r--r--docker/storperf-master/storperf/db/job_db.py2
-rw-r--r--docker/storperf-master/storperf/fio/fio_invoker.py15
-rw-r--r--docker/storperf-master/storperf/storperf_master.py19
-rw-r--r--docker/storperf-master/storperf/test_executor.py6
-rw-r--r--docker/storperf-master/storperf/workloads/_base_workload.py2
-rw-r--r--docker/storperf-master/tests/carbon_tests/emitter_test.py21
-rw-r--r--docker/storperf-master/tests/db_tests/graphite_db_test.py3
-rw-r--r--docker/storperf-master/tests/db_tests/job_db_test.py3
-rw-r--r--docker/storperf-master/tests/fio_tests/fio_invoker_test.py14
-rw-r--r--docker/storperf-master/tests/storperf_master_test.py6
-rw-r--r--docker/storperf-master/tests/utilities_tests/data_handler_test.py20
14 files changed, 81 insertions, 69 deletions
diff --git a/docker/storperf-master/Dockerfile b/docker/storperf-master/Dockerfile
index 9764a8d..a2e1a1d 100644
--- a/docker/storperf-master/Dockerfile
+++ b/docker/storperf-master/Dockerfile
@@ -16,12 +16,12 @@
#
ARG ARCH=x86_64
-ARG ALPINE_VERSION=v3.6
+ARG ALPINE_VERSION=v3.10
FROM multiarch/alpine:$ARCH-$ALPINE_VERSION as storperf-builder
RUN ulimit -n 1024
-LABEL version="7.0" description="OPNFV Storperf Docker container"
+LABEL version="8.0" description="OPNFV Storperf Docker container"
ARG BRANCH=master
@@ -47,28 +47,27 @@ RUN cd ${repos_dir}/fio && EXTFLAGS="-static" make -j $(grep -c ^processor /proc
RUN apk --no-cache add --update \
libffi-dev \
libressl-dev \
- python \
- py-pip \
- python-dev \
+ python3=3.7.5-r1 \
+ python3-dev=3.7.5-r1 \
alpine-sdk \
- linux-headers \
- bash
+ linux-headers
# Install StorPerf
COPY requirements.pip /storperf/
-RUN pip install --upgrade setuptools==33.1.1
-RUN pip install -r /storperf/requirements.pip
+RUN python3 -m pip install --upgrade setuptools==33.1.1
+RUN python3 -m pip install -r /storperf/requirements.pip
# Build stripped down StorPerf image
FROM multiarch/alpine:$ARCH-$ALPINE_VERSION as storperf-master
RUN apk --no-cache add --update \
- python \
+ libressl-dev \
+ python3=3.7.5-r1 \
bash
-COPY --from=storperf-builder /usr/lib/python2.7/site-packages /usr/lib/python2.7/site-packages
+COPY --from=storperf-builder /usr/lib/python3.7/site-packages /usr/lib/python3.7/site-packages
COPY --from=storperf-builder /usr/local/bin/fio /usr/local/bin/fio
COPY . /storperf
@@ -80,4 +79,4 @@ RUN chmod 600 storperf/resources/ssh/storperf_rsa
EXPOSE 5000
# Entry point
-CMD [ "python", "./rest_server.py" ]
+CMD [ "python3", "./rest_server.py" ]
diff --git a/docker/storperf-master/storperf/carbon/converter.py b/docker/storperf-master/storperf/carbon/converter.py
index 623c144..4b5e6aa 100644
--- a/docker/storperf-master/storperf/carbon/converter.py
+++ b/docker/storperf-master/storperf/carbon/converter.py
@@ -32,12 +32,12 @@ class Converter(object):
def resurse_to_flat_dictionary(self, json, prefix=None):
if type(json) == dict:
- for k, v in json.items():
+ for k, v in list(json.items()):
if prefix is None:
- key = k.decode("utf-8").replace(" ", "_")
+ key = k.replace(" ", "_")
else:
- key = prefix + "." + k.decode("utf-8").replace(" ", "_")
- if hasattr(v, '__iter__'):
+ key = prefix + "." + k.replace(" ", "_")
+ if type(v) is list or type(v) is dict:
self.resurse_to_flat_dictionary(v, key)
else:
self.flat_dictionary[key] = str(v).replace(" ", "_")
@@ -45,7 +45,7 @@ class Converter(object):
index = 0
for v in json:
index += 1
- if hasattr(v, '__iter__'):
+ if type(v) is list or type(v) is dict:
self.resurse_to_flat_dictionary(
v, prefix + "." + str(index))
else:
diff --git a/docker/storperf-master/storperf/carbon/emitter.py b/docker/storperf-master/storperf/carbon/emitter.py
index b196709..13503b2 100644
--- a/docker/storperf-master/storperf/carbon/emitter.py
+++ b/docker/storperf-master/storperf/carbon/emitter.py
@@ -40,19 +40,19 @@ class CarbonMetricTransmitter():
message = "%s %s %s\n" \
% (key, value, timestamp)
self.logger.debug("Metric: " + message.strip())
- carbon_socket.send(message)
+ carbon_socket.send(message.encode('utf-8'))
except ValueError:
self.logger.debug("Ignoring non numeric metric %s %s"
% (key, value))
message = "%s.commit-marker %s %s\n" \
% (commit_marker, timestamp, timestamp)
- carbon_socket.send(message)
+ carbon_socket.send(message.encode('utf-8'))
self.logger.debug("Marker %s" % message.strip())
self.logger.info("Sent metrics to %s:%s with timestamp %s"
% (self.host, self.port, timestamp))
- except Exception, e:
+ except Exception as e:
self.logger.error("While notifying carbon %s:%s %s"
% (self.host, self.port, e))
diff --git a/docker/storperf-master/storperf/db/job_db.py b/docker/storperf-master/storperf/db/job_db.py
index b029a35..c3632e4 100644
--- a/docker/storperf-master/storperf/db/job_db.py
+++ b/docker/storperf-master/storperf/db/job_db.py
@@ -220,7 +220,7 @@ class JobDB(object):
db = sqlite3.connect(JobDB.db_name)
cursor = db.cursor()
- for param, value in params.iteritems():
+ for param, value in params.items():
cursor.execute(
"""insert into job_params
(job_id,
diff --git a/docker/storperf-master/storperf/fio/fio_invoker.py b/docker/storperf-master/storperf/fio/fio_invoker.py
index c665598..2437763 100644
--- a/docker/storperf-master/storperf/fio/fio_invoker.py
+++ b/docker/storperf-master/storperf/fio/fio_invoker.py
@@ -45,6 +45,8 @@ class FIOInvoker(object):
self.json_body = ""
try:
for line in iter(stdout.readline, b''):
+ if type(line) == bytes:
+ line = line.decode('utf=8')
if line.startswith("fio"):
line = ""
continue
@@ -78,7 +80,8 @@ class FIOInvoker(object):
def stderr_handler(self, stderr):
self.logger.debug("Started")
for line in iter(stderr.readline, b''):
- self.logger.error("FIO Error: %s", line.rstrip())
+ if len(line) > 0:
+ self.logger.error("FIO Error: %s", line.rstrip())
self.stderr.append(line.rstrip())
# Sometime, FIO gets stuck and will give us this message:
@@ -137,10 +140,12 @@ class FIOInvoker(object):
ssh = self._ssh_client()
- command = "sudo killall fio"
-
- self.logger.debug("Executing on %s: %s" % (self.remote_host, command))
- (_, stdout, stderr) = ssh.exec_command(command)
+ kill_commands = ['sudo killall fio',
+ 'sudo pkill fio']
+ for command in kill_commands:
+ self.logger.debug("Executing on %s: %s" %
+ (self.remote_host, command))
+ (_, stdout, stderr) = ssh.exec_command(command)
for line in stdout.readlines():
self.logger.debug(line.strip())
diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py
index 3de8478..afcd018 100644
--- a/docker/storperf-master/storperf/storperf_master.py
+++ b/docker/storperf-master/storperf/storperf_master.py
@@ -7,26 +7,25 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-import StringIO
+
+from _io import StringIO
from datetime import datetime
-import json
-import logging
from multiprocessing.pool import ThreadPool
-import os
-import socket
-from time import sleep
-import uuid
-
-import paramiko
from scp import SCPClient
from snaps.config.stack import StackConfig
from snaps.openstack.create_stack import OpenStackHeatStack
from snaps.openstack.os_credentials import OSCreds
from snaps.openstack.utils import heat_utils, cinder_utils, glance_utils
from snaps.thread_utils import worker_pool
-
from storperf.db.job_db import JobDB
from storperf.test_executor import TestExecutor
+from time import sleep
+import json
+import logging
+import os
+import paramiko
+import socket
+import uuid
class ParameterError(Exception):
diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py
index 4b5bbd4..cb7e478 100644
--- a/docker/storperf-master/storperf/test_executor.py
+++ b/docker/storperf-master/storperf/test_executor.py
@@ -318,7 +318,7 @@ class TestExecutor(object):
workload = current_workload['workload']
self._thread_gate = ThreadGate(
len(self.slaves) * min(1, self.volume_count),
- workload.options['status-interval'])
+ float(workload.options['status-interval']))
self.current_workload = current_workload['name']
@@ -362,7 +362,7 @@ class TestExecutor(object):
workloads = []
if self._custom_workloads:
- for workload_name in self._custom_workloads.iterkeys():
+ for workload_name in self._custom_workloads.keys():
real_name = workload_name
if real_name.startswith('_'):
real_name = real_name.replace('_', '')
@@ -380,7 +380,7 @@ class TestExecutor(object):
workload.id = self.job_db.job_id
workload_params = self._custom_workloads[workload_name]
- for param, value in workload_params.iteritems():
+ for param, value in workload_params.items():
if param == "readwrite":
param = "rw"
if param in workload.fixed_options:
diff --git a/docker/storperf-master/storperf/workloads/_base_workload.py b/docker/storperf-master/storperf/workloads/_base_workload.py
index 7468fea..5aa596e 100644
--- a/docker/storperf-master/storperf/workloads/_base_workload.py
+++ b/docker/storperf-master/storperf/workloads/_base_workload.py
@@ -57,7 +57,7 @@ class _base_workload(object):
self.setup()
- for key, value in self.options.iteritems():
+ for key, value in self.options.items():
if value is not None:
args.append('--' + key + "=" + str(value))
else:
diff --git a/docker/storperf-master/tests/carbon_tests/emitter_test.py b/docker/storperf-master/tests/carbon_tests/emitter_test.py
index f5a78d1..7ea515b 100644
--- a/docker/storperf-master/tests/carbon_tests/emitter_test.py
+++ b/docker/storperf-master/tests/carbon_tests/emitter_test.py
@@ -11,7 +11,7 @@ import json
from time import strptime
import unittest
-import mock
+from unittest import mock
from storperf.carbon import converter
from storperf.carbon.emitter import CarbonMetricTransmitter
@@ -69,9 +69,15 @@ class CarbonMetricTransmitterTest(unittest.TestCase):
emitter.carbon_port = self.listen_port
emitter.transmit_metrics(result, None)
+ element = ""
+ for element in data:
+ element = element.decode('utf-8')
+ if element.startswith("host.run-name"):
+ break
+
self.assertEqual("host.run-name.key 123.0 975542400\n",
- data[1],
- data[1])
+ element,
+ data)
@mock.patch("socket.socket")
@mock.patch("time.gmtime")
@@ -90,9 +96,14 @@ class CarbonMetricTransmitterTest(unittest.TestCase):
emitter.carbon_port = self.listen_port
emitter.transmit_metrics(result, None)
+ element = ""
+ for element in data:
+ element = element.decode('utf-8')
+ if element.startswith("None.commit-marker"):
+ break
self.assertEqual("None.commit-marker 975542400 975542400\n",
- data[1],
- data[1])
+ element,
+ data)
@mock.patch("socket.socket")
def test_connect_fails(self, mock_socket):
diff --git a/docker/storperf-master/tests/db_tests/graphite_db_test.py b/docker/storperf-master/tests/db_tests/graphite_db_test.py
index d5fbbfc..2fabfd4 100644
--- a/docker/storperf-master/tests/db_tests/graphite_db_test.py
+++ b/docker/storperf-master/tests/db_tests/graphite_db_test.py
@@ -9,8 +9,7 @@
import unittest
-import mock
-
+from unittest import mock
from storperf.db.graphite_db import GraphiteDB
diff --git a/docker/storperf-master/tests/db_tests/job_db_test.py b/docker/storperf-master/tests/db_tests/job_db_test.py
index 25fda1f..5201963 100644
--- a/docker/storperf-master/tests/db_tests/job_db_test.py
+++ b/docker/storperf-master/tests/db_tests/job_db_test.py
@@ -11,8 +11,7 @@ import os
import sqlite3
import unittest
-import mock
-
+from unittest import mock
from storperf.db.job_db import JobDB
from storperf.workloads.rr import rr
diff --git a/docker/storperf-master/tests/fio_tests/fio_invoker_test.py b/docker/storperf-master/tests/fio_tests/fio_invoker_test.py
index 4672651..3a30500 100644
--- a/docker/storperf-master/tests/fio_tests/fio_invoker_test.py
+++ b/docker/storperf-master/tests/fio_tests/fio_invoker_test.py
@@ -7,11 +7,11 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from StringIO import StringIO
import json
import unittest
from storperf.fio.fio_invoker import FIOInvoker
+from io import BytesIO
class Test(unittest.TestCase):
@@ -34,7 +34,7 @@ class Test(unittest.TestCase):
self.fio_invoker.register(self.event)
string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
- output = StringIO(string + "\n")
+ output = BytesIO((string + "\n").encode('utf-8'))
self.fio_invoker.stdout_handler(output)
self.assertEqual(self.simple_dictionary, self.metric)
@@ -43,7 +43,7 @@ class Test(unittest.TestCase):
self.fio_invoker.register(self.event)
string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
terminating = "fio: terminating on signal 2\n"
- output = StringIO(terminating + string + "\n")
+ output = BytesIO((terminating + string + "\n").encode('utf-8'))
self.fio_invoker.stdout_handler(output)
self.assertEqual(self.simple_dictionary, self.metric)
@@ -52,7 +52,7 @@ class Test(unittest.TestCase):
self.fio_invoker.register(self.event)
string = "{'key': 'value'}"
- output = StringIO(string + "\n")
+ output = BytesIO((string + "\n").encode('utf-8'))
self.fio_invoker.stdout_handler(output)
self.assertEqual(None, self.metric)
@@ -61,7 +61,7 @@ class Test(unittest.TestCase):
self.fio_invoker.register(self.event)
string = "{'key':\n}"
- output = StringIO(string + "\n")
+ output = BytesIO((string + "\n").encode('utf-8'))
self.fio_invoker.stdout_handler(output)
self.assertEqual(None, self.metric)
@@ -71,7 +71,7 @@ class Test(unittest.TestCase):
string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
self.fio_invoker.terminated = True
- output = StringIO(string + "\n")
+ output = BytesIO((string + "\n").encode('utf-8'))
self.fio_invoker.stdout_handler(output)
self.assertEqual(None, self.metric)
@@ -81,7 +81,7 @@ class Test(unittest.TestCase):
self.fio_invoker.register(self.event)
string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
- output = StringIO(string + "\n")
+ output = BytesIO((string + "\n").encode('utf-8'))
self.fio_invoker.stdout_handler(output)
self.assertEqual(self.simple_dictionary, self.metric)
diff --git a/docker/storperf-master/tests/storperf_master_test.py b/docker/storperf-master/tests/storperf_master_test.py
index 03009d1..1edac6d 100644
--- a/docker/storperf-master/tests/storperf_master_test.py
+++ b/docker/storperf-master/tests/storperf_master_test.py
@@ -9,7 +9,7 @@
import unittest
-import mock
+from unittest.mock import patch
from storperf.storperf_master import StorPerfMaster
@@ -17,8 +17,8 @@ from storperf.storperf_master import StorPerfMaster
class StorPerfMasterTest(unittest.TestCase):
def setUp(self):
- with mock.patch("storperf.storperf_master.OSCreds"), \
- mock.patch(
+ with patch("storperf.storperf_master.OSCreds"), \
+ patch(
"storperf.storperf_master.OpenStackHeatStack") as oshs:
oshs.return_value.get_stack.return_value = None
diff --git a/docker/storperf-master/tests/utilities_tests/data_handler_test.py b/docker/storperf-master/tests/utilities_tests/data_handler_test.py
index 35150dd..7e8cbcc 100644
--- a/docker/storperf-master/tests/utilities_tests/data_handler_test.py
+++ b/docker/storperf-master/tests/utilities_tests/data_handler_test.py
@@ -10,7 +10,7 @@
import os
import unittest
-import mock
+from unittest import mock
from storperf.utilities.data_handler import DataHandler
@@ -311,10 +311,10 @@ class DataHandlerTest(unittest.TestCase):
def test_pass_criteria(self):
metadata = {
"details": {
- "steady_state": {
- "_warm_up.queue-depth.8.block-size.16384": False,
- "rw.queue-depth.4.block-size.16384": True
- }
+ "steady_state": {
+ "_warm_up.queue-depth.8.block-size.16384": False,
+ "rw.queue-depth.4.block-size.16384": True
+ }
},
}
criteria = self.data_handler._determine_criteria(metadata)
@@ -325,11 +325,11 @@ class DataHandlerTest(unittest.TestCase):
def test_fail_criteria(self):
metadata = {
"details": {
- "steady_state": {
- "_warm_up.queue-depth.8.block-size.16384": False,
- "rw.queue-depth.4.block-size.16384": True,
- "rw.queue-depth.8.block-size.16384": False
- }
+ "steady_state": {
+ "_warm_up.queue-depth.8.block-size.16384": False,
+ "rw.queue-depth.4.block-size.16384": True,
+ "rw.queue-depth.8.block-size.16384": False
+ }
},
}
criteria = self.data_handler._determine_criteria(metadata)