Update to Python3

Updates to make StorPerf Python3 compatible, including pulling
in SNAPS-OO as python3.

Change-Id: Ie4e7af684bd5a1b8db086893848b1889da6308ba
JIRA: STORPERF-268
Signed-off-by: Mark Beierl <mbeierl@vmware.com>

18 files changed, 91 insertions, 76 deletions

diff --git a/docker/local-docker-compose.yaml b/docker/local-docker-compose.yaml
index 6daa6e2..a4b69b4 100644
--- a/docker/local-docker-compose.yaml
+++ b/docker/local-docker-compose.yaml
@@ -17,8 +17,10 @@ services:
             args:
                 ARCH: ${ARCH}
         env_file: ${ENV_FILE}
+        user: ${CURRENT_UID}
         volumes:
             - ./storperf-master/:/storperf
+            - ./certs:/etc/ssl/certs/
         links:
             - storperf-graphite
 
@@ -28,6 +30,7 @@ services:
             context: storperf-reporting
             args:
                 ARCH: ${ARCH}
+        user: ${CURRENT_UID}
         volumes:
             - ./storperf-reporting/:/home/opnfv/storperf-reporting
 
diff --git a/docker/storperf-httpfrontend/Dockerfile b/docker/storperf-httpfrontend/Dockerfile
index 95188b5..6f072b0 100644
--- a/docker/storperf-httpfrontend/Dockerfile
+++ b/docker/storperf-httpfrontend/Dockerfile
@@ -13,7 +13,7 @@
 ##
 
 ARG ARCH=x86_64
-ARG ALPINE_VERSION=v3.6
+ARG ALPINE_VERSION=v3.10
 FROM nginx:alpine
 
 EXPOSE 80 443
diff --git a/docker/storperf-master/Dockerfile b/docker/storperf-master/Dockerfile
index 9764a8d..a2e1a1d 100644
--- a/docker/storperf-master/Dockerfile
+++ b/docker/storperf-master/Dockerfile
@@ -16,12 +16,12 @@
 #
 
 ARG ARCH=x86_64
-ARG ALPINE_VERSION=v3.6
+ARG ALPINE_VERSION=v3.10
 FROM multiarch/alpine:$ARCH-$ALPINE_VERSION as storperf-builder
 
 RUN ulimit -n 1024
 
-LABEL version="7.0" description="OPNFV Storperf Docker container"
+LABEL version="8.0" description="OPNFV Storperf Docker container"
 
 ARG BRANCH=master
 
@@ -47,28 +47,27 @@ RUN cd ${repos_dir}/fio && EXTFLAGS="-static" make -j $(grep -c ^processor /proc
 RUN apk --no-cache add --update \
     libffi-dev \
     libressl-dev \
-    python \
-    py-pip \
-    python-dev \
+    python3=3.7.5-r1 \
+    python3-dev=3.7.5-r1 \
     alpine-sdk \
-    linux-headers \
-    bash
+    linux-headers
 
 # Install StorPerf
 
 COPY requirements.pip /storperf/
-RUN pip install --upgrade setuptools==33.1.1
-RUN pip install -r /storperf/requirements.pip
+RUN python3 -m pip  install --upgrade setuptools==33.1.1
+RUN python3 -m pip  install -r /storperf/requirements.pip
 
 # Build stripped down StorPerf image
 
 FROM multiarch/alpine:$ARCH-$ALPINE_VERSION as storperf-master
 
 RUN apk --no-cache add --update \
-    python \
+    libressl-dev \
+    python3=3.7.5-r1 \
     bash
 
-COPY --from=storperf-builder /usr/lib/python2.7/site-packages /usr/lib/python2.7/site-packages
+COPY --from=storperf-builder /usr/lib/python3.7/site-packages /usr/lib/python3.7/site-packages
 COPY --from=storperf-builder /usr/local/bin/fio /usr/local/bin/fio
 COPY . /storperf
 
@@ -80,4 +79,4 @@ RUN chmod 600 storperf/resources/ssh/storperf_rsa
 EXPOSE 5000
 
 # Entry point
-CMD [ "python", "./rest_server.py" ]
+CMD [ "python3", "./rest_server.py" ]
diff --git a/docker/storperf-master/storperf/carbon/converter.py b/docker/storperf-master/storperf/carbon/converter.py
index 623c144..4b5e6aa 100644
--- a/docker/storperf-master/storperf/carbon/converter.py
+++ b/docker/storperf-master/storperf/carbon/converter.py
@@ -32,12 +32,12 @@ class Converter(object):
 
     def resurse_to_flat_dictionary(self, json, prefix=None):
         if type(json) == dict:
-            for k, v in json.items():
+            for k, v in list(json.items()):
                 if prefix is None:
-                    key = k.decode("utf-8").replace(" ", "_")
+                    key = k.replace(" ", "_")
                 else:
-                    key = prefix + "." + k.decode("utf-8").replace(" ", "_")
-                if hasattr(v, '__iter__'):
+                    key = prefix + "." + k.replace(" ", "_")
+                if type(v) is list or type(v) is dict:
                     self.resurse_to_flat_dictionary(v, key)
                 else:
                     self.flat_dictionary[key] = str(v).replace(" ", "_")
@@ -45,7 +45,7 @@ class Converter(object):
             index = 0
             for v in json:
                 index += 1
-                if hasattr(v, '__iter__'):
+                if type(v) is list or type(v) is dict:
                     self.resurse_to_flat_dictionary(
                         v, prefix + "." + str(index))
                 else:
diff --git a/docker/storperf-master/storperf/carbon/emitter.py b/docker/storperf-master/storperf/carbon/emitter.py
index b196709..13503b2 100644
--- a/docker/storperf-master/storperf/carbon/emitter.py
+++ b/docker/storperf-master/storperf/carbon/emitter.py
@@ -40,19 +40,19 @@ class CarbonMetricTransmitter():
                     message = "%s %s %s\n" \
                         % (key, value, timestamp)
                     self.logger.debug("Metric: " + message.strip())
-                    carbon_socket.send(message)
+                    carbon_socket.send(message.encode('utf-8'))
                 except ValueError:
                     self.logger.debug("Ignoring non numeric metric %s %s"
                                       % (key, value))
 
             message = "%s.commit-marker %s %s\n" \
                 % (commit_marker, timestamp, timestamp)
-            carbon_socket.send(message)
+            carbon_socket.send(message.encode('utf-8'))
             self.logger.debug("Marker %s" % message.strip())
             self.logger.info("Sent metrics to %s:%s with timestamp %s"
                              % (self.host, self.port, timestamp))
 
-        except Exception, e:
+        except Exception as e:
             self.logger.error("While notifying carbon %s:%s %s"
                               % (self.host, self.port, e))
 
diff --git a/docker/storperf-master/storperf/db/job_db.py b/docker/storperf-master/storperf/db/job_db.py
index b029a35..c3632e4 100644
--- a/docker/storperf-master/storperf/db/job_db.py
+++ b/docker/storperf-master/storperf/db/job_db.py
@@ -220,7 +220,7 @@ class JobDB(object):
 
             db = sqlite3.connect(JobDB.db_name)
             cursor = db.cursor()
-            for param, value in params.iteritems():
+            for param, value in params.items():
                 cursor.execute(
                     """insert into job_params
                                (job_id,
diff --git a/docker/storperf-master/storperf/fio/fio_invoker.py b/docker/storperf-master/storperf/fio/fio_invoker.py
index c665598..2437763 100644
--- a/docker/storperf-master/storperf/fio/fio_invoker.py
+++ b/docker/storperf-master/storperf/fio/fio_invoker.py
@@ -45,6 +45,8 @@ class FIOInvoker(object):
         self.json_body = ""
         try:
             for line in iter(stdout.readline, b''):
+                if type(line) == bytes:
+                    line = line.decode('utf=8')
                 if line.startswith("fio"):
                     line = ""
                     continue
@@ -78,7 +80,8 @@ class FIOInvoker(object):
     def stderr_handler(self, stderr):
         self.logger.debug("Started")
         for line in iter(stderr.readline, b''):
-            self.logger.error("FIO Error: %s", line.rstrip())
+            if len(line) > 0:
+                self.logger.error("FIO Error: %s", line.rstrip())
             self.stderr.append(line.rstrip())
 
             # Sometime, FIO gets stuck and will give us this message:
@@ -137,10 +140,12 @@ class FIOInvoker(object):
 
         ssh = self._ssh_client()
 
-        command = "sudo killall fio"
-
-        self.logger.debug("Executing on %s: %s" % (self.remote_host, command))
-        (_, stdout, stderr) = ssh.exec_command(command)
+        kill_commands = ['sudo killall fio',
+                         'sudo pkill fio']
+        for command in kill_commands:
+            self.logger.debug("Executing on %s: %s" %
+                              (self.remote_host, command))
+            (_, stdout, stderr) = ssh.exec_command(command)
 
         for line in stdout.readlines():
             self.logger.debug(line.strip())
diff --git a/docker/storperf-master/storperf/storperf_master.py b/docker/storperf-master/storperf/storperf_master.py
index 3de8478..afcd018 100644
--- a/docker/storperf-master/storperf/storperf_master.py
+++ b/docker/storperf-master/storperf/storperf_master.py
@@ -7,26 +7,25 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-import StringIO
+
+from _io import StringIO
 from datetime import datetime
-import json
-import logging
 from multiprocessing.pool import ThreadPool
-import os
-import socket
-from time import sleep
-import uuid
-
-import paramiko
 from scp import SCPClient
 from snaps.config.stack import StackConfig
 from snaps.openstack.create_stack import OpenStackHeatStack
 from snaps.openstack.os_credentials import OSCreds
 from snaps.openstack.utils import heat_utils, cinder_utils, glance_utils
 from snaps.thread_utils import worker_pool
-
 from storperf.db.job_db import JobDB
 from storperf.test_executor import TestExecutor
+from time import sleep
+import json
+import logging
+import os
+import paramiko
+import socket
+import uuid
 
 
 class ParameterError(Exception):
diff --git a/docker/storperf-master/storperf/test_executor.py b/docker/storperf-master/storperf/test_executor.py
index 4b5bbd4..cb7e478 100644
--- a/docker/storperf-master/storperf/test_executor.py
+++ b/docker/storperf-master/storperf/test_executor.py
@@ -318,7 +318,7 @@ class TestExecutor(object):
             workload = current_workload['workload']
             self._thread_gate = ThreadGate(
                 len(self.slaves) * min(1, self.volume_count),
-                workload.options['status-interval'])
+                float(workload.options['status-interval']))
 
             self.current_workload = current_workload['name']
 
@@ -362,7 +362,7 @@ class TestExecutor(object):
         workloads = []
 
         if self._custom_workloads:
-            for workload_name in self._custom_workloads.iterkeys():
+            for workload_name in self._custom_workloads.keys():
                 real_name = workload_name
                 if real_name.startswith('_'):
                     real_name = real_name.replace('_', '')
@@ -380,7 +380,7 @@ class TestExecutor(object):
                 workload.id = self.job_db.job_id
 
                 workload_params = self._custom_workloads[workload_name]
-                for param, value in workload_params.iteritems():
+                for param, value in workload_params.items():
                     if param == "readwrite":
                         param = "rw"
                     if param in workload.fixed_options:
diff --git a/docker/storperf-master/storperf/workloads/_base_workload.py b/docker/storperf-master/storperf/workloads/_base_workload.py
index 7468fea..5aa596e 100644
--- a/docker/storperf-master/storperf/workloads/_base_workload.py
+++ b/docker/storperf-master/storperf/workloads/_base_workload.py
@@ -57,7 +57,7 @@ class _base_workload(object):
 
         self.setup()
 
-        for key, value in self.options.iteritems():
+        for key, value in self.options.items():
             if value is not None:
                 args.append('--' + key + "=" + str(value))
             else:
diff --git a/docker/storperf-master/tests/carbon_tests/emitter_test.py b/docker/storperf-master/tests/carbon_tests/emitter_test.py
index f5a78d1..7ea515b 100644
--- a/docker/storperf-master/tests/carbon_tests/emitter_test.py
+++ b/docker/storperf-master/tests/carbon_tests/emitter_test.py
@@ -11,7 +11,7 @@ import json
 from time import strptime
 import unittest
 
-import mock
+from unittest import mock
 
 from storperf.carbon import converter
 from storperf.carbon.emitter import CarbonMetricTransmitter
@@ -69,9 +69,15 @@ class CarbonMetricTransmitterTest(unittest.TestCase):
         emitter.carbon_port = self.listen_port
         emitter.transmit_metrics(result, None)
 
+        element = ""
+        for element in data:
+            element = element.decode('utf-8')
+            if element.startswith("host.run-name"):
+                break
+
         self.assertEqual("host.run-name.key 123.0 975542400\n",
-                         data[1],
-                         data[1])
+                         element,
+                         data)
 
     @mock.patch("socket.socket")
     @mock.patch("time.gmtime")
@@ -90,9 +96,14 @@ class CarbonMetricTransmitterTest(unittest.TestCase):
         emitter.carbon_port = self.listen_port
         emitter.transmit_metrics(result, None)
 
+        element = ""
+        for element in data:
+            element = element.decode('utf-8')
+            if element.startswith("None.commit-marker"):
+                break
         self.assertEqual("None.commit-marker 975542400 975542400\n",
-                         data[1],
-                         data[1])
+                         element,
+                         data)
 
     @mock.patch("socket.socket")
     def test_connect_fails(self, mock_socket):
diff --git a/docker/storperf-master/tests/db_tests/graphite_db_test.py b/docker/storperf-master/tests/db_tests/graphite_db_test.py
index d5fbbfc..2fabfd4 100644
--- a/docker/storperf-master/tests/db_tests/graphite_db_test.py
+++ b/docker/storperf-master/tests/db_tests/graphite_db_test.py
@@ -9,8 +9,7 @@
 
 import unittest
 
-import mock
-
+from unittest import mock
 from storperf.db.graphite_db import GraphiteDB
 
 
diff --git a/docker/storperf-master/tests/db_tests/job_db_test.py b/docker/storperf-master/tests/db_tests/job_db_test.py
index 25fda1f..5201963 100644
--- a/docker/storperf-master/tests/db_tests/job_db_test.py
+++ b/docker/storperf-master/tests/db_tests/job_db_test.py
@@ -11,8 +11,7 @@ import os
 import sqlite3
 import unittest
 
-import mock
-
+from unittest import mock
 from storperf.db.job_db import JobDB
 from storperf.workloads.rr import rr
 
diff --git a/docker/storperf-master/tests/fio_tests/fio_invoker_test.py b/docker/storperf-master/tests/fio_tests/fio_invoker_test.py
index 4672651..3a30500 100644
--- a/docker/storperf-master/tests/fio_tests/fio_invoker_test.py
+++ b/docker/storperf-master/tests/fio_tests/fio_invoker_test.py
@@ -7,11 +7,11 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-from StringIO import StringIO
 import json
 import unittest
 
 from storperf.fio.fio_invoker import FIOInvoker
+from io import BytesIO
 
 
 class Test(unittest.TestCase):
@@ -34,7 +34,7 @@ class Test(unittest.TestCase):
         self.fio_invoker.register(self.event)
         string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
 
-        output = StringIO(string + "\n")
+        output = BytesIO((string + "\n").encode('utf-8'))
         self.fio_invoker.stdout_handler(output)
 
         self.assertEqual(self.simple_dictionary, self.metric)
@@ -43,7 +43,7 @@ class Test(unittest.TestCase):
         self.fio_invoker.register(self.event)
         string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
         terminating = "fio: terminating on signal 2\n"
-        output = StringIO(terminating + string + "\n")
+        output = BytesIO((terminating + string + "\n").encode('utf-8'))
         self.fio_invoker.stdout_handler(output)
 
         self.assertEqual(self.simple_dictionary, self.metric)
@@ -52,7 +52,7 @@ class Test(unittest.TestCase):
         self.fio_invoker.register(self.event)
         string = "{'key': 'value'}"
 
-        output = StringIO(string + "\n")
+        output = BytesIO((string + "\n").encode('utf-8'))
         self.fio_invoker.stdout_handler(output)
 
         self.assertEqual(None, self.metric)
@@ -61,7 +61,7 @@ class Test(unittest.TestCase):
         self.fio_invoker.register(self.event)
         string = "{'key':\n}"
 
-        output = StringIO(string + "\n")
+        output = BytesIO((string + "\n").encode('utf-8'))
         self.fio_invoker.stdout_handler(output)
 
         self.assertEqual(None, self.metric)
@@ -71,7 +71,7 @@ class Test(unittest.TestCase):
         string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
 
         self.fio_invoker.terminated = True
-        output = StringIO(string + "\n")
+        output = BytesIO((string + "\n").encode('utf-8'))
         self.fio_invoker.stdout_handler(output)
 
         self.assertEqual(None, self.metric)
@@ -81,7 +81,7 @@ class Test(unittest.TestCase):
         self.fio_invoker.register(self.event)
         string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
 
-        output = StringIO(string + "\n")
+        output = BytesIO((string + "\n").encode('utf-8'))
         self.fio_invoker.stdout_handler(output)
 
         self.assertEqual(self.simple_dictionary, self.metric)
diff --git a/docker/storperf-master/tests/storperf_master_test.py b/docker/storperf-master/tests/storperf_master_test.py
index 03009d1..1edac6d 100644
--- a/docker/storperf-master/tests/storperf_master_test.py
+++ b/docker/storperf-master/tests/storperf_master_test.py
@@ -9,7 +9,7 @@
 
 import unittest
 
-import mock
+from unittest.mock import patch
 
 from storperf.storperf_master import StorPerfMaster
 
@@ -17,8 +17,8 @@ from storperf.storperf_master import StorPerfMaster
 class StorPerfMasterTest(unittest.TestCase):
 
     def setUp(self):
-        with mock.patch("storperf.storperf_master.OSCreds"), \
-                mock.patch(
+        with patch("storperf.storperf_master.OSCreds"), \
+                patch(
                     "storperf.storperf_master.OpenStackHeatStack") as oshs:
             oshs.return_value.get_stack.return_value = None
 
diff --git a/docker/storperf-master/tests/utilities_tests/data_handler_test.py b/docker/storperf-master/tests/utilities_tests/data_handler_test.py
index 35150dd..7e8cbcc 100644
--- a/docker/storperf-master/tests/utilities_tests/data_handler_test.py
+++ b/docker/storperf-master/tests/utilities_tests/data_handler_test.py
@@ -10,7 +10,7 @@
 import os
 import unittest
 
-import mock
+from unittest import mock
 
 from storperf.utilities.data_handler import DataHandler
 
@@ -311,10 +311,10 @@ class DataHandlerTest(unittest.TestCase):
     def test_pass_criteria(self):
         metadata = {
             "details": {
-              "steady_state": {
-                "_warm_up.queue-depth.8.block-size.16384": False,
-                "rw.queue-depth.4.block-size.16384": True
-              }
+                "steady_state": {
+                    "_warm_up.queue-depth.8.block-size.16384": False,
+                    "rw.queue-depth.4.block-size.16384": True
+                }
             },
         }
         criteria = self.data_handler._determine_criteria(metadata)
@@ -325,11 +325,11 @@ class DataHandlerTest(unittest.TestCase):
     def test_fail_criteria(self):
         metadata = {
             "details": {
-              "steady_state": {
-                "_warm_up.queue-depth.8.block-size.16384": False,
-                "rw.queue-depth.4.block-size.16384": True,
-                "rw.queue-depth.8.block-size.16384": False
-              }
+                "steady_state": {
+                    "_warm_up.queue-depth.8.block-size.16384": False,
+                    "rw.queue-depth.4.block-size.16384": True,
+                    "rw.queue-depth.8.block-size.16384": False
+                }
             },
         }
         criteria = self.data_handler._determine_criteria(metadata)
diff --git a/docker/storperf-reporting/Dockerfile b/docker/storperf-reporting/Dockerfile
index ff28dd1..6d017ae 100644
--- a/docker/storperf-reporting/Dockerfile
+++ b/docker/storperf-reporting/Dockerfile
@@ -16,22 +16,22 @@
 
 
 ARG ARCH=x86_64
-ARG ALPINE_VERSION=v3.6
+ARG ALPINE_VERSION=v3.10
 FROM multiarch/alpine:$ARCH-$ALPINE_VERSION
 MAINTAINER Mark Beierl <mark.beierl@dell.com>
-LABEL version="0.1" description="OPNFV Storperf Reporting Container"
+LABEL version="8.0" description="OPNFV Storperf Reporting Container"
 
 ARG BRANCH=master
 
 RUN ulimit -n 1024
 
-RUN apk add --update python py-pip
+RUN apk add --update python3=3.7.5-r1
 
 COPY . /home/opnfv/storperf-reporting
 WORKDIR /home/opnfv/storperf-reporting/src
 
-RUN pip install -r /home/opnfv/storperf-reporting/requirements.txt
+RUN python3 -m pip install -r /home/opnfv/storperf-reporting/requirements.txt
 
-CMD ["python", "app.py"]
+CMD ["python3", "app.py"]
 
 EXPOSE 5000
diff --git a/docker/storperf-swaggerui/Dockerfile b/docker/storperf-swaggerui/Dockerfile
index 5d58a30..9f82890 100644
--- a/docker/storperf-swaggerui/Dockerfile
+++ b/docker/storperf-swaggerui/Dockerfile
@@ -13,7 +13,7 @@
 ##
 
 ARG ARCH=x86_64
-ARG ALPINE_VERSION=v3.6
+ARG ALPINE_VERSION=v3.10
 FROM node:10-alpine
 
 RUN ulimit -n 1024