Add support for Python 3

Porting to Python3 using Openstack guidelines:
https://wiki.openstack.org/wiki/Python3

This passes unittests on Python 3.5 and passes opnfv_smoke suite

Updates:
   use six for urlparse and urlopen
   fix exception.message attribute removal
   run unittests on python3
   use unitest.mock on python 3
   fix open mock for vsperf
   fix float division by using delta/eplison comparison
   use unicode in StringIO
   use plugin/sample_config.yaml relative path from test case
   fixed apexlake unittests
   upgraded to mock 2.0.0 to match python3 unittest.mock features
   fixed flake8 issues
   implement safe JSON decode with oslo_serialization.jsonutils.dump_as_bytes()
   implement safe unicode encode/decode with oslo_utils.encodeutils

heat: convert pub key file from bytes to unicode
    pkg_resources returns raw bytes, in python3
    we have to decode this to utf-8 unicode
    so JSON can encode it for heat template

JIRA: YARDSTICK-452

Change-Id: Ib80dd1d0c0eb0592acd832b82f6a7f8f7c20bfda
Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>

3 files changed, 76 insertions, 38 deletions

diff --git a/tests/unit/benchmark/scenarios/storage/test_fio.py b/tests/unit/benchmark/scenarios/storage/test_fio.py
index 153d15052..603ff389e 100644
--- a/tests/unit/benchmark/scenarios/storage/test_fio.py
+++ b/tests/unit/benchmark/scenarios/storage/test_fio.py
@@ -11,10 +11,13 @@
 
 # Unittest for yardstick.benchmark.scenarios.storage.fio.Fio
 
-import mock
-import unittest
-import json
+from __future__ import absolute_import
+
 import os
+import unittest
+
+import mock
+from oslo_serialization import jsonutils
 
 from yardstick.benchmark.scenarios.storage import fio
 
@@ -74,7 +77,7 @@ class FioTestCase(unittest.TestCase):
         expected_result = '{"read_bw": 83888, "read_iops": 20972,' \
             '"read_lat": 236.8, "write_bw": 84182, "write_iops": 21045,'\
             '"write_lat": 233.55}'
-        expected_result = json.loads(expected_result)
+        expected_result = jsonutils.loads(expected_result)
         self.assertEqual(result, expected_result)
 
     def test_fio_successful_read_no_sla(self, mock_ssh):
@@ -98,7 +101,7 @@ class FioTestCase(unittest.TestCase):
 
         expected_result = '{"read_bw": 36113, "read_iops": 9028,' \
             '"read_lat": 108.7}'
-        expected_result = json.loads(expected_result)
+        expected_result = jsonutils.loads(expected_result)
         self.assertEqual(result, expected_result)
 
     def test_fio_successful_write_no_sla(self, mock_ssh):
@@ -122,7 +125,7 @@ class FioTestCase(unittest.TestCase):
 
         expected_result = '{"write_bw": 35107, "write_iops": 8776,'\
             '"write_lat": 111.74}'
-        expected_result = json.loads(expected_result)
+        expected_result = jsonutils.loads(expected_result)
         self.assertEqual(result, expected_result)
 
     def test_fio_successful_lat_sla(self, mock_ssh):
@@ -150,10 +153,9 @@ class FioTestCase(unittest.TestCase):
         expected_result = '{"read_bw": 83888, "read_iops": 20972,' \
             '"read_lat": 236.8, "write_bw": 84182, "write_iops": 21045,'\
             '"write_lat": 233.55}'
-        expected_result = json.loads(expected_result)
+        expected_result = jsonutils.loads(expected_result)
         self.assertEqual(result, expected_result)
 
-
     def test_fio_unsuccessful_lat_sla(self, mock_ssh):
 
         options = {
@@ -200,7 +202,7 @@ class FioTestCase(unittest.TestCase):
         expected_result = '{"read_bw": 83888, "read_iops": 20972,' \
             '"read_lat": 236.8, "write_bw": 84182, "write_iops": 21045,'\
             '"write_lat": 233.55}'
-        expected_result = json.loads(expected_result)
+        expected_result = jsonutils.loads(expected_result)
         self.assertEqual(result, expected_result)
 
     def test_fio_unsuccessful_bw_iops_sla(self, mock_ssh):
@@ -248,8 +250,10 @@ class FioTestCase(unittest.TestCase):
             sample_output = f.read()
         return sample_output
 
+
 def main():
     unittest.main()
 
+
 if __name__ == '__main__':
     main()
diff --git a/tests/unit/benchmark/scenarios/storage/test_storagecapacity.py b/tests/unit/benchmark/scenarios/storage/test_storagecapacity.py
index ace0ca374..6fb5f5686 100644
--- a/tests/unit/benchmark/scenarios/storage/test_storagecapacity.py
+++ b/tests/unit/benchmark/scenarios/storage/test_storagecapacity.py
@@ -9,35 +9,41 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-# Unittest for yardstick.benchmark.scenarios.storage.storagecapacity.StorageCapacity
+# Unittest for
+# yardstick.benchmark.scenarios.storage.storagecapacity.StorageCapacity
+
+from __future__ import absolute_import
 
-import mock
 import unittest
-import os
-import json
+
+import mock
+from oslo_serialization import jsonutils
 
 from yardstick.benchmark.scenarios.storage import storagecapacity
 
-DISK_SIZE_SAMPLE_OUTPUT = '{"Numberf of devides": "2", "Total disk size in bytes": "1024000000"}'
+DISK_SIZE_SAMPLE_OUTPUT = \
+    '{"Numberf of devides": "2", "Total disk size in bytes": "1024000000"}'
 BLOCK_SIZE_SAMPLE_OUTPUT = '{"/dev/sda": 1024, "/dev/sdb": 4096}'
 DISK_UTIL_RAW_OUTPUT = "vda 10.00\nvda 0.00"
-DISK_UTIL_SAMPLE_OUTPUT = '{"vda": {"avg_util": 5.0, "max_util": 10.0, "min_util": 0.0}}'
+DISK_UTIL_SAMPLE_OUTPUT = \
+    '{"vda": {"avg_util": 5.0, "max_util": 10.0, "min_util": 0.0}}'
+
 
 @mock.patch('yardstick.benchmark.scenarios.storage.storagecapacity.ssh')
 class StorageCapacityTestCase(unittest.TestCase):
 
     def setUp(self):
         self.scn = {
-               "options": {
-                   'test_type': 'disk_size'
-               }
+            "options": {
+                'test_type': 'disk_size'
+            }
         }
         self.ctx = {
-                "host": {
-                    'ip': '172.16.0.137',
-                    'user': 'cirros',
-                    'password': "root"
-                }
+            "host": {
+                'ip': '172.16.0.137',
+                'user': 'cirros',
+                'password': "root"
+            }
         }
         self.result = {}
 
@@ -54,7 +60,8 @@ class StorageCapacityTestCase(unittest.TestCase):
 
         mock_ssh.SSH().execute.return_value = (0, DISK_SIZE_SAMPLE_OUTPUT, '')
         c.run(self.result)
-        expected_result = json.loads(DISK_SIZE_SAMPLE_OUTPUT)
+        expected_result = jsonutils.loads(
+            DISK_SIZE_SAMPLE_OUTPUT)
         self.assertEqual(self.result, expected_result)
 
     def test_capacity_block_size_successful(self, mock_ssh):
@@ -67,7 +74,8 @@ class StorageCapacityTestCase(unittest.TestCase):
 
         mock_ssh.SSH().execute.return_value = (0, BLOCK_SIZE_SAMPLE_OUTPUT, '')
         c.run(self.result)
-        expected_result = json.loads(BLOCK_SIZE_SAMPLE_OUTPUT)
+        expected_result = jsonutils.loads(
+            BLOCK_SIZE_SAMPLE_OUTPUT)
         self.assertEqual(self.result, expected_result)
 
     def test_capacity_disk_utilization_successful(self, mock_ssh):
@@ -82,7 +90,8 @@ class StorageCapacityTestCase(unittest.TestCase):
 
         mock_ssh.SSH().execute.return_value = (0, DISK_UTIL_RAW_OUTPUT, '')
         c.run(self.result)
-        expected_result = json.loads(DISK_UTIL_SAMPLE_OUTPUT)
+        expected_result = jsonutils.loads(
+            DISK_UTIL_SAMPLE_OUTPUT)
         self.assertEqual(self.result, expected_result)
 
     def test_capacity_unsuccessful_script_error(self, mock_ssh):
@@ -91,6 +100,7 @@ class StorageCapacityTestCase(unittest.TestCase):
         mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR')
         self.assertRaises(RuntimeError, c.run, self.result)
 
+
 def main():
     unittest.main()
 
diff --git a/tests/unit/benchmark/scenarios/storage/test_storperf.py b/tests/unit/benchmark/scenarios/storage/test_storperf.py
index 8fc97d2ed..adc9d47c6 100644
--- a/tests/unit/benchmark/scenarios/storage/test_storperf.py
+++ b/tests/unit/benchmark/scenarios/storage/test_storperf.py
@@ -11,43 +11,58 @@
 
 # Unittest for yardstick.benchmark.scenarios.storage.storperf.StorPerf
 
-import mock
+from __future__ import absolute_import
+
 import unittest
-import requests
-import json
+
+import mock
+from oslo_serialization import jsonutils
 
 from yardstick.benchmark.scenarios.storage import storperf
 
 
 def mocked_requests_config_post(*args, **kwargs):
     class MockResponseConfigPost:
+
         def __init__(self, json_data, status_code):
             self.content = json_data
             self.status_code = status_code
 
-    return MockResponseConfigPost('{"stack_id": "dac27db1-3502-4300-b301-91c64e6a1622","stack_created": "false"}', 200)
+    return MockResponseConfigPost(
+        '{"stack_id": "dac27db1-3502-4300-b301-91c64e6a1622",'
+        '"stack_created": "false"}',
+        200)
 
 
 def mocked_requests_config_get(*args, **kwargs):
     class MockResponseConfigGet:
+
         def __init__(self, json_data, status_code):
             self.content = json_data
             self.status_code = status_code
 
-    return MockResponseConfigGet('{"stack_id": "dac27db1-3502-4300-b301-91c64e6a1622","stack_created": "true"}', 200)
+    return MockResponseConfigGet(
+        '{"stack_id": "dac27db1-3502-4300-b301-91c64e6a1622",'
+        '"stack_created": "true"}',
+        200)
 
 
 def mocked_requests_job_get(*args, **kwargs):
     class MockResponseJobGet:
+
         def __init__(self, json_data, status_code):
             self.content = json_data
             self.status_code = status_code
 
-    return MockResponseJobGet('{"status": "completed", "_ssd_preconditioning.queue-depth.8.block-size.16384.duration": 6}', 200)
+    return MockResponseJobGet(
+        '{"status": "completed",\
+         "_ssd_preconditioning.queue-depth.8.block-size.16384.duration": 6}',
+        200)
 
 
 def mocked_requests_job_post(*args, **kwargs):
     class MockResponseJobPost:
+
         def __init__(self, json_data, status_code):
             self.content = json_data
             self.status_code = status_code
@@ -58,6 +73,7 @@ def mocked_requests_job_post(*args, **kwargs):
 
 def mocked_requests_job_delete(*args, **kwargs):
     class MockResponseJobDelete:
+
         def __init__(self, json_data, status_code):
             self.content = json_data
             self.status_code = status_code
@@ -67,6 +83,7 @@ def mocked_requests_job_delete(*args, **kwargs):
 
 def mocked_requests_delete(*args, **kwargs):
     class MockResponseDelete:
+
         def __init__(self, json_data, status_code):
             self.json_data = json_data
             self.status_code = status_code
@@ -76,6 +93,7 @@ def mocked_requests_delete(*args, **kwargs):
 
 def mocked_requests_delete_failed(*args, **kwargs):
     class MockResponseDeleteFailed:
+
         def __init__(self, json_data, status_code):
             self.json_data = json_data
             self.status_code = status_code
@@ -130,8 +148,9 @@ class StorPerfTestCase(unittest.TestCase):
                 side_effect=mocked_requests_job_post)
     @mock.patch('yardstick.benchmark.scenarios.storage.storperf.requests.get',
                 side_effect=mocked_requests_job_get)
-    @mock.patch('yardstick.benchmark.scenarios.storage.storperf.requests.delete',
-                side_effect=mocked_requests_job_delete)
+    @mock.patch(
+        'yardstick.benchmark.scenarios.storage.storperf.requests.delete',
+        side_effect=mocked_requests_job_delete)
     def test_successful_run(self, mock_post, mock_get, mock_delete):
         options = {
             "agent_count": 8,
@@ -152,15 +171,18 @@ class StorPerfTestCase(unittest.TestCase):
         s = storperf.StorPerf(args, self.ctx)
         s.setup_done = True
 
-        sample_output = '{"status": "completed", "_ssd_preconditioning.queue-depth.8.block-size.16384.duration": 6}'
+        sample_output = '{"status": "completed",\
+         "_ssd_preconditioning.queue-depth.8.block-size.16384.duration": 6}'
 
-        expected_result = json.loads(sample_output)
+        expected_result = jsonutils.loads(sample_output)
 
         s.run(self.result)
 
         self.assertEqual(self.result, expected_result)
 
-    @mock.patch('yardstick.benchmark.scenarios.storage.storperf.requests.delete', side_effect=mocked_requests_delete)
+    @mock.patch(
+        'yardstick.benchmark.scenarios.storage.storperf.requests.delete',
+        side_effect=mocked_requests_delete)
     def test_successful_teardown(self, mock_delete):
         options = {
             "agent_count": 8,
@@ -184,7 +206,9 @@ class StorPerfTestCase(unittest.TestCase):
 
         self.assertFalse(s.setup_done)
 
-    @mock.patch('yardstick.benchmark.scenarios.storage.storperf.requests.delete', side_effect=mocked_requests_delete_failed)
+    @mock.patch(
+        'yardstick.benchmark.scenarios.storage.storperf.requests.delete',
+        side_effect=mocked_requests_delete_failed)
     def test_failed_teardown(self, mock_delete):
         options = {
             "agent_count": 8,