Add support for Python 3

Porting to Python3 using Openstack guidelines:
https://wiki.openstack.org/wiki/Python3

This passes unittests on Python 3.5 and passes opnfv_smoke suite

Updates:
   use six for urlparse and urlopen
   fix exception.message attribute removal
   run unittests on python3
   use unitest.mock on python 3
   fix open mock for vsperf
   fix float division by using delta/eplison comparison
   use unicode in StringIO
   use plugin/sample_config.yaml relative path from test case
   fixed apexlake unittests
   upgraded to mock 2.0.0 to match python3 unittest.mock features
   fixed flake8 issues
   implement safe JSON decode with oslo_serialization.jsonutils.dump_as_bytes()
   implement safe unicode encode/decode with oslo_utils.encodeutils

heat: convert pub key file from bytes to unicode
    pkg_resources returns raw bytes, in python3
    we have to decode this to utf-8 unicode
    so JSON can encode it for heat template

JIRA: YARDSTICK-452

Change-Id: Ib80dd1d0c0eb0592acd832b82f6a7f8f7c20bfda
Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>

10 files changed, 74 insertions, 38 deletions

diff --git a/yardstick/benchmark/scenarios/compute/cachestat.py b/yardstick/benchmark/scenarios/compute/cachestat.py
index 20786ff61..0f60d466e 100644
--- a/yardstick/benchmark/scenarios/compute/cachestat.py
+++ b/yardstick/benchmark/scenarios/compute/cachestat.py
@@ -9,12 +9,14 @@
 
 """cache hit/miss ratio and usage statistics"""
 
+from __future__ import absolute_import
 import pkg_resources
 import logging
 import re
 import yardstick.ssh as ssh
 
 from yardstick.benchmark.scenarios import base
+from six.moves import zip
 
 LOG = logging.getLogger(__name__)
 
@@ -120,7 +122,7 @@ class CACHEstat(base.Scenario):
                 ite += 1
                 values = line[:]
                 if values and len(values) == len(fields):
-                    cachestat[cache] = dict(zip(fields, values))
+                    cachestat[cache] = dict(list(zip(fields, values)))
 
         for entry in cachestat:
             for item in average:
diff --git a/yardstick/benchmark/scenarios/compute/computecapacity.py b/yardstick/benchmark/scenarios/compute/computecapacity.py
index 7f0c58de1..9d518f7a0 100644
--- a/yardstick/benchmark/scenarios/compute/computecapacity.py
+++ b/yardstick/benchmark/scenarios/compute/computecapacity.py
@@ -6,9 +6,12 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
-import pkg_resources
+from __future__ import absolute_import
+
 import logging
-import json
+
+import pkg_resources
+from oslo_serialization import jsonutils
 
 import yardstick.ssh as ssh
 from yardstick.benchmark.scenarios import base
@@ -66,4 +69,4 @@ class ComputeCapacity(base.Scenario):
         if status:
             raise RuntimeError(stderr)
 
-        result.update(json.loads(stdout))
+        result.update(jsonutils.loads(stdout))
diff --git a/yardstick/benchmark/scenarios/compute/cpuload.py b/yardstick/benchmark/scenarios/compute/cpuload.py
index 9d71038ef..121d5a75d 100644
--- a/yardstick/benchmark/scenarios/compute/cpuload.py
+++ b/yardstick/benchmark/scenarios/compute/cpuload.py
@@ -9,13 +9,16 @@
 
 """Processor statistics and system load."""
 
+from __future__ import absolute_import
+
 import logging
-import time
 import re
-import yardstick.ssh as ssh
+import time
 
-from yardstick.benchmark.scenarios import base
+from six.moves import map, zip
 
+import yardstick.ssh as ssh
+from yardstick.benchmark.scenarios import base
 
 LOG = logging.getLogger(__name__)
 
@@ -145,7 +148,7 @@ class CPULoad(base.Scenario):
                     cpu = 'cpu' if line[0] == 'all' else 'cpu' + line[0]
                     values = line[1:]
                     if values and len(values) == len(fields):
-                        temp_dict = dict(zip(fields, values))
+                        temp_dict = dict(list(zip(fields, values)))
                         if cpu not in maximum:
                             maximum[cpu] = temp_dict
                         else:
@@ -177,7 +180,7 @@ class CPULoad(base.Scenario):
                     cpu = 'cpu' if line[0] == 'all' else 'cpu' + line[0]
                     values = line[1:]
                     if values and len(values) == len(fields):
-                        average[cpu] = dict(zip(fields, values))
+                        average[cpu] = dict(list(zip(fields, values)))
                     else:
                         raise RuntimeError("mpstat average: parse error",
                                            fields, line)
@@ -210,9 +213,9 @@ class CPULoad(base.Scenario):
 
             cpu = cur_list[0]
 
-            cur_stats = map(int, cur_list[1:])
+            cur_stats = list(map(int, cur_list[1:]))
             if self.interval > 0:
-                prev_stats = map(int, prev_list[1:])
+                prev_stats = list(map(int, prev_list[1:]))
             else:
                 prev_stats = [0] * len(cur_stats)
 
@@ -236,9 +239,9 @@ class CPULoad(base.Scenario):
                 else:
                     return "%.2f" % (100.0 * (x - y) / samples)
 
-            load = map(_percent, cur_stats, prev_stats)
+            load = list(map(_percent, cur_stats, prev_stats))
 
-            mpstat[cpu] = dict(zip(fields, load))
+            mpstat[cpu] = dict(list(zip(fields, load)))
 
         return {'mpstat': mpstat}
 
@@ -278,7 +281,7 @@ class CPULoad(base.Scenario):
 #     p = CPULoad(args, ctx)
 #     p.run(result)
 #     import json
-#     print json.dumps(result)
+#     print(oslo_serialization.jsonutils.dump_as_bytes(result))
 
 # if __name__ == '__main__':
 #     _test()
diff --git a/yardstick/benchmark/scenarios/compute/cyclictest.py b/yardstick/benchmark/scenarios/compute/cyclictest.py
index 568e6e7df..76bafff2b 100644
--- a/yardstick/benchmark/scenarios/compute/cyclictest.py
+++ b/yardstick/benchmark/scenarios/compute/cyclictest.py
@@ -6,12 +6,16 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
-import pkg_resources
+from __future__ import absolute_import
+from __future__ import print_function
+
 import logging
-import json
+import os
 import re
 import time
-import os
+
+import pkg_resources
+from oslo_serialization import jsonutils
 
 import yardstick.ssh as ssh
 from yardstick.benchmark.scenarios import base
@@ -183,7 +187,7 @@ class Cyclictest(base.Scenario):
         if status:
             raise RuntimeError(stderr)
 
-        result.update(json.loads(stdout))
+        result.update(jsonutils.loads(stdout))
 
         if "sla" in self.scenario_cfg:
             sla_error = ""
@@ -236,7 +240,8 @@ def _test():    # pragma: no cover
 
     cyclictest = Cyclictest(args, ctx)
     cyclictest.run(result)
-    print result
+    print(result)
+
 
 if __name__ == '__main__':    # pragma: no cover
     _test()
diff --git a/yardstick/benchmark/scenarios/compute/lmbench.py b/yardstick/benchmark/scenarios/compute/lmbench.py
index 518840c09..6a17ae8a1 100644
--- a/yardstick/benchmark/scenarios/compute/lmbench.py
+++ b/yardstick/benchmark/scenarios/compute/lmbench.py
@@ -6,9 +6,13 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
-import pkg_resources
+from __future__ import absolute_import
+from __future__ import print_function
+
 import logging
-import json
+
+import pkg_resources
+from oslo_serialization import jsonutils
 
 import yardstick.ssh as ssh
 from yardstick.benchmark.scenarios import base
@@ -130,9 +134,10 @@ class Lmbench(base.Scenario):
             raise RuntimeError(stderr)
 
         if test_type == 'latency':
-            result.update({"latencies": json.loads(stdout)})
+            result.update(
+                {"latencies": jsonutils.loads(stdout)})
         else:
-            result.update(json.loads(stdout))
+            result.update(jsonutils.loads(stdout))
 
         if "sla" in self.scenario_cfg:
             sla_error = ""
@@ -185,7 +190,8 @@ def _test():
 
     p = Lmbench(args, ctx)
     p.run(result)
-    print result
+    print(result)
+
 
 if __name__ == '__main__':
     _test()
diff --git a/yardstick/benchmark/scenarios/compute/memload.py b/yardstick/benchmark/scenarios/compute/memload.py
index e1ba93d02..35528d4ef 100644
--- a/yardstick/benchmark/scenarios/compute/memload.py
+++ b/yardstick/benchmark/scenarios/compute/memload.py
@@ -9,10 +9,12 @@
 
 """Memory load and statistics."""
 
+from __future__ import absolute_import
 import logging
 import yardstick.ssh as ssh
 
 from yardstick.benchmark.scenarios import base
+from six.moves import zip
 
 LOG = logging.getLogger(__name__)
 
@@ -88,7 +90,7 @@ class MEMLoad(base.Scenario):
                 ite += 1
                 values = line[1:]
                 if values and len(values) == len(fields):
-                    free[memory] = dict(zip(fields, values))
+                    free[memory] = dict(list(zip(fields, values)))
 
         for entry in free:
             for item in average:
diff --git a/yardstick/benchmark/scenarios/compute/perf.py b/yardstick/benchmark/scenarios/compute/perf.py
index 8f1a4d630..ae4990688 100644
--- a/yardstick/benchmark/scenarios/compute/perf.py
+++ b/yardstick/benchmark/scenarios/compute/perf.py
@@ -6,9 +6,13 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
-import pkg_resources
+from __future__ import absolute_import
+from __future__ import print_function
+
 import logging
-import json
+
+import pkg_resources
+from oslo_serialization import jsonutils
 
 import yardstick.ssh as ssh
 from yardstick.benchmark.scenarios import base
@@ -100,7 +104,7 @@ class Perf(base.Scenario):
         if status:
             raise RuntimeError(stdout)
 
-        result.update(json.loads(stdout))
+        result.update(jsonutils.loads(stdout))
 
         if "sla" in self.scenario_cfg:
             metric = self.scenario_cfg['sla']['metric']
@@ -140,7 +144,8 @@ def _test():
 
     p = Perf(args, ctx)
     p.run(result)
-    print result
+    print(result)
+
 
 if __name__ == '__main__':
     _test()
diff --git a/yardstick/benchmark/scenarios/compute/plugintest.py b/yardstick/benchmark/scenarios/compute/plugintest.py
index e7ec91c5c..c9d025964 100644
--- a/yardstick/benchmark/scenarios/compute/plugintest.py
+++ b/yardstick/benchmark/scenarios/compute/plugintest.py
@@ -6,8 +6,11 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
+from __future__ import absolute_import
+
 import logging
-import json
+
+from oslo_serialization import jsonutils
 
 import yardstick.ssh as ssh
 from yardstick.benchmark.scenarios import base
@@ -53,4 +56,4 @@ class PluginTest(base.Scenario):
         if status:
             raise RuntimeError(stderr)
 
-        result.update(json.loads(stdout))
+        result.update(jsonutils.loads(stdout))
diff --git a/yardstick/benchmark/scenarios/compute/ramspeed.py b/yardstick/benchmark/scenarios/compute/ramspeed.py
index db70af90b..4330202de 100644
--- a/yardstick/benchmark/scenarios/compute/ramspeed.py
+++ b/yardstick/benchmark/scenarios/compute/ramspeed.py
@@ -6,9 +6,12 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
-import pkg_resources
+from __future__ import absolute_import
+
 import logging
-import json
+
+import pkg_resources
+from oslo_serialization import jsonutils
 
 import yardstick.ssh as ssh
 from yardstick.benchmark.scenarios import base
@@ -131,7 +134,7 @@ class Ramspeed(base.Scenario):
         if status:
             raise RuntimeError(stderr)
 
-        result.update(json.loads(stdout))
+        result.update(jsonutils.loads(stdout))
 
         if "sla" in self.scenario_cfg:
             sla_error = ""
diff --git a/yardstick/benchmark/scenarios/compute/unixbench.py b/yardstick/benchmark/scenarios/compute/unixbench.py
index b22be29c9..4a2eb9766 100644
--- a/yardstick/benchmark/scenarios/compute/unixbench.py
+++ b/yardstick/benchmark/scenarios/compute/unixbench.py
@@ -6,9 +6,13 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
-import pkg_resources
+from __future__ import absolute_import
+from __future__ import print_function
+
 import logging
-import json
+
+import pkg_resources
+from oslo_serialization import jsonutils
 
 import yardstick.ssh as ssh
 from yardstick.benchmark.scenarios import base
@@ -115,7 +119,7 @@ class Unixbench(base.Scenario):
         if status:
             raise RuntimeError(stderr)
 
-        result.update(json.loads(stdout))
+        result.update(jsonutils.loads(stdout))
 
         if "sla" in self.scenario_cfg:
             sla_error = ""
@@ -152,7 +156,7 @@ def _test():  # pragma: no cover
 
     p = Unixbench(args, ctx)
     p.run(result)
-    print result
+    print(result)
 
 
 if __name__ == '__main__':