Merge "YAML fixes"

4 files changed, 58 insertions, 5 deletions

diff --git a/yardstick/common/task_template.py b/yardstick/common/task_template.py
index 9acc21336..f6c128609 100755
--- a/yardstick/common/task_template.py
+++ b/yardstick/common/task_template.py
@@ -11,6 +11,22 @@ from __future__ import absolute_import
 import re
 import jinja2
 import jinja2.meta
+import yaml
+
+
+def finalize_for_yaml(elem):
+    """Render Jinja2 output specifically for YAML files"""
+    # Jinaj2 by default converts None to 'None', we can't allow this
+    # we could convert to empty string '', or we can convert to null, aka ~
+    if elem is None:
+        return '~'
+    # convert data structures to inline YAML
+    # match builtin types because we shouldn't be trying to render complex types
+    if isinstance(elem, (dict, list)):
+        # remove newlines because we are injecting back into YAML
+        # use block style for single line
+        return yaml.safe_dump(elem, default_flow_style=True).replace('\n', '')
+    return elem
 
 
 class TaskTemplate(object):
@@ -38,7 +54,7 @@ class TaskTemplate(object):
             single_msg = ("Please specify template task argument:%s")
             raise TypeError((len(real_missing) > 1 and multi_msg or single_msg)
                             % ", ".join(real_missing))
-        return jinja2.Template(task_template).render(**kwargs)
+        return jinja2.Template(task_template, finalize=finalize_for_yaml).render(**kwargs)
 
 
 def is_really_missing(mis, task_template):
diff --git a/yardstick/common/template_format.py b/yardstick/common/template_format.py
index 98c0a0b3c..bd5d8376f 100644
--- a/yardstick/common/template_format.py
+++ b/yardstick/common/template_format.py
@@ -18,9 +18,10 @@ import yaml
 from oslo_serialization import jsonutils
 
 if hasattr(yaml, 'CSafeLoader'):
-    yaml_loader = yaml.CSafeLoader
+    # make a dynamic subclass so we don't override global yaml Loader
+    yaml_loader = type('HeatYamlLoader', (yaml.CSafeLoader,), {})
 else:
-    yaml_loader = yaml.SafeLoader
+    yaml_loader = type('HeatYamlLoader', (yaml.SafeLoader,), {})
 
 if hasattr(yaml, 'CSafeDumper'):
     yaml_dumper = yaml.CSafeDumper
@@ -28,10 +29,13 @@ else:
     yaml_dumper = yaml.SafeDumper
 
 
+# This breaks NetworkServiceTestCase yaml loading, because we need to conversion to
+# native Python str() objects because we use use Trex and Trex is has broken unicode handling
 def _construct_yaml_str(self, node):
     # Override the default string handling function
     # to always return unicode objects
     return self.construct_scalar(node)
+
 yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
 # Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type
 # datetime.data which causes problems in API layer when being processed by
diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py
index f2455be3a..c7ae9c1ef 100644
--- a/yardstick/common/utils.py
+++ b/yardstick/common/utils.py
@@ -30,7 +30,6 @@ import random
 import ipaddress
 from contextlib import closing
 
-import yaml
 import six
 from flask import jsonify
 from six.moves import configparser
@@ -38,6 +37,7 @@ from oslo_utils import importutils
 from oslo_serialization import jsonutils
 
 import yardstick
+from yardstick.common.yaml_loader import yaml_load
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -97,7 +97,7 @@ def import_modules_from_package(package):
 def parse_yaml(file_path):
     try:
         with open(file_path) as f:
-            value = yaml.safe_load(f)
+            value = yaml_load(f)
     except IOError:
         return {}
     except OSError as e:
diff --git a/yardstick/common/yaml_loader.py b/yardstick/common/yaml_loader.py
new file mode 100644
index 000000000..0572bd582
--- /dev/null
+++ b/yardstick/common/yaml_loader.py
@@ -0,0 +1,33 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# yardstick: this file is copied from python-heatclient and slightly modified
+
+from __future__ import absolute_import
+
+import yaml
+
+
+if hasattr(yaml, 'CSafeLoader'):
+    # make a dynamic subclass so we don't override global yaml Loader
+    yaml_loader = type('CustomLoader', (yaml.CSafeLoader,), {})
+else:
+    yaml_loader = type('CustomLoader', (yaml.SafeLoader,), {})
+
+if hasattr(yaml, 'CSafeDumper'):
+    yaml_dumper = yaml.CSafeDumper
+else:
+    yaml_dumper = yaml.SafeDumper
+
+
+def yaml_load(tmpl_str):
+    return yaml.load(tmpl_str, Loader=yaml_loader)