Merge "YAML fixes"

23 files changed, 290 insertions, 148 deletions

diff --git a/api/resources/v1/env.py b/api/resources/v1/env.py
index 8367fa9eb..98b8ec7e4 100644
--- a/api/resources/v1/env.py
+++ b/api/resources/v1/env.py
@@ -31,7 +31,7 @@ from yardstick.common import utils
 from yardstick.common.utils import result_handler
 from yardstick.common import openstack_utils
 from yardstick.common.httpClient import HttpClient
-
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.DEBUG)
@@ -393,7 +393,7 @@ class V1Env(ApiResource):
             return result_handler(consts.API_ERROR, 'file must be provided')
 
         LOG.info('Checking file')
-        data = yaml.safe_load(pod_file.read())
+        data = yaml_load(pod_file.read())
         if not isinstance(data, collections.Mapping):
             return result_handler(consts.API_ERROR, 'invalid yaml file')
 
diff --git a/api/resources/v2/pods.py b/api/resources/v2/pods.py
index f2316d353..d98238ca1 100644
--- a/api/resources/v2/pods.py
+++ b/api/resources/v2/pods.py
@@ -18,6 +18,7 @@ from api.database.v2.handlers import V2EnvironmentHandler
 from yardstick.common import constants as consts
 from yardstick.common.utils import result_handler
 from yardstick.common.task_template import TaskTemplate
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 LOG.setLevel(logging.DEBUG)
@@ -48,7 +49,7 @@ class V2Pods(ApiResource):
         upload_file.save(consts.POD_FILE)
 
         with open(consts.POD_FILE) as f:
-            data = yaml.safe_load(TaskTemplate.render(f.read()))
+            data = yaml_load(TaskTemplate.render(f.read()))
         LOG.debug('pod content is: %s', data)
 
         LOG.info('create pod in database')
diff --git a/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py b/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
index e7ba3ca73..680692fdc 100644
--- a/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
+++ b/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
@@ -18,7 +18,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
 
     @mock.patch('{}.GetNumaInfo._check_numa_node'.format(BASE))
     @mock.patch('{}.GetNumaInfo._get_current_host_name'.format(BASE))
-    @mock.patch('yaml.safe_load')
+    @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
     @mock.patch('yardstick.common.task_template.TaskTemplate.render')
     def test_get_numa_info(self,
                            mock_render,
@@ -44,7 +44,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
 
     @mock.patch('yardstick.ssh.SSH.from_node')
     @mock.patch('{}.GetNumaInfo._get_current_host_name'.format(BASE))
-    @mock.patch('yaml.safe_load')
+    @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
     @mock.patch('yardstick.common.task_template.TaskTemplate.render')
     def test_check_numa_node(self,
                              mock_render,
@@ -74,7 +74,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
 
     @mock.patch('{}.change_obj_to_dict'.format(BASE))
     @mock.patch('{}.get_nova_client'.format(BASE))
-    @mock.patch('yaml.safe_load')
+    @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
     @mock.patch('yardstick.common.task_template.TaskTemplate.render')
     def test_get_current_host_name(self,
                                    mock_render,
diff --git a/tests/unit/common/test_yaml_loader.py b/tests/unit/common/test_yaml_loader.py
new file mode 100644
index 000000000..90cbb8157
--- /dev/null
+++ b/tests/unit/common/test_yaml_loader.py
@@ -0,0 +1,32 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# yardstick: this file is copied from python-heatclient and slightly modified
+
+from __future__ import absolute_import
+import unittest
+
+from yardstick.common import yaml_loader
+
+
+class TemplateFormatTestCase(unittest.TestCase):
+
+    def test_parse_to_value_exception(self):
+
+        self.assertEquals(yaml_loader.yaml_load("string"), u"string")
+
+
+def main():
+    unittest.main()
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/unit/network_services/test_yang_model.py b/tests/unit/network_services/test_yang_model.py
index 28367f316..0b29da701 100644
--- a/tests/unit/network_services/test_yang_model.py
+++ b/tests/unit/network_services/test_yang_model.py
@@ -95,7 +95,7 @@ class YangModelTestCase(unittest.TestCase):
         y._get_entries()
         self.assertEqual(y._rules, '')
 
-    @mock.patch('yaml.safe_load')
+    @mock.patch('yardstick.network_services.yang_model.yaml_load')
     @mock.patch('yardstick.network_services.yang_model.open')
     def test__read_config(self, mock_open, mock_safe_load):
         cfg = "yang.yaml"
diff --git a/tests/unit/network_services/vnf_generic/test_vnfdgen.py b/tests/unit/network_services/vnf_generic/test_vnfdgen.py
index 44d9058dc..c2b923568 100644
--- a/tests/unit/network_services/vnf_generic/test_vnfdgen.py
+++ b/tests/unit/network_services/vnf_generic/test_vnfdgen.py
@@ -21,6 +21,7 @@ from __future__ import absolute_import
 import unittest
 from six.moves import range
 
+from yardstick.common.yaml_loader import yaml_load
 from yardstick.network_services.vnf_generic import vnfdgen
 
 TREX_VNFD_TEMPLATE = """
@@ -65,6 +66,8 @@ vnfd:vnfd-catalog:
                     dst_mac: '{{ interfaces.xe1.dst_mac }}'
                     bandwidth: 10 Gbps
                 vnfd-connection-point-ref: xe1
+            routing_table: {{ routing_table }}
+            nd_route_tbl: {{ nd_route_tbl }}
 
         benchmark:
             kpi:
@@ -126,6 +129,22 @@ COMPLETE_TREX_VNFD = \
                                          'vpci': '0000:00:10.1'},
                    'vnfd-connection-point-ref': 'xe1'}],
                  'id': 'trexgen-baremetal',
+                 'nd_route_tbl': [{'gateway': '0064:ff9b:0:0:0:0:9810:6414',
+                                   'if': 'xe0',
+                                   'netmask': '112',
+                                   'network': '0064:ff9b:0:0:0:0:9810:6414'},
+                                  {'gateway': '0064:ff9b:0:0:0:0:9810:2814',
+                                   'if': 'xe1',
+                                   'netmask': '112',
+                                   'network': '0064:ff9b:0:0:0:0:9810:2814'}],
+                 'routing_table': [{'gateway': '152.16.100.20',
+                                    'if': 'xe0',
+                                    'netmask': '255.255.255.0',
+                                    'network': '152.16.100.20'},
+                                   {'gateway': '152.16.40.20',
+                                    'if': 'xe1',
+                                    'netmask': '255.255.255.0',
+                                    'network': '152.16.40.20'}],
                  'name': 'trexgen-baremetal'}]}]}}
 
 NODE_CFG = {'ip': '1.1.1.1',
@@ -144,7 +163,24 @@ NODE_CFG = {'ip': '1.1.1.1',
                                    'dst_mac': '00:01:02:03:04:06',
                                    'local_ip': '2.1.1.2',
                                    'local_mac': '00:01:02:03:05:06',
-                                   'vpci': '0000:00:10.1'}}}
+                                   'vpci': '0000:00:10.1'}},
+            'nd_route_tbl': [{u'gateway': u'0064:ff9b:0:0:0:0:9810:6414',
+                              u'if': u'xe0',
+                              u'netmask': u'112',
+                              u'network': u'0064:ff9b:0:0:0:0:9810:6414'},
+                             {u'gateway': u'0064:ff9b:0:0:0:0:9810:2814',
+                              u'if': u'xe1',
+                              u'netmask': u'112',
+                              u'network': u'0064:ff9b:0:0:0:0:9810:2814'}],
+            'routing_table': [{u'gateway': u'152.16.100.20',
+                               u'if': u'xe0',
+                               u'netmask': u'255.255.255.0',
+                               u'network': u'152.16.100.20'},
+                              {u'gateway': u'152.16.40.20',
+                               u'if': u'xe1',
+                               u'netmask': u'255.255.255.0',
+                               u'network': u'152.16.40.20'}],
+            }
 
 
 TRAFFIC_PROFILE_TPL = """
@@ -169,6 +205,20 @@ TRAFFIC_PROFILE = {
                                          "1518B": '40'}}}}]}
 
 
+class TestRender(unittest.TestCase):
+
+    def test_render_none(self):
+
+        tmpl = "{{ routing_table }}"
+        self.assertEqual(vnfdgen.render(tmpl, routing_table=None), u'~')
+        self.assertEqual(yaml_load(vnfdgen.render(tmpl, routing_table=None)), None)
+
+    def test_render_unicode_dict(self):
+
+        tmpl = "{{ routing_table }}"
+        self.assertEqual(yaml_load(vnfdgen.render(tmpl, **NODE_CFG)), NODE_CFG["routing_table"])
+
+
 class TestVnfdGen(unittest.TestCase):
     """ Class to verify VNFS testcases """
 
diff --git a/yardstick/benchmark/contexts/node.py b/yardstick/benchmark/contexts/node.py
index 35c64335d..250032efc 100644
--- a/yardstick/benchmark/contexts/node.py
+++ b/yardstick/benchmark/contexts/node.py
@@ -17,12 +17,12 @@ import tempfile
 
 import six
 import pkg_resources
-import yaml
 
 from yardstick import ssh
 from yardstick.benchmark.contexts.base import Context
 from yardstick.common.constants import ANSIBLE_DIR, YARDSTICK_ROOT_PATH
 from yardstick.common.ansible_common import AnsibleCommon
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 
@@ -55,7 +55,7 @@ class NodeContext(Context):
 
         with open(self.file_path) as stream:
             LOG.info("Parsing pod file: %s", self.file_path)
-            cfg = yaml.safe_load(stream)
+            cfg = yaml_load(stream)
         return cfg
 
     def init(self, attrs):
diff --git a/yardstick/benchmark/contexts/standalone.py b/yardstick/benchmark/contexts/standalone.py
index ae1046974..f0ef1d560 100644
--- a/yardstick/benchmark/contexts/standalone.py
+++ b/yardstick/benchmark/contexts/standalone.py
@@ -18,12 +18,12 @@ import logging
 import os
 import errno
 import collections
-import yaml
 import time
 
 from yardstick.benchmark.contexts.base import Context
 from yardstick.common.constants import YARDSTICK_ROOT_PATH
 from yardstick.common.utils import import_modules_from_package, itersubclasses
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 
@@ -50,7 +50,7 @@ class StandaloneContext(Context):
 
         with open(self.file_path) as stream:
             LOG.info("Parsing pod file: %s", self.file_path)
-            cfg = yaml.safe_load(stream)
+            cfg = yaml_load(stream)
         return cfg
 
     def get_nfvi_obj(self):
diff --git a/yardstick/benchmark/core/plugin.py b/yardstick/benchmark/core/plugin.py
index a741d5e74..24f1b6b25 100644
--- a/yardstick/benchmark/core/plugin.py
+++ b/yardstick/benchmark/core/plugin.py
@@ -13,13 +13,13 @@ from __future__ import print_function
 from __future__ import absolute_import
 import os
 import sys
-import yaml
 import time
 import logging
 import pkg_resources
 import yardstick.ssh as ssh
 
 from yardstick.common.task_template import TaskTemplate
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 
@@ -153,7 +153,7 @@ class PluginParser(object):
                     raise e
                 print("Input plugin is:\n%s\n" % rendered_plugin)
 
-                cfg = yaml.safe_load(rendered_plugin)
+                cfg = yaml_load(rendered_plugin)
         except IOError as ioerror:
             sys.exit(ioerror)
 
diff --git a/yardstick/benchmark/core/task.py b/yardstick/benchmark/core/task.py
index ea3239edf..f689f2e63 100644
--- a/yardstick/benchmark/core/task.py
+++ b/yardstick/benchmark/core/task.py
@@ -29,6 +29,7 @@ from jinja2 import Environment
 
 from yardstick.benchmark.contexts.base import Context
 from yardstick.benchmark.runners import base as base_runner
+from yardstick.common.yaml_loader import yaml_load
 from yardstick.dispatcher.base import Base as DispatcherBase
 from yardstick.common.task_template import TaskTemplate
 from yardstick.common.utils import source_env
@@ -437,7 +438,7 @@ class TaskParser(object):       # pragma: no cover
 
         try:
             with open(self.path) as stream:
-                cfg = yaml.load(stream)
+                cfg = yaml_load(stream)
         except IOError as ioerror:
             sys.exit(ioerror)
 
@@ -501,7 +502,7 @@ class TaskParser(object):       # pragma: no cover
                     raise e
                 print("Input task is:\n%s\n" % rendered_task)
 
-                cfg = yaml.load(rendered_task)
+                cfg = yaml_load(rendered_task)
         except IOError as ioerror:
             sys.exit(ioerror)
 
@@ -657,7 +658,7 @@ def parse_task_args(src_name, args):
         return args
 
     try:
-        kw = args and yaml.safe_load(args)
+        kw = args and yaml_load(args)
         kw = {} if kw is None else kw
     except yaml.parser.ParserError as e:
         print_invalid_header(src_name, args)
diff --git a/yardstick/benchmark/core/testcase.py b/yardstick/benchmark/core/testcase.py
index 7ab1b08cf..501356726 100644
--- a/yardstick/benchmark/core/testcase.py
+++ b/yardstick/benchmark/core/testcase.py
@@ -12,11 +12,11 @@ from __future__ import absolute_import
 from __future__ import print_function
 
 import os
-import yaml
 import logging
 
 from yardstick.common.task_template import TaskTemplate
 from yardstick.common import constants as consts
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 
@@ -69,7 +69,7 @@ class Testcase(object):
     def _parse_testcase(self, testcase_info):
 
         rendered_testcase = TaskTemplate.render(testcase_info)
-        testcase_cfg = yaml.safe_load(rendered_testcase)
+        testcase_cfg = yaml_load(rendered_testcase)
 
         test_precondition = testcase_cfg.get('precondition', {})
         installer_type = test_precondition.get('installer_type', 'all')
diff --git a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py
index a20b26396..61698da43 100644
--- a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py
+++ b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py
@@ -8,11 +8,11 @@
 ##############################################################################
 from __future__ import absolute_import
 import pkg_resources
-import yaml
 import logging
 import os
 
 import yardstick.common.utils as utils
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 
@@ -56,7 +56,7 @@ class BaseAttacker(object):
     def __init__(self, config, context):
         if not BaseAttacker.attacker_cfgs:
             with open(attacker_conf_path) as stream:
-                BaseAttacker.attacker_cfgs = yaml.safe_load(stream)
+                BaseAttacker.attacker_cfgs = yaml_load(stream)
 
         self._config = config
         self._context = context
diff --git a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
index 6165aba74..0027925d6 100644
--- a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
+++ b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
@@ -13,7 +13,8 @@ import multiprocessing
 import time
 import os
 import yardstick.common.utils as utils
-import yaml
+
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 
@@ -74,7 +75,7 @@ class BaseMonitor(multiprocessing.Process):
     def __init__(self, config, context, data):
         if not BaseMonitor.monitor_cfgs:
             with open(monitor_conf_path) as stream:
-                BaseMonitor.monitor_cfgs = yaml.safe_load(stream)
+                BaseMonitor.monitor_cfgs = yaml_load(stream)
         multiprocessing.Process.__init__(self)
         self._config = config
         self._context = context
diff --git a/yardstick/benchmark/scenarios/availability/operation/baseoperation.py b/yardstick/benchmark/scenarios/availability/operation/baseoperation.py
index 4c2ce82d9..d21b857b5 100644
--- a/yardstick/benchmark/scenarios/availability/operation/baseoperation.py
+++ b/yardstick/benchmark/scenarios/availability/operation/baseoperation.py
@@ -8,11 +8,11 @@
 ##############################################################################
 from __future__ import absolute_import
 import pkg_resources
-import yaml
 import logging
 import os
 
 import yardstick.common.utils as utils
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 
@@ -54,7 +54,7 @@ class BaseOperation(object):
     def __init__(self, config, context):
         if not BaseOperation.operation_cfgs:
             with open(operation_conf_path) as stream:
-                BaseOperation.operation_cfgs = yaml.safe_load(stream)
+                BaseOperation.operation_cfgs = yaml_load(stream)
         self.key = ''
         self._config = config
         self._context = context
diff --git a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py
index ce34d8be0..05b660105 100644
--- a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py
+++ b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py
@@ -8,11 +8,11 @@
 ##############################################################################
 from __future__ import absolute_import
 import pkg_resources
-import yaml
 import logging
 import os
 
 import yardstick.common.utils as utils
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 
@@ -58,7 +58,7 @@ class BaseResultChecker(object):
     def __init__(self, config, context):
         if not BaseResultChecker.resultchecker_cfgs:
             with open(resultchecker_conf_path) as stream:
-                BaseResultChecker.resultchecker_cfgs = yaml.safe_load(stream)
+                BaseResultChecker.resultchecker_cfgs = yaml_load(stream)
         self.actualResult = object()
         self.expectedResult = object()
         self.success = False
diff --git a/yardstick/benchmark/scenarios/lib/get_numa_info.py b/yardstick/benchmark/scenarios/lib/get_numa_info.py
index 4e4a44d95..75a9e3506 100644
--- a/yardstick/benchmark/scenarios/lib/get_numa_info.py
+++ b/yardstick/benchmark/scenarios/lib/get_numa_info.py
@@ -13,7 +13,6 @@ from __future__ import absolute_import
 import logging
 import os
 
-import yaml
 from xml.etree import ElementTree as ET
 
 from yardstick import ssh
@@ -22,6 +21,7 @@ from yardstick.common import constants as consts
 from yardstick.common.utils import change_obj_to_dict
 from yardstick.common.openstack_utils import get_nova_client
 from yardstick.common.task_template import TaskTemplate
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 
@@ -47,7 +47,7 @@ class GetNumaInfo(base.Scenario):
                                  self.options.get('file'))
 
         with open(node_file) as f:
-            nodes = yaml.safe_load(TaskTemplate.render(f.read()))
+            nodes = yaml_load(TaskTemplate.render(f.read()))
         self.nodes = {a['host_name']: a for a in nodes['nodes']}
 
     def run(self, result):
diff --git a/yardstick/benchmark/scenarios/networking/vnf_generic.py b/yardstick/benchmark/scenarios/networking/vnf_generic.py
index e0dd36684..599835d56 100644
--- a/yardstick/benchmark/scenarios/networking/vnf_generic.py
+++ b/yardstick/benchmark/scenarios/networking/vnf_generic.py
@@ -21,12 +21,12 @@ import os
 
 import re
 from itertools import chain
-import yaml
 from operator import itemgetter
 from collections import defaultdict
 
 from yardstick.benchmark.scenarios import base
 from yardstick.common.utils import import_modules_from_package, itersubclasses
+from yardstick.common.yaml_loader import yaml_load
 from yardstick.network_services.collector.subscriber import Collector
 from yardstick.network_services.vnf_generic import vnfdgen
 from yardstick.network_services.vnf_generic.vnf.base import GenericVNF
@@ -119,7 +119,7 @@ class NetworkServiceTestCase(base.Scenario):
         # fixme: create schema to validate all fields have been provided
         with open_relative_file(scenario_cfg["topology"],
                                 scenario_cfg['task_path']) as stream:
-            topology_yaml = yaml.safe_load(stream)
+            topology_yaml = yaml_load(stream)
 
         self.topology = topology_yaml["nsd:nsd-catalog"]["nsd"][0]
         self.vnfs = []
@@ -129,7 +129,7 @@ class NetworkServiceTestCase(base.Scenario):
     def _get_traffic_flow(self):
         try:
             with open(self.scenario_cfg["traffic_options"]["flow"]) as fflow:
-                flow = yaml.safe_load(fflow)
+                flow = yaml_load(fflow)
         except (KeyError, IOError, OSError):
             flow = {}
         return flow
@@ -137,7 +137,7 @@ class NetworkServiceTestCase(base.Scenario):
     def _get_traffic_imix(self):
         try:
             with open(self.scenario_cfg["traffic_options"]["imix"]) as fimix:
-                imix = yaml.safe_load(fimix)
+                imix = yaml_load(fimix)
         except (KeyError, IOError, OSError):
             imix = {}
         return imix
diff --git a/yardstick/common/task_template.py b/yardstick/common/task_template.py
index 9acc21336..f6c128609 100755
--- a/yardstick/common/task_template.py
+++ b/yardstick/common/task_template.py
@@ -11,6 +11,22 @@ from __future__ import absolute_import
 import re
 import jinja2
 import jinja2.meta
+import yaml
+
+
+def finalize_for_yaml(elem):
+    """Render Jinja2 output specifically for YAML files"""
+    # Jinaj2 by default converts None to 'None', we can't allow this
+    # we could convert to empty string '', or we can convert to null, aka ~
+    if elem is None:
+        return '~'
+    # convert data structures to inline YAML
+    # match builtin types because we shouldn't be trying to render complex types
+    if isinstance(elem, (dict, list)):
+        # remove newlines because we are injecting back into YAML
+        # use block style for single line
+        return yaml.safe_dump(elem, default_flow_style=True).replace('\n', '')
+    return elem
 
 
 class TaskTemplate(object):
@@ -38,7 +54,7 @@ class TaskTemplate(object):
             single_msg = ("Please specify template task argument:%s")
             raise TypeError((len(real_missing) > 1 and multi_msg or single_msg)
                             % ", ".join(real_missing))
-        return jinja2.Template(task_template).render(**kwargs)
+        return jinja2.Template(task_template, finalize=finalize_for_yaml).render(**kwargs)
 
 
 def is_really_missing(mis, task_template):
diff --git a/yardstick/common/template_format.py b/yardstick/common/template_format.py
index 98c0a0b3c..bd5d8376f 100644
--- a/yardstick/common/template_format.py
+++ b/yardstick/common/template_format.py
@@ -18,9 +18,10 @@ import yaml
 from oslo_serialization import jsonutils
 
 if hasattr(yaml, 'CSafeLoader'):
-    yaml_loader = yaml.CSafeLoader
+    # make a dynamic subclass so we don't override global yaml Loader
+    yaml_loader = type('HeatYamlLoader', (yaml.CSafeLoader,), {})
 else:
-    yaml_loader = yaml.SafeLoader
+    yaml_loader = type('HeatYamlLoader', (yaml.SafeLoader,), {})
 
 if hasattr(yaml, 'CSafeDumper'):
     yaml_dumper = yaml.CSafeDumper
@@ -28,10 +29,13 @@ else:
     yaml_dumper = yaml.SafeDumper
 
 
+# This breaks NetworkServiceTestCase yaml loading, because we need to conversion to
+# native Python str() objects because we use use Trex and Trex is has broken unicode handling
 def _construct_yaml_str(self, node):
     # Override the default string handling function
     # to always return unicode objects
     return self.construct_scalar(node)
+
 yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
 # Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type
 # datetime.data which causes problems in API layer when being processed by
diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py
index f2455be3a..c7ae9c1ef 100644
--- a/yardstick/common/utils.py
+++ b/yardstick/common/utils.py
@@ -30,7 +30,6 @@ import random
 import ipaddress
 from contextlib import closing
 
-import yaml
 import six
 from flask import jsonify
 from six.moves import configparser
@@ -38,6 +37,7 @@ from oslo_utils import importutils
 from oslo_serialization import jsonutils
 
 import yardstick
+from yardstick.common.yaml_loader import yaml_load
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.DEBUG)
@@ -97,7 +97,7 @@ def import_modules_from_package(package):
 def parse_yaml(file_path):
     try:
         with open(file_path) as f:
-            value = yaml.safe_load(f)
+            value = yaml_load(f)
     except IOError:
         return {}
     except OSError as e:
diff --git a/yardstick/common/yaml_loader.py b/yardstick/common/yaml_loader.py
new file mode 100644
index 000000000..0572bd582
--- /dev/null
+++ b/yardstick/common/yaml_loader.py
@@ -0,0 +1,33 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# yardstick: this file is copied from python-heatclient and slightly modified
+
+from __future__ import absolute_import
+
+import yaml
+
+
+if hasattr(yaml, 'CSafeLoader'):
+    # make a dynamic subclass so we don't override global yaml Loader
+    yaml_loader = type('CustomLoader', (yaml.CSafeLoader,), {})
+else:
+    yaml_loader = type('CustomLoader', (yaml.SafeLoader,), {})
+
+if hasattr(yaml, 'CSafeDumper'):
+    yaml_dumper = yaml.CSafeDumper
+else:
+    yaml_dumper = yaml.SafeDumper
+
+
+def yaml_load(tmpl_str):
+    return yaml.load(tmpl_str, Loader=yaml_loader)
diff --git a/yardstick/network_services/vnf_generic/vnfdgen.py b/yardstick/network_services/vnf_generic/vnfdgen.py
index 474403dee..f42635006 100644
--- a/yardstick/network_services/vnf_generic/vnfdgen.py
+++ b/yardstick/network_services/vnf_generic/vnfdgen.py
@@ -14,26 +14,29 @@
 """ Generic file to map and build vnf discriptor """
 
 from __future__ import absolute_import
+
 from functools import reduce
 
 import jinja2
 import logging
-import yaml
 
+from yardstick.common.task_template import finalize_for_yaml
 from yardstick.common.utils import try_int
+from yardstick.common.yaml_loader import yaml_load
 
 LOG = logging.getLogger(__name__)
 
 
 def render(vnf_model, **kwargs):
     """Render jinja2 VNF template
+    Do not check for missing arguments
 
     :param vnf_model: string that contains template
     :param kwargs: Dict with template arguments
     :returns:rendered template str
     """
 
-    return jinja2.Template(vnf_model).render(**kwargs)
+    return jinja2.Template(vnf_model, finalize=finalize_for_yaml).render(**kwargs)
 
 
 def generate_vnfd(vnf_model, node):
@@ -54,7 +57,7 @@ def generate_vnfd(vnf_model, node):
     rendered_vnfd = render(vnf_model, **node)
     # This is done to get rid of issues with serializing node
     del node["get"]
-    filled_vnfd = yaml.safe_load(rendered_vnfd)
+    filled_vnfd = yaml_load(rendered_vnfd)
     return filled_vnfd
 
 
diff --git a/yardstick/network_services/yang_model.py b/yardstick/network_services/yang_model.py
index fbf224bd8..ec00c4513 100644
--- a/yardstick/network_services/yang_model.py
+++ b/yardstick/network_services/yang_model.py
@@ -1,107 +1,108 @@
-# Copyright (c) 2017 Intel Corporation

-#

-# Licensed under the Apache License, Version 2.0 (the "License");

-# you may not use this file except in compliance with the License.

-# You may obtain a copy of the License at

-#

-#      http://www.apache.org/licenses/LICENSE-2.0

-#

-# Unless required by applicable law or agreed to in writing, software

-# distributed under the License is distributed on an "AS IS" BASIS,

-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

-# See the License for the specific language governing permissions and

-# limitations under the License.

-

-from __future__ import absolute_import

-from __future__ import print_function

-import logging

-import ipaddress

-import yaml

-import six

-

-LOG = logging.getLogger(__name__)

-

-

-class YangModel(object):

-

-    RULE_TEMPLATE = "p acl add 1 {0} {1} {2} {3} {4} {5} {6} {7} 0 0 {8}"

-

-    def __init__(self, config_file):

-        super(YangModel, self).__init__()

-        self._config_file = config_file

-        self._options = {}

-        self._rules = ''

-

-    @property

-    def config_file(self):

-        return self._config_file

-

-    @config_file.setter

-    def config_file(self, value):

-        self._config_file = value

-        self._options = {}

-        self._rules = ''

-

-    def _read_config(self):

-        # TODO: add some error handling in case of empty or non-existing file

-        try:

-            with open(self._config_file) as f:

-                self._options = yaml.safe_load(f)

-        except Exception as e:

-            LOG.exception("Failed to load the yaml %s", e)

-            raise

-

-    def _get_entries(self):

-        if not self._options:

-            return ''

-

-        rule_list = []

-        for ace in self._options['access-list1']['acl']['access-list-entries']:

-            # TODO: resolve ports using topology file and nodes'

-            # ids: public or private.

-            matches = ace['ace']['matches']

-            dst_ipv4_net = matches['destination-ipv4-network']

-            dst_ipv4_net_ip = ipaddress.ip_interface(six.text_type(dst_ipv4_net))

-            port0_local_network = dst_ipv4_net_ip.network.network_address.exploded

-            port0_prefix = dst_ipv4_net_ip.network.prefixlen

-

-            src_ipv4_net = matches['source-ipv4-network']

-            src_ipv4_net_ip = ipaddress.ip_interface(six.text_type(src_ipv4_net))

-            port1_local_network = src_ipv4_net_ip.network.network_address.exploded

-            port1_prefix = src_ipv4_net_ip.network.prefixlen

-

-            lower_dport = matches['destination-port-range']['lower-port']

-            upper_dport = matches['destination-port-range']['upper-port']

-

-            lower_sport = matches['source-port-range']['lower-port']

-            upper_sport = matches['source-port-range']['upper-port']

-

-            # TODO: proto should be read from file also.

-            # Now all rules in sample ACL file are TCP.

-            rule_list.append('')  # get an extra new line

-            rule_list.append(self.RULE_TEMPLATE.format(port0_local_network,

-                                                       port0_prefix,

-                                                       port1_local_network,

-                                                       port1_prefix,

-                                                       lower_dport,

-                                                       upper_dport,

-                                                       lower_sport,

-                                                       upper_sport,

-                                                       0))

-            rule_list.append(self.RULE_TEMPLATE.format(port1_local_network,

-                                                       port1_prefix,

-                                                       port0_local_network,

-                                                       port0_prefix,

-                                                       lower_sport,

-                                                       upper_sport,

-                                                       lower_dport,

-                                                       upper_dport,

-                                                       1))

-

-        self._rules = '\n'.join(rule_list)

-

-    def get_rules(self):

-        if not self._rules:

-            self._read_config()

-            self._get_entries()

-        return self._rules

+# Copyright (c) 2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import print_function
+import logging
+import ipaddress
+import six
+
+from yardstick.common.yaml_loader import yaml_load
+
+LOG = logging.getLogger(__name__)
+
+
+class YangModel(object):
+
+    RULE_TEMPLATE = "p acl add 1 {0} {1} {2} {3} {4} {5} {6} {7} 0 0 {8}"
+
+    def __init__(self, config_file):
+        super(YangModel, self).__init__()
+        self._config_file = config_file
+        self._options = {}
+        self._rules = ''
+
+    @property
+    def config_file(self):
+        return self._config_file
+
+    @config_file.setter
+    def config_file(self, value):
+        self._config_file = value
+        self._options = {}
+        self._rules = ''
+
+    def _read_config(self):
+        # TODO: add some error handling in case of empty or non-existing file
+        try:
+            with open(self._config_file) as f:
+                self._options = yaml_load(f)
+        except Exception as e:
+            LOG.exception("Failed to load the yaml %s", e)
+            raise
+
+    def _get_entries(self):
+        if not self._options:
+            return ''
+
+        rule_list = []
+        for ace in self._options['access-list1']['acl']['access-list-entries']:
+            # TODO: resolve ports using topology file and nodes'
+            # ids: public or private.
+            matches = ace['ace']['matches']
+            dst_ipv4_net = matches['destination-ipv4-network']
+            dst_ipv4_net_ip = ipaddress.ip_interface(six.text_type(dst_ipv4_net))
+            port0_local_network = dst_ipv4_net_ip.network.network_address.exploded
+            port0_prefix = dst_ipv4_net_ip.network.prefixlen
+
+            src_ipv4_net = matches['source-ipv4-network']
+            src_ipv4_net_ip = ipaddress.ip_interface(six.text_type(src_ipv4_net))
+            port1_local_network = src_ipv4_net_ip.network.network_address.exploded
+            port1_prefix = src_ipv4_net_ip.network.prefixlen
+
+            lower_dport = matches['destination-port-range']['lower-port']
+            upper_dport = matches['destination-port-range']['upper-port']
+
+            lower_sport = matches['source-port-range']['lower-port']
+            upper_sport = matches['source-port-range']['upper-port']
+
+            # TODO: proto should be read from file also.
+            # Now all rules in sample ACL file are TCP.
+            rule_list.append('')  # get an extra new line
+            rule_list.append(self.RULE_TEMPLATE.format(port0_local_network,
+                                                       port0_prefix,
+                                                       port1_local_network,
+                                                       port1_prefix,
+                                                       lower_dport,
+                                                       upper_dport,
+                                                       lower_sport,
+                                                       upper_sport,
+                                                       0))
+            rule_list.append(self.RULE_TEMPLATE.format(port1_local_network,
+                                                       port1_prefix,
+                                                       port0_local_network,
+                                                       port0_prefix,
+                                                       lower_sport,
+                                                       upper_sport,
+                                                       lower_dport,
+                                                       upper_dport,
+                                                       1))
+
+        self._rules = '\n'.join(rule_list)
+
+    def get_rules(self):
+        if not self._rules:
+            self._read_config()
+            self._get_entries()
+        return self._rules