YAML fixes

There are multiple issues wiht YAML loading.

1. Jinja2 renders None values as a string 'None'.  This is not valid YAML
we need to render None values to '~' or 'null' which is the native YAML
None value.

2. Jinja2 renders dict and lists that contain unicode with
u'foo' values.  This is not value YAML syntax.
Because we are serializing dict and lists into YAML, we
need to encode them as valid YAML.  We can override Jinja2 finalize to
use yaml.dump to dump inline YAML.

We use yaml.safe_dump(elem, default_flow_style=True).replace('\n', '')
to generate valid single-line YAML dict and list values.

But this problem highlights the general difficulties with templating and
loading files.
We could avoid this Python->Jinja2->YAML->Python issue by directly
injecting the list or dict after the YAML is loaded.

I'm not sure of the real utility of these templates.

3. On Python 2 YAML loader is rendering all strings
as unicode.  This does not work for Trex because Trex is broken
and badly coded.  Trex does type checking against str() which
is different for Python 2 and Python 3.
The default YAML loader will return native string types, str() or unicode()
for Python 2 and Python 3 respectively.

The bad Trex codes is in convert_val:

https://github.com/cisco-system-traffic-generator/trex-core/blob/master/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py#L674

def convert_val (val):
    if is_integer(val):
        return val
    if type(val) == str:
        return ipv4_str_to_num (is_valid_ipv4(val))
    raise CTRexPacketBuildException(-11,("init val invalid %s ") % val  );

This code is doing type(val) == str.  This is bad and broken.
We can't fix Trex, so we have to render all strings as native str() types

The bug here was that the Heat template loader template_format.py
was overriding the global YAML loader to always return unicode.
We don't want this global override.

To fix this we have to use local subclasses of the yaml.SafeLoader
class.
But in order to dynamically subclass from CSafeLoader or SafeLoader
we have to use the type() builtin to define a new class at runtime.

Once we have new classes defined, we can safely isolate different
YAML constructors and return unicode or not depending on the case.

To be consistent we implement a new yaml_loader.py module to centralize
all non-Heat template yaml loading to ensure correct uncode/str
conversion

Change-Id: Iebf9cf78fbda390977c390436b0869e7bbf503eb
Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>
Signed-off-by: Deepak S <deepak.s@linux.intel.com>
Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>

4 files changed, 87 insertions, 5 deletions

diff --git a/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py b/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
index e7ba3ca73..680692fdc 100644
--- a/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
+++ b/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
@@ -18,7 +18,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
 
     @mock.patch('{}.GetNumaInfo._check_numa_node'.format(BASE))
     @mock.patch('{}.GetNumaInfo._get_current_host_name'.format(BASE))
-    @mock.patch('yaml.safe_load')
+    @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
     @mock.patch('yardstick.common.task_template.TaskTemplate.render')
     def test_get_numa_info(self,
                            mock_render,
@@ -44,7 +44,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
 
     @mock.patch('yardstick.ssh.SSH.from_node')
     @mock.patch('{}.GetNumaInfo._get_current_host_name'.format(BASE))
-    @mock.patch('yaml.safe_load')
+    @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
     @mock.patch('yardstick.common.task_template.TaskTemplate.render')
     def test_check_numa_node(self,
                              mock_render,
@@ -74,7 +74,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
 
     @mock.patch('{}.change_obj_to_dict'.format(BASE))
     @mock.patch('{}.get_nova_client'.format(BASE))
-    @mock.patch('yaml.safe_load')
+    @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
     @mock.patch('yardstick.common.task_template.TaskTemplate.render')
     def test_get_current_host_name(self,
                                    mock_render,
diff --git a/tests/unit/common/test_yaml_loader.py b/tests/unit/common/test_yaml_loader.py
new file mode 100644
index 000000000..90cbb8157
--- /dev/null
+++ b/tests/unit/common/test_yaml_loader.py
@@ -0,0 +1,32 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+# yardstick: this file is copied from python-heatclient and slightly modified
+
+from __future__ import absolute_import
+import unittest
+
+from yardstick.common import yaml_loader
+
+
+class TemplateFormatTestCase(unittest.TestCase):
+
+    def test_parse_to_value_exception(self):
+
+        self.assertEquals(yaml_loader.yaml_load("string"), u"string")
+
+
+def main():
+    unittest.main()
+
+if __name__ == '__main__':
+    main()
diff --git a/tests/unit/network_services/test_yang_model.py b/tests/unit/network_services/test_yang_model.py
index 28367f316..0b29da701 100644
--- a/tests/unit/network_services/test_yang_model.py
+++ b/tests/unit/network_services/test_yang_model.py
@@ -95,7 +95,7 @@ class YangModelTestCase(unittest.TestCase):
         y._get_entries()
         self.assertEqual(y._rules, '')
 
-    @mock.patch('yaml.safe_load')
+    @mock.patch('yardstick.network_services.yang_model.yaml_load')
     @mock.patch('yardstick.network_services.yang_model.open')
     def test__read_config(self, mock_open, mock_safe_load):
         cfg = "yang.yaml"
diff --git a/tests/unit/network_services/vnf_generic/test_vnfdgen.py b/tests/unit/network_services/vnf_generic/test_vnfdgen.py
index 44d9058dc..c2b923568 100644
--- a/tests/unit/network_services/vnf_generic/test_vnfdgen.py
+++ b/tests/unit/network_services/vnf_generic/test_vnfdgen.py
@@ -21,6 +21,7 @@ from __future__ import absolute_import
 import unittest
 from six.moves import range
 
+from yardstick.common.yaml_loader import yaml_load
 from yardstick.network_services.vnf_generic import vnfdgen
 
 TREX_VNFD_TEMPLATE = """
@@ -65,6 +66,8 @@ vnfd:vnfd-catalog:
                     dst_mac: '{{ interfaces.xe1.dst_mac }}'
                     bandwidth: 10 Gbps
                 vnfd-connection-point-ref: xe1
+            routing_table: {{ routing_table }}
+            nd_route_tbl: {{ nd_route_tbl }}
 
         benchmark:
             kpi:
@@ -126,6 +129,22 @@ COMPLETE_TREX_VNFD = \
                                          'vpci': '0000:00:10.1'},
                    'vnfd-connection-point-ref': 'xe1'}],
                  'id': 'trexgen-baremetal',
+                 'nd_route_tbl': [{'gateway': '0064:ff9b:0:0:0:0:9810:6414',
+                                   'if': 'xe0',
+                                   'netmask': '112',
+                                   'network': '0064:ff9b:0:0:0:0:9810:6414'},
+                                  {'gateway': '0064:ff9b:0:0:0:0:9810:2814',
+                                   'if': 'xe1',
+                                   'netmask': '112',
+                                   'network': '0064:ff9b:0:0:0:0:9810:2814'}],
+                 'routing_table': [{'gateway': '152.16.100.20',
+                                    'if': 'xe0',
+                                    'netmask': '255.255.255.0',
+                                    'network': '152.16.100.20'},
+                                   {'gateway': '152.16.40.20',
+                                    'if': 'xe1',
+                                    'netmask': '255.255.255.0',
+                                    'network': '152.16.40.20'}],
                  'name': 'trexgen-baremetal'}]}]}}
 
 NODE_CFG = {'ip': '1.1.1.1',
@@ -144,7 +163,24 @@ NODE_CFG = {'ip': '1.1.1.1',
                                    'dst_mac': '00:01:02:03:04:06',
                                    'local_ip': '2.1.1.2',
                                    'local_mac': '00:01:02:03:05:06',
-                                   'vpci': '0000:00:10.1'}}}
+                                   'vpci': '0000:00:10.1'}},
+            'nd_route_tbl': [{u'gateway': u'0064:ff9b:0:0:0:0:9810:6414',
+                              u'if': u'xe0',
+                              u'netmask': u'112',
+                              u'network': u'0064:ff9b:0:0:0:0:9810:6414'},
+                             {u'gateway': u'0064:ff9b:0:0:0:0:9810:2814',
+                              u'if': u'xe1',
+                              u'netmask': u'112',
+                              u'network': u'0064:ff9b:0:0:0:0:9810:2814'}],
+            'routing_table': [{u'gateway': u'152.16.100.20',
+                               u'if': u'xe0',
+                               u'netmask': u'255.255.255.0',
+                               u'network': u'152.16.100.20'},
+                              {u'gateway': u'152.16.40.20',
+                               u'if': u'xe1',
+                               u'netmask': u'255.255.255.0',
+                               u'network': u'152.16.40.20'}],
+            }
 
 
 TRAFFIC_PROFILE_TPL = """
@@ -169,6 +205,20 @@ TRAFFIC_PROFILE = {
                                          "1518B": '40'}}}}]}
 
 
+class TestRender(unittest.TestCase):
+
+    def test_render_none(self):
+
+        tmpl = "{{ routing_table }}"
+        self.assertEqual(vnfdgen.render(tmpl, routing_table=None), u'~')
+        self.assertEqual(yaml_load(vnfdgen.render(tmpl, routing_table=None)), None)
+
+    def test_render_unicode_dict(self):
+
+        tmpl = "{{ routing_table }}"
+        self.assertEqual(yaml_load(vnfdgen.render(tmpl, **NODE_CFG)), NODE_CFG["routing_table"])
+
+
 class TestVnfdGen(unittest.TestCase):
     """ Class to verify VNFS testcases """