summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRoss Brattain <ross.b.brattain@intel.com>2017-08-07 14:32:45 -0700
committerRoss Brattain <ross.b.brattain@intel.com>2017-08-15 20:36:32 -0700
commitc3d309865bb57e2aba2033d3bbe1c16adca4f0ba (patch)
treebebe3ecb46b1ac7983d2542867058c5420a06ab1
parent7d5cc542a9bfa35cba6d2b681e0cf5f90bc77d40 (diff)
YAML fixes
There are multiple issues wiht YAML loading. 1. Jinja2 renders None values as a string 'None'. This is not valid YAML we need to render None values to '~' or 'null' which is the native YAML None value. 2. Jinja2 renders dict and lists that contain unicode with u'foo' values. This is not value YAML syntax. Because we are serializing dict and lists into YAML, we need to encode them as valid YAML. We can override Jinja2 finalize to use yaml.dump to dump inline YAML. We use yaml.safe_dump(elem, default_flow_style=True).replace('\n', '') to generate valid single-line YAML dict and list values. But this problem highlights the general difficulties with templating and loading files. We could avoid this Python->Jinja2->YAML->Python issue by directly injecting the list or dict after the YAML is loaded. I'm not sure of the real utility of these templates. 3. On Python 2 YAML loader is rendering all strings as unicode. This does not work for Trex because Trex is broken and badly coded. Trex does type checking against str() which is different for Python 2 and Python 3. The default YAML loader will return native string types, str() or unicode() for Python 2 and Python 3 respectively. The bad Trex codes is in convert_val: https://github.com/cisco-system-traffic-generator/trex-core/blob/master/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py#L674 def convert_val (val): if is_integer(val): return val if type(val) == str: return ipv4_str_to_num (is_valid_ipv4(val)) raise CTRexPacketBuildException(-11,("init val invalid %s ") % val ); This code is doing type(val) == str. This is bad and broken. We can't fix Trex, so we have to render all strings as native str() types The bug here was that the Heat template loader template_format.py was overriding the global YAML loader to always return unicode. We don't want this global override. To fix this we have to use local subclasses of the yaml.SafeLoader class. But in order to dynamically subclass from CSafeLoader or SafeLoader we have to use the type() builtin to define a new class at runtime. Once we have new classes defined, we can safely isolate different YAML constructors and return unicode or not depending on the case. To be consistent we implement a new yaml_loader.py module to centralize all non-Heat template yaml loading to ensure correct uncode/str conversion Change-Id: Iebf9cf78fbda390977c390436b0869e7bbf503eb Signed-off-by: Ross Brattain <ross.b.brattain@intel.com> Signed-off-by: Deepak S <deepak.s@linux.intel.com> Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>
-rw-r--r--api/resources/v1/env.py4
-rw-r--r--api/resources/v2/pods.py3
-rw-r--r--tests/unit/benchmark/scenarios/lib/test_get_numa_info.py6
-rw-r--r--tests/unit/common/test_yaml_loader.py32
-rw-r--r--tests/unit/network_services/test_yang_model.py2
-rw-r--r--tests/unit/network_services/vnf_generic/test_vnfdgen.py52
-rw-r--r--yardstick/benchmark/contexts/node.py4
-rw-r--r--yardstick/benchmark/contexts/standalone.py4
-rw-r--r--yardstick/benchmark/core/plugin.py4
-rw-r--r--yardstick/benchmark/core/task.py7
-rw-r--r--yardstick/benchmark/core/testcase.py4
-rw-r--r--yardstick/benchmark/scenarios/availability/attacker/baseattacker.py4
-rw-r--r--yardstick/benchmark/scenarios/availability/monitor/basemonitor.py5
-rw-r--r--yardstick/benchmark/scenarios/availability/operation/baseoperation.py4
-rw-r--r--yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py4
-rw-r--r--yardstick/benchmark/scenarios/lib/get_numa_info.py4
-rw-r--r--yardstick/benchmark/scenarios/networking/vnf_generic.py8
-rwxr-xr-xyardstick/common/task_template.py18
-rw-r--r--yardstick/common/template_format.py8
-rw-r--r--yardstick/common/utils.py4
-rw-r--r--yardstick/common/yaml_loader.py33
-rw-r--r--yardstick/network_services/vnf_generic/vnfdgen.py9
-rw-r--r--yardstick/network_services/yang_model.py215
23 files changed, 290 insertions, 148 deletions
diff --git a/api/resources/v1/env.py b/api/resources/v1/env.py
index 8367fa9eb..98b8ec7e4 100644
--- a/api/resources/v1/env.py
+++ b/api/resources/v1/env.py
@@ -31,7 +31,7 @@ from yardstick.common import utils
from yardstick.common.utils import result_handler
from yardstick.common import openstack_utils
from yardstick.common.httpClient import HttpClient
-
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)
@@ -393,7 +393,7 @@ class V1Env(ApiResource):
return result_handler(consts.API_ERROR, 'file must be provided')
LOG.info('Checking file')
- data = yaml.safe_load(pod_file.read())
+ data = yaml_load(pod_file.read())
if not isinstance(data, collections.Mapping):
return result_handler(consts.API_ERROR, 'invalid yaml file')
diff --git a/api/resources/v2/pods.py b/api/resources/v2/pods.py
index f2316d353..d98238ca1 100644
--- a/api/resources/v2/pods.py
+++ b/api/resources/v2/pods.py
@@ -18,6 +18,7 @@ from api.database.v2.handlers import V2EnvironmentHandler
from yardstick.common import constants as consts
from yardstick.common.utils import result_handler
from yardstick.common.task_template import TaskTemplate
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)
@@ -48,7 +49,7 @@ class V2Pods(ApiResource):
upload_file.save(consts.POD_FILE)
with open(consts.POD_FILE) as f:
- data = yaml.safe_load(TaskTemplate.render(f.read()))
+ data = yaml_load(TaskTemplate.render(f.read()))
LOG.debug('pod content is: %s', data)
LOG.info('create pod in database')
diff --git a/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py b/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
index e7ba3ca73..680692fdc 100644
--- a/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
+++ b/tests/unit/benchmark/scenarios/lib/test_get_numa_info.py
@@ -18,7 +18,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
@mock.patch('{}.GetNumaInfo._check_numa_node'.format(BASE))
@mock.patch('{}.GetNumaInfo._get_current_host_name'.format(BASE))
- @mock.patch('yaml.safe_load')
+ @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
@mock.patch('yardstick.common.task_template.TaskTemplate.render')
def test_get_numa_info(self,
mock_render,
@@ -44,7 +44,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
@mock.patch('yardstick.ssh.SSH.from_node')
@mock.patch('{}.GetNumaInfo._get_current_host_name'.format(BASE))
- @mock.patch('yaml.safe_load')
+ @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
@mock.patch('yardstick.common.task_template.TaskTemplate.render')
def test_check_numa_node(self,
mock_render,
@@ -74,7 +74,7 @@ class GetNumaInfoTestCase(unittest.TestCase):
@mock.patch('{}.change_obj_to_dict'.format(BASE))
@mock.patch('{}.get_nova_client'.format(BASE))
- @mock.patch('yaml.safe_load')
+ @mock.patch('yardstick.benchmark.scenarios.lib.get_numa_info.yaml_load')
@mock.patch('yardstick.common.task_template.TaskTemplate.render')
def test_get_current_host_name(self,
mock_render,
diff --git a/tests/unit/common/test_yaml_loader.py b/tests/unit/common/test_yaml_loader.py
new file mode 100644
index 000000000..90cbb8157
--- /dev/null
+++ b/tests/unit/common/test_yaml_loader.py
@@ -0,0 +1,32 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# yardstick: this file is copied from python-heatclient and slightly modified
+
+from __future__ import absolute_import
+import unittest
+
+from yardstick.common import yaml_loader
+
+
+class TemplateFormatTestCase(unittest.TestCase):
+
+ def test_parse_to_value_exception(self):
+
+ self.assertEquals(yaml_loader.yaml_load("string"), u"string")
+
+
+def main():
+ unittest.main()
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/unit/network_services/test_yang_model.py b/tests/unit/network_services/test_yang_model.py
index 28367f316..0b29da701 100644
--- a/tests/unit/network_services/test_yang_model.py
+++ b/tests/unit/network_services/test_yang_model.py
@@ -95,7 +95,7 @@ class YangModelTestCase(unittest.TestCase):
y._get_entries()
self.assertEqual(y._rules, '')
- @mock.patch('yaml.safe_load')
+ @mock.patch('yardstick.network_services.yang_model.yaml_load')
@mock.patch('yardstick.network_services.yang_model.open')
def test__read_config(self, mock_open, mock_safe_load):
cfg = "yang.yaml"
diff --git a/tests/unit/network_services/vnf_generic/test_vnfdgen.py b/tests/unit/network_services/vnf_generic/test_vnfdgen.py
index 44d9058dc..c2b923568 100644
--- a/tests/unit/network_services/vnf_generic/test_vnfdgen.py
+++ b/tests/unit/network_services/vnf_generic/test_vnfdgen.py
@@ -21,6 +21,7 @@ from __future__ import absolute_import
import unittest
from six.moves import range
+from yardstick.common.yaml_loader import yaml_load
from yardstick.network_services.vnf_generic import vnfdgen
TREX_VNFD_TEMPLATE = """
@@ -65,6 +66,8 @@ vnfd:vnfd-catalog:
dst_mac: '{{ interfaces.xe1.dst_mac }}'
bandwidth: 10 Gbps
vnfd-connection-point-ref: xe1
+ routing_table: {{ routing_table }}
+ nd_route_tbl: {{ nd_route_tbl }}
benchmark:
kpi:
@@ -126,6 +129,22 @@ COMPLETE_TREX_VNFD = \
'vpci': '0000:00:10.1'},
'vnfd-connection-point-ref': 'xe1'}],
'id': 'trexgen-baremetal',
+ 'nd_route_tbl': [{'gateway': '0064:ff9b:0:0:0:0:9810:6414',
+ 'if': 'xe0',
+ 'netmask': '112',
+ 'network': '0064:ff9b:0:0:0:0:9810:6414'},
+ {'gateway': '0064:ff9b:0:0:0:0:9810:2814',
+ 'if': 'xe1',
+ 'netmask': '112',
+ 'network': '0064:ff9b:0:0:0:0:9810:2814'}],
+ 'routing_table': [{'gateway': '152.16.100.20',
+ 'if': 'xe0',
+ 'netmask': '255.255.255.0',
+ 'network': '152.16.100.20'},
+ {'gateway': '152.16.40.20',
+ 'if': 'xe1',
+ 'netmask': '255.255.255.0',
+ 'network': '152.16.40.20'}],
'name': 'trexgen-baremetal'}]}]}}
NODE_CFG = {'ip': '1.1.1.1',
@@ -144,7 +163,24 @@ NODE_CFG = {'ip': '1.1.1.1',
'dst_mac': '00:01:02:03:04:06',
'local_ip': '2.1.1.2',
'local_mac': '00:01:02:03:05:06',
- 'vpci': '0000:00:10.1'}}}
+ 'vpci': '0000:00:10.1'}},
+ 'nd_route_tbl': [{u'gateway': u'0064:ff9b:0:0:0:0:9810:6414',
+ u'if': u'xe0',
+ u'netmask': u'112',
+ u'network': u'0064:ff9b:0:0:0:0:9810:6414'},
+ {u'gateway': u'0064:ff9b:0:0:0:0:9810:2814',
+ u'if': u'xe1',
+ u'netmask': u'112',
+ u'network': u'0064:ff9b:0:0:0:0:9810:2814'}],
+ 'routing_table': [{u'gateway': u'152.16.100.20',
+ u'if': u'xe0',
+ u'netmask': u'255.255.255.0',
+ u'network': u'152.16.100.20'},
+ {u'gateway': u'152.16.40.20',
+ u'if': u'xe1',
+ u'netmask': u'255.255.255.0',
+ u'network': u'152.16.40.20'}],
+ }
TRAFFIC_PROFILE_TPL = """
@@ -169,6 +205,20 @@ TRAFFIC_PROFILE = {
"1518B": '40'}}}}]}
+class TestRender(unittest.TestCase):
+
+ def test_render_none(self):
+
+ tmpl = "{{ routing_table }}"
+ self.assertEqual(vnfdgen.render(tmpl, routing_table=None), u'~')
+ self.assertEqual(yaml_load(vnfdgen.render(tmpl, routing_table=None)), None)
+
+ def test_render_unicode_dict(self):
+
+ tmpl = "{{ routing_table }}"
+ self.assertEqual(yaml_load(vnfdgen.render(tmpl, **NODE_CFG)), NODE_CFG["routing_table"])
+
+
class TestVnfdGen(unittest.TestCase):
""" Class to verify VNFS testcases """
diff --git a/yardstick/benchmark/contexts/node.py b/yardstick/benchmark/contexts/node.py
index 35c64335d..250032efc 100644
--- a/yardstick/benchmark/contexts/node.py
+++ b/yardstick/benchmark/contexts/node.py
@@ -17,12 +17,12 @@ import tempfile
import six
import pkg_resources
-import yaml
from yardstick import ssh
from yardstick.benchmark.contexts.base import Context
from yardstick.common.constants import ANSIBLE_DIR, YARDSTICK_ROOT_PATH
from yardstick.common.ansible_common import AnsibleCommon
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
@@ -55,7 +55,7 @@ class NodeContext(Context):
with open(self.file_path) as stream:
LOG.info("Parsing pod file: %s", self.file_path)
- cfg = yaml.safe_load(stream)
+ cfg = yaml_load(stream)
return cfg
def init(self, attrs):
diff --git a/yardstick/benchmark/contexts/standalone.py b/yardstick/benchmark/contexts/standalone.py
index ae1046974..f0ef1d560 100644
--- a/yardstick/benchmark/contexts/standalone.py
+++ b/yardstick/benchmark/contexts/standalone.py
@@ -18,12 +18,12 @@ import logging
import os
import errno
import collections
-import yaml
import time
from yardstick.benchmark.contexts.base import Context
from yardstick.common.constants import YARDSTICK_ROOT_PATH
from yardstick.common.utils import import_modules_from_package, itersubclasses
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
@@ -50,7 +50,7 @@ class StandaloneContext(Context):
with open(self.file_path) as stream:
LOG.info("Parsing pod file: %s", self.file_path)
- cfg = yaml.safe_load(stream)
+ cfg = yaml_load(stream)
return cfg
def get_nfvi_obj(self):
diff --git a/yardstick/benchmark/core/plugin.py b/yardstick/benchmark/core/plugin.py
index a741d5e74..24f1b6b25 100644
--- a/yardstick/benchmark/core/plugin.py
+++ b/yardstick/benchmark/core/plugin.py
@@ -13,13 +13,13 @@ from __future__ import print_function
from __future__ import absolute_import
import os
import sys
-import yaml
import time
import logging
import pkg_resources
import yardstick.ssh as ssh
from yardstick.common.task_template import TaskTemplate
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
@@ -153,7 +153,7 @@ class PluginParser(object):
raise e
print("Input plugin is:\n%s\n" % rendered_plugin)
- cfg = yaml.safe_load(rendered_plugin)
+ cfg = yaml_load(rendered_plugin)
except IOError as ioerror:
sys.exit(ioerror)
diff --git a/yardstick/benchmark/core/task.py b/yardstick/benchmark/core/task.py
index ea3239edf..f689f2e63 100644
--- a/yardstick/benchmark/core/task.py
+++ b/yardstick/benchmark/core/task.py
@@ -29,6 +29,7 @@ from jinja2 import Environment
from yardstick.benchmark.contexts.base import Context
from yardstick.benchmark.runners import base as base_runner
+from yardstick.common.yaml_loader import yaml_load
from yardstick.dispatcher.base import Base as DispatcherBase
from yardstick.common.task_template import TaskTemplate
from yardstick.common.utils import source_env
@@ -437,7 +438,7 @@ class TaskParser(object): # pragma: no cover
try:
with open(self.path) as stream:
- cfg = yaml.load(stream)
+ cfg = yaml_load(stream)
except IOError as ioerror:
sys.exit(ioerror)
@@ -501,7 +502,7 @@ class TaskParser(object): # pragma: no cover
raise e
print("Input task is:\n%s\n" % rendered_task)
- cfg = yaml.load(rendered_task)
+ cfg = yaml_load(rendered_task)
except IOError as ioerror:
sys.exit(ioerror)
@@ -657,7 +658,7 @@ def parse_task_args(src_name, args):
return args
try:
- kw = args and yaml.safe_load(args)
+ kw = args and yaml_load(args)
kw = {} if kw is None else kw
except yaml.parser.ParserError as e:
print_invalid_header(src_name, args)
diff --git a/yardstick/benchmark/core/testcase.py b/yardstick/benchmark/core/testcase.py
index 7ab1b08cf..501356726 100644
--- a/yardstick/benchmark/core/testcase.py
+++ b/yardstick/benchmark/core/testcase.py
@@ -12,11 +12,11 @@ from __future__ import absolute_import
from __future__ import print_function
import os
-import yaml
import logging
from yardstick.common.task_template import TaskTemplate
from yardstick.common import constants as consts
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
@@ -69,7 +69,7 @@ class Testcase(object):
def _parse_testcase(self, testcase_info):
rendered_testcase = TaskTemplate.render(testcase_info)
- testcase_cfg = yaml.safe_load(rendered_testcase)
+ testcase_cfg = yaml_load(rendered_testcase)
test_precondition = testcase_cfg.get('precondition', {})
installer_type = test_precondition.get('installer_type', 'all')
diff --git a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py
index a20b26396..61698da43 100644
--- a/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py
+++ b/yardstick/benchmark/scenarios/availability/attacker/baseattacker.py
@@ -8,11 +8,11 @@
##############################################################################
from __future__ import absolute_import
import pkg_resources
-import yaml
import logging
import os
import yardstick.common.utils as utils
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
@@ -56,7 +56,7 @@ class BaseAttacker(object):
def __init__(self, config, context):
if not BaseAttacker.attacker_cfgs:
with open(attacker_conf_path) as stream:
- BaseAttacker.attacker_cfgs = yaml.safe_load(stream)
+ BaseAttacker.attacker_cfgs = yaml_load(stream)
self._config = config
self._context = context
diff --git a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
index 6165aba74..0027925d6 100644
--- a/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
+++ b/yardstick/benchmark/scenarios/availability/monitor/basemonitor.py
@@ -13,7 +13,8 @@ import multiprocessing
import time
import os
import yardstick.common.utils as utils
-import yaml
+
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
@@ -74,7 +75,7 @@ class BaseMonitor(multiprocessing.Process):
def __init__(self, config, context, data):
if not BaseMonitor.monitor_cfgs:
with open(monitor_conf_path) as stream:
- BaseMonitor.monitor_cfgs = yaml.safe_load(stream)
+ BaseMonitor.monitor_cfgs = yaml_load(stream)
multiprocessing.Process.__init__(self)
self._config = config
self._context = context
diff --git a/yardstick/benchmark/scenarios/availability/operation/baseoperation.py b/yardstick/benchmark/scenarios/availability/operation/baseoperation.py
index 4c2ce82d9..d21b857b5 100644
--- a/yardstick/benchmark/scenarios/availability/operation/baseoperation.py
+++ b/yardstick/benchmark/scenarios/availability/operation/baseoperation.py
@@ -8,11 +8,11 @@
##############################################################################
from __future__ import absolute_import
import pkg_resources
-import yaml
import logging
import os
import yardstick.common.utils as utils
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
@@ -54,7 +54,7 @@ class BaseOperation(object):
def __init__(self, config, context):
if not BaseOperation.operation_cfgs:
with open(operation_conf_path) as stream:
- BaseOperation.operation_cfgs = yaml.safe_load(stream)
+ BaseOperation.operation_cfgs = yaml_load(stream)
self.key = ''
self._config = config
self._context = context
diff --git a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py
index ce34d8be0..05b660105 100644
--- a/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py
+++ b/yardstick/benchmark/scenarios/availability/result_checker/baseresultchecker.py
@@ -8,11 +8,11 @@
##############################################################################
from __future__ import absolute_import
import pkg_resources
-import yaml
import logging
import os
import yardstick.common.utils as utils
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
@@ -58,7 +58,7 @@ class BaseResultChecker(object):
def __init__(self, config, context):
if not BaseResultChecker.resultchecker_cfgs:
with open(resultchecker_conf_path) as stream:
- BaseResultChecker.resultchecker_cfgs = yaml.safe_load(stream)
+ BaseResultChecker.resultchecker_cfgs = yaml_load(stream)
self.actualResult = object()
self.expectedResult = object()
self.success = False
diff --git a/yardstick/benchmark/scenarios/lib/get_numa_info.py b/yardstick/benchmark/scenarios/lib/get_numa_info.py
index 4e4a44d95..75a9e3506 100644
--- a/yardstick/benchmark/scenarios/lib/get_numa_info.py
+++ b/yardstick/benchmark/scenarios/lib/get_numa_info.py
@@ -13,7 +13,6 @@ from __future__ import absolute_import
import logging
import os
-import yaml
from xml.etree import ElementTree as ET
from yardstick import ssh
@@ -22,6 +21,7 @@ from yardstick.common import constants as consts
from yardstick.common.utils import change_obj_to_dict
from yardstick.common.openstack_utils import get_nova_client
from yardstick.common.task_template import TaskTemplate
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
@@ -47,7 +47,7 @@ class GetNumaInfo(base.Scenario):
self.options.get('file'))
with open(node_file) as f:
- nodes = yaml.safe_load(TaskTemplate.render(f.read()))
+ nodes = yaml_load(TaskTemplate.render(f.read()))
self.nodes = {a['host_name']: a for a in nodes['nodes']}
def run(self, result):
diff --git a/yardstick/benchmark/scenarios/networking/vnf_generic.py b/yardstick/benchmark/scenarios/networking/vnf_generic.py
index e0dd36684..599835d56 100644
--- a/yardstick/benchmark/scenarios/networking/vnf_generic.py
+++ b/yardstick/benchmark/scenarios/networking/vnf_generic.py
@@ -21,12 +21,12 @@ import os
import re
from itertools import chain
-import yaml
from operator import itemgetter
from collections import defaultdict
from yardstick.benchmark.scenarios import base
from yardstick.common.utils import import_modules_from_package, itersubclasses
+from yardstick.common.yaml_loader import yaml_load
from yardstick.network_services.collector.subscriber import Collector
from yardstick.network_services.vnf_generic import vnfdgen
from yardstick.network_services.vnf_generic.vnf.base import GenericVNF
@@ -119,7 +119,7 @@ class NetworkServiceTestCase(base.Scenario):
# fixme: create schema to validate all fields have been provided
with open_relative_file(scenario_cfg["topology"],
scenario_cfg['task_path']) as stream:
- topology_yaml = yaml.safe_load(stream)
+ topology_yaml = yaml_load(stream)
self.topology = topology_yaml["nsd:nsd-catalog"]["nsd"][0]
self.vnfs = []
@@ -129,7 +129,7 @@ class NetworkServiceTestCase(base.Scenario):
def _get_traffic_flow(self):
try:
with open(self.scenario_cfg["traffic_options"]["flow"]) as fflow:
- flow = yaml.safe_load(fflow)
+ flow = yaml_load(fflow)
except (KeyError, IOError, OSError):
flow = {}
return flow
@@ -137,7 +137,7 @@ class NetworkServiceTestCase(base.Scenario):
def _get_traffic_imix(self):
try:
with open(self.scenario_cfg["traffic_options"]["imix"]) as fimix:
- imix = yaml.safe_load(fimix)
+ imix = yaml_load(fimix)
except (KeyError, IOError, OSError):
imix = {}
return imix
diff --git a/yardstick/common/task_template.py b/yardstick/common/task_template.py
index 9acc21336..f6c128609 100755
--- a/yardstick/common/task_template.py
+++ b/yardstick/common/task_template.py
@@ -11,6 +11,22 @@ from __future__ import absolute_import
import re
import jinja2
import jinja2.meta
+import yaml
+
+
+def finalize_for_yaml(elem):
+ """Render Jinja2 output specifically for YAML files"""
+ # Jinaj2 by default converts None to 'None', we can't allow this
+ # we could convert to empty string '', or we can convert to null, aka ~
+ if elem is None:
+ return '~'
+ # convert data structures to inline YAML
+ # match builtin types because we shouldn't be trying to render complex types
+ if isinstance(elem, (dict, list)):
+ # remove newlines because we are injecting back into YAML
+ # use block style for single line
+ return yaml.safe_dump(elem, default_flow_style=True).replace('\n', '')
+ return elem
class TaskTemplate(object):
@@ -38,7 +54,7 @@ class TaskTemplate(object):
single_msg = ("Please specify template task argument:%s")
raise TypeError((len(real_missing) > 1 and multi_msg or single_msg)
% ", ".join(real_missing))
- return jinja2.Template(task_template).render(**kwargs)
+ return jinja2.Template(task_template, finalize=finalize_for_yaml).render(**kwargs)
def is_really_missing(mis, task_template):
diff --git a/yardstick/common/template_format.py b/yardstick/common/template_format.py
index 98c0a0b3c..bd5d8376f 100644
--- a/yardstick/common/template_format.py
+++ b/yardstick/common/template_format.py
@@ -18,9 +18,10 @@ import yaml
from oslo_serialization import jsonutils
if hasattr(yaml, 'CSafeLoader'):
- yaml_loader = yaml.CSafeLoader
+ # make a dynamic subclass so we don't override global yaml Loader
+ yaml_loader = type('HeatYamlLoader', (yaml.CSafeLoader,), {})
else:
- yaml_loader = yaml.SafeLoader
+ yaml_loader = type('HeatYamlLoader', (yaml.SafeLoader,), {})
if hasattr(yaml, 'CSafeDumper'):
yaml_dumper = yaml.CSafeDumper
@@ -28,10 +29,13 @@ else:
yaml_dumper = yaml.SafeDumper
+# This breaks NetworkServiceTestCase yaml loading, because we need to conversion to
+# native Python str() objects because we use use Trex and Trex is has broken unicode handling
def _construct_yaml_str(self, node):
# Override the default string handling function
# to always return unicode objects
return self.construct_scalar(node)
+
yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
# Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type
# datetime.data which causes problems in API layer when being processed by
diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py
index f2455be3a..c7ae9c1ef 100644
--- a/yardstick/common/utils.py
+++ b/yardstick/common/utils.py
@@ -30,7 +30,6 @@ import random
import ipaddress
from contextlib import closing
-import yaml
import six
from flask import jsonify
from six.moves import configparser
@@ -38,6 +37,7 @@ from oslo_utils import importutils
from oslo_serialization import jsonutils
import yardstick
+from yardstick.common.yaml_loader import yaml_load
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
@@ -97,7 +97,7 @@ def import_modules_from_package(package):
def parse_yaml(file_path):
try:
with open(file_path) as f:
- value = yaml.safe_load(f)
+ value = yaml_load(f)
except IOError:
return {}
except OSError as e:
diff --git a/yardstick/common/yaml_loader.py b/yardstick/common/yaml_loader.py
new file mode 100644
index 000000000..0572bd582
--- /dev/null
+++ b/yardstick/common/yaml_loader.py
@@ -0,0 +1,33 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# yardstick: this file is copied from python-heatclient and slightly modified
+
+from __future__ import absolute_import
+
+import yaml
+
+
+if hasattr(yaml, 'CSafeLoader'):
+ # make a dynamic subclass so we don't override global yaml Loader
+ yaml_loader = type('CustomLoader', (yaml.CSafeLoader,), {})
+else:
+ yaml_loader = type('CustomLoader', (yaml.SafeLoader,), {})
+
+if hasattr(yaml, 'CSafeDumper'):
+ yaml_dumper = yaml.CSafeDumper
+else:
+ yaml_dumper = yaml.SafeDumper
+
+
+def yaml_load(tmpl_str):
+ return yaml.load(tmpl_str, Loader=yaml_loader)
diff --git a/yardstick/network_services/vnf_generic/vnfdgen.py b/yardstick/network_services/vnf_generic/vnfdgen.py
index 474403dee..f42635006 100644
--- a/yardstick/network_services/vnf_generic/vnfdgen.py
+++ b/yardstick/network_services/vnf_generic/vnfdgen.py
@@ -14,26 +14,29 @@
""" Generic file to map and build vnf discriptor """
from __future__ import absolute_import
+
from functools import reduce
import jinja2
import logging
-import yaml
+from yardstick.common.task_template import finalize_for_yaml
from yardstick.common.utils import try_int
+from yardstick.common.yaml_loader import yaml_load
LOG = logging.getLogger(__name__)
def render(vnf_model, **kwargs):
"""Render jinja2 VNF template
+ Do not check for missing arguments
:param vnf_model: string that contains template
:param kwargs: Dict with template arguments
:returns:rendered template str
"""
- return jinja2.Template(vnf_model).render(**kwargs)
+ return jinja2.Template(vnf_model, finalize=finalize_for_yaml).render(**kwargs)
def generate_vnfd(vnf_model, node):
@@ -54,7 +57,7 @@ def generate_vnfd(vnf_model, node):
rendered_vnfd = render(vnf_model, **node)
# This is done to get rid of issues with serializing node
del node["get"]
- filled_vnfd = yaml.safe_load(rendered_vnfd)
+ filled_vnfd = yaml_load(rendered_vnfd)
return filled_vnfd
diff --git a/yardstick/network_services/yang_model.py b/yardstick/network_services/yang_model.py
index fbf224bd8..ec00c4513 100644
--- a/yardstick/network_services/yang_model.py
+++ b/yardstick/network_services/yang_model.py
@@ -1,107 +1,108 @@
-# Copyright (c) 2017 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import absolute_import
-from __future__ import print_function
-import logging
-import ipaddress
-import yaml
-import six
-
-LOG = logging.getLogger(__name__)
-
-
-class YangModel(object):
-
- RULE_TEMPLATE = "p acl add 1 {0} {1} {2} {3} {4} {5} {6} {7} 0 0 {8}"
-
- def __init__(self, config_file):
- super(YangModel, self).__init__()
- self._config_file = config_file
- self._options = {}
- self._rules = ''
-
- @property
- def config_file(self):
- return self._config_file
-
- @config_file.setter
- def config_file(self, value):
- self._config_file = value
- self._options = {}
- self._rules = ''
-
- def _read_config(self):
- # TODO: add some error handling in case of empty or non-existing file
- try:
- with open(self._config_file) as f:
- self._options = yaml.safe_load(f)
- except Exception as e:
- LOG.exception("Failed to load the yaml %s", e)
- raise
-
- def _get_entries(self):
- if not self._options:
- return ''
-
- rule_list = []
- for ace in self._options['access-list1']['acl']['access-list-entries']:
- # TODO: resolve ports using topology file and nodes'
- # ids: public or private.
- matches = ace['ace']['matches']
- dst_ipv4_net = matches['destination-ipv4-network']
- dst_ipv4_net_ip = ipaddress.ip_interface(six.text_type(dst_ipv4_net))
- port0_local_network = dst_ipv4_net_ip.network.network_address.exploded
- port0_prefix = dst_ipv4_net_ip.network.prefixlen
-
- src_ipv4_net = matches['source-ipv4-network']
- src_ipv4_net_ip = ipaddress.ip_interface(six.text_type(src_ipv4_net))
- port1_local_network = src_ipv4_net_ip.network.network_address.exploded
- port1_prefix = src_ipv4_net_ip.network.prefixlen
-
- lower_dport = matches['destination-port-range']['lower-port']
- upper_dport = matches['destination-port-range']['upper-port']
-
- lower_sport = matches['source-port-range']['lower-port']
- upper_sport = matches['source-port-range']['upper-port']
-
- # TODO: proto should be read from file also.
- # Now all rules in sample ACL file are TCP.
- rule_list.append('') # get an extra new line
- rule_list.append(self.RULE_TEMPLATE.format(port0_local_network,
- port0_prefix,
- port1_local_network,
- port1_prefix,
- lower_dport,
- upper_dport,
- lower_sport,
- upper_sport,
- 0))
- rule_list.append(self.RULE_TEMPLATE.format(port1_local_network,
- port1_prefix,
- port0_local_network,
- port0_prefix,
- lower_sport,
- upper_sport,
- lower_dport,
- upper_dport,
- 1))
-
- self._rules = '\n'.join(rule_list)
-
- def get_rules(self):
- if not self._rules:
- self._read_config()
- self._get_entries()
- return self._rules
+# Copyright (c) 2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from __future__ import print_function
+import logging
+import ipaddress
+import six
+
+from yardstick.common.yaml_loader import yaml_load
+
+LOG = logging.getLogger(__name__)
+
+
+class YangModel(object):
+
+ RULE_TEMPLATE = "p acl add 1 {0} {1} {2} {3} {4} {5} {6} {7} 0 0 {8}"
+
+ def __init__(self, config_file):
+ super(YangModel, self).__init__()
+ self._config_file = config_file
+ self._options = {}
+ self._rules = ''
+
+ @property
+ def config_file(self):
+ return self._config_file
+
+ @config_file.setter
+ def config_file(self, value):
+ self._config_file = value
+ self._options = {}
+ self._rules = ''
+
+ def _read_config(self):
+ # TODO: add some error handling in case of empty or non-existing file
+ try:
+ with open(self._config_file) as f:
+ self._options = yaml_load(f)
+ except Exception as e:
+ LOG.exception("Failed to load the yaml %s", e)
+ raise
+
+ def _get_entries(self):
+ if not self._options:
+ return ''
+
+ rule_list = []
+ for ace in self._options['access-list1']['acl']['access-list-entries']:
+ # TODO: resolve ports using topology file and nodes'
+ # ids: public or private.
+ matches = ace['ace']['matches']
+ dst_ipv4_net = matches['destination-ipv4-network']
+ dst_ipv4_net_ip = ipaddress.ip_interface(six.text_type(dst_ipv4_net))
+ port0_local_network = dst_ipv4_net_ip.network.network_address.exploded
+ port0_prefix = dst_ipv4_net_ip.network.prefixlen
+
+ src_ipv4_net = matches['source-ipv4-network']
+ src_ipv4_net_ip = ipaddress.ip_interface(six.text_type(src_ipv4_net))
+ port1_local_network = src_ipv4_net_ip.network.network_address.exploded
+ port1_prefix = src_ipv4_net_ip.network.prefixlen
+
+ lower_dport = matches['destination-port-range']['lower-port']
+ upper_dport = matches['destination-port-range']['upper-port']
+
+ lower_sport = matches['source-port-range']['lower-port']
+ upper_sport = matches['source-port-range']['upper-port']
+
+ # TODO: proto should be read from file also.
+ # Now all rules in sample ACL file are TCP.
+ rule_list.append('') # get an extra new line
+ rule_list.append(self.RULE_TEMPLATE.format(port0_local_network,
+ port0_prefix,
+ port1_local_network,
+ port1_prefix,
+ lower_dport,
+ upper_dport,
+ lower_sport,
+ upper_sport,
+ 0))
+ rule_list.append(self.RULE_TEMPLATE.format(port1_local_network,
+ port1_prefix,
+ port0_local_network,
+ port0_prefix,
+ lower_sport,
+ upper_sport,
+ lower_dport,
+ upper_dport,
+ 1))
+
+ self._rules = '\n'.join(rule_list)
+
+ def get_rules(self):
+ if not self._rules:
+ self._read_config()
+ self._get_entries()
+ return self._rules