From c3d309865bb57e2aba2033d3bbe1c16adca4f0ba Mon Sep 17 00:00:00 2001 From: Ross Brattain Date: Mon, 7 Aug 2017 14:32:45 -0700 Subject: YAML fixes There are multiple issues wiht YAML loading. 1. Jinja2 renders None values as a string 'None'. This is not valid YAML we need to render None values to '~' or 'null' which is the native YAML None value. 2. Jinja2 renders dict and lists that contain unicode with u'foo' values. This is not value YAML syntax. Because we are serializing dict and lists into YAML, we need to encode them as valid YAML. We can override Jinja2 finalize to use yaml.dump to dump inline YAML. We use yaml.safe_dump(elem, default_flow_style=True).replace('\n', '') to generate valid single-line YAML dict and list values. But this problem highlights the general difficulties with templating and loading files. We could avoid this Python->Jinja2->YAML->Python issue by directly injecting the list or dict after the YAML is loaded. I'm not sure of the real utility of these templates. 3. On Python 2 YAML loader is rendering all strings as unicode. This does not work for Trex because Trex is broken and badly coded. Trex does type checking against str() which is different for Python 2 and Python 3. The default YAML loader will return native string types, str() or unicode() for Python 2 and Python 3 respectively. The bad Trex codes is in convert_val: https://github.com/cisco-system-traffic-generator/trex-core/blob/master/scripts/automation/trex_control_plane/stl/trex_stl_lib/trex_stl_packet_builder_scapy.py#L674 def convert_val (val): if is_integer(val): return val if type(val) == str: return ipv4_str_to_num (is_valid_ipv4(val)) raise CTRexPacketBuildException(-11,("init val invalid %s ") % val ); This code is doing type(val) == str. This is bad and broken. We can't fix Trex, so we have to render all strings as native str() types The bug here was that the Heat template loader template_format.py was overriding the global YAML loader to always return unicode. We don't want this global override. To fix this we have to use local subclasses of the yaml.SafeLoader class. But in order to dynamically subclass from CSafeLoader or SafeLoader we have to use the type() builtin to define a new class at runtime. Once we have new classes defined, we can safely isolate different YAML constructors and return unicode or not depending on the case. To be consistent we implement a new yaml_loader.py module to centralize all non-Heat template yaml loading to ensure correct uncode/str conversion Change-Id: Iebf9cf78fbda390977c390436b0869e7bbf503eb Signed-off-by: Ross Brattain Signed-off-by: Deepak S Signed-off-by: Ross Brattain --- api/resources/v1/env.py | 4 ++-- api/resources/v2/pods.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'api') diff --git a/api/resources/v1/env.py b/api/resources/v1/env.py index 8367fa9eb..98b8ec7e4 100644 --- a/api/resources/v1/env.py +++ b/api/resources/v1/env.py @@ -31,7 +31,7 @@ from yardstick.common import utils from yardstick.common.utils import result_handler from yardstick.common import openstack_utils from yardstick.common.httpClient import HttpClient - +from yardstick.common.yaml_loader import yaml_load LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) @@ -393,7 +393,7 @@ class V1Env(ApiResource): return result_handler(consts.API_ERROR, 'file must be provided') LOG.info('Checking file') - data = yaml.safe_load(pod_file.read()) + data = yaml_load(pod_file.read()) if not isinstance(data, collections.Mapping): return result_handler(consts.API_ERROR, 'invalid yaml file') diff --git a/api/resources/v2/pods.py b/api/resources/v2/pods.py index f2316d353..d98238ca1 100644 --- a/api/resources/v2/pods.py +++ b/api/resources/v2/pods.py @@ -18,6 +18,7 @@ from api.database.v2.handlers import V2EnvironmentHandler from yardstick.common import constants as consts from yardstick.common.utils import result_handler from yardstick.common.task_template import TaskTemplate +from yardstick.common.yaml_loader import yaml_load LOG = logging.getLogger(__name__) LOG.setLevel(logging.DEBUG) @@ -48,7 +49,7 @@ class V2Pods(ApiResource): upload_file.save(consts.POD_FILE) with open(consts.POD_FILE) as f: - data = yaml.safe_load(TaskTemplate.render(f.read())) + data = yaml_load(TaskTemplate.render(f.read())) LOG.debug('pod content is: %s', data) LOG.info('create pod in database') -- cgit 1.2.3-korg