diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/process-templates.py | 125 | ||||
| -rwxr-xr-x | tools/yaml-nic-config-2-script.py | 219 |
2 files changed, 344 insertions, 0 deletions
diff --git a/tools/process-templates.py b/tools/process-templates.py new file mode 100755 index 00000000..a15b00e2 --- /dev/null +++ b/tools/process-templates.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import jinja2 +import os +import sys +import yaml + + +def parse_opts(argv): + parser = argparse.ArgumentParser( + description='Configure host network interfaces using a JSON' + ' config file format.') + parser.add_argument('-p', '--base_path', metavar='BASE_PATH', + help="""base path of templates to process.""", + default='.') + parser.add_argument('-r', '--roles-data', metavar='ROLES_DATA', + help="""relative path to the roles_data.yaml file.""", + default='roles_data.yaml') + parser.add_argument('--safe', + action='store_true', + help="""Enable safe mode (do not overwrite files).""", + default=False) + opts = parser.parse_args(argv[1:]) + + return opts + + +def _j2_render_to_file(j2_template, j2_data, outfile_name=None, + overwrite=True): + yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml') + print('rendering j2 template to file: %s' % outfile_name) + + if not overwrite and os.path.exists(outfile_name): + print('ERROR: path already exists for file: %s' % outfile_name) + sys.exit(1) + + try: + # Render the j2 template + template = jinja2.Environment().from_string(j2_template) + r_template = template.render(**j2_data) + except jinja2.exceptions.TemplateError as ex: + error_msg = ("Error rendering template %s : %s" + % (yaml_f, six.text_type(ex))) + print(error_msg) + raise Exception(error_msg) + with open(outfile_name, 'w') as out_f: + out_f.write(r_template) + + +def process_templates(template_path, role_data_path, overwrite): + + with open(role_data_path) as role_data_file: + role_data = yaml.safe_load(role_data_file) + + j2_excludes_path = os.path.join(template_path, 'j2_excludes.yaml') + with open(j2_excludes_path) as role_data_file: + j2_excludes = yaml.safe_load(role_data_file) + + role_names = [r.get('name') for r in role_data] + r_map = {} + for r in role_data: + r_map[r.get('name')] = r + excl_templates = ['%s/%s' % (template_path, e) + for e in j2_excludes.get('name')] + + if os.path.isdir(template_path): + for subdir, dirs, files in os.walk(template_path): + for f in files: + file_path = os.path.join(subdir, f) + # We do two templating passes here: + # 1. *.role.j2.yaml - we template just the role name + # and create multiple files (one per role) + # 2. *.j2.yaml - we template with all roles_data, + # and create one file common to all roles + if f.endswith('.role.j2.yaml'): + print("jinja2 rendering role template %s" % f) + with open(file_path) as j2_template: + template_data = j2_template.read() + print("jinja2 rendering roles %s" % "," + .join(role_names)) + for role in role_names: + j2_data = {'role': role} + # (dprince) For the undercloud installer we don't + # want to have heat check nova/glance API's + if r_map[role].get('disable_constraints', False): + j2_data['disable_constraints'] = True + out_f = "-".join( + [role.lower(), + os.path.basename(f).replace('.role.j2.yaml', + '.yaml')]) + out_f_path = os.path.join(subdir, out_f) + if not (out_f_path in excl_templates): + _j2_render_to_file(template_data, j2_data, + out_f_path, overwrite) + else: + print('skipping rendering of %s' % out_f_path) + elif f.endswith('.j2.yaml'): + print("jinja2 rendering normal template %s" % f) + with open(file_path) as j2_template: + template_data = j2_template.read() + j2_data = {'roles': role_data} + out_f = file_path.replace('.j2.yaml', '.yaml') + _j2_render_to_file(template_data, j2_data, out_f, + overwrite) + + else: + print('Unexpected argument %s' % template_path) + +opts = parse_opts(sys.argv) + +role_data_path = os.path.join(opts.base_path, opts.roles_data) + +process_templates(opts.base_path, role_data_path, (not opts.safe)) diff --git a/tools/yaml-nic-config-2-script.py b/tools/yaml-nic-config-2-script.py new file mode 100755 index 00000000..b8f07e4f --- /dev/null +++ b/tools/yaml-nic-config-2-script.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import copy +import os +import sys +import traceback +import yaml +import six +import re + + +#convert comments into 'comments<num>: ...' YAML +def to_commented_yaml(filename): + out_str = '' + last_non_comment_spaces = '' + with open(filename, 'r') as f: + comment_count = 0 + for line in f: + char_count = 0 + spaces = '' + for char in line: + char_count += 1 + if char == ' ': + spaces+=' ' + next; + elif char == '#': + comment_count += 1 + comment = line[char_count:-1] + out_str += "%scomment%i_%i: '%s'\n" % (last_non_comment_spaces, comment_count, len(spaces), comment) + break; + else: + last_non_comment_spaces = spaces + out_str += line + + #inline comments check + m = re.match(".*:.*#(.*)", line) + if m: + comment_count += 1 + out_str += "%s inline_comment%i: '%s'\n" % (last_non_comment_spaces, comment_count, m.group(1)) + break; + + with open(filename, 'w') as f: + f.write(out_str) + + return out_str + +#convert back to normal #commented YAML +def to_normal_yaml(filename): + + with open(filename, 'r') as f: + data = f.read() + + out_str = '' + next_line_break = False + for line in data.split('\n'): + m = re.match(" +comment[0-9]+_([0-9]+): '(.*)'.*", line) #normal comments + i = re.match(" +inline_comment[0-9]+: '(.*)'.*", line) #inline comments + if m: + if next_line_break: + out_str += '\n' + next_line_break = False + for x in range(0, int(m.group(1))): + out_str += " " + out_str += "#%s\n" % m.group(2) + elif i: + out_str += " #%s\n" % i.group(1) + next_line_break = False + else: + if next_line_break: + out_str += '\n' + out_str += line + next_line_break = True + + if next_line_break: + out_str += '\n' + + with open(filename, 'w') as f: + f.write(out_str) + + return out_str + + +class description(six.text_type): + pass + +# FIXME: Some of this duplicates code from build_endpoint_map.py, we should +# refactor to share the common code +class TemplateDumper(yaml.SafeDumper): + def represent_ordered_dict(self, data): + return self.represent_dict(data.items()) + + def description_presenter(self, data): + if '\n' in data: + style = '>' + else: + style = '' + return self.represent_scalar( + yaml.resolver.BaseResolver.DEFAULT_SCALAR_TAG, data, style=style) + + +# We load mappings into OrderedDict to preserve their order +class TemplateLoader(yaml.SafeLoader): + def construct_mapping(self, node): + self.flatten_mapping(node) + return collections.OrderedDict(self.construct_pairs(node)) + + +TemplateDumper.add_representer(description, + TemplateDumper.description_presenter) + +TemplateDumper.add_representer(collections.OrderedDict, + TemplateDumper.represent_ordered_dict) + + +TemplateLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + TemplateLoader.construct_mapping) + +def write_template(template, filename=None): + with open(filename, 'w') as f: + yaml.dump(template, f, TemplateDumper, width=120, default_flow_style=False) + +def exit_usage(): + print('Usage %s <yaml file>' % sys.argv[0]) + sys.exit(1) + +def convert(filename): + print('Converting %s' % filename) + try: + tpl = yaml.load(open(filename).read(), Loader=TemplateLoader) + except Exception: + print(traceback.format_exc()) + return 0 + + # Check which path we need for run-os-net-config.sh because we have + # nic config templates in the top-level and network/config + script_paths = ['network/scripts/run-os-net-config.sh', + '../../scripts/run-os-net-config.sh'] + script_path = None + for p in script_paths: + check_path = os.path.join(os.path.dirname(filename), p) + if os.path.isfile(check_path): + print("Found %s, using %s" % (check_path, p)) + script_path = p + if script_path is None: + print("Error couldn't find run-os-net-config.sh relative to filename") + exit_usage() + + for r in six.iteritems(tpl.get('resources', {})): + if (r[1].get('type') == 'OS::Heat::StructuredConfig' and + r[1].get('properties', {}).get('group') == 'os-apply-config' and + r[1].get('properties', {}).get('config', {}).get('os_net_config')): + #print("match %s" % r[0]) + new_r = collections.OrderedDict() + new_r['type'] = 'OS::Heat::SoftwareConfig' + new_r['properties'] = collections.OrderedDict() + new_r['properties']['group'] = 'script' + old_net_config = r[1].get( + 'properties', {}).get('config', {}).get('os_net_config') + new_config = {'str_replace': collections.OrderedDict()} + new_config['str_replace']['template'] = {'get_file': script_path} + new_config['str_replace']['params'] = {'$network_config': old_net_config} + new_r['properties']['config'] = new_config + tpl['resources'][r[0]] = new_r + else: + print("No match %s" % r[0]) + return 0 + + # Preserve typical HOT template key ordering + od_result = collections.OrderedDict() + # Need to bump the HOT version so str_replace supports serializing to json + od_result['heat_template_version'] = "2016-10-14" + if tpl.get('description'): + od_result['description'] = description(tpl['description']) + od_result['parameters'] = tpl['parameters'] + od_result['resources'] = tpl['resources'] + od_result['outputs'] = tpl['outputs'] + #print('Result:') + #print('%s' % yaml.dump(od_result, Dumper=TemplateDumper, width=120, default_flow_style=False)) + #print('---') + #replace = raw_input( + #"Replace file %s? Answer y/n" % filename).lower() == 'y' + #if replace: + #print("Replace %s" % filename) + write_template(od_result, filename) + #else: + # print("NOT replacing %s" % filename) + # return 0 + return 1 + +if len(sys.argv) < 2: + exit_usage() + +path_args = sys.argv[1:] +exit_val = 0 +num_converted = 0 + +for base_path in path_args: + if os.path.isfile(base_path) and base_path.endswith('.yaml'): + to_commented_yaml(base_path) + num_converted += convert(base_path) + to_normal_yaml(base_path) + else: + print('Unexpected argument %s' % base_path) + exit_usage() +if num_converted == 0: + exit_val = 1 +sys.exit(exit_val) |