diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/process-templates.py | 54 | ||||
| -rwxr-xr-x | tools/yaml-validate.py | 74 |
2 files changed, 121 insertions, 7 deletions
diff --git a/tools/process-templates.py b/tools/process-templates.py index 9a06812b..1c8c4ba6 100755 --- a/tools/process-templates.py +++ b/tools/process-templates.py @@ -14,10 +14,13 @@ import argparse import jinja2 import os +import shutil import six import sys import yaml +__tht_root_dir = os.path.dirname(os.path.dirname(__file__)) + def parse_opts(argv): parser = argparse.ArgumentParser( @@ -33,6 +36,9 @@ def parse_opts(argv): action='store_true', help="""Enable safe mode (do not overwrite files).""", default=False) + parser.add_argument('-o', '--output-dir', metavar='OUTPUT_DIR', + help="""Output dir for all the templates""", + default='') opts = parser.parse_args(argv[1:]) return opts @@ -47,9 +53,14 @@ def _j2_render_to_file(j2_template, j2_data, outfile_name=None, print('ERROR: path already exists for file: %s' % outfile_name) sys.exit(1) + # Search for templates relative to the current template path first + template_base = os.path.dirname(yaml_f) + j2_loader = jinja2.loaders.FileSystemLoader([template_base, __tht_root_dir]) + try: # Render the j2 template - template = jinja2.Environment().from_string(j2_template) + template = jinja2.Environment(loader=j2_loader).from_string( + j2_template) r_template = template.render(**j2_data) except jinja2.exceptions.TemplateError as ex: error_msg = ("Error rendering template %s : %s" @@ -60,7 +71,7 @@ def _j2_render_to_file(j2_template, j2_data, outfile_name=None, out_f.write(r_template) -def process_templates(template_path, role_data_path, overwrite): +def process_templates(template_path, role_data_path, output_dir, overwrite): with open(role_data_path) as role_data_file: role_data = yaml.safe_load(role_data_file) @@ -69,6 +80,11 @@ def process_templates(template_path, role_data_path, overwrite): with open(j2_excludes_path) as role_data_file: j2_excludes = yaml.safe_load(role_data_file) + if output_dir and not os.path.isdir(output_dir): + if os.path.exists(output_dir): + raise RuntimeError('Output dir %s is not a directory' % output_dir) + os.mkdir(output_dir) + role_names = [r.get('name') for r in role_data] r_map = {} for r in role_data: @@ -78,6 +94,29 @@ def process_templates(template_path, role_data_path, overwrite): if os.path.isdir(template_path): for subdir, dirs, files in os.walk(template_path): + + # NOTE(flaper87): Ignore hidden dirs as we don't + # generate templates for those. + # Note the slice assigment for `dirs` is necessary + # because we need to modify the *elements* in the + # dirs list rather than the reference to the list. + # This way we'll make sure os.walk will iterate over + # the shrunk list. os.walk doesn't have an API for + # filtering dirs at this point. + dirs[:] = [d for d in dirs if not d[0] == '.'] + files = [f for f in files if not f[0] == '.'] + + # NOTE(flaper87): We could have used shutil.copytree + # but it requires the dst dir to not be present. This + # approach is safer as it doesn't require us to delete + # the output_dir in advance and it allows for running + # the command multiple times with the same output_dir. + out_dir = subdir + if output_dir: + out_dir = os.path.join(output_dir, subdir) + if not os.path.exists(out_dir): + os.mkdir(out_dir) + for f in files: file_path = os.path.join(subdir, f) # We do two templating passes here: @@ -101,7 +140,7 @@ def process_templates(template_path, role_data_path, overwrite): [role.lower(), os.path.basename(f).replace('.role.j2.yaml', '.yaml')]) - out_f_path = os.path.join(subdir, out_f) + out_f_path = os.path.join(out_dir, out_f) if not (out_f_path in excl_templates): _j2_render_to_file(template_data, j2_data, out_f_path, overwrite) @@ -112,9 +151,12 @@ def process_templates(template_path, role_data_path, overwrite): with open(file_path) as j2_template: template_data = j2_template.read() j2_data = {'roles': role_data} - out_f = file_path.replace('.j2.yaml', '.yaml') - _j2_render_to_file(template_data, j2_data, out_f, + out_f = os.path.basename(f).replace('.j2.yaml', '.yaml') + out_f_path = os.path.join(out_dir, out_f) + _j2_render_to_file(template_data, j2_data, out_f_path, overwrite) + elif output_dir: + shutil.copy(os.path.join(subdir, f), out_dir) else: print('Unexpected argument %s' % template_path) @@ -123,4 +165,4 @@ opts = parse_opts(sys.argv) role_data_path = os.path.join(opts.base_path, opts.roles_data) -process_templates(opts.base_path, role_data_path, (not opts.safe)) +process_templates(opts.base_path, role_data_path, opts.output_dir, (not opts.safe)) diff --git a/tools/yaml-validate.py b/tools/yaml-validate.py index fd1f47de..0eacbc60 100755 --- a/tools/yaml-validate.py +++ b/tools/yaml-validate.py @@ -19,11 +19,41 @@ import yaml required_params = ['EndpointMap', 'ServiceNetMap', 'DefaultPasswords'] +envs_containing_endpoint_map = ['tls-endpoints-public-dns.yaml', + 'tls-endpoints-public-ip.yaml', + 'tls-everywhere-endpoints-dns.yaml'] +ENDPOINT_MAP_FILE = 'endpoint_map.yaml' + def exit_usage(): print('Usage %s <yaml file or directory>' % sys.argv[0]) sys.exit(1) +def get_base_endpoint_map(filename): + try: + tpl = yaml.load(open(filename).read()) + return tpl['parameters']['EndpointMap']['default'] + except Exception: + print(traceback.format_exc()) + return None + + +def get_endpoint_map_from_env(filename): + try: + tpl = yaml.load(open(filename).read()) + return { + 'file': filename, + 'map': tpl['parameter_defaults']['EndpointMap'] + } + except Exception: + print(traceback.format_exc()) + return None + + +def validate_endpoint_map(base_map, env_map): + return sorted(base_map.keys()) == sorted(env_map.keys()) + + def validate_mysql_connection(settings): no_op = lambda *args: False error_status = [0] @@ -36,7 +66,10 @@ def validate_mysql_connection(settings): def validate_mysql_uri(key, items): # Only consider a connection if it targets mysql - if key.endswith('connection') and \ + # TODO(owalsh): skip nova mysql uris,temporary workaround for + # tripleo/+bug/1662344 + if not key.startswith('nova') and \ + key.endswith('connection') and \ search(items, mysql_protocol, no_op): # Assume the "bind_address" option is one of # the token that made up the uri @@ -101,6 +134,13 @@ def validate(filename): try: tpl = yaml.load(open(filename).read()) + # The template alias version should be used instead a date, this validation + # will be applied to all templates not just for those in the services folder. + if 'heat_template_version' in tpl and not str(tpl['heat_template_version']).isalpha(): + print('ERROR: heat_template_version needs to be the release alias not a date: %s' + % filename) + return 1 + if (filename.startswith('./puppet/services/') and filename != './puppet/services/services.yaml'): retval = validate_service(filename, tpl) @@ -128,6 +168,8 @@ if len(sys.argv) < 2: path_args = sys.argv[1:] exit_val = 0 failed_files = [] +base_endpoint_map = None +env_endpoint_maps = list() for base_path in path_args: if os.path.isdir(base_path): @@ -139,6 +181,12 @@ for base_path in path_args: if failed: failed_files.append(file_path) exit_val |= failed + if f == ENDPOINT_MAP_FILE: + base_endpoint_map = get_base_endpoint_map(file_path) + if f in envs_containing_endpoint_map: + env_endpoint_map = get_endpoint_map_from_env(file_path) + if env_endpoint_map: + env_endpoint_maps.append(env_endpoint_map) elif os.path.isfile(base_path) and base_path.endswith('.yaml'): failed = validate(base_path) if failed: @@ -148,6 +196,30 @@ for base_path in path_args: print('Unexpected argument %s' % base_path) exit_usage() +if base_endpoint_map and \ + len(env_endpoint_maps) == len(envs_containing_endpoint_map): + for env_endpoint_map in env_endpoint_maps: + matches = validate_endpoint_map(base_endpoint_map, + env_endpoint_map['map']) + if not matches: + print("ERROR: %s needs to be updated to match changes in base " + "endpoint map" % env_endpoint_map['file']) + failed_files.append(env_endpoint_map['file']) + exit_val |= 1 + else: + print("%s matches base endpoint map" % env_endpoint_map['file']) +else: + print("ERROR: Can't validate endpoint maps since a file is missing. " + "If you meant to delete one of these files you should update this " + "tool as well.") + if not base_endpoint_map: + failed_files.append(ENDPOINT_MAP_FILE) + if len(env_endpoint_maps) != len(envs_containing_endpoint_map): + matched_files = set(os.path.basename(matched_env_file['file']) + for matched_env_file in env_endpoint_maps) + failed_files.extend(set(envs_containing_endpoint_map) - matched_files) + exit_val |= 1 + if failed_files: print('Validation failed on:') for f in failed_files: |