diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/process-templates.py | 46 | ||||
| -rwxr-xr-x | tools/releasenotes_tox.sh | 28 | ||||
| -rwxr-xr-x | tools/tox_install.sh | 30 | ||||
| -rwxr-xr-x | tools/yaml-validate.py | 66 |
4 files changed, 165 insertions, 5 deletions
diff --git a/tools/process-templates.py b/tools/process-templates.py index a15b00e2..7d66b35d 100755 --- a/tools/process-templates.py +++ b/tools/process-templates.py @@ -14,6 +14,8 @@ import argparse import jinja2 import os +import shutil +import six import sys import yaml @@ -32,6 +34,9 @@ def parse_opts(argv): action='store_true', help="""Enable safe mode (do not overwrite files).""", default=False) + parser.add_argument('-o', '--output-dir', metavar='OUTPUT_DIR', + help="""Output dir for all the templates""", + default='') opts = parser.parse_args(argv[1:]) return opts @@ -59,7 +64,7 @@ def _j2_render_to_file(j2_template, j2_data, outfile_name=None, out_f.write(r_template) -def process_templates(template_path, role_data_path, overwrite): +def process_templates(template_path, role_data_path, output_dir, overwrite): with open(role_data_path) as role_data_file: role_data = yaml.safe_load(role_data_file) @@ -68,6 +73,11 @@ def process_templates(template_path, role_data_path, overwrite): with open(j2_excludes_path) as role_data_file: j2_excludes = yaml.safe_load(role_data_file) + if output_dir and not os.path.isdir(output_dir): + if os.path.exists(output_dir): + raise RuntimeError('Output dir %s is not a directory' % output_dir) + os.mkdir(output_dir) + role_names = [r.get('name') for r in role_data] r_map = {} for r in role_data: @@ -77,6 +87,29 @@ def process_templates(template_path, role_data_path, overwrite): if os.path.isdir(template_path): for subdir, dirs, files in os.walk(template_path): + + # NOTE(flaper87): Ignore hidden dirs as we don't + # generate templates for those. + # Note the slice assigment for `dirs` is necessary + # because we need to modify the *elements* in the + # dirs list rather than the reference to the list. + # This way we'll make sure os.walk will iterate over + # the shrunk list. os.walk doesn't have an API for + # filtering dirs at this point. + dirs[:] = [d for d in dirs if not d[0] == '.'] + files = [f for f in files if not f[0] == '.'] + + # NOTE(flaper87): We could have used shutil.copytree + # but it requires the dst dir to not be present. This + # approach is safer as it doesn't require us to delete + # the output_dir in advance and it allows for running + # the command multiple times with the same output_dir. + out_dir = subdir + if output_dir: + out_dir = os.path.join(output_dir, subdir) + if not os.path.exists(out_dir): + os.mkdir(out_dir) + for f in files: file_path = os.path.join(subdir, f) # We do two templating passes here: @@ -100,7 +133,7 @@ def process_templates(template_path, role_data_path, overwrite): [role.lower(), os.path.basename(f).replace('.role.j2.yaml', '.yaml')]) - out_f_path = os.path.join(subdir, out_f) + out_f_path = os.path.join(out_dir, out_f) if not (out_f_path in excl_templates): _j2_render_to_file(template_data, j2_data, out_f_path, overwrite) @@ -111,9 +144,12 @@ def process_templates(template_path, role_data_path, overwrite): with open(file_path) as j2_template: template_data = j2_template.read() j2_data = {'roles': role_data} - out_f = file_path.replace('.j2.yaml', '.yaml') - _j2_render_to_file(template_data, j2_data, out_f, + out_f = os.path.basename(f).replace('.j2.yaml', '.yaml') + out_f_path = os.path.join(out_dir, out_f) + _j2_render_to_file(template_data, j2_data, out_f_path, overwrite) + elif output_dir: + shutil.copy(os.path.join(subdir, f), out_dir) else: print('Unexpected argument %s' % template_path) @@ -122,4 +158,4 @@ opts = parse_opts(sys.argv) role_data_path = os.path.join(opts.base_path, opts.roles_data) -process_templates(opts.base_path, role_data_path, (not opts.safe)) +process_templates(opts.base_path, role_data_path, opts.output_dir, (not opts.safe)) diff --git a/tools/releasenotes_tox.sh b/tools/releasenotes_tox.sh new file mode 100755 index 00000000..4fecfd92 --- /dev/null +++ b/tools/releasenotes_tox.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +rm -rf releasenotes/build + +sphinx-build -a -E -W \ + -d releasenotes/build/doctrees \ + -b html \ + releasenotes/source releasenotes/build/html +BUILD_RESULT=$? + +UNCOMMITTED_NOTES=$(git status --porcelain | \ + awk '$1 == "M" && $2 ~ /releasenotes\/notes/ {print $2}') + +if [ "${UNCOMMITTED_NOTES}" ] +then + cat <<EOF + +REMINDER: The following changes to release notes have not been committed: + +${UNCOMMITTED_NOTES} + +While that may be intentional, keep in mind that release notes are built from +committed changes, not the working directory. + +EOF +fi + +exit ${BUILD_RESULT} diff --git a/tools/tox_install.sh b/tools/tox_install.sh new file mode 100755 index 00000000..e61b63a8 --- /dev/null +++ b/tools/tox_install.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +# Client constraint file contains this client version pin that is in conflict +# with installing the client from source. We should remove the version pin in +# the constraints file before applying it for from-source installation. + +CONSTRAINTS_FILE="$1" +shift 1 + +set -e + +# NOTE(tonyb): Place this in the tox enviroment's log dir so it will get +# published to logs.openstack.org for easy debugging. +localfile="$VIRTUAL_ENV/log/upper-constraints.txt" + +if [[ "$CONSTRAINTS_FILE" != http* ]]; then + CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE" +fi +# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep +curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile" + +pip install -c"$localfile" openstack-requirements + +# This is the main purpose of the script: Allow local installation of +# the current repo. It is listed in constraints file and thus any +# install will be constrained and we need to unconstrain it. +edit-constraints "$localfile" -- "$CLIENT_NAME" + +pip install -c"$localfile" -U "$@" +exit $? diff --git a/tools/yaml-validate.py b/tools/yaml-validate.py index fd1f47de..63e3ce51 100755 --- a/tools/yaml-validate.py +++ b/tools/yaml-validate.py @@ -19,11 +19,41 @@ import yaml required_params = ['EndpointMap', 'ServiceNetMap', 'DefaultPasswords'] +envs_containing_endpoint_map = ['tls-endpoints-public-dns.yaml', + 'tls-endpoints-public-ip.yaml', + 'tls-everywhere-endpoints-dns.yaml'] +ENDPOINT_MAP_FILE = 'endpoint_map.yaml' + def exit_usage(): print('Usage %s <yaml file or directory>' % sys.argv[0]) sys.exit(1) +def get_base_endpoint_map(filename): + try: + tpl = yaml.load(open(filename).read()) + return tpl['parameters']['EndpointMap']['default'] + except Exception: + print(traceback.format_exc()) + return None + + +def get_endpoint_map_from_env(filename): + try: + tpl = yaml.load(open(filename).read()) + return { + 'file': filename, + 'map': tpl['parameter_defaults']['EndpointMap'] + } + except Exception: + print(traceback.format_exc()) + return None + + +def validate_endpoint_map(base_map, env_map): + return sorted(base_map.keys()) == sorted(env_map.keys()) + + def validate_mysql_connection(settings): no_op = lambda *args: False error_status = [0] @@ -64,6 +94,10 @@ def validate_mysql_connection(settings): def validate_service(filename, tpl): + if 'heat_template_version' in tpl and not str(tpl['heat_template_version']).isalpha(): + print('ERROR: heat_template_version needs to be the release alias not a date: %s' + % filename) + return 1 if 'outputs' in tpl and 'role_data' in tpl['outputs']: if 'value' not in tpl['outputs']['role_data']: print('ERROR: invalid role_data for filename: %s' @@ -128,6 +162,8 @@ if len(sys.argv) < 2: path_args = sys.argv[1:] exit_val = 0 failed_files = [] +base_endpoint_map = None +env_endpoint_maps = list() for base_path in path_args: if os.path.isdir(base_path): @@ -139,6 +175,12 @@ for base_path in path_args: if failed: failed_files.append(file_path) exit_val |= failed + if f == ENDPOINT_MAP_FILE: + base_endpoint_map = get_base_endpoint_map(file_path) + if f in envs_containing_endpoint_map: + env_endpoint_map = get_endpoint_map_from_env(file_path) + if env_endpoint_map: + env_endpoint_maps.append(env_endpoint_map) elif os.path.isfile(base_path) and base_path.endswith('.yaml'): failed = validate(base_path) if failed: @@ -148,6 +190,30 @@ for base_path in path_args: print('Unexpected argument %s' % base_path) exit_usage() +if base_endpoint_map and \ + len(env_endpoint_maps) == len(envs_containing_endpoint_map): + for env_endpoint_map in env_endpoint_maps: + matches = validate_endpoint_map(base_endpoint_map, + env_endpoint_map['map']) + if not matches: + print("ERROR: %s doesn't match base endpoint map" % + env_endpoint_map['file']) + failed_files.append(env_endpoint_map['file']) + exit_val |= 1 + else: + print("%s matches base endpoint map" % env_endpoint_map['file']) +else: + print("ERROR: Can't validate endpoint maps since a file is missing. " + "If you meant to delete one of these files you should update this " + "tool as well.") + if not base_endpoint_map: + failed_files.append(ENDPOINT_MAP_FILE) + if len(env_endpoint_maps) != len(envs_containing_endpoint_map): + matched_files = set(os.path.basename(matched_env_file['file']) + for matched_env_file in env_endpoint_maps) + failed_files.extend(set(envs_containing_endpoint_map) - matched_files) + exit_val |= 1 + if failed_files: print('Validation failed on:') for f in failed_files: |