summaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rwxr-xr-xtools/process-templates.py55
-rwxr-xr-xtools/releasenotes_tox.sh28
-rwxr-xr-xtools/tox_install.sh30
-rwxr-xr-xtools/yaml-validate.py115
4 files changed, 222 insertions, 6 deletions
diff --git a/tools/process-templates.py b/tools/process-templates.py
index a15b00e2..1c8c4ba6 100755
--- a/tools/process-templates.py
+++ b/tools/process-templates.py
@@ -14,9 +14,13 @@
import argparse
import jinja2
import os
+import shutil
+import six
import sys
import yaml
+__tht_root_dir = os.path.dirname(os.path.dirname(__file__))
+
def parse_opts(argv):
parser = argparse.ArgumentParser(
@@ -32,6 +36,9 @@ def parse_opts(argv):
action='store_true',
help="""Enable safe mode (do not overwrite files).""",
default=False)
+ parser.add_argument('-o', '--output-dir', metavar='OUTPUT_DIR',
+ help="""Output dir for all the templates""",
+ default='')
opts = parser.parse_args(argv[1:])
return opts
@@ -46,9 +53,14 @@ def _j2_render_to_file(j2_template, j2_data, outfile_name=None,
print('ERROR: path already exists for file: %s' % outfile_name)
sys.exit(1)
+ # Search for templates relative to the current template path first
+ template_base = os.path.dirname(yaml_f)
+ j2_loader = jinja2.loaders.FileSystemLoader([template_base, __tht_root_dir])
+
try:
# Render the j2 template
- template = jinja2.Environment().from_string(j2_template)
+ template = jinja2.Environment(loader=j2_loader).from_string(
+ j2_template)
r_template = template.render(**j2_data)
except jinja2.exceptions.TemplateError as ex:
error_msg = ("Error rendering template %s : %s"
@@ -59,7 +71,7 @@ def _j2_render_to_file(j2_template, j2_data, outfile_name=None,
out_f.write(r_template)
-def process_templates(template_path, role_data_path, overwrite):
+def process_templates(template_path, role_data_path, output_dir, overwrite):
with open(role_data_path) as role_data_file:
role_data = yaml.safe_load(role_data_file)
@@ -68,6 +80,11 @@ def process_templates(template_path, role_data_path, overwrite):
with open(j2_excludes_path) as role_data_file:
j2_excludes = yaml.safe_load(role_data_file)
+ if output_dir and not os.path.isdir(output_dir):
+ if os.path.exists(output_dir):
+ raise RuntimeError('Output dir %s is not a directory' % output_dir)
+ os.mkdir(output_dir)
+
role_names = [r.get('name') for r in role_data]
r_map = {}
for r in role_data:
@@ -77,6 +94,29 @@ def process_templates(template_path, role_data_path, overwrite):
if os.path.isdir(template_path):
for subdir, dirs, files in os.walk(template_path):
+
+ # NOTE(flaper87): Ignore hidden dirs as we don't
+ # generate templates for those.
+ # Note the slice assigment for `dirs` is necessary
+ # because we need to modify the *elements* in the
+ # dirs list rather than the reference to the list.
+ # This way we'll make sure os.walk will iterate over
+ # the shrunk list. os.walk doesn't have an API for
+ # filtering dirs at this point.
+ dirs[:] = [d for d in dirs if not d[0] == '.']
+ files = [f for f in files if not f[0] == '.']
+
+ # NOTE(flaper87): We could have used shutil.copytree
+ # but it requires the dst dir to not be present. This
+ # approach is safer as it doesn't require us to delete
+ # the output_dir in advance and it allows for running
+ # the command multiple times with the same output_dir.
+ out_dir = subdir
+ if output_dir:
+ out_dir = os.path.join(output_dir, subdir)
+ if not os.path.exists(out_dir):
+ os.mkdir(out_dir)
+
for f in files:
file_path = os.path.join(subdir, f)
# We do two templating passes here:
@@ -100,7 +140,7 @@ def process_templates(template_path, role_data_path, overwrite):
[role.lower(),
os.path.basename(f).replace('.role.j2.yaml',
'.yaml')])
- out_f_path = os.path.join(subdir, out_f)
+ out_f_path = os.path.join(out_dir, out_f)
if not (out_f_path in excl_templates):
_j2_render_to_file(template_data, j2_data,
out_f_path, overwrite)
@@ -111,9 +151,12 @@ def process_templates(template_path, role_data_path, overwrite):
with open(file_path) as j2_template:
template_data = j2_template.read()
j2_data = {'roles': role_data}
- out_f = file_path.replace('.j2.yaml', '.yaml')
- _j2_render_to_file(template_data, j2_data, out_f,
+ out_f = os.path.basename(f).replace('.j2.yaml', '.yaml')
+ out_f_path = os.path.join(out_dir, out_f)
+ _j2_render_to_file(template_data, j2_data, out_f_path,
overwrite)
+ elif output_dir:
+ shutil.copy(os.path.join(subdir, f), out_dir)
else:
print('Unexpected argument %s' % template_path)
@@ -122,4 +165,4 @@ opts = parse_opts(sys.argv)
role_data_path = os.path.join(opts.base_path, opts.roles_data)
-process_templates(opts.base_path, role_data_path, (not opts.safe))
+process_templates(opts.base_path, role_data_path, opts.output_dir, (not opts.safe))
diff --git a/tools/releasenotes_tox.sh b/tools/releasenotes_tox.sh
new file mode 100755
index 00000000..4fecfd92
--- /dev/null
+++ b/tools/releasenotes_tox.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+rm -rf releasenotes/build
+
+sphinx-build -a -E -W \
+ -d releasenotes/build/doctrees \
+ -b html \
+ releasenotes/source releasenotes/build/html
+BUILD_RESULT=$?
+
+UNCOMMITTED_NOTES=$(git status --porcelain | \
+ awk '$1 == "M" && $2 ~ /releasenotes\/notes/ {print $2}')
+
+if [ "${UNCOMMITTED_NOTES}" ]
+then
+ cat <<EOF
+
+REMINDER: The following changes to release notes have not been committed:
+
+${UNCOMMITTED_NOTES}
+
+While that may be intentional, keep in mind that release notes are built from
+committed changes, not the working directory.
+
+EOF
+fi
+
+exit ${BUILD_RESULT}
diff --git a/tools/tox_install.sh b/tools/tox_install.sh
new file mode 100755
index 00000000..e61b63a8
--- /dev/null
+++ b/tools/tox_install.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+# Client constraint file contains this client version pin that is in conflict
+# with installing the client from source. We should remove the version pin in
+# the constraints file before applying it for from-source installation.
+
+CONSTRAINTS_FILE="$1"
+shift 1
+
+set -e
+
+# NOTE(tonyb): Place this in the tox enviroment's log dir so it will get
+# published to logs.openstack.org for easy debugging.
+localfile="$VIRTUAL_ENV/log/upper-constraints.txt"
+
+if [[ "$CONSTRAINTS_FILE" != http* ]]; then
+ CONSTRAINTS_FILE="file://$CONSTRAINTS_FILE"
+fi
+# NOTE(tonyb): need to add curl to bindep.txt if the project supports bindep
+curl "$CONSTRAINTS_FILE" --insecure --progress-bar --output "$localfile"
+
+pip install -c"$localfile" openstack-requirements
+
+# This is the main purpose of the script: Allow local installation of
+# the current repo. It is listed in constraints file and thus any
+# install will be constrained and we need to unconstrain it.
+edit-constraints "$localfile" -- "$CLIENT_NAME"
+
+pip install -c"$localfile" -U "$@"
+exit $?
diff --git a/tools/yaml-validate.py b/tools/yaml-validate.py
index 95c7d025..1d0dba02 100755
--- a/tools/yaml-validate.py
+++ b/tools/yaml-validate.py
@@ -19,11 +19,81 @@ import yaml
required_params = ['EndpointMap', 'ServiceNetMap', 'DefaultPasswords']
+envs_containing_endpoint_map = ['tls-endpoints-public-dns.yaml',
+ 'tls-endpoints-public-ip.yaml',
+ 'tls-everywhere-endpoints-dns.yaml']
+ENDPOINT_MAP_FILE = 'endpoint_map.yaml'
+
def exit_usage():
print('Usage %s <yaml file or directory>' % sys.argv[0])
sys.exit(1)
+def get_base_endpoint_map(filename):
+ try:
+ tpl = yaml.load(open(filename).read())
+ return tpl['parameters']['EndpointMap']['default']
+ except Exception:
+ print(traceback.format_exc())
+ return None
+
+
+def get_endpoint_map_from_env(filename):
+ try:
+ tpl = yaml.load(open(filename).read())
+ return {
+ 'file': filename,
+ 'map': tpl['parameter_defaults']['EndpointMap']
+ }
+ except Exception:
+ print(traceback.format_exc())
+ return None
+
+
+def validate_endpoint_map(base_map, env_map):
+ return sorted(base_map.keys()) == sorted(env_map.keys())
+
+
+def validate_mysql_connection(settings):
+ no_op = lambda *args: False
+ error_status = [0]
+
+ def mysql_protocol(items):
+ return items == ['EndpointMap', 'MysqlInternal', 'protocol']
+
+ def client_bind_address(item):
+ return 'read_default_file' in item and \
+ 'read_default_group' in item
+
+ def validate_mysql_uri(key, items):
+ # Only consider a connection if it targets mysql
+ if key.endswith('connection') and \
+ search(items, mysql_protocol, no_op):
+ # Assume the "bind_address" option is one of
+ # the token that made up the uri
+ if not search(items, client_bind_address, no_op):
+ error_status[0] = 1
+ return False
+
+ def search(item, check_item, check_key):
+ if check_item(item):
+ return True
+ elif isinstance(item, list):
+ for i in item:
+ if search(i, check_item, check_key):
+ return True
+ elif isinstance(item, dict):
+ for k in item.keys():
+ if check_key(k, item[k]):
+ return True
+ elif search(item[k], check_item, check_key):
+ return True
+ return False
+
+ search(settings, no_op, validate_mysql_uri)
+ return error_status[0]
+
+
def validate_service(filename, tpl):
if 'outputs' in tpl and 'role_data' in tpl['outputs']:
if 'value' not in tpl['outputs']['role_data']:
@@ -41,6 +111,12 @@ def validate_service(filename, tpl):
print('ERROR: service_name should match file name for service: %s.'
% filename)
return 1
+ # if service connects to mysql, the uri should use option
+ # bind_address to avoid issues with VIP failover
+ if 'config_settings' in role_data and \
+ validate_mysql_connection(role_data['config_settings']):
+ print('ERROR: mysql connection uri should use option bind_address')
+ return 1
if 'parameters' in tpl:
for param in required_params:
if param not in tpl['parameters']:
@@ -56,6 +132,13 @@ def validate(filename):
try:
tpl = yaml.load(open(filename).read())
+ # The template alias version should be used instead a date, this validation
+ # will be applied to all templates not just for those in the services folder.
+ if 'heat_template_version' in tpl and not str(tpl['heat_template_version']).isalpha():
+ print('ERROR: heat_template_version needs to be the release alias not a date: %s'
+ % filename)
+ return 1
+
if (filename.startswith('./puppet/services/') and
filename != './puppet/services/services.yaml'):
retval = validate_service(filename, tpl)
@@ -83,6 +166,8 @@ if len(sys.argv) < 2:
path_args = sys.argv[1:]
exit_val = 0
failed_files = []
+base_endpoint_map = None
+env_endpoint_maps = list()
for base_path in path_args:
if os.path.isdir(base_path):
@@ -94,6 +179,12 @@ for base_path in path_args:
if failed:
failed_files.append(file_path)
exit_val |= failed
+ if f == ENDPOINT_MAP_FILE:
+ base_endpoint_map = get_base_endpoint_map(file_path)
+ if f in envs_containing_endpoint_map:
+ env_endpoint_map = get_endpoint_map_from_env(file_path)
+ if env_endpoint_map:
+ env_endpoint_maps.append(env_endpoint_map)
elif os.path.isfile(base_path) and base_path.endswith('.yaml'):
failed = validate(base_path)
if failed:
@@ -103,6 +194,30 @@ for base_path in path_args:
print('Unexpected argument %s' % base_path)
exit_usage()
+if base_endpoint_map and \
+ len(env_endpoint_maps) == len(envs_containing_endpoint_map):
+ for env_endpoint_map in env_endpoint_maps:
+ matches = validate_endpoint_map(base_endpoint_map,
+ env_endpoint_map['map'])
+ if not matches:
+ print("ERROR: %s needs to be updated to match changes in base "
+ "endpoint map" % env_endpoint_map['file'])
+ failed_files.append(env_endpoint_map['file'])
+ exit_val |= 1
+ else:
+ print("%s matches base endpoint map" % env_endpoint_map['file'])
+else:
+ print("ERROR: Can't validate endpoint maps since a file is missing. "
+ "If you meant to delete one of these files you should update this "
+ "tool as well.")
+ if not base_endpoint_map:
+ failed_files.append(ENDPOINT_MAP_FILE)
+ if len(env_endpoint_maps) != len(envs_containing_endpoint_map):
+ matched_files = set(os.path.basename(matched_env_file['file'])
+ for matched_env_file in env_endpoint_maps)
+ failed_files.extend(set(envs_containing_endpoint_map) - matched_files)
+ exit_val |= 1
+
if failed_files:
print('Validation failed on:')
for f in failed_files: