summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDan Prince <dprince@redhat.com>2016-11-07 07:45:15 -0500
committerDan Prince <dprince@redhat.com>2016-11-30 16:02:44 -0500
commit4b5b24462b7c37bd784502d8afa341181a9fab9a (patch)
tree99777eba8c1b08b9a93900609372ef9195d051ac
parent6df32707e9698da5a647aff2b20e6fc2617ea1d2 (diff)
Add local template generation tox task
This patch adds a local version of our template processing routine so that developers can more quickly view the templates that are actually getting generated. I've noticed multiple developers now do a full deployment with 'overcloud deploy' only to download the swift container with the generated templates. This simple task avoids that step by allowing developers to generate it locally. It also aims to preserve the ability to use t-h-t templates directly with Heat (instead of going through Mistral) should users wish to do that. The new undercloud heat installer requires the ability to generate templates without requiring Mistral and Swift to do so. Ideally the Mistral API workflow would use this same code so perhaps in the future we might modify that routine to: -download swift tarball containing the templates -run this local routine that lives in t-h-t -re-upload the tarball of templates to the swift container Change-Id: Ie664c9c5f455b7320a58a26f35bc403355408d9b
-rw-r--r--.gitignore15
-rw-r--r--puppet/role.role.j2.yaml6
-rw-r--r--requirements.txt1
-rwxr-xr-xtools/process-templates.py125
-rw-r--r--tox.ini3
5 files changed, 150 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
index 3d7aded8..0925145c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,3 +44,18 @@ doc/_build
# Built by pbr (python setup.py sdist):
AUTHORS
ChangeLog
+
+extraconfig/all_nodes/mac_hostname.yaml
+extraconfig/all_nodes/random_string.yaml
+extraconfig/all_nodes/swap-partition.yaml
+extraconfig/all_nodes/swap.yaml
+extraconfig/tasks/major_upgrade_pacemaker_init.yaml
+network/service_net_map.yaml
+overcloud-resource-registry-puppet.yaml
+overcloud.yaml
+puppet/blockstorage-config.yaml
+puppet/cephstorage-config.yaml
+puppet/compute-config.yaml
+puppet/controller-config.yaml
+puppet/objectstorage-config.yaml
+puppet/post.yaml
diff --git a/puppet/role.role.j2.yaml b/puppet/role.role.j2.yaml
index ad5e4794..74a63dd1 100644
--- a/puppet/role.role.j2.yaml
+++ b/puppet/role.role.j2.yaml
@@ -5,13 +5,17 @@ parameters:
description: Flavor for the {{role}} node.
default: baremetal
type: string
+{% if disable_constraints is not defined %}
constraints:
- custom_constraint: nova.flavor
+{% endif %}
{{role}}Image:
type: string
default: overcloud-full
+{% if disable_constraints is not defined %}
constraints:
- custom_constraint: glance.image
+{% endif %}
ImageUpdatePolicy:
default: 'REBUILD_PRESERVE_EPHEMERAL'
description: What policy to use when reconstructing instances. REBUILD for rebuilds, REBUILD_PRESERVE_EPHEMERAL to preserve /mnt.
@@ -20,8 +24,10 @@ parameters:
description: Name of an existing Nova key pair to enable SSH access to the instances
type: string
default: default
+{% if disable_constraints is not defined %}
constraints:
- custom_constraint: nova.keypair
+{% endif %}
ServiceNetMap:
default: {}
description: Mapping of service_name -> network name. Typically set
diff --git a/requirements.txt b/requirements.txt
index 4e46b891..9c4a708a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
pbr>=0.5.21,<1.0
+Jinja2>=2.8 # BSD License (3 clause)
diff --git a/tools/process-templates.py b/tools/process-templates.py
new file mode 100755
index 00000000..a15b00e2
--- /dev/null
+++ b/tools/process-templates.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import jinja2
+import os
+import sys
+import yaml
+
+
+def parse_opts(argv):
+ parser = argparse.ArgumentParser(
+ description='Configure host network interfaces using a JSON'
+ ' config file format.')
+ parser.add_argument('-p', '--base_path', metavar='BASE_PATH',
+ help="""base path of templates to process.""",
+ default='.')
+ parser.add_argument('-r', '--roles-data', metavar='ROLES_DATA',
+ help="""relative path to the roles_data.yaml file.""",
+ default='roles_data.yaml')
+ parser.add_argument('--safe',
+ action='store_true',
+ help="""Enable safe mode (do not overwrite files).""",
+ default=False)
+ opts = parser.parse_args(argv[1:])
+
+ return opts
+
+
+def _j2_render_to_file(j2_template, j2_data, outfile_name=None,
+ overwrite=True):
+ yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
+ print('rendering j2 template to file: %s' % outfile_name)
+
+ if not overwrite and os.path.exists(outfile_name):
+ print('ERROR: path already exists for file: %s' % outfile_name)
+ sys.exit(1)
+
+ try:
+ # Render the j2 template
+ template = jinja2.Environment().from_string(j2_template)
+ r_template = template.render(**j2_data)
+ except jinja2.exceptions.TemplateError as ex:
+ error_msg = ("Error rendering template %s : %s"
+ % (yaml_f, six.text_type(ex)))
+ print(error_msg)
+ raise Exception(error_msg)
+ with open(outfile_name, 'w') as out_f:
+ out_f.write(r_template)
+
+
+def process_templates(template_path, role_data_path, overwrite):
+
+ with open(role_data_path) as role_data_file:
+ role_data = yaml.safe_load(role_data_file)
+
+ j2_excludes_path = os.path.join(template_path, 'j2_excludes.yaml')
+ with open(j2_excludes_path) as role_data_file:
+ j2_excludes = yaml.safe_load(role_data_file)
+
+ role_names = [r.get('name') for r in role_data]
+ r_map = {}
+ for r in role_data:
+ r_map[r.get('name')] = r
+ excl_templates = ['%s/%s' % (template_path, e)
+ for e in j2_excludes.get('name')]
+
+ if os.path.isdir(template_path):
+ for subdir, dirs, files in os.walk(template_path):
+ for f in files:
+ file_path = os.path.join(subdir, f)
+ # We do two templating passes here:
+ # 1. *.role.j2.yaml - we template just the role name
+ # and create multiple files (one per role)
+ # 2. *.j2.yaml - we template with all roles_data,
+ # and create one file common to all roles
+ if f.endswith('.role.j2.yaml'):
+ print("jinja2 rendering role template %s" % f)
+ with open(file_path) as j2_template:
+ template_data = j2_template.read()
+ print("jinja2 rendering roles %s" % ","
+ .join(role_names))
+ for role in role_names:
+ j2_data = {'role': role}
+ # (dprince) For the undercloud installer we don't
+ # want to have heat check nova/glance API's
+ if r_map[role].get('disable_constraints', False):
+ j2_data['disable_constraints'] = True
+ out_f = "-".join(
+ [role.lower(),
+ os.path.basename(f).replace('.role.j2.yaml',
+ '.yaml')])
+ out_f_path = os.path.join(subdir, out_f)
+ if not (out_f_path in excl_templates):
+ _j2_render_to_file(template_data, j2_data,
+ out_f_path, overwrite)
+ else:
+ print('skipping rendering of %s' % out_f_path)
+ elif f.endswith('.j2.yaml'):
+ print("jinja2 rendering normal template %s" % f)
+ with open(file_path) as j2_template:
+ template_data = j2_template.read()
+ j2_data = {'roles': role_data}
+ out_f = file_path.replace('.j2.yaml', '.yaml')
+ _j2_render_to_file(template_data, j2_data, out_f,
+ overwrite)
+
+ else:
+ print('Unexpected argument %s' % template_path)
+
+opts = parse_opts(sys.argv)
+
+role_data_path = os.path.join(opts.base_path, opts.roles_data)
+
+process_templates(opts.base_path, role_data_path, (not opts.safe))
diff --git a/tox.ini b/tox.ini
index 5d09e0a4..21e9f406 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,3 +12,6 @@ commands = {posargs}
[testenv:pep8]
commands = python ./tools/yaml-validate.py .
+
+[testenv:templates]
+commands = python ./tools/process-templates.py .