Add local template generation tox task

This patch adds a local version of our template processing
routine so that developers can more quickly view the templates
that are actually getting generated. I've noticed multiple developers
now do a full deployment with 'overcloud deploy' only to download
the swift container with the generated templates. This simple task
avoids that step by allowing developers to generate it locally.

It also aims to preserve the ability to use t-h-t templates directly
with Heat (instead of going through Mistral) should users wish to do that.
The new undercloud heat installer requires the ability to generate
templates without requiring Mistral and Swift to do so.

Ideally the Mistral API workflow would use this same code
so perhaps in the future we might modify that routine to:

 -download swift tarball containing the templates
 -run this local routine that lives in t-h-t
 -re-upload the tarball of templates to the swift container

Change-Id: Ie664c9c5f455b7320a58a26f35bc403355408d9b

5 files changed, 150 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
index 3d7aded8..0925145c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,3 +44,18 @@ doc/_build
 # Built by pbr (python setup.py sdist):
 AUTHORS
 ChangeLog
+
+extraconfig/all_nodes/mac_hostname.yaml
+extraconfig/all_nodes/random_string.yaml
+extraconfig/all_nodes/swap-partition.yaml
+extraconfig/all_nodes/swap.yaml
+extraconfig/tasks/major_upgrade_pacemaker_init.yaml
+network/service_net_map.yaml
+overcloud-resource-registry-puppet.yaml
+overcloud.yaml
+puppet/blockstorage-config.yaml
+puppet/cephstorage-config.yaml
+puppet/compute-config.yaml
+puppet/controller-config.yaml
+puppet/objectstorage-config.yaml
+puppet/post.yaml
diff --git a/puppet/role.role.j2.yaml b/puppet/role.role.j2.yaml
index ad5e4794..74a63dd1 100644
--- a/puppet/role.role.j2.yaml
+++ b/puppet/role.role.j2.yaml
@@ -5,13 +5,17 @@ parameters:
     description: Flavor for the {{role}} node.
     default: baremetal
     type: string
+{% if disable_constraints is not defined %}
     constraints:
       - custom_constraint: nova.flavor
+{% endif %}
   {{role}}Image:
     type: string
     default: overcloud-full
+{% if disable_constraints is not defined %}
     constraints:
       - custom_constraint: glance.image
+{% endif %}
   ImageUpdatePolicy:
     default: 'REBUILD_PRESERVE_EPHEMERAL'
     description: What policy to use when reconstructing instances. REBUILD for rebuilds, REBUILD_PRESERVE_EPHEMERAL to preserve /mnt.
@@ -20,8 +24,10 @@ parameters:
     description: Name of an existing Nova key pair to enable SSH access to the instances
     type: string
     default: default
+{% if disable_constraints is not defined %}
     constraints:
       - custom_constraint: nova.keypair
+{% endif %}
   ServiceNetMap:
     default: {}
     description: Mapping of service_name -> network name. Typically set
diff --git a/requirements.txt b/requirements.txt
index 4e46b891..9c4a708a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
 pbr>=0.5.21,<1.0
+Jinja2>=2.8 # BSD License (3 clause)
diff --git a/tools/process-templates.py b/tools/process-templates.py
new file mode 100755
index 00000000..a15b00e2
--- /dev/null
+++ b/tools/process-templates.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import argparse
+import jinja2
+import os
+import sys
+import yaml
+
+
+def parse_opts(argv):
+    parser = argparse.ArgumentParser(
+        description='Configure host network interfaces using a JSON'
+        ' config file format.')
+    parser.add_argument('-p', '--base_path', metavar='BASE_PATH',
+                        help="""base path of templates to process.""",
+                        default='.')
+    parser.add_argument('-r', '--roles-data', metavar='ROLES_DATA',
+                        help="""relative path to the roles_data.yaml file.""",
+                        default='roles_data.yaml')
+    parser.add_argument('--safe',
+                        action='store_true',
+                        help="""Enable safe mode (do not overwrite files).""",
+                        default=False)
+    opts = parser.parse_args(argv[1:])
+
+    return opts
+
+
+def _j2_render_to_file(j2_template, j2_data, outfile_name=None,
+                       overwrite=True):
+    yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
+    print('rendering j2 template to file: %s' % outfile_name)
+
+    if not overwrite and os.path.exists(outfile_name):
+        print('ERROR: path already exists for file: %s' % outfile_name)
+        sys.exit(1)
+
+    try:
+        # Render the j2 template
+        template = jinja2.Environment().from_string(j2_template)
+        r_template = template.render(**j2_data)
+    except jinja2.exceptions.TemplateError as ex:
+        error_msg = ("Error rendering template %s : %s"
+                     % (yaml_f, six.text_type(ex)))
+        print(error_msg)
+        raise Exception(error_msg)
+    with open(outfile_name, 'w') as out_f:
+        out_f.write(r_template)
+
+
+def process_templates(template_path, role_data_path, overwrite):
+
+    with open(role_data_path) as role_data_file:
+        role_data = yaml.safe_load(role_data_file)
+
+    j2_excludes_path = os.path.join(template_path, 'j2_excludes.yaml')
+    with open(j2_excludes_path) as role_data_file:
+        j2_excludes = yaml.safe_load(role_data_file)
+
+    role_names = [r.get('name') for r in role_data]
+    r_map = {}
+    for r in role_data:
+        r_map[r.get('name')] = r
+    excl_templates = ['%s/%s' % (template_path, e)
+                      for e in j2_excludes.get('name')]
+
+    if os.path.isdir(template_path):
+        for subdir, dirs, files in os.walk(template_path):
+            for f in files:
+                file_path = os.path.join(subdir, f)
+                # We do two templating passes here:
+                # 1. *.role.j2.yaml - we template just the role name
+                #    and create multiple files (one per role)
+                # 2. *.j2.yaml - we template with all roles_data,
+                #    and create one file common to all roles
+                if f.endswith('.role.j2.yaml'):
+                    print("jinja2 rendering role template %s" % f)
+                    with open(file_path) as j2_template:
+                        template_data = j2_template.read()
+                        print("jinja2 rendering roles %s" % ","
+                              .join(role_names))
+                        for role in role_names:
+                            j2_data = {'role': role}
+                            # (dprince) For the undercloud installer we don't
+                            # want to have heat check nova/glance API's
+                            if r_map[role].get('disable_constraints', False):
+                                j2_data['disable_constraints'] = True
+                            out_f = "-".join(
+                                [role.lower(),
+                                 os.path.basename(f).replace('.role.j2.yaml',
+                                                             '.yaml')])
+                            out_f_path = os.path.join(subdir, out_f)
+                            if not (out_f_path in excl_templates):
+                                _j2_render_to_file(template_data, j2_data,
+                                                   out_f_path, overwrite)
+                            else:
+                                print('skipping rendering of %s' % out_f_path)
+                elif f.endswith('.j2.yaml'):
+                    print("jinja2 rendering normal template %s" % f)
+                    with open(file_path) as j2_template:
+                        template_data = j2_template.read()
+                        j2_data = {'roles': role_data}
+                        out_f = file_path.replace('.j2.yaml', '.yaml')
+                        _j2_render_to_file(template_data, j2_data, out_f,
+                                           overwrite)
+
+    else:
+        print('Unexpected argument %s' % template_path)
+
+opts = parse_opts(sys.argv)
+
+role_data_path = os.path.join(opts.base_path, opts.roles_data)
+
+process_templates(opts.base_path, role_data_path, (not opts.safe))
diff --git a/tox.ini b/tox.ini
index 5d09e0a4..21e9f406 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,3 +12,6 @@ commands = {posargs}
 
 [testenv:pep8]
 commands = python ./tools/yaml-validate.py .
+
+[testenv:templates]
+commands = python ./tools/process-templates.py .