Merge "Add local template generation tox task"

5 files changed, 150 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
index 3d7aded8..0925145c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -44,3 +44,18 @@ doc/_build
 # Built by pbr (python setup.py sdist):
 AUTHORS
 ChangeLog
+
+extraconfig/all_nodes/mac_hostname.yaml
+extraconfig/all_nodes/random_string.yaml
+extraconfig/all_nodes/swap-partition.yaml
+extraconfig/all_nodes/swap.yaml
+extraconfig/tasks/major_upgrade_pacemaker_init.yaml
+network/service_net_map.yaml
+overcloud-resource-registry-puppet.yaml
+overcloud.yaml
+puppet/blockstorage-config.yaml
+puppet/cephstorage-config.yaml
+puppet/compute-config.yaml
+puppet/controller-config.yaml
+puppet/objectstorage-config.yaml
+puppet/post.yaml
diff --git a/puppet/role.role.j2.yaml b/puppet/role.role.j2.yaml
index c2c322d8..27dfc49f 100644
--- a/puppet/role.role.j2.yaml
+++ b/puppet/role.role.j2.yaml
@@ -5,13 +5,17 @@ parameters:
     description: Flavor for the {{role}} node.
     default: baremetal
     type: string
+{% if disable_constraints is not defined %}
     constraints:
       - custom_constraint: nova.flavor
+{% endif %}
   {{role}}Image:
     type: string
     default: overcloud-full
+{% if disable_constraints is not defined %}
     constraints:
       - custom_constraint: glance.image
+{% endif %}
   ImageUpdatePolicy:
     default: 'REBUILD_PRESERVE_EPHEMERAL'
     description: What policy to use when reconstructing instances. REBUILD for rebuilds, REBUILD_PRESERVE_EPHEMERAL to preserve /mnt.
@@ -20,8 +24,10 @@ parameters:
     description: Name of an existing Nova key pair to enable SSH access to the instances
     type: string
     default: default
+{% if disable_constraints is not defined %}
     constraints:
       - custom_constraint: nova.keypair
+{% endif %}
   ServiceNetMap:
     default: {}
     description: Mapping of service_name -> network name. Typically set
diff --git a/requirements.txt b/requirements.txt
index 4e46b891..9c4a708a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1 +1,2 @@
 pbr>=0.5.21,<1.0
+Jinja2>=2.8 # BSD License (3 clause)
diff --git a/tools/process-templates.py b/tools/process-templates.py
new file mode 100755
index 00000000..a15b00e2
--- /dev/null
+++ b/tools/process-templates.py
@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import argparse
+import jinja2
+import os
+import sys
+import yaml
+
+
+def parse_opts(argv):
+    parser = argparse.ArgumentParser(
+        description='Configure host network interfaces using a JSON'
+        ' config file format.')
+    parser.add_argument('-p', '--base_path', metavar='BASE_PATH',
+                        help="""base path of templates to process.""",
+                        default='.')
+    parser.add_argument('-r', '--roles-data', metavar='ROLES_DATA',
+                        help="""relative path to the roles_data.yaml file.""",
+                        default='roles_data.yaml')
+    parser.add_argument('--safe',
+                        action='store_true',
+                        help="""Enable safe mode (do not overwrite files).""",
+                        default=False)
+    opts = parser.parse_args(argv[1:])
+
+    return opts
+
+
+def _j2_render_to_file(j2_template, j2_data, outfile_name=None,
+                       overwrite=True):
+    yaml_f = outfile_name or j2_template.replace('.j2.yaml', '.yaml')
+    print('rendering j2 template to file: %s' % outfile_name)
+
+    if not overwrite and os.path.exists(outfile_name):
+        print('ERROR: path already exists for file: %s' % outfile_name)
+        sys.exit(1)
+
+    try:
+        # Render the j2 template
+        template = jinja2.Environment().from_string(j2_template)
+        r_template = template.render(**j2_data)
+    except jinja2.exceptions.TemplateError as ex:
+        error_msg = ("Error rendering template %s : %s"
+                     % (yaml_f, six.text_type(ex)))
+        print(error_msg)
+        raise Exception(error_msg)
+    with open(outfile_name, 'w') as out_f:
+        out_f.write(r_template)
+
+
+def process_templates(template_path, role_data_path, overwrite):
+
+    with open(role_data_path) as role_data_file:
+        role_data = yaml.safe_load(role_data_file)
+
+    j2_excludes_path = os.path.join(template_path, 'j2_excludes.yaml')
+    with open(j2_excludes_path) as role_data_file:
+        j2_excludes = yaml.safe_load(role_data_file)
+
+    role_names = [r.get('name') for r in role_data]
+    r_map = {}
+    for r in role_data:
+        r_map[r.get('name')] = r
+    excl_templates = ['%s/%s' % (template_path, e)
+                      for e in j2_excludes.get('name')]
+
+    if os.path.isdir(template_path):
+        for subdir, dirs, files in os.walk(template_path):
+            for f in files:
+                file_path = os.path.join(subdir, f)
+                # We do two templating passes here:
+                # 1. *.role.j2.yaml - we template just the role name
+                #    and create multiple files (one per role)
+                # 2. *.j2.yaml - we template with all roles_data,
+                #    and create one file common to all roles
+                if f.endswith('.role.j2.yaml'):
+                    print("jinja2 rendering role template %s" % f)
+                    with open(file_path) as j2_template:
+                        template_data = j2_template.read()
+                        print("jinja2 rendering roles %s" % ","
+                              .join(role_names))
+                        for role in role_names:
+                            j2_data = {'role': role}
+                            # (dprince) For the undercloud installer we don't
+                            # want to have heat check nova/glance API's
+                            if r_map[role].get('disable_constraints', False):
+                                j2_data['disable_constraints'] = True
+                            out_f = "-".join(
+                                [role.lower(),
+                                 os.path.basename(f).replace('.role.j2.yaml',
+                                                             '.yaml')])
+                            out_f_path = os.path.join(subdir, out_f)
+                            if not (out_f_path in excl_templates):
+                                _j2_render_to_file(template_data, j2_data,
+                                                   out_f_path, overwrite)
+                            else:
+                                print('skipping rendering of %s' % out_f_path)
+                elif f.endswith('.j2.yaml'):
+                    print("jinja2 rendering normal template %s" % f)
+                    with open(file_path) as j2_template:
+                        template_data = j2_template.read()
+                        j2_data = {'roles': role_data}
+                        out_f = file_path.replace('.j2.yaml', '.yaml')
+                        _j2_render_to_file(template_data, j2_data, out_f,
+                                           overwrite)
+
+    else:
+        print('Unexpected argument %s' % template_path)
+
+opts = parse_opts(sys.argv)
+
+role_data_path = os.path.join(opts.base_path, opts.roles_data)
+
+process_templates(opts.base_path, role_data_path, (not opts.safe))
diff --git a/tox.ini b/tox.ini
index 5d09e0a4..21e9f406 100644
--- a/tox.ini
+++ b/tox.ini
@@ -12,3 +12,6 @@ commands = {posargs}
 
 [testenv:pep8]
 commands = python ./tools/yaml-validate.py .
+
+[testenv:templates]
+commands = python ./tools/process-templates.py .