Make merge.py invokable from another script

Until now this script has been used at the command line. Tuskar would like
to consume the merge.py functionality to generate the overcloud heat stack
template. A main() is thus added here as well as a few related nits like
ensuring paths are absolute for included files. The actual merge
functionality is split into its own method so it can be invoked more
cleanly from calling scripts. The main method is then used at the command
line as before

For more info about how this will all be used by Tuskar see
I578b4e9f238590ea245b827bc75d252568d194fe

Change-Id: Ia6b6416fe10358d23f2b120283eecaf4c1178cfd

1 files changed, 146 insertions, 132 deletions

diff --git a/merge.py b/merge.py
index fe42b8db..7827f12a 100644
--- a/merge.py
+++ b/merge.py
@@ -1,30 +1,20 @@
+import os
 import sys
 import yaml
 import argparse
 
-parser = argparse.ArgumentParser()
-parser.add_argument('templates', nargs='+')
-parser.add_argument('--master-role', nargs='?',
-                    help='Translate slave_roles to this')
-parser.add_argument('--slave-roles', nargs='*',
-                    help='Translate all of these to master_role')
 
-args = parser.parse_args()
-
-templates = args.templates
-
-def _translate_role(role):
-    global args
-    if not args.master_role:
+def _translate_role(role, master_role, slave_roles):
+    if not master_role:
         return role
-    if role == args.master_role:
+    if role == master_role:
         return role
-    if role not in args.slave_roles:
+    if role not in slave_roles:
         return role
-    return args.master_role
+    return master_role
 
-def translate_role(role):
-    r = _translate_role(role)
+def translate_role(role, master_role, slave_roles):
+    r = _translate_role(role, master_role, slave_roles)
     if not isinstance(r, basestring):
         raise Exception('%s -> %r' % (role, r))
     return r
@@ -86,123 +76,147 @@ def resolve_includes(template, params=None):
                 new_template[key] = value
     return new_template
 
-errors = []
-end_template={'HeatTemplateFormatVersion': '2012-12-12',
-              'Description': []}
-resource_changes=[]
-for template_path in templates:
-    template = yaml.safe_load(open(template_path))
-    # Resolve __include__ tags
-    template = resolve_includes(template)
-    end_template['Description'].append(template.get('Description',
-                                                    template_path))
-    new_parameters = template.get('Parameters', {})
-    for p, pbody in sorted(new_parameters.items()):
-        if p in end_template.get('Parameters', {}):
-            if pbody != end_template['Parameters'][p]:
-                errors.append('Parameter %s from %s conflicts.' % (p,
-                                                                   template_path))
-            continue
-        if 'Parameters' not in end_template:
-            end_template['Parameters'] = {}
-        end_template['Parameters'][p] = pbody
-
-    new_outputs = template.get('Outputs', {})
-    for o, obody in sorted(new_outputs.items()):
-        if o in end_template.get('Outputs', {}):
-            if pbody != end_template['Outputs'][p]:
-                errors.append('Output %s from %s conflicts.' % (o,
-                                                                   template_path))
-            continue
-        if 'Outputs' not in end_template:
-            end_template['Outputs'] = {}
-        end_template['Outputs'][o] = obody
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+    parser = argparse.ArgumentParser()
+    parser.add_argument('templates', nargs='+')
+    parser.add_argument('--master-role', nargs='?',
+                        help='Translate slave_roles to this')
+    parser.add_argument('--slave-roles', nargs='*',
+                        help='Translate all of these to master_role')
+    args = parser.parse_args(argv)
+    templates = args.templates
+    merged_template = merge(templates, args.master_role, args.slave_roles)
+    sys.stdout.write(merged_template)
 
-    new_resources = template.get('Resources', {})
-    for r, rbody in sorted(new_resources.items()):
-        if rbody['Type'] in MERGABLE_TYPES:
-            if 'image' in MERGABLE_TYPES[rbody['Type']]:
-                image_key = MERGABLE_TYPES[rbody['Type']]['image']
-                # XXX Assuming ImageId is always a Ref
-                ikey_val = end_template['Parameters'][rbody['Properties'][image_key]['Ref']]
-                del end_template['Parameters'][rbody['Properties'][image_key]['Ref']]
-            role = rbody.get('Metadata', {}).get('OpenStack::Role', r)
-            role = translate_role(role)
-            if role != r:
-                resource_changes.append((r, role))
-            if role in end_template.get('Resources', {}):
-                new_metadata = rbody.get('Metadata', {})
-                for m, mbody in iter(new_metadata.items()):
-                    if m in end_template['Resources'][role].get('Metadata', {}):
-                        if m == 'OpenStack::ImageBuilder::Elements':
-                            end_template['Resources'][role]['Metadata'][m].extend(mbody)
-                            continue
-                        if mbody != end_template['Resources'][role]['Metadata'][m]:
-                            errors.append('Role %s metadata key %s conflicts.' %
-                                          (role, m))
-                        continue
-                    end_template['Resources'][role]['Metadata'][m] = mbody
+def merge(templates, master_role=None, slave_roles=None):
+    errors = []
+    end_template={'HeatTemplateFormatVersion': '2012-12-12',
+                  'Description': []}
+    resource_changes=[]
+    for template_path in templates:
+        template = yaml.safe_load(open(template_path))
+        # Resolve __include__ tags
+        template = resolve_includes(template)
+        end_template['Description'].append(template.get('Description',
+                                                        template_path))
+        new_parameters = template.get('Parameters', {})
+        for p, pbody in sorted(new_parameters.items()):
+            if p in end_template.get('Parameters', {}):
+                if pbody != end_template['Parameters'][p]:
+                    errors.append('Parameter %s from %s conflicts.' % (p,
+                                                                       template_path))
                 continue
-            if 'Resources' not in end_template:
-                end_template['Resources'] = {}
-            end_template['Resources'][role] = rbody
-            if 'image' in MERGABLE_TYPES[rbody['Type']]:
-                ikey = '%sImage' % (role)
-                end_template['Resources'][role]['Properties'][image_key] = {'Ref': ikey}
-                end_template['Parameters'][ikey] = ikey_val
-        elif rbody['Type'] == 'FileInclude':
-            with open(rbody['Path']) as rfile:
-                include_content = yaml.safe_load(rfile.read())
-                subkeys = rbody.get('SubKey','').split('.')
-                while len(subkeys) and subkeys[0]:
-                    include_content = include_content[subkeys.pop(0)]
-                for replace_param, replace_value in iter(rbody.get('Parameters',
-                                                                   {}).items()):
-                    include_content = resolve_params(include_content,
-                                                     replace_param,
-                                                     replace_value)
-                end_template['Resources'][r] = include_content
-        else:
-            if r in end_template.get('Resources', {}):
-                if rbody != end_template['Resources'][r]:
-                    errors.append('Resource %s from %s conflicts' % (r,
-                                                                     template_path))
-                continue
-            if 'Resources' not in end_template:
-                end_template['Resources'] = {}
-            end_template['Resources'][r] = rbody
+            if 'Parameters' not in end_template:
+                end_template['Parameters'] = {}
+            end_template['Parameters'][p] = pbody
 
-def fix_ref(item, old, new):
-    if isinstance(item, dict):
-        copy_item = dict(item)
-        for k, v in sorted(copy_item.items()):
-            if k == 'Ref' and v == old:
-                item[k] = new
-                continue
-            if k == 'DependsOn' and v == old:
-                item[k] = new
+        new_outputs = template.get('Outputs', {})
+        for o, obody in sorted(new_outputs.items()):
+            if o in end_template.get('Outputs', {}):
+                if pbody != end_template['Outputs'][p]:
+                    errors.append('Output %s from %s conflicts.' % (o,
+                                                                       template_path))
                 continue
-            if k == 'Fn::GetAtt' and isinstance(v, list) and v[0] == old:
-                new_list = list(v)
-                new_list[0] = new
-                item[k] = new_list
-                continue
-            if k == 'AllowedResources' and isinstance(v, list) and old in v:
-                while old in v:
-                    pos = v.index(old)
-                    v[pos] = new
-                continue
-            fix_ref(v, old, new)
-    elif isinstance(item, list):
-        copy_item = list(item)
-        for v in item:
-            fix_ref(v, old, new)
+            if 'Outputs' not in end_template:
+                end_template['Outputs'] = {}
+            end_template['Outputs'][o] = obody
+
+        new_resources = template.get('Resources', {})
+        for r, rbody in sorted(new_resources.items()):
+            if rbody['Type'] in MERGABLE_TYPES:
+                if 'image' in MERGABLE_TYPES[rbody['Type']]:
+                    image_key = MERGABLE_TYPES[rbody['Type']]['image']
+                    # XXX Assuming ImageId is always a Ref
+                    ikey_val = end_template['Parameters'][rbody['Properties'][image_key]['Ref']]
+                    del end_template['Parameters'][rbody['Properties'][image_key]['Ref']]
+                role = rbody.get('Metadata', {}).get('OpenStack::Role', r)
+                role = translate_role(role, master_role, slave_roles)
+                if role != r:
+                    resource_changes.append((r, role))
+                if role in end_template.get('Resources', {}):
+                    new_metadata = rbody.get('Metadata', {})
+                    for m, mbody in iter(new_metadata.items()):
+                        if m in end_template['Resources'][role].get('Metadata', {}):
+                            if m == 'OpenStack::ImageBuilder::Elements':
+                                end_template['Resources'][role]['Metadata'][m].extend(mbody)
+                                continue
+                            if mbody != end_template['Resources'][role]['Metadata'][m]:
+                                errors.append('Role %s metadata key %s conflicts.' %
+                                              (role, m))
+                            continue
+                        end_template['Resources'][role]['Metadata'][m] = mbody
+                    continue
+                if 'Resources' not in end_template:
+                    end_template['Resources'] = {}
+                end_template['Resources'][role] = rbody
+                if 'image' in MERGABLE_TYPES[rbody['Type']]:
+                    ikey = '%sImage' % (role)
+                    end_template['Resources'][role]['Properties'][image_key] = {'Ref': ikey}
+                    end_template['Parameters'][ikey] = ikey_val
+            elif rbody['Type'] == 'FileInclude':
+                #make sure rbody['Path'] is absolute - required when this
+                #script invoked by import rather than command line
+                if os.path.dirname(rbody['Path']) == '':
+                    template_dir = os.path.dirname(__file__)
+                    filename = rbody['Path']
+                    rbody['Path'] = os.path.join(template_dir, filename)
+                with open(rbody['Path']) as rfile:
+                    include_content = yaml.safe_load(rfile.read())
+                    subkeys = rbody.get('SubKey','').split('.')
+                    while len(subkeys) and subkeys[0]:
+                        include_content = include_content[subkeys.pop(0)]
+                    for replace_param, replace_value in iter(rbody.get('Parameters',
+                                                                       {}).items()):
+                        include_content = resolve_params(include_content,
+                                                         replace_param,
+                                                         replace_value)
+                    end_template['Resources'][r] = include_content
+            else:
+                if r in end_template.get('Resources', {}):
+                    if rbody != end_template['Resources'][r]:
+                        errors.append('Resource %s from %s conflicts' % (r,
+                                                                         template_path))
+                    continue
+                if 'Resources' not in end_template:
+                    end_template['Resources'] = {}
+                end_template['Resources'][r] = rbody
+
+    def fix_ref(item, old, new):
+        if isinstance(item, dict):
+            copy_item = dict(item)
+            for k, v in sorted(copy_item.items()):
+                if k == 'Ref' and v == old:
+                    item[k] = new
+                    continue
+                if k == 'DependsOn' and v == old:
+                    item[k] = new
+                    continue
+                if k == 'Fn::GetAtt' and isinstance(v, list) and v[0] == old:
+                    new_list = list(v)
+                    new_list[0] = new
+                    item[k] = new_list
+                    continue
+                if k == 'AllowedResources' and isinstance(v, list) and old in v:
+                    while old in v:
+                        pos = v.index(old)
+                        v[pos] = new
+                    continue
+                fix_ref(v, old, new)
+        elif isinstance(item, list):
+            copy_item = list(item)
+            for v in item:
+                fix_ref(v, old, new)
+
+    for change in resource_changes:
+        fix_ref(end_template, change[0], change[1])
 
-for change in resource_changes:
-    fix_ref(end_template, change[0], change[1])
+    if errors:
+        for e in errors:
+            sys.stderr.write("ERROR: %s\n" % e)
+    end_template['Description'] = ','.join(end_template['Description'])
+    return yaml.safe_dump(end_template, default_flow_style=False)
 
-if errors:
-    for e in errors:
-        sys.stderr.write("ERROR: %s\n" % e)
-end_template['Description'] = ','.join(end_template['Description'])
-sys.stdout.write(yaml.safe_dump(end_template, default_flow_style=False))
+if __name__ == "__main__":
+      main()