import os import sys import yaml import argparse class Cfn(object): base_template = { 'HeatTemplateFormatVersion': '2012-12-12', 'Description': [] } get_resource = 'Ref' get_param = 'Ref' description = 'Description' parameters = 'Parameters' outputs = 'Outputs' resources = 'Resources' type = 'Type' properties = 'Properties' metadata = 'Metadata' depends_on = 'DependsOn' get_attr = 'Fn::GetAtt' class Hot(object): base_template = { 'heat_template_version': '2013-05-23', 'description': [] } get_resource = 'get_resource' get_param = 'get_param' description = 'description' parameters = 'parameters' outputs = 'outputs' resources = 'resources' type = 'type' properties = 'properties' metadata = 'metadata' depends_on = 'depends_on' get_attr = 'get_attr' lang = Cfn() def apply_maps(template): """Apply Merge::Map within template. Any dict {'Merge::Map': {'Foo': 'Bar', 'Baz': 'Quux'}} will resolve to ['Bar', 'Quux'] - that is a dict with key 'Merge::Map' is replaced entirely by that dict['Merge::Map'].values(). """ if isinstance(template, dict): if 'Merge::Map' in template: return sorted( apply_maps(value) for value in template['Merge::Map'].values() ) else: return dict((key, apply_maps(value)) for key, value in template.items()) elif isinstance(template, list): return [apply_maps(item) for item in template] else: return template def apply_scaling(template, scaling, in_copies=None): """Apply a set of scaling operations to template. This is a single pass recursive function: for each call we process one dict or list and recurse to handle children containers. Values are handled via scale_value. Keys in dicts are copied per the scaling rule. Values are either replaced or copied depending on whether the given scaling rule is in in_copies. in_copies is reset to None when a dict {'Merge::Map': someobject} is encountered. :param scaling: A dict of prefix -> (count, blacklists). """ in_copies = dict(in_copies or {}) # Shouldn't be needed but to avoid unexpected side effects/bugs we short # circuit no-ops. if not scaling: return template if isinstance(template, dict): if 'Merge::Map' in template: in_copies = None new_template = {} for key, value in template.items(): for prefix, copy_num, new_key in scale_value( key, scaling, in_copies): if prefix: # e.g. Compute0, 1, Compute1Foo in_copies[prefix] = prefix[:-1] + str(copy_num) if isinstance(value, (dict, list)): new_value = apply_scaling(value, scaling, in_copies) new_template[new_key] = new_value else: new_values = list(scale_value(value, scaling, in_copies)) # We have nowhere to multiply a non-container value of a # dict, so it may be copied or unchanged but not scaled. assert len(new_values) == 1 new_template[new_key] = new_values[0][2] if prefix: del in_copies[prefix] return new_template elif isinstance(template, list): new_template = [] for value in template: if isinstance(value, (dict, list)): new_template.append(apply_scaling(value, scaling, in_copies)) else: for _, _, new_value in scale_value(value, scaling, in_copies): new_template.append(new_value) return new_template else: raise Exception("apply_scaling called with non-container %r" % template) def scale_value(value, scaling, in_copies): """Scale out a value. :param value: The value to scale (not a container). :param scaling: The scaling map (prefix-> (copies, blacklist) to use. :param in_copies: What containers we're currently copying. :return: An iterator of the new values for the value as tuples: (prefix, copy_num, value). E.g. Compute0, 1, Compute1Foo prefix and copy_num are only set when: - a prefix in scaling matches value - and that prefix is not in in_copies """ if isinstance(value, (str, unicode)): for prefix, (copies, blacklist) in scaling.items(): if not value.startswith(prefix): continue suffix = value[len(prefix):] if prefix in in_copies: # Adjust to the copy number we're on yield None, None, in_copies[prefix] + suffix return else: for n in range(copies): if n not in blacklist: yield prefix, n, prefix[:-1] + str(n) + suffix return yield None, None, value else: yield None, None, value def parse_scaling(scaling_args): """Translate a list of scaling requests to a dict prefix:count.""" scaling_args = scaling_args or [] result = {} for item in scaling_args: key, values = item.split('=') values = values.split(',') value = int(values[0]) blacklist = frozenset(int(v) for v in values[1:] if v) result[key + '0'] = value, blacklist return result def _translate_role(role, master_role, slave_roles): if not master_role: return role if role == master_role: return role if role not in slave_roles: return role return master_role def translate_role(role, master_role, slave_roles): r = _translate_role(role, master_role, slave_roles) if not isinstance(r, basestring): raise Exception('%s -> %r' % (role, r)) return r def resolve_params(item, param, value): if item in ({lang.get_param: param}, {lang.get_resource: param}): return value if isinstance(item, dict): copy_item = dict(item) for k, v in iter(copy_item.items()): item[k] = resolve_params(v, param, value) elif isinstance(item, list): copy_item = list(item) new_item = [] for v in copy_item: new_item.append(resolve_params(v, param, value)) item = new_item return item MERGABLE_TYPES = {'OS::Nova::Server': {'image': 'image'}, 'AWS::EC2::Instance': {'image': 'ImageId'}, 'AWS::AutoScaling::LaunchConfiguration': {}, } INCLUDED_TEMPLATE_DIR = os.getcwd() def resolve_includes(template, params=None): new_template = {} if params is None: params = {} for key, value in iter(template.items()): if key == '__include__': new_params = dict(params) # do not propagate up the stack if not isinstance(value, dict): raise ValueError('__include__ must be a mapping') if 'path' not in value: raise ValueError('__include__ must have path') if 'params' in value: if not isinstance(value['params'], dict): raise ValueError('__include__ params must be a mapping') new_params.update(value['params']) with open(value['path']) as include_file: sub_template = yaml.safe_load(include_file.read()) if 'subkey' in value: if ((not isinstance(value['subkey'], int) and not isinstance(sub_template, dict))): raise RuntimeError('subkey requires mapping root or' ' integer for list root') sub_template = sub_template[value['subkey']] for k, v in iter(new_params.items()): sub_template = resolve_params(sub_template, k, v) new_template.update(resolve_includes(sub_template)) else: if isinstance(value, dict): new_template[key] = resolve_includes(value) else: new_template[key] = value return new_template def main(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser() parser.add_argument('templates', nargs='+') parser.add_argument('--master-role', nargs='?', help='Translate slave_roles to this') parser.add_argument('--slave-roles', nargs='*', help='Translate all of these to master_role') parser.add_argument('--included-template-dir', nargs='?', default=INCLUDED_TEMPLATE_DIR, help='Path for resolving included templates') parser.add_argument('--output', help='File to write output to. - for stdout', default='-') parser.add_argument('--scale', action="append", help="Names to scale out. Pass Prefix=2 to cause a key Prefix0Foo to " "be copied to Prefix1Foo in the output, and value Prefix0Bar to be" "renamed to Prefix1Bar inside that copy, or copied to Prefix1Bar " "outside of any copy. Pass Prefix=3,1 to cause Prefix1* to be elided" "when scaling Prefix out. Prefix=4,1,2 will likewise elide Prefix1 and" "Prefix2.") parser.add_argument( '--change-image-params', action='store_true', default=False, help="Change parameters in templates to match resource names. This was " " the default at one time but it causes issues when parameter " " names need to remain stable.") parser.add_argument( '--hot', action='store_true', default=False, help="Assume source templates are in the HOT format, and generate a " "HOT template artifact.") args = parser.parse_args(argv) if args.hot: global lang lang = Hot() templates = args.templates scaling = parse_scaling(args.scale) merged_template = merge(templates, args.master_role, args.slave_roles, args.included_template_dir, scaling=scaling, change_image_params=args.change_image_params) if args.output == '-': out_file = sys.stdout else: out_file = file(args.output, 'wt') out_file.write(merged_template) def merge(templates, master_role=None, slave_roles=None, included_template_dir=INCLUDED_TEMPLATE_DIR, scaling=None, change_image_params=None): scaling = scaling or {} errors = [] end_template = dict(lang.base_template) resource_changes=[] for template_path in templates: template = yaml.safe_load(open(template_path)) # Resolve __include__ tags template = resolve_includes(template) end_template[lang.description].append(template.get(lang.description, template_path)) new_parameters = template.get(lang.parameters, {}) for p, pbody in sorted(new_parameters.items()): if p in end_template.get(lang.parameters, {}): if pbody != end_template[lang.parameters][p]: errors.append('Parameter %s from %s conflicts.' % (p, template_path)) continue if lang.parameters not in end_template: end_template[lang.parameters] = {} end_template[lang.parameters][p] = pbody new_outputs = template.get(lang.outputs, {}) for o, obody in sorted(new_outputs.items()): if o in end_template.get(lang.outputs, {}): if pbody != end_template[lang.outputs][p]: errors.append('Output %s from %s conflicts.' % (o, template_path)) continue if lang.outputs not in end_template: end_template[lang.outputs] = {} end_template[lang.outputs][o] = obody new_resources = template.get(lang.resources, {}) for r, rbody in sorted(new_resources.items()): if rbody[lang.type] in MERGABLE_TYPES: if change_image_params: if 'image' in MERGABLE_TYPES[rbody[lang.type]]: image_key = MERGABLE_TYPES[rbody[lang.type]]['image'] # XXX Assuming ImageId is always a Ref ikey_val = end_template[lang.parameters][rbody[lang.properties][image_key][lang.get_param]] del end_template[lang.parameters][rbody[lang.properties][image_key][lang.get_param]] if 'OpenStack::Role' in rbody.get(lang.metadata, {}): sys.stderr.write("WARNING: OpenStack::Role is deprecated" " and will be removed in a later release\n"); role = rbody.get(lang.metadata, {}).get('OpenStack::Role', r) role = translate_role(role, master_role, slave_roles) if role != r: resource_changes.append((r, role)) if role in end_template.get(lang.resources, {}): new_metadata = rbody.get(lang.metadata, {}) for m, mbody in iter(new_metadata.items()): if m in end_template[lang.resources][role].get(lang.metadata, {}): if m == 'OpenStack::ImageBuilder::Elements': end_template[lang.resources][role][lang.metadata][m].extend(mbody) sys.stderr.write( "WARNING: OpenStack::ImageBuilder::Elements" " is deprecated and will be removed in a" " later release\n"); continue if mbody != end_template[lang.resources][role][lang.metadata][m]: errors.append('Role %s metadata key %s conflicts.' % (role, m)) continue role_res = end_template[lang.resources][role] if role_res[lang.type] == 'OS::Heat::StructuredConfig': end_template[lang.resources][role][lang.properties]['config'][m] = mbody else: end_template[lang.resources][role][lang.metadata][m] = mbody continue if lang.resources not in end_template: end_template[lang.resources] = {} end_template[lang.resources][role] = rbody if change_image_params: if 'image' in MERGABLE_TYPES[rbody[lang.type]]: ikey = '%sImage' % (role) end_template[lang.resources][role][lang.properties][image_key] = {lang.get_param: ikey} end_template[lang.parameters][ikey] = ikey_val elif rbody[lang.type] == 'FileInclude': # we trust os.path.join to DTRT: if FileInclude path isn't # absolute, join to included_template_dir (./) with open(os.path.join(included_template_dir, rbody['Path'])) as rfile: include_content = yaml.safe_load(rfile.read()) subkeys = rbody.get('SubKey','').split('.') while len(subkeys) and subkeys[0]: include_content = include_content[subkeys.pop(0)] for replace_param, replace_value in iter(rbody.get(lang.parameters, {}).items()): include_content = resolve_params(include_content, replace_param, replace_value) if lang.resources not in end_template: end_template[lang.resources] = {} end_template[lang.resources][r] = include_content else: if r in end_template.get(lang.resources, {}): if rbody != end_template[lang.resources][r]: errors.append('Resource %s from %s conflicts' % (r, template_path)) continue if lang.resources not in end_template: end_template[lang.resources] = {} end_template[lang.resources][r] = rbody end_template = apply_scaling(end_template, scaling) end_template = apply_maps(end_template) def fix_ref(item, old, new): if isinstance(item, dict): copy_item = dict(item) for k, v in sorted(copy_item.items()): if k == lang.get_resource and v == old: item[k] = new continue if k == lang.depends_on and v == old: item[k] = new continue if k == lang.get_attr and isinstance(v, list) and v[0] == old: new_list = list(v) new_list[0] = new item[k] = new_list continue if k == 'AllowedResources' and isinstance(v, list) and old in v: while old in v: pos = v.index(old) v[pos] = new continue fix_ref(v, old, new) elif isinstance(item, list): copy_item = list(item) for v in item: fix_ref(v, old, new) for change in resource_changes: fix_ref(end_template, change[0], change[1]) if errors: for e in errors: sys.stderr.write("ERROR: %s\n" % e) end_template[lang.description] = ','.join(end_template[lang.description]) return yaml.safe_dump(end_template, default_flow_style=False) if __name__ == "__main__": main()