path: root/tripleo_heat_merge/merge.py

import os
import sys
import yaml
import argparse


class Cfn(object):

    base_template = {
        'HeatTemplateFormatVersion': '2012-12-12',
        'Description': []
    }
    get_resource = 'Ref'
    get_param = 'Ref'
    description = 'Description'
    parameters = 'Parameters'
    outputs = 'Outputs'
    resources = 'Resources'
    type = 'Type'
    properties = 'Properties'
    metadata = 'Metadata'
    depends_on = 'DependsOn'
    get_attr = 'Fn::GetAtt'


class Hot(object):

    base_template = {
        'heat_template_version': '2013-05-23',
        'description': []
    }
    get_resource = 'get_resource'
    get_param = 'get_param'
    description = 'description'
    parameters = 'parameters'
    outputs = 'outputs'
    resources = 'resources'
    type = 'type'
    properties = 'properties'
    metadata = 'metadata'
    depends_on = 'depends_on'
    get_attr = 'get_attr'


lang = Cfn()


def apply_maps(template):
    """Apply Merge::Map within template.

    Any dict {'Merge::Map': {'Foo': 'Bar', 'Baz': 'Quux'}}
    will resolve to ['Bar', 'Quux'] - that is a dict with key
    'Merge::Map' is replaced entirely by that dict['Merge::Map'].values().
    """
    if isinstance(template, dict):
        if 'Merge::Map' in template:
            return sorted(
                apply_maps(value) for value in template['Merge::Map'].values()
                )
        else:
            return dict((key, apply_maps(value))
                for key, value in template.items())
    elif isinstance(template, list):
        return [apply_maps(item) for item in template]
    else:
        return template


def apply_scaling(template, scaling, in_copies=None):
    """Apply a set of scaling operations to template.

    This is a single pass recursive function: for each call we process one
    dict or list and recurse to handle children containers.

    Values are handled via scale_value.

    Keys in dicts are copied per the scaling rule.
    Values are either replaced or copied depending on whether the given
    scaling rule is in in_copies.

    in_copies is reset to None when a dict {'Merge::Map': someobject} is
    encountered.

    :param scaling: A dict of prefix -> (count, blacklists).
    """
    in_copies = dict(in_copies or {})
    # Shouldn't be needed but to avoid unexpected side effects/bugs we short
    # circuit no-ops.
    if not scaling:
        return template
    if isinstance(template, dict):
        if 'Merge::Map' in template:
            in_copies = None
        new_template = {}
        for key, value in template.items():
            for prefix, copy_num, new_key in scale_value(
                key, scaling, in_copies):
                if prefix:
                    # e.g. Compute0, 1, Compute1Foo
                    in_copies[prefix] = prefix[:-1] + str(copy_num)
                if isinstance(value, (dict, list)):
                    new_value = apply_scaling(value, scaling, in_copies)
                    new_template[new_key] = new_value
                else:
                    new_values = list(scale_value(value, scaling, in_copies))
                    # We have nowhere to multiply a non-container value of a
                    # dict, so it may be copied or unchanged but not scaled.
                    assert len(new_values) == 1
                    new_template[new_key] = new_values[0][2]
                if prefix:
                    del in_copies[prefix]
        return new_template
    elif isinstance(template, list):
        new_template = []
        for value in template:
            if isinstance(value, (dict, list)):
                new_template.append(apply_scaling(value, scaling, in_copies))
            else:
                for _, _, new_value in scale_value(value, scaling, in_copies):
                    new_template.append(new_value)
        return new_template
    else:
        raise Exception("apply_scaling called with non-container %r" % template)


def scale_value(value, scaling, in_copies):
    """Scale out a value.

    :param value: The value to scale (not a container).
    :param scaling: The scaling map (prefix-> (copies, blacklist) to use.
    :param in_copies: What containers we're currently copying.
    :return: An iterator of the new values for the value as tuples:
        (prefix, copy_num, value). E.g. Compute0, 1, Compute1Foo
        prefix and copy_num are only set when:
         - a prefix in scaling matches value
         - and that prefix is not in in_copies
    """
    if isinstance(value, (str, unicode)):
        for prefix, (copies, blacklist) in scaling.items():
            if not value.startswith(prefix):
                continue
            suffix = value[len(prefix):]
            if prefix in in_copies:
                # Adjust to the copy number we're on
                yield None, None, in_copies[prefix] + suffix
                return
            else:
                for n in range(copies):
                    if n not in blacklist:
                        yield prefix, n, prefix[:-1] + str(n) + suffix
                return
        yield None, None, value
    else:
        yield None, None, value


def parse_scaling(scaling_args):
    """Translate a list of scaling requests to a dict prefix:count."""
    scaling_args = scaling_args or []
    result = {}
    for item in scaling_args:
        key, values = item.split('=')
        values = values.split(',')
        value = int(values[0])
        blacklist = frozenset(int(v) for v in values[1:] if v)
        result[key + '0'] = value, blacklist
    return result


def _translate_role(role, master_role, slave_roles):
    if not master_role:
        return role
    if role == master_role:
        return role
    if role not in slave_roles:
        return role
    return master_role

def translate_role(role, master_role, slave_roles):
    r = _translate_role(role, master_role, slave_roles)
    if not isinstance(r, basestring):
        raise Exception('%s -> %r' % (role, r))
    return r

def resolve_params(item, param, value):
    if item in ({lang.get_param: param}, {lang.get_resource: param}):
        return value
    if isinstance(item, dict):
        copy_item = dict(item)
        for k, v in iter(copy_item.items()):
            item[k] = resolve_params(v, param, value)
    elif isinstance(item, list):
        copy_item = list(item)
        new_item = []
        for v in copy_item:
            new_item.append(resolve_params(v, param, value))
        item = new_item
    return item

MERGABLE_TYPES = {'OS::Nova::Server':
                  {'image': 'image'},
                  'AWS::EC2::Instance':
                  {'image': 'ImageId'},
                  'AWS::AutoScaling::LaunchConfiguration':
                  {},
                 }
INCLUDED_TEMPLATE_DIR = os.getcwd()


def resolve_includes(template, params=None):
    new_template = {}
    if params is None:
        params = {}
    for key, value in iter(template.items()):
        if key == '__include__':
            new_params = dict(params) # do not propagate up the stack
            if not isinstance(value, dict):
                raise ValueError('__include__ must be a mapping')
            if 'path' not in value:
                raise ValueError('__include__ must have path')
            if 'params' in value:
                if not isinstance(value['params'], dict):
                    raise ValueError('__include__ params must be a mapping')
                new_params.update(value['params'])
            with open(value['path']) as include_file:
                sub_template = yaml.safe_load(include_file.read())
                if 'subkey' in value:
                    if ((not isinstance(value['subkey'], int)
                         and not isinstance(sub_template, dict))):
                        raise RuntimeError('subkey requires mapping root or'
                                           ' integer for list root')
                    sub_template = sub_template[value['subkey']]
                for k, v in iter(new_params.items()):
                    sub_template = resolve_params(sub_template, k, v)
                new_template.update(resolve_includes(sub_template))
        else:
            if isinstance(value, dict):
                new_template[key] = resolve_includes(value)
            else:
                new_template[key] = value
    return new_template

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser()
    parser.add_argument('templates', nargs='+')
    parser.add_argument('--master-role', nargs='?',
                        help='Translate slave_roles to this')
    parser.add_argument('--slave-roles', nargs='*',
                        help='Translate all of these to master_role')
    parser.add_argument('--included-template-dir', nargs='?',
                        default=INCLUDED_TEMPLATE_DIR,
                        help='Path for resolving included templates')
    parser.add_argument('--output',
                        help='File to write output to. - for stdout',
                        default='-')
    parser.add_argument('--scale', action="append",
        help="Names to scale out. Pass Prefix=2 to cause a key Prefix0Foo to "
        "be copied to Prefix1Foo in the output, and value Prefix0Bar to be"
        "renamed to Prefix1Bar inside that copy, or copied to Prefix1Bar "
        "outside of any copy. Pass Prefix=3,1 to cause Prefix1* to be elided"
        "when scaling Prefix out. Prefix=4,1,2 will likewise elide Prefix1 and"
        "Prefix2.")
    parser.add_argument(
        '--change-image-params', action='store_true', default=False,
        help="Change parameters in templates to match resource names. This was "
             " the default at one time but it causes issues when parameter "
             " names need to remain stable.")
    parser.add_argument(
        '--hot', action='store_true', default=False,
        help="Assume source templates are in the HOT format, and generate a "
             "HOT template artifact.")
    args = parser.parse_args(argv)
    if args.hot:
        global lang
        lang = Hot()

    templates = args.templates
    scaling = parse_scaling(args.scale)
    merged_template = merge(templates, args.master_role, args.slave_roles,
                            args.included_template_dir, scaling=scaling,
                            change_image_params=args.change_image_params)
    if args.output == '-':
        out_file = sys.stdout
    else:
        out_file = file(args.output, 'wt')
    out_file.write(merged_template)


def merge(templates, master_role=None, slave_roles=None,
          included_template_dir=INCLUDED_TEMPLATE_DIR,
          scaling=None, change_image_params=None):
    scaling = scaling or {}
    errors = []
    end_template = dict(lang.base_template)
    resource_changes=[]
    for template_path in templates:
        template = yaml.safe_load(open(template_path))
        # Resolve __include__ tags
        template = resolve_includes(template)
        end_template[lang.description].append(template.get(lang.description,
                                                        template_path))
        new_parameters = template.get(lang.parameters, {})
        for p, pbody in sorted(new_parameters.items()):
            if p in end_template.get(lang.parameters, {}):
                if pbody != end_template[lang.parameters][p]:
                    errors.append('Parameter %s from %s conflicts.' % (p,
                                                                       template_path))
                continue
            if lang.parameters not in end_template:
                end_template[lang.parameters] = {}
            end_template[lang.parameters][p] = pbody

        new_outputs = template.get(lang.outputs, {})
        for o, obody in sorted(new_outputs.items()):
            if o in end_template.get(lang.outputs, {}):
                if pbody != end_template[lang.outputs][p]:
                    errors.append('Output %s from %s conflicts.' % (o,
                                                                       template_path))
                continue
            if lang.outputs not in end_template:
                end_template[lang.outputs] = {}
            end_template[lang.outputs][o] = obody

        new_resources = template.get(lang.resources, {})
        for r, rbody in sorted(new_resources.items()):
            if rbody[lang.type] in MERGABLE_TYPES:
                if change_image_params:
                    if 'image' in MERGABLE_TYPES[rbody[lang.type]]:
                        image_key = MERGABLE_TYPES[rbody[lang.type]]['image']
                        # XXX Assuming ImageId is always a Ref
                        ikey_val = end_template[lang.parameters][rbody[lang.properties][image_key][lang.get_param]]
                        del end_template[lang.parameters][rbody[lang.properties][image_key][lang.get_param]]
                if 'OpenStack::Role' in rbody.get(lang.metadata, {}):
                    sys.stderr.write("WARNING: OpenStack::Role is deprecated"
                                     " and will be removed in a later release\n");
                role = rbody.get(lang.metadata, {}).get('OpenStack::Role', r)
                role = translate_role(role, master_role, slave_roles)
                if role != r:
                    resource_changes.append((r, role))
                if role in end_template.get(lang.resources, {}):
                    new_metadata = rbody.get(lang.metadata, {})
                    for m, mbody in iter(new_metadata.items()):
                        if m in end_template[lang.resources][role].get(lang.metadata, {}):
                            if m == 'OpenStack::ImageBuilder::Elements':
                                end_template[lang.resources][role][lang.metadata][m].extend(mbody)
                                sys.stderr.write(
                                    "WARNING: OpenStack::ImageBuilder::Elements"
                                    " is deprecated and will be removed in a"
                                    " later release\n");
                                continue
                            if mbody != end_template[lang.resources][role][lang.metadata][m]:
                                errors.append('Role %s metadata key %s conflicts.' %
                                              (role, m))
                            continue
                        role_res = end_template[lang.resources][role]
                        if role_res[lang.type] == 'OS::Heat::StructuredConfig':
                            end_template[lang.resources][role][lang.properties]['config'][m] = mbody
                        else:
                            end_template[lang.resources][role][lang.metadata][m] = mbody
                    continue
                if lang.resources not in end_template:
                    end_template[lang.resources] = {}
                end_template[lang.resources][role] = rbody
                if change_image_params:
                    if 'image' in MERGABLE_TYPES[rbody[lang.type]]:
                        ikey = '%sImage' % (role)
                        end_template[lang.resources][role][lang.properties][image_key] = {lang.get_param: ikey}
                        end_template[lang.parameters][ikey] = ikey_val
            elif rbody[lang.type] == 'FileInclude':
                # we trust os.path.join to DTRT: if FileInclude path isn't
                # absolute, join to included_template_dir (./)
                with open(os.path.join(included_template_dir, rbody['Path'])) as rfile:
                    include_content = yaml.safe_load(rfile.read())
                    subkeys = rbody.get('SubKey','').split('.')
                    while len(subkeys) and subkeys[0]:
                        include_content = include_content[subkeys.pop(0)]
                    for replace_param, replace_value in iter(rbody.get(lang.parameters,
                                                                       {}).items()):
                        include_content = resolve_params(include_content,
                                                         replace_param,
                                                         replace_value)
                    if lang.resources not in end_template:
                        end_template[lang.resources] = {}
                    end_template[lang.resources][r] = include_content
            else:
                if r in end_template.get(lang.resources, {}):
                    if rbody != end_template[lang.resources][r]:
                        errors.append('Resource %s from %s conflicts' % (r,
                                                                         template_path))
                    continue
                if lang.resources not in end_template:
                    end_template[lang.resources] = {}
                end_template[lang.resources][r] = rbody

    end_template = apply_scaling(end_template, scaling)
    end_template = apply_maps(end_template)

    def fix_ref(item, old, new):
        if isinstance(item, dict):
            copy_item = dict(item)
            for k, v in sorted(copy_item.items()):
                if k == lang.get_resource and v == old:
                    item[k] = new
                    continue
                if k == lang.depends_on and v == old:
                    item[k] = new
                    continue
                if k == lang.get_attr and isinstance(v, list) and v[0] == old:
                    new_list = list(v)
                    new_list[0] = new
                    item[k] = new_list
                    continue
                if k == 'AllowedResources' and isinstance(v, list) and old in v:
                    while old in v:
                        pos = v.index(old)
                        v[pos] = new
                    continue
                fix_ref(v, old, new)
        elif isinstance(item, list):
            copy_item = list(item)
            for v in item:
                fix_ref(v, old, new)

    for change in resource_changes:
        fix_ref(end_template, change[0], change[1])

    if errors:
        for e in errors:
            sys.stderr.write("ERROR: %s\n" % e)
    end_template[lang.description] = ','.join(end_template[lang.description])
    return yaml.safe_dump(end_template, default_flow_style=False)

if __name__ == "__main__":
      main()