From c9ce27c981d74c18f844fa71339c0177c3f76b28 Mon Sep 17 00:00:00 2001 From: marios Date: Fri, 29 Nov 2013 15:12:33 +0200 Subject: Make merge.py installable as a package using pbr Now that merge.py is invokable from another script (Ia6b6416fe10358d23f2b120283eecaf4c1178cfd) and from comments at that review, it makes sense to offer a nicer way to consume the merge functionality. Once you git clone tripleo-heat-templates you can python setup.py install and get /usr/bin/tripleo_heat_merge as well as a tripleo_heat_merge package in python2.7/site-packages. Makefile edits required because we moved merge.py into the tripleo_heat_merge directory for the packaging. Change-Id: I587fa7a826f93f89e8e5c266af7f5765438fe738 --- .gitignore | 4 + Makefile | 10 +- merge.py | 224 ----------------------------------------- setup.cfg | 10 +- test_merge.bash | 9 +- tripleo_heat_merge/__init__.py | 0 tripleo_heat_merge/merge.py | 224 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 243 insertions(+), 238 deletions(-) delete mode 100644 merge.py create mode 100644 tripleo_heat_merge/__init__.py create mode 100644 tripleo_heat_merge/merge.py diff --git a/.gitignore b/.gitignore index 08015990..90c6090d 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,7 @@ nosetests.xml *.swp doc/_build + +# Built by pbr (python setup.py sdist): +AUTHORS +ChangeLog diff --git a/Makefile b/Makefile index 38e495e5..b05f951a 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,23 @@ overcloud.yaml: overcloud-source.yaml nova-compute-instance.yaml swift-source.yaml # $^ won't work here because we want to list nova-compute-instance.yaml as # a prerequisite but don't want to pass it into merge.py - python merge.py overcloud-source.yaml swift-source.yaml > $@.tmp + python ./tripleo_heat_merge/merge.py overcloud-source.yaml swift-source.yaml > $@.tmp mv $@.tmp $@ undercloud-vm.yaml: undercloud-source.yaml undercloud-vm-source.yaml - python merge.py $^ > $@.tmp + python ./tripleo_heat_merge/merge.py $^ > $@.tmp mv $@.tmp $@ undercloud-bm.yaml: undercloud-source.yaml undercloud-bm-source.yaml - python merge.py $^ > $@.tmp + python ./tripleo_heat_merge/merge.py $^ > $@.tmp mv $@.tmp $@ undercloud-vm-tuskar.yaml: undercloud-source.yaml undercloud-vm-source.yaml tuskar-source.yaml - python merge.py $^ > $@.tmp + python ./tripleo_heat_merge/merge.py $^ > $@.tmp mv $@.tmp $@ undercloud-vm-ironic.yaml: undercloud-source.yaml undercloud-vm-source.yaml ironic-source.yaml - python merge.py $^ > $@.tmp + python ./tripleo_heat_merge/merge.py $^ > $@.tmp mv $@.tmp $@ test: diff --git a/merge.py b/merge.py deleted file mode 100644 index 053a683a..00000000 --- a/merge.py +++ /dev/null @@ -1,224 +0,0 @@ -import os -import sys -import yaml -import argparse - - -def _translate_role(role, master_role, slave_roles): - if not master_role: - return role - if role == master_role: - return role - if role not in slave_roles: - return role - return master_role - -def translate_role(role, master_role, slave_roles): - r = _translate_role(role, master_role, slave_roles) - if not isinstance(r, basestring): - raise Exception('%s -> %r' % (role, r)) - return r - -def resolve_params(item, param, value): - if item == {'Ref': param}: - return value - if isinstance(item, dict): - copy_item = dict(item) - for k, v in iter(copy_item.items()): - item[k] = resolve_params(v, param, value) - elif isinstance(item, list): - copy_item = list(item) - new_item = [] - for v in copy_item: - new_item.append(resolve_params(v, param, value)) - item = new_item - return item - -MERGABLE_TYPES = {'OS::Nova::Server': - {'image': 'image'}, - 'AWS::EC2::Instance': - {'image': 'ImageId'}, - 'AWS::AutoScaling::LaunchConfiguration': - {}, - } -INCLUDED_TEMPLATE_DIR = os.getcwd() - - -def resolve_includes(template, params=None): - new_template = {} - if params is None: - params = {} - for key, value in iter(template.items()): - if key == '__include__': - new_params = dict(params) # do not propagate up the stack - if not isinstance(value, dict): - raise ValueError('__include__ must be a mapping') - if 'path' not in value: - raise ValueError('__include__ must have path') - if 'params' in value: - if not isinstance(value['params'], dict): - raise ValueError('__include__ params must be a mapping') - new_params.update(value['params']) - with open(value['path']) as include_file: - sub_template = yaml.safe_load(include_file.read()) - if 'subkey' in value: - if ((not isinstance(value['subkey'], int) - and not isinstance(sub_template, dict))): - raise RuntimeError('subkey requires mapping root or' - ' integer for list root') - sub_template = sub_template[value['subkey']] - for k, v in iter(new_params.items()): - sub_template = resolve_params(sub_template, k, v) - new_template.update(resolve_includes(sub_template)) - else: - if isinstance(value, dict): - new_template[key] = resolve_includes(value) - else: - new_template[key] = value - return new_template - -def main(argv=None): - if argv is None: - argv = sys.argv[1:] - parser = argparse.ArgumentParser() - parser.add_argument('templates', nargs='+') - parser.add_argument('--master-role', nargs='?', - help='Translate slave_roles to this') - parser.add_argument('--slave-roles', nargs='*', - help='Translate all of these to master_role') - parser.add_argument('--included-template-dir', nargs='?', - default=INCLUDED_TEMPLATE_DIR, - help='Path for resolving included templates') - args = parser.parse_args(argv) - templates = args.templates - merged_template = merge(templates, args.master_role, args.slave_roles, - args.included_template_dir) - sys.stdout.write(merged_template) - -def merge(templates, master_role=None, slave_roles=None, - included_template_dir=INCLUDED_TEMPLATE_DIR): - errors = [] - end_template={'HeatTemplateFormatVersion': '2012-12-12', - 'Description': []} - resource_changes=[] - for template_path in templates: - template = yaml.safe_load(open(template_path)) - # Resolve __include__ tags - template = resolve_includes(template) - end_template['Description'].append(template.get('Description', - template_path)) - new_parameters = template.get('Parameters', {}) - for p, pbody in sorted(new_parameters.items()): - if p in end_template.get('Parameters', {}): - if pbody != end_template['Parameters'][p]: - errors.append('Parameter %s from %s conflicts.' % (p, - template_path)) - continue - if 'Parameters' not in end_template: - end_template['Parameters'] = {} - end_template['Parameters'][p] = pbody - - new_outputs = template.get('Outputs', {}) - for o, obody in sorted(new_outputs.items()): - if o in end_template.get('Outputs', {}): - if pbody != end_template['Outputs'][p]: - errors.append('Output %s from %s conflicts.' % (o, - template_path)) - continue - if 'Outputs' not in end_template: - end_template['Outputs'] = {} - end_template['Outputs'][o] = obody - - new_resources = template.get('Resources', {}) - for r, rbody in sorted(new_resources.items()): - if rbody['Type'] in MERGABLE_TYPES: - if 'image' in MERGABLE_TYPES[rbody['Type']]: - image_key = MERGABLE_TYPES[rbody['Type']]['image'] - # XXX Assuming ImageId is always a Ref - ikey_val = end_template['Parameters'][rbody['Properties'][image_key]['Ref']] - del end_template['Parameters'][rbody['Properties'][image_key]['Ref']] - role = rbody.get('Metadata', {}).get('OpenStack::Role', r) - role = translate_role(role, master_role, slave_roles) - if role != r: - resource_changes.append((r, role)) - if role in end_template.get('Resources', {}): - new_metadata = rbody.get('Metadata', {}) - for m, mbody in iter(new_metadata.items()): - if m in end_template['Resources'][role].get('Metadata', {}): - if m == 'OpenStack::ImageBuilder::Elements': - end_template['Resources'][role]['Metadata'][m].extend(mbody) - continue - if mbody != end_template['Resources'][role]['Metadata'][m]: - errors.append('Role %s metadata key %s conflicts.' % - (role, m)) - continue - end_template['Resources'][role]['Metadata'][m] = mbody - continue - if 'Resources' not in end_template: - end_template['Resources'] = {} - end_template['Resources'][role] = rbody - if 'image' in MERGABLE_TYPES[rbody['Type']]: - ikey = '%sImage' % (role) - end_template['Resources'][role]['Properties'][image_key] = {'Ref': ikey} - end_template['Parameters'][ikey] = ikey_val - elif rbody['Type'] == 'FileInclude': - # we trust os.path.join to DTRT: if FileInclude path isn't - # absolute, join to included_template_dir (./) - with open(os.path.join(included_template_dir, rbody['Path'])) as rfile: - include_content = yaml.safe_load(rfile.read()) - subkeys = rbody.get('SubKey','').split('.') - while len(subkeys) and subkeys[0]: - include_content = include_content[subkeys.pop(0)] - for replace_param, replace_value in iter(rbody.get('Parameters', - {}).items()): - include_content = resolve_params(include_content, - replace_param, - replace_value) - end_template['Resources'][r] = include_content - else: - if r in end_template.get('Resources', {}): - if rbody != end_template['Resources'][r]: - errors.append('Resource %s from %s conflicts' % (r, - template_path)) - continue - if 'Resources' not in end_template: - end_template['Resources'] = {} - end_template['Resources'][r] = rbody - - def fix_ref(item, old, new): - if isinstance(item, dict): - copy_item = dict(item) - for k, v in sorted(copy_item.items()): - if k == 'Ref' and v == old: - item[k] = new - continue - if k == 'DependsOn' and v == old: - item[k] = new - continue - if k == 'Fn::GetAtt' and isinstance(v, list) and v[0] == old: - new_list = list(v) - new_list[0] = new - item[k] = new_list - continue - if k == 'AllowedResources' and isinstance(v, list) and old in v: - while old in v: - pos = v.index(old) - v[pos] = new - continue - fix_ref(v, old, new) - elif isinstance(item, list): - copy_item = list(item) - for v in item: - fix_ref(v, old, new) - - for change in resource_changes: - fix_ref(end_template, change[0], change[1]) - - if errors: - for e in errors: - sys.stderr.write("ERROR: %s\n" % e) - end_template['Description'] = ','.join(end_template['Description']) - return yaml.safe_dump(end_template, default_flow_style=False) - -if __name__ == "__main__": - main() diff --git a/setup.cfg b/setup.cfg index 1a5cb59b..28adf7c2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,8 +17,8 @@ classifier = Environment :: Console [files] -data_files = - share/tripleo-heat-tempaltes/examples = examples/* - share/tripleo-heat-templates = - Makefile - *.yaml +packages = + tripleo_heat_merge +[entry_points] +console_scripts = + tripleo-heat-merge = tripleo_heat_merge.merge:main diff --git a/test_merge.bash b/test_merge.bash index e6bfb192..2f8a4cc9 100755 --- a/test_merge.bash +++ b/test_merge.bash @@ -23,10 +23,11 @@ run_test() { cleanup } echo -run_test "python merge.py examples/source.yaml" examples/source_lib_result.yaml -run_test "python merge.py examples/source2.yaml" examples/source2_lib_result.yaml -run_test "python merge.py examples/source_include_subkey.yaml" examples/source_include_subkey_result.yaml -run_test "python merge.py examples/launchconfig1.yaml examples/launchconfig2.yaml" examples/launchconfig_result.yaml +merge_py="./tripleo_heat_merge/merge.py" +run_test "python $merge_py examples/source.yaml" examples/source_lib_result.yaml +run_test "python $merge_py examples/source2.yaml" examples/source2_lib_result.yaml +run_test "python $merge_py examples/source_include_subkey.yaml" examples/source_include_subkey_result.yaml +run_test "python $merge_py examples/launchconfig1.yaml examples/launchconfig2.yaml" examples/launchconfig_result.yaml echo trap - EXIT exit $fail diff --git a/tripleo_heat_merge/__init__.py b/tripleo_heat_merge/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tripleo_heat_merge/merge.py b/tripleo_heat_merge/merge.py new file mode 100644 index 00000000..053a683a --- /dev/null +++ b/tripleo_heat_merge/merge.py @@ -0,0 +1,224 @@ +import os +import sys +import yaml +import argparse + + +def _translate_role(role, master_role, slave_roles): + if not master_role: + return role + if role == master_role: + return role + if role not in slave_roles: + return role + return master_role + +def translate_role(role, master_role, slave_roles): + r = _translate_role(role, master_role, slave_roles) + if not isinstance(r, basestring): + raise Exception('%s -> %r' % (role, r)) + return r + +def resolve_params(item, param, value): + if item == {'Ref': param}: + return value + if isinstance(item, dict): + copy_item = dict(item) + for k, v in iter(copy_item.items()): + item[k] = resolve_params(v, param, value) + elif isinstance(item, list): + copy_item = list(item) + new_item = [] + for v in copy_item: + new_item.append(resolve_params(v, param, value)) + item = new_item + return item + +MERGABLE_TYPES = {'OS::Nova::Server': + {'image': 'image'}, + 'AWS::EC2::Instance': + {'image': 'ImageId'}, + 'AWS::AutoScaling::LaunchConfiguration': + {}, + } +INCLUDED_TEMPLATE_DIR = os.getcwd() + + +def resolve_includes(template, params=None): + new_template = {} + if params is None: + params = {} + for key, value in iter(template.items()): + if key == '__include__': + new_params = dict(params) # do not propagate up the stack + if not isinstance(value, dict): + raise ValueError('__include__ must be a mapping') + if 'path' not in value: + raise ValueError('__include__ must have path') + if 'params' in value: + if not isinstance(value['params'], dict): + raise ValueError('__include__ params must be a mapping') + new_params.update(value['params']) + with open(value['path']) as include_file: + sub_template = yaml.safe_load(include_file.read()) + if 'subkey' in value: + if ((not isinstance(value['subkey'], int) + and not isinstance(sub_template, dict))): + raise RuntimeError('subkey requires mapping root or' + ' integer for list root') + sub_template = sub_template[value['subkey']] + for k, v in iter(new_params.items()): + sub_template = resolve_params(sub_template, k, v) + new_template.update(resolve_includes(sub_template)) + else: + if isinstance(value, dict): + new_template[key] = resolve_includes(value) + else: + new_template[key] = value + return new_template + +def main(argv=None): + if argv is None: + argv = sys.argv[1:] + parser = argparse.ArgumentParser() + parser.add_argument('templates', nargs='+') + parser.add_argument('--master-role', nargs='?', + help='Translate slave_roles to this') + parser.add_argument('--slave-roles', nargs='*', + help='Translate all of these to master_role') + parser.add_argument('--included-template-dir', nargs='?', + default=INCLUDED_TEMPLATE_DIR, + help='Path for resolving included templates') + args = parser.parse_args(argv) + templates = args.templates + merged_template = merge(templates, args.master_role, args.slave_roles, + args.included_template_dir) + sys.stdout.write(merged_template) + +def merge(templates, master_role=None, slave_roles=None, + included_template_dir=INCLUDED_TEMPLATE_DIR): + errors = [] + end_template={'HeatTemplateFormatVersion': '2012-12-12', + 'Description': []} + resource_changes=[] + for template_path in templates: + template = yaml.safe_load(open(template_path)) + # Resolve __include__ tags + template = resolve_includes(template) + end_template['Description'].append(template.get('Description', + template_path)) + new_parameters = template.get('Parameters', {}) + for p, pbody in sorted(new_parameters.items()): + if p in end_template.get('Parameters', {}): + if pbody != end_template['Parameters'][p]: + errors.append('Parameter %s from %s conflicts.' % (p, + template_path)) + continue + if 'Parameters' not in end_template: + end_template['Parameters'] = {} + end_template['Parameters'][p] = pbody + + new_outputs = template.get('Outputs', {}) + for o, obody in sorted(new_outputs.items()): + if o in end_template.get('Outputs', {}): + if pbody != end_template['Outputs'][p]: + errors.append('Output %s from %s conflicts.' % (o, + template_path)) + continue + if 'Outputs' not in end_template: + end_template['Outputs'] = {} + end_template['Outputs'][o] = obody + + new_resources = template.get('Resources', {}) + for r, rbody in sorted(new_resources.items()): + if rbody['Type'] in MERGABLE_TYPES: + if 'image' in MERGABLE_TYPES[rbody['Type']]: + image_key = MERGABLE_TYPES[rbody['Type']]['image'] + # XXX Assuming ImageId is always a Ref + ikey_val = end_template['Parameters'][rbody['Properties'][image_key]['Ref']] + del end_template['Parameters'][rbody['Properties'][image_key]['Ref']] + role = rbody.get('Metadata', {}).get('OpenStack::Role', r) + role = translate_role(role, master_role, slave_roles) + if role != r: + resource_changes.append((r, role)) + if role in end_template.get('Resources', {}): + new_metadata = rbody.get('Metadata', {}) + for m, mbody in iter(new_metadata.items()): + if m in end_template['Resources'][role].get('Metadata', {}): + if m == 'OpenStack::ImageBuilder::Elements': + end_template['Resources'][role]['Metadata'][m].extend(mbody) + continue + if mbody != end_template['Resources'][role]['Metadata'][m]: + errors.append('Role %s metadata key %s conflicts.' % + (role, m)) + continue + end_template['Resources'][role]['Metadata'][m] = mbody + continue + if 'Resources' not in end_template: + end_template['Resources'] = {} + end_template['Resources'][role] = rbody + if 'image' in MERGABLE_TYPES[rbody['Type']]: + ikey = '%sImage' % (role) + end_template['Resources'][role]['Properties'][image_key] = {'Ref': ikey} + end_template['Parameters'][ikey] = ikey_val + elif rbody['Type'] == 'FileInclude': + # we trust os.path.join to DTRT: if FileInclude path isn't + # absolute, join to included_template_dir (./) + with open(os.path.join(included_template_dir, rbody['Path'])) as rfile: + include_content = yaml.safe_load(rfile.read()) + subkeys = rbody.get('SubKey','').split('.') + while len(subkeys) and subkeys[0]: + include_content = include_content[subkeys.pop(0)] + for replace_param, replace_value in iter(rbody.get('Parameters', + {}).items()): + include_content = resolve_params(include_content, + replace_param, + replace_value) + end_template['Resources'][r] = include_content + else: + if r in end_template.get('Resources', {}): + if rbody != end_template['Resources'][r]: + errors.append('Resource %s from %s conflicts' % (r, + template_path)) + continue + if 'Resources' not in end_template: + end_template['Resources'] = {} + end_template['Resources'][r] = rbody + + def fix_ref(item, old, new): + if isinstance(item, dict): + copy_item = dict(item) + for k, v in sorted(copy_item.items()): + if k == 'Ref' and v == old: + item[k] = new + continue + if k == 'DependsOn' and v == old: + item[k] = new + continue + if k == 'Fn::GetAtt' and isinstance(v, list) and v[0] == old: + new_list = list(v) + new_list[0] = new + item[k] = new_list + continue + if k == 'AllowedResources' and isinstance(v, list) and old in v: + while old in v: + pos = v.index(old) + v[pos] = new + continue + fix_ref(v, old, new) + elif isinstance(item, list): + copy_item = list(item) + for v in item: + fix_ref(v, old, new) + + for change in resource_changes: + fix_ref(end_template, change[0], change[1]) + + if errors: + for e in errors: + sys.stderr.write("ERROR: %s\n" % e) + end_template['Description'] = ','.join(end_template['Description']) + return yaml.safe_dump(end_template, default_flow_style=False) + +if __name__ == "__main__": + main() -- cgit 1.2.3-korg