From c071255ad6bf1fb2c9db6799e96b509295f53fc9 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 28 Jan 2014 19:20:40 +1300 Subject: Make scaling out a feature for merge.py. This is a simple implementation designed to work with our current non-HOT approach. We need to adjust our templates a little to support this - though future work could make that better, it's orthogonal to this effort. Change-Id: I555617e5f24a5882de915f057dc02e008c81e753 --- examples/scale1.yaml | 30 +++++++ examples/scale2.yaml | 66 +++++++++++++++ examples/scale_result.yaml | 190 ++++++++++++++++++++++++++++++++++++++++++++ test_merge.bash | 1 + tripleo_heat_merge/merge.py | 106 +++++++++++++++++++++++- 5 files changed, 391 insertions(+), 2 deletions(-) create mode 100644 examples/scale1.yaml create mode 100644 examples/scale2.yaml create mode 100644 examples/scale_result.yaml diff --git a/examples/scale1.yaml b/examples/scale1.yaml new file mode 100644 index 00000000..c0a0763d --- /dev/null +++ b/examples/scale1.yaml @@ -0,0 +1,30 @@ +Resources: + ComputeUser: + Type: AWS::IAM::User + Properties: + Policies: [ { Ref: ComputeAccessPolicy } ] + GlobalAccessPolicy: + Type: OS::Heat::AccessPolicy + NovaCompute0Key: + Type: FileInclude + Path: examples/scale2.yaml + SubKey: Resources.NovaCompute0Key + NovaCompute0CompletionCondition: + Type: FileInclude + Path: examples/scale2.yaml + SubKey: Resources.NovaCompute0CompletionCondition + NovaCompute0CompletionHandle: + Type: FileInclude + Path: examples/scale2.yaml + SubKey: Resources.NovaCompute0CompletionHandle + NovaCompute0Config: + Type: FileInclude + Path: examples/scale2.yaml + SubKey: Resources.NovaCompute0Config + Parameters: + ComputeImage: "123" + RabbitPassword: "guest" + NovaCompute0: + Type: FileInclude + Path: examples/scale2.yaml + SubKey: Resources.NovaCompute0 diff --git a/examples/scale2.yaml b/examples/scale2.yaml new file mode 100644 index 00000000..d1a81fe5 --- /dev/null +++ b/examples/scale2.yaml @@ -0,0 +1,66 @@ +HeatTemplateFormatVersion: '2012-12-12' +Parameters: + ComputeImage: + Type: String + RabbitPassword: + Type: String + NoEcho: true +Resources: + ComputeAccessPolicy: + Type: OS::Heat::AccessPolicy + Properties: + AllowedResources: [ NovaCompute0 ] + NovaCompute0Key: + Type: AWS::IAM::AccessKey + Properties: + UserName: + Ref: ComputeUser + NovaCompute0CompletionCondition: + Type: AWS::CloudFormation::WaitCondition + DependsOn: notcompute + Properties: + Handle: {Ref: NovaCompute0CompletionHandle} + Count: '1' + Timeout: '1800' + NovaCompute0CompletionHandle: + Type: AWS::CloudFormation::WaitConditionHandle + NovaCompute0: + Type: OS::Nova::Server + Properties: + image: + Ref: ComputeImage + Metadata: + os-collect-config: + cfn: + access_key_id: + Ref: NovaCompute0Key + secret_access_key: + Fn::GetAtt: [ NovaCompute0Key, SecretAccessKey ] + stack_name: {Ref: 'AWS::StackName'} + path: NovaCompute0Config.Metadata + NovaCompute0Config: + Type: AWS::AutoScaling::LaunchConfiguration + Metadata: + completion-handle: + Ref: NovaCompute0CompletionHandle + os-collect-config: + cfn: + access_key_id: + Ref: NovaCompute0Key + secret_access_key: + Fn::GetAtt: [ NovaCompute0Key, SecretAccessKey ] + stack_name: {Ref: 'AWS::StackName'} + path: NovaCompute0Config.Metadata + neutron: + ovs: + local_ip: + Fn::Select: + - 0 + - Fn::Select: + - ctlplane + - Fn::GetAtt: + - NovaCompute0 + - networks + rabbit: + password: {Ref: RabbitPassword} + diff --git a/examples/scale_result.yaml b/examples/scale_result.yaml new file mode 100644 index 00000000..f22d864e --- /dev/null +++ b/examples/scale_result.yaml @@ -0,0 +1,190 @@ +Description: examples/scale1.yaml +HeatTemplateFormatVersion: '2012-12-12' +Resources: + ComputeUser: + Properties: + Policies: + - Ref: ComputeAccessPolicy + Type: AWS::IAM::User + GlobalAccessPolicy: + Type: OS::Heat::AccessPolicy + NovaCompute0: + Metadata: + os-collect-config: + cfn: + access_key_id: + Ref: NovaCompute0Key + path: NovaCompute0Config.Metadata + secret_access_key: + Fn::GetAtt: + - NovaCompute0Key + - SecretAccessKey + stack_name: + Ref: AWS::StackName + Properties: + image: + Ref: ComputeImage + Type: OS::Nova::Server + NovaCompute0CompletionCondition: + DependsOn: notcompute + Properties: + Count: '1' + Handle: + Ref: NovaCompute0CompletionHandle + Timeout: '1800' + Type: AWS::CloudFormation::WaitCondition + NovaCompute0CompletionHandle: + Type: AWS::CloudFormation::WaitConditionHandle + NovaCompute0Config: + Metadata: + completion-handle: + Ref: NovaCompute0CompletionHandle + neutron: + ovs: + local_ip: + Fn::Select: + - 0 + - Fn::Select: + - ctlplane + - Fn::GetAtt: + - NovaCompute0 + - networks + os-collect-config: + cfn: + access_key_id: + Ref: NovaCompute0Key + path: NovaCompute0Config.Metadata + secret_access_key: + Fn::GetAtt: + - NovaCompute0Key + - SecretAccessKey + stack_name: + Ref: AWS::StackName + rabbit: + password: guest + Type: AWS::AutoScaling::LaunchConfiguration + NovaCompute0Key: + Properties: + UserName: + Ref: ComputeUser + Type: AWS::IAM::AccessKey + NovaCompute1: + Metadata: + os-collect-config: + cfn: + access_key_id: + Ref: NovaCompute1Key + path: NovaCompute1Config.Metadata + secret_access_key: + Fn::GetAtt: + - NovaCompute1Key + - SecretAccessKey + stack_name: + Ref: AWS::StackName + Properties: + image: + Ref: ComputeImage + Type: OS::Nova::Server + NovaCompute1CompletionCondition: + DependsOn: notcompute + Properties: + Count: '1' + Handle: + Ref: NovaCompute1CompletionHandle + Timeout: '1800' + Type: AWS::CloudFormation::WaitCondition + NovaCompute1CompletionHandle: + Type: AWS::CloudFormation::WaitConditionHandle + NovaCompute1Config: + Metadata: + completion-handle: + Ref: NovaCompute1CompletionHandle + neutron: + ovs: + local_ip: + Fn::Select: + - 0 + - Fn::Select: + - ctlplane + - Fn::GetAtt: + - NovaCompute1 + - networks + os-collect-config: + cfn: + access_key_id: + Ref: NovaCompute1Key + path: NovaCompute1Config.Metadata + secret_access_key: + Fn::GetAtt: + - NovaCompute1Key + - SecretAccessKey + stack_name: + Ref: AWS::StackName + rabbit: + password: guest + Type: AWS::AutoScaling::LaunchConfiguration + NovaCompute1Key: + Properties: + UserName: + Ref: ComputeUser + Type: AWS::IAM::AccessKey + NovaCompute2: + Metadata: + os-collect-config: + cfn: + access_key_id: + Ref: NovaCompute2Key + path: NovaCompute2Config.Metadata + secret_access_key: + Fn::GetAtt: + - NovaCompute2Key + - SecretAccessKey + stack_name: + Ref: AWS::StackName + Properties: + image: + Ref: ComputeImage + Type: OS::Nova::Server + NovaCompute2CompletionCondition: + DependsOn: notcompute + Properties: + Count: '1' + Handle: + Ref: NovaCompute2CompletionHandle + Timeout: '1800' + Type: AWS::CloudFormation::WaitCondition + NovaCompute2CompletionHandle: + Type: AWS::CloudFormation::WaitConditionHandle + NovaCompute2Config: + Metadata: + completion-handle: + Ref: NovaCompute2CompletionHandle + neutron: + ovs: + local_ip: + Fn::Select: + - 0 + - Fn::Select: + - ctlplane + - Fn::GetAtt: + - NovaCompute2 + - networks + os-collect-config: + cfn: + access_key_id: + Ref: NovaCompute2Key + path: NovaCompute2Config.Metadata + secret_access_key: + Fn::GetAtt: + - NovaCompute2Key + - SecretAccessKey + stack_name: + Ref: AWS::StackName + rabbit: + password: guest + Type: AWS::AutoScaling::LaunchConfiguration + NovaCompute2Key: + Properties: + UserName: + Ref: ComputeUser + Type: AWS::IAM::AccessKey diff --git a/test_merge.bash b/test_merge.bash index 61462f66..f9bc7fb6 100755 --- a/test_merge.bash +++ b/test_merge.bash @@ -28,6 +28,7 @@ run_test "python $merge_py examples/source.yaml" examples/source_lib_result.yaml run_test "python $merge_py examples/source2.yaml" examples/source2_lib_result.yaml run_test "python $merge_py examples/source_include_subkey.yaml" examples/source_include_subkey_result.yaml run_test "python $merge_py examples/launchconfig1.yaml examples/launchconfig2.yaml" examples/launchconfig_result.yaml +run_test "python $merge_py --scale NovaCompute=3 examples/scale1.yaml" examples/scale_result.yaml echo trap - EXIT exit $fail diff --git a/tripleo_heat_merge/merge.py b/tripleo_heat_merge/merge.py index 2975bd02..7b5951a3 100644 --- a/tripleo_heat_merge/merge.py +++ b/tripleo_heat_merge/merge.py @@ -4,6 +4,97 @@ import yaml import argparse +def apply_scaling(template, scaling, in_copies=None): + """Apply a set of scaling operations to template. + + This is a single pass recursive function: for each call we process one + dict or list and recurse to handle children containers. + + Values are handled via scale_value. + + Keys in dicts are always copied per the scaling rule. + Values are either replaced or copied depending on whether the given + scaling rule is in in_copies. + """ + in_copies = dict(in_copies or {}) + # Shouldn't be needed but to avoid unexpected side effects/bugs we short + # circuit no-ops. + if not scaling: + return template + if isinstance(template, dict): + new_template = {} + for key, value in template.items(): + for prefix, copy_num, new_key in scale_value( + key, scaling, in_copies): + if prefix: + # e.g. Compute0, 1, Compute1Foo + in_copies[prefix] = prefix[:-1] + str(copy_num) + if isinstance(value, (dict, list)): + new_value = apply_scaling(value, scaling, in_copies) + new_template[new_key] = new_value + else: + new_values = list(scale_value(value, scaling, in_copies)) + # We have nowhere to multiply a non-container value of a + # dict, so it may be copied or unchanged but not scaled. + assert len(new_values) == 1 + new_template[new_key] = new_values[0][2] + if prefix: + del in_copies[prefix] + return new_template + elif isinstance(template, list): + new_template = [] + for value in template: + if isinstance(value, (dict, list)): + new_template.append(apply_scaling(value, scaling, in_copies)) + else: + for _, _, new_value in scale_value(value, scaling, in_copies): + new_template.append(new_value) + return new_template + else: + raise Exception("apply_scaling called with non-container %r" % template) + + +def scale_value(value, scaling, in_copies): + """Scale out a value. + + :param value: The value to scale (not a container). + :param scaling: The scaling map to use. + :param in_copies: What containers we're currently copying. + :return: An iterator of the new values for the value as tuples: + (prefix, copy_num, value). E.g. Compute0, 1, Compute1Foo + prefix and copy_num are only set when: + - a prefix in scaling matches value + - and that prefix is not in in_copies + """ + if isinstance(value, (str, unicode)): + for prefix, copies in scaling.items(): + if not value.startswith(prefix): + continue + suffix = value[len(prefix):] + if prefix in in_copies: + # Adjust to the copy number we're on + yield None, None, in_copies[prefix] + suffix + return + else: + for n in range(copies): + yield prefix, n, prefix[:-1] + str(n) + suffix + return + yield None, None, value + else: + yield None, None, value + + +def parse_scaling(scaling_args): + """Translate a list of scaling requests to a dict prefix:count.""" + scaling_args = scaling_args or [] + result = {} + for item in scaling_args: + key, value = item.split('=') + value = int(value) + result[key + '0'] = value + return result + + def _translate_role(role, master_role, slave_roles): if not master_role: return role @@ -92,18 +183,27 @@ def main(argv=None): parser.add_argument('--output', help='File to write output to. - for stdout', default='-') + parser.add_argument('--scale', action="append", + help="Names to scale out. Pass Prefix=1 to cause a key Prefix0Foo to " + "be copied to Prefix1Foo in the output, and value Prefix0Bar to be" + "renamed to Prefix1Bar inside that copy, or copied to Prefix1Bar " + "outside of any copy.") args = parser.parse_args(argv) templates = args.templates + scaling = parse_scaling(args.scale) merged_template = merge(templates, args.master_role, args.slave_roles, - args.included_template_dir) + args.included_template_dir, scaling=scaling) if args.output == '-': out_file = sys.stdout else: out_file = file(args.output, 'wt') out_file.write(merged_template) + def merge(templates, master_role=None, slave_roles=None, - included_template_dir=INCLUDED_TEMPLATE_DIR): + included_template_dir=INCLUDED_TEMPLATE_DIR, + scaling=None): + scaling = scaling or {} errors = [] end_template={'HeatTemplateFormatVersion': '2012-12-12', 'Description': []} @@ -192,6 +292,8 @@ def merge(templates, master_role=None, slave_roles=None, end_template['Resources'] = {} end_template['Resources'][r] = rbody + end_template = apply_scaling(end_template, scaling) + def fix_ref(item, old, new): if isinstance(item, dict): copy_item = dict(item) -- cgit 1.2.3-korg