Make scaling out a feature for merge.py.

This is a simple implementation designed to work with our current
non-HOT approach. We need to adjust our templates a little to support
this - though future work could make that better, it's orthogonal to
this effort.

Change-Id: I555617e5f24a5882de915f057dc02e008c81e753

5 files changed, 391 insertions, 2 deletions

diff --git a/examples/scale1.yaml b/examples/scale1.yaml
new file mode 100644
index 00000000..c0a0763d
--- /dev/null
+++ b/examples/scale1.yaml
@@ -0,0 +1,30 @@
+Resources:
+  ComputeUser:
+    Type: AWS::IAM::User
+    Properties:
+      Policies: [ { Ref: ComputeAccessPolicy } ]
+  GlobalAccessPolicy:
+    Type: OS::Heat::AccessPolicy
+  NovaCompute0Key:
+    Type: FileInclude
+    Path: examples/scale2.yaml
+    SubKey: Resources.NovaCompute0Key
+  NovaCompute0CompletionCondition:
+    Type: FileInclude
+    Path: examples/scale2.yaml
+    SubKey: Resources.NovaCompute0CompletionCondition
+  NovaCompute0CompletionHandle:
+    Type: FileInclude
+    Path: examples/scale2.yaml
+    SubKey: Resources.NovaCompute0CompletionHandle
+  NovaCompute0Config:
+    Type: FileInclude
+    Path: examples/scale2.yaml
+    SubKey: Resources.NovaCompute0Config
+    Parameters:
+        ComputeImage: "123"
+        RabbitPassword: "guest"
+  NovaCompute0:
+    Type: FileInclude
+    Path: examples/scale2.yaml
+    SubKey: Resources.NovaCompute0
diff --git a/examples/scale2.yaml b/examples/scale2.yaml
new file mode 100644
index 00000000..d1a81fe5
--- /dev/null
+++ b/examples/scale2.yaml
@@ -0,0 +1,66 @@
+HeatTemplateFormatVersion: '2012-12-12'
+Parameters:
+  ComputeImage:
+    Type: String
+  RabbitPassword:
+    Type: String
+    NoEcho: true
+Resources:
+  ComputeAccessPolicy:
+    Type: OS::Heat::AccessPolicy
+    Properties:
+      AllowedResources: [ NovaCompute0 ]
+  NovaCompute0Key:
+    Type: AWS::IAM::AccessKey
+    Properties:
+      UserName:
+        Ref: ComputeUser
+  NovaCompute0CompletionCondition:
+    Type: AWS::CloudFormation::WaitCondition
+    DependsOn: notcompute
+    Properties:
+      Handle: {Ref: NovaCompute0CompletionHandle}
+      Count: '1'
+      Timeout: '1800'
+  NovaCompute0CompletionHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+  NovaCompute0:
+    Type: OS::Nova::Server
+    Properties:
+      image:
+        Ref: ComputeImage
+    Metadata:
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute0Key
+          secret_access_key:
+            Fn::GetAtt: [ NovaCompute0Key, SecretAccessKey ]
+          stack_name: {Ref: 'AWS::StackName'}
+          path: NovaCompute0Config.Metadata
+  NovaCompute0Config:
+    Type: AWS::AutoScaling::LaunchConfiguration
+    Metadata:
+      completion-handle:
+        Ref: NovaCompute0CompletionHandle
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute0Key
+          secret_access_key:
+            Fn::GetAtt: [ NovaCompute0Key, SecretAccessKey ]
+          stack_name: {Ref: 'AWS::StackName'}
+          path: NovaCompute0Config.Metadata
+      neutron:
+        ovs:
+          local_ip:
+            Fn::Select:
+              - 0
+              - Fn::Select:
+                - ctlplane
+                - Fn::GetAtt:
+                  - NovaCompute0
+                  - networks
+      rabbit:
+        password: {Ref: RabbitPassword}
+
diff --git a/examples/scale_result.yaml b/examples/scale_result.yaml
new file mode 100644
index 00000000..f22d864e
--- /dev/null
+++ b/examples/scale_result.yaml
@@ -0,0 +1,190 @@
+Description: examples/scale1.yaml
+HeatTemplateFormatVersion: '2012-12-12'
+Resources:
+  ComputeUser:
+    Properties:
+      Policies:
+      - Ref: ComputeAccessPolicy
+    Type: AWS::IAM::User
+  GlobalAccessPolicy:
+    Type: OS::Heat::AccessPolicy
+  NovaCompute0:
+    Metadata:
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute0Key
+          path: NovaCompute0Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute0Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+    Properties:
+      image:
+        Ref: ComputeImage
+    Type: OS::Nova::Server
+  NovaCompute0CompletionCondition:
+    DependsOn: notcompute
+    Properties:
+      Count: '1'
+      Handle:
+        Ref: NovaCompute0CompletionHandle
+      Timeout: '1800'
+    Type: AWS::CloudFormation::WaitCondition
+  NovaCompute0CompletionHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+  NovaCompute0Config:
+    Metadata:
+      completion-handle:
+        Ref: NovaCompute0CompletionHandle
+      neutron:
+        ovs:
+          local_ip:
+            Fn::Select:
+            - 0
+            - Fn::Select:
+              - ctlplane
+              - Fn::GetAtt:
+                - NovaCompute0
+                - networks
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute0Key
+          path: NovaCompute0Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute0Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+      rabbit:
+        password: guest
+    Type: AWS::AutoScaling::LaunchConfiguration
+  NovaCompute0Key:
+    Properties:
+      UserName:
+        Ref: ComputeUser
+    Type: AWS::IAM::AccessKey
+  NovaCompute1:
+    Metadata:
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute1Key
+          path: NovaCompute1Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute1Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+    Properties:
+      image:
+        Ref: ComputeImage
+    Type: OS::Nova::Server
+  NovaCompute1CompletionCondition:
+    DependsOn: notcompute
+    Properties:
+      Count: '1'
+      Handle:
+        Ref: NovaCompute1CompletionHandle
+      Timeout: '1800'
+    Type: AWS::CloudFormation::WaitCondition
+  NovaCompute1CompletionHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+  NovaCompute1Config:
+    Metadata:
+      completion-handle:
+        Ref: NovaCompute1CompletionHandle
+      neutron:
+        ovs:
+          local_ip:
+            Fn::Select:
+            - 0
+            - Fn::Select:
+              - ctlplane
+              - Fn::GetAtt:
+                - NovaCompute1
+                - networks
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute1Key
+          path: NovaCompute1Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute1Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+      rabbit:
+        password: guest
+    Type: AWS::AutoScaling::LaunchConfiguration
+  NovaCompute1Key:
+    Properties:
+      UserName:
+        Ref: ComputeUser
+    Type: AWS::IAM::AccessKey
+  NovaCompute2:
+    Metadata:
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute2Key
+          path: NovaCompute2Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute2Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+    Properties:
+      image:
+        Ref: ComputeImage
+    Type: OS::Nova::Server
+  NovaCompute2CompletionCondition:
+    DependsOn: notcompute
+    Properties:
+      Count: '1'
+      Handle:
+        Ref: NovaCompute2CompletionHandle
+      Timeout: '1800'
+    Type: AWS::CloudFormation::WaitCondition
+  NovaCompute2CompletionHandle:
+    Type: AWS::CloudFormation::WaitConditionHandle
+  NovaCompute2Config:
+    Metadata:
+      completion-handle:
+        Ref: NovaCompute2CompletionHandle
+      neutron:
+        ovs:
+          local_ip:
+            Fn::Select:
+            - 0
+            - Fn::Select:
+              - ctlplane
+              - Fn::GetAtt:
+                - NovaCompute2
+                - networks
+      os-collect-config:
+        cfn:
+          access_key_id:
+            Ref: NovaCompute2Key
+          path: NovaCompute2Config.Metadata
+          secret_access_key:
+            Fn::GetAtt:
+            - NovaCompute2Key
+            - SecretAccessKey
+          stack_name:
+            Ref: AWS::StackName
+      rabbit:
+        password: guest
+    Type: AWS::AutoScaling::LaunchConfiguration
+  NovaCompute2Key:
+    Properties:
+      UserName:
+        Ref: ComputeUser
+    Type: AWS::IAM::AccessKey
diff --git a/test_merge.bash b/test_merge.bash
index 61462f66..f9bc7fb6 100755
--- a/test_merge.bash
+++ b/test_merge.bash
@@ -28,6 +28,7 @@ run_test "python $merge_py examples/source.yaml" examples/source_lib_result.yaml
 run_test "python $merge_py examples/source2.yaml" examples/source2_lib_result.yaml
 run_test "python $merge_py examples/source_include_subkey.yaml" examples/source_include_subkey_result.yaml
 run_test "python $merge_py examples/launchconfig1.yaml examples/launchconfig2.yaml" examples/launchconfig_result.yaml
+run_test "python $merge_py --scale NovaCompute=3 examples/scale1.yaml" examples/scale_result.yaml
 echo
 trap - EXIT
 exit $fail
diff --git a/tripleo_heat_merge/merge.py b/tripleo_heat_merge/merge.py
index 2975bd02..7b5951a3 100644
--- a/tripleo_heat_merge/merge.py
+++ b/tripleo_heat_merge/merge.py
@@ -4,6 +4,97 @@ import yaml
 import argparse
 
 
+def apply_scaling(template, scaling, in_copies=None):
+    """Apply a set of scaling operations to template.
+
+    This is a single pass recursive function: for each call we process one
+    dict or list and recurse to handle children containers.
+
+    Values are handled via scale_value.
+
+    Keys in dicts are always copied per the scaling rule.
+    Values are either replaced or copied depending on whether the given
+    scaling rule is in in_copies.
+    """
+    in_copies = dict(in_copies or {})
+    # Shouldn't be needed but to avoid unexpected side effects/bugs we short
+    # circuit no-ops.
+    if not scaling:
+        return template
+    if isinstance(template, dict):
+        new_template = {}
+        for key, value in template.items():
+            for prefix, copy_num, new_key in scale_value(
+                key, scaling, in_copies):
+                if prefix:
+                    # e.g. Compute0, 1, Compute1Foo
+                    in_copies[prefix] = prefix[:-1] + str(copy_num)
+                if isinstance(value, (dict, list)):
+                    new_value = apply_scaling(value, scaling, in_copies)
+                    new_template[new_key] = new_value
+                else:
+                    new_values = list(scale_value(value, scaling, in_copies))
+                    # We have nowhere to multiply a non-container value of a
+                    # dict, so it may be copied or unchanged but not scaled.
+                    assert len(new_values) == 1
+                    new_template[new_key] = new_values[0][2]
+                if prefix:
+                    del in_copies[prefix]
+        return new_template
+    elif isinstance(template, list):
+        new_template = []
+        for value in template:
+            if isinstance(value, (dict, list)):
+                new_template.append(apply_scaling(value, scaling, in_copies))
+            else:
+                for _, _, new_value in scale_value(value, scaling, in_copies):
+                    new_template.append(new_value)
+        return new_template
+    else:
+        raise Exception("apply_scaling called with non-container %r" % template)
+
+
+def scale_value(value, scaling, in_copies):
+    """Scale out a value.
+
+    :param value: The value to scale (not a container).
+    :param scaling: The scaling map to use.
+    :param in_copies: What containers we're currently copying.
+    :return: An iterator of the new values for the value as tuples:
+        (prefix, copy_num, value). E.g. Compute0, 1, Compute1Foo
+        prefix and copy_num are only set when:
+         - a prefix in scaling matches value
+         - and that prefix is not in in_copies
+    """
+    if isinstance(value, (str, unicode)):
+        for prefix, copies in scaling.items():
+            if not value.startswith(prefix):
+                continue
+            suffix = value[len(prefix):]
+            if prefix in in_copies:
+                # Adjust to the copy number we're on
+                yield None, None, in_copies[prefix] + suffix
+                return
+            else:
+                for n in range(copies):
+                    yield prefix, n, prefix[:-1] + str(n) + suffix
+                return
+        yield None, None, value
+    else:
+        yield None, None, value
+
+
+def parse_scaling(scaling_args):
+    """Translate a list of scaling requests to a dict prefix:count."""
+    scaling_args = scaling_args or []
+    result = {}
+    for item in scaling_args:
+        key, value = item.split('=')
+        value = int(value)
+        result[key + '0'] = value
+    return result
+
+
 def _translate_role(role, master_role, slave_roles):
     if not master_role:
         return role
@@ -92,18 +183,27 @@ def main(argv=None):
     parser.add_argument('--output',
                         help='File to write output to. - for stdout',
                         default='-')
+    parser.add_argument('--scale', action="append",
+        help="Names to scale out. Pass Prefix=1 to cause a key Prefix0Foo to "
+        "be copied to Prefix1Foo in the output, and value Prefix0Bar to be"
+        "renamed to Prefix1Bar inside that copy, or copied to Prefix1Bar "
+        "outside of any copy.")
     args = parser.parse_args(argv)
     templates = args.templates
+    scaling = parse_scaling(args.scale)
     merged_template = merge(templates, args.master_role, args.slave_roles,
-                            args.included_template_dir)
+                            args.included_template_dir, scaling=scaling)
     if args.output == '-':
         out_file = sys.stdout
     else:
         out_file = file(args.output, 'wt')
     out_file.write(merged_template)
 
+
 def merge(templates, master_role=None, slave_roles=None,
-          included_template_dir=INCLUDED_TEMPLATE_DIR):
+          included_template_dir=INCLUDED_TEMPLATE_DIR,
+          scaling=None):
+    scaling = scaling or {}
     errors = []
     end_template={'HeatTemplateFormatVersion': '2012-12-12',
                   'Description': []}
@@ -192,6 +292,8 @@ def merge(templates, master_role=None, slave_roles=None,
                     end_template['Resources'] = {}
                 end_template['Resources'][r] = rbody
 
+    end_template = apply_scaling(end_template, scaling)
+
     def fix_ref(item, old, new):
         if isinstance(item, dict):
             copy_item = dict(item)