Add templating support for generating config files

 - Remove unneeded method
 - Write result to a file and not STDOUT
 - Add documentation
 - Remove trailing whitespace
 - Documentation corrections

Change-Id: I7532222d3512380c4f1129bd05dc2ba37b409dc2
Signed-off-by: Peter Barabas <peter.barabas@ericsson.com>

2 files changed, 446 insertions, 0 deletions

diff --git a/deploy/README.templater b/deploy/README.templater
new file mode 100644
index 000000000..964872fb7
--- /dev/null
+++ b/deploy/README.templater
@@ -0,0 +1,277 @@
+##############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# peter.barabas@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+======== TEMPLATING SUPPORT IN YAML CONFIGURATION FILES ========
+
+deploy/templater.py makes it possible to use templates to generate configuration
+files. It takes 2 input YAML files and an output file as arguments. One being
+the dictionary (called the base file), which is used to look up values in; the
+other file is the template, where the substitution will take place. Templater
+will write the result to an output file, specified as the 3rd argument.
+
+
+======== SYNTAX OF TEMPLATE FILES ========
+
+A template file can contain any valid YAML data and template variables, whose
+syntax is described below:
+
+1. Single value references
+
+   %{title}
+
+   %{environment/net_segment_type}
+
+   Either a root element, or a path can be specified.
+
+2. YAML sections
+
+   %{nodes}
+
+   %{network/networking_parameters}
+
+   Either a root element, or a path can be specified.
+
+3. Interface lookup for network
+
+   %{interface(storage)}
+
+   Specify a network type as argument to interface().
+
+4. Interface lookup for network and role
+
+   %{interface(public,compute)}
+
+   Specify a network type and a role as arguments to interface().
+
+
+======== EXAMPLES ========
+
+Base YAML file (excerpt):
+
+title: Deployment Environment Adapter (DEA)
+version: 1.1
+created: Wed Mar 30 08:16:04 2016
+environment:
+  name: vCity
+  net_segment_type: tun
+wanted_release: Liberty on Ubuntu 14.04
+nodes:
+- id: 1
+  interfaces: interfaces_1
+  role: ceph-osd,compute
+  transformations: transformations_1
+- id: 2
+  interfaces: interfaces_1
+  role: ceph-osd,compute
+  transformations: transformations_1
+- id: 3
+  interfaces: interfaces_1
+  role: ceph-osd,compute
+  transformations: transformations_1
+- id: 4
+  interfaces: interfaces_2
+  role: controller,mongo
+  transformations: transformations_2
+- id: 5
+  interfaces: interfaces_2
+  role: controller,mongo
+  transformations: transformations_2
+- id: 6
+  interfaces: interfaces_2
+  role: controller,mongo
+  transformations: transformations_2
+interfaces_1:
+  ens3:
+  - fuelweb_admin
+  - management
+  ens4:
+  - storage
+  ens5:
+  - private
+  ens6:
+  - public
+interfaces_2:
+  ens3:
+  - fuelweb_admin
+  - management
+  ens4:
+  - storage
+  - private
+  - public
+network:
+  networks:
+  - cidr: 172.16.0.0/24
+    gateway: 172.16.0.1
+    ip_ranges:
+    - - 172.16.0.2
+      - 172.16.0.126
+    meta:
+      cidr: 172.16.0.0/24
+      configurable: true
+      floating_range_var: floating_ranges
+      ip_range:
+      - 172.16.0.2
+      - 172.16.0.126
+      map_priority: 1
+      name: public
+      notation: ip_ranges
+      render_addr_mask: public
+      render_type: null
+      use_gateway: true
+      vips:
+      - haproxy
+      - vrouter
+      vlan_start: null
+    name: public
+    vlan_start: null
+  - cidr: 192.168.1.0/24
+    gateway: null
+    ip_ranges:
+    - - 192.168.1.1
+      - 192.168.1.254
+    meta:
+      cidr: 192.168.1.0/24
+      configurable: true
+      map_priority: 2
+      name: storage
+      notation: cidr
+      render_addr_mask: storage
+      render_type: cidr
+      use_gateway: false
+      vlan_start: 102
+    name: storage
+    vlan_start: 102
+
+
+--- Example 1 ---
+
+Template file:
+
+deployment-scenario-metadata:
+  title: %{title}
+  version: 0.1
+dea-override-config:
+  environment:
+    net_segment_type: %{environment/net_segment_type}
+  nodes:
+  %{nodes}
+
+
+Result:
+
+deployment-scenario-metadata:
+  title: Deployment Environment Adapter (DEA)
+  version: 0.1
+dea-override-config:
+  environment:
+    net_segment_type: tun
+  nodes:
+  - id: 1
+    interfaces: interfaces_1
+    role: ceph-osd,compute
+    transformations: transformations_1
+  - id: 2
+    interfaces: interfaces_1
+    role: ceph-osd,compute
+    transformations: transformations_1
+  - id: 3
+    interfaces: interfaces_1
+    role: ceph-osd,compute
+    transformations: transformations_1
+  - id: 4
+    interfaces: interfaces_2
+    role: controller,mongo
+    transformations: transformations_2
+  - id: 5
+    interfaces: interfaces_2
+    role: controller,mongo
+    transformations: transformations_2
+  - id: 6
+    interfaces: interfaces_2
+    role: controller,mongo
+    transformations: transformations_2
+
+
+--- Example 2 ---
+
+Template file:
+
+dea-override-config:
+  network:
+    networks:
+    %{network/networks}
+
+
+Result:
+
+dea-override-config:
+  network:
+    networks:
+    - cidr: 172.16.0.0/24
+      gateway: 172.16.0.1
+      ip_ranges:
+      - - 172.16.0.2
+        - 172.16.0.126
+      meta:
+        cidr: 172.16.0.0/24
+        configurable: true
+        floating_range_var: floating_ranges
+        ip_range:
+        - 172.16.0.2
+        - 172.16.0.126
+        map_priority: 1
+        name: public
+        notation: ip_ranges
+        render_addr_mask: public
+        render_type: null
+        use_gateway: true
+        vips:
+        - haproxy
+        - vrouter
+        vlan_start: null
+      name: public
+      vlan_start: null
+    - cidr: 192.168.1.0/24
+      gateway: null
+      ip_ranges:
+      - - 192.168.1.1
+        - 192.168.1.254
+      meta:
+        cidr: 192.168.1.0/24
+        configurable: true
+        map_priority: 2
+        name: storage
+        notation: cidr
+        render_addr_mask: storage
+        render_type: cidr
+        use_gateway: false
+        vlan_start: 102
+      name: storage
+      vlan_start: 102
+
+
+--- Example 3 ---
+
+Template file:
+
+storage_if: %{interface(storage)}
+compute_private_if: %{interface(private,compute)}
+# Management interface of a mongo node
+mongo_mgmt_if: %{interface(management,mongo)}
+controller_private_if: %{interface(private,controller)}
+
+
+Result:
+
+storage_if: ens4
+compute_private_if: ens5
+# Management interface of a mongo node
+mongo_mgmt_if: ens3
+controller_private_if: ens4
+
diff --git a/deploy/templater.py b/deploy/templater.py
new file mode 100755
index 000000000..2ad6e05ba
--- /dev/null
+++ b/deploy/templater.py
@@ -0,0 +1,169 @@
+#!/usr/bin/env python
+###############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# peter.barabas@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+###############################################################################
+
+
+import io
+import re
+import yaml
+from common import(
+    err,
+    ArgParser,
+)
+
+
+TAG_START = '%{'
+TAG_END = '}'
+DELIMITER = '/'
+
+
+class Templater(object):
+    def __init__(self, base_file, template_file, output_file):
+        self.template_file = template_file
+        self.output_file = output_file
+        self.base = self.load_yaml(base_file)
+
+    def load_yaml(self, filename):
+        try:
+            with io.open(filename) as yaml_file:
+                return yaml.load(yaml_file)
+        except Exception as error:
+            err('Error opening YAML file: %s' % error)
+
+    def save_yaml(self, filename, content):
+        try:
+            with io.open(filename, 'w') as yaml_file:
+                yaml_file.write(content)
+        except Exception as error:
+            err('Error writing YAML file: %s' % error)
+
+    def get_indent(self, line):
+        return len(line) - len(line.lstrip(' '))
+
+    def format_fragment(self, fragment, indent):
+        result = ''
+        is_first_line = True
+
+        for line in fragment.splitlines():
+            # Skip indenting the first line as it is already indented
+            if is_first_line:
+                line += '\n'
+                is_first_line = False
+            else:
+                line = ' ' * indent + line + '\n'
+
+            result += line
+
+        return result.rstrip('\n')
+
+    def format_substitution(self, string):
+        if isinstance(string, basestring):
+            return string
+        else:
+            return yaml.dump(string, default_flow_style=False)
+
+    def parse_interface_tag(self, tag):
+        # Remove 'interface(' prefix, trailing ')' and split arguments
+        args = tag[len('interface('):].rstrip(')').split(',')
+
+        if len(args) == 1 and not args[0]:
+            err('No arguments for interface().')
+        elif len(args) == 2 and (not args[0] or not args[1]):
+            err('Empty argument for interface().')
+        elif len(args) > 2:
+            err('Too many arguments for interface().')
+        else:
+            return args
+
+    def get_interface_from_network(self, interfaces, network):
+        nics = self.base[interfaces]
+        for nic in nics:
+            if network in nics[nic]:
+                return nic
+
+        err('Network not found: %s' % network)
+
+    def get_role_interfaces(self, role):
+        nodes = self.base['nodes']
+        for node in nodes:
+            if role in node['role']:
+                return node['interfaces']
+
+        err('Role not found: %s' % role)
+
+    def lookup_interface(self, args):
+        nodes = self.base['nodes']
+
+        if len(args) == 1:
+            interfaces = nodes[0]['interfaces']
+        if len(args) == 2:
+            interfaces = self.get_role_interfaces(args[1])
+
+        return self.get_interface_from_network(interfaces, args[0])
+
+    def parse_tag(self, tag, indent):
+        fragment = ''
+
+        if 'interface(' in tag:
+            args = self.parse_interface_tag(tag)
+            fragment = self.lookup_interface(args)
+        else:
+            path = tag.split(DELIMITER)
+            fragment = self.base
+            for i in path:
+                if i in fragment:
+                    fragment = fragment.get(i)
+                else:
+                    err('Error: key "%s" does not exist in base YAML file' % i)
+
+            fragment = self.format_substitution(fragment)
+
+        return self.format_fragment(fragment, indent)
+
+    def run(self):
+        result = ''
+
+        regex = re.compile(re.escape(TAG_START) + r'([a-z].+)' + re.escape(TAG_END),
+                           flags=re.IGNORECASE)
+        with io.open(self.template_file) as f:
+            for line in f:
+                indent = self.get_indent(line)
+                result += re.sub(regex,
+                                 lambda match: self.parse_tag(match.group(1), indent),
+                                 line)
+
+        self.save_yaml(self.output_file, result)
+
+
+def parse_arguments():
+    description = '''Process 'template_file' using 'base_file' as source for
+template variable substitution and write the results to 'output_file'.'''
+
+    parser = ArgParser(prog='python %s' % __file__,
+                       description=description)
+    parser.add_argument('base_file',
+                        help='Base YAML filename')
+    parser.add_argument('template_file',
+                        help='Fragment filename')
+    parser.add_argument('output_file',
+                        help='Output filename')
+
+    args = parser.parse_args()
+    return(args.base_file, args.template_file, args.output_file)
+
+
+def main():
+    base_file, template_file, output_file = parse_arguments()
+
+    templater = Templater(base_file, template_file, output_file)
+    templater.run()
+
+
+if __name__ == '__main__':
+    main()