Render IP map and host maps according to network_data.yaml

This change renders the network IP maps and hostname maps for
all networks defined in network_data.yaml. This should make it
possible to create custom networks that will be rendered for
all applicable roles.

Note that at this time all networks will be rendered whether
they are enabled or not. All networks will be present in all
roles, but ports will be associated with noop.yaml in roles
that do not use the network. This is in accordance with
previous behavior, although we may wish to change this in
the future to limit the size of the role definitions and
reduce the number of placeholder resources in deployments
with many networks.

Note that this patch is a replacement for original patch
https://review.openstack.org/#/c/486280, which I was having
trouble rebasing to current.

Change-Id: I445b008fc1240af57c2b76a5dbb6c751a05b7a2a
Depends-on: I662e8d0b3737c7807d18c8917bfce1e25baa3d8a
Partially-implements: blueprint composable-networks

1 files changed, 190 insertions, 0 deletions

diff --git a/network/ports/net_ip_list_map.j2.yaml b/network/ports/net_ip_list_map.j2.yaml
new file mode 100644
index 00000000..e929ab2c
--- /dev/null
+++ b/network/ports/net_ip_list_map.j2.yaml
@@ -0,0 +1,190 @@
+heat_template_version: pike
+
+parameters:
+  ControlPlaneIpList:
+    default: []
+    type: comma_delimited_list
+{%- for network in networks %}
+  {{network.name}}IpList:
+    default: []
+    type: comma_delimited_list
+{%- endfor %}
+  EnabledServices:
+    default: []
+    type: comma_delimited_list
+  ServiceNetMap:
+    default: {}
+    type: json
+  ServiceHostnameList:
+    default: []
+    type: comma_delimited_list
+  NetworkHostnameMap:
+    default: []
+    type: json
+
+  InternalApiNetName:
+    default: internal_api
+    description: The name of the internal_api network.
+    type: string
+  ExternalNetName:
+    default: external
+    description: The name of the external network.
+    type: string
+  ManagementNetName:
+    default: management
+    description: The name of the management network.
+    type: string
+  StorageNetName:
+    default: storage
+    description: The name of the storage network.
+    type: string
+  StorageMgmtNetName:
+    default: storage_mgmt
+    description: The name of the storage_mgmt network.
+    type: string
+  TenantNetName:
+    default: tenant
+    description: The name of the tenant network.
+{%- for network in networks %}
+  {{network.name}}NetName:
+    default: {{network.name_lower}}
+    description: The name of the {{network.name_lower}} network.
+    type: string
+{%- endfor %}
+
+
+resources:
+   # This adds the extra "services" on for keystone
+   # so that keystone_admin_api_network and
+   # keystone_public_api_network point to the correct
+   # network on the nodes running the "keystone" service
+  EnabledServicesValue:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          expression: let(root => $) -> $.data.extra_services.items().where($[0] in $root.data.enabled_services).select($[1]).flatten() + $root.data.enabled_services
+          data:
+            enabled_services: {get_param: EnabledServices}
+            extra_services:
+              # If anything other than keystone needs this
+              # then we should add an extra_networks interface
+              # to the service templates role_data but for
+              # now we hard-code the keystone special case
+              keystone:
+                - keystone_admin_api
+                - keystone_public_api
+
+  NetIpMapValue:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        map_replace:
+          - ctlplane: {get_param: ControlPlaneIpList}
+{%- for network in networks %}
+            {{network.name_lower}}: {get_param: {{network.name}}IpList}
+{%- endfor %}
+          - keys:
+{%- for network in networks %}
+              {{network.name_lower}}: {get_param: {{network.name}}NetName}
+{%- endfor %}
+
+outputs:
+  net_ip_map:
+    description: >
+      A Hash containing a mapping of network names to assigned lists
+      of IP addresses.
+    value: {get_attr: [NetIpMapValue, value]}
+  service_ips:
+    description: >
+      Map of enabled services to a list of their IP addresses
+    value:
+      yaql:
+        # This filters any entries where the value hasn't been substituted for
+        # a list, e.g it's still $service_network.  This happens when there is
+        # no network defined for the service in the ServiceNetMap, which is OK
+        # as not all services have to be bound to a network, so we filter them
+        expression: dict($.data.map.items().where(not isString($[1])))
+        data:
+          map:
+            map_replace:
+              - map_replace:
+                  - map_merge:
+                      repeat:
+                        template:
+                          SERVICE_node_ips: SERVICE_network
+                        for_each:
+                          SERVICE: {get_attr: [EnabledServicesValue, value]}
+                  - values: {get_param: ServiceNetMap}
+              - values: {get_attr: [NetIpMapValue, value]}
+  ctlplane_service_ips:
+    description: >
+      Map of enabled services to a list of their ctlplane IP addresses
+    value:
+      yaql:
+        expression: dict($.data.map.items().where(len($[1]) > 0))
+        data:
+          map:
+            map_merge:
+              repeat:
+                template:
+                  SERVICE_ctlplane_node_ips: {get_param: ControlPlaneIpList}
+                for_each:
+                  SERVICE: {get_attr: [EnabledServicesValue, value]}
+  service_hostnames:
+    description: >
+      Map of enabled services to a list of hostnames where they're running
+    value:
+      map_replace:
+        - yaql:
+            # This filters any entries where the value hasn't been substituted for
+            # a list, e.g it's still $service_network.  This happens when there is
+            # no network defined for the service in the ServiceNetMap, which is OK
+            # as not all services have to be bound to a network, so we filter them
+            expression: dict($.data.map.items().where(not $[1].endsWith("_network")))
+            data:
+              map:
+                map_replace:
+                  - map_merge:
+                      repeat:
+                        template:
+                          SERVICE_node_names: SERVICE_network
+                        for_each:
+                          SERVICE: {get_attr: [EnabledServicesValue, value]}
+                  - values: {get_param: ServiceNetMap}
+        - values: {get_param: NetworkHostnameMap}
+  short_service_hostnames:
+    description: >
+      Map of enabled services to a list of hostnames where they're running regardless of the network
+    value:
+      yaql:
+        # If ServiceHostnameList is empty the role is deployed with zero nodes
+        # therefore we don't want to add any *_node_names to the map
+        expression: dict($.data.map.items().where(len($[1]) > 0))
+        data:
+          map:
+            map_merge:
+              repeat:
+                template:
+                  SERVICE_short_node_names: {get_param: ServiceHostnameList}
+                for_each:
+                    SERVICE: {get_attr: [EnabledServicesValue, value]}
+  short_service_bootstrap_hostnames:
+    description: >
+      Map of enabled services to a list of hostnames where they're running regardless of the network
+      Used for bootstrap purposes
+    value:
+      yaql:
+        # If ServiceHostnameList is empty the role is deployed with zero nodes
+        # therefore we don't want to add any *_node_names to the map
+        expression: dict($.data.map.items().where(len($[1]) > 0))
+        data:
+          map:
+            map_merge:
+              repeat:
+                template:
+                  SERVICE_short_bootstrap_node_name: {get_param: ServiceHostnameList}
+                for_each:
+                    SERVICE: {get_attr: [EnabledServicesValue, value]}