Create composable mapping between enabled services and role ips

Currently we have a hard-coded list of ips for various services that
run on the controller, instead we can dynamically generate that list
of per-service ips, initially only for the controller but this approach
can be extended so it works for any role.

Change-Id: I3c8a946e439539d239ad7281a1395414df0893eb
Partially-Implements: blueprint custom-roles

3 files changed, 44 insertions, 228 deletions

diff --git a/network/ports/net_ip_list_map.yaml b/network/ports/net_ip_list_map.yaml
index 32272bd6..36f3358e 100644
--- a/network/ports/net_ip_list_map.yaml
+++ b/network/ports/net_ip_list_map.yaml
@@ -1,4 +1,4 @@
-heat_template_version: 2015-04-30
+heat_template_version: 2016-10-14
 
 parameters:
   ControlPlaneIpList:
@@ -22,6 +22,12 @@ parameters:
   ManagementIpList:
     default: []
     type: comma_delimited_list
+  EnabledServices:
+    default: []
+    type: comma_delimited_list
+  ServiceNetMap:
+    default: {}
+    type: json
 
 outputs:
   net_ip_map:
@@ -36,3 +42,32 @@ outputs:
       storage_mgmt: {get_param: StorageMgmtIpList}
       tenant: {get_param: TenantIpList}
       management: {get_param: ManagementIpList}
+  service_ips:
+    description: >
+      Map of enabled services to a list of their IP addresses
+    value:
+      yaql:
+        # This filters any entries where the value hasn't been substituted for
+        # a list, e.g it's still $service_network.  This happens when there is
+        # no network defined for the service in the ServiceNetMap, which is OK
+        # as not all services have to be bound to a network, so we filter them
+        expression: dict($.data.map.items().where(not isString($[1])))
+        data:
+          map:
+            map_replace:
+              - map_replace:
+                  - map_merge:
+                      repeat:
+                        template:
+                          SERVICE_node_ips: SERVICE_network
+                        for_each:
+                          SERVICE: {get_param: EnabledServices}
+                  - values: {get_param: ServiceNetMap}
+              - values:
+                  ctlplane: {get_param: ControlPlaneIpList}
+                  external: {get_param: ExternalIpList}
+                  internal_api: {get_param: InternalApiIpList}
+                  storage: {get_param: StorageIpList}
+                  storage_mgmt: {get_param: StorageMgmtIpList}
+                  tenant: {get_param: TenantIpList}
+                  management: {get_param: ManagementIpList}
diff --git a/overcloud.yaml b/overcloud.yaml
index 419ac837..e034e8af 100644
--- a/overcloud.yaml
+++ b/overcloud.yaml
@@ -587,6 +587,8 @@ resources:
       StorageMgmtIpList: {get_attr: [Controller, storage_mgmt_ip_address]}
       TenantIpList: {get_attr: [Controller, tenant_ip_address]}
       ManagementIpList: {get_attr: [Controller, management_ip_address]}
+      EnabledServices: {get_attr: [ControllerServiceChain, role_data, service_names]}
+      ServiceNetMap: {get_attr: [ServiceNetMap, service_net_map_lower]}
 
   allNodesConfig:
     type: OS::TripleO::AllNodes::SoftwareConfig
@@ -617,30 +619,13 @@ resources:
           - {get_attr: [CephStorageServiceChain, role_data, service_names]}
       controller_ips: {get_attr: [Controller, ip_address]}
       controller_names: {get_attr: [Controller, hostname]}
+      service_ips: {get_attr: [ControllerIpListMap, service_ips]}
+      # FIXME(shardy): These require further work to move into service_ips
       rabbit_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, RabbitmqNetwork]}]}
-      mongo_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, MongodbNetwork]}]}
-      redis_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, RedisNetwork]}]}
       memcache_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, MemcachedNetwork]}]}
-      mysql_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, MysqlNetwork]}]}
-      horizon_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, HorizonNetwork]}]}
-      heat_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, HeatApiNetwork]}]}
-      swift_proxy_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, SwiftProxyNetwork]}]}
-      ceilometer_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, CeilometerApiNetwork]}]}
-      aodh_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, AodhApiNetwork]}]}
-      gnocchi_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, GnocchiApiNetwork]}]}
-      nova_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, NovaApiNetwork]}]}
-      nova_metadata_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, NovaMetadataNetwork]}]}
-      glance_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, GlanceApiNetwork]}]}
-      glance_registry_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, GlanceRegistryNetwork]}]}
-      cinder_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, CinderApiNetwork]}]}
-      manila_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, ManilaApiNetwork]}]}
-      neutron_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, NeutronApiNetwork]}]}
       keystone_public_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, KeystonePublicApiNetwork]}]}
       keystone_admin_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, KeystoneAdminApiNetwork]}]}
-      sahara_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, SaharaApiNetwork]}]}
-      ironic_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, IronicApiNetwork]}]}
       ceph_mon_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, CephPublicNetwork]}]}
-      opendaylight_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, OpenDaylightApiNetwork]}]}
       ceph_mon_node_names: {get_attr: [Controller, hostname]}
       DeployIdentifier: {get_param: DeployIdentifier}
       UpdateIdentifier: {get_param: UpdateIdentifier}
diff --git a/puppet/all-nodes-config.yaml b/puppet/all-nodes-config.yaml
index 20e50d4e..644c1938 100644
--- a/puppet/all-nodes-config.yaml
+++ b/puppet/all-nodes-config.yaml
@@ -10,58 +10,22 @@ parameters:
     type: string
   controller_ips:
     type: comma_delimited_list
+  service_ips:
+    type: json
   controller_names:
     type: comma_delimited_list
   rabbit_node_ips:
     type: comma_delimited_list
-  mongo_node_ips:
-    type: comma_delimited_list
-  redis_node_ips:
-    type: comma_delimited_list
   memcache_node_ips:
     type: comma_delimited_list
-  mysql_node_ips:
-    type: comma_delimited_list
-  horizon_node_ips:
-    type: comma_delimited_list
-  heat_api_node_ips:
-    type: comma_delimited_list
-  swift_proxy_node_ips:
-    type: comma_delimited_list
-  ceilometer_api_node_ips:
-    type: comma_delimited_list
-  aodh_api_node_ips:
-    type: comma_delimited_list
-  nova_api_node_ips:
-    type: comma_delimited_list
-  nova_metadata_node_ips:
-    type: comma_delimited_list
-  glance_api_node_ips:
-    type: comma_delimited_list
-  glance_registry_node_ips:
-    type: comma_delimited_list
-  gnocchi_api_node_ips:
-    type: comma_delimited_list
-  cinder_api_node_ips:
-    type: comma_delimited_list
-  manila_api_node_ips:
-    type: comma_delimited_list
-  neutron_api_node_ips:
-    type: comma_delimited_list
   keystone_public_api_node_ips:
     type: comma_delimited_list
   keystone_admin_api_node_ips:
     type: comma_delimited_list
-  sahara_api_node_ips:
-    type: comma_delimited_list
-  ironic_api_node_ips:
-    type: comma_delimited_list
   ceph_mon_node_ips:
     type: comma_delimited_list
   ceph_mon_node_names:
     type: comma_delimited_list
-  opendaylight_api_node_ips:
-    type: comma_delimited_list
   DeployIdentifier:
     type: string
     description: >
@@ -112,6 +76,8 @@ resources:
                         for_each:
                           SERVICE:
                             str_split: [',', {get_param: enabled_services}]
+                  # provides a mapping of service_name_ips to a list of IPs
+                  - {get_param: service_ips}
                   - controller_node_ips:
                       list_join:
                       - ','
@@ -132,30 +98,6 @@ resources:
                             list_join:
                             - "','"
                             - {get_param: rabbit_node_ips}
-                    mongodb_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: mongo_node_ips}
-                    redis_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: redis_node_ips}
-                    memcached_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: memcache_node_ips}
                     memcached_node_ips_v6:
                       str_replace:
                         template: "['inet6:[SERVERS_LIST]']"
@@ -164,127 +106,6 @@ resources:
                             list_join:
                             - "]','inet6:["
                             - {get_param: memcache_node_ips}
-                    mysql_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: mysql_node_ips}
-                    horizon_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: horizon_node_ips}
-                    heat_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: heat_api_node_ips}
-                    swift_proxy_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: swift_proxy_node_ips}
-                    ceilometer_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: ceilometer_api_node_ips}
-                    aodh_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: aodh_api_node_ips}
-                    gnocchi_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: gnocchi_api_node_ips}
-                    nova_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: nova_api_node_ips}
-                    nova_metadata_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: nova_metadata_node_ips}
-                    glance_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: glance_api_node_ips}
-                    glance_registry_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: glance_registry_node_ips}
-                    cinder_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: cinder_api_node_ips}
-                    manila_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: manila_api_node_ips}
-                    neutron_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: neutron_api_node_ips}
-                    # TODO: pass a `midonet_api_node_ips` var
-                    midonet_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: neutron_api_node_ips}
                     keystone_public_api_node_ips:
                       str_replace:
                         template: "['SERVERS_LIST']"
@@ -301,31 +122,6 @@ resources:
                             list_join:
                             - "','"
                             - {get_param: keystone_admin_api_node_ips}
-                    sahara_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: sahara_api_node_ips}
-                    ironic_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: ironic_api_node_ips}
-                    opendaylight_api_node_ips:
-                      str_replace:
-                        template: "['SERVERS_LIST']"
-                        params:
-                          SERVERS_LIST:
-                            list_join:
-                            - "','"
-                            - {get_param: opendaylight_api_node_ips}
-
                     tripleo::profile::base::ceph::ceph_mon_initial_members:
                       list_join:
                       - ','