puppet controller role: per service VIP settings

This patch refactors the puppet controller role so that it
makes use of per service VIP settings for each service.

Previously the VIP for the ctlplane was hard wired to
many of the controller service. With this patch we have
the ability to isolate traffic for services which
made use of the ctlplane and public VIPs for their
settings.

The implementation includes:

 * stops the use of the VirtualIP and PublicVirtualIP within the
   controller role. These parameters have now been replaced with
   per service heat parameters for the controller nested stack which
   are determined via VipMap based on per service settings in the heat
   environment.

 * All VIP configuration is now moved into puppet/vip-config.yaml.
   This made sense so we could deprecate the use of the VirtualIP
   and PublicVirtualIP settings above.

 * The puppet manifests for the controller were cleaned up for several
   to use Hiera directly instead of constructing URLs based on the
   static controller and public network VIPs. This improvement
   was something we wanted to do anyways and made the implementation
   cleaner.

Change-Id: I9b9a15be67f74bec97366408f7047acfd6ea0ec6

6 files changed, 83 insertions, 57 deletions

diff --git a/controller.yaml b/controller.yaml
index 4e541e92..8c1b93d2 100644
--- a/controller.yaml
+++ b/controller.yaml
@@ -418,6 +418,21 @@ parameters:
   VirtualIP:
     type: string
     default: ''  # Has to be here because of the ignored empty value bug
+  HeatApiVirtualIP:
+    type: string
+    default: ''
+  GlanceApiVirtualIP:
+    type: string
+    default: ''
+  MysqlVirtualIP:
+    type: string
+    default: ''
+  KeystonePublicApiVirtualIP:
+    type: string
+    default: ''
+  NeutronApiVirtualIP:
+    type: string
+    default: ''
   ServiceNetMap:
     default: {}
     description: Mapping of service_name -> network name. Typically set
diff --git a/overcloud-without-mergepy.yaml b/overcloud-without-mergepy.yaml
index 4b969dfc..4b2e0b90 100644
--- a/overcloud-without-mergepy.yaml
+++ b/overcloud-without-mergepy.yaml
@@ -662,9 +662,14 @@ resources:
           SwiftPartPower: {get_param: SwiftPartPower}
           SwiftPassword: {get_param: SwiftPassword}
           SwiftReplicas: { get_param: SwiftReplicas}
-          VirtualIP: {get_attr: [ControlVirtualIP, fixed_ips, 0, ip_address]}
-          PublicVirtualIP: {get_attr: [PublicVirtualIP, fixed_ips, 0, ip_address]}
+          VirtualIP: {get_attr: [ControlVirtualIP, fixed_ips, 0, ip_address]} # deprecated. Use per service VIP settings instead now.
+          PublicVirtualIP: {get_attr: [PublicVirtualIP, fixed_ips, 0, ip_address]} # deprecated. Use per service VIP settings instead now.
           ServiceNetMap: {get_param: ServiceNetMap}
+          HeatApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, HeatApiNetwork]}]}
+          GlanceApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, GlanceApiNetwork]}]}
+          MysqlVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
+          KeystonePublicApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, KeystonePublicApiNetwork]}]}
+          NeutronApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NeutronApiNetwork]}]}
 
   Compute:
     type: OS::Heat::ResourceGroup
@@ -910,6 +915,8 @@ resources:
         mysql_vip: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
         rabbit_vip: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, RabbitMqNetwork]}]}
         # direct configuration of Virtual IPs for each network
+        control_virtual_ip: {get_attr: [ControlVirtualIP, fixed_ips, 0, ip_address]}
+        public_virtual_ip: {get_attr: [PublicVirtualIP, fixed_ips, 0, ip_address]}
         internal_api_virtual_ip: {get_attr: [InternalApiVirtualIP, ip_address]}
         storage_virtual_ip: {get_attr: [StorageVirtualIP, ip_address]}
         storage_mgmt_virtual_ip: {get_attr: [StorageMgmtVirtualIP, ip_address]}
diff --git a/puppet/controller-puppet.yaml b/puppet/controller-puppet.yaml
index 0da10fc8..aa33cbfc 100644
--- a/puppet/controller-puppet.yaml
+++ b/puppet/controller-puppet.yaml
@@ -342,7 +342,7 @@ parameters:
         Specifies the interface where the public-facing virtual ip will be assigned.
         This should be int_public when a VLAN is being used.
     type: string
-  PublicVirtualIP:
+  PublicVirtualIP: # DEPRECATED: use per service settings instead
     type: string
     default: ''  # Has to be here because of the ignored empty value bug
   RabbitCookie:
@@ -419,9 +419,24 @@ parameters:
     type: number
     default: 3
     description: How many replicas to use in the swift rings.
-  VirtualIP:
+  VirtualIP: # DEPRECATED: use per service settings instead
     type: string
     default: ''  # Has to be here because of the ignored empty value bug
+  HeatApiVirtualIP:
+    type: string
+    default: ''
+  GlanceApiVirtualIP:
+    type: string
+    default: ''
+  MysqlVirtualIP:
+    type: string
+    default: ''
+  KeystonePublicApiVirtualIP:
+    type: string
+    default: ''
+  NeutronApiVirtualIP:
+    type: string
+    default: ''
   EnablePackageInstall:
     default: 'false'
     description: Set to true to enable package installation via Puppet
@@ -510,25 +525,24 @@ resources:
       server: {get_resource: Controller}
       input_values:
         bootstack_nodeid: {get_attr: [Controller, name]}
-        controller_virtual_ip: {get_param: VirtualIP}
         neutron_enable_tunneling: {get_param: NeutronEnableTunnelling}
         heat.watch_server_url:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: HeatApiVirtualIP}
               - ':8003'
         heat.metadata_server_url:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: HeatApiVirtualIP}
               - ':8000'
         heat.waitcondition_server_url:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: HeatApiVirtualIP}
               - ':8000/v1/waitcondition'
         heat_auth_encryption_key: {get_param: HeatAuthEncryptionKey}
         horizon_secret: {get_param: HorizonSecret}
@@ -547,13 +561,11 @@ resources:
             - - 'mysql://cinder:'
               - {get_param: CinderPassword}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/cinder'
         glance_port: {get_param: GlancePort}
-        glance_protocol: {get_param: GlanceProtocol}
         glance_password: {get_param: GlancePassword}
         glance_backend: {get_param: GlanceBackend}
-        glance_swift_store_auth_address: {list_join: ['', ['http://', {get_param: VirtualIP} , ':5000/v2.0']]}
         glance_notifier_strategy: {get_param: GlanceNotifierStrategy}
         glance_log_file: {get_param: GlanceLogFile}
         glance_dsn:
@@ -562,7 +574,7 @@ resources:
             - - 'mysql://glance:'
               - {get_param: GlancePassword}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/glance'
         heat_password: {get_param: HeatPassword}
         heat_stack_domain_admin_password: {get_param: HeatStackDomainAdminPassword}
@@ -572,8 +584,9 @@ resources:
             - - 'mysql://heat:'
               - {get_param: HeatPassword}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/heat'
+        keystone_auth_address: {list_join: ['', ['http://', {get_param: KeystonePublicApiVirtualIP} , ':5000/v2.0']]}
         keystone_ca_certificate: {get_param: KeystoneCACertificate}
         keystone_signing_key: {get_param: KeystoneSigningKey}
         keystone_signing_certificate: {get_param: KeystoneSigningCertificate}
@@ -585,19 +598,19 @@ resources:
             - - 'mysql://keystone:'
               - {get_param: AdminToken}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/keystone'
         keystone_identity_uri:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: KeystonePublicApiVirtualIP}
               - ':35357/'
         keystone_auth_uri:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: KeystonePublicApiVirtualIP}
               - ':5000/v2.0/'
         enable_galera: {get_param: EnableGalera}
         enable_ceph_storage: {get_param: EnableCephStorage}
@@ -632,19 +645,19 @@ resources:
             - - 'mysql://neutron:'
               - {get_param: NeutronPassword}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/ovs_neutron?charset=utf8'
         neutron_url:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: NeutronApiVirtualIP}
               - ':9696'
         neutron_admin_auth_url:
           list_join:
             - ''
             - - 'http://'
-              - {get_param: VirtualIP}
+              - {get_param: KeystonePublicApiVirtualIP}
               - ':35357/v2.0'
         ceilometer_backend: {get_param: CeilometerBackend}
         ceilometer_metering_secret: {get_param: CeilometerMeteringSecret}
@@ -653,7 +666,7 @@ resources:
           list_join:
             - ''
             - - 'mysql://ceilometer:unset@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/ceilometer'
         snmpd_readonly_user_name: {get_param: SnmpdReadonlyUserName}
         snmpd_readonly_user_password: {get_param: SnmpdReadonlyUserPassword}
@@ -664,7 +677,7 @@ resources:
             - - 'mysql://nova:'
               - {get_param: NovaPassword}
               - '@'
-              - {get_param: VirtualIP}
+              - {get_param: MysqlVirtualIP}
               - '/nova'
         pcsd_password: {get_param: PcsdPassword}
         rabbit_username: {get_param: RabbitUserName}
@@ -679,7 +692,6 @@ resources:
               server: {get_param: NtpServer}
         control_virtual_interface: {get_param: ControlVirtualInterface}
         public_virtual_interface: {get_param: PublicVirtualInterface}
-        public_virtual_ip: {get_param: PublicVirtualIP}
         swift_hash_suffix: {get_param: SwiftHashSuffix}
         swift_password: {get_param: SwiftPassword}
         swift_part_power: {get_param: SwiftPartPower}
@@ -693,6 +705,14 @@ resources:
         cinder_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, CinderApiNetwork]}]}
         glance_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, GlanceApiNetwork]}]}
         glance_registry_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, GlanceRegistryNetwork]}]}
+        glance_api_servers:
+          list_join:
+            - ''
+            - - {get_param: GlanceProtocol}
+              - '://'
+              - {get_param: GlanceApiVirtualIP}
+              - ':'
+              - {get_param: GlancePort}
         heat_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, HeatApiNetwork]}]}
         keystone_public_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, KeystonePublicApiNetwork]}]}
         keystone_admin_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, KeystoneAdminApiNetwork]}]}
@@ -724,6 +744,7 @@ resources:
             - ceph
             - bootstrap_node # provided by BootstrapNodeConfig
             - all_nodes # provided by allNodesConfig
+            - vip_data # provided by vip-config
             - '"%{::osfamily}"'
             - common
           datafiles:
@@ -782,9 +803,6 @@ resources:
                 glance::api::registry_host: {get_input: glance_registry_network}
                 glance::api::keystone_password: {get_input: glance_password}
                 glance::api::debug: {get_input: debug}
-                # used to construct glance_api_servers
-                glance_port: {get_input: glance_port}
-                glance_protocol: {get_input: glance_protocol}
                 glance_notifier_strategy: {get_input: glance_notifier_strategy}
                 glance_log_file: {get_input: glance_log_file}
                 glance_log_file: {get_input: glance_log_file}
@@ -795,7 +813,7 @@ resources:
                 glance::registry::auth_uri: {get_input: keystone_auth_uri}
                 glance::registry::identity_uri: {get_input: keystone_identity_uri}
                 glance::registry::debug: {get_input: debug}
-                glance::backend::swift::swift_store_auth_address: {get_input: glance_swift_store_auth_address}
+                glance::backend::swift::swift_store_auth_address: {get_input: keystone_auth_address}
                 glance::backend::swift::swift_store_user: service:glance
                 glance::backend::swift::swift_store_key: {get_input: glance_password}
                 glance_backend: {get_input: glance_backend}
@@ -857,7 +875,7 @@ resources:
                 neutron::agents::ml2::ovs::local_ip: {get_input: neutron_local_ip}
                 neutron_flat_networks: {get_input: neutron_flat_networks}
                 neutron::agents::metadata::shared_secret: {get_input: neutron_metadata_proxy_shared_secret}
-                neutron::agents::metadata::metadata_ip: {get_input: controller_virtual_ip}
+                neutron::agents::metadata::metadata_ip: {get_input: neutron_api_network}
                 neutron_agent_mode: {get_input: neutron_agent_mode}
                 neutron_router_distributed: {get_input: neutron_router_distributed}
                 neutron_mechanism_drivers: {get_input: neutron_mechanism_drivers}
@@ -875,6 +893,7 @@ resources:
                 neutron::agents::metadata::auth_password: {get_input: neutron_password}
                 neutron_dnsmasq_options: {get_input: neutron_dnsmasq_options}
                 neutron_dsn: {get_input: neutron_dsn}
+                neutron::agents::metadata::auth_url: {get_input: keystone_identity_uri}
 
                 # Ceilometer
                 ceilometer_backend: {get_input: ceilometer_backend}
@@ -890,6 +909,7 @@ resources:
                 ceilometer::api::keystone_auth_uri: {get_input: keystone_auth_uri}
                 ceilometer::api::keystone_identity_uri: {get_input: keystone_identity_uri}
                 ceilometer::agent::auth::auth_password: {get_input: ceilometer_password}
+                ceilometer::agent::auth::auth_url: {get_input: keystone_auth_address}
                 snmpd_readonly_user_name: {get_input: snmpd_readonly_user_name}
                 snmpd_readonly_user_password: {get_input: snmpd_readonly_user_password}
 
@@ -905,6 +925,7 @@ resources:
                 nova::api::metadata_listen: {get_input: nova_metadata_network}
                 nova::api::admin_password: {get_input: nova_password}
                 nova::database_connection: {get_input: nova_dsn}
+                nova::glance_api_servers: {get_input: glance_api_servers}
                 nova::api::neutron_metadata_proxy_shared_secret: {get_input: neutron_metadata_proxy_shared_secret}
                 nova::network::neutron::neutron_admin_password: {get_input: neutron_password}
                 nova::network::neutron::neutron_url: {get_input: neutron_url}
@@ -926,13 +947,9 @@ resources:
                 neutron_public_interface_ip: {get_input: neutron_public_interface_ip}
                 ntp::servers: {get_input: ntp_servers}
                 control_virtual_interface: {get_input: control_virtual_interface}
-                controller_virtual_ip: {get_input: controller_virtual_ip}
                 public_virtual_interface: {get_input: public_virtual_interface}
-                public_virtual_ip: {get_input: public_virtual_ip}
                 tripleo::loadbalancer::control_virtual_interface: {get_input: control_virtual_interface}
-                tripleo::loadbalancer::controller_virtual_ip: {get_input: controller_virtual_ip}
                 tripleo::loadbalancer::public_virtual_interface: {get_input: public_virtual_interface}
-                tripleo::loadbalancer::public_virtual_ip: {get_input: public_virtual_ip}
                 enable_package_install: {get_input: enable_package_install}
 
 outputs:
@@ -984,11 +1001,11 @@ outputs:
       str_replace:
         template: 'r1z1-IP:%PORT%/d1'
         params:
-          IP: {get_attr: [Controller, networks, ctlplane, 0]}
+          IP: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftMgmtNetwork]}]}
   swift_proxy_memcache:
     description: Swift proxy-memcache value
     value:
       str_replace:
         template: "IP:11211"
         params:
-          IP: {get_attr: [Controller, networks, ctlplane, 0]}
+          IP: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MemcachedNetwork]}]}
diff --git a/puppet/manifests/overcloud_controller.pp b/puppet/manifests/overcloud_controller.pp
index 9316ae14..7070e05c 100644
--- a/puppet/manifests/overcloud_controller.pp
+++ b/puppet/manifests/overcloud_controller.pp
@@ -76,9 +76,7 @@ if hiera('step') >= 2 {
   if count($redis_node_ips) > 1 {
     Class['::tripleo::redis_notification'] -> Service['redis-sentinel']
     include ::redis::sentinel
-    class {'::tripleo::redis_notification' :
-      haproxy_monitor_ip => hiera('tripleo::loadbalancer::controller_virtual_ip'),
-    }
+    include ::tripleo::redis_notification
   }
 
   if str2bool(hiera('enable_galera', 'true')) {
@@ -253,10 +251,7 @@ if hiera('step') >= 3 {
   include ::glance::registry
   include join(['::glance::backend::', $glance_backend])
 
-  class { 'nova':
-    glance_api_servers     => join([hiera('glance_protocol'), '://', hiera('controller_virtual_ip'), ':', hiera('glance_port')]),
-  }
-
+  include ::nova
   include ::nova::api
   include ::nova::cert
   include ::nova::conductor
@@ -269,9 +264,7 @@ if hiera('step') >= 3 {
   include ::neutron::server
   include ::neutron::agents::l3
   include ::neutron::agents::dhcp
-  class { 'neutron::agents::metadata':
-    auth_url => join(['http://', hiera('controller_virtual_ip'), ':35357/v2.0']),
-  }
+  include ::neutron::agents::metadata
 
   file { '/etc/neutron/dnsmasq-neutron.conf':
     content => hiera('neutron_dnsmasq_options'),
@@ -398,12 +391,10 @@ if hiera('step') >= 3 {
   include ::ceilometer::alarm::evaluator
   include ::ceilometer::expirer
   include ::ceilometer::collector
+  include ceilometer::agent::auth
   class { '::ceilometer::db' :
     database_connection => $ceilometer_database_connection,
   }
-  class { 'ceilometer::agent::auth':
-    auth_url => join(['http://', hiera('controller_virtual_ip'), ':5000/v2.0']),
-  }
 
   Cron <| title == 'ceilometer-expirer' |> { command => "sleep $((\$(od -A n -t d -N 3 /dev/urandom) % 86400)) && ${::ceilometer::params::expirer_command}" }
 
diff --git a/puppet/manifests/overcloud_controller_pacemaker.pp b/puppet/manifests/overcloud_controller_pacemaker.pp
index e30cda82..05ea8752 100644
--- a/puppet/manifests/overcloud_controller_pacemaker.pp
+++ b/puppet/manifests/overcloud_controller_pacemaker.pp
@@ -237,9 +237,7 @@ if hiera('step') >= 2 {
   if count($redis_node_ips) > 1 {
     Class['::tripleo::redis_notification'] -> Service['redis-sentinel']
     include ::redis::sentinel
-    class {'::tripleo::redis_notification' :
-      haproxy_monitor_ip => hiera('tripleo::loadbalancer::controller_virtual_ip'),
-    }
+    include ::tripleo::redis_notification
   }
 
   exec { 'galera-ready' :
@@ -426,9 +424,7 @@ if hiera('step') >= 3 {
   }
   include join(['::glance::backend::', $glance_backend])
 
-  class { 'nova':
-    glance_api_servers     => join([hiera('glance_protocol'), '://', hiera('controller_virtual_ip'), ':', hiera('glance_port')]),
-  }
+  include ::nova
 
   class { '::nova::api' :
     sync_db => $sync_db,
@@ -473,7 +469,6 @@ if hiera('step') >= 3 {
     enabled => false,
   }
   class { 'neutron::agents::metadata':
-    auth_url => join(['http://', hiera('controller_virtual_ip'), ':35357/v2.0']),
     manage_service => false,
     enabled => false,
   }
@@ -496,7 +491,6 @@ if hiera('step') >= 3 {
     tunnel_types     => split(hiera('neutron_tunnel_types'), ','),
   }
 
-
   include ::cinder
   class { '::cinder::api':
     sync_db => $sync_db,
@@ -645,9 +639,7 @@ if hiera('step') >= 3 {
     database_connection => $ceilometer_database_connection,
     sync_db             => $sync_db,
   }
-  class { 'ceilometer::agent::auth':
-    auth_url => join(['http://', hiera('controller_virtual_ip'), ':5000/v2.0']),
-  }
+  include ceilometer::agent::auth
 
   Cron <| title == 'ceilometer-expirer' |> { command => "sleep $((\$(od -A n -t d -N 3 /dev/urandom) % 86400)) && ${::ceilometer::params::expirer_command}" }
 
diff --git a/puppet/vip-config.yaml b/puppet/vip-config.yaml
index a84def5d..c083821e 100644
--- a/puppet/vip-config.yaml
+++ b/puppet/vip-config.yaml
@@ -28,9 +28,13 @@ resources:
                 redis_vip: {get_input: redis_vip}
                 mysql_vip: {get_input: mysql_vip}
                 mysql_vip: {get_input: mysql_vip}
+                tripleo::loadbalancer::public_virtual_ip: {get_input: public_virtual_ip}
+                tripleo::loadbalancer::controller_virtual_ip: {get_input: control_virtual_ip}
                 tripleo::loadbalancer::internal_api_virtual_ip: {get_input: internal_api_virtual_ip}
                 tripleo::loadbalancer::storage_virtual_ip: {get_input: storage_virtual_ip}
                 tripleo::loadbalancer::storage_mgmt_virtual_ip: {get_input: storage_mgmt_virtual_ip}
+                tripleo::redis_notification::haproxy_monitor_ip: {get_input: control_virtual_ip}
+
 
 outputs:
   OS::stack_id: