From 787685101179f13f6074901070cf307bd9bb0731 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Mon, 10 Oct 2016 21:09:01 -0400 Subject: Hiera optimization: use a new hiera hook This patch optimizes how we deploy hiera by using a new heat hook specifically designed to help compose hiera within heat templates. As part of this change: - we update all the 'hiera' software configurations to set the group to hiera instead of os-apply-config. - The new format uses JSON instead of YAML. The hook actually writes out the hiera JSON directly so no conversion takes place. Arrays, Strings, Booleans all stay in their native formats. As such we can avoid having to do many of the awkward string and list conversions in t-h-t to support the previous YAML formatting. - The new hook prefers JSON over YAML so upgrading users will have the new files prefered. (we will post a cleanup routine for the old files soon but this isn't a new behavior, JSON is now simply prefered.) - A lot of services required edits to account for default settings that worked in YAML that no longer work correctly in the native JSON format. In almost all these cases I think the resulting codes looks cleaner and is more explicit with regards to what is getting configured in hiera on the actual nodes. Depends-On: I6a383b1ad4ec29458569763bd3f56fd3f2bd726b Closes-bug: #1596373 Change-Id: Ibe7e2044e200e2c947223286fdf4fd5bcf98c2e1 --- puppet/all-nodes-config.yaml | 244 +++++++++++++++++++++---------------------- 1 file changed, 118 insertions(+), 126 deletions(-) (limited to 'puppet/all-nodes-config.yaml') diff --git a/puppet/all-nodes-config.yaml b/puppet/all-nodes-config.yaml index cc5e4eac..0a8bec6e 100644 --- a/puppet/all-nodes-config.yaml +++ b/puppet/all-nodes-config.yaml @@ -71,140 +71,132 @@ resources: allNodesConfigImpl: type: OS::Heat::StructuredConfig properties: - group: os-apply-config + group: hiera config: - hiera: - datafiles: - bootstrap_node: - mapped_data: - bootstrap_nodeid: {get_input: bootstrap_nodeid} - bootstrap_nodeid_ip: {get_input: bootstrap_nodeid_ip} - all_nodes: - mapped_data: - map_merge: - - tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: logging_sources} - - tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: logging_groups} - - enabled_services: {get_param: enabled_services} - # This writes out a mapping of service_name_enabled: 'true' - # For any services not enabled, hiera foo_enabled will - # return nil, as it's undefined - - map_merge: - repeat: - template: - # Note this must be string 'true' due to - # https://bugs.launchpad.net/heat/+bug/1617203 - SERVICE_enabled: 'true' - for_each: - SERVICE: - str_split: [',', {get_param: enabled_services}] - # Dynamically generate per-service network data - # This works as follows (outer->inner functions) - # yaql - filters services where no mapping exists in ServiceNetMap - # map_replace: substitute e.g heat_api_network with network name from ServiceNetMap - # map_merge/repeat: generate a per-service mapping - - yaql: - # This filters any entries where the value hasn't been substituted for - # a list, e.g it's still $service_network. This happens when there is - # no network defined for the service in the ServiceNetMap, which is OK - # as not all services have to be bound to a network, so we filter them - expression: dict($.data.map.items().where(isString($[1]) and not $[1].endsWith("_network"))) - data: - map: - map_replace: + datafiles: + bootstrap_node: + bootstrap_nodeid: {get_input: bootstrap_nodeid} + bootstrap_nodeid_ip: {get_input: bootstrap_nodeid_ip} + all_nodes: + map_merge: + - tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: logging_sources} + - tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: logging_groups} + - enabled_services: {get_param: enabled_services} + # This writes out a mapping of service_name_enabled: 'true' + # For any services not enabled, hiera foo_enabled will + # return nil, as it's undefined + - map_merge: + repeat: + template: + # Note this must be string 'true' due to + # https://bugs.launchpad.net/heat/+bug/1617203 + SERVICE_enabled: 'true' + for_each: + SERVICE: + str_split: [',', {get_param: enabled_services}] + # Dynamically generate per-service network data + # This works as follows (outer->inner functions) + # yaql - filters services where no mapping exists in ServiceNetMap + # map_replace: substitute e.g heat_api_network with network name from ServiceNetMap + # map_merge/repeat: generate a per-service mapping + - yaql: + # This filters any entries where the value hasn't been substituted for + # a list, e.g it's still $service_network. This happens when there is + # no network defined for the service in the ServiceNetMap, which is OK + # as not all services have to be bound to a network, so we filter them + expression: dict($.data.map.items().where(isString($[1]) and not $[1].endsWith("_network"))) + data: + map: + map_replace: + - map_merge: + repeat: + template: + SERVICE_network: SERVICE_network + for_each: + SERVICE: + str_split: [',', {get_param: enabled_services}] + - values: {get_param: ServiceNetMap} + # Keystone doesn't provide separate entries for the public + # and admin endpoints, so we need to add them here manually + # like we do in the vip-config below + - keystone_admin_api_network: {get_param: [ServiceNetMap, keystone_admin_api_network]} + keystone_public_api_network: {get_param: [ServiceNetMap, keystone_public_api_network]} + # provides a mapping of service_name_ips to a list of IPs + - {get_param: service_ips} + - {get_param: service_node_names} + - {get_param: short_service_node_names} + - controller_node_ips: + list_join: + - ',' + - {get_param: controller_ips} + controller_node_names: + list_join: + - ',' + - {get_param: controller_names} + memcached_node_ips_v6: + repeat: + template: "inet6:[NAME]" + for_each: + NAME: {get_param: memcache_node_ips} + deploy_identifier: {get_param: DeployIdentifier} + update_identifier: {get_param: UpdateIdentifier} + stack_action: {get_param: StackAction} + vip_data: + map_merge: + # Dynamically generate per-service VIP data based on enabled_services + # This works as follows (outer->inner functions) + # yaql - filters services where no mapping exists in ServiceNetMap + # map_replace: substitute e.g internal_api with the IP from NetVipMap + # map_replace: substitute e.g heat_api_network with network name from ServiceNetMap + # map_merge/repeat: generate a per-service mapping + - yaql: + # This filters any entries where the value hasn't been substituted for + # a list, e.g it's still $service_network. This happens when there is + # no network defined for the service in the ServiceNetMap, which is OK + # as not all services have to be bound to a network, so we filter them + expression: dict($.data.map.items().where(isString($[1]) and not $[1].endsWith("_network"))) + data: + map: + map_replace: + - map_replace: - map_merge: repeat: template: - SERVICE_network: SERVICE_network + SERVICE_vip: SERVICE_network for_each: SERVICE: str_split: [',', {get_param: enabled_services}] - values: {get_param: ServiceNetMap} - # Keystone doesn't provide separate entries for the public - # and admin endpoints, so we need to add them here manually - # like we do in the vip-config below - - keystone_admin_api_network: {get_param: [ServiceNetMap, keystone_admin_api_network]} - keystone_public_api_network: {get_param: [ServiceNetMap, keystone_public_api_network]} - # provides a mapping of service_name_ips to a list of IPs - - {get_param: service_ips} - - {get_param: service_node_names} - - {get_param: short_service_node_names} - - controller_node_ips: - list_join: - - ',' - - {get_param: controller_ips} - controller_node_names: - list_join: - - ',' - - {get_param: controller_names} - memcached_node_ips_v6: - str_replace: - template: "['inet6:[SERVERS_LIST]']" - params: - SERVERS_LIST: - list_join: - - "]','inet6:[" - - {get_param: memcache_node_ips} - - deploy_identifier: {get_param: DeployIdentifier} - update_identifier: {get_param: UpdateIdentifier} - stack_action: {get_param: StackAction} - vip_data: - mapped_data: - map_merge: - # Dynamically generate per-service VIP data based on enabled_services - # This works as follows (outer->inner functions) - # yaql - filters services where no mapping exists in ServiceNetMap - # map_replace: substitute e.g internal_api with the IP from NetVipMap - # map_replace: substitute e.g heat_api_network with network name from ServiceNetMap - # map_merge/repeat: generate a per-service mapping - - yaql: - # This filters any entries where the value hasn't been substituted for - # a list, e.g it's still $service_network. This happens when there is - # no network defined for the service in the ServiceNetMap, which is OK - # as not all services have to be bound to a network, so we filter them - expression: dict($.data.map.items().where(isString($[1]) and not $[1].endsWith("_network"))) - data: - map: - map_replace: - - map_replace: - - map_merge: - repeat: - template: - SERVICE_vip: SERVICE_network - for_each: - SERVICE: - str_split: [',', {get_param: enabled_services}] - - values: {get_param: ServiceNetMap} - - values: {get_param: NetVipMap} - - keystone_admin_api_vip: - get_param: [NetVipMap, {get_param: [ServiceNetMap, keystone_admin_api_network]}] - keystone_public_api_vip: - get_param: [NetVipMap, {get_param: [ServiceNetMap, keystone_public_api_network]}] - public_virtual_ip: {get_param: [NetVipMap, external]} - controller_virtual_ip: {get_param: [NetVipMap, ctlplane]} - internal_api_virtual_ip: {get_param: [NetVipMap, internal_api]} - storage_virtual_ip: {get_param: [NetVipMap, storage]} - storage_mgmt_virtual_ip: {get_param: [NetVipMap, storage_mgmt]} - redis_vip: {get_param: RedisVirtualIP} - # public_virtual_ip and controller_virtual_ip are needed in - # both HAproxy & keepalived. - tripleo::haproxy::public_virtual_ip: {get_param: [NetVipMap, external]} - tripleo::haproxy::controller_virtual_ip: {get_param: [NetVipMap, ctlplane]} - tripleo::keepalived::public_virtual_ip: {get_param: [NetVipMap, external]} - tripleo::keepalived::controller_virtual_ip: {get_param: [NetVipMap, ctlplane]} - tripleo::keepalived::internal_api_virtual_ip: {get_param: [NetVipMap, internal_api]} - tripleo::keepalived::storage_virtual_ip: {get_param: [NetVipMap, storage]} - tripleo::keepalived::storage_mgmt_virtual_ip: {get_param: [NetVipMap, storage_mgmt]} - tripleo::keepalived::redis_virtual_ip: {get_param: RedisVirtualIP} - tripleo::redis_notification::haproxy_monitor_ip: {get_param: [NetVipMap, ctlplane]} - cloud_name_external: {get_param: cloud_name_external} - cloud_name_internal_api: {get_param: cloud_name_internal_api} - cloud_name_storage: {get_param: cloud_name_storage} - cloud_name_storage_mgmt: {get_param: cloud_name_storage_mgmt} - cloud_name_ctlplane: {get_param: cloud_name_ctlplane} - # TLS parameters - certmonger_ca: {get_param: CertmongerCA} - enable_internal_tls: {get_param: EnableInternalTLS} + - values: {get_param: NetVipMap} + - keystone_admin_api_vip: + get_param: [NetVipMap, {get_param: [ServiceNetMap, keystone_admin_api_network]}] + keystone_public_api_vip: + get_param: [NetVipMap, {get_param: [ServiceNetMap, keystone_public_api_network]}] + public_virtual_ip: {get_param: [NetVipMap, external]} + controller_virtual_ip: {get_param: [NetVipMap, ctlplane]} + internal_api_virtual_ip: {get_param: [NetVipMap, internal_api]} + storage_virtual_ip: {get_param: [NetVipMap, storage]} + storage_mgmt_virtual_ip: {get_param: [NetVipMap, storage_mgmt]} + redis_vip: {get_param: RedisVirtualIP} + # public_virtual_ip and controller_virtual_ip are needed in + # both HAproxy & keepalived. + tripleo::haproxy::public_virtual_ip: {get_param: [NetVipMap, external]} + tripleo::haproxy::controller_virtual_ip: {get_param: [NetVipMap, ctlplane]} + tripleo::keepalived::public_virtual_ip: {get_param: [NetVipMap, external]} + tripleo::keepalived::controller_virtual_ip: {get_param: [NetVipMap, ctlplane]} + tripleo::keepalived::internal_api_virtual_ip: {get_param: [NetVipMap, internal_api]} + tripleo::keepalived::storage_virtual_ip: {get_param: [NetVipMap, storage]} + tripleo::keepalived::storage_mgmt_virtual_ip: {get_param: [NetVipMap, storage_mgmt]} + tripleo::keepalived::redis_virtual_ip: {get_param: RedisVirtualIP} + tripleo::redis_notification::haproxy_monitor_ip: {get_param: [NetVipMap, ctlplane]} + cloud_name_external: {get_param: cloud_name_external} + cloud_name_internal_api: {get_param: cloud_name_internal_api} + cloud_name_storage: {get_param: cloud_name_storage} + cloud_name_storage_mgmt: {get_param: cloud_name_storage_mgmt} + cloud_name_ctlplane: {get_param: cloud_name_ctlplane} + # TLS parameters + certmonger_ca: {get_param: CertmongerCA} + enable_internal_tls: {get_param: EnableInternalTLS} outputs: config_id: -- cgit 1.2.3-korg