From 316cc2572d0a255d466fb7f9aa74de3b8ccb6831 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Thu, 8 Jun 2017 19:07:51 +0100 Subject: Remove duplicate docker/puppet services.yaml Move to one common services.yaml not only reduces the duplication, but it should improve performance for the docker/services.yaml case, because we were creating two ResourceChains with $many services which we know can be really slow (especially since we seem to be missing concurrent: true on one) Change-Id: I76f188438bfc6449b152c2861d99738e6eb3c61b --- common/README | 1 + docker/services/services.yaml | 105 ----------------- environments/docker-services-tls-everywhere.yaml | 2 - environments/docker.yaml | 3 - overcloud-resource-registry-puppet.j2.yaml | 2 +- puppet/services/services.yaml | 129 -------------------- services.yaml | 144 +++++++++++++++++++++++ tools/yaml-validate.py | 6 +- 8 files changed, 148 insertions(+), 244 deletions(-) create mode 100644 common/README delete mode 100644 docker/services/services.yaml delete mode 100644 puppet/services/services.yaml create mode 100644 services.yaml diff --git a/common/README b/common/README new file mode 100644 index 00000000..6a523118 --- /dev/null +++ b/common/README @@ -0,0 +1 @@ +This will contain some common templates but it needs to be added to the RPM spec first diff --git a/docker/services/services.yaml b/docker/services/services.yaml deleted file mode 100644 index 2ad3b63d..00000000 --- a/docker/services/services.yaml +++ /dev/null @@ -1,105 +0,0 @@ -heat_template_version: pike - -description: > - Utility stack to convert an array of services into a set of combined - role configs. - -parameters: - Services: - default: [] - description: | - List nested stack service templates. - type: comma_delimited_list - ServiceNetMap: - default: {} - description: Mapping of service_name -> network name. Typically set - via parameter_defaults in the resource registry. This - mapping overrides those in ServiceNetMapDefaults. - type: json - EndpointMap: - default: {} - description: Mapping of service endpoint -> protocol. Typically set - via parameter_defaults in the resource registry. - type: json - DefaultPasswords: - default: {} - description: Mapping of service -> default password. Used to help - pass top level passwords managed by Heat into services. - type: json - RoleName: - default: '' - description: Role name on which the service is applied - type: string - RoleParameters: - default: {} - description: Parameters specific to the role - type: json - -resources: - - PuppetServices: - type: ../../puppet/services/services.yaml - properties: - Services: {get_param: Services} - ServiceNetMap: {get_param: ServiceNetMap} - EndpointMap: {get_param: EndpointMap} - DefaultPasswords: {get_param: DefaultPasswords} - RoleName: {get_param: RoleName} - RoleParameters: {get_param: RoleParameters} - - ServiceChain: - type: OS::Heat::ResourceChain - properties: - resources: {get_param: Services} - concurrent: true - resource_properties: - ServiceNetMap: {get_param: ServiceNetMap} - EndpointMap: {get_param: EndpointMap} - DefaultPasswords: {get_param: DefaultPasswords} - RoleName: {get_param: RoleName} - RoleParameters: {get_param: RoleParameters} - -outputs: - role_data: - description: Combined Role data for this set of services. - value: - service_names: - {get_attr: [PuppetServices, role_data, service_names]} - monitoring_subscriptions: - {get_attr: [PuppetServices, role_data, monitoring_subscriptions]} - logging_sources: - {get_attr: [PuppetServices, role_data, logging_sources]} - logging_groups: - {get_attr: [PuppetServices, role_data, logging_groups]} - service_config_settings: - {get_attr: [PuppetServices, role_data, service_config_settings]} - config_settings: - {get_attr: [PuppetServices, role_data, config_settings]} - global_config_settings: - {get_attr: [PuppetServices, role_data, global_config_settings]} - step_config: - {get_attr: [ServiceChain, role_data, step_config]} - puppet_config: {get_attr: [ServiceChain, role_data, puppet_config]} - kolla_config: - map_merge: {get_attr: [ServiceChain, role_data, kolla_config]} - docker_config: - {get_attr: [ServiceChain, role_data, docker_config]} - docker_puppet_tasks: - {get_attr: [ServiceChain, role_data, docker_puppet_tasks]} - host_prep_tasks: - yaql: - # Note we use distinct() here to filter any identical tasks - expression: $.data.where($ != null).select($.get('host_prep_tasks')).where($ != null).flatten().distinct() - data: {get_attr: [ServiceChain, role_data]} - upgrade_tasks: - yaql: - # Note we use distinct() here to filter any identical tasks, e.g yum update for all services - expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct() - data: {get_attr: [ServiceChain, role_data]} - upgrade_batch_tasks: - yaql: - # Note we use distinct() here to filter any identical tasks, e.g yum update for all services - expression: $.data.where($ != null).select($.get('upgrade_batch_tasks')).where($ != null).flatten().distinct() - data: {get_attr: [ServiceChain, role_data]} - service_metadata_settings: - get_attr: [PuppetServices, role_data, service_metadata_settings] diff --git a/environments/docker-services-tls-everywhere.yaml b/environments/docker-services-tls-everywhere.yaml index 2740664c..f1e3d96b 100644 --- a/environments/docker-services-tls-everywhere.yaml +++ b/environments/docker-services-tls-everywhere.yaml @@ -35,8 +35,6 @@ resource_registry: OS::TripleO::PostDeploySteps: ../docker/post.yaml OS::TripleO::PostUpgradeSteps: ../docker/post-upgrade.yaml - OS::TripleO::Services: ../docker/services/services.yaml - parameter_defaults: # Defaults to 'tripleoupstream'. Specify a local docker registry # Example: 192.168.24.1:8787/tripleoupstream diff --git a/environments/docker.yaml b/environments/docker.yaml index 28527945..bb1ebdc3 100644 --- a/environments/docker.yaml +++ b/environments/docker.yaml @@ -2,7 +2,6 @@ resource_registry: # This can be used when you don't want to run puppet on the host, # e.g atomic, but it has been replaced with OS::TripleO::Services::Docker # OS::TripleO::NodeUserData: ../docker/firstboot/setup_docker_host.yaml - OS::TripleO::Services::Docker: ../puppet/services/docker.yaml # The compute node still needs extra initialization steps OS::TripleO::Compute::NodeUserData: ../docker/firstboot/setup_docker_host.yaml @@ -51,8 +50,6 @@ resource_registry: OS::TripleO::PostDeploySteps: ../docker/post.yaml OS::TripleO::PostUpgradeSteps: ../docker/post-upgrade.yaml - OS::TripleO::Services: ../docker/services/services.yaml - parameter_defaults: # To specify a local docker registry, enable these # where 192.168.24.1 is the host running docker-distribution diff --git a/overcloud-resource-registry-puppet.j2.yaml b/overcloud-resource-registry-puppet.j2.yaml index d4c301bb..b2660527 100644 --- a/overcloud-resource-registry-puppet.j2.yaml +++ b/overcloud-resource-registry-puppet.j2.yaml @@ -106,7 +106,7 @@ resource_registry: OS::TripleO::UpgradeConfig: puppet/upgrade_config.yaml # services - OS::TripleO::Services: puppet/services/services.yaml + OS::TripleO::Services: services.yaml OS::TripleO::Services::Apache: puppet/services/apache.yaml OS::TripleO::Services::CACerts: puppet/services/ca-certs.yaml OS::TripleO::Services::CephMds: OS::Heat::None diff --git a/puppet/services/services.yaml b/puppet/services/services.yaml deleted file mode 100644 index 0e7b6d2b..00000000 --- a/puppet/services/services.yaml +++ /dev/null @@ -1,129 +0,0 @@ -heat_template_version: pike - -description: > - Utility stack to convert an array of services into a set of combined - role configs. - -parameters: - Services: - default: [] - description: | - List nested stack service templates. - type: comma_delimited_list - ServiceNetMap: - default: {} - description: Mapping of service_name -> network name. Typically set - via parameter_defaults in the resource registry. This - mapping overrides those in ServiceNetMapDefaults. - type: json - EndpointMap: - default: {} - description: Mapping of service endpoint -> protocol. Typically set - via parameter_defaults in the resource registry. - type: json - DefaultPasswords: - default: {} - description: Mapping of service -> default password. Used to help - pass top level passwords managed by Heat into services. - type: json - RoleName: - default: '' - description: Role name on which the service is applied - type: string - RoleParameters: - description: Role Specific parameters to be provided to service - default: {} - type: json - -resources: - - ServiceChain: - type: OS::Heat::ResourceChain - properties: - resources: {get_param: Services} - concurrent: true - resource_properties: - ServiceNetMap: {get_param: ServiceNetMap} - EndpointMap: {get_param: EndpointMap} - DefaultPasswords: {get_param: DefaultPasswords} - RoleName: {get_param: RoleName} - RoleParameters: {get_param: RoleParameters} - - LoggingConfiguration: - type: OS::TripleO::LoggingConfiguration - - ServiceServerMetadataHook: - type: OS::TripleO::ServiceServerMetadataHook - properties: - RoleData: {get_attr: [ServiceChain, role_data]} - -outputs: - role_data: - description: Combined Role data for this set of services. - value: - service_names: - {get_attr: [ServiceChain, role_data, service_name]} - monitoring_subscriptions: - yaql: - expression: list($.data.role_data.where($ != null).select($.get('monitoring_subscription')).where($ != null)) - data: {role_data: {get_attr: [ServiceChain, role_data]}} - logging_sources: - # Transform the individual logging_source configuration from - # each service in the chain into a global list, adding some - # default configuration at the same time. - yaql: - expression: > - let( - default_format => $.data.default_format, - pos_file_path => $.data.pos_file_path, - sources => $.data.sources.flatten() - ) -> - $sources.where($ != null).select({ - 'type' => 'tail', - 'tag' => $.tag, - 'path' => $.path, - 'format' => $.get('format', $default_format), - 'pos_file' => $.get('pos_file', $pos_file_path + '/' + $.tag + '.pos') - }) - data: - sources: - - {get_attr: [LoggingConfiguration, LoggingDefaultSources]} - - yaql: - expression: list($.data.role_data.where($ != null).select($.get('logging_source')).where($ != null)) - data: {role_data: {get_attr: [ServiceChain, role_data]}} - - - {get_attr: [LoggingConfiguration, LoggingExtraSources]} - default_format: {get_attr: [LoggingConfiguration, LoggingDefaultFormat]} - pos_file_path: {get_attr: [LoggingConfiguration, LoggingPosFilePath]} - logging_groups: - # Build a list of unique groups to which we should add the - # fluentd user. - yaql: - expression: > - set(($.data.default + $.data.extra + $.data.role_data.where($ != null).select($.get('logging_groups'))).flatten()).where($) - data: - default: {get_attr: [LoggingConfiguration, LoggingDefaultGroups]} - extra: {get_attr: [LoggingConfiguration, LoggingExtraGroups]} - role_data: {get_attr: [ServiceChain, role_data]} - config_settings: {map_merge: {get_attr: [ServiceChain, role_data, config_settings]}} - global_config_settings: - map_merge: - yaql: - expression: list($.data.role_data.where($ != null).select($.get('global_config_settings')).where($ != null)) - data: {role_data: {get_attr: [ServiceChain, role_data]}} - service_config_settings: - yaql: - expression: $.data.role_data.where($ != null).select($.get('service_config_settings')).where($ != null).reduce($1.mergeWith($2), {}) - data: {role_data: {get_attr: [ServiceChain, role_data]}} - step_config: {get_attr: [ServiceChain, role_data, step_config]} - upgrade_tasks: - yaql: - # Note we use distinct() here to filter any identical tasks, e.g yum update for all services - expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct() - data: {get_attr: [ServiceChain, role_data]} - upgrade_batch_tasks: - yaql: - # Note we use distinct() here to filter any identical tasks, e.g yum update for all services - expression: $.data.where($ != null).select($.get('upgrade_batch_tasks')).where($ != null).flatten().distinct() - data: {get_attr: [ServiceChain, role_data]} - service_metadata_settings: {get_attr: [ServiceServerMetadataHook, metadata]} diff --git a/services.yaml b/services.yaml new file mode 100644 index 00000000..724727bb --- /dev/null +++ b/services.yaml @@ -0,0 +1,144 @@ +#FIXME move into common when specfile adds it +heat_template_version: pike + +description: > + Utility stack to convert an array of services into a set of combined + role configs. + +parameters: + Services: + default: [] + description: | + List nested stack service templates. + type: comma_delimited_list + ServiceNetMap: + default: {} + description: Mapping of service_name -> network name. Typically set + via parameter_defaults in the resource registry. This + mapping overrides those in ServiceNetMapDefaults. + type: json + EndpointMap: + default: {} + description: Mapping of service endpoint -> protocol. Typically set + via parameter_defaults in the resource registry. + type: json + DefaultPasswords: + default: {} + description: Mapping of service -> default password. Used to help + pass top level passwords managed by Heat into services. + type: json + RoleName: + default: '' + description: Role name on which the service is applied + type: string + RoleParameters: + description: Role Specific parameters to be provided to service + default: {} + type: json + +resources: + + ServiceChain: + type: OS::Heat::ResourceChain + properties: + resources: {get_param: Services} + concurrent: true + resource_properties: + ServiceNetMap: {get_param: ServiceNetMap} + EndpointMap: {get_param: EndpointMap} + DefaultPasswords: {get_param: DefaultPasswords} + RoleName: {get_param: RoleName} + RoleParameters: {get_param: RoleParameters} + + LoggingConfiguration: + type: OS::TripleO::LoggingConfiguration + + ServiceServerMetadataHook: + type: OS::TripleO::ServiceServerMetadataHook + properties: + RoleData: {get_attr: [ServiceChain, role_data]} + +outputs: + role_data: + description: Combined Role data for this set of services. + value: + service_names: + {get_attr: [ServiceChain, role_data, service_name]} + monitoring_subscriptions: + yaql: + expression: list($.data.role_data.where($ != null).select($.get('monitoring_subscription')).where($ != null)) + data: {role_data: {get_attr: [ServiceChain, role_data]}} + logging_sources: + # Transform the individual logging_source configuration from + # each service in the chain into a global list, adding some + # default configuration at the same time. + yaql: + expression: > + let( + default_format => $.data.default_format, + pos_file_path => $.data.pos_file_path, + sources => $.data.sources.flatten() + ) -> + $sources.where($ != null).select({ + 'type' => 'tail', + 'tag' => $.tag, + 'path' => $.path, + 'format' => $.get('format', $default_format), + 'pos_file' => $.get('pos_file', $pos_file_path + '/' + $.tag + '.pos') + }) + data: + sources: + - {get_attr: [LoggingConfiguration, LoggingDefaultSources]} + - yaql: + expression: list($.data.role_data.where($ != null).select($.get('logging_source')).where($ != null)) + data: {role_data: {get_attr: [ServiceChain, role_data]}} + + - {get_attr: [LoggingConfiguration, LoggingExtraSources]} + default_format: {get_attr: [LoggingConfiguration, LoggingDefaultFormat]} + pos_file_path: {get_attr: [LoggingConfiguration, LoggingPosFilePath]} + logging_groups: + # Build a list of unique groups to which we should add the + # fluentd user. + yaql: + expression: > + set(($.data.default + $.data.extra + $.data.role_data.where($ != null).select($.get('logging_groups'))).flatten()).where($) + data: + default: {get_attr: [LoggingConfiguration, LoggingDefaultGroups]} + extra: {get_attr: [LoggingConfiguration, LoggingExtraGroups]} + role_data: {get_attr: [ServiceChain, role_data]} + config_settings: {map_merge: {get_attr: [ServiceChain, role_data, config_settings]}} + global_config_settings: + map_merge: + yaql: + expression: list($.data.role_data.where($ != null).select($.get('global_config_settings')).where($ != null)) + data: {role_data: {get_attr: [ServiceChain, role_data]}} + service_config_settings: + yaql: + expression: $.data.role_data.where($ != null).select($.get('service_config_settings')).where($ != null).reduce($1.mergeWith($2), {}) + data: {role_data: {get_attr: [ServiceChain, role_data]}} + step_config: {get_attr: [ServiceChain, role_data, step_config]} + upgrade_tasks: + yaql: + # Note we use distinct() here to filter any identical tasks, e.g yum update for all services + expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct() + data: {get_attr: [ServiceChain, role_data]} + upgrade_batch_tasks: + yaql: + # Note we use distinct() here to filter any identical tasks, e.g yum update for all services + expression: $.data.where($ != null).select($.get('upgrade_batch_tasks')).where($ != null).flatten().distinct() + data: {get_attr: [ServiceChain, role_data]} + service_metadata_settings: {get_attr: [ServiceServerMetadataHook, metadata]} + + # Keys to support docker/services + puppet_config: {get_attr: [ServiceChain, role_data, puppet_config]} + kolla_config: + map_merge: {get_attr: [ServiceChain, role_data, kolla_config]} + docker_config: + {get_attr: [ServiceChain, role_data, docker_config]} + docker_puppet_tasks: + {get_attr: [ServiceChain, role_data, docker_puppet_tasks]} + host_prep_tasks: + yaql: + # Note we use distinct() here to filter any identical tasks + expression: $.data.where($ != null).select($.get('host_prep_tasks')).where($ != null).flatten().distinct() + data: {get_attr: [ServiceChain, role_data]} diff --git a/tools/yaml-validate.py b/tools/yaml-validate.py index 92d76d23..0fd4bcc3 100755 --- a/tools/yaml-validate.py +++ b/tools/yaml-validate.py @@ -219,12 +219,10 @@ def validate(filename): # qdr aliases rabbitmq service to provide alternative messaging backend if (filename.startswith('./puppet/services/') and - filename not in ['./puppet/services/services.yaml', - './puppet/services/qdr.yaml']): + filename not in ['./puppet/services/qdr.yaml']): retval = validate_service(filename, tpl) - if (filename.startswith('./docker/services/') and - filename != './docker/services/services.yaml'): + if filename.startswith('./docker/services/'): retval = validate_docker_service(filename, tpl) if filename.endswith('hyperconverged-ceph.yaml'): -- cgit 1.2.3-korg