Move services.yaml output calculation into Value resources

This stores the result of the yaql queries etc for easier debugging, and
also so there's no risk we constantly re-evaluate the expensive query
which can happen with some heat versions and configurations.

This also gives a nicer error when things go wrong as when a query fails
you know which resource had an error, and also the validation on resources
is currently stricter due to bug #1599114.  We also get some additional
type validation from each OS::Heat::Value resource, e.g it checks if the
calculated value is a valid map or list.

The final advantage (and the original motivation for doing this) is that
we can easily filter null values for any outputs where this isn't already
done, which makes the config data written via openstack overcloud config
download cleaner.

Change-Id: Ia6697cf2e47f3f7b727d620536e0873a985c98c4

1 files changed, 126 insertions, 41 deletions

diff --git a/common/services.yaml b/common/services.yaml
index e415f4b9..ab4ac361 100644
--- a/common/services.yaml
+++ b/common/services.yaml
@@ -89,26 +89,20 @@ resources:
             service_names: {get_attr: [ServiceChain, role_data, service_names]}
             docker_config: {get_attr: [ServiceChain, role_data, docker_config]}
 
-outputs:
-  role_data:
-    description: Combined Role data for this set of services.
-    value:
-      service_names:
-        {get_attr: [ServiceChain, role_data, service_name]}
-      monitoring_subscriptions:
-        yaql:
-          expression: list($.data.role_data.where($ != null).select($.get('monitoring_subscription')).where($ != null))
-          data: {role_data: {get_attr: [ServiceChain, role_data]}}
-      logging_sources:
+  LoggingSourcesConfig:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
         # Transform the individual logging_source configuration from
         # each service in the chain into a global list, adding some
         # default configuration at the same time.
         yaql:
           expression: >
             let(
-            default_format => $.data.default_format,
-            pos_file_path => $.data.pos_file_path,
-            sources => $.data.sources.flatten()
+            default_format => coalesce($.data.default_format, ''),
+            pos_file_path => coalesce($.data.pos_file_path, ''),
+            sources => coalesce($.data.sources, {}).flatten()
             ) ->
             $sources.where($ != null).select({
             'type' => 'tail',
@@ -121,59 +115,150 @@ outputs:
             sources:
               - {get_attr: [LoggingConfiguration, LoggingDefaultSources]}
               - yaql:
-                  expression: list($.data.role_data.where($ != null).select($.get('logging_source')).where($ != null))
+                  expression: list(coalesce($.data.role_data, []).where($ != null).select($.get('logging_source')).where($ != null))
                   data: {role_data: {get_attr: [ServiceChain, role_data]}}
 
               - {get_attr: [LoggingConfiguration, LoggingExtraSources]}
             default_format: {get_attr: [LoggingConfiguration, LoggingDefaultFormat]}
             pos_file_path: {get_attr: [LoggingConfiguration, LoggingPosFilePath]}
-      logging_groups:
+
+  LoggingGroupsConfig:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
         # Build a list of unique groups to which we should add the
         # fluentd user.
         yaql:
           expression: >
-            set(($.data.default + $.data.extra + $.data.role_data.where($ != null).select($.get('logging_groups'))).flatten()).where($)
+            set((coalesce($.data.default, []) + coalesce($.data.extra, []) + coalesce($.data.role_data, []).where($ != null).select($.get('logging_groups'))).flatten()).where($)
           data:
             default: {get_attr: [LoggingConfiguration, LoggingDefaultGroups]}
             extra: {get_attr: [LoggingConfiguration, LoggingExtraGroups]}
             role_data: {get_attr: [ServiceChain, role_data]}
-      config_settings: {map_merge: {get_attr: [ServiceChain, role_data, config_settings]}}
-      global_config_settings:
+
+  MonitoringSubscriptionsConfig:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          expression: list(coalesce($.data.role_data, []).where($ != null).select($.get('monitoring_subscription')).where($ != null))
+          data: {role_data: {get_attr: [ServiceChain, role_data]}}
+
+  ServiceNames:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        filter:
+          - [null]
+          - {get_attr: [ServiceChain, role_data, service_name]}
+
+  GlobalConfigSettings:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
         map_merge:
           yaql:
-            expression: list($.data.role_data.where($ != null).select($.get('global_config_settings')).where($ != null))
+            expression: list(coalesce($.data.role_data, []).where($ != null).select($.get('global_config_settings')).where($ != null))
             data: {role_data: {get_attr: [ServiceChain, role_data]}}
-      service_config_settings:
+
+  ServiceConfigSettings:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
         yaql:
-          expression: $.data.role_data.where($ != null).select($.get('service_config_settings')).where($ != null).reduce($1.mergeWith($2), {})
+          expression: coalesce($.data.role_data, []).where($ != null).select($.get('service_config_settings')).where($ != null).reduce($1.mergeWith($2), {})
           data: {role_data: {get_attr: [ServiceChain, role_data]}}
-      service_workflow_tasks:
+
+  ServiceWorkflowTasks:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
         yaql:
-          expression: $.data.role_data.where($ != null).select($.get('service_workflow_tasks')).where($ != null).reduce($1.mergeWith($2), {})
+          expression: coalesce($.data.role_data, []).where($ != null).select($.get('service_workflow_tasks')).where($ != null).reduce($1.mergeWith($2), {})
           data: {role_data: {get_attr: [ServiceChain, role_data]}}
-      step_config: {get_attr: [PuppetStepConfig, value]}
-      upgrade_tasks:
+
+  UpgradeTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
         yaql:
           # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
-          expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct()
+          expression: coalesce($.data, []).where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct()
           data: {get_attr: [ServiceChain, role_data]}
-      upgrade_batch_tasks:
+
+  UpgradeBatchTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
         yaql:
-          # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
-          expression: $.data.where($ != null).select($.get('upgrade_batch_tasks')).where($ != null).flatten().distinct()
+          expression: coalesce($.data, []).where($ != null).select($.get('upgrade_batch_tasks')).where($ != null).flatten().distinct()
           data: {get_attr: [ServiceChain, role_data]}
-      service_metadata_settings: {get_attr: [ServiceServerMetadataHook, metadata]}
 
-      # Keys to support docker/services
-      puppet_config: {get_attr: [ServiceChain, role_data, puppet_config]}
-      kolla_config:
-        map_merge: {get_attr: [ServiceChain, role_data, kolla_config]}
-      docker_config:
-        {get_attr: [DockerConfig, value]}
-      docker_puppet_tasks:
-        {get_attr: [ServiceChain, role_data, docker_puppet_tasks]}
-      host_prep_tasks:
+  PuppetConfig:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          expression: coalesce($.data, []).where($ != null).select($.get('puppet_config')).where($ != null).distinct()
+          data: {get_attr: [ServiceChain, role_data]}
+
+  KollaConfig:
+    type: OS::Heat::Value
+    properties:
+      type: json
+      value:
+        yaql:
+          expression: coalesce($.data.role_data, []).where($ != null).select($.get('kolla_config')).where($ != null).reduce($1.mergeWith($2), {})
+          data: {role_data: {get_attr: [ServiceChain, role_data]}}
+
+  DockerPuppetTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
+        yaql:
+          expression: coalesce($.data, []).where($ != null).select($.get('docker_puppet_tasks')).where($ != null).distinct()
+          data: {get_attr: [ServiceChain, role_data]}
+
+  HostPrepTasks:
+    type: OS::Heat::Value
+    properties:
+      type: comma_delimited_list
+      value:
         yaql:
           # Note we use distinct() here to filter any identical tasks
-          expression: $.data.where($ != null).select($.get('host_prep_tasks')).where($ != null).flatten().distinct()
+          expression: coalesce($.data, []).where($ != null).select($.get('host_prep_tasks')).where($ != null).flatten().distinct()
           data: {get_attr: [ServiceChain, role_data]}
+
+outputs:
+  role_data:
+    description: Combined Role data for this set of services.
+    value:
+      service_names: {get_attr: [ServiceNames, value]}
+      monitoring_subscriptions: {get_attr: [MonitoringSubscriptionsConfig, value]}
+      logging_sources: {get_attr: [LoggingSourcesConfig, value]}
+      logging_groups: {get_attr: [LoggingGroupsConfig, value]}
+      config_settings: {map_merge: {get_attr: [ServiceChain, role_data, config_settings]}}
+      global_config_settings: {get_attr: [GlobalConfigSettings, value]}
+      service_config_settings: {get_attr: [ServiceConfigSettings, value]}
+      service_workflow_tasks: {get_attr: [ServiceWorkflowTasks, value]}
+      step_config: {get_attr: [PuppetStepConfig, value]}
+      upgrade_tasks: {get_attr: [UpgradeTasks, value]}
+      upgrade_batch_tasks: {get_attr: [UpgradeBatchTasks, value]}
+      service_metadata_settings: {get_attr: [ServiceServerMetadataHook, metadata]}
+
+      # Keys to support docker/services
+      puppet_config: {get_attr: [PuppetConfig, value]}
+      kolla_config: {get_attr: [KollaConfig, value]}
+      docker_config: {get_attr: [DockerConfig, value]}
+      docker_puppet_tasks: {get_attr: [DockerPuppetTasks, value]}
+      host_prep_tasks: {get_attr: [HostPrepTasks, value]}