restore missing fluentd client functionality

in the great rebase following the JINJA ALL THE THINGS changes we lost
critical functionality in the fluentd client service.  This review
restores the missing features.

Change-Id: I7c23f16f81e75f3da6a24587b2eb8385b3e920a4
Closes-bug: 1630692

9 files changed, 26 insertions, 44 deletions

diff --git a/overcloud.j2.yaml b/overcloud.j2.yaml
index fc756617..08289d19 100644
--- a/overcloud.j2.yaml
+++ b/overcloud.j2.yaml
@@ -292,8 +292,6 @@ resources:
                     services: {get_attr: [{{role.name}}ServiceChain, role_data, service_names]}
           ServiceNames: {get_attr: [{{role.name}}ServiceChain, role_data, service_names]}
           MonitoringSubscriptions: {get_attr: [{{role.name}}ServiceChain, role_data, monitoring_subscriptions]}
-          LoggingSources: {get_attr: [{{role.name}}ServiceChain, role_data, logging_sources]}
-          LoggingGroups: {get_attr: [{{role.name}}ServiceChain, role_data, logging_groups]}
 {% endfor %}
 
   allNodesConfig:
@@ -316,6 +314,24 @@ resources:
 {% for role in roles %}
           - {get_attr: [{{role.name}}ServiceChain, role_data, service_names]}
 {% endfor %}
+      logging_groups:
+        yaql:
+          expression: >
+            $.data.groups.flatten()
+          data:
+            groups:
+{% for role in roles %}
+              - {get_attr: [{{role.name}}ServiceChain, role_data, logging_groups]}
+{% endfor %}
+      logging_sources:
+        yaql:
+          expression: >
+            $.data.sources.flatten()
+          data:
+            sources:
+{% for role in roles %}
+              - {get_attr: [{{role.name}}ServiceChain, role_data, logging_sources]}
+{% endfor %}
       controller_ips: {get_attr: [Controller, ip_address]}
       controller_names: {get_attr: [Controller, hostname]}
       service_ips:
diff --git a/puppet/all-nodes-config.yaml b/puppet/all-nodes-config.yaml
index 67dc056b..34ad6fc7 100644
--- a/puppet/all-nodes-config.yaml
+++ b/puppet/all-nodes-config.yaml
@@ -20,6 +20,10 @@ parameters:
     type: string
   controller_ips:
     type: comma_delimited_list
+  logging_groups:
+    type: json
+  logging_sources:
+    type: json
   service_ips:
     type: json
   service_node_names:
@@ -79,6 +83,8 @@ resources:
             all_nodes:
               mapped_data:
                 map_merge:
+                  - tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: logging_sources}
+                  - tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: logging_groups}
                   - enabled_services: {get_param: enabled_services}
                   # This writes out a mapping of service_name_enabled: 'true'
                   # For any services not enabled, hiera foo_enabled will
diff --git a/puppet/blockstorage-role.yaml b/puppet/blockstorage-role.yaml
index a66ea08b..f5118c2c 100644
--- a/puppet/blockstorage-role.yaml
+++ b/puppet/blockstorage-role.yaml
@@ -96,12 +96,6 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
-  LoggingSources:
-    type: json
-    default: []
-  LoggingGroups:
-    type: comma_delimited_list
-    default: []
 
 resources:
   BlockStorage:
@@ -277,8 +271,6 @@ resources:
             volume:
               mapped_data:
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
-                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
-                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   # Resource for site-specific injection of root certificate
   NodeTLSCAData:
diff --git a/puppet/cephstorage-role.yaml b/puppet/cephstorage-role.yaml
index 03a53b00..62748f94 100644
--- a/puppet/cephstorage-role.yaml
+++ b/puppet/cephstorage-role.yaml
@@ -102,12 +102,6 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
-  LoggingSources:
-    type: json
-    default: []
-  LoggingGroups:
-    type: comma_delimited_list
-    default: []
 
 resources:
   CephStorage:
@@ -281,8 +275,6 @@ resources:
             ceph:
               mapped_data:
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
-                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
-                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   # Resource for site-specific injection of root certificate
   NodeTLSCAData:
diff --git a/puppet/compute-role.yaml b/puppet/compute-role.yaml
index 0205d0a6..05b8d065 100644
--- a/puppet/compute-role.yaml
+++ b/puppet/compute-role.yaml
@@ -114,12 +114,6 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
-  LoggingSources:
-    type: json
-    default: []
-  LoggingGroups:
-    type: comma_delimited_list
-    default: []
 
 resources:
 
@@ -295,8 +289,6 @@ resources:
             compute:
               mapped_data:
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
-                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
-                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   NovaComputeDeployment:
     type: OS::TripleO::SoftwareDeployment
diff --git a/puppet/controller-role.yaml b/puppet/controller-role.yaml
index ccb517f8..33ed51c0 100644
--- a/puppet/controller-role.yaml
+++ b/puppet/controller-role.yaml
@@ -128,12 +128,6 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
-  LoggingSources:
-    type: json
-    default: []
-  LoggingGroups:
-    type: comma_delimited_list
-    default: []
 
 parameter_groups:
 - label: deprecated
@@ -355,8 +349,6 @@ resources:
                 # Misc
                 tripleo::haproxy::service_certificate: {get_attr: [NodeTLSData, deployed_ssl_certificate_path]}
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
-                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
-                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
   # Hook for site-specific additional pre-deployment config, e.g extra hieradata
   ControllerExtraConfigPre:
diff --git a/puppet/objectstorage-role.yaml b/puppet/objectstorage-role.yaml
index 899ba66d..9eb66314 100644
--- a/puppet/objectstorage-role.yaml
+++ b/puppet/objectstorage-role.yaml
@@ -96,12 +96,6 @@ parameters:
     type: string
     description: Command which will be run whenever configuration data changes
     default: os-refresh-config --timeout 14400
-  LoggingSources:
-    type: json
-    default: []
-  LoggingGroups:
-    type: comma_delimited_list
-    default: []
 
 resources:
 
@@ -266,8 +260,6 @@ resources:
             object:
               mapped_data: # data supplied directly to this deployment configuration, etc
                 tripleo::packages::enable_upgrade: {get_input: enable_package_upgrade}
-                tripleo::profile::base::logging::fluentd::fluentd_sources: {get_param: LoggingSources}
-                tripleo::profile::base::logging::fluentd::fluentd_groups: {get_param: LoggingGroups}
 
 
   SwiftStorageHieraDeploy:
diff --git a/puppet/services/logging/fluentd-config.yaml b/puppet/services/logging/fluentd-config.yaml
index e051781e..58b423fd 100644
--- a/puppet/services/logging/fluentd-config.yaml
+++ b/puppet/services/logging/fluentd-config.yaml
@@ -70,7 +70,7 @@ parameters:
       - tag_pattern: '**'
         type: record_transformer
         record:
-          nodename: '${hostname}'
+          host: '${hostname}'
 
       - tag_pattern: 'openstack.**'
         type: record_transformer
diff --git a/puppet/services/services.yaml b/puppet/services/services.yaml
index 7b5fa40c..176fd235 100644
--- a/puppet/services/services.yaml
+++ b/puppet/services/services.yaml
@@ -78,7 +78,7 @@ outputs:
             sources:
               - {get_attr: [LoggingConfiguration, LoggingDefaultSources]}
               - yaql:
-                  expression: list($.data.where($ != null).select($.get('logging_sources')).where($ != null))
+                  expression: list($.data.where($ != null).select($.get('logging_source')).where($ != null))
                   data: {get_attr: [ServiceChain, role_data]}
               - {get_attr: [LoggingConfiguration, LoggingExtraSources]}
             default_format: {get_attr: [LoggingConfiguration, LoggingDefaultFormat]}