Add fluentd client service

This implements support for installing fluentd agents as a composable
service on the overcloud.

Depends-On: I2e1abe4d8c8359e56ff626255ee50c9cacca1940

Implements: tripleo-opstools-centralized-logging
Change-Id: I23b0e23881b742158fcfb6b8c145a3211d45086e

3 files changed, 254 insertions, 0 deletions

diff --git a/puppet/services/logging/fluentd-base.yaml b/puppet/services/logging/fluentd-base.yaml
new file mode 100644
index 00000000..583dca33
--- /dev/null
+++ b/puppet/services/logging/fluentd-base.yaml
@@ -0,0 +1,36 @@
+heat_template_version: 2016-04-08
+
+description: Fluentd base service
+
+parameters:
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: >
+      Mapping of service endpoint -> protocol. Typically set
+      via parameter_defaults in the resource registry.
+    type: json
+
+
+outputs:
+  role_data:
+    description: Role data for the Fluentd role.
+    value:
+      service_name: fluentd_base
+      config_settings:
+        fluentd::package_name: fluentd
+        fluentd::service_name: fluentd
+        fluentd::config_file: /etc/fluentd/fluent.conf
+        fluentd::config_owner: fluentd
+        fluentd::config_group: fluentd
+        fluentd::config_path: /etc/fluentd/config.d
+        fluentd::plugin_provider: yum
+        fluentd::repo_install: false
diff --git a/puppet/services/logging/fluentd-client.yaml b/puppet/services/logging/fluentd-client.yaml
new file mode 100644
index 00000000..3ae7110f
--- /dev/null
+++ b/puppet/services/logging/fluentd-client.yaml
@@ -0,0 +1,64 @@
+heat_template_version: 2016-10-14
+
+description: Fluentd client configured with Puppet
+
+parameters:
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: >
+      Mapping of service endpoint -> protocol. Typically set
+      via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+  FluentdBase:
+    type: ./fluentd-base.yaml
+    properties:
+      EndpointMap: {get_param: EndpointMap}
+      ServiceNetMap: {get_param: ServiceNetMap}
+      DefaultPasswords: {get_param: DefaultPasswords}
+
+  LoggingConfiguration:
+    type: OS::TripleO::LoggingConfiguration
+
+outputs:
+  role_data:
+    description: Role data for the Fluentd client role.
+    value:
+      service_name: fluentd_client
+      config_settings:
+        map_merge:
+          - get_attr: [FluentdBase, role_data, config_settings]
+          - tripleo::profile::base::logging::fluentd::fluentd_servers:
+              get_attr: [LoggingConfiguration, LoggingServers]
+            tripleo::profile::base::logging::fluentd::fluentd_filters:
+              yaql:
+                expression: >
+                  $.data.filters.flatten().where($)
+                data:
+                  filters:
+                    - get_attr: [LoggingConfiguration, LoggingDefaultFilters]
+                    - get_attr: [LoggingConfiguration, LoggingExtraFilters]
+            tripleo::profile::base::logging::fluentd::fluentd_pos_file_path:
+              get_attr: [LoggingConfiguration, LoggingPosFilePath]
+            tripleo::profile::base::logging::fluentd::fluentd_use_ssl:
+              get_attr: [LoggingConfiguration, LoggingUsesSSL]
+            tripleo::profile::base::logging::fluentd::fluentd_ssl_certificate:
+              get_attr: [LoggingConfiguration, LoggingSSLCertificate]
+            tripleo::profile::base::logging::fluentd::fluentd_ssl_key:
+              get_attr: [LoggingConfiguration, LoggingSSLKey]
+            tripleo::profile::base::logging::fluentd::fluentd_ssl_key_passphrase:
+              get_attr: [LoggingConfiguration, LoggingSSLKeyPassphrase]
+            tripleo::profile::base::logging::fluentd::fluentd_shared_key:
+              get_attr: [LoggingConfiguration, LoggingSharedKey]
+      step_config: |
+        include ::tripleo::profile::base::logging::fluentd
diff --git a/puppet/services/logging/fluentd-config.yaml b/puppet/services/logging/fluentd-config.yaml
new file mode 100644
index 00000000..e051781e
--- /dev/null
+++ b/puppet/services/logging/fluentd-config.yaml
@@ -0,0 +1,154 @@
+heat_template_version: 2016-10-14
+
+description: Fluentd logging configuration
+
+parameters:
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: >
+      Mapping of service endpoint -> protocol. Typically set
+      via parameter_defaults in the resource registry.
+    type: json
+  LoggingDefaultFormat:
+    description: >
+      Default format used to parse messages from log files.
+    type: string
+    default: >-
+      /(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+)
+      (?<pid>\d+)
+      (?<priority>\S+)
+      (?<message>.*)$/
+  LoggingPosFilePath:
+    description: >
+      Directory in which to place fluentd pos_file files (used to track
+      file position for the 'tail' input type).
+    type: string
+    default: /var/cache/fluentd
+  LoggingDefaultGroups:
+    description: >
+      Make fluentd user a member of these groups. Only override this parameter
+      if you want to modify the default list of groups.  Use
+      LoggingExtraGroups to add the fluentd user to additional groups.
+    type: comma_delimited_list
+    default:
+      - root
+  LoggingExtraGroups:
+    description: >
+      Make fluentd user a member of these groups (in addition to
+      LoggingDefaultGroups and the groups provided by individual
+      composable services).
+    type: comma_delimited_list
+    default: []
+  LoggingServers:
+    description: |
+      A list of destinations to which fluentd will forward log messages.  Expects
+      a list of dictionaries of the form:
+
+          - host: loghost1.example.com
+            port: 24224
+          - host: loghost2.example.com
+            port: 24224
+    type: json
+    default: []
+  LoggingDefaultFilters:
+    description: >
+      A list of fluentd default filters. This will be passed verbatim
+      to the 'filter' key of a fluentd::config resource.  Only override this
+      if you do not want the default set of filters; use LoggingExtraFilters
+      if you just want to add additional servers.
+    type: json
+    default:
+      - tag_pattern: '**'
+        type: record_transformer
+        record:
+          nodename: '${hostname}'
+
+      - tag_pattern: 'openstack.**'
+        type: record_transformer
+        record:
+          component: '${tag_parts[1]}'
+  LoggingExtraFilters:
+    description: >
+      A list of additional fluentd filters. This will be passed
+      verbatim to the 'filter' key of a fluentd::config resource.
+    type: json
+    default: []
+  LoggingUsesSSL:
+    description: >
+      A boolean value indicating whether or not we should forward log messages
+      use the secure_forward plugin.
+    type: boolean
+    default: false
+  LoggingSSLCertificate:
+    description: >
+      PEM-encoded SSL CA certificate for fluentd.
+    type: string
+    default: ""
+  LoggingSSLKey:
+    description: >
+      PEM-encoded key for fluentd CA certificate (used by in_secure_forward).
+    type: string
+    default: ""
+  LoggingSSLKeyPassphrase:
+    description: >
+      Passphrase for LoggingSSLKey (used by in_secure_forward).
+    type: string
+    default: ""
+  LoggingSharedKey:
+    description: >
+      Shared secret for fluentd secure-forward plugin.
+    type: string
+    default: ""
+  LoggingDefaultSources:
+    description: >
+      A list of default logging sources for fluentd.  You should only override
+      this parameter if you wish to disable the default logging sources.  Use
+      LoggingExtraSources to define additional source configurations.
+    type: json
+    default: []
+  LoggingExtraSources:
+    description: >
+      A list of additional logging sources for fluentd.  These will be combined
+      with the LoggingDefaultSources and any logging sources defined by
+      composable services.
+    type: json
+    default: []
+
+outputs:
+  LoggingDefaultFormat:
+    value: {get_param: LoggingDefaultFormat}
+  LoggingDefaultFilters:
+    value: {get_param: LoggingDefaultFilters}
+  LoggingExtraFilters:
+    value: {get_param: LoggingExtraFilters}
+  LoggingDefaultGroups:
+    value: {get_param: LoggingDefaultGroups}
+  LoggingExtraGroups:
+    value: {get_param: LoggingExtraGroups}
+  LoggingPosFilePath:
+    value: {get_param: LoggingPosFilePath}
+  LoggingSSLCertificate:
+    value: {get_param: LoggingSSLCertificate}
+  LoggingSSLKey:
+    value: {get_param: LoggingSSLKey}
+  LoggingSSLKeyPassphrase:
+    value: {get_param: LoggingSSLKeyPassphrase}
+  LoggingServers:
+    value: {get_param: LoggingServers}
+  LoggingSharedKey:
+    value: {get_param: LoggingSharedKey}
+  LoggingUsesSSL:
+    value: {get_param: LoggingUsesSSL}
+  LoggingDefaultSources:
+    value: {get_param: LoggingDefaultSources}
+  LoggingExtraSources:
+    value: {get_param: LoggingExtraSources}