Composable Sahara services

Adds new puppet and puppet pacemaker specific services for Sahara API
and Sahara Engine.

The Pacemaker templates extend the default Sahara services and swap
in the pacemaker specific puppet-tripleo profile instead.

Change-Id: I1adda514e9592d149a3d45743a9a00b59c28ca38
Depends-On: I0c8bd68f9a98626e9d67ef713c72c9dd05b7cc12
Implements: blueprint composable-services-within-roles

10 files changed, 206 insertions, 100 deletions

diff --git a/overcloud-resource-registry-puppet.yaml b/overcloud-resource-registry-puppet.yaml
index 950f1b68..5a9d6e3a 100644
--- a/overcloud-resource-registry-puppet.yaml
+++ b/overcloud-resource-registry-puppet.yaml
@@ -136,6 +136,8 @@ resource_registry:
   OS::TripleO::Services::Loadbalancer: puppet/services/loadbalancer.yaml
   OS::TripleO::Services::Memcached: puppet/services/memcached.yaml
   OS::TripleO::Services::SwiftProxy: puppet/services/swift-proxy.yaml
+  OS::TripleO::Services::SaharaApi: puppet/services/sahara-api.yaml
+  OS::TripleO::Services::SaharaEngine: puppet/services/sahara-engine.yaml
 
 parameter_defaults:
   EnablePackageInstall: false
diff --git a/overcloud.yaml b/overcloud.yaml
index c6772ab3..686ad91f 100644
--- a/overcloud.yaml
+++ b/overcloud.yaml
@@ -524,10 +524,6 @@ parameters:
     type: number
     default: 3
     description: How many replicas to use in the swift rings.
-  SaharaPassword:
-    description: The password for the sahara service account.
-    type: string
-    hidden: true
 
 # Compute-specific params
   CeilometerComputeAgent:
@@ -956,7 +952,6 @@ resources:
           RabbitClientUseSSL: {get_param: RabbitClientUseSSL}
           RabbitClientPort: {get_param: RabbitClientPort}
           RedisPassword: {get_param: RedisPassword}
-          SaharaPassword: {get_param: SaharaPassword}
           SnmpdReadonlyUserName: {get_param: SnmpdReadonlyUserName}
           SnmpdReadonlyUserPassword: {get_param: SnmpdReadonlyUserPassword}
           RedisVirtualIP: {get_attr: [RedisVirtualIP, ip_address]}
diff --git a/puppet/controller.yaml b/puppet/controller.yaml
index 42687c53..6bd6c39f 100644
--- a/puppet/controller.yaml
+++ b/puppet/controller.yaml
@@ -270,11 +270,6 @@ parameters:
   SaharaApiVirtualIP:
     type: string
     default: ''
-  SaharaPassword:
-    default: unset
-    description: The password for the sahara service account, used by sahara-api.
-    type: string
-    hidden: true
   MysqlClusterUniquePart:
     description: A unique identifier of the MySQL cluster the controller is in.
     type: string
@@ -1056,19 +1051,6 @@ resources:
         swift_mount_check: {get_param: SwiftMountCheck}
         enable_package_install: {get_param: EnablePackageInstall}
         enable_package_upgrade: {get_attr: [UpdateDeployment, update_managed_packages]}
-        sahara_password: {get_param: SaharaPassword}
-        sahara_public_url: {get_param: [EndpointMap, SaharaPublic, uri]}
-        sahara_internal_url: {get_param: [EndpointMap, SaharaInternal, uri]}
-        sahara_admin_url: {get_param: [EndpointMap, SaharaAdmin, uri]}
-        sahara_dsn:
-          list_join:
-            - ''
-            - - {get_param: [EndpointMap, MysqlInternal, protocol]}
-              - '://sahara:'
-              - {get_param: SaharaPassword}
-              - '@'
-              - {get_param: [EndpointMap, MysqlInternal, host]}
-              - '/sahara'
         swift_proxy_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftProxyNetwork]}]}
         swift_management_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftMgmtNetwork]}]}
         cinder_iscsi_network:
@@ -1426,33 +1408,6 @@ resources:
                 horizon::bind_address: {get_input: horizon_network}
                 horizon::keystone_url: {get_input: keystone_auth_uri}
 
-                # Sahara
-                sahara::host: {get_input: sahara_api_network}
-                sahara::plugins:
-                  - cdh
-                  - hdp
-                  - mapr
-                  - vanilla
-                  - spark
-                  - storm
-                sahara::admin_password: {get_input: sahara_password}
-                sahara::auth_uri: {get_input: keystone_auth_uri}
-                sahara::admin_user: sahara
-                sahara::identity_uri: {get_input: keystone_identity_uri}
-                sahara::use_neutron: true
-                sahara::database_connection: {get_input: sahara_dsn}
-                sahara::debug: {get_input: debug}
-                sahara::rpc_backend: rabbit
-                sahara::rabbit_userid: {get_input: rabbit_username}
-                sahara::rabbit_password: {get_input: rabbit_password}
-                sahara::rabbit_use_ssl: {get_input: rabbit_client_use_ssl}
-                sahara::rabbit_port: {get_input: rabbit_client_port}
-                sahara::db::mysql::password: {get_input: sahara_password}
-                sahara::keystone::auth::public_url: {get_input: sahara_public_url }
-                sahara::keystone::auth::internal_url: {get_input: sahara_internal_url }
-                sahara::keystone::auth::admin_url: {get_input: sahara_admin_url }
-                sahara::keystone::auth::password: {get_input: sahara_password }
-                sahara::keystone::auth::region: {get_input: keystone_region}
                 # RabbitMQ
                 rabbitmq::node_ip_address: {get_input: rabbitmq_network}
                 rabbitmq::erlang_cookie: {get_input: rabbit_cookie}
diff --git a/puppet/manifests/overcloud_controller.pp b/puppet/manifests/overcloud_controller.pp
index 8263ec56..5fec47a1 100644
--- a/puppet/manifests/overcloud_controller.pp
+++ b/puppet/manifests/overcloud_controller.pp
@@ -108,7 +108,6 @@ if hiera('step') >= 2 {
   include ::nova::db::mysql_api
   include ::neutron::db::mysql
   include ::cinder::db::mysql
-  include ::sahara::db::mysql
   if downcase(hiera('gnocchi_indexer_backend')) == 'mysql' {
     include ::gnocchi::db::mysql
   }
@@ -504,11 +503,6 @@ if hiera('step') >= 4 {
   include ::aodh::listener
   include ::aodh::client
 
-  # Sahara
-  include ::sahara
-  include ::sahara::service::api
-  include ::sahara::service::engine
-
   # Horizon
   include ::apache::mod::remoteip
   if 'cisco_n1kv' in hiera('neutron::plugins::ml2::mechanism_drivers') {
diff --git a/puppet/manifests/overcloud_controller_pacemaker.pp b/puppet/manifests/overcloud_controller_pacemaker.pp
index b9bd6b1d..0892da13 100644
--- a/puppet/manifests/overcloud_controller_pacemaker.pp
+++ b/puppet/manifests/overcloud_controller_pacemaker.pp
@@ -25,8 +25,7 @@ Service <|
   tag == 'ceilometer-service' or
   tag == 'gnocchi-service' or
   tag == 'neutron-service' or
-  tag == 'nova-service' or
-  tag == 'sahara-service'
+  tag == 'nova-service'
 |> {
   hasrestart => true,
   restart    => '/bin/true',
@@ -313,9 +312,6 @@ if hiera('step') >= 2 {
         require => Exec['galera-ready'],
       }
     }
-    class { '::sahara::db::mysql':
-      require       => Exec['galera-ready'],
-    }
   }
 
   # Ceph
@@ -683,18 +679,6 @@ MYSQL_HOST=localhost\n",
     enabled_backends => union($cinder_enabled_backends, hiera('cinder_user_enabled_backends')),
   }
 
-  class { '::sahara':
-    sync_db => $sync_db,
-  }
-  class { '::sahara::service::api':
-    manage_service => false,
-    enabled        => false,
-  }
-  class { '::sahara::service::engine':
-    manage_service => false,
-    enabled        => false,
-  }
-
   # swift storage
   if str2bool(hiera('enable_swift_storage', true)) {
     class {'::swift::storage::all':
@@ -971,33 +955,6 @@ password=\"${mysql_root_password}\"",
                   Pacemaker::Resource::Service[$::cinder::params::volume_service]],
     }
 
-    # Sahara
-    pacemaker::resource::service { $::sahara::params::api_service_name :
-      clone_params => 'interleave=true',
-      require      => Pacemaker::Resource::Ocf['openstack-core'],
-    }
-    pacemaker::resource::service { $::sahara::params::engine_service_name :
-      clone_params => 'interleave=true',
-    }
-    pacemaker::constraint::base { 'keystone-then-sahara-api-constraint':
-      constraint_type => 'order',
-      first_resource  => 'openstack-core-clone',
-      second_resource => "${::sahara::params::api_service_name}-clone",
-      first_action    => 'start',
-      second_action   => 'start',
-      require         => [Pacemaker::Resource::Service[$::sahara::params::api_service_name],
-                          Pacemaker::Resource::Ocf['openstack-core']],
-    }
-    pacemaker::constraint::base { 'sahara-api-then-sahara-engine-constraint':
-      constraint_type => 'order',
-      first_resource  => "${::sahara::params::api_service_name}-clone",
-      second_resource => "${::sahara::params::engine_service_name}-clone",
-      first_action    => 'start',
-      second_action   => 'start',
-      require         => [Pacemaker::Resource::Service[$::sahara::params::api_service_name],
-                          Pacemaker::Resource::Service[$::sahara::params::engine_service_name]],
-    }
-
     if hiera('neutron::enable_ovs_agent', true) {
       pacemaker::resource::service { $::neutron::params::ovs_agent_service:
         clone_params => 'interleave=true',
diff --git a/puppet/services/pacemaker/sahara-api.yaml b/puppet/services/pacemaker/sahara-api.yaml
new file mode 100644
index 00000000..a5db77c4
--- /dev/null
+++ b/puppet/services/pacemaker/sahara-api.yaml
@@ -0,0 +1,30 @@
+heat_template_version: 2016-04-08
+
+description: >
+  OpenStack Sahara API service with Pacemaker configured with Puppet
+
+parameters:
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+
+  SaharaApiBase:
+    type: ../sahara-api.yaml
+    properties:
+      EndpointMap: {get_param: EndpointMap}
+
+outputs:
+  role_data:
+    description: Role data for the Sahara API role.
+    value:
+      config_settings:
+        map_merge:
+          - get_attr: [SaharaApiBase, role_data, config_settings]
+          - sahara::service::api::manage_service: false
+            sahara::service::api::enabled: false
+      step_config: |
+        include ::tripleo::profile::pacemaker::sahara
diff --git a/puppet/services/pacemaker/sahara-engine.yaml b/puppet/services/pacemaker/sahara-engine.yaml
new file mode 100644
index 00000000..129f88bf
--- /dev/null
+++ b/puppet/services/pacemaker/sahara-engine.yaml
@@ -0,0 +1,32 @@
+heat_template_version: 2016-04-08
+
+description: >
+  OpenStack Sahara Engine service with Pacemaker configured with Puppet
+
+parameters:
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+
+  SaharaEngineBase:
+    type: ../sahara-engine.yaml
+    properties:
+      EndpointMap: {get_param: EndpointMap}
+
+outputs:
+  role_data:
+    description: Role data for the Sahara Engine role.
+    value:
+      config_settings:
+        map_merge:
+          - get_attr: [SaharaEngineBase, role_data, config_settings]
+          - sahara::service::engine::manage_service: false
+            sahara::service::engine::enabled: false
+      # No puppet manifests since sahara-engine is included in
+      # ::tripleo::profile::pacemaker::sahara which is maintained alongside of
+      # pacemaker/sahara-api.yaml.
+      step_config:
diff --git a/puppet/services/sahara-api.yaml b/puppet/services/sahara-api.yaml
new file mode 100644
index 00000000..93bf7385
--- /dev/null
+++ b/puppet/services/sahara-api.yaml
@@ -0,0 +1,52 @@
+heat_template_version: 2016-04-08
+
+description: >
+  OpenStack Sahara API service configured with Puppet
+
+parameters:
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+  SaharaPassword:
+    default: unset
+    description: The password for the sahara service account, used by sahara-api.
+    type: string
+    hidden: true
+  SaharaWorkers:
+    default: 0
+    description: The number of workers for the sahara-api.
+    type: number
+  SaharaApiVirtualIP:
+    type: string
+    default: ''
+  KeystoneRegion:
+    type: string
+    default: 'regionOne'
+    description: Keystone region for endpoint
+
+resources:
+  SaharaBase:
+    type: ./sahara-base.yaml
+
+outputs:
+  role_data:
+    description: Role data for the Sahara API role.
+    value:
+      config_settings:
+        map_merge:
+          - get_attr: [SaharaBase, role_data, config_settings]
+          - sahara::host: {get_param: SaharaApiVirtualIP}
+            sahara::port: {get_param: [EndpointMap, SaharaInternal, port]}
+            sahara::auth_uri: {get_param: [EndpointMap, KeystoneInternal, uri] }
+            sahara::identity_uri: { get_param: [EndpointMap, KeystoneAdmin, uri_no_suffix] }
+            sahara::admin_password: {get_param: SaharaPassword}
+            sahara::service::api::api_workers: {get_param: SaharaApiWorkers}
+            sahara::keystone::auth::public_url: {get_param: [EndpointMap, SaharaPublic, uri]}
+            sahara::keystone::auth::internal_url: {get_param: [EndpointMap, SaharaInternal, uri]}
+            sahara::keystone::auth::admin_url: {get_param: [EndpointMap, SaharaAdmin, uri]}
+            sahara::keystone::auth::password: {get_param: SaharaPassword }
+            sahara::keystone::auth::region: {get_param: KeystoneRegion}
+      step_config: |
+        include ::tripleo::profile::base::sahara::api
diff --git a/puppet/services/sahara-base.yaml b/puppet/services/sahara-base.yaml
new file mode 100644
index 00000000..275d7536
--- /dev/null
+++ b/puppet/services/sahara-base.yaml
@@ -0,0 +1,48 @@
+heat_template_version: 2016-04-08
+
+description: >
+  OpenStack Sahara base service. Shared for all Sahara services.
+
+parameters:
+  RabbitPassword:
+    description: The password for RabbitMQ
+    type: string
+    hidden: true
+  RabbitUserName:
+    default: guest
+    description: The username for RabbitMQ
+    type: string
+  RabbitClientUseSSL:
+    default: false
+    description: >
+        Rabbit client subscriber parameter to specify
+        an SSL connection to the RabbitMQ host.
+    type: string
+  RabbitClientPort:
+    default: 5672
+    description: Set rabbit subscriber port, change this if using SSL
+    type: number
+  Debug:
+    type: string
+    default: ''
+    description: Set to True to enable debugging on all services.
+
+outputs:
+  role_data:
+    description: Role data for the Sahara base service.
+    value:
+      config_settings:
+        sahara::rabbit_password: {get_param: RabbitPassword}
+        sahara::rabbit_user: {get_param: RabbitUserName}
+        sahara::rabbit_use_ssl: {get_param: RabbitClientUseSSL}
+        sahara::rabbit_port: {get_param: RabbitClientPort}
+        sahara::debug: {get_param: Debug}
+        sahara::use_neutron: true
+        sahara::plugins:
+          - cdh
+          - hdp
+          - mapr
+          - vanilla
+          - spark
+          - storm
+        sahara::rpc_backend: rabbit
diff --git a/puppet/services/sahara-engine.yaml b/puppet/services/sahara-engine.yaml
new file mode 100644
index 00000000..17ef49fa
--- /dev/null
+++ b/puppet/services/sahara-engine.yaml
@@ -0,0 +1,41 @@
+heat_template_version: 2016-04-08
+
+description: >
+  OpenStack Sahara Engine service configured with Puppet
+
+parameters:
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+  SaharaPassword:
+    default: unset
+    description: The password for the sahara service account, used by sahara-api.
+    type: string
+    hidden: true
+
+resources:
+  SaharaBase:
+    type: ./sahara-base.yaml
+
+outputs:
+  role_data:
+    description: Role data for the Sahara Engine role.
+    value:
+      config_settings:
+        map_merge:
+          - get_attr: [SaharaBase, role_data, config_settings]
+          - sahara_dsn: &sahara_dsn
+              list_join:
+                - ''
+                - - {get_param: [EndpointMap, MysqlVirtual, protocol]}
+                  - '://sahara:'
+                  - {get_param: SaharaPassword}
+                  - '@'
+                  - {get_param: [EndpointMap, MysqlVirtual, host]}
+                  - '/sahara'
+            sahara::database_connection: *sahara_dsn
+            sahara::db::mysql::password: {get_param: SaharaPassword}
+      step_config: |
+        include ::tripleo::profile::base::sahara::engine