Sahara Integration

Integration of OpenStack data processing service (sahara) with
TripleO.

- Deploys sahara in distributed mode (separate api and engine
  processes on each controller node)
- Load balancing w/haproxy
- RabbitMQ/MySQL supported per current TripleO standard
- Minimal configurability at this time

Change-Id: I77a6a69ed5691e3b1ba34e9ebb4d88c80019642c
Partially-implements: blueprint sahara-integration
Depends-On: I0f0a1dc2eaa57d8226bad8cfb250110296ab9614
Depends-On: Ib84cc59667616ec94e7edce2715cbd7dd944f4ae
Depends-On: I9fe321fd4284f7bfd55bd2e69dcfe623ed6f8a2a

7 files changed, 103 insertions, 0 deletions

diff --git a/puppet/all-nodes-config.yaml b/puppet/all-nodes-config.yaml
index 9dd43680..3dd3d5c9 100644
--- a/puppet/all-nodes-config.yaml
+++ b/puppet/all-nodes-config.yaml
@@ -50,6 +50,8 @@ parameters:
     type: comma_delimited_list
   keystone_admin_api_node_ips:
     type: comma_delimited_list
+  sahara_api_node_ips:
+    type: comma_delimited_list
 
   DeployIdentifier:
     type: string
@@ -241,6 +243,14 @@ resources:
                         list_join:
                         - "','"
                         - {get_param: keystone_admin_api_node_ips}
+                sahara_api_node_ips:
+                  str_replace:
+                    template: "['SERVERS_LIST']"
+                    params:
+                      SERVERS_LIST:
+                        list_join:
+                        - "','"
+                        - {get_param: sahara_api_node_ips}
 
                 # NOTE(gfidente): interpolation with %{} in the
                 # hieradata file can't be used as it returns string
@@ -250,6 +260,7 @@ resources:
                 neutron::rabbit_hosts: *rabbit_nodes_array
                 nova::rabbit_hosts: *rabbit_nodes_array
                 keystone::rabbit_hosts: *rabbit_nodes_array
+                sahara::rabbit_hosts: *rabbit_nodes_array
 
                 deploy_identifier: {get_param: DeployIdentifier}
                 update_identifier: {get_param: UpdateIdentifier}
diff --git a/puppet/controller.yaml b/puppet/controller.yaml
index a825f582..8a4fe21b 100644
--- a/puppet/controller.yaml
+++ b/puppet/controller.yaml
@@ -315,6 +315,14 @@ parameters:
     default: 0
     description: Number of workers for Keystone service.
     type: number
+  SaharaApiVirtualIP:
+    type: string
+    default: ''
+  SaharaPassword:
+    default: unset
+    description: The password for the sahara service account, used by sahara-api.
+    type: string
+    hidden: true
   MysqlClusterUniquePart:
     description: A unique identifier of the MySQL cluster the controller is in.
     type: string
@@ -822,6 +830,7 @@ resources:
       server: {get_resource: Controller}
       NodeIndex: {get_param: NodeIndex}
 
+
   ControllerDeployment:
     type: OS::TripleO::SoftwareDeployment
     depends_on: NetworkDeployment
@@ -1089,6 +1098,15 @@ resources:
         swift_mount_check: {get_param: SwiftMountCheck}
         enable_package_install: {get_param: EnablePackageInstall}
         enable_package_upgrade: {get_attr: [UpdateDeployment, update_managed_packages]}
+        sahara_password: {get_param: SaharaPassword}
+        sahara_dsn:
+          list_join:
+            - ''
+            - - 'mysql://sahara:'
+              - {get_param: SaharaPassword}
+              - '@'
+              - {get_param: MysqlVirtualIP}
+              - '/sahara'
         swift_proxy_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftProxyNetwork]}]}
         swift_management_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftMgmtNetwork]}]}
         cinder_iscsi_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, CinderIscsiNetwork]}]}
@@ -1111,6 +1129,7 @@ resources:
         rabbitmq_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, RabbitMqNetwork]}]}
         redis_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, RedisNetwork]}]}
         redis_vip: {get_param: RedisVirtualIP}
+        sahara_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
         memcached_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MemcachedNetwork]}]}
         mysql_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
         mysql_virtual_ip: {get_param: MysqlVirtualIP}
@@ -1424,6 +1443,29 @@ resources:
                 horizon::bind_address: {get_input: horizon_network}
                 horizon::keystone_url: {get_input: keystone_auth_uri}
 
+                # Sahara
+                sahara::host: {get_input: sahara_api_network}
+                sahara::plugins:
+                  - cdh
+                  - hdp
+                  - mapr
+                  - vanilla
+                  - spark
+                  - storm
+                sahara::admin_password: {get_input: sahara_password}
+                sahara::auth_uri: {get_input: keystone_auth_uri}
+                sahara::admin_user: sahara
+                sahara::identity_uri: {get_input: keystone_identity_uri}
+                sahara::use_neutron: true
+                sahara::database_connection: {get_input: sahara_dsn}
+                sahara::debug: {get_input: debug}
+                sahara::rpc_backend: rabbit
+                sahara::rabbit_userid: {get_input: rabbit_username}
+                sahara::rabbit_password: {get_input: rabbit_password}
+                sahara::rabbit_use_ssl: {get_input: rabbit_client_use_ssl}
+                sahara::rabbit_port: {get_input: rabbit_client_port}
+                sahara::db::mysql::password: {get_input: sahara_password}
+
                 # Rabbit
                 rabbitmq::node_ip_address: {get_input: rabbitmq_network}
                 rabbitmq::erlang_cookie: {get_input: rabbit_cookie}
diff --git a/puppet/hieradata/controller.yaml b/puppet/hieradata/controller.yaml
index c9f3a417..2f082390 100644
--- a/puppet/hieradata/controller.yaml
+++ b/puppet/hieradata/controller.yaml
@@ -38,6 +38,7 @@ cinder::api::keystone_tenant: 'service'
 swift::proxy::authtoken::admin_tenant_name: 'service'
 ceilometer::api::keystone_tenant: 'service'
 heat::keystone_tenant: 'service'
+sahara::admin_tenant_name: 'service'
 
 # keystone
 keystone::cron::token_flush::maxdelay: 3600
@@ -120,6 +121,7 @@ tripleo::loadbalancer::nova_metadata: true
 tripleo::loadbalancer::nova_novncproxy: true
 tripleo::loadbalancer::mysql: true
 tripleo::loadbalancer::redis: true
+tripleo::loadbalancer::sahara: true
 tripleo::loadbalancer::swift_proxy_server: true
 tripleo::loadbalancer::ceilometer: true
 tripleo::loadbalancer::heat_api: true
diff --git a/puppet/hieradata/database.yaml b/puppet/hieradata/database.yaml
index 7e925d90..89577505 100644
--- a/puppet/hieradata/database.yaml
+++ b/puppet/hieradata/database.yaml
@@ -53,3 +53,10 @@ ceilometer::db::mysql::dbname: ceilometer
 ceilometer::db::mysql::allowed_hosts:
   - '%'
   - "%{hiera('mysql_bind_host')}"
+
+sahara::db::mysql::user: sahara
+sahara::db::mysql::host: "%{hiera('mysql_virtual_ip')}"
+sahara::db::mysql::dbname: sahara
+sahara::db::mysql::allowed_hosts:
+  - '%'
+  - "%{hiera('mysql_bind_host')}"
diff --git a/puppet/manifests/overcloud_controller.pp b/puppet/manifests/overcloud_controller.pp
index 913bcb63..14daf9bf 100644
--- a/puppet/manifests/overcloud_controller.pp
+++ b/puppet/manifests/overcloud_controller.pp
@@ -101,6 +101,7 @@ if hiera('step') >= 2 {
   include ::neutron::db::mysql
   include ::cinder::db::mysql
   include ::heat::db::mysql
+  include ::sahara::db::mysql
   if downcase(hiera('ceilometer_backend')) == 'mysql' {
     include ::ceilometer::db::mysql
   }
@@ -541,6 +542,11 @@ if hiera('step') >= 3 {
   include ::heat::api_cloudwatch
   include ::heat::engine
 
+  # Sahara
+  include ::sahara
+  include ::sahara::service::api
+  include ::sahara::service::engine
+
   # Horizon
   if 'cisco_n1kv' in hiera('neutron::plugins::ml2::mechanism_drivers') {
     $_profile_support = 'cisco'
diff --git a/puppet/manifests/overcloud_controller_pacemaker.pp b/puppet/manifests/overcloud_controller_pacemaker.pp
index e6ee85ae..b0a9fe10 100644
--- a/puppet/manifests/overcloud_controller_pacemaker.pp
+++ b/puppet/manifests/overcloud_controller_pacemaker.pp
@@ -443,6 +443,10 @@ MYSQL_HOST=localhost\n",
         require => Exec['galera-ready'],
       }
     }
+
+    class { '::sahara::db::mysql':
+      require       => Exec['galera-ready'],
+    }
   }
 
   # pre-install swift here so we can build rings
@@ -852,6 +856,18 @@ if hiera('step') >= 3 {
     enabled_backends => $cinder_enabled_backends,
   }
 
+  class { '::sahara':
+    sync_db => $sync_db,
+  }
+  class { '::sahara::service::api':
+    manage_service => false,
+    enabled        => false,
+  }
+  class { '::sahara::service::engine':
+    manage_service => false,
+    enabled        => false,
+  }
+
   # swift proxy
   class { '::swift::proxy' :
     manage_service => $non_pcmk_start,
@@ -1097,6 +1113,24 @@ if hiera('step') >= 4 {
                   Pacemaker::Resource::Service[$::cinder::params::volume_service]],
     }
 
+    # Sahara
+    pacemaker::resource::service { $::sahara::params::api_service_name :
+      clone_params => 'interleave=true',
+      require      => Pacemaker::Resource::Service[$::keystone::params::service_name],
+    }
+    pacemaker::resource::service { $::sahara::params::engine_service_name :
+      clone_params => 'interleave=true',
+    }
+    pacemaker::constraint::base { 'keystone-then-sahara-api-constraint':
+      constraint_type => 'order',
+      first_resource  => "${::keystone::params::service_name}-clone",
+      second_resource => "${::sahara::params::api_service_name}-clone",
+      first_action    => 'start',
+      second_action   => 'start',
+      require         => [Pacemaker::Resource::Service[$::sahara::params::api_service_name],
+                          Pacemaker::Resource::Service[$::keystone::params::service_name]],
+    }
+
     # Glance
     pacemaker::resource::service { $::glance::params::registry_service_name :
       clone_params => 'interleave=true',
diff --git a/puppet/vip-config.yaml b/puppet/vip-config.yaml
index 1dec489c..c49a1047 100644
--- a/puppet/vip-config.yaml
+++ b/puppet/vip-config.yaml
@@ -19,6 +19,7 @@ resources:
                 cinder_api_vip: {get_input: cinder_api_vip}
                 glance_api_vip: {get_input: glance_api_vip}
                 glance_registry_vip: {get_input: glance_registry_vip}
+                sahara_api_vip: {get_input: sahara_api_vip}
                 swift_proxy_vip: {get_input: swift_proxy_vip}
                 nova_api_vip: {get_input: nova_api_vip}
                 nova_metadata_vip: {get_input: nova_metadata_vip}