summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorEthan Gafford <egafford@redhat.com>2015-08-26 17:43:52 -0400
committerEthan Gafford <egafford@redhat.com>2016-01-08 13:50:28 -0500
commitd69868ea9eec3d1f0a59b1fd6635a2c14df6e609 (patch)
treead47a36140503e81e241c573b730aaf4e9330fcd
parent86a77ebd42a3fd3bdddc0c3d242691b45641ade0 (diff)
Sahara Integration
Integration of OpenStack data processing service (sahara) with TripleO. - Deploys sahara in distributed mode (separate api and engine processes on each controller node) - Load balancing w/haproxy - RabbitMQ/MySQL supported per current TripleO standard - Minimal configurability at this time Change-Id: I77a6a69ed5691e3b1ba34e9ebb4d88c80019642c Partially-implements: blueprint sahara-integration Depends-On: I0f0a1dc2eaa57d8226bad8cfb250110296ab9614 Depends-On: Ib84cc59667616ec94e7edce2715cbd7dd944f4ae Depends-On: I9fe321fd4284f7bfd55bd2e69dcfe623ed6f8a2a
-rw-r--r--network/endpoints/endpoint_map.yaml34
-rw-r--r--overcloud.yaml15
-rw-r--r--puppet/all-nodes-config.yaml11
-rw-r--r--puppet/controller.yaml42
-rw-r--r--puppet/hieradata/controller.yaml2
-rw-r--r--puppet/hieradata/database.yaml7
-rw-r--r--puppet/manifests/overcloud_controller.pp6
-rw-r--r--puppet/manifests/overcloud_controller_pacemaker.pp34
-rw-r--r--puppet/vip-config.yaml1
9 files changed, 152 insertions, 0 deletions
diff --git a/network/endpoints/endpoint_map.yaml b/network/endpoints/endpoint_map.yaml
index 0ff0a8e5..f6063c0e 100644
--- a/network/endpoints/endpoint_map.yaml
+++ b/network/endpoints/endpoint_map.yaml
@@ -40,6 +40,9 @@ parameters:
SwiftProxyVirtualIP:
type: string
default: ''
+ SaharaApiVirtualIP:
+ type: string
+ default: ''
EndpointMap:
type: json
default:
@@ -74,6 +77,9 @@ parameters:
SwiftAdmin: {protocol: 'http', port: '8080', host: 'IP_ADDRESS'}
SwiftInternal: {protocol: 'http', port: '8080', host: 'IP_ADDRESS'}
SwiftPublic: {protocol: 'http', port: '8080', host: 'IP_ADDRESS'}
+ SaharaAdmin: {protocol: 'http', port: '8386', host: 'IP_ADDRESS'}
+ SaharaInternal: {protocol: 'http', port: '8386', host: 'IP_ADDRESS'}
+ SaharaPublic: {protocol: 'http', port: '8386', host: 'IP_ADDRESS'}
description: Mapping of service endpoint -> protocol. Typically set
via parameter_defaults in the resource registry.
CloudName:
@@ -404,6 +410,31 @@ resources:
IP: {get_param: SwiftProxyVirtualIP}
CloudName: {get_param: CloudName}
+ SaharaInternal:
+ type: OS::TripleO::Endpoint
+ properties:
+ EndpointName: SaharaInternal
+ EndpointMap: { get_param: EndpointMap }
+ IP: {get_param: SaharaApiVirtualIP}
+ CloudName: {get_param: CloudName}
+ UriSuffix: '/v1.1/%(tenant_id)s'
+ SaharaPublic:
+ type: OS::TripleO::Endpoint
+ properties:
+ EndpointName: SaharaPublic
+ EndpointMap: { get_param: EndpointMap }
+ IP: {get_param: SaharaApiVirtualIP}
+ CloudName: {get_param: CloudName}
+ UriSuffix: '/v1.1/%(tenant_id)s'
+ SaharaAdmin:
+ type: OS::TripleO::Endpoint
+ properties:
+ EndpointName: SaharaAdmin
+ EndpointMap: { get_param: EndpointMap }
+ IP: {get_param: SaharaApiVirtualIP}
+ CloudName: {get_param: CloudName}
+ UriSuffix: '/v1.1/%(tenant_id)s'
+
outputs:
endpoint_map:
value:
@@ -448,3 +479,6 @@ outputs:
SwiftS3Internal: {get_attr: [ SwiftS3Internal, endpoint] }
SwiftS3Public: {get_attr: [ SwiftS3Public, endpoint] }
SwiftS3Admin: {get_attr: [ SwiftS3Admin, endpoint] }
+ SaharaInternal: {get_attr: [ SaharaInternal, endpoint] }
+ SaharaPublic: {get_attr: [ SaharaPublic, endpoint] }
+ SaharaAdmin: {get_attr: [ SaharaAdmin, endpoint] } \ No newline at end of file
diff --git a/overcloud.yaml b/overcloud.yaml
index 738dcfb6..0e8318e9 100644
--- a/overcloud.yaml
+++ b/overcloud.yaml
@@ -529,6 +529,12 @@ parameters:
type: number
default: 3
description: How many replicas to use in the swift rings.
+ SaharaPassword:
+ description: The password for the sahara service account.
+ # TODO(egafford): Remove default on merge of https://review.openstack.org/#/c/221418/ (added to avoid circular dep)
+ default: unset
+ type: string
+ hidden: true
# Compute-specific params
CeilometerComputeAgent:
@@ -619,6 +625,7 @@ parameters:
NovaVncProxyNetwork: internal_api
SwiftMgmtNetwork: storage_mgmt
SwiftProxyNetwork: storage
+ SaharaApiNetwork: internal_api
HorizonNetwork: internal_api
MemcachedNetwork: internal_api
RabbitMqNetwork: internal_api
@@ -812,6 +819,7 @@ resources:
MysqlVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
NeutronApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NeutronApiNetwork]}]}
NovaApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NovaApiNetwork]}]}
+ SaharaApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
SwiftProxyVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SwiftProxyNetwork]}]}
PublicVirtualIP: {get_attr: [VipMap, net_ip_map, external]}
@@ -912,6 +920,7 @@ resources:
RabbitClientUseSSL: {get_param: RabbitClientUseSSL}
RabbitClientPort: {get_param: RabbitClientPort}
RabbitFDLimit: {get_param: RabbitFDLimit}
+ SaharaPassword: {get_param: SaharaPassword}
SnmpdReadonlyUserName: {get_param: SnmpdReadonlyUserName}
SnmpdReadonlyUserPassword: {get_param: SnmpdReadonlyUserPassword}
RedisVirtualIP: {get_attr: [RedisVirtualIP, ip_address]}
@@ -937,6 +946,7 @@ resources:
KeystonePublicApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, KeystonePublicApiNetwork]}]}
NeutronApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NeutronApiNetwork]}]}
NovaApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NovaApiNetwork]}]}
+ SaharaApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
UpdateIdentifier: {get_param: UpdateIdentifier}
Hostname:
str_replace:
@@ -1158,6 +1168,7 @@ resources:
neutron_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, NeutronApiNetwork]}]}
keystone_public_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, KeystonePublicApiNetwork]}]}
keystone_admin_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, KeystoneAdminApiNetwork]}]}
+ sahara_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
DeployIdentifier: {get_param: DeployIdentifier}
UpdateIdentifier: {get_param: UpdateIdentifier}
@@ -1270,6 +1281,7 @@ resources:
control_virtual_ip: {get_attr: [VipMap, net_ip_map, ctlplane]}
public_virtual_ip: {get_attr: [VipMap, net_ip_map, external]}
internal_api_virtual_ip: {get_attr: [VipMap, net_ip_map, internal_api]}
+ sahara_api_vip: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
storage_virtual_ip: {get_attr: [VipMap, net_ip_map, storage]}
storage_mgmt_virtual_ip: {get_attr: [VipMap, net_ip_map, storage_mgmt]}
@@ -1527,6 +1539,9 @@ outputs:
NovaInternalVip:
description: VIP for Nova API internal endpoint
value: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NovaApiNetwork]}]}
+ SaharaInternalVip:
+ description: VIP for Sahara API internal endpoint
+ value: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
SwiftInternalVip:
description: VIP for Swift Proxy internal endpoint
value: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SwiftProxyNetwork]}]}
diff --git a/puppet/all-nodes-config.yaml b/puppet/all-nodes-config.yaml
index 9dd43680..3dd3d5c9 100644
--- a/puppet/all-nodes-config.yaml
+++ b/puppet/all-nodes-config.yaml
@@ -50,6 +50,8 @@ parameters:
type: comma_delimited_list
keystone_admin_api_node_ips:
type: comma_delimited_list
+ sahara_api_node_ips:
+ type: comma_delimited_list
DeployIdentifier:
type: string
@@ -241,6 +243,14 @@ resources:
list_join:
- "','"
- {get_param: keystone_admin_api_node_ips}
+ sahara_api_node_ips:
+ str_replace:
+ template: "['SERVERS_LIST']"
+ params:
+ SERVERS_LIST:
+ list_join:
+ - "','"
+ - {get_param: sahara_api_node_ips}
# NOTE(gfidente): interpolation with %{} in the
# hieradata file can't be used as it returns string
@@ -250,6 +260,7 @@ resources:
neutron::rabbit_hosts: *rabbit_nodes_array
nova::rabbit_hosts: *rabbit_nodes_array
keystone::rabbit_hosts: *rabbit_nodes_array
+ sahara::rabbit_hosts: *rabbit_nodes_array
deploy_identifier: {get_param: DeployIdentifier}
update_identifier: {get_param: UpdateIdentifier}
diff --git a/puppet/controller.yaml b/puppet/controller.yaml
index a825f582..8a4fe21b 100644
--- a/puppet/controller.yaml
+++ b/puppet/controller.yaml
@@ -315,6 +315,14 @@ parameters:
default: 0
description: Number of workers for Keystone service.
type: number
+ SaharaApiVirtualIP:
+ type: string
+ default: ''
+ SaharaPassword:
+ default: unset
+ description: The password for the sahara service account, used by sahara-api.
+ type: string
+ hidden: true
MysqlClusterUniquePart:
description: A unique identifier of the MySQL cluster the controller is in.
type: string
@@ -822,6 +830,7 @@ resources:
server: {get_resource: Controller}
NodeIndex: {get_param: NodeIndex}
+
ControllerDeployment:
type: OS::TripleO::SoftwareDeployment
depends_on: NetworkDeployment
@@ -1089,6 +1098,15 @@ resources:
swift_mount_check: {get_param: SwiftMountCheck}
enable_package_install: {get_param: EnablePackageInstall}
enable_package_upgrade: {get_attr: [UpdateDeployment, update_managed_packages]}
+ sahara_password: {get_param: SaharaPassword}
+ sahara_dsn:
+ list_join:
+ - ''
+ - - 'mysql://sahara:'
+ - {get_param: SaharaPassword}
+ - '@'
+ - {get_param: MysqlVirtualIP}
+ - '/sahara'
swift_proxy_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftProxyNetwork]}]}
swift_management_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftMgmtNetwork]}]}
cinder_iscsi_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, CinderIscsiNetwork]}]}
@@ -1111,6 +1129,7 @@ resources:
rabbitmq_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, RabbitMqNetwork]}]}
redis_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, RedisNetwork]}]}
redis_vip: {get_param: RedisVirtualIP}
+ sahara_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
memcached_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MemcachedNetwork]}]}
mysql_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
mysql_virtual_ip: {get_param: MysqlVirtualIP}
@@ -1424,6 +1443,29 @@ resources:
horizon::bind_address: {get_input: horizon_network}
horizon::keystone_url: {get_input: keystone_auth_uri}
+ # Sahara
+ sahara::host: {get_input: sahara_api_network}
+ sahara::plugins:
+ - cdh
+ - hdp
+ - mapr
+ - vanilla
+ - spark
+ - storm
+ sahara::admin_password: {get_input: sahara_password}
+ sahara::auth_uri: {get_input: keystone_auth_uri}
+ sahara::admin_user: sahara
+ sahara::identity_uri: {get_input: keystone_identity_uri}
+ sahara::use_neutron: true
+ sahara::database_connection: {get_input: sahara_dsn}
+ sahara::debug: {get_input: debug}
+ sahara::rpc_backend: rabbit
+ sahara::rabbit_userid: {get_input: rabbit_username}
+ sahara::rabbit_password: {get_input: rabbit_password}
+ sahara::rabbit_use_ssl: {get_input: rabbit_client_use_ssl}
+ sahara::rabbit_port: {get_input: rabbit_client_port}
+ sahara::db::mysql::password: {get_input: sahara_password}
+
# Rabbit
rabbitmq::node_ip_address: {get_input: rabbitmq_network}
rabbitmq::erlang_cookie: {get_input: rabbit_cookie}
diff --git a/puppet/hieradata/controller.yaml b/puppet/hieradata/controller.yaml
index c9f3a417..2f082390 100644
--- a/puppet/hieradata/controller.yaml
+++ b/puppet/hieradata/controller.yaml
@@ -38,6 +38,7 @@ cinder::api::keystone_tenant: 'service'
swift::proxy::authtoken::admin_tenant_name: 'service'
ceilometer::api::keystone_tenant: 'service'
heat::keystone_tenant: 'service'
+sahara::admin_tenant_name: 'service'
# keystone
keystone::cron::token_flush::maxdelay: 3600
@@ -120,6 +121,7 @@ tripleo::loadbalancer::nova_metadata: true
tripleo::loadbalancer::nova_novncproxy: true
tripleo::loadbalancer::mysql: true
tripleo::loadbalancer::redis: true
+tripleo::loadbalancer::sahara: true
tripleo::loadbalancer::swift_proxy_server: true
tripleo::loadbalancer::ceilometer: true
tripleo::loadbalancer::heat_api: true
diff --git a/puppet/hieradata/database.yaml b/puppet/hieradata/database.yaml
index 7e925d90..89577505 100644
--- a/puppet/hieradata/database.yaml
+++ b/puppet/hieradata/database.yaml
@@ -53,3 +53,10 @@ ceilometer::db::mysql::dbname: ceilometer
ceilometer::db::mysql::allowed_hosts:
- '%'
- "%{hiera('mysql_bind_host')}"
+
+sahara::db::mysql::user: sahara
+sahara::db::mysql::host: "%{hiera('mysql_virtual_ip')}"
+sahara::db::mysql::dbname: sahara
+sahara::db::mysql::allowed_hosts:
+ - '%'
+ - "%{hiera('mysql_bind_host')}"
diff --git a/puppet/manifests/overcloud_controller.pp b/puppet/manifests/overcloud_controller.pp
index 913bcb63..14daf9bf 100644
--- a/puppet/manifests/overcloud_controller.pp
+++ b/puppet/manifests/overcloud_controller.pp
@@ -101,6 +101,7 @@ if hiera('step') >= 2 {
include ::neutron::db::mysql
include ::cinder::db::mysql
include ::heat::db::mysql
+ include ::sahara::db::mysql
if downcase(hiera('ceilometer_backend')) == 'mysql' {
include ::ceilometer::db::mysql
}
@@ -541,6 +542,11 @@ if hiera('step') >= 3 {
include ::heat::api_cloudwatch
include ::heat::engine
+ # Sahara
+ include ::sahara
+ include ::sahara::service::api
+ include ::sahara::service::engine
+
# Horizon
if 'cisco_n1kv' in hiera('neutron::plugins::ml2::mechanism_drivers') {
$_profile_support = 'cisco'
diff --git a/puppet/manifests/overcloud_controller_pacemaker.pp b/puppet/manifests/overcloud_controller_pacemaker.pp
index e6ee85ae..b0a9fe10 100644
--- a/puppet/manifests/overcloud_controller_pacemaker.pp
+++ b/puppet/manifests/overcloud_controller_pacemaker.pp
@@ -443,6 +443,10 @@ MYSQL_HOST=localhost\n",
require => Exec['galera-ready'],
}
}
+
+ class { '::sahara::db::mysql':
+ require => Exec['galera-ready'],
+ }
}
# pre-install swift here so we can build rings
@@ -852,6 +856,18 @@ if hiera('step') >= 3 {
enabled_backends => $cinder_enabled_backends,
}
+ class { '::sahara':
+ sync_db => $sync_db,
+ }
+ class { '::sahara::service::api':
+ manage_service => false,
+ enabled => false,
+ }
+ class { '::sahara::service::engine':
+ manage_service => false,
+ enabled => false,
+ }
+
# swift proxy
class { '::swift::proxy' :
manage_service => $non_pcmk_start,
@@ -1097,6 +1113,24 @@ if hiera('step') >= 4 {
Pacemaker::Resource::Service[$::cinder::params::volume_service]],
}
+ # Sahara
+ pacemaker::resource::service { $::sahara::params::api_service_name :
+ clone_params => 'interleave=true',
+ require => Pacemaker::Resource::Service[$::keystone::params::service_name],
+ }
+ pacemaker::resource::service { $::sahara::params::engine_service_name :
+ clone_params => 'interleave=true',
+ }
+ pacemaker::constraint::base { 'keystone-then-sahara-api-constraint':
+ constraint_type => 'order',
+ first_resource => "${::keystone::params::service_name}-clone",
+ second_resource => "${::sahara::params::api_service_name}-clone",
+ first_action => 'start',
+ second_action => 'start',
+ require => [Pacemaker::Resource::Service[$::sahara::params::api_service_name],
+ Pacemaker::Resource::Service[$::keystone::params::service_name]],
+ }
+
# Glance
pacemaker::resource::service { $::glance::params::registry_service_name :
clone_params => 'interleave=true',
diff --git a/puppet/vip-config.yaml b/puppet/vip-config.yaml
index 1dec489c..c49a1047 100644
--- a/puppet/vip-config.yaml
+++ b/puppet/vip-config.yaml
@@ -19,6 +19,7 @@ resources:
cinder_api_vip: {get_input: cinder_api_vip}
glance_api_vip: {get_input: glance_api_vip}
glance_registry_vip: {get_input: glance_registry_vip}
+ sahara_api_vip: {get_input: sahara_api_vip}
swift_proxy_vip: {get_input: swift_proxy_vip}
nova_api_vip: {get_input: nova_api_vip}
nova_metadata_vip: {get_input: nova_metadata_vip}