From d69868ea9eec3d1f0a59b1fd6635a2c14df6e609 Mon Sep 17 00:00:00 2001 From: Ethan Gafford Date: Wed, 26 Aug 2015 17:43:52 -0400 Subject: Sahara Integration Integration of OpenStack data processing service (sahara) with TripleO. - Deploys sahara in distributed mode (separate api and engine processes on each controller node) - Load balancing w/haproxy - RabbitMQ/MySQL supported per current TripleO standard - Minimal configurability at this time Change-Id: I77a6a69ed5691e3b1ba34e9ebb4d88c80019642c Partially-implements: blueprint sahara-integration Depends-On: I0f0a1dc2eaa57d8226bad8cfb250110296ab9614 Depends-On: Ib84cc59667616ec94e7edce2715cbd7dd944f4ae Depends-On: I9fe321fd4284f7bfd55bd2e69dcfe623ed6f8a2a --- puppet/all-nodes-config.yaml | 11 ++++++ puppet/controller.yaml | 42 ++++++++++++++++++++++ puppet/hieradata/controller.yaml | 2 ++ puppet/hieradata/database.yaml | 7 ++++ puppet/manifests/overcloud_controller.pp | 6 ++++ puppet/manifests/overcloud_controller_pacemaker.pp | 34 ++++++++++++++++++ puppet/vip-config.yaml | 1 + 7 files changed, 103 insertions(+) (limited to 'puppet') diff --git a/puppet/all-nodes-config.yaml b/puppet/all-nodes-config.yaml index 9dd43680..3dd3d5c9 100644 --- a/puppet/all-nodes-config.yaml +++ b/puppet/all-nodes-config.yaml @@ -50,6 +50,8 @@ parameters: type: comma_delimited_list keystone_admin_api_node_ips: type: comma_delimited_list + sahara_api_node_ips: + type: comma_delimited_list DeployIdentifier: type: string @@ -241,6 +243,14 @@ resources: list_join: - "','" - {get_param: keystone_admin_api_node_ips} + sahara_api_node_ips: + str_replace: + template: "['SERVERS_LIST']" + params: + SERVERS_LIST: + list_join: + - "','" + - {get_param: sahara_api_node_ips} # NOTE(gfidente): interpolation with %{} in the # hieradata file can't be used as it returns string @@ -250,6 +260,7 @@ resources: neutron::rabbit_hosts: *rabbit_nodes_array nova::rabbit_hosts: *rabbit_nodes_array keystone::rabbit_hosts: *rabbit_nodes_array + sahara::rabbit_hosts: *rabbit_nodes_array deploy_identifier: {get_param: DeployIdentifier} update_identifier: {get_param: UpdateIdentifier} diff --git a/puppet/controller.yaml b/puppet/controller.yaml index a825f582..8a4fe21b 100644 --- a/puppet/controller.yaml +++ b/puppet/controller.yaml @@ -315,6 +315,14 @@ parameters: default: 0 description: Number of workers for Keystone service. type: number + SaharaApiVirtualIP: + type: string + default: '' + SaharaPassword: + default: unset + description: The password for the sahara service account, used by sahara-api. + type: string + hidden: true MysqlClusterUniquePart: description: A unique identifier of the MySQL cluster the controller is in. type: string @@ -822,6 +830,7 @@ resources: server: {get_resource: Controller} NodeIndex: {get_param: NodeIndex} + ControllerDeployment: type: OS::TripleO::SoftwareDeployment depends_on: NetworkDeployment @@ -1089,6 +1098,15 @@ resources: swift_mount_check: {get_param: SwiftMountCheck} enable_package_install: {get_param: EnablePackageInstall} enable_package_upgrade: {get_attr: [UpdateDeployment, update_managed_packages]} + sahara_password: {get_param: SaharaPassword} + sahara_dsn: + list_join: + - '' + - - 'mysql://sahara:' + - {get_param: SaharaPassword} + - '@' + - {get_param: MysqlVirtualIP} + - '/sahara' swift_proxy_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftProxyNetwork]}]} swift_management_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftMgmtNetwork]}]} cinder_iscsi_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, CinderIscsiNetwork]}]} @@ -1111,6 +1129,7 @@ resources: rabbitmq_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, RabbitMqNetwork]}]} redis_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, RedisNetwork]}]} redis_vip: {get_param: RedisVirtualIP} + sahara_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]} memcached_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MemcachedNetwork]}]} mysql_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]} mysql_virtual_ip: {get_param: MysqlVirtualIP} @@ -1424,6 +1443,29 @@ resources: horizon::bind_address: {get_input: horizon_network} horizon::keystone_url: {get_input: keystone_auth_uri} + # Sahara + sahara::host: {get_input: sahara_api_network} + sahara::plugins: + - cdh + - hdp + - mapr + - vanilla + - spark + - storm + sahara::admin_password: {get_input: sahara_password} + sahara::auth_uri: {get_input: keystone_auth_uri} + sahara::admin_user: sahara + sahara::identity_uri: {get_input: keystone_identity_uri} + sahara::use_neutron: true + sahara::database_connection: {get_input: sahara_dsn} + sahara::debug: {get_input: debug} + sahara::rpc_backend: rabbit + sahara::rabbit_userid: {get_input: rabbit_username} + sahara::rabbit_password: {get_input: rabbit_password} + sahara::rabbit_use_ssl: {get_input: rabbit_client_use_ssl} + sahara::rabbit_port: {get_input: rabbit_client_port} + sahara::db::mysql::password: {get_input: sahara_password} + # Rabbit rabbitmq::node_ip_address: {get_input: rabbitmq_network} rabbitmq::erlang_cookie: {get_input: rabbit_cookie} diff --git a/puppet/hieradata/controller.yaml b/puppet/hieradata/controller.yaml index c9f3a417..2f082390 100644 --- a/puppet/hieradata/controller.yaml +++ b/puppet/hieradata/controller.yaml @@ -38,6 +38,7 @@ cinder::api::keystone_tenant: 'service' swift::proxy::authtoken::admin_tenant_name: 'service' ceilometer::api::keystone_tenant: 'service' heat::keystone_tenant: 'service' +sahara::admin_tenant_name: 'service' # keystone keystone::cron::token_flush::maxdelay: 3600 @@ -120,6 +121,7 @@ tripleo::loadbalancer::nova_metadata: true tripleo::loadbalancer::nova_novncproxy: true tripleo::loadbalancer::mysql: true tripleo::loadbalancer::redis: true +tripleo::loadbalancer::sahara: true tripleo::loadbalancer::swift_proxy_server: true tripleo::loadbalancer::ceilometer: true tripleo::loadbalancer::heat_api: true diff --git a/puppet/hieradata/database.yaml b/puppet/hieradata/database.yaml index 7e925d90..89577505 100644 --- a/puppet/hieradata/database.yaml +++ b/puppet/hieradata/database.yaml @@ -53,3 +53,10 @@ ceilometer::db::mysql::dbname: ceilometer ceilometer::db::mysql::allowed_hosts: - '%' - "%{hiera('mysql_bind_host')}" + +sahara::db::mysql::user: sahara +sahara::db::mysql::host: "%{hiera('mysql_virtual_ip')}" +sahara::db::mysql::dbname: sahara +sahara::db::mysql::allowed_hosts: + - '%' + - "%{hiera('mysql_bind_host')}" diff --git a/puppet/manifests/overcloud_controller.pp b/puppet/manifests/overcloud_controller.pp index 913bcb63..14daf9bf 100644 --- a/puppet/manifests/overcloud_controller.pp +++ b/puppet/manifests/overcloud_controller.pp @@ -101,6 +101,7 @@ if hiera('step') >= 2 { include ::neutron::db::mysql include ::cinder::db::mysql include ::heat::db::mysql + include ::sahara::db::mysql if downcase(hiera('ceilometer_backend')) == 'mysql' { include ::ceilometer::db::mysql } @@ -541,6 +542,11 @@ if hiera('step') >= 3 { include ::heat::api_cloudwatch include ::heat::engine + # Sahara + include ::sahara + include ::sahara::service::api + include ::sahara::service::engine + # Horizon if 'cisco_n1kv' in hiera('neutron::plugins::ml2::mechanism_drivers') { $_profile_support = 'cisco' diff --git a/puppet/manifests/overcloud_controller_pacemaker.pp b/puppet/manifests/overcloud_controller_pacemaker.pp index e6ee85ae..b0a9fe10 100644 --- a/puppet/manifests/overcloud_controller_pacemaker.pp +++ b/puppet/manifests/overcloud_controller_pacemaker.pp @@ -443,6 +443,10 @@ MYSQL_HOST=localhost\n", require => Exec['galera-ready'], } } + + class { '::sahara::db::mysql': + require => Exec['galera-ready'], + } } # pre-install swift here so we can build rings @@ -852,6 +856,18 @@ if hiera('step') >= 3 { enabled_backends => $cinder_enabled_backends, } + class { '::sahara': + sync_db => $sync_db, + } + class { '::sahara::service::api': + manage_service => false, + enabled => false, + } + class { '::sahara::service::engine': + manage_service => false, + enabled => false, + } + # swift proxy class { '::swift::proxy' : manage_service => $non_pcmk_start, @@ -1097,6 +1113,24 @@ if hiera('step') >= 4 { Pacemaker::Resource::Service[$::cinder::params::volume_service]], } + # Sahara + pacemaker::resource::service { $::sahara::params::api_service_name : + clone_params => 'interleave=true', + require => Pacemaker::Resource::Service[$::keystone::params::service_name], + } + pacemaker::resource::service { $::sahara::params::engine_service_name : + clone_params => 'interleave=true', + } + pacemaker::constraint::base { 'keystone-then-sahara-api-constraint': + constraint_type => 'order', + first_resource => "${::keystone::params::service_name}-clone", + second_resource => "${::sahara::params::api_service_name}-clone", + first_action => 'start', + second_action => 'start', + require => [Pacemaker::Resource::Service[$::sahara::params::api_service_name], + Pacemaker::Resource::Service[$::keystone::params::service_name]], + } + # Glance pacemaker::resource::service { $::glance::params::registry_service_name : clone_params => 'interleave=true', diff --git a/puppet/vip-config.yaml b/puppet/vip-config.yaml index 1dec489c..c49a1047 100644 --- a/puppet/vip-config.yaml +++ b/puppet/vip-config.yaml @@ -19,6 +19,7 @@ resources: cinder_api_vip: {get_input: cinder_api_vip} glance_api_vip: {get_input: glance_api_vip} glance_registry_vip: {get_input: glance_registry_vip} + sahara_api_vip: {get_input: sahara_api_vip} swift_proxy_vip: {get_input: swift_proxy_vip} nova_api_vip: {get_input: nova_api_vip} nova_metadata_vip: {get_input: nova_metadata_vip} -- cgit 1.2.3-korg