Add Galera as a Pacemaker resource when EnablePacemaker

This commit aims to support the creation of the galera cluster via
Pacemaker. With this commit in, three use-cases will be supported.

 * Non HA setup / Non Pacemaker setup : The deployment will take place
as it is currently the case in f20puppet-nonha. Nothing changes.

 * Non HA setup / Pacemaker setup : Even though it is a non ha setup,
galera cluster via pacemaker  will be deployed with a cluster nbr of 1.

 * HA setup / Non Pacemaker setup : N/A

 * HA setup / Pacemaker setup : It is assumed that HA setup will
always be with pacemaker.  So in this situation pacemaker will deploy a
cluster of 3 galera master nodes.

Depends-On: I7aed9acec11486e0f4f67e4d522727476c767d83
Change-Id: If0c37a86fa8b5aa6d452129bccf7341a3a3ba667

7 files changed, 207 insertions, 72 deletions

diff --git a/controller.yaml b/controller.yaml
index d7ba1a10..bbac3a68 100644
--- a/controller.yaml
+++ b/controller.yaml
@@ -208,6 +208,10 @@ parameters:
     description: Keystone key for signing tokens.
     type: string
     hidden: true
+  MysqlClustercheckPassword:
+    type: string
+    hidden: true
+    default: ''  # Has to be here because of the ignored empty value bug
   MysqlClusterUniquePart:
     description: A unique identifier of the MySQL cluster the controller is in.
     type: string
diff --git a/overcloud-without-mergepy.yaml b/overcloud-without-mergepy.yaml
index 70e197d2..d3f8c59d 100644
--- a/overcloud-without-mergepy.yaml
+++ b/overcloud-without-mergepy.yaml
@@ -598,6 +598,7 @@ resources:
           MysqlClusterUniquePart: {get_attr: [MysqlClusterUniquePart, value]}
           MysqlInnodbBufferPoolSize: {get_param: MysqlInnodbBufferPoolSize}
           MysqlRootPassword: {get_attr: [MysqlRootPassword, value]}
+          MysqlClustercheckPassword: {get_attr: [MysqlClustercheckPassword, value]}
           NeutronPublicInterfaceIP: {get_param: NeutronPublicInterfaceIP}
           NeutronFlatNetworks: {get_param: NeutronFlatNetworks}
           NeutronBridgeMappings: {get_param: NeutronBridgeMappings}
@@ -760,6 +761,11 @@ resources:
     properties:
       length: 10
 
+  MysqlClustercheckPassword:
+    type: OS::Heat::RandomString
+    properties:
+      length: 10
+
   MysqlClusterUniquePart:
     type: OS::Heat::RandomString
     properties:
diff --git a/puppet/all-nodes-config.yaml b/puppet/all-nodes-config.yaml
index 35bd4196..c6dcaecb 100644
--- a/puppet/all-nodes-config.yaml
+++ b/puppet/all-nodes-config.yaml
@@ -54,6 +54,14 @@ resources:
                   list_join:
                   - ','
                   - {get_param: controller_ips}
+                controller_node_names:
+                  list_join:
+                  - ','
+                  - {get_param: controller_names}
+                galera_node_names:
+                  list_join:
+                  - ','
+                  - {get_param: controller_names}
                 rabbit_node_ips:
                   list_join:
                   - ','
diff --git a/puppet/controller-post-puppet.yaml b/puppet/controller-post-puppet.yaml
index 361e6a43..7aab7f5a 100644
--- a/puppet/controller-post-puppet.yaml
+++ b/puppet/controller-post-puppet.yaml
@@ -63,10 +63,19 @@ resources:
       input_values:
         step: 3
 
+  ControllerDeploymentOvercloudServices_Step5:
+    type: OS::Heat::StructuredDeployments
+    depends_on: ControllerDeploymentOvercloudServices_Step4
+    properties:
+      servers:  {get_param: servers}
+      config: {get_resource: ControllerPuppetConfig}
+      input_values:
+        step: 4
+
   # Note, this should come last, so use depends_on to ensure
   # this is created after any other resources.
   ExtraConfig:
-    depends_on: ControllerDeploymentOvercloudServices_Step4
+    depends_on: ControllerDeploymentOvercloudServices_Step5
     type: OS::TripleO::NodeExtraConfigPost
     properties:
         servers: {get_param: servers}
diff --git a/puppet/controller-puppet.yaml b/puppet/controller-puppet.yaml
index 64e7fc72..09a26d58 100644
--- a/puppet/controller-puppet.yaml
+++ b/puppet/controller-puppet.yaml
@@ -226,6 +226,10 @@ parameters:
     type: string
     hidden: true
     default: ''  # Has to be here because of the ignored empty value bug
+  MysqlClustercheckPassword:
+    type: string
+    hidden: true
+    default: ''  # Has to be here because of the ignored empty value bug
   NeutronBridgeMappings:
     description: >
       The OVS logical->physical bridge mappings to use. See the Neutron
@@ -553,6 +557,7 @@ resources:
         enable_swift_storage: {get_param: EnableSwiftStorage}
         mysql_innodb_buffer_pool_size: {get_param: MysqlInnodbBufferPoolSize}
         mysql_root_password: {get_param: MysqlRootPassword}
+        mysql_clustercheck_password: {get_param: MysqlClustercheckPassword}
         mysql_cluster_name:
           str_replace:
             template: tripleo-CLUSTER
@@ -775,6 +780,7 @@ resources:
                 mysql_innodb_buffer_pool_size: {get_input: mysql_innodb_buffer_pool_size}
                 mysql::server::root_password: {get_input: mysql_root_password}
                 mysql_cluster_name: {get_input: mysql_cluster_name}
+                mysql_clustercheck_password: {get_input: mysql_clustercheck_password}
 
                 # Neutron
                 neutron::bind_host: {get_input: controller_host}
diff --git a/puppet/hieradata/controller.yaml b/puppet/hieradata/controller.yaml
index 76488457..69e62d9e 100644
--- a/puppet/hieradata/controller.yaml
+++ b/puppet/hieradata/controller.yaml
@@ -84,6 +84,10 @@ horizon::allowed_hosts: '*'
 
 mysql::server::manage_config_file: true
 
+
+tripleo::loadbalancer::galera_master_ip: "%{hiera('bootstrap_nodeid_ip')}"
+tripleo::loadbalancer::galera_master_hostname: "%{hiera('bootstrap_nodeid')}"
+
 tripleo::loadbalancer::keystone_admin: true
 tripleo::loadbalancer::keystone_public: true
 tripleo::loadbalancer::neutron: true
diff --git a/puppet/manifests/overcloud_controller_pacemaker.pp b/puppet/manifests/overcloud_controller_pacemaker.pp
index ba5f80a6..7f180295 100644
--- a/puppet/manifests/overcloud_controller_pacemaker.pp
+++ b/puppet/manifests/overcloud_controller_pacemaker.pp
@@ -33,10 +33,11 @@ if $::hostname == downcase(hiera('bootstrap_nodeid')) {
 if hiera('step') >= 1 {
 
   $controller_node_ips = split(hiera('controller_node_ips'), ',')
-
+  $controller_node_names = split(downcase(hiera('controller_node_names')), ',')
   class { '::tripleo::loadbalancer' :
-    controller_hosts => $controller_node_ips,
-    manage_vip       => false,
+    controller_hosts       => $controller_node_ips,
+    controller_hosts_names => $controller_node_names,
+    manage_vip             => false,
   }
 
   $pacemaker_cluster_members = regsubst(hiera('controller_node_ips'), ',', ' ', 'G')
@@ -140,76 +141,163 @@ if hiera('step') >= 2 {
   } else {
     $mysql_config_file = '/etc/my.cnf.d/server.cnf'
   }
-  # TODO Galara
-  class { 'mysql::server':
-    config_file => $mysql_config_file,
-    override_options => {
-      'mysqld' => {
-        'bind-address' => hiera('controller_host')
-      }
+  $galera_nodes = downcase(hiera('galera_node_names', $::hostname))
+  $galera_nodes_count = count(split($galera_nodes, ','))
+  $clustercheck_password = hiera('mysql_clustercheck_password')
+  $mysql_root_password = hiera('mysql::server::root_password')
+
+  $mysqld_options = {
+    'mysqld' => {
+      'skip-name-resolve'             => '1',
+      'binlog_format'                 => 'ROW',
+      'default-storage-engine'        => 'innodb',
+      'innodb_autoinc_lock_mode'      => '2',
+      'innodb_locks_unsafe_for_binlog'=> '1',
+      'query_cache_size'              => '0',
+      'query_cache_type'              => '0',
+      'bind-address'                  => hiera('controller_host'),
+      'wsrep_provider'                => '/usr/lib64/galera/libgalera_smm.so',
+      'wsrep_cluster_name'            => 'galera_cluster',
+      'wsrep_slave_threads'           => '1',
+      'wsrep_certify_nonPK'           => '1',
+      'wsrep_max_ws_rows'             => '131072',
+      'wsrep_max_ws_size'             => '1073741824',
+      'wsrep_debug'                   => '0',
+      'wsrep_convert_LOCK_to_trx'     => '0',
+      'wsrep_retry_autocommit'        => '1',
+      'wsrep_auto_increment_control'  => '1',
+      'wsrep_drupal_282555_workaround'=> '0',
+      'wsrep_causal_reads'            => '0',
+      'wsrep_notify_cmd'              => '',
+      'wsrep_sst_method'              => 'rsync',
+    }
+  }
+
+  class { '::mysql::server':
+    create_root_user   => false,
+    create_root_my_cnf => false,
+    config_file        => $mysql_config_file,
+    override_options   => $mysqld_options,
+    service_manage     => false,
+  }
+
+  if $pacemaker_master {
+    $sync_db = true
+
+    pacemaker::resource::ocf { 'galera' :
+      resource_name => 'heartbeat:galera',
+      options       => "enable_creation=true wsrep_cluster_address='gcomm://${galera_nodes}' meta master-max=${galera_nodes_count} ordered=true op promote timeout=300s on-fail=block --master",
+      require       => Class['::mysql::server'],
+      before        => Exec['galera-ready'],
     }
+
+    mysql_user { 'clustercheckuser@localhost' :
+      password_hash => mysql_password($clustercheck_password),
+      require       => Exec['galera-ready'],
+    }
+  } else {
+    $sync_db = false
+  }
+
+  exec { 'galera-ready' :
+    command     => '/bin/mysql -e "SHOW GLOBAL VARIABLES LIKE \'read_only\'" | /bin/grep -i off',
+    timeout     => 3600,
+    tries       => 60,
+    try_sleep   => 60,
+    environment => 'HOME=/root',
+    require     => Class['::mysql::server'],
+  }
+
+  file { '/etc/sysconfig/clustercheck' :
+    ensure  => file,
+    content => "MYSQL_USERNAME=clustercheckuser\n
+MYSQL_PASSWORD=${clustercheck_password}\n
+MYSQL_HOST=localhost\n",
+    require       => Exec['galera-ready'],
+  }
+
+  xinetd::service { 'galera-monitor' :
+    port           => '9200',
+    server         => '/usr/bin/clustercheck',
+    per_source     => 'UNLIMITED',
+    log_on_success => '',
+    log_on_failure => 'HOST',
+    flags          => 'REUSE',
+    service_type   => 'UNLISTED',
+    user           => 'root',
+    group          => 'root',
+    require        => File['/etc/sysconfig/clustercheck'],
   }
 
   # FIXME: this should only occur on the bootstrap host (ditto for db syncs)
   # Create all the database schemas
   # Example DSN format: mysql://user:password@host/dbname
-  $allowed_hosts = ['%',hiera('controller_host')]
-  $keystone_dsn = split(hiera('keystone::database_connection'), '[@:/?]')
-  class { 'keystone::db::mysql':
-    user          => $keystone_dsn[3],
-    password      => $keystone_dsn[4],
-    host          => $keystone_dsn[5],
-    dbname        => $keystone_dsn[6],
-    allowed_hosts => $allowed_hosts,
-  }
-  $glance_dsn = split(hiera('glance::api::database_connection'), '[@:/?]')
-  class { 'glance::db::mysql':
-    user          => $glance_dsn[3],
-    password      => $glance_dsn[4],
-    host          => $glance_dsn[5],
-    dbname        => $glance_dsn[6],
-    allowed_hosts => $allowed_hosts,
-  }
-  $nova_dsn = split(hiera('nova::database_connection'), '[@:/?]')
-  class { 'nova::db::mysql':
-    user          => $nova_dsn[3],
-    password      => $nova_dsn[4],
-    host          => $nova_dsn[5],
-    dbname        => $nova_dsn[6],
-    allowed_hosts => $allowed_hosts,
-  }
-  $neutron_dsn = split(hiera('neutron::server::database_connection'), '[@:/?]')
-  class { 'neutron::db::mysql':
-    user          => $neutron_dsn[3],
-    password      => $neutron_dsn[4],
-    host          => $neutron_dsn[5],
-    dbname        => $neutron_dsn[6],
-    allowed_hosts => $allowed_hosts,
-  }
-  $cinder_dsn = split(hiera('cinder::database_connection'), '[@:/?]')
-  class { 'cinder::db::mysql':
-    user          => $cinder_dsn[3],
-    password      => $cinder_dsn[4],
-    host          => $cinder_dsn[5],
-    dbname        => $cinder_dsn[6],
-    allowed_hosts => $allowed_hosts,
-  }
-  $heat_dsn = split(hiera('heat::database_connection'), '[@:/?]')
-  class { 'heat::db::mysql':
-    user          => $heat_dsn[3],
-    password      => $heat_dsn[4],
-    host          => $heat_dsn[5],
-    dbname        => $heat_dsn[6],
-    allowed_hosts => $allowed_hosts,
-  }
-  if downcase(hiera('ceilometer_backend')) == 'mysql' {
-    $ceilometer_dsn = split(hiera('ceilometer_mysql_conn_string'), '[@:/?]')
-    class { 'ceilometer::db::mysql':
-      user          => $ceilometer_dsn[3],
-      password      => $ceilometer_dsn[4],
-      host          => $ceilometer_dsn[5],
-      dbname        => $ceilometer_dsn[6],
+  if $sync_db {
+    $allowed_hosts = ['%',hiera('controller_host')]
+    $keystone_dsn = split(hiera('keystone::database_connection'), '[@:/?]')
+    class { 'keystone::db::mysql':
+      user          => $keystone_dsn[3],
+      password      => $keystone_dsn[4],
+      host          => $keystone_dsn[5],
+      dbname        => $keystone_dsn[6],
+      allowed_hosts => $allowed_hosts,
+      require       => Exec['galera-ready'],
+    }
+    $glance_dsn = split(hiera('glance::api::database_connection'), '[@:/?]')
+    class { 'glance::db::mysql':
+      user          => $glance_dsn[3],
+      password      => $glance_dsn[4],
+      host          => $glance_dsn[5],
+      dbname        => $glance_dsn[6],
+      allowed_hosts => $allowed_hosts,
+      require       => Exec['galera-ready'],
+    }
+    $nova_dsn = split(hiera('nova::database_connection'), '[@:/?]')
+    class { 'nova::db::mysql':
+      user          => $nova_dsn[3],
+      password      => $nova_dsn[4],
+      host          => $nova_dsn[5],
+      dbname        => $nova_dsn[6],
+      allowed_hosts => $allowed_hosts,
+      require       => Exec['galera-ready'],
+    }
+    $neutron_dsn = split(hiera('neutron::server::database_connection'), '[@:/?]')
+    class { 'neutron::db::mysql':
+      user          => $neutron_dsn[3],
+      password      => $neutron_dsn[4],
+      host          => $neutron_dsn[5],
+      dbname        => $neutron_dsn[6],
+      allowed_hosts => $allowed_hosts,
+      require       => Exec['galera-ready'],
+    }
+    $cinder_dsn = split(hiera('cinder::database_connection'), '[@:/?]')
+    class { 'cinder::db::mysql':
+      user          => $cinder_dsn[3],
+      password      => $cinder_dsn[4],
+      host          => $cinder_dsn[5],
+      dbname        => $cinder_dsn[6],
       allowed_hosts => $allowed_hosts,
+      require       => Exec['galera-ready'],
+    }
+    $heat_dsn = split(hiera('heat::database_connection'), '[@:/?]')
+    class { 'heat::db::mysql':
+      user          => $heat_dsn[3],
+      password      => $heat_dsn[4],
+      host          => $heat_dsn[5],
+      dbname        => $heat_dsn[6],
+      allowed_hosts => $allowed_hosts,
+      require       => Exec['galera-ready'],
+    }
+    if downcase(hiera('ceilometer_backend')) == 'mysql' {
+      $ceilometer_dsn = split(hiera('ceilometer_mysql_conn_string'), '[@:/?]')
+      class { 'ceilometer::db::mysql':
+        user          => $ceilometer_dsn[3],
+        password      => $ceilometer_dsn[4],
+        host          => $ceilometer_dsn[5],
+        dbname        => $ceilometer_dsn[6],
+        allowed_hosts => $allowed_hosts,
+        require       => Exec['galera-ready'],
+      }
     }
   }
 
@@ -257,7 +345,8 @@ if hiera('step') >= 2 {
 
 } #END STEP 2
 
-if hiera('step') >= 3 {
+if (hiera('step') >= 3 and $::hostname == downcase(hiera('bootstrap_nodeid')))
+   or hiera('step') >= 4 {
 
   include ::keystone
 
@@ -306,14 +395,18 @@ if hiera('step') >= 3 {
   class { 'glance::api':
     known_stores => [$glance_store]
   }
-  include ::glance::registry
+  class { '::glance::registry' :
+    sync_db => $sync_db,
+  }
   include join(['::glance::backend::', $glance_backend])
 
   class { 'nova':
     glance_api_servers     => join([hiera('glance_protocol'), '://', hiera('controller_virtual_ip'), ':', hiera('glance_port')]),
   }
 
-  include ::nova::api
+  class { '::nova::api' :
+    sync_db => $sync_db,
+  }
   include ::nova::cert
   include ::nova::conductor
   include ::nova::consoleauth
@@ -322,7 +415,9 @@ if hiera('step') >= 3 {
   include ::nova::scheduler
 
   include ::neutron
-  include ::neutron::server
+  class { '::neutron::server' :
+    sync_db => $sync_db,
+  }
   include ::neutron::agents::dhcp
   include ::neutron::agents::l3
 
@@ -458,6 +553,7 @@ if hiera('step') >= 3 {
   include ::ceilometer::collector
   class { '::ceilometer::db' :
     database_connection => $ceilometer_database_connection,
+    sync_db             => $sync_db,
   }
   class { 'ceilometer::agent::auth':
     auth_url => join(['http://', hiera('controller_virtual_ip'), ':5000/v2.0']),
@@ -466,7 +562,9 @@ if hiera('step') >= 3 {
   Cron <| title == 'ceilometer-expirer' |> { command => "sleep $((\$(od -A n -t d -N 3 /dev/urandom) % 86400)) && ${::ceilometer::params::expirer_command}" }
 
   # Heat
-  include ::heat
+  class { '::heat' :
+    sync_db => $sync_db,
+  }
   include ::heat::api
   include ::heat::api_cfn
   include ::heat::api_cloudwatch