Write restart flags to restart services only when necessary

Write restart flag file for services managed by Pacemaker into
/var/lib/tripleo/pacemaker-restarts directory. The name of the file must
match the name of the clone resource defined in pacemaker. The
post-puppet restart script will restart each service having a restart
flag file and remove those files.

This approach focuses on $pacemaker_master only (we don't want to
restart the pacemaker services 3 times when we have 3 controllers), so
it relies on the assumption that we're making the matching config
changes across the pacemaker nodes.

Change-Id: I6369ab0c82dbf3c8f21043f8aa9ab810744ddc12

9 files changed, 83 insertions, 0 deletions

diff --git a/manifests/pacemaker/resource_restart_flag.pp b/manifests/pacemaker/resource_restart_flag.pp
new file mode 100644
index 0000000..aded802
--- /dev/null
+++ b/manifests/pacemaker/resource_restart_flag.pp
@@ -0,0 +1,35 @@
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == Define: tripleo::pacemaker::resource_restart_flag
+#
+# Creates a flag file on the filesystem to signify that a pacemaker
+# resource needs restarting (usually to pick up config changes after
+# they've been written on all nodes).
+#
+# === Parameters
+#
+# [*title*]
+#   The resource name in Pacemaker to restart. If it's a cloned
+#   resource, the name should include the '-clone' part.
+#
+define tripleo::pacemaker::resource_restart_flag() {
+
+  exec { "${title} resource restart flag":
+    command     => "touch /var/lib/tripleo/pacemaker-restarts/${title}",
+    path        => ['/bin','/usr/bin','/sbin','/usr/sbin'],
+    refreshonly => true,
+  }
+
+}
diff --git a/manifests/profile/base/pacemaker.pp b/manifests/profile/base/pacemaker.pp
index 2c70eab..0d628b5 100644
--- a/manifests/profile/base/pacemaker.pp
+++ b/manifests/profile/base/pacemaker.pp
@@ -75,6 +75,10 @@ class tripleo::profile::base::pacemaker (
     Pacemaker::Resource::Service {
       op_params => 'start timeout=200s stop timeout=200s',
     }
+
+    file { '/var/lib/tripleo/pacemaker-restarts':
+      ensure => directory,
+    } ~> Tripleo::Pacemaker::Resource_restart_flag<||>
   }
 
   if $step >= 2 {
diff --git a/manifests/profile/pacemaker/cinder/backup.pp b/manifests/profile/pacemaker/cinder/backup.pp
index 20a0104..72ec456 100644
--- a/manifests/profile/pacemaker/cinder/backup.pp
+++ b/manifests/profile/pacemaker/cinder/backup.pp
@@ -47,6 +47,12 @@ class tripleo::profile::pacemaker::cinder::backup (
 
   include ::tripleo::profile::base::cinder::backup
 
+  if $step >= 3 and $pacemaker_master and hiera('stack_action') == 'UPDATE' {
+    Cinder_config<||>
+    ~>
+    tripleo::pacemaker::resource_restart_flag { "${::cinder::params::backup_service}": }
+  }
+
   if $step >= 5 and $pacemaker_master {
     pacemaker::resource::service { $::cinder::params::backup_service : }
   }
diff --git a/manifests/profile/pacemaker/cinder/volume.pp b/manifests/profile/pacemaker/cinder/volume.pp
index 5a581eb..82e2522 100644
--- a/manifests/profile/pacemaker/cinder/volume.pp
+++ b/manifests/profile/pacemaker/cinder/volume.pp
@@ -46,6 +46,12 @@ class tripleo::profile::pacemaker::cinder::volume (
 
   include ::tripleo::profile::base::cinder::volume
 
+  if $step >= 3 and $pacemaker_master and hiera('stack_action') == 'UPDATE' {
+    Cinder_api_paste_ini<||> ~> Tripleo::Pacemaker::Resource_restart_flag["${::cinder::params::volume_service}"]
+    Cinder_config<||> ~> Tripleo::Pacemaker::Resource_restart_flag["${::cinder::params::volume_service}"]
+    tripleo::pacemaker::resource_restart_flag { "${::cinder::params::volume_service}": }
+  }
+
   if $step >= 5 and $pacemaker_master {
     pacemaker::resource::service { $::cinder::params::volume_service : }
   }
diff --git a/manifests/profile/pacemaker/database/mysql.pp b/manifests/profile/pacemaker/database/mysql.pp
index fceb415..4e60f2f 100644
--- a/manifests/profile/pacemaker/database/mysql.pp
+++ b/manifests/profile/pacemaker/database/mysql.pp
@@ -71,6 +71,12 @@ class tripleo::profile::pacemaker::database::mysql (
     mysql_server_options    => $mysqld_options,
   }
 
+  if $step >= 1 and $pacemaker_master and hiera('stack_action') == 'UPDATE' {
+    tripleo::pacemaker::resource_restart_flag { 'galera-master':
+      subscribe => File['mysql-config-file'],
+    }
+  }
+
   if $step >= 2 {
     if $pacemaker_master {
       pacemaker::resource::ocf { 'galera' :
diff --git a/manifests/profile/pacemaker/database/redis.pp b/manifests/profile/pacemaker/database/redis.pp
index 8a37ce9..e081516 100644
--- a/manifests/profile/pacemaker/database/redis.pp
+++ b/manifests/profile/pacemaker/database/redis.pp
@@ -44,6 +44,14 @@ class tripleo::profile::pacemaker::database::redis (
 
   if $step >= 1 {
     include ::redis
+
+    if $pacemaker_master and hiera('stack_action') == 'UPDATE' {
+      tripleo::pacemaker::resource_restart_flag { 'redis-master':
+        # ouch, but trying to stay close how notification works in
+        # puppet-redis when pacemaker is not being used
+        subscribe => Exec["cp -p ${::redis::config_file_orig} ${::redis::config_file}"]
+      }
+    }
   }
 
   if $step >= 2 and $pacemaker_master {
diff --git a/manifests/profile/pacemaker/haproxy.pp b/manifests/profile/pacemaker/haproxy.pp
index 1b83d9b..b2e127b 100644
--- a/manifests/profile/pacemaker/haproxy.pp
+++ b/manifests/profile/pacemaker/haproxy.pp
@@ -44,6 +44,12 @@ class tripleo::profile::pacemaker::haproxy (
     $pacemaker_master = false
   }
 
+  if $step >= 1 and $pacemaker_master and hiera('stack_action') == 'UPDATE' {
+    tripleo::pacemaker::resource_restart_flag { 'haproxy-clone':
+      subscribe => Concat['/etc/haproxy/haproxy.cfg'],
+    }
+  }
+
   if $step >= 2 and $pacemaker_master and $enable_load_balancer {
       # FIXME: we should not have to access tripleo::haproxy class
       # parameters here to configure pacemaker VIPs. The configuration
diff --git a/manifests/profile/pacemaker/manila.pp b/manifests/profile/pacemaker/manila.pp
index 37c67ab..a7f91fc 100644
--- a/manifests/profile/pacemaker/manila.pp
+++ b/manifests/profile/pacemaker/manila.pp
@@ -202,6 +202,12 @@ class tripleo::profile::pacemaker::manila (
     class { '::manila::backends' :
       enabled_share_backends => $manila_enabled_backends,
     }
+
+    if $pacemaker_master and hiera('stack_action') == 'UPDATE' {
+      Manila_api_paste_ini<||> ~> Tripleo::Pacemaker::Resource_restart_flag["${::manila::params::share_service}"]
+      Manila_config<||> ~> Tripleo::Pacemaker::Resource_restart_flag["${::manila::params::share_service}"]
+      tripleo::pacemaker::resource_restart_flag { "${::manila::params::share_service}": }
+    }
   }
 
   if $step >= 5 and $pacemaker_master {
diff --git a/manifests/profile/pacemaker/rabbitmq.pp b/manifests/profile/pacemaker/rabbitmq.pp
index 93edec9..1f25e8b 100644
--- a/manifests/profile/pacemaker/rabbitmq.pp
+++ b/manifests/profile/pacemaker/rabbitmq.pp
@@ -54,6 +54,12 @@ class tripleo::profile::pacemaker::rabbitmq (
     require => Class['::rabbitmq'],
   }
 
+  if $step >= 1 and $pacemaker_master and hiera('stack_action') == 'UPDATE' {
+    tripleo::pacemaker::resource_restart_flag { 'rabbitmq-clone':
+      subscribe => Class['rabbitmq::service'],
+    }
+  }
+
   if $step >= 2 and $pacemaker_master {
     pacemaker::resource::ocf { 'rabbitmq':
       ocf_agent_name  => 'heartbeat:rabbitmq-cluster',