Fixes ceph key import failures

There is an issue with HA deployments where sometimes key imports fail
for Ceph which seem to occur around 50% of the time.  When logging in
after a failure, the key import seems to work which indicates it may be
a race condition.  In addition, sometimes the keyring that is created
is missing the "caps" section of the file, which will also fail import.

This patch adds a retries for a minute to try to import the key.  It
also moves creating/importing to the same Exec because there is
evidence that the file is being modified by some other process right
after the file content is created in the previous exec.

JIRA: APEX-563

Change-Id: Ie8cfeb4803f6bed95f9e612eeb37c5cdf2d76617
Signed-off-by: Tim Rozet <trozet@redhat.com>

2 files changed, 78 insertions, 0 deletions

diff --git a/build/overcloud-full.sh b/build/overcloud-full.sh
index 527e39ea..e50fc863 100755
--- a/build/overcloud-full.sh
+++ b/build/overcloud-full.sh
@@ -145,6 +145,8 @@ LIBGUESTFS_BACKEND=direct $VIRT_CUSTOMIZE \
     --install python-etcd,puppet-etcd \
     --install patch \
     --install docker,kubelet,kubeadm,kubectl,kubernetes-cni \
+    --upload ${BUILD_ROOT}/patches/puppet-ceph.patch:/etc/puppet/modules/ceph/ \
+    --run-command "cd /etc/puppet/modules/ceph && patch -p1 < puppet-ceph.patch" \
     -a overcloud-full_build.qcow2
 
     # upload and install barometer packages
diff --git a/build/patches/puppet-ceph.patch b/build/patches/puppet-ceph.patch
new file mode 100644
index 00000000..18bf9ee4
--- /dev/null
+++ b/build/patches/puppet-ceph.patch
@@ -0,0 +1,76 @@
+From 99a0bcc818ed801f6cb9e07a9904ee40e624bdab Mon Sep 17 00:00:00 2001
+From: Tim Rozet <trozet@redhat.com>
+Date: Mon, 5 Mar 2018 17:03:00 -0500
+Subject: [PATCH] Fixes ceph key import failures by adding multiple attempts
+
+Signed-off-by: Tim Rozet <trozet@redhat.com>
+---
+ manifests/key.pp | 42 +++++++++++++++++-------------------------
+ 1 file changed, 17 insertions(+), 25 deletions(-)
+
+diff --git a/manifests/key.pp b/manifests/key.pp
+index 911df1a..d47a4c3 100644
+--- a/manifests/key.pp
++++ b/manifests/key.pp
+@@ -123,22 +123,6 @@ define ceph::key (
+     }
+   }
+ 
+-  # ceph-authtool --add-key is idempotent, will just update pre-existing keys
+-  exec { "ceph-key-${name}":
+-    command   => "/bin/true # comment to satisfy puppet syntax requirements
+-set -ex
+-ceph-authtool ${keyring_path} --name '${name}' --add-key '${secret}' ${caps}",
+-    unless    => "/bin/true # comment to satisfy puppet syntax requirements
+-set -x
+-NEW_KEYRING=\$(mktemp)
+-ceph-authtool \$NEW_KEYRING --name '${name}' --add-key '${secret}' ${caps}
+-diff -N \$NEW_KEYRING ${keyring_path}
+-rv=\$?
+-rm \$NEW_KEYRING
+-exit \$rv",
+-    require   => [ File[$keyring_path], ],
+-    logoutput => true,
+-  }
+ 
+   if $inject {
+ 
+@@ -162,18 +146,26 @@ exit \$rv",
+     exec { "ceph-injectkey-${name}":
+       command   => "/bin/true # comment to satisfy puppet syntax requirements
+ set -ex
++cat ${keyring_path}
++ceph-authtool ${keyring_path} --name '${name}' --add-key '${secret}' ${caps}
++cat ${keyring_path}
+ ceph ${cluster_option} ${inject_id_option} ${inject_keyring_option} auth import -i ${keyring_path}",
+-      unless    => "/bin/true # comment to satisfy puppet syntax requirements
+-set -x
+-OLD_KEYRING=\$(mktemp)
+-ceph ${cluster_option} ${inject_id_option} ${inject_keyring_option} auth get ${name} -o \$OLD_KEYRING || true
+-diff -N \$OLD_KEYRING ${keyring_path}
+-rv=$?
+-rm \$OLD_KEYRING
+-exit \$rv",
+-      require   => [ Class['ceph'], Exec["ceph-key-${name}"], ],
++      require   => [ File[$keyring_path], Class['ceph'] ],
+       logoutput => true,
++      tries     => 6,
++      try_sleep => 10
+     }
+ 
++  } else {
++
++    # ceph-authtool --add-key is idempotent, will just update pre-existing keys
++    exec { "ceph-key-${name}":
++      command   => "/bin/true # comment to satisfy puppet syntax requirements
++set -ex
++ceph-authtool ${keyring_path} --name '${name}' --add-key '${secret}' ${caps}
++cat ${keyring_path}",
++      require   => [ File[$keyring_path], ],
++      logoutput => true,
++    }
+   }
+ }
+-- 
+2.14.3
+