summaryrefslogtreecommitdiffstats
path: root/build
diff options
context:
space:
mode:
authorTim Rozet <trozet@redhat.com>2018-03-05 17:08:03 -0500
committerTim Rozet <trozet@redhat.com>2018-03-09 09:34:10 -0500
commit32550560a1b6e00565db8d995c84f304d4cb9893 (patch)
treec9b1f4e7dbd73a9cee34718d6e42ddd7365bd36f /build
parent3ed11a41b3aa792fc11e79b010f2366eb94f9d49 (diff)
Fixes ceph key import failures
There is an issue with HA deployments where sometimes key imports fail for Ceph which seem to occur around 50% of the time. When logging in after a failure, the key import seems to work which indicates it may be a race condition. In addition, sometimes the keyring that is created is missing the "caps" section of the file, which will also fail import. This patch adds a retries for a minute to try to import the key. It also moves creating/importing to the same Exec because there is evidence that the file is being modified by some other process right after the file content is created in the previous exec. JIRA: APEX-563 Change-Id: Ie8cfeb4803f6bed95f9e612eeb37c5cdf2d76617 Signed-off-by: Tim Rozet <trozet@redhat.com>
Diffstat (limited to 'build')
-rwxr-xr-xbuild/overcloud-full.sh2
-rw-r--r--build/patches/puppet-ceph.patch76
2 files changed, 78 insertions, 0 deletions
diff --git a/build/overcloud-full.sh b/build/overcloud-full.sh
index 527e39ea..e50fc863 100755
--- a/build/overcloud-full.sh
+++ b/build/overcloud-full.sh
@@ -145,6 +145,8 @@ LIBGUESTFS_BACKEND=direct $VIRT_CUSTOMIZE \
--install python-etcd,puppet-etcd \
--install patch \
--install docker,kubelet,kubeadm,kubectl,kubernetes-cni \
+ --upload ${BUILD_ROOT}/patches/puppet-ceph.patch:/etc/puppet/modules/ceph/ \
+ --run-command "cd /etc/puppet/modules/ceph && patch -p1 < puppet-ceph.patch" \
-a overcloud-full_build.qcow2
# upload and install barometer packages
diff --git a/build/patches/puppet-ceph.patch b/build/patches/puppet-ceph.patch
new file mode 100644
index 00000000..18bf9ee4
--- /dev/null
+++ b/build/patches/puppet-ceph.patch
@@ -0,0 +1,76 @@
+From 99a0bcc818ed801f6cb9e07a9904ee40e624bdab Mon Sep 17 00:00:00 2001
+From: Tim Rozet <trozet@redhat.com>
+Date: Mon, 5 Mar 2018 17:03:00 -0500
+Subject: [PATCH] Fixes ceph key import failures by adding multiple attempts
+
+Signed-off-by: Tim Rozet <trozet@redhat.com>
+---
+ manifests/key.pp | 42 +++++++++++++++++-------------------------
+ 1 file changed, 17 insertions(+), 25 deletions(-)
+
+diff --git a/manifests/key.pp b/manifests/key.pp
+index 911df1a..d47a4c3 100644
+--- a/manifests/key.pp
++++ b/manifests/key.pp
+@@ -123,22 +123,6 @@ define ceph::key (
+ }
+ }
+
+- # ceph-authtool --add-key is idempotent, will just update pre-existing keys
+- exec { "ceph-key-${name}":
+- command => "/bin/true # comment to satisfy puppet syntax requirements
+-set -ex
+-ceph-authtool ${keyring_path} --name '${name}' --add-key '${secret}' ${caps}",
+- unless => "/bin/true # comment to satisfy puppet syntax requirements
+-set -x
+-NEW_KEYRING=\$(mktemp)
+-ceph-authtool \$NEW_KEYRING --name '${name}' --add-key '${secret}' ${caps}
+-diff -N \$NEW_KEYRING ${keyring_path}
+-rv=\$?
+-rm \$NEW_KEYRING
+-exit \$rv",
+- require => [ File[$keyring_path], ],
+- logoutput => true,
+- }
+
+ if $inject {
+
+@@ -162,18 +146,26 @@ exit \$rv",
+ exec { "ceph-injectkey-${name}":
+ command => "/bin/true # comment to satisfy puppet syntax requirements
+ set -ex
++cat ${keyring_path}
++ceph-authtool ${keyring_path} --name '${name}' --add-key '${secret}' ${caps}
++cat ${keyring_path}
+ ceph ${cluster_option} ${inject_id_option} ${inject_keyring_option} auth import -i ${keyring_path}",
+- unless => "/bin/true # comment to satisfy puppet syntax requirements
+-set -x
+-OLD_KEYRING=\$(mktemp)
+-ceph ${cluster_option} ${inject_id_option} ${inject_keyring_option} auth get ${name} -o \$OLD_KEYRING || true
+-diff -N \$OLD_KEYRING ${keyring_path}
+-rv=$?
+-rm \$OLD_KEYRING
+-exit \$rv",
+- require => [ Class['ceph'], Exec["ceph-key-${name}"], ],
++ require => [ File[$keyring_path], Class['ceph'] ],
+ logoutput => true,
++ tries => 6,
++ try_sleep => 10
+ }
+
++ } else {
++
++ # ceph-authtool --add-key is idempotent, will just update pre-existing keys
++ exec { "ceph-key-${name}":
++ command => "/bin/true # comment to satisfy puppet syntax requirements
++set -ex
++ceph-authtool ${keyring_path} --name '${name}' --add-key '${secret}' ${caps}
++cat ${keyring_path}",
++ require => [ File[$keyring_path], ],
++ logoutput => true,
++ }
+ }
+ }
+--
+2.14.3
+