aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMichael Polenchuk <mpolenchuk@mirantis.com>2017-03-17 14:24:28 +0400
committerMichael Polenchuk <mpolenchuk@mirantis.com>2017-03-17 14:38:30 +0400
commit2fff9fec802617a34aa4ceac8f328d34101189c6 (patch)
treeba384b29c2eb8046ba632b7dd4fbdc5d9e79fc24
parent4b647cf926eeb373a90dee77d1dc915ae5a6c665 (diff)
Catch nodes flapping issue
Wait for nodes to become online to row down the bug https://bugs.launchpad.net/fuel/+bug/1672969 Change-Id: Ic0621a415461c1552ebfc9cd7d1ab2061ea7a8cb Signed-off-by: Michael Polenchuk <mpolenchuk@mirantis.com>
-rw-r--r--deploy/cloud/deployment.py13
1 files changed, 7 insertions, 6 deletions
diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
index 28bcfdf1d..7f791cbc6 100644
--- a/deploy/cloud/deployment.py
+++ b/deploy/cloud/deployment.py
@@ -24,6 +24,8 @@ LOG_FILE = '/var/log/puppet.log'
GREP_LINES_OF_LEADING_CONTEXT = 100
GREP_LINES_OF_TRAILING_CONTEXT = 100
LIST_OF_CHAR_TO_BE_ESCAPED = ['[', ']', '"']
+ERROR_MSGS = ['Critical nodes are not available for deployment',
+ 'offline. Remove them from environment and try again.']
class DeployNotStart(Exception):
@@ -112,8 +114,8 @@ class Deployment(object):
ready = False
timeout = False
- attempts = 0
- while attempts < 3:
+ attempts = 5
+ while attempts > 0:
try:
if time.time() > start + abort_after:
timeout = True
@@ -132,7 +134,7 @@ class Deployment(object):
time.sleep(SLEEP_TIME)
except (DeployNotStart, NodesGoOffline) as e:
log(e)
- attempts += 1
+ attempts -= 1
deploy_id = None
time.sleep(SLEEP_TIME * attempts)
@@ -164,8 +166,7 @@ class Deployment(object):
def _deployment_status(self, id):
task = self._task_fields(id)
if task['status'] == 'error':
- if task['message'].endswith(
- 'offline. Remove them from environment and try again.'):
+ if any(msg in task['message'] for msg in ERROR_MSGS):
raise NodesGoOffline(task['message'])
return task['status'], task['progress'], task['message']
@@ -190,7 +191,7 @@ class Deployment(object):
exec_cmd('rm -f /var/log/remote/fuel-snapshot-*', False)
exec_cmd('rm -f /root/deploy-*', False)
log('Generating Fuel deploy snap-shot')
- if exec_cmd('fuel snapshot < /dev/null &> snapshot.log', False)[1] <> 0:
+ if exec_cmd('fuel snapshot < /dev/null &> snapshot.log', False)[1] != 0:
log('Could not create a Fuel snapshot')
else:
exec_cmd('mv /root/fuel-snapshot* /var/log/remote/', False)