diff options
author | 2017-03-17 14:24:28 +0400 | |
---|---|---|
committer | 2017-03-17 23:17:35 +0000 | |
commit | 73ef2acc09d512b338f1a1dd7c2c4abb2d3413bf (patch) | |
tree | ba384b29c2eb8046ba632b7dd4fbdc5d9e79fc24 /deploy/cloud | |
parent | a8ede12de7d762f1753ad72c6339040755c57dc9 (diff) |
Catch nodes flapping issue
Wait for nodes to become online to row down the bug
https://bugs.launchpad.net/fuel/+bug/1672969
Change-Id: Ic0621a415461c1552ebfc9cd7d1ab2061ea7a8cb
Signed-off-by: Michael Polenchuk <mpolenchuk@mirantis.com>
(cherry picked from commit 2fff9fec802617a34aa4ceac8f328d34101189c6)
Diffstat (limited to 'deploy/cloud')
-rw-r--r-- | deploy/cloud/deployment.py | 13 |
1 files changed, 7 insertions, 6 deletions
diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py index 28bcfdf1d..7f791cbc6 100644 --- a/deploy/cloud/deployment.py +++ b/deploy/cloud/deployment.py @@ -24,6 +24,8 @@ LOG_FILE = '/var/log/puppet.log' GREP_LINES_OF_LEADING_CONTEXT = 100 GREP_LINES_OF_TRAILING_CONTEXT = 100 LIST_OF_CHAR_TO_BE_ESCAPED = ['[', ']', '"'] +ERROR_MSGS = ['Critical nodes are not available for deployment', + 'offline. Remove them from environment and try again.'] class DeployNotStart(Exception): @@ -112,8 +114,8 @@ class Deployment(object): ready = False timeout = False - attempts = 0 - while attempts < 3: + attempts = 5 + while attempts > 0: try: if time.time() > start + abort_after: timeout = True @@ -132,7 +134,7 @@ class Deployment(object): time.sleep(SLEEP_TIME) except (DeployNotStart, NodesGoOffline) as e: log(e) - attempts += 1 + attempts -= 1 deploy_id = None time.sleep(SLEEP_TIME * attempts) @@ -164,8 +166,7 @@ class Deployment(object): def _deployment_status(self, id): task = self._task_fields(id) if task['status'] == 'error': - if task['message'].endswith( - 'offline. Remove them from environment and try again.'): + if any(msg in task['message'] for msg in ERROR_MSGS): raise NodesGoOffline(task['message']) return task['status'], task['progress'], task['message'] @@ -190,7 +191,7 @@ class Deployment(object): exec_cmd('rm -f /var/log/remote/fuel-snapshot-*', False) exec_cmd('rm -f /root/deploy-*', False) log('Generating Fuel deploy snap-shot') - if exec_cmd('fuel snapshot < /dev/null &> snapshot.log', False)[1] <> 0: + if exec_cmd('fuel snapshot < /dev/null &> snapshot.log', False)[1] != 0: log('Could not create a Fuel snapshot') else: exec_cmd('mv /root/fuel-snapshot* /var/log/remote/', False) |