aboutsummaryrefslogtreecommitdiffstats
path: root/deploy/cloud
diff options
context:
space:
mode:
authorJonas Bjurel <jonas.bjurel@ericsson.com>2016-05-11 13:38:31 +0200
committerJonas Bjurel <jonas.bjurel@ericsson.com>2016-06-16 10:09:34 +0000
commit1fc07d1d0ad750e6d1049f5b763320db2de1b396 (patch)
tree1528a92ca20bef6ad09721373c8bb7be5f15bbcc /deploy/cloud
parent3cddf277c5966c5cc584a493d5923cc5dcdc7820 (diff)
Introducing collection of all fuel and stack deployment logs.
The purpose of this patch is to collect all available Fuel snapshots- and stack/node ldeployment logs for later off-line troubleshooting. The intention is that Jenkins, or other deployment robots will be able to collect all logs from the deployment and store it at some repository where developers can fetch it and perform off-line post deployment trouble-shooting. Following script arguments have been added: CI Arg changes: Added an argument to ci/deploy.sh: -L [Deploy log path and file name], E.g. -L ~/jenkins/deploy/deploy-888.log.tar.gz This will create an tar gzip archive at the path and filename pointed out. If -L is not specified, the log archive will be placed under the CI directory with the following name convention: deploy-YYMMDD-HHMMSS.log.tar.gz Fuel Internal deploy changes: Added an argument to ci/deploy.py -log [Deploy log path and file name], E.g. -log ~/jenkins/deploy/deploy-888.log.tar.gz This will create an tar gzip archive at the path and filename pointed out. If -log is not specified, the log archive will be placed under the CI directory with the following name convention: deploy-YYMMDD-HHMMSS.log.tar.gz READY TO MERGE! VERIFIED! Change-Id: Icb75d9d2e66bdd47f75dcca29071943444d5c823 Signed-off-by: Jonas Bjurel <jonas.bjurel@ericsson.com>
Diffstat (limited to 'deploy/cloud')
-rw-r--r--deploy/cloud/deployment.py52
1 files changed, 38 insertions, 14 deletions
diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
index 0127d2a52..f8e1617f8 100644
--- a/deploy/cloud/deployment.py
+++ b/deploy/cloud/deployment.py
@@ -7,7 +7,6 @@
# http://www.apache.org/licenses/LICENSE-2.0
###############################################################################
-
import time
import re
@@ -16,6 +15,8 @@ from common import (
E,
exec_cmd,
run_proc,
+ run_proc_wait_terminated,
+ run_proc_kill,
parse,
err,
log,
@@ -30,6 +31,7 @@ LIST_OF_CHAR_TO_BE_ESCAPED = ['[', ']', '"']
class Deployment(object):
+
def __init__(self, dea, yaml_config_dir, env_id, node_id_roles_dict,
no_health_check, deploy_timeout):
self.dea = dea
@@ -41,6 +43,7 @@ class Deployment(object):
self.pattern = re.compile(
'\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')
+
def collect_error_logs(self):
for node_id, roles_blade in self.node_id_roles_dict.iteritems():
log_list = []
@@ -96,13 +99,14 @@ class Deployment(object):
for log_msg in log_list:
print(log_msg + '\n')
+
def run_deploy(self):
SLEEP_TIME = 60
LOG_FILE = 'cloud.log'
log('Starting deployment of environment %s' % self.env_id)
- p = run_proc('fuel --env %s deploy-changes | strings > %s'
- % (self.env_id, LOG_FILE))
+ deploy_proc = run_proc('fuel --env %s deploy-changes | strings > %s'
+ % (self.env_id, LOG_FILE))
ready = False
for i in range(int(self.deploy_timeout)):
@@ -120,19 +124,37 @@ class Deployment(object):
else:
time.sleep(SLEEP_TIME)
- p.poll()
- if p.returncode == None:
- log('The process deploying the changes has not yet finished.')
- log('''The file %s won't be deleted''' % LOG_FILE)
- else:
- delete(LOG_FILE)
+ if (env[0][E['status']] <> 'operational'
+ and env[0][E['status']] <> 'error'
+ and env[0][E['status']] <> 'stopped'):
+ err('Deployment timed out, environment %s is not operational, snapshot will not be performed'
+ % self.env_id, self.collect_logs)
+
+ run_proc_wait_terminated(deploy_proc)
+ delete(LOG_FILE)
if ready:
log('Environment %s successfully deployed' % self.env_id)
else:
self.collect_error_logs()
err('Deployment failed, environment %s is not operational'
- % self.env_id)
+ % self.env_id, self.collect_logs)
+
+
+ def collect_logs(self):
+ log('Cleaning out any previous deployment logs')
+ exec_cmd('rm -f /var/log/remote/fuel-snapshot-*', False)
+ exec_cmd('rm -f /root/deploy-*', False)
+ log('Generating Fuel deploy snap-shot')
+ if exec_cmd('fuel snapshot < /dev/null &> snapshot.log', False)[1] <> 0:
+ log('Could not create a Fuel snapshot')
+ else:
+ exec_cmd('mv /root/fuel-snapshot* /var/log/remote/', False)
+
+ log('Collecting all Fuel Snapshot & deploy log files')
+ r, _ = exec_cmd('tar -czhf /root/deploy-%s.log.tar.gz /var/log/remote' % time.strftime("%Y%m%d-%H%M%S"), False)
+ log(r)
+
def verify_node_status(self):
node_list = parse(exec_cmd('fuel node list'))
@@ -145,18 +167,20 @@ class Deployment(object):
summary = ''
for node, status in failed_nodes:
summary += '[node %s, status %s]\n' % (node, status)
- err('Deployment failed: %s' % summary)
+ err('Deployment failed: %s' % summary, self.collect_logs)
+
def health_check(self):
log('Now running sanity and smoke health checks')
- r = exec_cmd('fuel health --env %s --check sanity,smoke --force'
- % self.env_id)
+ r = exec_cmd('fuel health --env %s --check sanity,smoke --force' % self.env_id)
log(r)
if 'failure' in r:
- err('Healthcheck failed!')
+ err('Healthcheck failed!', self.collect_logs)
+
def deploy(self):
self.run_deploy()
self.verify_node_status()
if not self.no_health_check:
self.health_check()
+ self.collect_logs()