Merge "Introducing collection of all fuel and stack deployment logs."

7 files changed, 95 insertions, 34 deletions

diff --git a/ci/deploy.sh b/ci/deploy.sh
index dc13f1c77..c7a1d1858 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -43,6 +43,7 @@ OPTIONS:
   -h  Print this message and exit
   -H  No health check
   -l  Lab-name
+  -L  Deployment log path and file name
   -p  Pod-name
   -s  Deploy-scenario short-name/base-file-name
   -S  Storage dir for VM images
@@ -66,6 +67,7 @@ Input parameters to the build script is:
 -h Print this message and exit
 -H Do not run fuel built in health-check after successfull deployment
 -l Lab name as defined in the configuration directory, e.g. lf
+-L Deployment log path and name, eg. -L /home/jenkins/logs/job888.log.tar.gz
 -p POD name as defined in the configuration directory, e.g. pod-1
 -s Deployment-scenario, this points to a deployment/test scenario file as
    defined in the configuration directory:
@@ -118,7 +120,7 @@ DRY_RUN=0
 ############################################################################
 # BEGIN of main
 #
-while getopts "b:B:dfFHl:p:s:S:i:he" OPTION
+while getopts "b:B:dfFHl:L:p:s:S:i:he" OPTION
 do
     case $OPTION in
         b)
@@ -155,6 +157,9 @@ do
         l)
             TARGET_LAB=${OPTARG}
             ;;
+        L)
+            DEPLOY_LOG="-log ${OPTARG}"
+            ;;
         p)
             TARGET_POD=${OPTARG}
             ;;
@@ -235,8 +240,8 @@ if [ $DRY_RUN -eq 0 ]; then
         ISO=${SCRIPT_PATH}/ISO/image.iso
     fi
     # Start deployment
-    echo "python deploy.py $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO"
-    python deploy.py $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO
+    echo "python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO"
+    python deploy.py $DEPLOY_LOG $STORAGE_DIR $PXE_BRIDGE $USE_EXISTING_FUEL $FUEL_CREATION_ONLY $NO_HEALTH_CHECK $NO_DEPLOY_ENVIRONMENT -dea ${SCRIPT_PATH}/config/dea.yaml -dha ${SCRIPT_PATH}/config/dha.yaml -iso $ISO
 fi
 popd > /dev/null
 
diff --git a/deploy/README b/deploy/README
index 773b1c78f..40f95ef92 100644
--- a/deploy/README
+++ b/deploy/README
@@ -84,41 +84,41 @@ optional arguments:
   -np                   Do not install Fuel Plugins
   -dt DEPLOY_TIMEOUT    Deployment timeout (in minutes) [default: 240]
   -nde                  Do not launch environment deployment
-
+  -log [LOG_FILE]      Deployment log path and file name
 
 * EXAMPLES:
 
 - Install Fuel Master and deploy OPNFV Cloud from scratch on Hardware Environment:
 
-    sudo python deploy.py -iso ~/ISO/opnfv.iso -dea ~/CONF/hardware/dea.yaml -dha ~/CONF/hardware/dha.yaml -s /mnt/images -b pxebr
+    sudo python deploy.py -iso ~/ISO/opnfv.iso -dea ~/CONF/hardware/dea.yaml -dha ~/CONF/hardware/dha.yaml -s /mnt/images -b pxebr -log ~/Deployment-888.log.tar.gz
 
 
 - Install Fuel Master and deploy OPNFV Cloud from scratch on Virtual Environment:
 
-    sudo python deploy.py -iso ~/ISO/opnfv.iso -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml -s /mnt/images
+    sudo python deploy.py -iso ~/ISO/opnfv.iso -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml -s /mnt/images -log ~/Deployment-888.log.tar.gz
 
 
 - Deploy OPNFV Cloud on an already active Environment where Fuel Master VM is running so no need to install Fuel again:
 
-    sudo python deploy.py -nf -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml
+    sudo python deploy.py -nf -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml -log ~/Deployment-888.log.tar.gz
 
     => with plugin installation
-    sudo python deploy.py -nf -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml
+    sudo python deploy.py -nf -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml -log ~/Deployment-888.log.tar.gz
 
     => with cleanup after deployment is finished
-    sudo python deploy.py -nf -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml -c
+    sudo python deploy.py -nf -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml -c -log ~/Deployment-888.log.tar.gz
 
     => no healthcheck after deployment is completed
-    sudo python deploy.py -nf -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml -nh
+    sudo python deploy.py -nf -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml -nh -log ~/Deployment-888.log.tar.gz
 
 
 - Install Fuel Master only (and Node VMs when using virtual environment):
 
     => for virtual environment:
-    sudo python deploy.py -iso ~/ISO/opnfv.iso -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml -s /mnt/images
+    sudo python deploy.py -iso ~/ISO/opnfv.iso -dea ~/CONF/virtual/dea.yaml -dha ~/CONF/virtual/dha.yaml -s /mnt/images -log ~/Deployment-888.log.tar.gz
 
     => for hardware environment:
-    sudo python deploy.py -iso ~/ISO/opnfv.iso -dea ~/CONF/hardware/dea.yaml -dha ~/CONF/hardware/dha.yaml -s /mnt/images -b pxebr
+    sudo python deploy.py -iso ~/ISO/opnfv.iso -dea ~/CONF/hardware/dea.yaml -dha ~/CONF/hardware/dha.yaml -s /mnt/images -b pxebr -log ~/Deployment-888.log.tar.gz
 
 
 - Cleanup a running OPNFV environment:
diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py
index 0127d2a52..f8e1617f8 100644
--- a/deploy/cloud/deployment.py
+++ b/deploy/cloud/deployment.py
@@ -7,7 +7,6 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ###############################################################################
 
-
 import time
 import re
 
@@ -16,6 +15,8 @@ from common import (
     E,
     exec_cmd,
     run_proc,
+    run_proc_wait_terminated,
+    run_proc_kill,
     parse,
     err,
     log,
@@ -30,6 +31,7 @@ LIST_OF_CHAR_TO_BE_ESCAPED = ['[', ']', '"']
 
 class Deployment(object):
 
+
     def __init__(self, dea, yaml_config_dir, env_id, node_id_roles_dict,
                  no_health_check, deploy_timeout):
         self.dea = dea
@@ -41,6 +43,7 @@ class Deployment(object):
         self.pattern = re.compile(
             '\d\d\d\d-\d\d-\d\d\s\d\d:\d\d:\d\d')
 
+
     def collect_error_logs(self):
         for node_id, roles_blade in self.node_id_roles_dict.iteritems():
             log_list = []
@@ -96,13 +99,14 @@ class Deployment(object):
                 for log_msg in log_list:
                     print(log_msg + '\n')
 
+
     def run_deploy(self):
         SLEEP_TIME = 60
         LOG_FILE = 'cloud.log'
 
         log('Starting deployment of environment %s' % self.env_id)
-        p = run_proc('fuel --env %s deploy-changes | strings > %s'
-                     % (self.env_id, LOG_FILE))
+        deploy_proc = run_proc('fuel --env %s deploy-changes | strings > %s'
+                               % (self.env_id, LOG_FILE))
 
         ready = False
         for i in range(int(self.deploy_timeout)):
@@ -120,19 +124,37 @@ class Deployment(object):
             else:
                 time.sleep(SLEEP_TIME)
 
-        p.poll()
-        if p.returncode == None:
-            log('The process deploying the changes has not yet finished.')
-            log('''The file %s won't be deleted''' % LOG_FILE)
-        else:
-            delete(LOG_FILE)
+        if (env[0][E['status']] <> 'operational'
+            and env[0][E['status']] <> 'error'
+            and env[0][E['status']] <> 'stopped'):
+            err('Deployment timed out, environment %s is not operational, snapshot will not be performed'
+                % self.env_id, self.collect_logs)
+
+        run_proc_wait_terminated(deploy_proc)
+        delete(LOG_FILE)
 
         if ready:
             log('Environment %s successfully deployed' % self.env_id)
         else:
             self.collect_error_logs()
             err('Deployment failed, environment %s is not operational'
-                % self.env_id)
+                % self.env_id, self.collect_logs)
+
+
+    def collect_logs(self):
+        log('Cleaning out any previous deployment logs')
+        exec_cmd('rm -f /var/log/remote/fuel-snapshot-*', False)
+        exec_cmd('rm -f /root/deploy-*', False)
+        log('Generating Fuel deploy snap-shot')
+        if exec_cmd('fuel snapshot < /dev/null &> snapshot.log', False)[1] <> 0:
+            log('Could not create a Fuel snapshot')
+        else:
+            exec_cmd('mv /root/fuel-snapshot* /var/log/remote/', False)
+
+        log('Collecting all Fuel Snapshot & deploy log files')
+        r, _ = exec_cmd('tar -czhf /root/deploy-%s.log.tar.gz /var/log/remote' % time.strftime("%Y%m%d-%H%M%S"), False)
+        log(r)
+
 
     def verify_node_status(self):
         node_list = parse(exec_cmd('fuel node list'))
@@ -145,18 +167,20 @@ class Deployment(object):
             summary = ''
             for node, status in failed_nodes:
                 summary += '[node %s, status %s]\n' % (node, status)
-            err('Deployment failed: %s' % summary)
+            err('Deployment failed: %s' % summary, self.collect_logs)
+
 
     def health_check(self):
         log('Now running sanity and smoke health checks')
-        r = exec_cmd('fuel health --env %s --check sanity,smoke --force'
-                     % self.env_id)
+        r = exec_cmd('fuel health --env %s --check sanity,smoke --force' % self.env_id)
         log(r)
         if 'failure' in r:
-            err('Healthcheck failed!')
+            err('Healthcheck failed!', self.collect_logs)
+
 
     def deploy(self):
         self.run_deploy()
         self.verify_node_status()
         if not self.no_health_check:
             self.health_check()
+        self.collect_logs()
diff --git a/deploy/common.py b/deploy/common.py
index 3cd3e0e6e..9c0f8abd1 100644
--- a/deploy/common.py
+++ b/deploy/common.py
@@ -77,6 +77,17 @@ def run_proc(cmd):
     return process
 
 
+def run_proc_wait_terminated(process):
+    response = process.communicate()[0].strip()
+    return_code = process.returncode
+    return response, return_code
+
+
+def run_proc_kill(process):
+    response = process.kill()
+    return response
+
+
 def parse(printout):
     parsed_list = []
     lines = printout.splitlines()
@@ -99,8 +110,10 @@ def clean(lines):
     return parsed if len(parsed_list) == 1 else parsed_list
 
 
-def err(message):
+def err(message, fun = None, *args):
     LOG.error('%s\n' % message)
+    if fun:
+        fun(*args)
     sys.exit(1)
 
 
diff --git a/deploy/deploy.py b/deploy/deploy.py
index 179ee7bcb..8064af993 100755
--- a/deploy/deploy.py
+++ b/deploy/deploy.py
@@ -30,6 +30,7 @@ from common import (
     err,
     warn,
     check_file_exists,
+    check_dir_exists,
     create_dir_if_not_exists,
     delete,
     check_if_root,
@@ -61,7 +62,7 @@ class AutoDeploy(object):
     def __init__(self, no_fuel, fuel_only, no_health_check, cleanup_only,
                  cleanup, storage_dir, pxe_bridge, iso_file, dea_file,
                  dha_file, fuel_plugins_dir, fuel_plugins_conf_dir,
-                 no_plugins, deploy_timeout, no_deploy_environment):
+                 no_plugins, deploy_timeout, no_deploy_environment, deploy_log):
         self.no_fuel = no_fuel
         self.fuel_only = fuel_only
         self.no_health_check = no_health_check
@@ -77,6 +78,7 @@ class AutoDeploy(object):
         self.no_plugins = no_plugins
         self.deploy_timeout = deploy_timeout
         self.no_deploy_environment = no_deploy_environment
+        self.deploy_log = deploy_log
         self.dea = (DeploymentEnvironmentAdapter(dea_file)
                     if not cleanup_only else None)
         self.dha = DeploymentHardwareAdapter(dha_file)
@@ -202,7 +204,7 @@ class AutoDeploy(object):
                           self.fuel_username, self.fuel_password,
                           self.dea_file, self.fuel_plugins_conf_dir,
                           WORK_DIR, self.no_health_check, self.deploy_timeout,
-                          self.no_deploy_environment)
+                          self.no_deploy_environment, self.deploy_log)
         return dep.deploy()
 
     def setup_execution_environment(self):
@@ -332,12 +334,17 @@ def parse_arguments():
     parser.add_argument('-nde', dest='no_deploy_environment',
                         action='store_true', default=False,
                         help=('Do not launch environment deployment'))
+    parser.add_argument('-log', dest='deploy_log',
+                        action='store', default='../ci/.',
+                        help=('Path and name of the deployment log archive'))
 
     args = parser.parse_args()
     log(args)
 
     check_file_exists(args.dha_file)
 
+    check_dir_exists(os.path.dirname(args.deploy_log))
+
     if not args.cleanup_only:
         check_file_exists(args.dea_file)
         check_fuel_plugins_dir(args.fuel_plugins_dir)
@@ -350,6 +357,7 @@ def parse_arguments():
         create_dir_if_not_exists(args.storage_dir)
         check_bridge(args.pxe_bridge, args.dha_file)
 
+
     kwargs = {'no_fuel': args.no_fuel, 'fuel_only': args.fuel_only,
               'no_health_check': args.no_health_check,
               'cleanup_only': args.cleanup_only, 'cleanup': args.cleanup,
@@ -360,7 +368,8 @@ def parse_arguments():
               'fuel_plugins_conf_dir': args.fuel_plugins_conf_dir,
               'no_plugins': args.no_plugins,
               'deploy_timeout': args.deploy_timeout,
-              'no_deploy_environment': args.no_deploy_environment}
+              'no_deploy_environment': args.no_deploy_environment,
+              'deploy_log': args.deploy_log}
     return kwargs
 
 
diff --git a/deploy/deploy_env.py b/deploy/deploy_env.py
index 5eeaf11e0..93dc3959b 100644
--- a/deploy/deploy_env.py
+++ b/deploy/deploy_env.py
@@ -20,6 +20,7 @@ from ssh_client import SSHClient
 from common import (
     err,
     log,
+    exec_cmd,
     parse,
     N,
     E,
@@ -35,7 +36,7 @@ class CloudDeploy(object):
 
     def __init__(self, dea, dha, fuel_ip, fuel_username, fuel_password,
                  dea_file, fuel_plugins_conf_dir, work_dir, no_health_check,
-                 deploy_timeout, no_deploy_environment):
+                 deploy_timeout, no_deploy_environment, deploy_log):
         self.dea = dea
         self.dha = dha
         self.fuel_ip = fuel_ip
@@ -51,6 +52,7 @@ class CloudDeploy(object):
         self.no_health_check = no_health_check
         self.deploy_timeout = deploy_timeout
         self.no_deploy_environment = no_deploy_environment
+        self.deploy_log = deploy_log
         self.file_dir = os.path.dirname(os.path.realpath(__file__))
         self.ssh = SSHClient(self.fuel_ip, self.fuel_username,
                              self.fuel_password)
@@ -256,6 +258,10 @@ class CloudDeploy(object):
         self.set_boot_order(['pxe', 'disk'])
         self.power_on_nodes()
 
+    def get_put_deploy_log(self):
+        with self.ssh as s:
+            s.scp_get("deploy-*", local=self.deploy_log)
+
     def deploy(self):
 
         self.set_boot_order_nodes()
@@ -272,4 +278,8 @@ class CloudDeploy(object):
 
         delete(self.updated_dea_file)
 
-        return self.run_cloud_deploy(CLOUD_DEPLOY_FILE)
+        rc = self.run_cloud_deploy(CLOUD_DEPLOY_FILE)
+
+        self.get_put_deploy_log()
+
+        return rc
diff --git a/deploy/ssh_client.py b/deploy/ssh_client.py
index df780961f..f6888d52d 100644
--- a/deploy/ssh_client.py
+++ b/deploy/ssh_client.py
@@ -85,14 +85,14 @@ class SSHClient(object):
 
     def scp_get(self, remote, local='.', dir=False):
         try:
-            with scp.SCPClient(self.client.get_transport()) as _scp:
+            with scp.SCPClient(self.client.get_transport(), sanitize=lambda x: x) as _scp:
                 _scp.get(remote, local, dir)
         except Exception as e:
             err(e)
 
     def scp_put(self, local, remote='.', dir=False):
         try:
-            with scp.SCPClient(self.client.get_transport()) as _scp:
+            with scp.SCPClient(self.client.get_transport(), sanitize=lambda x: x) as _scp:
                 _scp.put(local, remote, dir)
         except Exception as e:
             err(e)