summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAlexandru Avadanii <Alexandru.Avadanii@enea.com>2017-10-03 20:07:20 +0200
committerAlexandru Avadanii <Alexandru.Avadanii@enea.com>2017-10-03 22:50:58 +0200
commit50cd40d868f326cecd490a761c56e9a838668b45 (patch)
tree834503bdd1fb9bc81a689a6f8aba193e2beab79a
parentf86d3e96d033044849da3d43554f778b97c40508 (diff)
fuel, armband: Add post mortem log collection
Debugging test suite failures is often hard without physical access to the POD, and even then the environment might be automatically redeployed before successfully debugging the problem. So, collect all contents of </var/log> from all cluster nodes (including the Salt Master node) after all test suites have been executed as part of the CI job. Change-Id: Ie5e2526c74f4ed21be9c76c6f6101eb3f434be36 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
-rw-r--r--jjb/armband/armband-ci-jobs.yml10
-rw-r--r--jjb/fuel/fuel-daily-jobs.yml52
-rwxr-xr-xjjb/fuel/fuel-logs.sh36
3 files changed, 98 insertions, 0 deletions
diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml
index 0154c7a71..64c633a3a 100644
--- a/jjb/armband/armband-ci-jobs.yml
+++ b/jjb/armband/armband-ci-jobs.yml
@@ -158,6 +158,16 @@
build-step-failure-threshold: 'never'
failure-threshold: 'never'
unstable-threshold: 'FAILURE'
+ # Armband uses Fuel's log collection project job, no need to duplicate
+ - trigger-builds:
+ - project: 'fuel-collect-logs-{pod}-daily-{stream}'
+ current-parameters: false
+ block: true
+ same-node: true
+ block-thresholds:
+ build-step-failure-threshold: 'never'
+ failure-threshold: 'never'
+ unstable-threshold: 'FAILURE'
- job-template:
name: '{installer}-deploy-{pod}-daily-{stream}'
diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml
index d489f7e55..b6a1bc469 100644
--- a/jjb/fuel/fuel-daily-jobs.yml
+++ b/jjb/fuel/fuel-daily-jobs.yml
@@ -115,6 +115,7 @@
jobs:
- 'fuel-{scenario}-{pod}-daily-{stream}'
- 'fuel-deploy-{pod}-daily-{stream}'
+ - 'fuel-collect-logs-{pod}-daily-{stream}'
########################
# job templates
@@ -215,6 +216,22 @@
build-step-failure-threshold: 'never'
failure-threshold: 'never'
unstable-threshold: 'FAILURE'
+ - conditional-step:
+ condition-kind: not
+ condition-operand:
+ condition-kind: regex-match
+ regex: 'danube'
+ label: '{stream}'
+ steps:
+ - trigger-builds:
+ - project: 'fuel-collect-logs-{pod}-daily-{stream}'
+ current-parameters: false
+ block: true
+ same-node: true
+ block-thresholds:
+ build-step-failure-threshold: 'never'
+ failure-threshold: 'never'
+ unstable-threshold: 'FAILURE'
publishers:
- email:
@@ -280,6 +297,41 @@
recipients: peter.barabas@ericsson.com fzhadaev@mirantis.com
- email-jenkins-admins-on-failure
+- job-template:
+ name: 'fuel-collect-logs-{pod}-daily-{stream}'
+
+ disabled: '{obj:disabled}'
+
+ concurrent: true
+
+ properties:
+ - logrotate-default
+
+ parameters:
+ - project-parameter:
+ project: '{project}'
+ branch: '{branch}'
+ - '{installer}-defaults':
+ gs-pathname: '{gs-pathname}'
+ - '{slave-label}-defaults':
+ installer: '{installer}'
+
+ scm:
+ - git-scm
+
+ wrappers:
+ - build-name:
+ name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO'
+
+ builders:
+ - description-setter:
+ description: "Built on $NODE_NAME"
+ - shell:
+ !include-raw-escape: ./fuel-logs.sh
+
+ publishers:
+ - email-jenkins-admins-on-failure
+
########################
# trigger macros
########################
diff --git a/jjb/fuel/fuel-logs.sh b/jjb/fuel/fuel-logs.sh
new file mode 100755
index 000000000..a7d852685
--- /dev/null
+++ b/jjb/fuel/fuel-logs.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2017 Ericsson AB, Mirantis Inc., Enea Software AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -o nounset
+set -o pipefail
+
+# Fuel requires deploy script to be ran with sudo, Armband does not
+SUDO='sudo -E'
+[ "${PROJECT}" = 'fuel' ] || SUDO=
+
+# Log file name
+FUEL_PM_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}_pm.log.tar.gz"
+
+# Construct the command
+LOG_COMMAND="${SUDO} ${WORKSPACE}/mcp/scripts/log.sh \
+ ${WORKSPACE}/${FUEL_PM_LOG_FILENAME}"
+
+# Log info to console
+echo "Collecting post mortem logs ..."
+echo "--------------------------------------------------------"
+echo "${LOG_COMMAND}"
+
+${LOG_COMMAND}
+
+# Upload logs for both baremetal and virtual deployments
+echo "Uploading deployment logs"
+echo "--------------------------------------------------------"
+gsutil cp "${WORKSPACE}/${FUEL_PM_LOG_FILENAME}" \
+ "gs://${GS_URL}/logs/${FUEL_PM_LOG_FILENAME}" > /dev/null 2>&1
+echo "Logs are available at http://${GS_URL}/logs/${FUEL_PM_LOG_FILENAME}"