From 50cd40d868f326cecd490a761c56e9a838668b45 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Tue, 3 Oct 2017 20:07:20 +0200 Subject: fuel, armband: Add post mortem log collection Debugging test suite failures is often hard without physical access to the POD, and even then the environment might be automatically redeployed before successfully debugging the problem. So, collect all contents of from all cluster nodes (including the Salt Master node) after all test suites have been executed as part of the CI job. Change-Id: Ie5e2526c74f4ed21be9c76c6f6101eb3f434be36 Signed-off-by: Alexandru Avadanii --- jjb/armband/armband-ci-jobs.yml | 10 ++++++++ jjb/fuel/fuel-daily-jobs.yml | 52 +++++++++++++++++++++++++++++++++++++++++ jjb/fuel/fuel-logs.sh | 36 ++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+) create mode 100755 jjb/fuel/fuel-logs.sh (limited to 'jjb') diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index 0154c7a71..64c633a3a 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -158,6 +158,16 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' + # Armband uses Fuel's log collection project job, no need to duplicate + - trigger-builds: + - project: 'fuel-collect-logs-{pod}-daily-{stream}' + current-parameters: false + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' - job-template: name: '{installer}-deploy-{pod}-daily-{stream}' diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml index d489f7e55..b6a1bc469 100644 --- a/jjb/fuel/fuel-daily-jobs.yml +++ b/jjb/fuel/fuel-daily-jobs.yml @@ -115,6 +115,7 @@ jobs: - 'fuel-{scenario}-{pod}-daily-{stream}' - 'fuel-deploy-{pod}-daily-{stream}' + - 'fuel-collect-logs-{pod}-daily-{stream}' ######################## # job templates @@ -215,6 +216,22 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' + - conditional-step: + condition-kind: not + condition-operand: + condition-kind: regex-match + regex: 'danube' + label: '{stream}' + steps: + - trigger-builds: + - project: 'fuel-collect-logs-{pod}-daily-{stream}' + current-parameters: false + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' publishers: - email: @@ -280,6 +297,41 @@ recipients: peter.barabas@ericsson.com fzhadaev@mirantis.com - email-jenkins-admins-on-failure +- job-template: + name: 'fuel-collect-logs-{pod}-daily-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - logrotate-default + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - '{installer}-defaults': + gs-pathname: '{gs-pathname}' + - '{slave-label}-defaults': + installer: '{installer}' + + scm: + - git-scm + + wrappers: + - build-name: + name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - shell: + !include-raw-escape: ./fuel-logs.sh + + publishers: + - email-jenkins-admins-on-failure + ######################## # trigger macros ######################## diff --git a/jjb/fuel/fuel-logs.sh b/jjb/fuel/fuel-logs.sh new file mode 100755 index 000000000..a7d852685 --- /dev/null +++ b/jjb/fuel/fuel-logs.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2017 Ericsson AB, Mirantis Inc., Enea Software AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o nounset +set -o pipefail + +# Fuel requires deploy script to be ran with sudo, Armband does not +SUDO='sudo -E' +[ "${PROJECT}" = 'fuel' ] || SUDO= + +# Log file name +FUEL_PM_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}_pm.log.tar.gz" + +# Construct the command +LOG_COMMAND="${SUDO} ${WORKSPACE}/mcp/scripts/log.sh \ + ${WORKSPACE}/${FUEL_PM_LOG_FILENAME}" + +# Log info to console +echo "Collecting post mortem logs ..." +echo "--------------------------------------------------------" +echo "${LOG_COMMAND}" + +${LOG_COMMAND} + +# Upload logs for both baremetal and virtual deployments +echo "Uploading deployment logs" +echo "--------------------------------------------------------" +gsutil cp "${WORKSPACE}/${FUEL_PM_LOG_FILENAME}" \ + "gs://${GS_URL}/logs/${FUEL_PM_LOG_FILENAME}" > /dev/null 2>&1 +echo "Logs are available at http://${GS_URL}/logs/${FUEL_PM_LOG_FILENAME}" -- cgit 1.2.3-korg