From 65954c4dd99697344fca78ef2c6dda50416f55f0 Mon Sep 17 00:00:00 2001 From: Markos Chandras Date: Wed, 18 Jan 2017 14:54:07 +0000 Subject: jjb: infra: bifrost-upload-logs.sh: Consolidate upload step to main job In case of job failures, the upload builder is not executed so the published links are dead. We move the upload step to the EXIT handler which is always executed despite the main job exit code. Moreover, The gsutil tool can compress the files itself but in such a way that the web browser can still display the contents without the need for the user to download and uncompress the file himself. This makes debugging more convenient. We also make the whole upload process somewhat less noisy. Finally, we include the $GERRIT_PATCHSET_NUMBER in the title and hyperlink since we provide and retain build results for each patchset instead of a single set just for the latest one. Change-Id: I43efd519a5743661d1f258aa338fb0769f59b6ed Suggested-by: Julien Zhang # compression only Signed-off-by: Markos Chandras --- jjb/infra/bifrost-upload-logs.sh | 52 --------------------------------------- jjb/infra/bifrost-verify-jobs.yml | 7 ------ jjb/infra/bifrost-verify.sh | 51 +++++++++++++++++++++++++++++++++++++- 3 files changed, 50 insertions(+), 60 deletions(-) delete mode 100755 jjb/infra/bifrost-upload-logs.sh diff --git a/jjb/infra/bifrost-upload-logs.sh b/jjb/infra/bifrost-upload-logs.sh deleted file mode 100755 index 8e1982dff..000000000 --- a/jjb/infra/bifrost-upload-logs.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# SPDX-license-identifier: Apache-2.0 -############################################################################## -# Copyright (c) 2016 SUSE. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -set -eu -set -o pipefail - -BIFROST_CONSOLE_LOG="${BUILD_URL}/consoleText" -BIFROST_GS_URL=${BIFROST_LOG_URL/http:/gs:} -BIFROST_COMPRESS_SUFFIX="tar.gz" -BIFROST_COMPRESSED_LOGS=() - -echo "Uploading build logs to ${BIFROST_LOG_URL}" - -echo "Uploading console output" -curl -L ${BIFROST_CONSOLE_LOG} | gsutil cp - ${BIFROST_GS_URL}/build_log.txt - -[[ ! -d ${WORKSPACE}/logs ]] && exit 0 - -pushd ${WORKSPACE}/logs/ &> /dev/null -for x in *.log; do - echo "Compressing and uploading $x" - tar -czf - $x | gsutil cp - ${BIFROST_GS_URL}/${x}.${BIFROST_COMPRESS_SUFFIX} 1>/dev/null - BIFROST_COMPRESSED_LOGS+=(${x}.${BIFROST_COMPRESS_SUFFIX}) -done -popd &> /dev/null - -echo "Generating the landing page" -cat > index.html << EOF - -

Build results for $GERRIT_NAME/$GERRIT_CHANGE_NUMBER

-

Job: $JOB_NAME

-
    -
  • build_log.txt
    • -EOF - -for x in ${BIFROST_COMPRESSED_LOGS[@]}; do - echo "
  • ${x}
    • " >> index.html -done - -cat >> index.html << EOF -
- -EOF - -gsutil cp index.html ${BIFROST_GS_URL}/index.html diff --git a/jjb/infra/bifrost-verify-jobs.yml b/jjb/infra/bifrost-verify-jobs.yml index 6232859ee..8184b6343 100644 --- a/jjb/infra/bifrost-verify-jobs.yml +++ b/jjb/infra/bifrost-verify-jobs.yml @@ -142,7 +142,6 @@ builders: - bifrost-set-name - bifrost-build - - bifrost-artifacts-upload publishers: - email: @@ -220,9 +219,3 @@ builders: - shell: !include-raw: ./bifrost-verify.sh - -- builder: - name: bifrost-artifacts-upload - builders: - - shell: - !include-raw: ./bifrost-upload-logs.sh diff --git a/jjb/infra/bifrost-verify.sh b/jjb/infra/bifrost-verify.sh index 9fbb1d0d9..2cb41a45b 100755 --- a/jjb/infra/bifrost-verify.sh +++ b/jjb/infra/bifrost-verify.sh @@ -11,7 +11,49 @@ set -o errexit set -o nounset set -o pipefail -trap fix_ownership EXIT +trap cleanup_and_upload EXIT + +function upload_logs() { + BIFROST_CONSOLE_LOG="${BUILD_URL}/consoleText" + BIFROST_GS_URL=${BIFROST_LOG_URL/http:/gs:} + + echo "Uploading build logs to ${BIFROST_LOG_URL}" + + echo "Uploading console output" + curl -s -L ${BIFROST_CONSOLE_LOG} | gsutil -q cp -Z - ${BIFROST_GS_URL}/build_log.txt + + [[ ! -d ${WORKSPACE}/logs ]] && exit 0 + + pushd ${WORKSPACE}/logs/ &> /dev/null + for x in *.log; do + echo "Compressing and uploading $x" + gsutil -q cp -Z ${x} ${BIFROST_GS_URL}/${x} + done + + echo "Generating the landing page" + cat > index.html < +

Build results for $GERRIT_NAME/$GERRIT_CHANGE_NUMBER/$GERRIT_PATCHSET_NUMBER

+

Job: $JOB_NAME

+
    +
  • build_log.txt
    • +EOF + + for x in *.log; do + echo "
  • ${x}
    • " >> index.html + done + + cat >> index.html << EOF +
+ +EOF + + gsutil -q cp index.html ${BIFROST_GS_URL}/index.html + + rm index.html + + popd &> /dev/null +} function fix_ownership() { if [ -z "${JOB_URL+x}" ]; then @@ -25,6 +67,13 @@ function fix_ownership() { fi } +function cleanup_and_upload() { + original_exit=$? + fix_ownership + upload_logs + exit $original_exit +} + # check distro to see if we support it if [[ ! "$DISTRO" =~ (trusty|centos7|suse) ]]; then echo "Distro $DISTRO is not supported!" -- cgit 1.2.3-korg