From df6c1db2695030b1ba68184bac9aab9fb2d41e71 Mon Sep 17 00:00:00 2001 From: Jonas Bjurel Date: Thu, 12 Mar 2015 15:18:06 +0100 Subject: *** FIRST VERSION OF CI BUILD SCRIPT *** - Provides build cache management to a remote location - Provides automatic cashe repopulation - See /fuel/ci/README - Rebased to master - Bug around cache handling fixed - New option -r introduced to invoke external script/program instead of standard http(s)/fttp get and put, eg. scripts or programs to integrate with dropbox, or similar. The script needs to anheaer to curl argument list. - Stefan Berg's comments implemented - Fatih Degirmenci's comments implemented with one exemption: the revision state .iso naming convention still remains the same - we need to discuss this. With the implementation of Fatih's comments the build.sh argument structure is slightly changed, please see ./build.sh -h TODO: - Further testing - Finalize cache invalidation logic - Make refactoring JIRA: Change-Id: I4efdbdd10f726c0a3229849a63040b6d2702db5c Signed-off-by: Jonas Bjurel --- fuel/ci/build.sh | 478 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 478 insertions(+) mode change 100644 => 100755 fuel/ci/build.sh (limited to 'fuel/ci/build.sh') diff --git a/fuel/ci/build.sh b/fuel/ci/build.sh old mode 100644 new mode 100755 index e69de29..bb0d124 --- a/fuel/ci/build.sh +++ b/fuel/ci/build.sh @@ -0,0 +1,478 @@ +#!/bin/bash +set -e +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +trap 'echo "Exiting ..."; \ +if [ -f ${LOCK_FILE} ]; then \ + if [ $(cat ${LOCK_FILE}) -eq $$ ]; then \ + rm -f ${LOCK_FILE}; \ + fi; \ +fi;' EXIT + +usage () +{ +cat << EOF +$0 Builds the Fuel@OPNFV stack + +usage: $0 [-s spec-file] [-c cache-URI] [-l log-file] [-f Flags] build-directory + +OPTIONS: + -s spec-file ($BUILD_SPEC), define the build-spec file, default ../build/config.mk + -c cache base URI ($BUILD_CACHE_URI), specifies the base URI to a build cache to be used/updated - the name is automatically generated from the md5sum of the spec-file, http://, ftp://, file://[absolute path] suported. + + -l log-file ($BUILD_LOG), specifies the output log-file (stdout and stderr), if not specified logs are output to console as normal + -v version tag to be applied to the build result + -r alternative remote access method script/program. curl is default. + -f build flags ($BUILD_FLAGS): + o s: Do nothing, succeed + o f: Do nothing, fail + o t: run build unit tests + o i: run interactive (-t flag to docker run) + o P: Populate a new local cache and push it to the (-c cache-URI) cache artifactory if -c option is present, currently file://, http:// and ftp:// are supported + o d: Detatch - NOT YET SUPPORTED + + build-directory ($BUILD_DIR), specifies the directory for the output artifacts (.iso file). + + -h help, prints this help text + +Description: +build.sh builds opnfv .iso artifact. +To reduce build time it uses build cache on a local or remote location. The cache is rebuilt and uploaded if either of the below conditions are met: +1) The P(opulate) flag is set and the -c cache-base-URI is provided, if -c is not provided the cache will stay local. +2) If the cache is invalidated by one of the following conditions: + - The config spec md5sum does not compare to the md5sum for the spec which the cache was built. + - The git Commit-Id on the remote repos/HEAD defined in the spec file does not correspont with the Commit-Id for what the cache was built with. +3) A valid cache does not exist on the specified -c cache-base-URI. + +The cache URI object name is fuel_cache-"md5sum(spec file)" + +Logging by default to console, but can be directed elsewhere with the -l option in which case both stdout and stderr is redirected to that destination. + +Built in unit testing of components is enabled by adding the t(est) flag. + +Return codes: + - 0 Success! + - 1-99 Unspecified build error + - 100-199 Build system internal error (not build it self) + o 101 Build system instance busy + - 200 Build failure + +Examples: +build -c http://opnfv.org/artifactory/fuel/cache -d ~/jenkins/genesis/fuel/ci/output -f ti +NOTE: At current the build scope is set to the git root of the repository, -d destination locations outside that scope will not work +EOF +} + + + +debug_make () { + make -C ${BUILD_BASE} clean + echo "This is a fake debug fuel .iso image" > ${BUILD_BASE}/fuel-6.0.1.iso + echo "This is a fake debug versions file" > ${BUILD_BASE}/.versions + rm -rf ${BUILD_BASE}/release + mkdir ${BUILD_BASE}/release + echo "This is a fake debug OPNFV .iso image" > ${BUILD_BASE}/release/fake-debug.iso + echo "This is a fake debug OPNFV .iso.txt message" > ${BUILD_BASE}/release/fake-debug.iso.txt + echo "This a fake debug odl build history" > ${BUILD_BASE}/opendaylight/.odl-build-history + echo "This a fake debug odl build log" > ${BUILD_BASE}/opendaylight/.odl-build.log +} + +cache_check () { + if [ ! -f ${BUILD_BASE}/fuel-6.0.1.iso ] || \ + [ ! -f ${BUILD_BASE}/.versions ] || \ + [ ! -f ${BUILD_BASE}/opendaylight/.odl-build-history ]; \ + [ ! -f ${BUILD_BASE}/opendaylight/.odl-build.log ]; then + echo "Cache not present in the build system" + echo "TEST FAILED" + exit $rc + fi +} + +integration-test () { + +##### Always succeed integration test #### + make -C ${BUILD_BASE} clean + echo "TEST - $0 - ALWAYS SUCCEED" + set +e + $0 -f s tmp/output + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "TEST FAILED" + rc=150 + exit $rc + fi + +##### Always fail integration test #### + make -C ${BUILD_BASE} clean + echo "TEST - $0 - ALWAYS FAIL" + set +e + $0 -f f tmp/output + rc=$? + set -e + if [ $rc -eq 0 ]; then + echo "TEST FAILED" + rc=151 + exit $rc + fi + +##### Populate cache integration test #### + make -C ${BUILD_BASE} clean + echo "TEST - $0 - POPULATE CACHE" + rm -rf tmp + mkdir tmp + mkdir tmp/cache + set +e + $0 -c ${BUILD_CACHE_URI} -f PD tmp/output + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "TEST FAILED" + rc=152 + exit $rc + fi + + if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then + echo "TEST FAILED" + rc=153 + exit $rc + fi + rc=154 + cache_check + +##### Build uinge cache integration test #### + make -C ${BUILD_BASE} clean + echo "TEST - $0 - BUILD USING CACHE" + set +e + $0 -c ${BUILD_CACHE_URI} -f D tmp/output + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "TEST FAILED" + rc=155 + exit $rc + fi + + if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then + echo "TEST FAILED" + rc=156 + exit $rc + fi + rc=157 + cache_check + +#### Repopulate cache because cache non existant #### + make -C ${BUILD_BASE} clean + echo "TEST - $0 - BUILD USING CACHE" + rm -rf tmp/cache/* + set +e + $0 -c ${BUILD_CACHE_URI} -f D tmp/output + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "TEST FAILED" + rc=158 + exit $rc + fi + + if [ ! -f tmp/output/fake-debug.iso ] || [ ! -f tmp/output/fake-debug.iso ]; then + echo "TEST FAILED" + rc=160 + exit $rc + fi + rc=161 + cache_check + +# Repopulate cache because cach is ivalidated +#TBD +rm -rf tmp +echo "All tests passed!" +rc=0 +exit $rc +} + +#DEFAULT VALUES +BUILD_BASE=$(readlink -e ../build/) +RESULT_DIR="${BUILD_BASE}/release" +BUILD_SPEC="${BUILD_BASE}/config.mk" +CACHE_DIR="cache" +LOCAL_CACHE_ARCH_NAME="fuel-cache" +REMOTE_CACHE_ARCH_NAME="fuel_cache-$(md5sum ${BUILD_SPEC}| cut -f1 -d " ")" +REMOTE_ACCESS_METHD=curl + +#SCRIPT ASSIGNED VARIABLES +SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +LOCK_FILE="${SCRIPT_DIR}/.build.lck" +TEST_SUCCEED=0 +TEST_FAIL=0 +UNIT_TEST=0 +UPDATE_CACHE=0 +POPULATE_CACHE=0 +RECURSIV=0 +DETACH=0 +DEBUG=0 +INTEGRATION_TEST=0 +INTERACTIVE=0 +BUILD_CACHE_URI= +BUILD_SPEC= +BUILD_DIR= +BUILD_LOG= +BUILD_VERSION= +MAKE_ARGS= + +while getopts "s:c:v:f:l:r:RTh" OPTION +do + case $OPTION in + h) + usage + rc=0 + exit $rc + ;; + + s) + BUILD_SPEC=${OPTARG} + ;; + + c) + BUILD_CACHE_URI=${OPTARG} + ;; + + l) + BUILD_LOG=${OPTARG} + ;; + + v) + BUILD_VERSION=${OPTARG} + ;; + + f) + BUILD_FLAGS=${OPTARG} + ;; + + r) REMOTE_ACCESS_METHD=${OPTARG} + ;; + + R) + RECURSIVE=1 + ;; + + T) + INTEGRATION_TEST=1 + ;; + + *) + echo "${OPTION} is not a valid argument" + rc=100 + exit $rc + ;; + esac +done + +if [ -z $BUILD_DIR ]; then + BUILD_DIR=$(echo $@ | cut -d ' ' -f ${OPTIND}) +fi + +for ((i=0; i<${#BUILD_FLAGS};i++)); do + case ${BUILD_FLAGS:$i:1} in + s) + rc=0 + exit $rc + ;; + + f) + rc=1 + exit $rc + ;; + + t) + UNIT_TEST=1 + ;; + + i) + INTERACTIVE=1 + ;; + + P) + POPULATE_CACHE=1 + ;; + + d) + DETACH=1 + echo "Detach is not yet supported - exiting ...." + rc=100 + exit $rc + ;; + + D) + DEBUG=1 + ;; + + *) + echo "${BUILD_FLAGS:$i:1} is not a valid build flag" + rc=100 + exit $rc + ;; + esac +done + +if [ ${INTEGRATION_TEST} -eq 1 ]; then + integration-test + rc=0 + exit $rc +fi + +if [ ! -f ${BUILD_SPEC} ]; then + echo "spec file does not exist: $BUILD_SPEC" + rc=100 + exit $rc +fi + +if [ -z ${BUILD_DIR} ]; then + echo "Missing build directory" + rc=100 + exit $rc +fi + +if [ ! -z ${BUILD_LOG} ]; then + if [[ ${RECURSIVE} -ne 1 ]]; then + set +e + eval $0 -R $@ > ${BUILD_LOG} 2>&1 + rc=$? + set -e + if [ $rc -ne 0]; then + exit $rc + fi + fi +fi + +if [ ${TEST_SUCCEED} -eq 1 ]; then + sleep 1 + rc=0 + exit $rc +fi + +if [ ${TEST_FAIL} -eq 1 ]; then + sleep 1 + rc=1 + exit $rc +fi + +if [ -e ${LOCK_FILE} ]; then + echo "A build job is already running, exiting....." + rc=101 + exit $rc +fi + +echo $$ > ${LOCK_FILE} + +if [ ! -z ${BUILD_CACHE_URI} ]; then + if [ ${POPULATE_CACHE} -ne 1 ]; then + rm -rf /tmp/cache + mkdir /tmp/cache + echo "Downloading cach file ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} ..." + set +e + ${REMOTE_ACCESS_METHD} -o /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "Remote cache does not exist, or is not accessible - a new cache will be built ..." + POPULATE_CACHE=1 + else + echo "Unpacking cache file ..." + tar -C /tmp/cache -xvf /tmp/cache/${LOCAL_CACHE_ARCH_NAME}.tgz + cp /tmp/cache/cache/.versions ${BUILD_BASE}/. + echo "Validating cache content ..." + set +e + make -C ${BUILD_BASE} validate-cache; + rc=$? + if [ $rc -ne 0 ]; then + echo "Cache invalid - a new cache will be built " + POPULATE_CACHE=1 + else + cp -rf /tmp/cache/cache/. ${BUILD_BASE} + fi + rm -rf /tmp/cache + fi + fi +fi + +if [ ${POPULATE_CACHE} -eq 1 ]; then + if [ ${DEBUG} -eq 0 ]; then + set +e + cd ${BUILD_BASE} && make clean + rc=$? + set -e + if [ $rc -ne 0 ]; then + echo "Build - make clean failed, exiting ..." + rc=100 + exit $rc + fi + fi +fi + +if [ ! -z ${BUILD_VERSION} ]; then + MAKE_ARGS+="REVSTATE=${BUILD_VERSION} " +fi + +if [ ${UNIT_TEST} -eq 1 ]; then + MAKE_ARGS+="UNIT_TEST=TRUE " +else + MAKE_ARGS+="UNIT_TEST=FALSE " +fi + +if [ ${INTERACTIVE} -eq 1 ]; then + MAKE_ARGS+="INTERACTIVE=TRUE " +else + MAKE_ARGS+="INTERACTIVE=FALSE " +fi + +MAKE_ARGS+=all + +if [ ${DEBUG} -eq 0 ]; then + set +e + cd ${BUILD_BASE} && make ${MAKE_ARGS} + rc=$? + set -e + if [ $rc -gt 0 ]; then + echo "Build: make all failed, exiting ..." + rc=200 + exit $rc + fi +else +debug_make +fi +set +e +make -C ${BUILD_BASE} prepare-cache +rc=$? +set -e + +if [ $rc -gt 0 ]; then + echo "Build: make prepare-cache failed, exiting ..." + rc=100 + exit $rc +fi +echo "Copying built OPNFV .iso file to target directory ${BUILD_DIR} ..." +rm -rf ${BUILD_DIR} +mkdir ${BUILD_DIR} +cp ${BUILD_BASE}/.versions ${BUILD_DIR} +cp ${RESULT_DIR}/*.iso* ${BUILD_DIR} + +if [ $POPULATE_CACHE -eq 1 ]; then + if [ ! -z ${BUILD_CACHE_URI} ]; then + echo "Building cache ..." + tar --dereference -C ${BUILD_BASE} -caf ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${CACHE_DIR} + echo "Uploading cache ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}" + ${REMOTE_ACCESS_METHD} -T ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} + rm ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz + fi +fi +echo "Success!!!" +exit 0 -- cgit 1.2.3-korg