Merge "Create Compass build script."

5 files changed, 536 insertions, 0 deletions

diff --git a/compass/build/Makefile b/compass/build/Makefile
new file mode 100755
index 0000000..7448dc4
--- /dev/null
+++ b/compass/build/Makefile
@@ -0,0 +1,117 @@
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# dradez@redhat.com
+# chigang@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+SHELL = /bin/bash
+############################################################################
+# BEGIN of variables to customize
+#
+#Input args
+export UNIT_TEST = FALSE
+export INTERACTIVE = TRUE
+export ISOSRC = file:$(shell pwd)/ubuntu
+export ISOCACHE = $(shell pwd)/$(shell basename $(ISOSRC))
+export PRODNO = "OPNFV_BGS"
+export REVSTATE = "P0000"
+export RELEASE_DIR = $(shell pwd)/release
+
+# Note! Invoke with "make REVSTATE=RXXXX all" to make release build!
+# Invoke with ICOCACHE=/full/path/to/iso if cached ISO is in non-standard location.
+
+#Build variables
+export BUILD_BASE := $(shell pwd)
+export CACHE_DIR := $(BUILD_BASE)/cache
+export INSTALL_DIR := $(BUILD_BASE)
+export VERSION_FILE := $(BUILD_BASE)/.versions
+export TOPDIR := $(shell pwd)
+
+export OLDISO_DIR := $(TOPDIR)/oldiso
+export NEWISO_DIR := $(TOPDIR)/newiso
+export NEWIMAGE_DIR := $(TOPDIR)/newiso/image
+export NEWFILESYSTEM := $(TOPDIR)/newiso/filesystem
+export MANIFEST_DIR = $(shell find $(NEWISO_DIR) -name filesystem.manifest)
+export SQUASHFS_DIR = $(shell find $(NEWISO_DIR) -name filesystem.squashfs)
+export FSSIZE_DIR = $(shell find $(NEWISO_DIR) -name filesystem.size)
+
+#
+# END of variables to customize
+#############################################################################
+
+.PHONY: all
+all: iso
+	@echo "Versions of cached build results built by" $(shell hostname) "at" $(shell date -u) > $(VERSION_FILE)
+	@echo "cache.mk" $(shell md5sum $(BUILD_BASE)/cache.mk | cut -f1 -d " ") >> $(VERSION_FILE)
+	@echo "config.mk" $(shell md5sum $(BUILD_BASE)/config.mk | cut -f1 -d " ") >> $(VERSION_FILE)
+
+############################################################################
+# BEGIN of Include definitions
+#
+include config.mk
+include cache.mk
+#
+# END Include definitions
+#############################################################################
+
+.PHONY: prepare-cache
+prepare-cache: 
+	@echo "prepare-cache to be done"
+
+.PHONY: mount-ubuntuiso
+mount-ubuntuiso:
+	@echo "===Mounting ubuntu ISO in $(OLDISO_DIR)"
+	-mkdir -p $(OLDISO_DIR) $(NEWIMAGE_DIR)
+	@fuseiso $(ISOCACHE)/*.iso $(OLDISO_DIR)
+	cp $(OLDISO_DIR)/. $(NEWIMAGE_DIR) -rp
+
+.PHONY: umount-ubuntuiso
+umount-ubuntuiso:
+	@set +e
+	@echo "===Unmounting ubuntu ISO from $(OLDISO_DIR)"
+	@fusermount -u $(OLDISO_DIR)
+	@set -e
+
+.PHONY: install-package
+install-package:
+	@echo "===uncompress file system to add new files"
+	@find $(NEWISO_DIR) -name "filesystem.squashfs" |xargs unsquashfs
+	@mv squashfs-root $(NEWFILESYSTEM)
+	cp -f /etc/resolv.conf $(NEWFILESYSTEM)/run/resolvconf/
+	cp /etc/hosts $(NEWFILESYSTEM)/etc/
+	cp $(INSTALL_DIR)/install.sh $(NEWFILESYSTEM)/
+	@echo "===install package on filesystem for newiso"
+	#@chroot $(NEWFILESYSTEM) sh ./install.sh
+	@chmod +w $(MANIFEST_DIR)
+	@chroot $(NEWFILESYSTEM) dpkg-query -W --showformat='$${Package} $${Version}\n' | tee ${MANIFEST_DIR}
+	@rm $(SQUASHFS_DIR)
+	@mksquashfs $(NEWFILESYSTEM) $(SQUASHFS_DIR)
+	@chmod +w $(FSSIZE_DIR)
+	cd $(NEWISO_DIR); \
+	(du -sx --block-size=1 $(NEWFILESYSTEM) | cut -f1 ) | tee ${FSSIZE_DIR}
+	cd $(NEWIMAGE_DIR); \
+	find . -type f -print0 | xargs -0 md5sum | grep -v "\./md5sum.txt" | tee ./md5sum.txt
+
+.PHONY: make-iso
+make-iso:
+	@echo "===Building OPNFV iso"
+	cd $(NEWIMAGE_DIR); \
+	mkisofs -r -V "OPNFV" -cache-inodes -J -l -b isolinux/isolinux.bin -c isolinux/boot.cat -no-emul-boot -boot-load-size 4 -boot-info-table -o ../ubuntu-14.04-amd64-opnfv.iso .
+
+.PHONY: build-clean
+build-clean:
+	-rm -Rf $(OLDISO_DIR)
+	-rm -Rf $(NEWISO_DIR)
+	-rm -Rf $(RELEASE_DIR)
+
+.PHONY: iso
+iso: build-clean mount-ubuntuiso umount-ubuntuiso install-package make-iso
+	-mkdir $(RELEASE_DIR)
+	@mv $(NEWISO_DIR)/*.iso $(RELEASE_DIR)
+	@printf "\n\nISO is built successfully!\n\n"
diff --git a/compass/build/cache.mk b/compass/build/cache.mk
new file mode 100755
index 0000000..e69de29
--- /dev/null
+++ b/compass/build/cache.mk
diff --git a/compass/build/config.mk b/compass/build/config.mk
new file mode 100755
index 0000000..e69de29
--- /dev/null
+++ b/compass/build/config.mk
diff --git a/compass/build/install.sh b/compass/build/install.sh
new file mode 100755
index 0000000..4a8b893
--- /dev/null
+++ b/compass/build/install.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+#####################################################################################
+# Copyright (c) 2015 Huawei Technologies Co.,Ltd.
+# chigang@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+#####################################################################################
+
+# some packages or tools maybe use below filesystem
+mount -t proc none /proc
+mount -t sysfs none /sys
+mount -t devpts none /dev/pts
+
+# install/remove packages
+sudo apt-get update
+sudo apt-get -y upgrade
+sudo apt-get -y dist-upgrade
+sudo apt-get install libxslt-dev libxml2-dev libvirt-dev build-essential qemu-utils qemu-kvm libvirt-bin virtinst -y
+
+#rm  /etc/resolv.conf
+#rm -rf /tmp/*
+
+umount /proc
+umount /sys
+umount /dev/pts
\ No newline at end of file
diff --git a/compass/ci/build.sh b/compass/ci/build.sh
index e69de29..4e5b87b 100644..100755
--- a/compass/ci/build.sh
+++ b/compass/ci/build.sh
@@ -0,0 +1,392 @@
+#!/bin/bash
+set -e
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# chigang@huawei.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+trap 'echo "Exiting ..."; \
+if [ -f ${LOCK_FILE} ]; then \
+   if [ $(cat ${LOCK_FILE}) -eq $$ ]; then \
+      rm -f ${LOCK_FILE}; \
+   fi; \
+fi;' EXIT
+
+############################################################################
+# BEGIN of usage description
+#
+usage ()
+{
+cat << EOF
+$0 Builds the Compass@OPNFV stack
+
+usage: $0 [-s spec-file] [-c cache-URI] [-l log-file] [-f Flags] build-directory
+
+OPTIONS:
+  -s spec-file ($BUILD_SPEC), define the build-spec file, default ../build/config.mk
+  -c cache base URI ($BUILD_CACHE_URI), specifies the base URI to a build cache to be used/updated - the name is automatically generated from the md5sum of the spec-file, http://, ftp://, file://[absolute path] suported.
+
+  -l log-file ($BUILD_LOG), specifies the output log-file (stdout and stderr), if not specified logs are output to console as normal
+  -v version tag to be applied to the build result
+  -r alternative remote access method script/program. curl is default.
+  -t run small build-script unit test.
+  -T run large build-script unit test.
+  -f build flags ($BUILD_FLAGS):
+     o s: Do nothing, succeed
+     o f: Do nothing, fail
+     o t: run build unit tests
+     o i: run interactive (-t flag to docker run)
+     o P: Populate a new local cache and push it to the (-c cache-URI) cache artifactory if -c option is present, currently file://, http:// and ftp:// are supported
+     o d: Detatch - NOT YET SUPPORTED
+
+  build-directory ($BUILD_DIR), specifies the directory for the output artifacts (.iso file).
+
+  -h help, prints this help text
+
+Description:
+build.sh builds opnfv .iso artifact.
+To reduce build time it uses build cache on a local or remote location. The cache is rebuilt and uploaded if either of the below conditions are met:
+1) The P(opulate) flag is set and the -c cache-base-URI is provided, if -c is not provided the cache will stay local.
+2) If the cache is invalidated by one of the following conditions:
+   - The config spec md5sum does not compare to the md5sum for the spec which the cache was built.
+   - The git Commit-Id on the remote repos/HEAD defined in the spec file does not correspont with the Commit-Id for what the cache was built with.
+3) A valid cache does not exist on the specified -c cache-base-URI.
+
+The cache URI object name is compass_cache-"md5sum(spec file)"
+
+Logging by default to console, but can be directed elsewhere with the -l option in which case both stdout and stderr is redirected to that destination.
+
+Built in unit testing of components is enabled by adding the t(est) flag.
+
+Return codes:
+ - 0 Success!
+ - 1-99 Unspecified build error
+ - 100-199 Build system internal error (not build it self)
+   o 101 Build system instance busy
+ - 200 Build failure
+
+Examples:
+build -c http://opnfv.org/artifactory/compass/cache -d ~/jenkins/genesis/compass/ci/output -f ti
+NOTE: At current the build scope is set to the git root of the repository, -d destination locations outside that scope will not work
+EOF
+}
+#
+# END of usage description
+############################################################################
+
+############################################################################
+# BEGIN of variables to customize
+#
+BUILD_BASE=$(readlink -e ../build/)
+RESULT_DIR="${BUILD_BASE}/release"
+BUILD_SPEC="${BUILD_BASE}/config.mk"
+CACHE_DIR="cache"
+LOCAL_CACHE_ARCH_NAME="compass-cache"
+REMOTE_CACHE_ARCH_NAME="compass_cache-$(md5sum ${BUILD_SPEC}| cut -f1 -d " ")"
+REMOTE_ACCESS_METHD=curl
+INCLUDE_DIR=../include
+#
+# END of variables to customize
+############################################################################
+
+############################################################################
+# BEGIN of script assigned variables
+#
+SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+LOCK_FILE="${SCRIPT_DIR}/.build.lck"
+CACHE_TMP="${SCRIPT_DIR}/tmp"
+TEST_SUCCEED=0
+TEST_FAIL=0
+UNIT_TEST=0
+UPDATE_CACHE=0
+POPULATE_CACHE=0
+RECURSIV=0
+DETACH=0
+DEBUG=0
+INTEGRATION_TEST=0
+FULL_INTEGRATION_TEST=0
+INTERACTIVE=0
+BUILD_CACHE_URI=
+BUILD_SPEC=
+BUILD_DIR=
+BUILD_LOG=
+BUILD_VERSION=
+MAKE_ARGS=
+#
+# END of script assigned variables
+############################################################################
+
+############################################################################
+# BEGIN of include pragmas
+#
+source ${INCLUDE_DIR}/build.sh.debug
+#
+# END of include
+############################################################################
+
+############################################################################
+# BEGIN of main
+#
+while getopts "s:c:v:f:l:r:RtTh" OPTION
+do
+    case $OPTION in
+	h)
+	    usage
+	    rc=0
+	    exit $rc
+	    ;;
+
+	s)
+	    BUILD_SPEC=${OPTARG}
+	    ;;
+
+	c)
+	    BUILD_CACHE_URI=${OPTARG}
+	    ;;
+
+	l)
+	    BUILD_LOG=${OPTARG}
+	    ;;
+
+	v)
+	    BUILD_VERSION=${OPTARG}
+	    ;;
+
+	f)
+	    BUILD_FLAGS=${OPTARG}
+	    ;;
+
+	r)  REMOTE_ACCESS_METHD=${OPTARG}
+	    ;;
+
+	R)
+	    RECURSIVE=1
+	    ;;
+
+	t)
+	    INTEGRATION_TEST=1
+	    ;;
+
+	T)
+	    INTEGRATION_TEST=1
+	    FULL_INTEGRATION_TEST=1
+	    ;;
+
+	*)
+	    echo "${OPTION} is not a valid argument"
+	    rc=100
+	    exit $rc
+	    ;;
+    esac
+done
+
+if [ -z $BUILD_DIR ]; then
+    BUILD_DIR=$(echo $@ | cut -d ' ' -f ${OPTIND})
+fi
+
+for ((i=0; i<${#BUILD_FLAGS};i++)); do
+    case ${BUILD_FLAGS:$i:1} in
+	s)
+	    rc=0
+	    exit $rc
+	    ;;
+
+	f)
+	    rc=1
+	    exit $rc
+	    ;;
+
+	t)
+	    UNIT_TEST=1
+	    ;;
+
+	i)
+	    INTERACTIVE=1
+	    ;;
+
+	P)
+	    POPULATE_CACHE=1
+	    ;;
+
+	d)
+	    DETACH=1
+	    echo "Detach is not yet supported - exiting ...."
+	    rc=100
+	    exit $rc
+	    ;;
+
+	D)
+ 	    DEBUG=1
+	    ;;
+
+	*)
+	    echo "${BUILD_FLAGS:$i:1} is not a valid build flag - exiting ...."
+	    rc=100
+	    exit $rc
+	    ;;
+    esac
+done
+
+if [ ${INTEGRATION_TEST} -eq 1 ]; then
+    integration-test
+    rc=0
+    exit $rc
+fi
+
+if [ ! -f ${BUILD_SPEC} ]; then
+    echo "spec file does not exist: $BUILD_SPEC - exiting ...."
+    rc=100
+    exit $rc
+fi
+
+if [ -z ${BUILD_DIR} ]; then
+    echo "Missing build directory - exiting ...."
+    rc=100
+    exit $rc
+fi
+
+if [ ! -z ${BUILD_LOG} ]; then
+    if [[ ${RECURSIVE} -ne 1 ]]; then
+	set +e
+	eval $0 -R $@ > ${BUILD_LOG} 2>&1
+	rc=$?
+	set -e
+	if [ $rc -ne 0 ]; then
+	    exit $rc
+	fi
+    fi
+fi
+
+if [ ${TEST_SUCCEED} -eq 1 ]; then
+    sleep 1
+    rc=0
+    exit $rc
+fi
+
+if [ ${TEST_FAIL} -eq 1 ]; then
+    sleep 1
+    rc=1
+    exit $rc
+fi
+
+if [ -e ${LOCK_FILE} ]; then
+    echo "A build job is already running, exiting....."
+    rc=101
+    exit $rc
+fi
+
+echo $$ > ${LOCK_FILE}
+
+if [ ! -z ${BUILD_CACHE_URI} ]; then
+    if [ ${POPULATE_CACHE} -ne 1 ]; then
+	rm -rf ${CACHE_TMP}/cache
+	mkdir -p ${CACHE_TMP}/cache
+	echo "Downloading cach file ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME} ..."
+	set +e
+	${REMOTE_ACCESS_METHD} -o ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz
+	rc=$?
+	set -e
+	if [ $rc -ne 0 ]; then
+		echo "Remote cache does not exist, or is not accessible - a new cache will be built ..."
+		POPULATE_CACHE=1
+	else
+	    echo "Unpacking cache file ..."
+	    tar -C ${CACHE_TMP}/cache -xvf ${CACHE_TMP}/cache/${LOCAL_CACHE_ARCH_NAME}.tgz
+	    cp ${CACHE_TMP}/cache/cache/.versions ${BUILD_BASE}/.
+	    set +e
+       	    make -C ${BUILD_BASE} validate-cache;
+	    rc=$?
+	    set -e
+
+	    if [ $rc -ne 0 ]; then
+		echo "Cache invalid - a new cache will be built "
+		POPULATE_CACHE=1
+	    else
+		cp -rf ${CACHE_TMP}/cache/cache/. ${BUILD_BASE}
+	    fi
+	    rm -rf ${CACHE_TMP}/cache
+	fi
+    fi
+fi
+
+if [ ${POPULATE_CACHE} -eq 1 ]; then
+    if [ ${DEBUG} -eq 0 ]; then
+	set +e
+	cd ${BUILD_BASE} && make clean
+	rc=$?
+	set -e
+	if [ $rc -ne 0 ]; then
+	    echo "Build - make clean failed, exiting ..."
+	    rc=100
+	    exit $rc
+	fi
+    fi
+fi
+
+if [ ! -z ${BUILD_VERSION} ]; then
+    MAKE_ARGS+="REVSTATE=${BUILD_VERSION} "
+fi
+
+if [ ${UNIT_TEST} -eq 1 ]; then
+    MAKE_ARGS+="UNIT_TEST=TRUE "
+else
+    MAKE_ARGS+="UNIT_TEST=FALSE "
+fi
+
+if [ ${INTERACTIVE} -eq 1 ]; then
+    MAKE_ARGS+="INTERACTIVE=TRUE "
+else
+    MAKE_ARGS+="INTERACTIVE=FALSE "
+fi
+
+MAKE_ARGS+=all
+
+if [ ${DEBUG} -eq 0 ]; then
+    set +e
+    cd ${BUILD_BASE} && make ${MAKE_ARGS}
+    rc=$?
+    set -e
+    if [ $rc -gt 0 ]; then
+	echo "Build: make all failed, exiting ..."
+	rc=200
+	exit $rc
+    fi
+else
+debug_make
+fi
+set +e
+make -C ${BUILD_BASE} prepare-cache
+rc=$?
+set -e
+
+if [ $rc -gt 0 ]; then
+    echo "Build: make prepare-cache failed - exiting ..."
+    rc=100
+    exit $rc
+fi
+echo "Copying built OPNFV .iso file to target directory ${BUILD_DIR} ..."
+rm -rf ${BUILD_DIR}
+mkdir -p ${BUILD_DIR}
+cp ${BUILD_BASE}/.versions ${BUILD_DIR}
+cp ${RESULT_DIR}/*.iso* ${BUILD_DIR}
+
+if [ $POPULATE_CACHE -eq 1 ]; then
+    if [ ! -z ${BUILD_CACHE_URI} ]; then
+	echo "Building cache ..."
+	tar --dereference -C ${BUILD_BASE} -caf ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${CACHE_DIR}
+	echo "Uploading cache ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}"
+	${REMOTE_ACCESS_METHD} -T ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz ${BUILD_CACHE_URI}/${REMOTE_CACHE_ARCH_NAME}.tgz
+	rm ${BUILD_BASE}/${LOCAL_CACHE_ARCH_NAME}.tgz
+    fi
+fi
+echo "Success!!!"
+exit 0
+#
+# END of main
+############################################################################