Simplification and cleanup of build process

Docker build image:
- Uplift of the base Ubuntu version to 14.04.
- Instead of building the Ubuntu image with debootstrap, we use the official
  Docker repository version of Ubuntu.
- Cleanup of the way how runcontext operates
- Fix: Make sure that the build system can be run as root

Top Makefile:
- Building the Fuel ISO on Docker local /tmp directory not to pollute local
  directory with files owned by root.
- Removed unused targets.

Change-Id: I684348b2a15546eadf98d810239a84ff9798b1e8
Signed-off-by: Stefan K. Berg <stefan.k.berg@ericsson.com>

10 files changed, 64 insertions, 335 deletions

diff --git a/fuel-build/DOC/BUILD/README.build b/fuel-build/DOC/BUILD/README.build
index 6cd87e7..a5e1112 100644
--- a/fuel-build/DOC/BUILD/README.build
+++ b/fuel-build/DOC/BUILD/README.build
@@ -21,7 +21,6 @@ How to setup and use the OPNFV/BGS/FUEL build system - work in progress
 1.2 Host SW dependencies
 ------------------------
 On the host, the following packages must be installed:
-- debootstrap (simply available through apt-get install debootstrap)
 - docker - see https://docs.docker.com/installation/ubuntulinux/ for installation notes for 12.04 and 14.04
 - git (simply available through apt-get install git)
 
diff --git a/fuel-build/Makefile b/fuel-build/Makefile
index ea403c8..98b108c 100644
--- a/fuel-build/Makefile
+++ b/fuel-build/Makefile
@@ -12,20 +12,19 @@
 # BEGIN of variables to customize
 #
 SHELL = /bin/bash
-
-
 ISOSRC = file:$(shell pwd)/fuel-5.1.1.iso
 ISOCACHE = $(shell pwd)/$(shell basename $(ISOSRC))
 PRODUCT_NAME = "OPNFV_BGS"
 VERSION = "P0000"
 NEWISO = $(shell pwd)/release/opnfv-${VERSION}.iso
-DOCKERIMG = opnfv.org/ubuntu-builder:12.04
+DOCKERIMG = opnfv.org/ubuntu-builder:14.04
 # Note! Invoke with "make VERSION=RXXXX iso" to make release build!
 # Invoke with ICOCACHE=/full/path/to/iso if cached ISO is in non-standard location.
 #
 # END of variables to customize
 #############################################################################
-ORIGDIR = $(shell pwd)/origiso
+TOPDIR := $(shell pwd)
+ORIGDIR := $(TOPDIR)/origiso
 
 SUBDIRS := f_isoroot
 SUBDIRS += f_opnfv_puppet
@@ -43,38 +42,27 @@ SUBCLEAN = $(addsuffix .clean,$(SUBDIRS))
 .PHONY: all
 all:	
 	@docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1)
-	make -C docker
+	@make -C docker
 	@docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS) iso
 
 $(ISOCACHE):
-	# Clone Fuel
-	sudo rm -rf fuel-main
-	git clone https://github.com/stackforge/fuel-main
-	cd fuel-main && git checkout 5.1.1
+	# Clone Fuel to non-persistent location and build
+	cd /tmp && git clone https://github.com/stackforge/fuel-main
+	cd /tmp/fuel-main && git checkout 5.1.1
 	# Setup cgroups for docker-in-docker
 	sudo /root/enable_dockerx2
-	# Need to patch to fix race condition in build
-	cd fuel-main && patch -p1 < ../fuel-main.patches
+	# Patch to fix race condition when doing "Docker-in-Docker" build
+	cd /tmp/fuel-main && patch -p1 < $(TOPDIR)/fuel-main.patches
+	# Remove Docker optimizations, otherwise multistrap will fail during
+	# Fuel build.
+	sudo rm -f /etc/apt/apt.conf.d/docker*
 	#
-	cd fuel-main && ./prepare-build-env.sh
-	# Need to patch not supported start-stop-daemon argument
-	sudo sed -i 's/--no-close//' /etc/init.d/docker
-	sudo /etc/init.d/docker start
-	cd fuel-main && make iso
-	mv fuel-main/build/artifacts/fuel*.iso .
-	# Remove Fuel
-	sudo rm -rf fuel-main
-
-# Used the "docker" target to build within the context of the Docker
-# container created by the Makefile in the docker subdirectory.
-.PHONY: docker
-docker: /.dockerinit
+	cd /tmp/fuel-main && ./prepare-build-env.sh
+	cd /tmp/fuel-main && make iso
+	mv /tmp/fuel-main/build/artifacts/fuel*.iso .
 
-/.dockerinit:
-	docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS)
-
-.PHONY: get-iso mount-origiso umount-origiso
-mount-origiso:
+.PHONY: mount-origiso umount-origiso
+mount-origiso: $(ISOCACHE)
 	@echo "Mounting original ISO in $(ORIGDIR)"
 	@mkdir -p $(ORIGDIR)
 	@fuseiso $(ISOCACHE) $(ORIGDIR)
@@ -84,13 +72,6 @@ umount-origiso:
 	@fusermount -u $(ORIGDIR)
 	@rmdir $(ORIGDIR)
 
-get-iso:
-	wget $(ISOSRC) -O $(ISOCACHE)
-
-
-	@echo "ISO data of source ISO:" >> release/.isodata
-	@isoinfo -d -i $(ISOCACHE) >> release/.isodata
-
 .PHONY: $(SUBDIRS)
 $(SUBDIRS):
 	@mkdir -p release/packages/ubuntu/pool/main release/puppet/modules release/isoroot
@@ -110,14 +91,6 @@ clean: $(SUBCLEAN)
 $(SUBCLEAN): %.clean:
 	$(MAKE) -C $* -f Makefile clean
 
-.PHONY: install
-install:$(SUBDIRS) patch-packages
-	install/install.sh live
-
-.PHONY: uninstall
-uninstall:
-	install/uninstall.sh
-
 # Todo: Make things smarter - we shouldn't need to clean everything
 # betwen make invocations.
 .PHONY: iso
diff --git a/fuel-build/docker/Makefile b/fuel-build/docker/Makefile
index 4f00de8..360bb54 100644
--- a/fuel-build/docker/Makefile
+++ b/fuel-build/docker/Makefile
@@ -8,25 +8,16 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-CONTEXT1 := opnfv.org/ubuntu:12.04
-CONTEXT2 := opnfv.org/ubuntu-builder:12.04
+FILES = $(wildcard ubuntu-builder/*) runcontext
 
-.PHONY: docker all
+.PHONY: all
+all:	.docker
 
-all:	create-docker-ubuntu populate-docker-ubuntu
-	./runcontext ${CONTEXT2} make test
+.docker: $(FILES)
+	/usr/bin/docker build --rm=true -t opnfv.org/ubuntu-builder:14.04 ubuntu-builder
+	/usr/bin/docker tag -f opnfv.org/ubuntu-builder:14.04 opnfv.org/ubuntu-builder
+	touch .docker
 
-.PHONY:	create-docker-ubuntu
-create-docker-ubuntu:
-	/usr/bin/docker run --rm -i -t ${CONTEXT1} date || \
-	sudo mkimage/mkimage.sh -t opnfv.org/ubuntu:12.04 debootstrap --include=ubuntu-minimal --components=main,universe precise
-
-.PHONY: populate-docker-ubuntu
-populate-docker-ubuntu:
-	/usr/bin/docker build --rm=true -t opnfv.org/ubuntu-builder:12.04 ubuntu-builder
-	/usr/bin/docker tag -f opnfv.org/ubuntu-builder:12.04 opnfv.org/ubuntu-builder
-
-.PHONY:	test
-test:
-	@hostname
-	@date
+.PHONY:	clean
+clean:
+	rm .docker
diff --git a/fuel-build/docker/README b/fuel-build/docker/README
index 5d22bc9..e5ccdfa 100644
--- a/fuel-build/docker/README
+++ b/fuel-build/docker/README
@@ -13,13 +13,12 @@ used for building in Ubuntu 12.04 context.
 
 There are two requirements before running make on this level:
 
-1. You need to have Docker installed on your system, and make sure
-to have a valid DNS config in your /etc/default/docker.
-e.g:
-DOCKER_OPTS=" --dns=8.8.8.8 --dns=8.8.8.4"
+1. You need to have Docker installed on your system
 
+2. If in a corporate network, make sure to have a valid DNS config in
+   your /etc/default/docker, e.g:
 
-2. You must have installed the package "debootstrap".
+   DOCKER_OPTS=" --dns=8.8.8.8 --dns=8.8.8.4"
 
 The top level makefile will refer to the "runcontext" file in order to
 run the build process as the current user with the current working
diff --git a/fuel-build/docker/mkimage/debootstrap b/fuel-build/docker/mkimage/debootstrap
deleted file mode 100755
index 0862120..0000000
--- a/fuel-build/docker/mkimage/debootstrap
+++ /dev/null
@@ -1,126 +0,0 @@
-#!/usr/bin/env bash
-#
-set -e
-
-rootfsDir="$1"
-shift
-
-# we have to do a little fancy footwork to make sure "rootfsDir" becomes the second non-option argument to debootstrap
-
-before=()
-while [ $# -gt 0 ] && [[ "$1" == -* ]]; do
-	before+=( "$1" )
-	shift
-done
-
-suite="$1"
-shift
-
-(
-	set -x
-	debootstrap "${before[@]}" "$suite" "$rootfsDir" "$@"
-)
-
-# now for some Docker-specific tweaks
-
-# prevent init scripts from running during install/update
-echo >&2 "+ cat > '$rootfsDir/usr/sbin/policy-rc.d'"
-cat > "$rootfsDir/usr/sbin/policy-rc.d" <<'EOF'
-#!/bin/sh
-exit 101
-EOF
-chmod +x "$rootfsDir/usr/sbin/policy-rc.d"
-
-# prevent upstart scripts from running during install/update
-(
-	set -x
-	chroot "$rootfsDir" dpkg-divert --local --rename --add /sbin/initctl
-	ln -sf /bin/true "$rootfsDir/sbin/initctl"
-)
-
-# shrink the image, since apt makes us fat (wheezy: ~157.5MB vs ~120MB)
-( set -x; chroot "$rootfsDir" apt-get clean )
-
-# Ubuntu 10.04 sucks... :)
-if strings "$rootfsDir/usr/bin/dpkg" | grep -q unsafe-io; then
-	# force dpkg not to call sync() after package extraction (speeding up installs)
-	echo >&2 "+ echo force-unsafe-io > '$rootfsDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup'"
-	echo 'force-unsafe-io' > "$rootfsDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup"
-fi
-
-if [ -d "$rootfsDir/etc/apt/apt.conf.d" ]; then
-	# _keep_ us lean by effectively running "apt-get clean" after every install
-	aptGetClean='"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true";'
-	echo >&2 "+ cat > '$rootfsDir/etc/apt/apt.conf.d/docker-clean'"
-	cat > "$rootfsDir/etc/apt/apt.conf.d/docker-clean" <<-EOF
-		DPkg::Post-Invoke { ${aptGetClean} };
-		APT::Update::Post-Invoke { ${aptGetClean} };
-
-		Dir::Cache::pkgcache "";
-		Dir::Cache::srcpkgcache "";
-	EOF
-
-	# remove apt-cache translations for fast "apt-get update"
-	echo >&2 "+ cat > '$rootfsDir/etc/apt/apt.conf.d/docker-no-languages'"
-	echo 'Acquire::Languages "none";' > "$rootfsDir/etc/apt/apt.conf.d/docker-no-languages"
-fi
-
-if [ -z "$DONT_TOUCH_SOURCES_LIST" ]; then
-	# tweak sources.list, where appropriate
-	lsbDist=
-	if [ -z "$lsbDist" -a -r "$rootfsDir/etc/os-release" ]; then
-		lsbDist="$(. "$rootfsDir/etc/os-release" && echo "$ID")"
-	fi
-	if [ -z "$lsbDist" -a -r "$rootfsDir/etc/lsb-release" ]; then
-		lsbDist="$(. "$rootfsDir/etc/lsb-release" && echo "$DISTRIB_ID")"
-	fi
-	if [ -z "$lsbDist" -a -r "$rootfsDir/etc/debian_version" ]; then
-		lsbDist='Debian'
-	fi
-	case "$lsbDist" in
-		debian|Debian)
-			# updates and security!
-			if [ "$suite" != 'sid' -a "$suite" != 'unstable' ]; then
-				(
-					set -x
-					sed -i "p; s/ $suite main$/ ${suite}-updates main/" "$rootfsDir/etc/apt/sources.list"
-					echo "deb http://security.debian.org $suite/updates main" >> "$rootfsDir/etc/apt/sources.list"
-				)
-			fi
-			;;
-		ubuntu|Ubuntu)
-			# add the universe, updates, and security repositories
-			(
-				set -x
-				sed -i "
-					s/ $suite main$/ $suite main universe/; p;
-					s/ $suite main/ ${suite}-updates main/; p;
-					s/ $suite-updates main/ ${suite}-security main/
-				" "$rootfsDir/etc/apt/sources.list"
-			)
-			;;
-		tanglu|Tanglu)
-			# add the updates repository
-			if [ "$suite" != 'devel' ]; then
-				(
-					set -x
-					sed -i "p; s/ $suite main$/ ${suite}-updates main/" "$rootfsDir/etc/apt/sources.list"
-				)
-			fi
-			;;
-		steamos|SteamOS)
-			# add contrib and non-free
-			(
-				set -x
-				sed -i "s/ $suite main$/ $suite main contrib non-free/" "$rootfsDir/etc/apt/sources.list"
-			)
-			;;
-	esac
-fi
-
-# make sure we're fully up-to-date, too
-(
-	set -x
-	chroot "$rootfsDir" apt-get update
-	chroot "$rootfsDir" apt-get dist-upgrade -y
-)
diff --git a/fuel-build/docker/mkimage/mkimage.sh b/fuel-build/docker/mkimage/mkimage.sh
deleted file mode 100755
index 3424258..0000000
--- a/fuel-build/docker/mkimage/mkimage.sh
+++ /dev/null
@@ -1,116 +0,0 @@
-#!/usr/bin/env bash
-##############################################################################
-# Copyright (c) 2015 Ericsson AB and others.
-# stefan.k.berg@ericsson.com
-# jonas.bjurel@ericsson.com
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-set -e
-
-mkimg="$(basename "$0")"
-
-usage() {
-        echo >&2 "usage: $mkimg [-d dir] [-t tag] script [script-args]"
-        echo >&2 "   ie: $mkimg -t someuser/debian debootstrap --variant=minbase jessie"
-        echo >&2 "       $mkimg -t someuser/ubuntu debootstrap --include=ubuntu-minimal trusty"
-        echo >&2 "       $mkimg -t someuser/busybox busybox-static"
-        echo >&2 "       $mkimg -t someuser/centos:5 rinse --distribution centos-5"
-        exit 1
-}
-
-scriptDir="$(dirname "$(readlink -f "$BASH_SOURCE")")"
-
-optTemp=$(getopt --options '+d:t:h' --longoptions 'dir:,tag:,help' --name "$mkimg" -- "$@")
-eval set -- "$optTemp"
-unset optTemp
-
-dir=
-tag=
-while true; do
-        case "$1" in
-                -d|--dir) dir="$2" ; shift 2 ;;
-                -t|--tag) tag="$2" ; shift 2 ;;
-                -h|--help) usage ;;
-                --) shift ; break ;;
-        esac
-done
-
-script="$1"
-[ "$script" ] || usage
-shift
-
-if [ ! -x "$scriptDir/$script" ]; then
-        echo >&2 "error: $script does not exist or is not executable"
-        echo >&2 "  see $scriptDir for possible scripts"
-        exit 1
-fi
-
-# don't mistake common scripts like .febootstrap-minimize as image-creators
-if [[ "$script" == .* ]]; then
-        echo >&2 "error: $script is a script helper, not a script"
-        echo >&2 "  see $scriptDir for possible scripts"
-        exit 1
-fi
-
-delDir=
-if [ -z "$dir" ]; then
-        dir="$(mktemp -d ${TMPDIR:-/tmp}/docker-mkimage.XXXXXXXXXX)"
-        delDir=1
-fi
-
-rootfsDir="$dir/rootfs"
-( set -x; mkdir -p "$rootfsDir" )
-
-# pass all remaining arguments to $script
-"$scriptDir/$script" "$rootfsDir" "$@"
-
-# Docker mounts tmpfs at /dev and procfs at /proc so we can remove them
-rm -rf "$rootfsDir/dev" "$rootfsDir/proc"
-mkdir -p "$rootfsDir/dev" "$rootfsDir/proc"
-
-# make sure /etc/resolv.conf has something useful in it
-mkdir -p "$rootfsDir/etc"
-cat > "$rootfsDir/etc/resolv.conf" <<'EOF'
-nameserver 8.8.8.8
-nameserver 8.8.4.4
-EOF
-
-tarFile="$dir/rootfs.tar.xz"
-touch "$tarFile"
-
-(
-        set -x
-        tar --numeric-owner -caf "$tarFile" -C "$rootfsDir" --transform='s,^./,,' .
-)
-
-echo >&2 "+ cat > '$dir/Dockerfile'"
-cat > "$dir/Dockerfile" <<'EOF'
-FROM scratch
-ADD rootfs.tar.xz /
-EOF
-
-# if our generated image has a decent shell, let's set a default command
-for shell in /bin/bash /usr/bin/fish /usr/bin/zsh /bin/sh; do
-        if [ -x "$rootfsDir/$shell" ]; then
-                ( set -x; echo 'CMD ["'"$shell"'"]' >> "$dir/Dockerfile" )
-                break
-        fi
-done
-
-umount $rootfsDir/run/shm
-( set -x; rm -rf "$rootfsDir" )
-
-if [ "$tag" ]; then
-        ( set -x; docker build -t "$tag" "$dir" )
-elif [ "$delDir" ]; then
-        # if we didn't specify a tag and we're going to delete our dir, let's just build an untagged image so that we did _something_
-        ( set -x; docker build "$dir" )
-fi
-
-if [ "$delDir" ]; then
-        ( set -x; rm -rf "$dir" )
-fi
diff --git a/fuel-build/docker/runcontext b/fuel-build/docker/runcontext
index 45a7645..d4e8b9c 100755
--- a/fuel-build/docker/runcontext
+++ b/fuel-build/docker/runcontext
@@ -15,16 +15,11 @@ GID=`id -g`
 USER=`whoami`
 res=`docker build -q --rm -  <<EOF
 FROM $context
-RUN echo "invalidate cache" && echo "done"
-RUN /usr/sbin/groupadd --gid $GID $USER
-RUN /usr/sbin/adduser --system --uid=$UID --gid=$GID --home $HOME --shell /bin/bash $USER
-RUN /usr/sbin/usermod -a -G fuse $USER
-RUN echo "export HOME=$HOME" >> /etc/bash.bashrc
-RUN echo "$@" >> /etc/bash.bashrc
-RUN echo 'exit \\$?' >> /etc/bash.bashrc
+RUN date || date
+RUN /root/setcontext $USER $UID $GID $HOME
 EOF`
 CID=`echo $res | sed 's/.* //'`
-docker run --privileged=true --rm -i -t -u $USER -w $PWD -v ${HOME}/.ssh:${HOME}/.ssh -v $PWD:$PWD $CID bash
+docker run --privileged=true --rm -i -t -e HOME=$HOME -u $USER -w $PWD -v ${HOME}/.ssh:${HOME}/.ssh -v $PWD:$PWD $CID "$@"
 rc=$?
 docker rmi $CID > /dev/null
 exit $rc
diff --git a/fuel-build/docker/ubuntu-builder/Dockerfile b/fuel-build/docker/ubuntu-builder/Dockerfile
index e91afbd..3ec934d 100644
--- a/fuel-build/docker/ubuntu-builder/Dockerfile
+++ b/fuel-build/docker/ubuntu-builder/Dockerfile
@@ -1,3 +1,4 @@
+#!/bin/bash
 ##############################################################################
 # Copyright (c) 2015 Ericsson AB and others.
 # stefan.k.berg@ericsson.com
@@ -8,11 +9,17 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-FROM opnfv.org/ubuntu:12.04
-ADD install.sh /root/install.sh
-RUN /root/install.sh
+FROM ubuntu:14.04
+RUN apt-get update
+RUN apt-get install -y make python-setuptools python-all dpkg-dev debhelper \
+    fuseiso git genisoimage bind9-host wget curl lintian tmux lxc iptables \
+    ca-certificates sudo apt-utils lsb-release
 
+RUN echo "ALL ALL=NOPASSWD: ALL" > /etc/sudoers.d/open-sudo
+RUN chmod 0440 /etc/sudoers.d/open-sudo
+
+ADD ./setcontext /root/setcontext
+RUN chmod +x /root/setcontext
 ADD ./enable_dockerx2 /root/enable_dockerx2
 RUN chmod +x /root/enable_dockerx2
 VOLUME /var/lib/docker
-CMD ["/root/enable_dockerx2"]
diff --git a/fuel-build/docker/ubuntu-builder/enable_dockerx2 b/fuel-build/docker/ubuntu-builder/enable_dockerx2
index b6bccdf..e79c8b7 100644
--- a/fuel-build/docker/ubuntu-builder/enable_dockerx2
+++ b/fuel-build/docker/ubuntu-builder/enable_dockerx2
@@ -1,6 +1,15 @@
 #!/bin/sh
+##############################################################################
+# Copyright (c) 2015 Ericsson AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+#
 # Needed to expose underlying cgroups to container
-
 mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup
 mount -t securityfs none /sys/kernel/security
 
diff --git a/fuel-build/docker/ubuntu-builder/install.sh b/fuel-build/docker/ubuntu-builder/setcontext
index 667ab27..bc28994 100755..100644
--- a/fuel-build/docker/ubuntu-builder/install.sh
+++ b/fuel-build/docker/ubuntu-builder/setcontext
@@ -8,15 +8,13 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
+#
+SETUSER=$1
+SETUID=$2
+SETGID=$3
+SETHOME=$4
 
-BASE_PACKAGES="make python-setuptools python-all dpkg-dev debhelper
-fuseiso git genisoimage bind9-host wget curl lintian tmux lxc iptables
-ca-certificates"
-
-apt-get update || exit 1
-apt-get upgrade -y || exit 1
-
-apt-get install -y $BASE_PACKAGES || exit 1
-
-echo "ALL ALL=NOPASSWD: ALL" > /etc/sudoers.d/open-sudo
-chmod 0440 /etc/sudoers.d/open-sudo
+getent group $SETUSER || /usr/sbin/groupadd --gid $SETGID $SETUSER
+getent passwd $SETUSER || /usr/sbin/adduser --system --uid=$SETUID --gid=$SETGID --home $SETHOME --shell /bin/bash $SETUSER
+/usr/sbin/usermod -a -G fuse $SETUSER
+exit 0