diff options
| -rw-r--r-- | fuel-build/DOC/BUILD/README.build | 1 | ||||
| -rw-r--r-- | fuel-build/Makefile | 61 | ||||
| -rw-r--r-- | fuel-build/docker/Makefile | 29 | ||||
| -rw-r--r-- | fuel-build/docker/README | 9 | ||||
| -rwxr-xr-x | fuel-build/docker/mkimage/debootstrap | 126 | ||||
| -rwxr-xr-x | fuel-build/docker/mkimage/mkimage.sh | 116 | ||||
| -rwxr-xr-x | fuel-build/docker/runcontext | 11 | ||||
| -rw-r--r-- | fuel-build/docker/ubuntu-builder/Dockerfile | 15 | ||||
| -rw-r--r-- | fuel-build/docker/ubuntu-builder/enable_dockerx2 | 11 | ||||
| -rw-r--r--[-rwxr-xr-x] | fuel-build/docker/ubuntu-builder/setcontext (renamed from fuel-build/docker/ubuntu-builder/install.sh) | 20 |
10 files changed, 64 insertions, 335 deletions
diff --git a/fuel-build/DOC/BUILD/README.build b/fuel-build/DOC/BUILD/README.build index 6cd87e7..a5e1112 100644 --- a/fuel-build/DOC/BUILD/README.build +++ b/fuel-build/DOC/BUILD/README.build @@ -21,7 +21,6 @@ How to setup and use the OPNFV/BGS/FUEL build system - work in progress 1.2 Host SW dependencies ------------------------ On the host, the following packages must be installed: -- debootstrap (simply available through apt-get install debootstrap) - docker - see https://docs.docker.com/installation/ubuntulinux/ for installation notes for 12.04 and 14.04 - git (simply available through apt-get install git) diff --git a/fuel-build/Makefile b/fuel-build/Makefile index ea403c8..98b108c 100644 --- a/fuel-build/Makefile +++ b/fuel-build/Makefile @@ -12,20 +12,19 @@ # BEGIN of variables to customize # SHELL = /bin/bash - - ISOSRC = file:$(shell pwd)/fuel-5.1.1.iso ISOCACHE = $(shell pwd)/$(shell basename $(ISOSRC)) PRODUCT_NAME = "OPNFV_BGS" VERSION = "P0000" NEWISO = $(shell pwd)/release/opnfv-${VERSION}.iso -DOCKERIMG = opnfv.org/ubuntu-builder:12.04 +DOCKERIMG = opnfv.org/ubuntu-builder:14.04 # Note! Invoke with "make VERSION=RXXXX iso" to make release build! # Invoke with ICOCACHE=/full/path/to/iso if cached ISO is in non-standard location. # # END of variables to customize ############################################################################# -ORIGDIR = $(shell pwd)/origiso +TOPDIR := $(shell pwd) +ORIGDIR := $(TOPDIR)/origiso SUBDIRS := f_isoroot SUBDIRS += f_opnfv_puppet @@ -43,38 +42,27 @@ SUBCLEAN = $(addsuffix .clean,$(SUBDIRS)) .PHONY: all all: @docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1) - make -C docker + @make -C docker @docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS) iso $(ISOCACHE): - # Clone Fuel - sudo rm -rf fuel-main - git clone https://github.com/stackforge/fuel-main - cd fuel-main && git checkout 5.1.1 + # Clone Fuel to non-persistent location and build + cd /tmp && git clone https://github.com/stackforge/fuel-main + cd /tmp/fuel-main && git checkout 5.1.1 # Setup cgroups for docker-in-docker sudo /root/enable_dockerx2 - # Need to patch to fix race condition in build - cd fuel-main && patch -p1 < ../fuel-main.patches + # Patch to fix race condition when doing "Docker-in-Docker" build + cd /tmp/fuel-main && patch -p1 < $(TOPDIR)/fuel-main.patches + # Remove Docker optimizations, otherwise multistrap will fail during + # Fuel build. + sudo rm -f /etc/apt/apt.conf.d/docker* # - cd fuel-main && ./prepare-build-env.sh - # Need to patch not supported start-stop-daemon argument - sudo sed -i 's/--no-close//' /etc/init.d/docker - sudo /etc/init.d/docker start - cd fuel-main && make iso - mv fuel-main/build/artifacts/fuel*.iso . - # Remove Fuel - sudo rm -rf fuel-main - -# Used the "docker" target to build within the context of the Docker -# container created by the Makefile in the docker subdirectory. -.PHONY: docker -docker: /.dockerinit + cd /tmp/fuel-main && ./prepare-build-env.sh + cd /tmp/fuel-main && make iso + mv /tmp/fuel-main/build/artifacts/fuel*.iso . -/.dockerinit: - docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS) - -.PHONY: get-iso mount-origiso umount-origiso -mount-origiso: +.PHONY: mount-origiso umount-origiso +mount-origiso: $(ISOCACHE) @echo "Mounting original ISO in $(ORIGDIR)" @mkdir -p $(ORIGDIR) @fuseiso $(ISOCACHE) $(ORIGDIR) @@ -84,13 +72,6 @@ umount-origiso: @fusermount -u $(ORIGDIR) @rmdir $(ORIGDIR) -get-iso: - wget $(ISOSRC) -O $(ISOCACHE) - - - @echo "ISO data of source ISO:" >> release/.isodata - @isoinfo -d -i $(ISOCACHE) >> release/.isodata - .PHONY: $(SUBDIRS) $(SUBDIRS): @mkdir -p release/packages/ubuntu/pool/main release/puppet/modules release/isoroot @@ -110,14 +91,6 @@ clean: $(SUBCLEAN) $(SUBCLEAN): %.clean: $(MAKE) -C $* -f Makefile clean -.PHONY: install -install:$(SUBDIRS) patch-packages - install/install.sh live - -.PHONY: uninstall -uninstall: - install/uninstall.sh - # Todo: Make things smarter - we shouldn't need to clean everything # betwen make invocations. .PHONY: iso diff --git a/fuel-build/docker/Makefile b/fuel-build/docker/Makefile index 4f00de8..360bb54 100644 --- a/fuel-build/docker/Makefile +++ b/fuel-build/docker/Makefile @@ -8,25 +8,16 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -CONTEXT1 := opnfv.org/ubuntu:12.04 -CONTEXT2 := opnfv.org/ubuntu-builder:12.04 +FILES = $(wildcard ubuntu-builder/*) runcontext -.PHONY: docker all +.PHONY: all +all: .docker -all: create-docker-ubuntu populate-docker-ubuntu - ./runcontext ${CONTEXT2} make test +.docker: $(FILES) + /usr/bin/docker build --rm=true -t opnfv.org/ubuntu-builder:14.04 ubuntu-builder + /usr/bin/docker tag -f opnfv.org/ubuntu-builder:14.04 opnfv.org/ubuntu-builder + touch .docker -.PHONY: create-docker-ubuntu -create-docker-ubuntu: - /usr/bin/docker run --rm -i -t ${CONTEXT1} date || \ - sudo mkimage/mkimage.sh -t opnfv.org/ubuntu:12.04 debootstrap --include=ubuntu-minimal --components=main,universe precise - -.PHONY: populate-docker-ubuntu -populate-docker-ubuntu: - /usr/bin/docker build --rm=true -t opnfv.org/ubuntu-builder:12.04 ubuntu-builder - /usr/bin/docker tag -f opnfv.org/ubuntu-builder:12.04 opnfv.org/ubuntu-builder - -.PHONY: test -test: - @hostname - @date +.PHONY: clean +clean: + rm .docker diff --git a/fuel-build/docker/README b/fuel-build/docker/README index 5d22bc9..e5ccdfa 100644 --- a/fuel-build/docker/README +++ b/fuel-build/docker/README @@ -13,13 +13,12 @@ used for building in Ubuntu 12.04 context. There are two requirements before running make on this level: -1. You need to have Docker installed on your system, and make sure -to have a valid DNS config in your /etc/default/docker. -e.g: -DOCKER_OPTS=" --dns=8.8.8.8 --dns=8.8.8.4" +1. You need to have Docker installed on your system +2. If in a corporate network, make sure to have a valid DNS config in + your /etc/default/docker, e.g: -2. You must have installed the package "debootstrap". + DOCKER_OPTS=" --dns=8.8.8.8 --dns=8.8.8.4" The top level makefile will refer to the "runcontext" file in order to run the build process as the current user with the current working diff --git a/fuel-build/docker/mkimage/debootstrap b/fuel-build/docker/mkimage/debootstrap deleted file mode 100755 index 0862120..0000000 --- a/fuel-build/docker/mkimage/debootstrap +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash -# -set -e - -rootfsDir="$1" -shift - -# we have to do a little fancy footwork to make sure "rootfsDir" becomes the second non-option argument to debootstrap - -before=() -while [ $# -gt 0 ] && [[ "$1" == -* ]]; do - before+=( "$1" ) - shift -done - -suite="$1" -shift - -( - set -x - debootstrap "${before[@]}" "$suite" "$rootfsDir" "$@" -) - -# now for some Docker-specific tweaks - -# prevent init scripts from running during install/update -echo >&2 "+ cat > '$rootfsDir/usr/sbin/policy-rc.d'" -cat > "$rootfsDir/usr/sbin/policy-rc.d" <<'EOF' -#!/bin/sh -exit 101 -EOF -chmod +x "$rootfsDir/usr/sbin/policy-rc.d" - -# prevent upstart scripts from running during install/update -( - set -x - chroot "$rootfsDir" dpkg-divert --local --rename --add /sbin/initctl - ln -sf /bin/true "$rootfsDir/sbin/initctl" -) - -# shrink the image, since apt makes us fat (wheezy: ~157.5MB vs ~120MB) -( set -x; chroot "$rootfsDir" apt-get clean ) - -# Ubuntu 10.04 sucks... :) -if strings "$rootfsDir/usr/bin/dpkg" | grep -q unsafe-io; then - # force dpkg not to call sync() after package extraction (speeding up installs) - echo >&2 "+ echo force-unsafe-io > '$rootfsDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup'" - echo 'force-unsafe-io' > "$rootfsDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" -fi - -if [ -d "$rootfsDir/etc/apt/apt.conf.d" ]; then - # _keep_ us lean by effectively running "apt-get clean" after every install - aptGetClean='"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true";' - echo >&2 "+ cat > '$rootfsDir/etc/apt/apt.conf.d/docker-clean'" - cat > "$rootfsDir/etc/apt/apt.conf.d/docker-clean" <<-EOF - DPkg::Post-Invoke { ${aptGetClean} }; - APT::Update::Post-Invoke { ${aptGetClean} }; - - Dir::Cache::pkgcache ""; - Dir::Cache::srcpkgcache ""; - EOF - - # remove apt-cache translations for fast "apt-get update" - echo >&2 "+ cat > '$rootfsDir/etc/apt/apt.conf.d/docker-no-languages'" - echo 'Acquire::Languages "none";' > "$rootfsDir/etc/apt/apt.conf.d/docker-no-languages" -fi - -if [ -z "$DONT_TOUCH_SOURCES_LIST" ]; then - # tweak sources.list, where appropriate - lsbDist= - if [ -z "$lsbDist" -a -r "$rootfsDir/etc/os-release" ]; then - lsbDist="$(. "$rootfsDir/etc/os-release" && echo "$ID")" - fi - if [ -z "$lsbDist" -a -r "$rootfsDir/etc/lsb-release" ]; then - lsbDist="$(. "$rootfsDir/etc/lsb-release" && echo "$DISTRIB_ID")" - fi - if [ -z "$lsbDist" -a -r "$rootfsDir/etc/debian_version" ]; then - lsbDist='Debian' - fi - case "$lsbDist" in - debian|Debian) - # updates and security! - if [ "$suite" != 'sid' -a "$suite" != 'unstable' ]; then - ( - set -x - sed -i "p; s/ $suite main$/ ${suite}-updates main/" "$rootfsDir/etc/apt/sources.list" - echo "deb http://security.debian.org $suite/updates main" >> "$rootfsDir/etc/apt/sources.list" - ) - fi - ;; - ubuntu|Ubuntu) - # add the universe, updates, and security repositories - ( - set -x - sed -i " - s/ $suite main$/ $suite main universe/; p; - s/ $suite main/ ${suite}-updates main/; p; - s/ $suite-updates main/ ${suite}-security main/ - " "$rootfsDir/etc/apt/sources.list" - ) - ;; - tanglu|Tanglu) - # add the updates repository - if [ "$suite" != 'devel' ]; then - ( - set -x - sed -i "p; s/ $suite main$/ ${suite}-updates main/" "$rootfsDir/etc/apt/sources.list" - ) - fi - ;; - steamos|SteamOS) - # add contrib and non-free - ( - set -x - sed -i "s/ $suite main$/ $suite main contrib non-free/" "$rootfsDir/etc/apt/sources.list" - ) - ;; - esac -fi - -# make sure we're fully up-to-date, too -( - set -x - chroot "$rootfsDir" apt-get update - chroot "$rootfsDir" apt-get dist-upgrade -y -) diff --git a/fuel-build/docker/mkimage/mkimage.sh b/fuel-build/docker/mkimage/mkimage.sh deleted file mode 100755 index 3424258..0000000 --- a/fuel-build/docker/mkimage/mkimage.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash -############################################################################## -# Copyright (c) 2015 Ericsson AB and others. -# stefan.k.berg@ericsson.com -# jonas.bjurel@ericsson.com -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -set -e - -mkimg="$(basename "$0")" - -usage() { - echo >&2 "usage: $mkimg [-d dir] [-t tag] script [script-args]" - echo >&2 " ie: $mkimg -t someuser/debian debootstrap --variant=minbase jessie" - echo >&2 " $mkimg -t someuser/ubuntu debootstrap --include=ubuntu-minimal trusty" - echo >&2 " $mkimg -t someuser/busybox busybox-static" - echo >&2 " $mkimg -t someuser/centos:5 rinse --distribution centos-5" - exit 1 -} - -scriptDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" - -optTemp=$(getopt --options '+d:t:h' --longoptions 'dir:,tag:,help' --name "$mkimg" -- "$@") -eval set -- "$optTemp" -unset optTemp - -dir= -tag= -while true; do - case "$1" in - -d|--dir) dir="$2" ; shift 2 ;; - -t|--tag) tag="$2" ; shift 2 ;; - -h|--help) usage ;; - --) shift ; break ;; - esac -done - -script="$1" -[ "$script" ] || usage -shift - -if [ ! -x "$scriptDir/$script" ]; then - echo >&2 "error: $script does not exist or is not executable" - echo >&2 " see $scriptDir for possible scripts" - exit 1 -fi - -# don't mistake common scripts like .febootstrap-minimize as image-creators -if [[ "$script" == .* ]]; then - echo >&2 "error: $script is a script helper, not a script" - echo >&2 " see $scriptDir for possible scripts" - exit 1 -fi - -delDir= -if [ -z "$dir" ]; then - dir="$(mktemp -d ${TMPDIR:-/tmp}/docker-mkimage.XXXXXXXXXX)" - delDir=1 -fi - -rootfsDir="$dir/rootfs" -( set -x; mkdir -p "$rootfsDir" ) - -# pass all remaining arguments to $script -"$scriptDir/$script" "$rootfsDir" "$@" - -# Docker mounts tmpfs at /dev and procfs at /proc so we can remove them -rm -rf "$rootfsDir/dev" "$rootfsDir/proc" -mkdir -p "$rootfsDir/dev" "$rootfsDir/proc" - -# make sure /etc/resolv.conf has something useful in it -mkdir -p "$rootfsDir/etc" -cat > "$rootfsDir/etc/resolv.conf" <<'EOF' -nameserver 8.8.8.8 -nameserver 8.8.4.4 -EOF - -tarFile="$dir/rootfs.tar.xz" -touch "$tarFile" - -( - set -x - tar --numeric-owner -caf "$tarFile" -C "$rootfsDir" --transform='s,^./,,' . -) - -echo >&2 "+ cat > '$dir/Dockerfile'" -cat > "$dir/Dockerfile" <<'EOF' -FROM scratch -ADD rootfs.tar.xz / -EOF - -# if our generated image has a decent shell, let's set a default command -for shell in /bin/bash /usr/bin/fish /usr/bin/zsh /bin/sh; do - if [ -x "$rootfsDir/$shell" ]; then - ( set -x; echo 'CMD ["'"$shell"'"]' >> "$dir/Dockerfile" ) - break - fi -done - -umount $rootfsDir/run/shm -( set -x; rm -rf "$rootfsDir" ) - -if [ "$tag" ]; then - ( set -x; docker build -t "$tag" "$dir" ) -elif [ "$delDir" ]; then - # if we didn't specify a tag and we're going to delete our dir, let's just build an untagged image so that we did _something_ - ( set -x; docker build "$dir" ) -fi - -if [ "$delDir" ]; then - ( set -x; rm -rf "$dir" ) -fi diff --git a/fuel-build/docker/runcontext b/fuel-build/docker/runcontext index 45a7645..d4e8b9c 100755 --- a/fuel-build/docker/runcontext +++ b/fuel-build/docker/runcontext @@ -15,16 +15,11 @@ GID=`id -g` USER=`whoami` res=`docker build -q --rm - <<EOF FROM $context -RUN echo "invalidate cache" && echo "done" -RUN /usr/sbin/groupadd --gid $GID $USER -RUN /usr/sbin/adduser --system --uid=$UID --gid=$GID --home $HOME --shell /bin/bash $USER -RUN /usr/sbin/usermod -a -G fuse $USER -RUN echo "export HOME=$HOME" >> /etc/bash.bashrc -RUN echo "$@" >> /etc/bash.bashrc -RUN echo 'exit \\$?' >> /etc/bash.bashrc +RUN date || date +RUN /root/setcontext $USER $UID $GID $HOME EOF` CID=`echo $res | sed 's/.* //'` -docker run --privileged=true --rm -i -t -u $USER -w $PWD -v ${HOME}/.ssh:${HOME}/.ssh -v $PWD:$PWD $CID bash +docker run --privileged=true --rm -i -t -e HOME=$HOME -u $USER -w $PWD -v ${HOME}/.ssh:${HOME}/.ssh -v $PWD:$PWD $CID "$@" rc=$? docker rmi $CID > /dev/null exit $rc diff --git a/fuel-build/docker/ubuntu-builder/Dockerfile b/fuel-build/docker/ubuntu-builder/Dockerfile index e91afbd..3ec934d 100644 --- a/fuel-build/docker/ubuntu-builder/Dockerfile +++ b/fuel-build/docker/ubuntu-builder/Dockerfile @@ -1,3 +1,4 @@ +#!/bin/bash ############################################################################## # Copyright (c) 2015 Ericsson AB and others. # stefan.k.berg@ericsson.com @@ -8,11 +9,17 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -FROM opnfv.org/ubuntu:12.04 -ADD install.sh /root/install.sh -RUN /root/install.sh +FROM ubuntu:14.04 +RUN apt-get update +RUN apt-get install -y make python-setuptools python-all dpkg-dev debhelper \ + fuseiso git genisoimage bind9-host wget curl lintian tmux lxc iptables \ + ca-certificates sudo apt-utils lsb-release +RUN echo "ALL ALL=NOPASSWD: ALL" > /etc/sudoers.d/open-sudo +RUN chmod 0440 /etc/sudoers.d/open-sudo + +ADD ./setcontext /root/setcontext +RUN chmod +x /root/setcontext ADD ./enable_dockerx2 /root/enable_dockerx2 RUN chmod +x /root/enable_dockerx2 VOLUME /var/lib/docker -CMD ["/root/enable_dockerx2"] diff --git a/fuel-build/docker/ubuntu-builder/enable_dockerx2 b/fuel-build/docker/ubuntu-builder/enable_dockerx2 index b6bccdf..e79c8b7 100644 --- a/fuel-build/docker/ubuntu-builder/enable_dockerx2 +++ b/fuel-build/docker/ubuntu-builder/enable_dockerx2 @@ -1,6 +1,15 @@ #!/bin/sh +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# # Needed to expose underlying cgroups to container - mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup mount -t securityfs none /sys/kernel/security diff --git a/fuel-build/docker/ubuntu-builder/install.sh b/fuel-build/docker/ubuntu-builder/setcontext index 667ab27..bc28994 100755..100644 --- a/fuel-build/docker/ubuntu-builder/install.sh +++ b/fuel-build/docker/ubuntu-builder/setcontext @@ -8,15 +8,13 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +# +SETUSER=$1 +SETUID=$2 +SETGID=$3 +SETHOME=$4 -BASE_PACKAGES="make python-setuptools python-all dpkg-dev debhelper -fuseiso git genisoimage bind9-host wget curl lintian tmux lxc iptables -ca-certificates" - -apt-get update || exit 1 -apt-get upgrade -y || exit 1 - -apt-get install -y $BASE_PACKAGES || exit 1 - -echo "ALL ALL=NOPASSWD: ALL" > /etc/sudoers.d/open-sudo -chmod 0440 /etc/sudoers.d/open-sudo +getent group $SETUSER || /usr/sbin/groupadd --gid $SETGID $SETUSER +getent passwd $SETUSER || /usr/sbin/adduser --system --uid=$SETUID --gid=$SETGID --home $SETHOME --shell /bin/bash $SETUSER +/usr/sbin/usermod -a -G fuse $SETUSER +exit 0 |