From 41048f09b545b34ff1ed398b1dd76aad48a98718 Mon Sep 17 00:00:00 2001 From: "Stefan K. Berg" Date: Thu, 12 Feb 2015 12:12:34 +0100 Subject: Simplification and cleanup of build process Docker build image: - Uplift of the base Ubuntu version to 14.04. - Instead of building the Ubuntu image with debootstrap, we use the official Docker repository version of Ubuntu. - Cleanup of the way how runcontext operates - Fix: Make sure that the build system can be run as root Top Makefile: - Building the Fuel ISO on Docker local /tmp directory not to pollute local directory with files owned by root. - Removed unused targets. Change-Id: I684348b2a15546eadf98d810239a84ff9798b1e8 Signed-off-by: Stefan K. Berg --- fuel-build/docker/Makefile | 29 ++---- fuel-build/docker/README | 9 +- fuel-build/docker/mkimage/debootstrap | 126 ----------------------- fuel-build/docker/mkimage/mkimage.sh | 116 --------------------- fuel-build/docker/runcontext | 11 +- fuel-build/docker/ubuntu-builder/Dockerfile | 15 ++- fuel-build/docker/ubuntu-builder/enable_dockerx2 | 11 +- fuel-build/docker/ubuntu-builder/install.sh | 22 ---- fuel-build/docker/ubuntu-builder/setcontext | 20 ++++ 9 files changed, 58 insertions(+), 301 deletions(-) delete mode 100755 fuel-build/docker/mkimage/debootstrap delete mode 100755 fuel-build/docker/mkimage/mkimage.sh delete mode 100755 fuel-build/docker/ubuntu-builder/install.sh create mode 100644 fuel-build/docker/ubuntu-builder/setcontext (limited to 'fuel-build/docker') diff --git a/fuel-build/docker/Makefile b/fuel-build/docker/Makefile index 4f00de8..360bb54 100644 --- a/fuel-build/docker/Makefile +++ b/fuel-build/docker/Makefile @@ -8,25 +8,16 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -CONTEXT1 := opnfv.org/ubuntu:12.04 -CONTEXT2 := opnfv.org/ubuntu-builder:12.04 +FILES = $(wildcard ubuntu-builder/*) runcontext -.PHONY: docker all +.PHONY: all +all: .docker -all: create-docker-ubuntu populate-docker-ubuntu - ./runcontext ${CONTEXT2} make test +.docker: $(FILES) + /usr/bin/docker build --rm=true -t opnfv.org/ubuntu-builder:14.04 ubuntu-builder + /usr/bin/docker tag -f opnfv.org/ubuntu-builder:14.04 opnfv.org/ubuntu-builder + touch .docker -.PHONY: create-docker-ubuntu -create-docker-ubuntu: - /usr/bin/docker run --rm -i -t ${CONTEXT1} date || \ - sudo mkimage/mkimage.sh -t opnfv.org/ubuntu:12.04 debootstrap --include=ubuntu-minimal --components=main,universe precise - -.PHONY: populate-docker-ubuntu -populate-docker-ubuntu: - /usr/bin/docker build --rm=true -t opnfv.org/ubuntu-builder:12.04 ubuntu-builder - /usr/bin/docker tag -f opnfv.org/ubuntu-builder:12.04 opnfv.org/ubuntu-builder - -.PHONY: test -test: - @hostname - @date +.PHONY: clean +clean: + rm .docker diff --git a/fuel-build/docker/README b/fuel-build/docker/README index 5d22bc9..e5ccdfa 100644 --- a/fuel-build/docker/README +++ b/fuel-build/docker/README @@ -13,13 +13,12 @@ used for building in Ubuntu 12.04 context. There are two requirements before running make on this level: -1. You need to have Docker installed on your system, and make sure -to have a valid DNS config in your /etc/default/docker. -e.g: -DOCKER_OPTS=" --dns=8.8.8.8 --dns=8.8.8.4" +1. You need to have Docker installed on your system +2. If in a corporate network, make sure to have a valid DNS config in + your /etc/default/docker, e.g: -2. You must have installed the package "debootstrap". + DOCKER_OPTS=" --dns=8.8.8.8 --dns=8.8.8.4" The top level makefile will refer to the "runcontext" file in order to run the build process as the current user with the current working diff --git a/fuel-build/docker/mkimage/debootstrap b/fuel-build/docker/mkimage/debootstrap deleted file mode 100755 index 0862120..0000000 --- a/fuel-build/docker/mkimage/debootstrap +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash -# -set -e - -rootfsDir="$1" -shift - -# we have to do a little fancy footwork to make sure "rootfsDir" becomes the second non-option argument to debootstrap - -before=() -while [ $# -gt 0 ] && [[ "$1" == -* ]]; do - before+=( "$1" ) - shift -done - -suite="$1" -shift - -( - set -x - debootstrap "${before[@]}" "$suite" "$rootfsDir" "$@" -) - -# now for some Docker-specific tweaks - -# prevent init scripts from running during install/update -echo >&2 "+ cat > '$rootfsDir/usr/sbin/policy-rc.d'" -cat > "$rootfsDir/usr/sbin/policy-rc.d" <<'EOF' -#!/bin/sh -exit 101 -EOF -chmod +x "$rootfsDir/usr/sbin/policy-rc.d" - -# prevent upstart scripts from running during install/update -( - set -x - chroot "$rootfsDir" dpkg-divert --local --rename --add /sbin/initctl - ln -sf /bin/true "$rootfsDir/sbin/initctl" -) - -# shrink the image, since apt makes us fat (wheezy: ~157.5MB vs ~120MB) -( set -x; chroot "$rootfsDir" apt-get clean ) - -# Ubuntu 10.04 sucks... :) -if strings "$rootfsDir/usr/bin/dpkg" | grep -q unsafe-io; then - # force dpkg not to call sync() after package extraction (speeding up installs) - echo >&2 "+ echo force-unsafe-io > '$rootfsDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup'" - echo 'force-unsafe-io' > "$rootfsDir/etc/dpkg/dpkg.cfg.d/docker-apt-speedup" -fi - -if [ -d "$rootfsDir/etc/apt/apt.conf.d" ]; then - # _keep_ us lean by effectively running "apt-get clean" after every install - aptGetClean='"rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true";' - echo >&2 "+ cat > '$rootfsDir/etc/apt/apt.conf.d/docker-clean'" - cat > "$rootfsDir/etc/apt/apt.conf.d/docker-clean" <<-EOF - DPkg::Post-Invoke { ${aptGetClean} }; - APT::Update::Post-Invoke { ${aptGetClean} }; - - Dir::Cache::pkgcache ""; - Dir::Cache::srcpkgcache ""; - EOF - - # remove apt-cache translations for fast "apt-get update" - echo >&2 "+ cat > '$rootfsDir/etc/apt/apt.conf.d/docker-no-languages'" - echo 'Acquire::Languages "none";' > "$rootfsDir/etc/apt/apt.conf.d/docker-no-languages" -fi - -if [ -z "$DONT_TOUCH_SOURCES_LIST" ]; then - # tweak sources.list, where appropriate - lsbDist= - if [ -z "$lsbDist" -a -r "$rootfsDir/etc/os-release" ]; then - lsbDist="$(. "$rootfsDir/etc/os-release" && echo "$ID")" - fi - if [ -z "$lsbDist" -a -r "$rootfsDir/etc/lsb-release" ]; then - lsbDist="$(. "$rootfsDir/etc/lsb-release" && echo "$DISTRIB_ID")" - fi - if [ -z "$lsbDist" -a -r "$rootfsDir/etc/debian_version" ]; then - lsbDist='Debian' - fi - case "$lsbDist" in - debian|Debian) - # updates and security! - if [ "$suite" != 'sid' -a "$suite" != 'unstable' ]; then - ( - set -x - sed -i "p; s/ $suite main$/ ${suite}-updates main/" "$rootfsDir/etc/apt/sources.list" - echo "deb http://security.debian.org $suite/updates main" >> "$rootfsDir/etc/apt/sources.list" - ) - fi - ;; - ubuntu|Ubuntu) - # add the universe, updates, and security repositories - ( - set -x - sed -i " - s/ $suite main$/ $suite main universe/; p; - s/ $suite main/ ${suite}-updates main/; p; - s/ $suite-updates main/ ${suite}-security main/ - " "$rootfsDir/etc/apt/sources.list" - ) - ;; - tanglu|Tanglu) - # add the updates repository - if [ "$suite" != 'devel' ]; then - ( - set -x - sed -i "p; s/ $suite main$/ ${suite}-updates main/" "$rootfsDir/etc/apt/sources.list" - ) - fi - ;; - steamos|SteamOS) - # add contrib and non-free - ( - set -x - sed -i "s/ $suite main$/ $suite main contrib non-free/" "$rootfsDir/etc/apt/sources.list" - ) - ;; - esac -fi - -# make sure we're fully up-to-date, too -( - set -x - chroot "$rootfsDir" apt-get update - chroot "$rootfsDir" apt-get dist-upgrade -y -) diff --git a/fuel-build/docker/mkimage/mkimage.sh b/fuel-build/docker/mkimage/mkimage.sh deleted file mode 100755 index 3424258..0000000 --- a/fuel-build/docker/mkimage/mkimage.sh +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env bash -############################################################################## -# Copyright (c) 2015 Ericsson AB and others. -# stefan.k.berg@ericsson.com -# jonas.bjurel@ericsson.com -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -set -e - -mkimg="$(basename "$0")" - -usage() { - echo >&2 "usage: $mkimg [-d dir] [-t tag] script [script-args]" - echo >&2 " ie: $mkimg -t someuser/debian debootstrap --variant=minbase jessie" - echo >&2 " $mkimg -t someuser/ubuntu debootstrap --include=ubuntu-minimal trusty" - echo >&2 " $mkimg -t someuser/busybox busybox-static" - echo >&2 " $mkimg -t someuser/centos:5 rinse --distribution centos-5" - exit 1 -} - -scriptDir="$(dirname "$(readlink -f "$BASH_SOURCE")")" - -optTemp=$(getopt --options '+d:t:h' --longoptions 'dir:,tag:,help' --name "$mkimg" -- "$@") -eval set -- "$optTemp" -unset optTemp - -dir= -tag= -while true; do - case "$1" in - -d|--dir) dir="$2" ; shift 2 ;; - -t|--tag) tag="$2" ; shift 2 ;; - -h|--help) usage ;; - --) shift ; break ;; - esac -done - -script="$1" -[ "$script" ] || usage -shift - -if [ ! -x "$scriptDir/$script" ]; then - echo >&2 "error: $script does not exist or is not executable" - echo >&2 " see $scriptDir for possible scripts" - exit 1 -fi - -# don't mistake common scripts like .febootstrap-minimize as image-creators -if [[ "$script" == .* ]]; then - echo >&2 "error: $script is a script helper, not a script" - echo >&2 " see $scriptDir for possible scripts" - exit 1 -fi - -delDir= -if [ -z "$dir" ]; then - dir="$(mktemp -d ${TMPDIR:-/tmp}/docker-mkimage.XXXXXXXXXX)" - delDir=1 -fi - -rootfsDir="$dir/rootfs" -( set -x; mkdir -p "$rootfsDir" ) - -# pass all remaining arguments to $script -"$scriptDir/$script" "$rootfsDir" "$@" - -# Docker mounts tmpfs at /dev and procfs at /proc so we can remove them -rm -rf "$rootfsDir/dev" "$rootfsDir/proc" -mkdir -p "$rootfsDir/dev" "$rootfsDir/proc" - -# make sure /etc/resolv.conf has something useful in it -mkdir -p "$rootfsDir/etc" -cat > "$rootfsDir/etc/resolv.conf" <<'EOF' -nameserver 8.8.8.8 -nameserver 8.8.4.4 -EOF - -tarFile="$dir/rootfs.tar.xz" -touch "$tarFile" - -( - set -x - tar --numeric-owner -caf "$tarFile" -C "$rootfsDir" --transform='s,^./,,' . -) - -echo >&2 "+ cat > '$dir/Dockerfile'" -cat > "$dir/Dockerfile" <<'EOF' -FROM scratch -ADD rootfs.tar.xz / -EOF - -# if our generated image has a decent shell, let's set a default command -for shell in /bin/bash /usr/bin/fish /usr/bin/zsh /bin/sh; do - if [ -x "$rootfsDir/$shell" ]; then - ( set -x; echo 'CMD ["'"$shell"'"]' >> "$dir/Dockerfile" ) - break - fi -done - -umount $rootfsDir/run/shm -( set -x; rm -rf "$rootfsDir" ) - -if [ "$tag" ]; then - ( set -x; docker build -t "$tag" "$dir" ) -elif [ "$delDir" ]; then - # if we didn't specify a tag and we're going to delete our dir, let's just build an untagged image so that we did _something_ - ( set -x; docker build "$dir" ) -fi - -if [ "$delDir" ]; then - ( set -x; rm -rf "$dir" ) -fi diff --git a/fuel-build/docker/runcontext b/fuel-build/docker/runcontext index 45a7645..d4e8b9c 100755 --- a/fuel-build/docker/runcontext +++ b/fuel-build/docker/runcontext @@ -15,16 +15,11 @@ GID=`id -g` USER=`whoami` res=`docker build -q --rm - <> /etc/bash.bashrc -RUN echo "$@" >> /etc/bash.bashrc -RUN echo 'exit \\$?' >> /etc/bash.bashrc +RUN date || date +RUN /root/setcontext $USER $UID $GID $HOME EOF` CID=`echo $res | sed 's/.* //'` -docker run --privileged=true --rm -i -t -u $USER -w $PWD -v ${HOME}/.ssh:${HOME}/.ssh -v $PWD:$PWD $CID bash +docker run --privileged=true --rm -i -t -e HOME=$HOME -u $USER -w $PWD -v ${HOME}/.ssh:${HOME}/.ssh -v $PWD:$PWD $CID "$@" rc=$? docker rmi $CID > /dev/null exit $rc diff --git a/fuel-build/docker/ubuntu-builder/Dockerfile b/fuel-build/docker/ubuntu-builder/Dockerfile index e91afbd..3ec934d 100644 --- a/fuel-build/docker/ubuntu-builder/Dockerfile +++ b/fuel-build/docker/ubuntu-builder/Dockerfile @@ -1,3 +1,4 @@ +#!/bin/bash ############################################################################## # Copyright (c) 2015 Ericsson AB and others. # stefan.k.berg@ericsson.com @@ -8,11 +9,17 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -FROM opnfv.org/ubuntu:12.04 -ADD install.sh /root/install.sh -RUN /root/install.sh +FROM ubuntu:14.04 +RUN apt-get update +RUN apt-get install -y make python-setuptools python-all dpkg-dev debhelper \ + fuseiso git genisoimage bind9-host wget curl lintian tmux lxc iptables \ + ca-certificates sudo apt-utils lsb-release +RUN echo "ALL ALL=NOPASSWD: ALL" > /etc/sudoers.d/open-sudo +RUN chmod 0440 /etc/sudoers.d/open-sudo + +ADD ./setcontext /root/setcontext +RUN chmod +x /root/setcontext ADD ./enable_dockerx2 /root/enable_dockerx2 RUN chmod +x /root/enable_dockerx2 VOLUME /var/lib/docker -CMD ["/root/enable_dockerx2"] diff --git a/fuel-build/docker/ubuntu-builder/enable_dockerx2 b/fuel-build/docker/ubuntu-builder/enable_dockerx2 index b6bccdf..e79c8b7 100644 --- a/fuel-build/docker/ubuntu-builder/enable_dockerx2 +++ b/fuel-build/docker/ubuntu-builder/enable_dockerx2 @@ -1,6 +1,15 @@ #!/bin/sh +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# # Needed to expose underlying cgroups to container - mount -n -t tmpfs -o uid=0,gid=0,mode=0755 cgroup /sys/fs/cgroup mount -t securityfs none /sys/kernel/security diff --git a/fuel-build/docker/ubuntu-builder/install.sh b/fuel-build/docker/ubuntu-builder/install.sh deleted file mode 100755 index 667ab27..0000000 --- a/fuel-build/docker/ubuntu-builder/install.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2015 Ericsson AB and others. -# stefan.k.berg@ericsson.com -# jonas.bjurel@ericsson.com -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -BASE_PACKAGES="make python-setuptools python-all dpkg-dev debhelper -fuseiso git genisoimage bind9-host wget curl lintian tmux lxc iptables -ca-certificates" - -apt-get update || exit 1 -apt-get upgrade -y || exit 1 - -apt-get install -y $BASE_PACKAGES || exit 1 - -echo "ALL ALL=NOPASSWD: ALL" > /etc/sudoers.d/open-sudo -chmod 0440 /etc/sudoers.d/open-sudo diff --git a/fuel-build/docker/ubuntu-builder/setcontext b/fuel-build/docker/ubuntu-builder/setcontext new file mode 100644 index 0000000..bc28994 --- /dev/null +++ b/fuel-build/docker/ubuntu-builder/setcontext @@ -0,0 +1,20 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# +SETUSER=$1 +SETUID=$2 +SETGID=$3 +SETHOME=$4 + +getent group $SETUSER || /usr/sbin/groupadd --gid $SETGID $SETUSER +getent passwd $SETUSER || /usr/sbin/adduser --system --uid=$SETUID --gid=$SETGID --home $SETHOME --shell /bin/bash $SETUSER +/usr/sbin/usermod -a -G fuse $SETUSER +exit 0 -- cgit 1.2.3-korg