diff options
Diffstat (limited to 'build')
70 files changed, 3859 insertions, 0 deletions
diff --git a/build/Makefile b/build/Makefile new file mode 100644 index 000000000..59d9096c6 --- /dev/null +++ b/build/Makefile @@ -0,0 +1,259 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +SHELL = /bin/bash + +# This file will be created if needed by runcontext to contain proxy +# settings. +-include environment.mk + +############################################################################ +# BEGIN of variables to customize +# +#Input args +export MOSVERSION = 9.0 +export ISOSRC = file:$(shell pwd)/fuel-$(MOSVERSION).iso +export ISOCACHE = $(shell pwd)/$(shell basename $(ISOSRC)) +export PRODNO = "OPNFV_FUEL" +export REVSTATE = "P0000" +export USER ?= $(shell whoami) +export BUILD_DATE = $(shell date --utc +%Y-%m-%d:%H:%M) +export OPNFV_GIT_SHA = $(shell git rev-parse HEAD) +# Store in /etc/fuel_build_id on fuel master +export BUILD_ID := $(PRODNO)_$(BUILD_DATE)_$(OPNFV_GIT_SHA) + +ifdef BUILD_FUEL_PLUGINS +$(warning Overriding plugin build selection to $(BUILD_FUEL_PLUGINS)) +export NEWISO = $(shell pwd)/release/unofficial-opnfv-${REVSTATE}.iso +else +export NEWISO = $(shell pwd)/release/opnfv-${REVSTATE}.iso +endif + +# Note! Invoke with "make REVSTATE=RXXXX all" to make release build! +# Invoke with ICOCACHE=/full/path/to/iso if cached ISO is in non-standard location. + +# Build variables +export BUILD_BASE := $(shell pwd) +export DEB_DEST := $(BUILD_BASE)/release/packages/ubuntu/pool/main +export UDEB_DEST := $(BUILD_BASE)/release/packages/ubuntu/pool/debian-installer +export PUPPET_DEST := $(BUILD_BASE)/release/puppet/modules +export VERSION_FILE := $(BUILD_BASE)/.versions +export DOCKERIMG = opnfv.org/ubuntu-builder:14.04 +export TOPDIR := $(shell pwd) +export REPOINFO := $(BUILD_BASE)/repo_info.sh + +# Use snapshots +# Use nearby repositories +export MIRROR_UBUNTU_URL := $(shell ./f_isoroot/f_repobuild/select_ubuntu_repo.sh --url) +export MIRROR_UBUNTU := $(shell echo "$(MIRROR_UBUNTU_URL)" | cut -d'/' -f3 ) +export MIRROR_UBUNTU_ROOT := $(shell echo -n '/' ; echo "$(MIRROR_UBUNTU_URL)" | cut -d'/' -f4-) + +export LATEST_MIRROR_ID_URL := http://$(shell ./select_closest_fuel_mirror.py) + +export MIRROR_MOS_UBUNTU ?= $(shell echo "$(LATEST_MIRROR_ID_URL)" | cut -d'/' -f3) +export LATEST_TARGET_UBUNTU := $(shell curl -sSf "$(MIRROR_MOS_UBUNTU)/mos-repos/ubuntu/$(MOSVERSION).target.txt" | head -1) +export MIRROR_MOS_UBUNTU_ROOT := /mos-repos/ubuntu/$(LATEST_TARGET_UBUNTU) + +export LATEST_TARGET_CENTOS := $(shell curl -sSf "$(LATEST_MIRROR_ID_URL)/mos-repos/centos/mos$(MOSVERSION)-centos7/os.target.txt" | head -1) +export MIRROR_FUEL := "$(LATEST_MIRROR_ID_URL)/mos-repos/centos/mos$(MOSVERSION)-centos7/$(LATEST_TARGET_CENTOS)/x86_64" + +# uncomment and use: make print-VARIABLE +#print-% : ; @echo $* = $($*) + +#Build subclasses + +SUBDIRS := f_isoroot + +# f_example is only an example of how to generate a .deb package and +# should not be enabled in official builds. +#SUBDIRS += f_example + +ORIGDIR := $(TOPDIR)/origiso +# +# END of variables to customize +############################################################################# + +# Fuel-main destination path and fuel-* submodule patching, for full list check: +# https://github.com/openstack/fuel-main/blob/stable/mitaka/repos.mk#L32-L44 +FUEL_MAIN_DIR := /tmp/fuel-main +FUEL_PATCHES := $(shell find $(BUILD_BASE)/patch-repos -name '*.patch' | sort) + +SUBCLEAN = $(addsuffix .clean,$(SUBDIRS)) + +.PHONY: all +all: + @docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1) + @make -C docker + @lsb_release -a | grep Ubuntu > /dev/null 2>&1 || (echo 'Ubuntu is the only supported Linux distribution for this build system'; exit 1) + @KERNEL_VER=$$(uname -r); \ + KERNEL_ARRAY=($${KERNEL_VER//./ }); \ + if [ $${KERNEL_ARRAY[0]} -lt 3 ] || [[ $${KERNEL_ARRAY[0]} -eq 3 && $${KERNEL_ARRAY[1]} -lt 19 ]] ; then (echo 'Kernel version must be 3.19 or newer'; exit 1) ; fi + @docker/runcontext $(DOCKERIMG) wget -q www.google.com -O /dev/null || (echo 'No docker network connectivity or name server - check your network- and docker settings'; exit 1) + @docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS) iso + + +############################################################################ +# BEGIN of Include definitions +# +include config.mk +include cache.mk +# +# END Include definitions +############################################################################# + +$(ISOCACHE): + # Clone Fuel to non-persistent location and build + sudo rm -rf $(FUEL_MAIN_DIR) + git clone $(FUEL_MAIN_REPO) $(FUEL_MAIN_DIR) + git -C $(FUEL_MAIN_DIR) checkout $(FUEL_MAIN_TAG) + @echo "fuel" `git -C $(FUEL_MAIN_DIR) rev-parse HEAD` >> $(VERSION_FILE) + # Remove Docker optimizations, otherwise multistrap will fail during + # Fuel build. + sudo rm -f /etc/apt/apt.conf.d/docker* + # + # Need to replace the old Docker v. 1.5.0 with a later version to be + # able to access the Docker remote repository! + sudo apt-get install apt-transport-https ca-certificates -y + sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D + echo "deb https://apt.dockerproject.org/repo ubuntu-trusty main" | sudo tee /etc/apt/sources.list.d/docker.list + sudo apt-get update -y + sudo apt-get install docker-engine -y + sudo sh -c 'echo DOCKER_OPTS=\"--bip 172.45.0.1/24\" > /etc/default/docker' + sudo service docker stop || exit 0 + sudo service docker start + + cd $(FUEL_MAIN_DIR) && ./prepare-build-env.sh + # Verify that Docker is alive + sudo docker info + # fuel-main Makefiles do not like `make -C` + cd $(FUEL_MAIN_DIR) && make repos + $(REPOINFO) -r $(FUEL_MAIN_DIR) > gitinfo_fuel.txt + # OPNFV patches at Fuel build time + # Need to be commited in order for them to be considered by the Fuel + # build system + $(foreach patch,$(FUEL_PATCHES),git \ + -C $(subst $(BUILD_BASE)/patch-repos,$(FUEL_MAIN_DIR),$(dir $(patch))) \ + am --whitespace=nowarn --committer-date-is-author-date $(patch) || \ + (echo 'Error: Failed patching Fuel repos!' ; exit 1);) + + # Repeat build up to three times + sudo -E ./fuel_build_loop + cp $(FUEL_MAIN_DIR)/build/artifacts/fuel*.iso . + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +.PHONY: mount-origiso umount-origiso +mount-origiso: $(ISOCACHE) + @echo "Mounting original ISO in $(ORIGDIR)" + @mkdir -p $(ORIGDIR) + @fuseiso $(ISOCACHE) $(ORIGDIR) + +umount-origiso: + @echo "Unmounting original ISO from $(ORIGDIR)" + @fusermount -u $(ORIGDIR) + @rmdir $(ORIGDIR) + +.PHONY: $(SUBDIRS) +$(SUBDIRS): + @mkdir -p release/packages/ubuntu/pool/main release/packages/ubuntu/pool/debian-installer release/puppet/modules release/isoroot + $(MAKE) -C $@ -f Makefile release + +.PHONY: patch-packages +patch-packages: + ORIGISO=$(ISOCACHE) REVSTATE=$(REVSTATE) $(MAKE) -C $@ -f Makefile release + +.PHONY: clean $(SUBCLEAN) +clean: $(SUBCLEAN) + $(MAKE) -C patch-packages -f Makefile clean + @rm -f *.iso + @rm -Rf release + @rm -Rf newiso + @rm -Rf .versions + @rm -f $(NEWISO) + @rm -f $(BUILD_BASE)/gitinfo_*.txt + +.PHONY: deepclean +deepclean: clean clean-cache + make -C docker clean + docker rmi opnfv.org/ubuntu-builder:14.04 &>/dev/null || exit 0 + docker rmi opnfv.org/ubuntu-builder:latest &>/dev/null || exit 0 + docker rmi ubuntu:14.04 &>/dev/null || exit 0 + @if docker images | grep -q "ubuntu *14.04"; then \ + echo "Error: ubuntu:14.04 still present!"; \ + exit 1; \ + fi + @if docker images | grep -q "opnfv.org/ubuntu-builder"; then \ + echo "Error: opnfv.org/ubuntu-builder still present!"; \ + exit 1; \ + fi + +$(SUBCLEAN): %.clean: + $(MAKE) -C $* -f Makefile clean + +.PHONY: setup-env +setup-env: + @if [ -f environment.mk ]; then \ + sudo bash -c "cat environment.mk >> /etc/environment"; \ + fi + +# Todo: Make things smarter - we shouldn't need to clean everything +# betwen make invocations. +.PHONY: iso +iso: setup-env $(ISOCACHE) $(SUBDIRS) patch-packages + $(REPOINFO) . > gitinfo_main.txt + install/install.sh iso $(ISOCACHE) $(NEWISO) $(PRODNO) $(REVSTATE) + @printf "\n\nProduct ISO is $(NEWISO)\n\n" + +# Start a bash shell in docker for Makefile debugging +.PHONY: debug +debug: + @docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1) + @make -C docker + docker/runcontext $(DOCKERIMG) debug + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + +# Create a unique hash to be used for getting and putting cache, based on: +# - The commit ID of the full Fuel repo structre +# - The contents of all local Fuel patches +.cacheid: + git clone $(FUEL_MAIN_REPO) $(FUEL_MAIN_DIR) + git -C $(FUEL_MAIN_DIR) checkout $(FUEL_MAIN_TAG) + make -C $(FUEL_MAIN_DIR) repos + $(REPOINFO) -r $(FUEL_MAIN_DIR) > .cachedata + $(foreach patch,$(FUEL_PATCHES),sha1sum $(patch) >> .cachedata;) + sha1sum fuel_build_loop >> .cachedata + sha1sum config.mk >> .cachedata + sha1sum Makefile >> .cachedata + $(CACHETOOL) getbiweek >> .cachedata + cat .cachedata | $(CACHETOOL) getid > .cacheid + # Not removing fuel-main as it is re-used in build + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: $(SUBCLEANCACHE) + rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - fuel*.iso gitinfo_fuel.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/README b/build/README new file mode 100644 index 000000000..a6e15694c --- /dev/null +++ b/build/README @@ -0,0 +1,22 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +This purpose of this framework is to: + +1) Build fuel baseline from upstream OpenStack Fuel repos (builds in a docker container to isolate dependencies from host + and full re-producability) +2) Apply arbitrary changes to the fuel baseline, consistent and non volatile: + - Add arbitrary packages with puppet modules for install and config. to be applied on controllers as well as computes. + - Change any stack-, or other pack config. using puppet + - Apply patches to the baseline. + - Etc. +3) Re factor/rebuild the .iso image for deployment (also builds in a container, for the same reason as mentioned above) + +For detailed instructions on how to add content, configuration, build and deply - please see: DOC/ diff --git a/build/add_opnfv_packages b/build/add_opnfv_packages new file mode 100644 index 000000000..44af92ea7 --- /dev/null +++ b/build/add_opnfv_packages @@ -0,0 +1,23 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Format of this file: +# +#TEXT>old package>url for replacement package +# +# Note! No spaces! +# The TEXT can be anything it is just for information display. +# Iff the packages is new, i.e. not replacing an existing package +# the old package should be specified as NONE. +# Note! For new packages a puppet manifest must actually install it. +# Easiest is to just add it to add_packages.pp. (The same might +# of course be true for an exising package if it isn't already +# is being installed). +# diff --git a/build/apply_patches b/build/apply_patches new file mode 100644 index 000000000..6925ff6b7 --- /dev/null +++ b/build/apply_patches @@ -0,0 +1,18 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Format of this file: +# +#TR>old package>url for replacement package +# +# Note! No spaces! +# Example: +#FIX>foo_0.0.1.deb>https://example.org/foo_0.0.2.deb + diff --git a/build/cache.mk b/build/cache.mk new file mode 100644 index 000000000..f6db01797 --- /dev/null +++ b/build/cache.mk @@ -0,0 +1,55 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +############################################################################# +# Cache operations - only used when building through ci/build.sh +# +# This is the global cache implementation, providing the main target "cache" +# which is called from ci/build.sh, and recursively calling the cache +# operations clean-cache, get-cache and put-cache on all $(SUBDIRS). +############################################################################# + + +export CACHETOOL := $(BUILD_BASE)/cache.sh + +# Call sub caches +SUBGETCACHE = $(addsuffix .getcache,$(SUBDIRS)) +$(SUBGETCACHE): %.getcache: + $(MAKE) -C $* -f Makefile get-cache + +SUBPUTCACHE = $(addsuffix .putcache,$(SUBDIRS)) +$(SUBPUTCACHE): %.putcache: + $(MAKE) -C $* -f Makefile put-cache + +SUBCLEANCACHE = $(addsuffix .cleancache,$(SUBDIRS)) +$(SUBCLEANCACHE): %.cleancache: + $(MAKE) -C $* -f Makefile clean-cache + +# Overlay implementation: +# - clean +# - clean cache identities +# - get caches +# - build iso +# - store caches +.PHONY: cached-all +cached-all: clean clean-cache $(SUBCLEANCACHE) get-cache $(SUBGETCACHE) iso put-cache $(SUBPUTCACHE) + @echo "Cached build is complete" + + +# cache: The target for ci/build.sh +.PHONY: cache +cache: + @if [ -z "${CACHEBASE}" ]; then \ + echo "CACHEBASE not set, are you really building through build.sh?"; \ + exit 1; \ + fi + @docker version >/dev/null 2>&1 || (echo 'No Docker installation available'; exit 1) + @make -C docker + docker/runcontext $(DOCKERIMG) $(MAKE) $(MAKEFLAGS) cached-all diff --git a/build/cache.sh b/build/cache.sh new file mode 100755 index 000000000..d4b2c45dd --- /dev/null +++ b/build/cache.sh @@ -0,0 +1,203 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +exit_trap() { + if [ -d "$TMPDIR" ]; then + rm -rf $TMPDIR + fi +} + +trap exit_trap EXIT + +CACHETRANSPORT=${CACHETRANSPORT:-"curl --silent"} +CACHEMAXAGE=${CACHEMAXAGE:-$[14*24*3600]} +CACHEDEBUG=${CACHEDEBUG:-1} +PLUGINS_MATCH="${BUILD_BASE}/f_isoroot/*/" + +debugmsg () { + if [ "$CACHEDEBUG" -eq 1 ]; then + echo "$@" >&2 + fi +} + +errorexit () { + echo "$@" >&2 + exit 1 +} + +# Generate a unique number every two weeks - a service routine that +# can be used when generating the SHA1 to make sure that the cache is +# rebuilt bi-weekly even if no pruning of the cache is taking place. +getbiweek () { + echo "$(date +'%G')$[10#$(date +'%V')/2]" +} + +# Get a SHA1 based on what's piped into the cache command +getid() { + debugmsg "Generating sha1sum" + sha1sum | sed 's/ .*//' +} + + +# Put in cache +put() { + if check $1; then + debugmsg "SHA1 $1 already in cache, skipping storage" + else + debugmsg "Storing SHA1 $1 in cache" + ${CACHETRANSPORT} -T - ${CACHEBASE}/$1.blob + echo "Expires: $[`date +"%s"` + $CACHEMAXAGE]" | ${CACHETRANSPORT} -T - ${CACHEBASE}/$1.meta + fi + exit 0 +} + +# Get from cache +get() { + local rc + + ${CACHETRANSPORT} -o - ${CACHEBASE}/$1.blob 2>/dev/null + rc=$? + + if [ $rc -eq 0 ]; then + echo "Got SHA1 $1 from cache" 2>/dev/null + else + echo "Tried to get SHA1 $1 from cache but failed" 2>/dev/null + fi + + return $? +} + +# Check if in cache +check() { + local rc + + ${CACHETRANSPORT} ${CACHEBASE}/$1.meta &>/dev/null + rc=$? + + if [ $rc -eq 0 ]; then + debugmsg "Checking for SHA1 $1 in cache and found it, rc = $rc" + else + debugmsg "Checking for SHA1 $1 in cache and failed, rc = $rc" + fi + + return $rc +} + +# Verify that SHA1 seems to be a SHA1... +validSHA1() { + if [ $(echo $1 | wc -c) -ne 41 ]; then + return 1 + else + return 0 + fi +} + +# Figure out commit ID from URI and tag/branch/commit ID +getcommitid() { + if echo $2 | grep -q '^refs/changes/'; then + REF=`echo $2 | sed "s,refs\/changes\/\(.*\),\1,"` + else + REF=$2 + fi + + echo "Repo is $1, ref is ${REF}" >&2 + + HEADMATCH=`git ls-remote $1 | grep "refs/heads/${REF}$" | awk '{ print $1 }'` + TAGMATCH=`git ls-remote $1 | grep "refs/tags/${REF}$" | awk '{ print $1 }'` + CHANGEMATCH=`git ls-remote $1 | grep "refs/changes/${REF}$" | awk '{ print $1 }'` + + if [ -n "$HEADMATCH" ]; then + echo "$HEADMATCH" + elif [ -n "$TAGMATCH" ]; then + echo "$TAGMATCH" + elif [ -n "$CHANGEMATCH" ]; then + echo "Warning: ${REF} is a change!" >&2 + TMPDIR=`mktemp -d /tmp/cacheXXXXX` + cd $TMPDIR + git clone $1 &>/dev/null || errorexit "Could not clone $1" + cd * || errorexit "Could not enter clone of $1" + git fetch $1 refs/changes/$REF &>/dev/null || errorexit "Could not fetch change" + git checkout FETCH_HEAD &>/dev/null || errorexit "Could not checkout FETCH_HEAD" + git show HEAD &>/dev/null || errorexit "Could not find commit $2" + git show HEAD | head -1 | awk '{ print $2 }' + else + TMPDIR=`mktemp -d /tmp/cacheXXXXX` + cd $TMPDIR + git clone $1 &>/dev/null || errorexit "Could not clone $1" + cd * || errorexit "Could not enter clone of $1" + git show $2 &>/dev/null || errorexit "Could not find commit $2" + git show $2 | head -1 | awk '{ print $2 }' + fi +} + +packages() { + local PLUGINS_SHA1='' + + # globbing expansion is alphabetical + for plugin in $PLUGINS_MATCH ; do + if [ -f "${plugin}packages.yaml" ] + then + PLUGINS_SHA1+=$(sha1sum ${plugin}packages.yaml) + fi + done + + if [ -n "${PLUGINS_SHA1}" ] + then + echo -n $PLUGINS_SHA1 | sha1sum + fi +} + +if [ -z "$CACHEBASE" ]; then + errorexit "CACHEBASE not set - exiting..." +fi + +case $1 in + getbiweek) + if [ $# -ne 1 ]; then + errorexit "No arguments can be given to getbiweek!" + fi + getbiweek + ;; + getcommitid) + if [ $# -ne 3 ]; then + errorexit "Arg 1 needs to be URI and arg 2 tag/branch/commit" + fi + shift + getcommitid $@ + ;; + getid) + if [ $# -ne 1 ]; then + errorexit "No arguments can be given to getid!" + fi + getid + ;; + get|check|put) + if [ $# -ne 2 ]; then + errorexit "Only one argument, the SHA1 sum, can be given to getid!" + else + if ! validSHA1 $2; then + errorexit "Invalid SHA1 format!" + fi + fi + + $1 $2 + exit $rc + ;; + packages) + if [ $# -ne 1 ]; then + errorexit "No arguments can be given to packages!" + fi + packages + ;; + *) + errorexit "I only know about getcommitid, getid, check, get and put!" +esac diff --git a/build/check_dependencies.sh b/build/check_dependencies.sh new file mode 100755 index 000000000..cbcb98ab3 --- /dev/null +++ b/build/check_dependencies.sh @@ -0,0 +1,41 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Given a file as input, this script verifies that all URIs in the file can +# be fetched. + +if [ $# -ne 1 ]; then + echo "Usage: $(basename $0) <filename>" + exit 1 +fi + +if [ ! -e $1 ]; then + echo "Could not open $1" + exit 1 +fi + +echo "Checking dependencies in $1" +rc=0 +for uri in `cat $1` +do + if ! curl -sfr 0-100 $uri > /dev/null; then + echo "Failed fetching $uri" >&2 + rc=1 + fi +done + +if [ $rc -ne 0 ]; then + echo "ERROR checking dependencies in $1" +else + echo "Dependencies OK" +fi + +exit $rc diff --git a/build/config.mk b/build/config.mk new file mode 100644 index 000000000..f78043e18 --- /dev/null +++ b/build/config.mk @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +FUEL_MAIN_REPO := https://github.com/openstack/fuel-main +FUEL_MAIN_TAG := 9.0.1 +MOS_VERSION = 9.0 +OPENSTACK_VERSION = mitaka-9.0 + +# Pinning down exact Fuel repo versions for Fuel 9.0.1 +export FUELLIB_COMMIT?=e283b62750d9e26355981b3ad3be7c880944ae0f +export NAILGUN_COMMIT?=e2b85bafb68c348f25cb7cceda81edc668ba2e64 +export PYTHON_FUELCLIENT_COMMIT?=67d8c693a670d27c239d5d175f3ea2a0512c498c +export FUEL_AGENT_COMMIT?=7ffbf39caf5845bd82b8ce20a7766cf24aa803fb +export FUEL_NAILGUN_AGENT_COMMIT?=46fa0db0f8944f9e67699d281d462678aaf4db26 +export ASTUTE_COMMIT?=390b257240d49cc5e94ed5c4fcd940b5f2f6ec64 +export OSTF_COMMIT?=f09c98ff7cc71ee612b2450f68a19f2f9c64345a +export FUEL_MIRROR_COMMIT?=d1ef06b530ce2149230953bb3810a88ecaff870c +export FUELMENU_COMMIT?=0ed9e206ed1c6271121d3acf52a6bf757411286b +export SHOTGUN_COMMIT?=781a8cfa0b6eb290e730429fe2792f2b6f5e0c11 +export NETWORKCHECKER_COMMIT?=fcb47dd095a76288aacf924de574e39709e1f3ca +export FUELUPGRADE_COMMIT?=c1c4bac6a467145ac4fac73e4a7dd2b00380ecfb +export FUEL_UI_COMMIT?=90de7ef4477230cb7335453ed26ed4306ca6f04f + +# for the patches applying purposes +export GIT_COMMITTER_NAME?=Fuel OPNFV +export GIT_COMMITTER_EMAIL?=fuel@opnfv.org + +DOCKER_REPO := http://get.docker.com/builds/Linux/x86_64 +DOCKER_TAG := docker-latest + +.PHONY: get-fuel-repo +get-fuel-repo: + @echo $(FUEL_MAIN_REPO) $(FUEL_MAIN_TAG) + diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile new file mode 100644 index 000000000..b38ea4c64 --- /dev/null +++ b/build/docker/Dockerfile @@ -0,0 +1,43 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +FROM ubuntu:14.04 +ENV http_proxy INSERT_HTTP_PROXY +ENV https_proxy INSERT_HTTPS_PROXY +ENV no_proxy INSERT_NO_PROXY +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update +RUN apt-get install -y software-properties-common python-software-properties \ + make python-setuptools python-all dpkg-dev debhelper ruby-json \ + fuseiso git genisoimage bind9-host wget curl lintian tmux lxc iptables \ + ca-certificates sudo apt-utils lsb-release dosfstools debmirror p7zip-full \ + build-essential ruby-dev rubygems-integration python-pip git rpm createrepo dpkg-dev + +RUN gem install fpm +RUN pip install fuel-plugin-builder + +RUN echo "ALL ALL=NOPASSWD: ALL" > /etc/sudoers.d/open-sudo +RUN echo "Defaults env_keep += \"ftp_proxy http_proxy https_proxy no_proxy RSYNC_PROXY RSYNC_CONNECT_PROG npm_config_registry\"" > /etc/sudoers.d/keep-proxies +# Keeping PWD is needed to build as root +RUN echo "Defaults env_keep += \"PWD\"" > /etc/sudoers.d/keep-pwd +# Keeping variables for ISO build +RUN echo "Defaults env_keep += \"MIRROR_UBUNTU MIRROR_UBUNTU_ROOT MIRROR_MOS_UBUNTU MIRROR_MOS_UBUNTU_ROOT MIRROR_FUEL LATEST_TARGET_UBUNTU UBUNTU_ARCH\"" > /etc/sudoers.d/keep-mos +RUN chmod 0440 /etc/sudoers.d/open-sudo +RUN chmod 0440 /etc/sudoers.d/keep-proxies +RUN chmod 0440 /etc/sudoers.d/keep-pwd +RUN chmod 0440 /etc/sudoers.d/keep-mos +RUN chmod 4755 /bin/fusermount + +ADD ./setcontext /root/setcontext +RUN chmod +x /root/setcontext + +VOLUME /var/lib/docker diff --git a/build/docker/Makefile b/build/docker/Makefile new file mode 100644 index 000000000..d4423b0a2 --- /dev/null +++ b/build/docker/Makefile @@ -0,0 +1,42 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +SHELL = /bin/bash +FILES = $(wildcard ubuntu-builder/*) runcontext +DOCKER_VER := $(shell [[ "$$(docker version --format '{{.Client.Version}}')" =~ ([0-9]+)\.([0-9]+) ]] && echo $$(( $${BASH_REMATCH[1]} * 100 + $${BASH_REMATCH[2]} ))) + +# Don't use -f flag when docker is newer than 1.9 +# https://docs.docker.com/engine/deprecated/#/f-flag-on-docker-tag +ifeq ($(shell echo "$(DOCKER_VER)>109" | bc), 1) + tag_flags := +else + tag_flags := -f +endif + +.PHONY: all +all: .docker + +.docker: $(FILES) + cp Dockerfile ubuntu-builder/Dockerfile + # Only add proxy ENVs where set in host - needed to pull the base Ubuntu image + test -n "${http_proxy}" && sed -i "s;INSERT_HTTP_PROXY;${http_proxy};" ubuntu-builder/Dockerfile || exit 0 + test -n "${https_proxy}" && sed -i "s;INSERT_HTTPS_PROXY;${https_proxy};" ubuntu-builder/Dockerfile || exit 0 + test -n "${no_proxy}" && sed -i "s;INSERT_NO_PROXY;${no_proxy};" ubuntu-builder/Dockerfile || exit 0 + test -n "${HTTP_PROXY}" && sed -i "s;INSERT_HTTP_PROXY;${HTTP_PROXY};" ubuntu-builder/Dockerfile || exit 0 + test -n "${HTTPS_PROXY}" && sed -i "s;INSERT_HTTPS_PROXY;${HTTPS_PROXY};" ubuntu-builder/Dockerfile || exit 0 + test -n "${NO_PROXY}" && sed -i "s;INSERT_NO_PROXY;${NO_PROXY};" ubuntu-builder/Dockerfile || exit 0 + sed -i '/INSERT_/d' ubuntu-builder/Dockerfile + /usr/bin/docker build --rm=true --no-cache=true -t opnfv.org/ubuntu-builder:14.04 ubuntu-builder + /usr/bin/docker tag ${tag_flags} opnfv.org/ubuntu-builder:14.04 opnfv.org/ubuntu-builder + touch .docker + +.PHONY: clean +clean: + rm -f .docker ubuntu-builder/Dockerfile diff --git a/build/docker/README b/build/docker/README new file mode 100644 index 000000000..e5ccdfab6 --- /dev/null +++ b/build/docker/README @@ -0,0 +1,25 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +The makefile on this level is used for *creating* the Docker image +used for building in Ubuntu 12.04 context. + +There are two requirements before running make on this level: + +1. You need to have Docker installed on your system + +2. If in a corporate network, make sure to have a valid DNS config in + your /etc/default/docker, e.g: + + DOCKER_OPTS=" --dns=8.8.8.8 --dns=8.8.8.4" + +The top level makefile will refer to the "runcontext" file in order to +run the build process as the current user with the current working +directory available to the container. diff --git a/build/docker/runcontext b/build/docker/runcontext new file mode 100755 index 000000000..daad663ac --- /dev/null +++ b/build/docker/runcontext @@ -0,0 +1,130 @@ +#!/bin/bash +set -e +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# + +############################################################################ +# BEGIN of Exit handlers +# + +do_exit () { + CID=`cat $CID_FILE </dev/null` + rm -f $CID_FILE + rm -rf $CONTEXT_DIR + set +e + docker kill $CID > /dev/null 2>&1 + docker rm -f $CID > /dev/null 2>&1 + docker rmi -f $IID > /dev/null 2>&1 + set -e +} + +# +# End of Exit handlers +############################################################################ + +trap do_exit SIGINT SIGTERM EXIT + +context=$1 +shift +USER_ID=`id -u` +USER=`whoami` +GROUP_ID=`id -g` + +GITROOT=`git rev-parse --show-toplevel` +CID_FILE=`mktemp -u -t runcontext.XXXXXXXXXX` +CONTEXT_DIR=`mktemp -d ${GITROOT}/.docker_contextXXXXXX` + +# If RSYNC_CONNECT_PROG is used, we need to copy all of +# the SSH structure, should one of the keys need to be +# used. +if [ -n "$RSYNC_CONNECT_PROG" -a -x $HOME/.ssh ]; then + cp -rp $HOME/.ssh $CONTEXT_DIR + rm -f $CONTEXT_DIR/.ssh/known_hosts +else + mkdir $CONTEXT_DIR/.ssh +fi + +# Disable verification of unknown keys +cat >> $CONTEXT_DIR/.ssh/config <<EOF +StrictHostKeyChecking=no +EOF + +cat > $CONTEXT_DIR/Dockerfile <<EOF +FROM $context +$(env | egrep -i 'proxy|rsync' | sed 's/^/ENV /' | sed 's/=/ /') +RUN date || date +COPY .ssh $HOME/.ssh +RUN chown -R $USER_ID:$GROUP_ID $HOME/.ssh +RUN chown -R $USER_ID:$GROUP_ID $HOME +RUN chmod 700 $HOME/.ssh +RUN /root/setcontext $USER $USER_ID $GROUP_ID $HOME +EOF + +res=`docker build -q --force-rm $CONTEXT_DIR` +IID=`echo $res | sed 's/.* //'` + +# Handle proxy settings passed to the context +if env | grep -iq .*proxy; then + envfile="$(readlink -f $(dirname $0)/..)/environment.mk" + + test -n "$HTTP_PROXY" && my_http_proxy=$HTTP_PROXY + test -n "$http_proxy" && my_http_proxy=$http_proxy + + test -n "$HTTPS_PROXY" && my_https_proxy=$HTTPS_PROXY + test -n "$https_proxy" && my_https_proxy=$https_proxy + + test -n "$NO_PROXY" && my_no_proxy=$NO_PROXY + test -n "$no_proxy" && my_no_proxy=$no_proxy + + # Make sure to add the Docker socket in no_proxy + if [ -n "$my_no_proxy" ]; then + my_no_proxy+=",/var/run/docker.sock" + else + my_no_proxy="/var/run/docker.sock" + fi + + echo "Creating $envfile" + echo "# This file is automatically generated by runcontext, do not edit!" > $envfile + test -n "$my_http_proxy" && echo "export http_proxy=$my_http_proxy" >> $envfile + test -n "$my_https_proxy" && echo "export https_proxy=$my_https_proxy" >> $envfile + test -n "$my_no_proxy" && echo "export no_proxy=$my_no_proxy" >> $envfile + test -n "$RSYNC_PROXY" && echo "export RSYNC_PROXY=$RSYNC_PROXY" >> $envfile + test -n "$RSYNC_CONNECT_PROG" && echo "export RSYNC_CONNECT_PROG=$RSYNC_CONNECT_PROG" >> $envfile + echo "export npm_config_registry=http://registry.npmjs.org/" >> $envfile +else + echo "No need to generate environment.mk" + rm -f $envfile +fi + +# Evaluate the need for bind mounting the cache directory +if [ -n "$CACHEBASE" ]; then + if echo $CACHEBASE | grep -q '^file://'; then + CACHEMOUNT="-v $(echo $CACHEBASE | sed 's;file://;;'):$(echo $CACHEBASE | sed 's;file://;;')" + fi +fi + +RUN_CONTEXT_OPT="--cidfile $CID_FILE --privileged=true --rm \ + -e HOME=$HOME -e CACHEDEBUG -e CACHETRANSPORT -e CACHEMAXAGE -e CACHEBASE \ + -e BUILD_FUEL_PLUGINS -e MIRROR_UBUNTU -e MIRROR_UBUNTU_ROOT \ + -e MIRROR_MOS_UBUNTU -e MIRROR_MOS_UBUNTU_ROOT -e MIRROR_FUEL \ + -e LATEST_TARGET_UBUNTU -e UBUNTU_ARCH \ + -u $USER_ID:$GROUP_ID -w $PWD \ + -v $GITROOT:$GITROOT -v /sys/fs/cgroup:/sys/fs/cgroup:ro $CACHEMOUNT" + +# Passing "debug" puts up an interactive bash shell +if [ "$1" == "debug" ]; then + echo command: docker run ${RUN_CONTEXT_OPT} $IID bash + docker run -i -t ${RUN_CONTEXT_OPT} $IID bash +else + echo command: docker run ${RUN_CONTEXT_OPT} $IID $@ + docker run -t ${RUN_CONTEXT_OPT} $IID $@ +fi + diff --git a/build/docker/ubuntu-builder/install_docker.sh b/build/docker/ubuntu-builder/install_docker.sh new file mode 100755 index 000000000..47c3a52ec --- /dev/null +++ b/build/docker/ubuntu-builder/install_docker.sh @@ -0,0 +1,36 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# From prepare-build-env.sh of Fuel +# Check if docker is installed +if hash docker 2>/dev/null; then + echo "Docker binary found, checking if service is running..." + ps cax | grep docker > /dev/null + if [ $? -eq 0 ]; then + echo "Docker is running." + else + echo "Process is not running, starting it..." + sudo service docker start + fi +else + # Install docker repository + # Check that HTTPS transport is available to APT + if [ ! -e /usr/lib/apt/methods/https ]; then + sudo apt-get update + sudo apt-get -y install -y apt-transport-https + fi + # Add the repository to APT sources + echo deb http://mirror.yandex.ru/mirrors/docker/ docker main | sudo tee /etc/apt/sources.list.d/docker.list + # Import the repository key + sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 + # Install docker + sudo apt-get update + sudo apt-get -y install lxc-docker-1.7.1 +fi diff --git a/build/docker/ubuntu-builder/setcontext b/build/docker/ubuntu-builder/setcontext new file mode 100755 index 000000000..bc28994a9 --- /dev/null +++ b/build/docker/ubuntu-builder/setcontext @@ -0,0 +1,20 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# +SETUSER=$1 +SETUID=$2 +SETGID=$3 +SETHOME=$4 + +getent group $SETUSER || /usr/sbin/groupadd --gid $SETGID $SETUSER +getent passwd $SETUSER || /usr/sbin/adduser --system --uid=$SETUID --gid=$SETGID --home $SETHOME --shell /bin/bash $SETUSER +/usr/sbin/usermod -a -G fuse $SETUSER +exit 0 diff --git a/build/f_isoroot/Makefile b/build/f_isoroot/Makefile new file mode 100644 index 000000000..448f4158c --- /dev/null +++ b/build/f_isoroot/Makefile @@ -0,0 +1,51 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Add Fuel plugin build targets here +# Plugins disabled for the Fuel Mitaka uplift. Please re-enable your plugin as you have +# verified it! +PLUGINS ?= f_odlpluginbuild f_onosfwpluginbuild f_vsperfpluginbuild f_ovs-nsh-dpdk-pluginbuild f_yardstick-pluginbuild f_kvm-pluginbuild f_bgpvpn-pluginbuild f_collectd-ceilometer-pluginbuild +#PLUGINS = f_odlpluginbuild f_onosfwpluginbuild f_ovsnfv-dpdk-pluginbuild f_vsperfpluginbuild f_ovs-nsh-dpdk-pluginbuild f_bgpvpn-pluginbuild +export PLUGINS +#NON_8-0_REBASED_PLUGINS = f_bgpvpn-pluginbuild + +# If the BUILD_FUEL_PLUGINS environment variable is set, only build the plugins +# indicated therein. +# Temporarilu disabling f_repobuild that currently have some issues on +# Fuel Mitaka. +ifdef BUILD_FUEL_PLUGINS +export BUILD_FUEL_PLUGINS +SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS) +$(warning Overriding plugin build selection to $(BUILD_FUEL_PLUGINS)) +else +SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS) +endif +SUBCLEAN = $(addsuffix .clean,$(SUBDIRS)) + +.PHONY: all +all: $(SUBDIRS) + +.PHONY: $(SUBDIRS) +$(SUBDIRS): + @mkdir -p release + $(MAKE) -C $@ -f Makefile release + +.PHONY: clean $(SUBCLEAN) +clean: $(SUBCLEAN) + @rm -Rf release + +$(SUBCLEAN): %.clean: + $(MAKE) -C $* -f Makefile clean + +.PHONY: release +release: $(SUBDIRS) + @cp -Rvp release/* ../release/isoroot + +include cache.mk diff --git a/build/f_isoroot/README b/build/f_isoroot/README new file mode 100644 index 000000000..eb54c0827 --- /dev/null +++ b/build/f_isoroot/README @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +This is to (a) replace/patch existing Fuel ISO files as well +as adding new information directly into the ISO structure + +Please separate changes into "patches" and new content into +"additions". Both directories will be anchored at the ISO file +root. diff --git a/build/f_isoroot/cache.mk b/build/f_isoroot/cache.mk new file mode 100644 index 000000000..2df3b6bd1 --- /dev/null +++ b/build/f_isoroot/cache.mk @@ -0,0 +1,37 @@ +############################################################################## +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +############################################################################# +# Cache operations - only used when building through ci/build.sh +# +# This file is only meant for a top Makefile which is only calling its +# own SUBDIRS, without building any cachable artifact by itself. +############################################################################# + +# Call sub caches +SUBGETCACHE = $(addsuffix .getcache,$(SUBDIRS)) +$(SUBGETCACHE): %.getcache: + $(MAKE) -C $* -f Makefile get-cache + +SUBPUTCACHE = $(addsuffix .putcache,$(SUBDIRS)) +$(SUBPUTCACHE): %.putcache: + $(MAKE) -C $* -f Makefile put-cache + +SUBCLEANCACHE = $(addsuffix .cleancache,$(SUBDIRS)) +$(SUBCLEANCACHE): %.cleancache: + $(MAKE) -C $* -f Makefile clean-cache + +.PHONY: get-cache +get-cache: $(SUBGETCACHE) + +.PHONY: put-cache +put-cache: $(SUBPUTCACHE) + +.PHONY: clean-cache +clean-cache: $(SUBCLEANCACHE) diff --git a/build/f_isoroot/f_bgpvpn-pluginbuild/Makefile b/build/f_isoroot/f_bgpvpn-pluginbuild/Makefile new file mode 100644 index 000000000..8ce28fcf8 --- /dev/null +++ b/build/f_isoroot/f_bgpvpn-pluginbuild/Makefile @@ -0,0 +1,91 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +############################################################################ +# BEGIN of Include definitions +# +include config.mk +# +# END Include definitions +############################################################################# + + +.PHONY: all +all: .bgpvpnbuild + +.PHONY: clean +clean: + @rm -f .bgpvpnbuild ../release/opnfv/bgpvpn*.rpm bgpvpn*.rpm + @rm -f $(BUILD_BASE)/gitinfo_bgpvpnplugin.txt gitinfo_bgpvpnplugin.txt + +.PHONY: release +release:.bgpvpnbuild + @rm -f ../release/opnfv/bgpvpn*.rpm + @mkdir -p ../release/bgpvpn + @cp bgpvpn*.rpm ../release/opnfv/ + cp gitinfo_bgpvpnplugin.txt $(BUILD_BASE) + +.bgpvpnbuild: + rm -rf fuel-plugin-bgpvpn + git clone $(BGPVPN_REPO) + cd fuel-plugin-bgpvpn; \ + git checkout $(BGPVPN_BRANCH); \ + if [ ! -z $(BGPVPN_CHANGE) ]; then \ + git fetch $(BGPVPN_REPO) $(BGPVPN_CHANGE); \ + git checkout FETCH_HEAD; \ + fi + fpb --debug --build fuel-plugin-bgpvpn/ + mv fuel-plugin-bgpvpn/bgpvpn*.rpm . + $(REPOINFO) -r . > gitinfo_bgpvpnplugin.txt + rm -rf fuel-plugin-bgpvpn + touch .bgpvpnbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + + +# Create a unique hash to be used for getting and putting cache, based on: +# - The SHA1 hash of the HEAD on the plugin repo's $(BGPVPN_BRANCH) +# - The contents of this Makefile +.cacheid: + @if [ ! -z $(BGPVPN_CHANGE) ]; then \ + $(CACHETOOL) getcommitid $(BGPVPN_REPO) $(BGPVPN_CHANGE) > .cachedata; \ + else \ + $(CACHETOOL) getcommitid $(BGPVPN_REPO) $(BGPVPN_BRANCH) > .cachedata; \ + fi + sha1sum Makefile >> .cachedata + sha1sum config.mk >> .cachedata + echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - .bgpvpnbuild bgpvpn*.rpm gitinfo_bgpvpnplugin.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk b/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk new file mode 100644 index 000000000..ba6fb73ff --- /dev/null +++ b/build/f_isoroot/f_bgpvpn-pluginbuild/config.mk @@ -0,0 +1,12 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +BGPVPN_BRANCH?=master +BGPVPN_REPO?="https://github.com/openstack/fuel-plugin-bgpvpn.git" +BGPVPN_CHANGE?=3349842af5724be63a74a82c9060848d9d3d299e diff --git a/build/f_isoroot/f_bootstrap/Makefile b/build/f_isoroot/f_bootstrap/Makefile new file mode 100644 index 000000000..1c08405ab --- /dev/null +++ b/build/f_isoroot/f_bootstrap/Makefile @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +.PHONY: all +all: + @mkdir -p release/opnfv/bootstrap/pre.d + @mkdir -p release/opnfv/bootstrap/post.d + @mkdir -p release/usr/local/sbin + @cp pre-scripts/* release/opnfv/bootstrap/pre.d + @cp post-scripts/* release/opnfv/bootstrap/post.d + +.PHONY: clean +clean: + @rm -rf release + +.PHONY: release +release:all + @cp -Rvp release/* ../release + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @echo "clean-cache not implemented" + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: + @echo "get-cache not implemented" + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: + @echo "put-cache not implemented" diff --git a/build/f_isoroot/f_bootstrap/README b/build/f_isoroot/f_bootstrap/README new file mode 100644 index 000000000..5da954ca4 --- /dev/null +++ b/build/f_isoroot/f_bootstrap/README @@ -0,0 +1,24 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +This function modifies the adds hooks to the beginning and end of +/usr/local/sbin/bootstrap_admin_nodes.sh, which is run as the +last post step from ks.cfg + +This enables other functions to place scripts into two directories +run either at the beginning or the end of the bootstrap procedure: + + /opt/opnfv/bootstrap/pre.d + /opt/opnfv/bootstrap/post.d + +These will be run in lexical order at pre or post. + +CI note: Using pre.d, it would be possible to "inject" a pre-defined +astute.yaml for Fuel. diff --git a/build/f_isoroot/f_bootstrap/post-scripts/00_post_example.sh b/build/f_isoroot/f_bootstrap/post-scripts/00_post_example.sh new file mode 100755 index 000000000..4dfeca58e --- /dev/null +++ b/build/f_isoroot/f_bootstrap/post-scripts/00_post_example.sh @@ -0,0 +1,14 @@ +#/bin/sh +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +date +echo "This is an example file run at post-bootstrap." +exit 0 diff --git a/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh b/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh new file mode 100755 index 000000000..be17e7e0a --- /dev/null +++ b/build/f_isoroot/f_bootstrap/post-scripts/03_install_repo.sh @@ -0,0 +1,34 @@ +#/bin/sh +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +echo "Installing pre-build repo" +if [ ! -d /opt/opnfv/nailgun ]; then + echo "Error - found no repo!" + exit 1 +fi + +mkdir -p /var/www/nailgun +mv /opt/opnfv/nailgun/* /var/www/nailgun +if [ $? -ne 0 ]; then + echo "Error moving repos to their correct location!" + exit 1 +fi +rmdir /opt/opnfv/nailgun +if [ $? -ne 0 ]; then + echo "Error removing /opt/opnfv/nailgun directory!" + exit 1 +fi +mv /opt/opnfv/fuel_bootstrap_cli.yaml /etc/fuel-bootstrap-cli/fuel_bootstrap_cli.yaml +if [ $? -ne 0 ]; then + echo "Error moving bootstrap image configuration!" + exit 1 +fi +echo "Done installing pre-build repo" diff --git a/build/f_isoroot/f_bootstrap/pre-scripts/00_pre_example.sh b/build/f_isoroot/f_bootstrap/pre-scripts/00_pre_example.sh new file mode 100755 index 000000000..3eaffa523 --- /dev/null +++ b/build/f_isoroot/f_bootstrap/pre-scripts/00_pre_example.sh @@ -0,0 +1,14 @@ +#/bin/sh +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +date +echo "This is an example file run at pre-bootstrap." +exit 0 diff --git a/build/f_isoroot/f_collectd-ceilometer-pluginbuild/Makefile b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/Makefile new file mode 100644 index 000000000..17bc36d91 --- /dev/null +++ b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/Makefile @@ -0,0 +1,91 @@ +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# ruijing.guo@intel.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +############################################################################ +# BEGIN of Include definitions +# +include config.mk +# +# END Include definitions +############################################################################# + +.PHONY: all +all: .fastpathmetrics + +.PHONY: clean +clean: + @rm -f .fastpathmetrics ../release/opnfv/fuel-plugin-collectd-ceilometer-*.rpm fuel-plugin-collectd-ceilometer-*.rpm + @rm -f $(BUILD_BASE)/gitinfo_collectd-ceilometer-plugin.txt gitinfo_collectd-ceilometer-plugin.txt + +.PHONY: release +release:.fastpathmetrics + @rm -f ../release/opnfv/fuel-plugin-collectd-ceilometer-*.rpm + @mkdir -p ../release/opnfv + @cp fuel-plugin-collectd-ceilometer*.rpm ../release/opnfv/ + cp gitinfo_collectd-ceilometer-plugin.txt $(BUILD_BASE) + +.fastpathmetrics: + @rm -rf fastpathmetrics + git clone $(COLLECTD_CEILOMETER_REPO) + cd fastpathmetrics; \ + git checkout $(COLLECTD_CEILOMETER_BRANCH); \ + if [ ! -z $(COLLECTD_CEILOMETER_CHANGE) ]; then \ + git fetch $(COLLECTD_CEILOMETER_REPO) $(COLLECTD_CEILOMETER_CHANGE); \ + git checkout FETCH_HEAD; \ + fi + fpb --debug --build fastpathmetrics/src/fuel-plugin + @mv fastpathmetrics/src/fuel-plugin/fuel-plugin-collectd-ceilometer*.rpm . + $(REPOINFO) -r . > gitinfo_collectd-ceilometer-plugin.txt + @rm -rf fastpathmetrics + @touch .fastpathmetrics + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + + +# Create a unique hash to be used for getting and putting cache, based on: +# - The SHA1 hash of the HEAD on the plugin repo's $(COLLECTD_CEILOMETER_BRANCH) +# - The contents of this Makefile +.cacheid: + @if [ ! -z $(COLLECTD_CEILOMETER_CHANGE) ]; then \ + $(CACHETOOL) getcommitid $(COLLECTD_CEILOMETER_REPO) $(COLLECTD_CEILOMETER_CHANGE) > .cachedata; \ + else \ + $(CACHETOOL) getcommitid $(COLLECTD_CEILOMETER_REPO) $(COLLECTD_CEILOMETER_BRANCH) > .cachedata; \ + fi + @sha1sum Makefile | awk {'print $$1'} >> .cachedata + @sha1sum config.mk | awk {'print $$1'} >> .cachedata + @echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + @cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - .fastpathmetrics fuel-plugin-collectd-ceilometer*.rpm gitinfo_collectd-ceilometer-plugin.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk new file mode 100644 index 000000000..0106213d3 --- /dev/null +++ b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/config.mk @@ -0,0 +1,13 @@ +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# ruijing.guo@intel.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +COLLECTD_CEILOMETER_BRANCH?=3ddf95235ef62de5fb9a520b3aa8f120518b0077 +COLLECTD_CEILOMETER_REPO?=http://gerrit.opnfv.org/gerrit/fastpathmetrics +COLLECTD_CEILOMETER_CHANGE?= diff --git a/build/f_isoroot/f_collectd-ceilometer-pluginbuild/packages.yaml b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/packages.yaml new file mode 100644 index 000000000..1ba7201f4 --- /dev/null +++ b/build/f_isoroot/f_collectd-ceilometer-pluginbuild/packages.yaml @@ -0,0 +1,9 @@ +############################################################################## +# Copyright (c) 2016 OPNFV. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +packages: + - "collectd" diff --git a/build/f_isoroot/f_isolinux/Makefile b/build/f_isoroot/f_isolinux/Makefile new file mode 100644 index 000000000..f3e09b234 --- /dev/null +++ b/build/f_isoroot/f_isolinux/Makefile @@ -0,0 +1,49 @@ +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +.PHONY: all +all: isolinux.cfg.iso + @mkdir -p release/isolinux + grep -q OpenStack_Fuel isolinux.cfg.iso + sed "s/OpenStack_Fuel/$(PRODNO)_$(REVSTATE)/g" isolinux.cfg.iso > isolinux.cfg + @cp isolinux.cfg release/isolinux/isolinux.cfg + @cp isolinux.cfg.iso release/isolinux/isolinux.cfg.orig + +isolinux.cfg.iso: + 7z -so x $(ISOCACHE) isolinux/isolinux.cfg > isolinux.cfg.iso + +.PHONY: clean +clean: + @rm -rf release isolinux.cfg.iso isolinux.cfg + +.PHONY: release +release: all + @cp -Rvp release/* ../release + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @echo "clean-cache not implemented" + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: + @echo "get-cache not implemented" + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: + @echo "put-cache not implemented" diff --git a/build/f_isoroot/f_kscfg/Makefile b/build/f_isoroot/f_kscfg/Makefile new file mode 100644 index 000000000..cfd433f8d --- /dev/null +++ b/build/f_isoroot/f_kscfg/Makefile @@ -0,0 +1,50 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +.PHONY: all +all: ks.cfg.iso + @mkdir -p release + grep -q OpenStack_Fuel ks.cfg.iso + sed "s/OpenStack_Fuel/$(PRODNO)_$(REVSTATE)/g" ks.cfg.iso > ks.cfg + /usr/bin/patch -p0 < ks.cfg.patch + @cp ks.cfg.iso release/ks.cfg.orig + @cp ks.cfg release/ks.cfg + +ks.cfg.iso: + 7z -so x $(ISOCACHE) ks.cfg > ks.cfg.iso + +.PHONY: clean +clean: + @rm -rf release ks.cfg.iso ks.cfg + +.PHONY: release +release: all + @cp -Rvp release/* ../release + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @echo "clean-cache not implemented" + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: + @echo "get-cache not implemented" + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: + @echo "put-cache not implemented" diff --git a/build/f_isoroot/f_kscfg/README b/build/f_isoroot/f_kscfg/README new file mode 100644 index 000000000..c85efde77 --- /dev/null +++ b/build/f_isoroot/f_kscfg/README @@ -0,0 +1,18 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +This function modifies the ks.cfg of the Fuel master to +make copy whatever is in the top level opnfv directory +of the ISO (populated by, for example, f_configfrontend) to +/opt/opnfv. In the end this is where the installation +parts will evolve. + +This is also the place to force the installation to overwrite +the disk (set forceformat="yes" instead of "no"). diff --git a/build/f_isoroot/f_kscfg/ks.cfg.patch b/build/f_isoroot/f_kscfg/ks.cfg.patch new file mode 100644 index 000000000..a6840e479 --- /dev/null +++ b/build/f_isoroot/f_kscfg/ks.cfg.patch @@ -0,0 +1,31 @@ +*** /dev/null 2016-04-26 10:10:11.481587709 +0200 +--- ks.cfg 2016-04-26 10:10:11.481587709 +0200 +*************** +*** 579,584 **** +--- 579,592 ---- + + EOF + ++ ######### OPNFV addition BEGIN ############ ++ # Copy data into /opt/opnfv ++ # TODO: This ought to be a package instead! ++ mkdir -p /opt/opnfv ++ cp -r ${SOURCE}/opnfv /opt ++ cp ${SOURCE}/gitinfo.txt / ++ ######### OPNFV addition END ############ ++ + # Unmounting source + umount -f ${SOURCE} + rm -rf ${SOURCE} +*************** +*** 647,650 **** + /etc/sysconfig/network-scripts/ifcfg-${ifname%%:*} > \ + /mnt/sysimage/etc/sysconfig/network-scripts/ifcfg-${adminif} + fi +! %end +\ No newline at end of file +--- 655,658 ---- + /etc/sysconfig/network-scripts/ifcfg-${ifname%%:*} > \ + /mnt/sysimage/etc/sysconfig/network-scripts/ifcfg-${adminif} + fi +! %end diff --git a/build/f_isoroot/f_kvm-pluginbuild/Makefile b/build/f_isoroot/f_kvm-pluginbuild/Makefile new file mode 100644 index 000000000..81309ef87 --- /dev/null +++ b/build/f_isoroot/f_kvm-pluginbuild/Makefile @@ -0,0 +1,92 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +############################################################################ +# BEGIN of Include definitions +# +include config.mk +# +# END Include definitions +############################################################################# + +.PHONY: all +all: .kvmbuild + +.PHONY: clean +clean: + @rm -f .kvmbuild ../release/opnfv/fuel-plugin-kvm*.rpm fuel-plugin-kvm*.rpm + @rm -f $(BUILD_BASE)/gitinfo_kvm-plugin.txt gitinfo_kvm-plugin.txt + +.PHONY: release +release:.kvmbuild + @rm -f ../release/opnfv/fuel-plugin-kvm*.rpm + @mkdir -p ../release/opnfv + @cp fuel-plugin-kvm*.rpm ../release/opnfv/ + cp gitinfo_kvm-plugin.txt $(BUILD_BASE) + +.kvmbuild: + @rm -rf kvmfornfv + ../../docker/ubuntu-builder/install_docker.sh + git clone $(KVMFORNFV_REPO) + cd kvmfornfv; \ + git checkout $(KVMFORNFV_BRANCH); \ + if [ ! -z $(KVMFORNFV_CHANGE) ]; then \ + git fetch $(KVMFORNFV_REPO) $(KVMFORNFV_CHANGE); \ + git checkout FETCH_HEAD; \ + fi + cd kvmfornfv/fuel-plugin; \ + INCLUDE_DEPENDENCIES=true fpb --debug --build ./ + @mv kvmfornfv/fuel-plugin/fuel-plugin-kvm*.rpm . + $(REPOINFO) -r . > gitinfo_kvm-plugin.txt + @rm -rf kvmfornfv + @touch .kvmbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + + +# Create a unique hash to be used for getting and putting cache, based on: +# - The SHA1 hash of the HEAD on the plugin repo's $(KVMFORNFV_BRANCH) +# - The contents of this Makefile +.cacheid: + @if [ ! -z $(KVMFORNFV_CHANGE) ]; then \ + $(CACHETOOL) getcommitid $(KVMFORNFV_REPO) $(KVMFORNFV_CHANGE) > .cachedata; \ + else \ + $(CACHETOOL) getcommitid $(KVMFORNFV_REPO) $(KVMFORNFV_BRANCH) > .cachedata; \ + fi + @sha1sum Makefile | awk {'print $$1'} >> .cachedata + @sha1sum config.mk | awk {'print $$1'} >> .cachedata + @echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + @cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - .kvmbuild fuel-plugin-kvm*.rpm gitinfo_kvm-plugin.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_kvm-pluginbuild/config.mk b/build/f_isoroot/f_kvm-pluginbuild/config.mk new file mode 100644 index 000000000..eaef3d414 --- /dev/null +++ b/build/f_isoroot/f_kvm-pluginbuild/config.mk @@ -0,0 +1,12 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +KVMFORNFV_BRANCH=stable/colorado +KVMFORNFV_REPO=https://gerrit.opnfv.org/gerrit/kvmfornfv +KVMFORNFV_CHANGE=e6588d1d59d7ac24f25f644650e6845561dd441b diff --git a/build/f_isoroot/f_odlpluginbuild/Makefile b/build/f_isoroot/f_odlpluginbuild/Makefile new file mode 100644 index 000000000..bf811f25f --- /dev/null +++ b/build/f_isoroot/f_odlpluginbuild/Makefile @@ -0,0 +1,82 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# mskalski@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +include config.mk + +.PHONY: all +all: .odlbuild + +.PHONY: clean +clean: + @rm -f .odlbuild ../release/opnfv/opendaylight*.rpm opendaylight*.rpm + @rm -f $(BUILD_BASE)/gitinfo_odlplugin.txt gitinfo_odlplugin.txt + @rm -rf fuel-plugins + +.PHONY: release +release:.odlbuild + @rm -f ../release/opnfv/opendaylight*.rpm + @mkdir -p ../release/opnfv + @cp opendaylight*.rpm ../release/opnfv/ + cp gitinfo_odlplugin.txt $(BUILD_BASE) +.odlbuild: + rm -rf fuel-plugin-opendaylight + git clone -b $(FUEL_PLUGIN_ODL_BRANCH) $(FUEL_PLUGIN_ODL_REPO) + cd fuel-plugin-opendaylight; \ + if [ -n $(FUEL_PLUGIN_ODL_CHANGE) ]; then \ + git checkout $(FUEL_PLUGIN_ODL_CHANGE); \ + fi + fpb --debug --build fuel-plugin-opendaylight/ + mv fuel-plugin-opendaylight/opendaylight*.rpm . + $(REPOINFO) -r . > gitinfo_odlplugin.txt + rm -rf fuel-plugin-opendaylight + touch .odlbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + + +# Create a unique hash to be used for getting and putting cache, based on: +# - The SHA1 hash of the HEAD on the plugin repo's $(FUEL_PLUGIN_ODL_BRANCH) +# - The contents of this Makefile +.cacheid: + if [ -n $(FUEL_PLUGIN_ODL_CHANGE) ]; then \ + $(CACHETOOL) getcommitid $(FUEL_PLUGIN_ODL_REPO) $(FUEL_PLUGIN_ODL_CHANGE) > .cachedata; \ + else \ + $(CACHETOOL) getcommitid $(FUEL_PLUGIN_ODL_REPO) $(FUEL_PLUGIN_ODL_BRANCH) > .cachedata; \ + fi + @sha1sum Makefile | awk {'print $$1'} >> .cachedata + @sha1sum config.mk | awk {'print $$1'} >> .cachedata + @echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - .odlbuild opendaylight*.rpm gitinfo_odlplugin.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_odlpluginbuild/config.mk b/build/f_isoroot/f_odlpluginbuild/config.mk new file mode 100644 index 000000000..7074ee2e0 --- /dev/null +++ b/build/f_isoroot/f_odlpluginbuild/config.mk @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +FUEL_PLUGIN_ODL_BRANCH ?= master +FUEL_PLUGIN_ODL_CHANGE ?= 58cf41ff4e9ed928bf024b166776b9df1a2e517b +FUEL_PLUGIN_ODL_REPO ?= https://github.com/openstack/fuel-plugin-opendaylight.git + +export ODL_TARBALL_LOCATION?=https://nexus.opendaylight.org/content/repositories/public/org/opendaylight/integration/distribution-karaf/0.4.3-Beryllium-SR3/distribution-karaf-0.4.3-Beryllium-SR3.tar.gz +export ODL_VERSION_NUMBER?=0.4.3 +export ODL_BORON_TARBALL_LOCATION?=http://artifacts.opnfv.org/fuel/colorado/vendor/distribution-karaf-0.5.0-Boron-RC3.5.tar.gz diff --git a/build/f_isoroot/f_onosfwpluginbuild/Makefile b/build/f_isoroot/f_onosfwpluginbuild/Makefile new file mode 100644 index 000000000..02ba02951 --- /dev/null +++ b/build/f_isoroot/f_onosfwpluginbuild/Makefile @@ -0,0 +1,90 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +############################################################################ +# BEGIN of Include definitions +# +include config.mk +# +# END Include definitions +############################################################################# + +.PHONY: all +all: .onosbuild + +.PHONY: clean +clean: + @rm -f .onosbuild ../release/opnfv/onos*.rpm onos*.rpm + @rm -f $(BUILD_BASE)/gitinfo_onos-plugin.txt gitinfo_onos-plugin.txt + +.PHONY: release +release:.onosbuild + @rm -f ../release/opnfv/onos*.rpm + @mkdir -p ../release/opnfv + @cp onos*.rpm ../release/opnfv/ + cp gitinfo_onos-plugin.txt $(BUILD_BASE) + +.onosbuild: + @rm -rf fuel-plugin-onos + git clone $(ONOS_REPO) + cd fuel-plugin-onos; \ + git checkout $(ONOS_BRANCH); \ + if [ ! -z $(ONOS_CHANGE) ]; then \ + git fetch $(ONOS_REPO) $(ONOS_CHANGE); \ + git checkout FETCH_HEAD; \ + fi + INCLUDE_DEPENDENCIES=true fpb --debug --build fuel-plugin-onos/ + @mv fuel-plugin-onos/onos*.rpm . + $(REPOINFO) -r . > gitinfo_onos-plugin.txt + @rm -rf fuel-plugin-onos + @touch .onosbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + + +# Create a unique hash to be used for getting and putting cache, based on: +# - The SHA1 hash of the HEAD on the plugin repo's $(ONOS_BRANCH) +# - The contents of this Makefile +.cacheid: + @if [ ! -z $(ONOS_CHANGE) ]; then \ + $(CACHETOOL) getcommitid $(ONOS_REPO) $(ONOS_CHANGE) > .cachedata; \ + else \ + $(CACHETOOL) getcommitid $(ONOS_REPO) $(ONOS_BRANCH) > .cachedata; \ + fi + @sha1sum Makefile | awk {'print $$1'} >> .cachedata + @sha1sum config.mk | awk {'print $$1'} >> .cachedata + @echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + @cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - .onosbuild onos*.rpm gitinfo_onos-plugin.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_onosfwpluginbuild/config.mk b/build/f_isoroot/f_onosfwpluginbuild/config.mk new file mode 100644 index 000000000..19d9d00d0 --- /dev/null +++ b/build/f_isoroot/f_onosfwpluginbuild/config.mk @@ -0,0 +1,11 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +ONOS_BRANCH=Mitaka +ONOS_REPO=git://git.openstack.org/openstack/fuel-plugin-onos diff --git a/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/Makefile b/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/Makefile new file mode 100644 index 000000000..2d0d202af --- /dev/null +++ b/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/Makefile @@ -0,0 +1,91 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +############################################################################ +# BEGIN of Include definitions +# +include config.mk +# +# END Include definitions +############################################################################# + +.PHONY: all +all: .ovsbuild + +.PHONY: clean +clean: + @rm -f .ovsbuild ../release/opnfv/fuel-plugin-ovs-*.rpm fuel-plugin-ovs-*.rpm + @rm -f $(BUILD_BASE)/gitinfo_ovs-nsh-dpdk-plugin.txt gitinfo_ovs-nsh-dpdk-plugin.txt + +.PHONY: release +release:.ovsbuild + @rm -f ../release/opnfv/fuel-plugin-ovs-*.rpm + @mkdir -p ../release/opnfv + @cp fuel-plugin-ovs*.rpm ../release/opnfv/ + cp gitinfo_ovs-nsh-dpdk-plugin.txt $(BUILD_BASE) + +.ovsbuild: + @rm -rf fuel-plugin-ovs + ../../docker/ubuntu-builder/install_docker.sh + git clone $(OVS_NSH_DPDK_REPO) + cd fuel-plugin-ovs; \ + git checkout $(OVS_NSH_DPDK_BRANCH); \ + if [ ! -z $(OVS_NSH_DPDK_CHANGE) ]; then \ + git fetch $(OVS_NSH_DPDK_REPO) $(OVS_NSH_DPDK_CHANGE); \ + git checkout FETCH_HEAD; \ + fi + INCLUDE_DEPENDENCIES=true fpb --debug --build fuel-plugin-ovs/ + @mv fuel-plugin-ovs/fuel-plugin-ovs*.rpm . + $(REPOINFO) -r . > gitinfo_ovs-nsh-dpdk-plugin.txt + @rm -rf fuel-plugin-ovs + @touch .ovsbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + + +# Create a unique hash to be used for getting and putting cache, based on: +# - The SHA1 hash of the HEAD on the plugin repo's $(OVS_NSH_DPDK_BRANCH) +# - The contents of this Makefile +.cacheid: + @if [ ! -z $(OVS_NSH_DPDK_CHANGE) ]; then \ + $(CACHETOOL) getcommitid $(OVS_NSH_DPDK_REPO) $(OVS_NSH_DPDK_CHANGE) > .cachedata; \ + else \ + $(CACHETOOL) getcommitid $(OVS_NSH_DPDK_REPO) $(OVS_NSH_DPDK_BRANCH) > .cachedata; \ + fi + @sha1sum Makefile | awk {'print $$1'} >> .cachedata + @sha1sum config.mk | awk {'print $$1'} >> .cachedata + @echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + @cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - .ovsbuild fuel-plugin-ovs*.rpm gitinfo_ovs-nsh-dpdk-plugin.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/config.mk b/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/config.mk new file mode 100644 index 000000000..8effbd26b --- /dev/null +++ b/build/f_isoroot/f_ovs-nsh-dpdk-pluginbuild/config.mk @@ -0,0 +1,12 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +OVS_NSH_DPDK_BRANCH?=074aadce66ef3cf322a00a3cff5c57deb84088b5 +OVS_NSH_DPDK_REPO?=https://review.openstack.org/openstack/fuel-plugin-ovs +OVS_NSH_DPDK_CHANGE?= diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile new file mode 100644 index 000000000..7c6b16304 --- /dev/null +++ b/build/f_isoroot/f_repobuild/Makefile @@ -0,0 +1,91 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +SHELL := /bin/bash +TOP := $(shell pwd) +TMP_ROOT_DIR := $(shell echo "$(MIRROR_UBUNTU_ROOT)" | cut -d "/" -f2) + +include ../../config.mk +include config.mk + +export MOS_VERSION +export OPENSTACK_VERSION + +.PHONY: all +all: nailgun + +nailgun: + sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml + rm -Rf nailgun + sudo mkdir -p /var/www/nailgun + git clone $(FUEL_MIRROR_REPO) + cd fuel-mirror && git checkout -q $(FUEL_MIRROR_COMMIT) + sudo pip install -U -r ./fuel-mirror/requirements.txt + sudo pip install ./fuel-mirror + sudo pip install ./fuel-mirror/contrib/fuel_mirror + ./opnfv_mirror_conf.py + sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group ubuntu --pattern=ubuntu + sudo chmod -R 755 /var/www/nailgun + cp -Rp /var/www/nailgun . + # On the end we want to have ubuntu repository in mirrors/ubuntu directory + -if [ "$(MIRROR_UBUNTU_ROOT)" != "/ubuntu/" ]; then \ + mkdir -p nailgun/mirrors/ubuntu;\ + mv nailgun/mirrors$(MIRROR_UBUNTU_ROOT)* nailgun/mirrors/ubuntu;\ + [ "$(MIRROR_UBUNTU_ROOT)" != "/" ] && rm -rf nailgun/mirrors/$(TMP_ROOT_DIR);\ + fi + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +.PHONY: clean +clean: + @rm -rf ../release/opnfv/nailgun nailgun fuel-mirror opnfv-config.yaml ubuntu.yaml + +.PHONY: release +release:nailgun + @rm -Rf ../release/opnfv/nailgun + @mkdir -p ../release/opnfv + @cp -Rp nailgun ../release/opnfv/nailgun + @cp fuel_bootstrap_cli.yaml ../release/opnfv/ + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + +# Create a unique hash to be used for getting and putting cache, based on: +# - Year and week (causing the cache to be rebuilt weekly) +# - The contents of this Makefile +.cacheid: + date +"Repocache %G%V" > .cachedata + sha1sum Makefile >> .cachedata + sha1sum config.mk >> .cachedata + $(CACHETOOL) packages >> .cachedata + echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - nailgun | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_repobuild/config.mk b/build/f_isoroot/f_repobuild/config.mk new file mode 100644 index 000000000..d81732849 --- /dev/null +++ b/build/f_isoroot/f_repobuild/config.mk @@ -0,0 +1,13 @@ +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# mskalski@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +FUEL_MIRROR_REPO?=https://github.com/openstack/fuel-mirror +# Point to the commit where 302 redirects are handled +# https://bugs.launchpad.net/fuel/mitaka/+bug/1593674 +FUEL_MIRROR_COMMIT?=192a3d9f8f993afb12c5108dd9339c6688c23e11 diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml new file mode 100644 index 000000000..fcf4257f1 --- /dev/null +++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2015,2016 Ericsson AB and others. +# mskalski@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- + root_ssh_authorized_file: /root/.ssh/id_rsa.pub + extend_kopts: "biosdevname=0 net.ifnames=1 debug ignore_loglevel log_buf_len=10M print_fatal_signals=1 LOGLEVEL=8" + ubuntu_release: trusty + extra_dirs: + - /usr/share/fuel_bootstrap_cli/files/trusty + output_dir: /tmp/ + kernel_flavor: linux-image-generic-lts-trusty + packages: + - fuel-agent + - hwloc + - hpsa-dkms + - i40e-dkms + - linux-firmware + - linux-firmware-nonfree + - linux-headers-generic + - live-boot + - live-boot-initramfs-tools + - mc + - mcollective + - msmtp-mta + - multipath-tools + - multipath-tools-boot + - nailgun-agent + - nailgun-mcagents + - network-checker + - ntp + - openssh-client + - openssh-server + - squashfs-tools + - ubuntu-minimal + - vim + - wget + - xz-utils + bootstrap_images_dir: /var/www/nailgun/bootstraps + active_bootstrap_symlink: /var/www/nailgun/bootstraps/active_bootstrap + flavor: ubuntu + http_proxy: "" + https_proxy: "" + repos: + - name: ubuntu + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty + type: deb + - name: ubuntu-updates + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty-updates + type: deb + - name: ubuntu-security + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty-security + type: deb + - name: mos + section: "main restricted" + uri: "http://127.0.0.1:8080/ubuntu/x86_64" + priority: 1050 + suite: mos9.0 + type: deb + skip_default_img_build: false + direct_repo_addresses: + - "127.0.0.1" diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py new file mode 100755 index 000000000..1c7eb591a --- /dev/null +++ b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# mskalski@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import yaml + +current_snapshot = os.environ["LATEST_TARGET_UBUNTU"] +mos_version = os.environ['MOS_VERSION'] +openstack_version = os.environ['OPENSTACK_VERSION'] +mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU'] +mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT'] +mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL'] +if os.environ.get('BUILD_FUEL_PLUGINS'): + plugins = os.environ['BUILD_FUEL_PLUGINS'] +else: + plugins = os.environ['PLUGINS'] + + +configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read() +conf = yaml.load(configuration_file) +conf['pattern_dir'] = '.' +conf['openstack_version'] = openstack_version +conf['mos_version'] = mos_version + +with open('opnfv-config.yaml', 'w') as outfile: + outfile.write( yaml.dump(conf, default_flow_style=False) ) + +pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read() +pattern = yaml.load(pattern_file) +pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) +pattern['ubuntu_baseurl'] = mirror_ubuntu +for group in pattern['groups']['mos']: + group['uri'] = pattern['mos_baseurl'] +for group in pattern['groups']['ubuntu']: + group['uri'] = pattern['ubuntu_baseurl'] + +for plugin in plugins.split(): + path = "../{}/packages.yaml".format(plugin) + if os.path.isfile(path): + f = open(path).read() + plugin_yaml = yaml.load(f) + plugin_set = set(plugin_yaml['packages']) + main_set = set(pattern['packages']) + new_packages = plugin_set - main_set + print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages)) + pattern['packages'] = pattern['packages'] + list(new_packages) + +pattern['requirements']['ubuntu'] = pattern['packages'] + +with open('ubuntu.yaml', 'w') as outfile: + outfile.write( yaml.safe_dump(pattern, default_flow_style=False) ) diff --git a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh new file mode 100755 index 000000000..20b3485a1 --- /dev/null +++ b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh @@ -0,0 +1,40 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015,2016 Ericsson AB and others. +# mskalski@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Try to choose close ubuntu mirror from mirrors.txt, but "whitewash" this +# against the full repo list to removed mirrors not up-to-date. + +# Some Ubuntu mirrors seem less reliable for this type of mirroring - +# as they are discoved they can be added to the blacklist below in order +# for them not to be considered. +BLACKLIST="mirror.clibre.uqam.ca" + +#NOTE: For now the mirror selection is disabled due to issues not yet +# understood/resolved. +#for url in $((curl -s https://launchpad.net/ubuntu/+archivemirrors | \ +# grep -P -B8 "statusUP|statusSIX" | \ +# grep -o -P "(f|ht)tp.*\"" | \ +# sed 's/"$//' | sort | uniq; \ +# curl -s http://mirrors.ubuntu.com/mirrors.txt | sort | uniq) | \ +# sort | uniq -d) +#do +# host=$(echo $url | cut -d'/' -f3) +# echo ${BLACKLIST} | grep -q ${host} && continue +# if curl -s -o /dev/null --head --fail "$url"; then +# echo $url +# exit 0 +# else +# continue +# fi +#done + +# If no suitable local mirror can be found, +# the default archive is returned instead. +echo "http://archive.ubuntu.com/ubuntu/" diff --git a/build/f_isoroot/f_vsperfpluginbuild/Makefile b/build/f_isoroot/f_vsperfpluginbuild/Makefile new file mode 100644 index 000000000..73de611ff --- /dev/null +++ b/build/f_isoroot/f_vsperfpluginbuild/Makefile @@ -0,0 +1,92 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +############################################################################ +# BEGIN of Include definitions +# +include config.mk +# +# END Include definitions +############################################################################# + +.PHONY: all +all: .vsperfbuild + +.PHONY: clean +clean: + @rm -f .vsperfbuild ../release/opnfv/fuel-plugin-vsperf*.rpm fuel-plugin-vsperf*.rpm + @rm -f $(BUILD_BASE)/gitinfo_vsperf-plugin.txt gitinfo_vsperf-plugin.txt + +.PHONY: release +release:.vsperfbuild + @rm -f ../release/opnfv/fuel-plugin-vsperf*.rpm + @mkdir -p ../release/opnfv + @cp fuel-plugin-vsperf*.rpm ../release/opnfv/ + cp gitinfo_vsperf-plugin.txt $(BUILD_BASE) + +.vsperfbuild: + @rm -rf vswitchperf + ../../docker/ubuntu-builder/install_docker.sh + git clone $(VSPERF_REPO) vswitchperf + cd vswitchperf; \ + git checkout $(VSPERF_BRANCH); \ + if [ ! -z $(VSPERF_CHANGE) ]; then \ + git fetch $(VSPERF_REPO) $(VSPERF_CHANGE); \ + git checkout FETCH_HEAD; \ + fi + cd vswitchperf; \ + INCLUDE_DEPENDENCIES=true fpb --debug --build fuel-plugin-vsperf/ + @mv vswitchperf/fuel-plugin-vsperf/fuel-plugin-vsperf*.rpm . + $(REPOINFO) -r . > gitinfo_vsperf-plugin.txt + @rm -rf fuel-plugin-vsperf + @touch .vsperfbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + + +# Create a unique hash to be used for getting and putting cache, based on: +# - The SHA1 hash of the HEAD on the plugin repo's $(VSPERF_BRANCH) +# - The contents of this Makefile +.cacheid: + @if [ ! -z $(VSPERF_CHANGE) ]; then \ + $(CACHETOOL) getcommitid $(VSPERF_REPO) $(VSPERF_CHANGE) > .cachedata; \ + else \ + $(CACHETOOL) getcommitid $(VSPERF_REPO) $(VSPERF_BRANCH) > .cachedata; \ + fi + @sha1sum Makefile | awk {'print $$1'} >> .cachedata + @sha1sum config.mk | awk {'print $$1'} >> .cachedata + @echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + @cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - .vsperfbuild fuel-plugin-vsperf*.rpm gitinfo_vsperf-plugin.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_vsperfpluginbuild/config.mk b/build/f_isoroot/f_vsperfpluginbuild/config.mk new file mode 100644 index 000000000..35f4059f1 --- /dev/null +++ b/build/f_isoroot/f_vsperfpluginbuild/config.mk @@ -0,0 +1,12 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +VSPERF_BRANCH?=5a0c6772d255aeb0ab1e6246c91bbeb56bb3a239 +VSPERF_REPO?=https://gerrit.opnfv.org/gerrit/vswitchperf.git +VSPERF_CHANGE?= diff --git a/build/f_isoroot/f_yardstick-pluginbuild/Makefile b/build/f_isoroot/f_yardstick-pluginbuild/Makefile new file mode 100644 index 000000000..32de3b3f4 --- /dev/null +++ b/build/f_isoroot/f_yardstick-pluginbuild/Makefile @@ -0,0 +1,92 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# ruijing.guo@intel.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +############################################################################ +# BEGIN of Include definitions +# +include config.mk +# +# END Include definitions +############################################################################# + +.PHONY: all +all: .yardstickbuild + +.PHONY: clean +clean: + @rm -f .yardstickbuild ../release/opnfv/fuel-plugin-yardstick*.rpm fuel-plugin-yardstick*.rpm + @rm -f $(BUILD_BASE)/gitinfo_yardstick-plugin.txt gitinfo_yardstick-plugin.txt + +.PHONY: release +release:.yardstickbuild + @rm -f ../release/opnfv/fuel-plugin-yardstick*.rpm + @mkdir -p ../release/opnfv + @cp fuel-plugin-yardstick*.rpm ../release/opnfv/ + cp gitinfo_yardstick-plugin.txt $(BUILD_BASE) + +.yardstickbuild: + @rm -rf yardstick + git clone $(YARDSTICK_REPO) yardstick + cd yardstick; \ + git checkout $(YARDSTICK_BRANCH); \ + if [ ! -z $(YARDSTICK_CHANGE) ]; then \ + git fetch $(YARDSTICK_REPO) $(YARDSTICK_CHANGE); \ + git checkout FETCH_HEAD; \ + fi + cd yardstick; \ + INCLUDE_DEPENDENCIES=true fpb --debug --build fuel-plugin/ + @mv yardstick/fuel-plugin/fuel-plugin-yardstick*.rpm . + $(REPOINFO) -r . > gitinfo_yardstick-plugin.txt + @rm -rf fuel-plugin-yardstick + @touch .yardstickbuild + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + + +# Create a unique hash to be used for getting and putting cache, based on: +# - The SHA1 hash of the HEAD on the plugin repo's $(YARDSTICK_BRANCH) +# - The contents of this Makefile +.cacheid: + @if [ ! -z $(YARDSTICK_CHANGE) ]; then \ + $(CACHETOOL) getcommitid $(YARDSTICK_REPO) $(YARDSTICK_CHANGE) > .cachedata; \ + else \ + $(CACHETOOL) getcommitid $(YARDSTICK_REPO) $(YARDSTICK_BRANCH) > .cachedata; \ + fi + @sha1sum Makefile | awk {'print $$1'} >> .cachedata + @sha1sum config.mk | awk {'print $$1'} >> .cachedata + @echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + @cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + @rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - .yardstickbuild fuel-plugin-yardstick*.rpm gitinfo_yardstick-plugin.txt | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_yardstick-pluginbuild/config.mk b/build/f_isoroot/f_yardstick-pluginbuild/config.mk new file mode 100644 index 000000000..88b1ea604 --- /dev/null +++ b/build/f_isoroot/f_yardstick-pluginbuild/config.mk @@ -0,0 +1,13 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jonas.bjurel@eicsson.com +# ruijing.guo@intel.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +YARDSTICK_BRANCH?=86b9fc4f434c72bd276e4bf1ce7d7f5a04b150b2 +YARDSTICK_REPO?=https://gerrit.opnfv.org/gerrit/yardstick.git +YARDSTICK_CHANGE?= diff --git a/build/fuel_build_loop b/build/fuel_build_loop new file mode 100755 index 000000000..324f28c1a --- /dev/null +++ b/build/fuel_build_loop @@ -0,0 +1,31 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +echo "MIRROR_MOS_UBUNTU_ROOT=${MIRROR_MOS_UBUNTU_ROOT}" +echo "MIRROR_FUEL=${MIRROR_FUEL}" + +maxcount=3 +cnt=0 +rc=1 +while [ $cnt -lt $maxcount ] && [ $rc -ne 0 ] +do + cnt=$[cnt + 1] + echo -e "\n\n\n*** Starting build attempt # $cnt" + cd /tmp/fuel-main + make iso + rc=$? + if [ $rc -ne 0 ]; then + echo "### Build failed with rc $rc ###" + else + echo "### Build successful at attempt # $cnt" + fi +done +exit $rc diff --git a/build/install/apt-ftparchive-deb.conf b/build/install/apt-ftparchive-deb.conf new file mode 100644 index 000000000..0d15aec6f --- /dev/null +++ b/build/install/apt-ftparchive-deb.conf @@ -0,0 +1,35 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +Dir { + ArchiveDir "."; +}; + +TreeDefault { + Directory "pool"; +}; + +BinDirectory "pool/main" { + Packages "dists/trusty/main/binary-amd64/Packages"; + BinOverride "./indices/override.trusty.main"; + ExtraOverride "./indices/override.trusty.extra.main"; +}; + +Default { + Packages { + Extensions ".deb"; + Compress ". gzip"; + }; +}; + +Contents { + Compress "gzip"; +}; + diff --git a/build/install/apt-ftparchive-release.conf b/build/install/apt-ftparchive-release.conf new file mode 100644 index 000000000..02706bd7d --- /dev/null +++ b/build/install/apt-ftparchive-release.conf @@ -0,0 +1,18 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +APT::FTPArchive::Release::Origin "Ubuntu"; +APT::FTPArchive::Release::Label "Ubuntu"; +APT::FTPArchive::Release::Suite "trusty"; +APT::FTPArchive::Release::Version "1.04"; +APT::FTPArchive::Release::Codename "trusty"; +APT::FTPArchive::Release::Architectures "amd64"; +APT::FTPArchive::Release::Components "main"; +APT::FTPArchive::Release::Description "Ubuntu Trusty Tahr 14.04 LTS"; diff --git a/build/install/apt-ftparchive-udeb.conf b/build/install/apt-ftparchive-udeb.conf new file mode 100644 index 000000000..3b5b239a6 --- /dev/null +++ b/build/install/apt-ftparchive-udeb.conf @@ -0,0 +1,33 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +Dir { + ArchiveDir "."; +}; + +TreeDefault { + Directory "pool"; +}; + +BinDirectory "pool/debian-installer" { + Packages "dists/trusty/main/debian-installer/binary-amd64/Packages"; + BinOverride "./indices/override.trusty.main.debian-installer"; +}; + +Default { + Packages { + Extensions ".udeb"; + Compress ". gzip"; + }; +}; + +Contents { + Compress "gzip"; +}; diff --git a/build/install/install.sh b/build/install/install.sh new file mode 100755 index 000000000..c63241974 --- /dev/null +++ b/build/install/install.sh @@ -0,0 +1,482 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +my_exit() { + rc=$? + + cd /tmp + + if [ -n "$TMP_HOSTMOUNT" ]; then + if [ -d "$TMP_HOSTMOUNT" ]; then + fusermount -u $TMP_HOSTMOUNT + rmdir $TMP_HOSTMOUNT + fi + fi + + if [ -d "$TMP_OLDISO" ]; then + fusermount -u $TMP_OLDISO + rmdir $TMP_OLDISO + fi + + if [ -f "$CONF" ]; then + rm $CONF + fi + + if [ -d "$TMP_ISOPUPPET" ]; then + rm -Rf $TMP_ISOPUPPET + fi +} + +get_deb_name() { + ar p $1 control.tar.gz | tar xzO ./control | grep "^Package:.* " | sed 's/.* //' + if [ $PIPESTATUS -ne 0 ]; then + echo "Error in get_deb_name($1)" + exit 1 + fi +} + +get_deb_rev() { + ar p $1 control.tar.gz | tar xzO ./control | grep "^Version:.* " | sed 's/.* //' + if [ $PIPESTATUS -ne 0 ]; then + echo "Error in get_deb_rev($1)" + exit 1 + fi +} + + +# Added logic for ".orig" files: +# 1. Is there an .orig file for the new file? +# 2. If the file is not present in base ISO -> Error! +# 3. If the file is changed i base ISO -> Error! (need manual rebase) +# 4. If there is no .orig file, but file is present in base ISO: Error! +verify_orig_files() { + OLDDIR=$1 + NEWDIR=$2 + + pushd $NEWDIR >/dev/null + for f in $(find * -type f -name '*.orig') + do + origfile=$NEWDIR/$f + oldfile=$OLDDIR/$(echo $f | sed 's/.orig$//') + newfile=$NEWDIR/$(echo $f | sed 's/.orig$//') + + origfile=${newfile}.orig + # If no corresponding base file, error + if [ ! -f $oldfile ]; then + printf "\n\n\n\n" + echo "Error: found ${newfile} but no" + echo "Error: ${oldfile}" + echo "Error: Manual rebase is needed!" + printf "\n\n\n\n" + exit 1 + fi + + # If orig file differs from base file, error + if ! diff -q $origfile $oldfile > /dev/null; then + printf "\n\n\n\n" + echo "Error: $origfile differs from" + echo "Error: $oldfile!" + echo "Error: Manual rebase is needed!" + printf "\n\n\n\n" + exit 1 + fi + + done + + + # Check that there we're not overwriting oldfiles without having a ".orig" copy + for f in $(find * -type f ! -name '*.orig') + do + newfile=$NEWDIR/$(echo $f | sed 's/.orig$//') + origfile=${newfile}.orig + oldfile=$OLDDIR/$f + if [ -f $oldfile ]; then + if [ ! -f $origfile ]; then + printf "\n\n\n\n" + echo "Error: Will overwrite $oldfile, but there is no" + echo "Error: $origfile!" + echo "Error: You need to create the `basename $origfile`!" + printf "\n\n\n\n" + exit 1 + fi + fi + done + + + popd >/dev/null +} + +prep_make_live() { + DEST=$TMP_HOSTMOUNT + REPO=$DEST/var/www/nailgun/ubuntu/fuelweb/x86_64 + echo "Live install" + ssh-copy-id root@$FUELHOST + sshfs root@1${FUELHOST}:/ $TMP_HOSTMOUNT + + if [ -f $REPO/dists/trusty/main/binary-amd64/Packages.backup ]; then + echo "Error - found backup file for Packages!" + exit 1 + fi + + if [ -f $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup ]; then + echo "Error - found backup file for Packages.gz!" + exit 1 + fi + + if [ -f $REPO/dists/trusty/Release.backup ]; then + echo "Error - found backup file for Release!" + exit 1 + fi + + if [ -d $DEST/etc/puppet.backup ]; then + echo "Error - found backup file for Puppet!" + exit 1 + fi + + cp $REPO/dists/trusty/main/binary-amd64/Packages $REPO/dists/trusty/main/binary-amd64/Packages.backup + cp $REPO/dists/trusty/main/binary-amd64/Packages.gz $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup + cp $REPO/dists/trusty/Release $REPO/dists/trusty/Release.backup + cp -Rvp $DEST/etc/puppet $DEST/etc/puppet.backup +} + +post_make_live() { + if [ -d $TOP/release/puppet/modules ]; then + echo "Installing into Puppet:" + cd $TOP/release/puppet/modules + if [ `ls -1 | wc -l` -gt 0 ]; then + for dir in * + do + echo " $dir" + cp -Rp $dir $DEST/etc/puppet/modules + done + fi + fi +} + +make_live() { + prep_make_live + copy_packages + post_make_live +} + + +prep_make_iso() { + DEST=$TOP/newiso + REPO=$DEST/ubuntu + echo "Preparing ISO..." + echo "Unpack of old ISO..." + if [ -d newiso ]; then + chmod -R 755 newiso + rm -rf newiso + fi + mkdir newiso + fusermount -u $TMP_OLDISO 2>/dev/null || cat /dev/null + fuseiso -p $ORIGISO $TMP_OLDISO + sleep 1 + cd $TMP_OLDISO + find . | cpio -pd $TOP/newiso + cd .. + fusermount -u $TMP_OLDISO + rm -Rf $TMP_OLDISO + chmod -R 755 $TOP/newiso +} + +make_iso_image() { + echo "Making ISO..." + cd $DEST + find . -name TRANS.TBL -exec rm {} \; + rm -rf rr_moved + + mkisofs --quiet -r -V "$VOLUMEID" -publisher "$PUBLISHER" \ + -p `git rev-parse --verify HEAD` -J -R -b isolinux/isolinux.bin \ + -no-emul-boot \ + -boot-load-size 4 -boot-info-table \ + --hide-rr-moved \ + --joliet-long \ + -x "lost+found" -o $NEWISO . + + isoinfo -d -i $NEWISO +} + +# iso_copy_puppet: Create a new puppet-slave.tgz for the iso +iso_copy_puppet() { + echo "Installing into Puppet..." + mkdir -p $TMP_ISOPUPPET/release/puppet + cd $TMP_ISOPUPPET/release/puppet + tar xzf $DEST/puppet-slave.tgz + cd $TOP/release/puppet/modules + + # Remove all .orig files before copying as they now have been verfied + + if [ -d $TOP/release/puppet/modules ]; then + if [ `ls -1 | wc -l` -gt 0 ]; then + verify_orig_files $TMP_ISOPUPPET/release/puppet $TOP/release/puppet/modules + find $TOP/release/puppet/modules -type f -name '*.orig' -exec rm {} \; + for dir in $TOP/release/puppet/modules/* + do + echo " $dir" + cp -Rp $dir $TMP_ISOPUPPET/release/puppet + done + fi + fi + + cd $TMP_ISOPUPPET/release/puppet + tar czf $DEST/puppet-slave.tgz . + cd $TOP + rm -Rf $TMP_ISOPUPPET +} + +# iso_modify_image: Add/patch files in the ISO root +iso_modify_image () { + # TODO: Add logic for ".orig" files (hey! make a function!), which would look + # something like: + # 1. Is there an .orig file? + # 2. If the file is not present in origiso -> Error exit + # 3. If the file is changed in origiso -> Error exit (need manual rebase) + # 4. Otherwise continue, but don't copy orig file (or maybe we should?) + # ... and corresponding reverse logic: + # 1. If there is no .orig file, but file is present in origiso -> Error exit + echo "Modify ISO files (wild copy)..." + + verify_orig_files $DEST $TOP/release/isoroot + # Remove all .orig files before copying as they now have been verfied + find $TOP/release/isoroot -type f -name '*.orig' -exec rm {} \; + + cd $TOP/release/isoroot + cp -Rvp . $DEST + + # Add all Git info files + sort $TOP/gitinfo*.txt > $DEST/gitinfo.txt + cp $DEST/gitinfo.txt $REPORTFILE +} + +make_iso() { + prep_make_iso + copy_packages + #iso_copy_puppet + iso_modify_image + make_iso_image +} + +copy_packages() { + echo "Copying Debian packages..." + cd $TOP/release/packages/ubuntu/pool/debian-installer + + for udeb in `ls -1 | grep '\.udeb$'` + do + echo " $udeb" + cp $udeb $REPO/pool/debian-installer + echo "Did not expect a package here, not supported" + exit 1 + done + + cd $TOP/release/packages/ubuntu/pool/main + for deb in `ls -1 | grep '\.deb$'` + do + echo " $deb" + cp $deb $REPO/pool/main + echo "Did not expect a package here, not supported" + exit 1 + done + + echo "Running Fuel package patch file" + pushd $REPO/pool/main > /dev/null + + for line in `cat $TOP/apply_patches | grep -v "^#" | grep -v "^$"`; do + echo "Line is $line" + echo "Did not expect a line here, not supported" + exit 1 + ref=`echo $line | cut -d '>' -f 1` + origpkg=`echo $line| cut -d '>' -f 2` + url=`echo $line | cut -d '>' -f 3` + + if [ -z "$origpkg" ]; then + echo "Error: No origpkg for patching" + exit 1 + fi + + if [ -z "$url" ]; then + echo "Error: No url for patching" + exit 1 + fi + + if [ -z "$ref" ]; then + echo "Error: No reference text for patching" + exit 1 + fi + + echo "CM: Patching Fuel package for $ref" | tee -a $REPORTFILE + echo "CM: Replacing package $origpkg with $url" | tee -a $REPORTFILE + oldrev=`get_deb_rev $origpkg` + rm $origpkg + wget --quiet $url + topkg=`basename $url` + echo "CM: MD5 of new package:" | tee -a $REPORTFILE + md5sum $topkg | tee -a $REPORTFILE + + patchname=`get_deb_name $topkg` + patchrev=`get_deb_rev $topkg` + echo "Correcting dependencies towards $patchname rev $patchrev - old rev $oldrev" | tee -a $REPORTFILE + $TOP/patch-packages/tools/correct_deps $patchname $oldrev $patchrev | tee -a $REPORTFILE + if [ $PIPESTATUS -ne 0 ]; then + exit 1 + fi + done + + printf "Done running Fuel patch file\n\n" + echo "Running add packages file" + for line in `cat $TOP/add_opnfv_packages | grep -v "^#" | grep -v "^$"`; do + echo "Line is $line" + echo "Did not expect a line here, not supported" + exit 1 + ref=`echo $line | cut -d '>' -f 1` + origpkg=`echo $line| cut -d '>' -f 2` + url=`echo $line | cut -d '>' -f 3` + + if [ -z "$origpkg" ]; then + echo "Error: No origpkg for patching" + exit 1 + fi + + if [ -z "$url" ]; then + echo "Error: No url for patching" + exit 1 + fi + + if [ -z "$ref" ]; then + echo "Error: No reference text for patching" + exit 1 + fi + + if [ "$origpkg" != "NONE" ]; then + echo "CM: Patching added package for $ref" | tee -a $REPORTFILE + echo "CM: Replacing package $origpkg with $url" | tee -a $REPORTFILE + oldrev=`get_deb_rev $origpkg` + rm $origpkg + else + echo "CM: Adding previoulsy uninstalled package for $ref" tee -a $REPORTFILE + fi + wget --quiet $url + topkg=`basename $url` + echo "CM: MD5 of new package:" | tee -a $REPORTFILE + md5sum $topkg | tee -a $REPORTFILE + if [ "$origpkg" != "NONE" ]; then + patchname=`get_deb_name $topkg` + patchrev=`get_deb_rev $topkg` + echo "Correcting dependencies towards $patchname rev $patchrev - old rev $oldrev" | tee -a $REPORTFILE + $TOP/patch-packages/tools/correct_deps $patchname $oldrev $patchrev | tee -a $REPORTFILE + if [ $PIPESTATUS -ne 0 ]; then + exit 1 + fi + fi + done + printf "Done running add packages file\n\n" + + popd > /dev/null + + if [ -f $TOP/patch-packages/release/patch-replacements ]; then + echo "Applying package patches" | tee -a $REPORTFILE + pushd $REPO/pool/main > /dev/null + printf "\n\n" | tee -a $REPORTFILE + for line in `cat $TOP/patch-packages/release/patch-replacements` + do + echo "Did not expect a line here, not supported" + exit 1 + frompkg=`echo $line | cut -d ">" -f 1` + topkg=`echo $line | cut -d ">" -f 2` + echo "CM: Applying patch to $frompkg" | tee -a $REPORTFILE + echo "CM: New package rev after patch: $topkg" | tee -a $REPORTFILE + + if [ ! -f $frompkg ]; then + echo "Error: Can't find $frompkg in repo" + exit 1 + else + oldrev=`get_deb_rev $frompkg` + echo "Removing $frompkg from repo" + rm $frompkg + fi + + if [ ! -f $TOP/patch-packages/release/packages/$topkg ]; then + echo "Error: Can't find $topkg in patch release" + exit 1 + else + echo "Adding $topkg to repo" + cp $TOP/patch-packages/release/packages/$topkg . + fi + + patchname=`get_deb_name $topkg` + patchrev=`get_deb_rev $topkg` + echo "Correcting dependencies towards $patchname rev $patchrev - old rev $oldrev" | tee -a $REPORTFILE + $TOP/patch-packages/tools/correct_deps $patchname $oldrev $patchrev | tee -a $REPORTFILE + if [ $PIPESTATUS -ne 0 ]; then + exit 1 + fi + done + popd > /dev/null + fi + + echo "Generating metadata..." + pushd $REPO > /dev/null + + # The below methods are from 15B + APT_REL_CONF="$TOP/install/apt-ftparchive-release.conf" + APT_DEB_CONF="$TOP/install/apt-ftparchive-deb.conf" + APT_UDEB_CONF="$TOP/install/apt-ftparchive-udeb.conf" + + echo Not running echo apt-ftparchive -c "${APT_REL_CONF}" generate "${APT_DEB_CONF}" + echo Not running apt-ftparchive -c "${APT_REL_CONF}" generate "${APT_DEB_CONF}" + echo Not running apt-ftparchive generate "${APT_UDEB_CONF}" + echo Not running apt-ftparchive generate "${APT_UDEB_CONF}" + + # Fuel also needs this index file + # cat dists/trusty/main/binary-amd64/Packages | \ + # awk '/^Package:/{pkg=$2} + # /^Version:/{print pkg ": \"" $2 "\""}' > ubuntu-versions.yaml + # cp ubuntu-versions.yaml $DEST + + # apt-ftparchive -c "${APT_REL_CONF}" release dists/trusty/ > dists/trusty/Release + # gzip -9cf dists/trusty/Release > dists/trusty/Release.gz + + popd > /dev/null + +} + + +############################################################################# + +trap my_exit EXIT + +CONF=`mktemp /tmp/XXXXXXX` +MODE=$1 +TOP=`pwd` + +if [ $MODE = "iso" ]; then + PUBLISHER="OPNFV" + TMP_OLDISO=`mktemp -d /tmp/XXXXXXX` + TMP_ISOPUPPET=`mktemp -d /tmp/XXXXXXX` + ORIGISO=$2 + NEWISO=$3 + VOLUMEID="$4_$5" + REPORTFILE="${NEWISO}.txt" + echo "Opening reportfile at $REPORTFILE" + touch $REPORTFILE + if [ ! -f $ORIGISO ]; then + echo "Can't find original iso at $ORIGISO" + rm $CONF + exit 1 + fi + + make_iso +else + echo "Unknown mode: $MODE" + exit 1 +fi diff --git a/build/install/uninstall.sh b/build/install/uninstall.sh new file mode 100755 index 000000000..a9e74bc39 --- /dev/null +++ b/build/install/uninstall.sh @@ -0,0 +1,79 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +my_exit() { + cd /tmp + if [ -d "$MOUNT" ]; then + fusermount -u $MOUNT + rmdir $MOUNT + fi +} +trap my_exit EXIT + +echo "Live uninstall is currently disabled as it is not tested" +exit 1 + +TOP=`pwd` +MOUNT=`mktemp -d /tmp/XXXXXXX` +ssh-copy-id root@10.20.0.2 +sshfs root@10.20.0.2:/ $MOUNT + +DEST=$MOUNT +REPO=$DEST/var/www/nailgun/ubuntu/fuelweb/x86_64 + +cd $REPO +if [ ! -f $REPO/dists/trusty/main/binary-amd64/Packages.backup ]; then + echo "Error - didn't find backup file for Packages!" + exit 1 +fi + +if [ ! -f $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup ]; then + echo "Error - didn't find backup file for Packages.gz!" + exit 1 +fi + +if [ ! -f $REPO/dists/trusty/Release.backup ]; then + echo "Error - didn't find backup file for Release!" + exit 1 +fi + +if [ ! -f $DEST/etc/puppet/manifests/site.pp.backup ]; then + echo "Error - didn't find backup file for site.pp!" + exit 1 +fi + +echo "Removing Debian packages:" +cd $TOP/release/pool/main +for deb in *.deb +do + echo " $deb" + rm -Rf $REPO/pool/main/$deb +done +cd $REPO + +echo "Removing Puppet modules:" +cd $TOP/puppet/modules +for dir in * +do + echo " $dir" + rm -Rf $DEST/etc/puppet/modules/$dir +done +cd $REPO + +echo "Restoring backups of datafiles" + +rm -f $REPO/dists/trusty/main/binary-amd64/Packages $REPO/dists/trusty/main/binary-amd64/Packages.gz +rm -f $REPO/dists/trusty/Release $DEST/etc/puppet/manifests/site.pp +mv $REPO/dists/trusty/main/binary-amd64/Packages.backup $REPO/dists/trusty/main/binary-amd64/Packages +mv $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup $REPO/dists/trusty/main/binary-amd64/Packages.gz +mv $REPO/dists/trusty/Release.backup $REPO/dists/trusty/Release +mv $DEST/etc/puppet/manifests/site.pp.backup $DEST/etc/puppet/manifests/site.pp diff --git a/build/patch-packages/Makefile b/build/patch-packages/Makefile new file mode 100644 index 000000000..339c9e7cf --- /dev/null +++ b/build/patch-packages/Makefile @@ -0,0 +1,26 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +SUBDIRS := +SUBCLEAN = $(addsuffix .clean,$(SUBDIRS)) + +.PHONY: $(SUBDIRS) $(SUBCLEAN) clean +$(SUBDIRS): + @mkdir -p release/packages + ORIGISO=$(ORIGISO) REVSTATE=$(REVSTATE) $(MAKE) -C $@ -f Makefile release + +release: $(SUBDIRS) + @echo $(ORIGISO) + +clean: $(SUBCLEAN) + @rm -Rf release + +$(SUBCLEAN): %.clean: + $(MAKE) -C $* -f Makefile clean diff --git a/build/patch-packages/tools/correct_deps b/build/patch-packages/tools/correct_deps new file mode 100755 index 000000000..cfb7d538f --- /dev/null +++ b/build/patch-packages/tools/correct_deps @@ -0,0 +1,78 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +patch_package () { + deb=$1 + pkgdep=$2 + newrev=$3 + + + tmpdir=`mktemp -d /tmp/patchXXXXX` + + cp $deb $tmpdir + pushd $tmpdir > /dev/null + + mkdir -p repack + dpkg -x $deb repack + + mkdir -p repack/DEBIAN + dpkg -e $deb repack/DEBIAN + + + pushd repack/DEBIAN > /dev/null + + echo "Before: `cat control | grep '^Depends:'`" + sed -i "s/$pkgdep (\([^ ]*\) [^)]*)/$pkgdep (\1 $newrev)/" control + echo "After: `cat control | grep '^Depends:'`" + popd > /dev/null + + fakeroot dpkg-deb --build repack + + popd > /dev/null + + cp $tmpdir/repack.deb $deb + rm -Rf $tmpdir +} + +# Name of package for which to check dependencies to +PKGDEP=$1 +# The old revision of the package in question +OLDREV=$2 +# The new revision of the package in question +NEWREV=$3 + +if [ -z "$PKGDEP" ]; then + echo "No package dependency name" + exit 1 +fi + +if [ -z "$OLDREV" ]; then + echo "No old rev" + exit 1 +fi + +if [ -z "$NEWREV" ]; then + echo "No new rev" + exit 1 +fi + + +for deb in *.deb +do + ar p $deb control.tar.gz | tar xzO ./control | grep -q "^Depends:.* ${PKGDEP} ([^ ]* ${OLDREV})" + if [ $? -eq 0 ]; then + name=`ar p $deb control.tar.gz | tar xzO ./control | grep "^Package:.* " | sed 's/.* //'` + echo "**** Changing dependencies line in $deb ($name) ****" + patch_package $deb $PKGDEP $NEWREV + fi +done + diff --git a/build/patch-packages/tools/deb_pack b/build/patch-packages/tools/deb_pack new file mode 100755 index 000000000..f5d42dec3 --- /dev/null +++ b/build/patch-packages/tools/deb_pack @@ -0,0 +1,63 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +my_exit() { + + if [ -d "$ISOROOT" ]; then + fusermount -u $ISOROOT + fi +} + +ME=$(basename $0) + +trap my_exit EXIT + +REV="$(echo $1 | tr '_' '-')" +if [ -z "$REV" ]; then + echo "$ME: Error - no revision info provided" + exit 1 +fi + +if [ ! -d package ]; then + echo "$ME: Error - package directory does not exist" + exit 1 +fi + +echo "Stepping revision number to $REV" + + +revision_step () +{ + REV=$1 + + sed -i "s/^\(Version:.*$\)/\1-eri$REV/" package/DEBIAN/control +} + +md5sum_gen () +{ + pushd package + find * -type f | grep -v "^DEBIAN/" | xargs md5sum | sort > DEBIAN/md5sums + popd +} + +revision_step $REV +md5sum_gen +fakeroot dpkg-deb --build package +PKGNAME=`grep "^Package: " package/DEBIAN/control | awk '{ print $2 }'` +PKGREV=`grep "^Version: " package/DEBIAN/control | awk '{ print $2 }'` +mv package.deb ${PKGNAME}_${PKGREV}.deb + +ORIGPKG=`cat .package` + +# Add patch into file read by install.sh +echo "$ORIGPKG>${PKGNAME}_${PKGREV}.deb" > patch-replacements + diff --git a/build/patch-packages/tools/deb_unpack b/build/patch-packages/tools/deb_unpack new file mode 100755 index 000000000..aaa60b743 --- /dev/null +++ b/build/patch-packages/tools/deb_unpack @@ -0,0 +1,58 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +my_exit() { + + if [ -d "$ISOROOT" ]; then + fusermount -u $ISOROOT + fi +} + +ME=$(basename $0) + +trap my_exit EXIT + + +if [ -z "$1" ]; then + echo "$ME: Error - No package specified" + exit 1 +fi + +if [ -z "$2" ]; then + echo "$ME: Error - No ISO path specified" + exit 1 +fi + +DEB=$1 +ORIGISO=$2 +DEST=package + +if [ -e $DEST -o -d $DEST ]; then + echo "$ME: Error - $DEST already exists" + exit 1 +fi + +ISOROOT=`mktemp -d /tmp/XXXXXXX` +fuseiso -p $ORIGISO $ISOROOT +sleep 1 + +if [ ! -f $ISOROOT/ubuntu/pool/main/$DEB ];then + echo "Could not find package $DEB in source ISO!" +fi + +mkdir -p $DEST +dpkg -x $ISOROOT/ubuntu/pool/main/$DEB $DEST + +mkdir -p $DEST/DEBIAN +dpkg -e $ISOROOT/ubuntu/pool/main/$DEB $DEST/DEBIAN + +echo $DEB > .package
\ No newline at end of file diff --git a/build/patch-packages/tools/udeb_pack b/build/patch-packages/tools/udeb_pack new file mode 100755 index 000000000..e961a7320 --- /dev/null +++ b/build/patch-packages/tools/udeb_pack @@ -0,0 +1,38 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +my_exit() { + + if [ -d "$ISOROOT" ]; then + fusermount -u $ISOROOT + fi +} + +ME=$(basename $0) + +trap my_exit EXIT + +if [ ! -d udebPackage ]; then + echo "$ME: Error - package directory does not exist" + exit 1 +fi + +echo "Stepping revision number to $REV" + + +fakeroot dpkg-deb --build -Zgzip udebPackage +PKGNAME=`grep "^Package: " udebPackage/DEBIAN/control | awk '{ print $2 }'` +PKGREV=`grep "^Version: " udebPackage/DEBIAN/control | awk '{ print $2 }'` +ARCH=`grep "^Architecture: " udebPackage/DEBIAN/control | awk '{ print $2 }'` +mv udebPackage.deb ${PKGNAME}_${PKGREV}_${ARCH}.udeb + +ORIGPKG=`cat .udebpackage` diff --git a/build/patch-packages/tools/udeb_unpack b/build/patch-packages/tools/udeb_unpack new file mode 100755 index 000000000..ed9cd21aa --- /dev/null +++ b/build/patch-packages/tools/udeb_unpack @@ -0,0 +1,58 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +my_exit() { + + if [ -d "$ISOROOT" ]; then + fusermount -u $ISOROOT + fi +} + +ME=$(basename $0) + +trap my_exit EXIT + + +if [ -z "$1" ]; then + echo "$ME: Error - No package specified" + exit 1 +fi + +if [ -z "$2" ]; then + echo "$ME: Error - No ISO path specified" + exit 1 +fi + +DEB=$1 +ORIGISO=$2 +DEST=udebPackage + +if [ -e $DEST -o -d $DEST ]; then + echo "$ME: Error - $DEST already exists" + exit 1 +fi + +ISOROOT=`mktemp -d /tmp/XXXXXXX` +fuseiso -p $ORIGISO $ISOROOT +sleep 1 + +if [ ! -f $ISOROOT/ubuntu/pool/debian-installer/$DEB ];then + echo "Could not find package $DEB in source ISO!" +fi + +mkdir -p $DEST +dpkg -x $ISOROOT/ubuntu/pool/debian-installer/$DEB $DEST + +mkdir -p $DEST/DEBIAN +dpkg -e $ISOROOT/ubuntu/pool/debian-installer/$DEB $DEST/DEBIAN + +echo $DEB > .udebpackage diff --git a/build/patch-packages/tr_example/Makefile b/build/patch-packages/tr_example/Makefile new file mode 100644 index 000000000..b7cab3590 --- /dev/null +++ b/build/patch-packages/tr_example/Makefile @@ -0,0 +1,30 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +TOP := $(shell pwd) + +.PHONY: all +all: + +.PHONY: clean +clean: + @rm -rf package + @rm -rf *.deb + @rm -rf patch-replacements + @rm -rf .package + +.PHONY: release +release: + ../tools/deb_unpack python-oslo.messaging_1.3.0-fuel5.1~mira4_all.deb $(ORIGISO) + @mkdir -p package/etc + @echo "Hello, world" > package/etc/hello.txt + ../tools/deb_pack $(REVSTATE) + @cp *.deb ../release/packages + @cat patch-replacements >> ../release/patch-replacements diff --git a/build/patch-repos/0010-bootstrap_admin_node.sh.patch b/build/patch-repos/0010-bootstrap_admin_node.sh.patch new file mode 100644 index 000000000..d3769adfb --- /dev/null +++ b/build/patch-repos/0010-bootstrap_admin_node.sh.patch @@ -0,0 +1,83 @@ +From: Fuel OPNFV <fuel@opnfv.org> +Date: Mon, 13 Jun 2016 22:23:57 +0200 +Subject: OPNFV: Additions to bootstrap_admin_node.sh + +--- +diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh +index 3197c91..e035145 100755 +--- a/iso/bootstrap_admin_node.sh ++++ b/iso/bootstrap_admin_node.sh +@@ -339,8 +339,22 @@ fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail + set +x + echo "Done!" + ++### OPNFV addition BEGIN ++shopt -s nullglob ++for script in /opt/opnfv/bootstrap/pre.d/*.sh ++do ++ echo "Pre script: $script" >> /root/pre.log 2>&1 ++ $script >> /root/pre.log 2>&1 ++done ++shopt -u nullglob ++### OPNFV addition END ++ ++# Enable sshd ++systemctl enable sshd ++systemctl start sshd ++ + if [[ "$showmenu" == "yes" || "$showmenu" == "YES" ]]; then +- fuelmenu || fail ++ fuelmenu + else + # Give user 15 seconds to enter fuelmenu or else continue + echo +@@ -360,9 +374,10 @@ else + fi + fi + ++# OPNFV: Disabled to speedup installation in offline env. + # Enable online base MOS repos (security, updates) if we run an ISO installation +-[ -f /etc/fuel_build_id ] && \ +- yum-config-manager --enable mos${FUEL_RELEASE}-security mos${FUEL_RELEASE}-updates --save ++#[ -f /etc/fuel_build_id ] && \ ++# yum-config-manager --enable mos${FUEL_RELEASE}-security mos${FUEL_RELEASE}-updates --save + + if [ ! -f "${ASTUTE_YAML}" ]; then + echo ${fuelmenu_fail_message} +@@ -377,7 +392,5 @@ if [ ! -f /etc/fuel_build_id ]; then + +-# Enable sshd +-systemctl enable sshd +-systemctl start sshd ++systemctl reload sshd + + # Enable iptables + systemctl enable iptables.service +@@ -532,6 +545,16 @@ systemctl start ntpd + + bash /etc/rc.local + ++### OPNFV addition BEGIN ++shopt -s nullglob ++for script in /opt/opnfv/bootstrap/post.d/*.sh ++do ++ echo "Post script: $script" >> /root/post.log 2>&1 ++ $script >> /root/post.log 2>&1 ++done ++shopt -u nullglob ++### OPNFV addition END ++ + if [ "`get_bootstrap_skip`" = "False" ]; then + build_ubuntu_bootstrap bs_status || true + else +@@ -571,8 +571,8 @@ + fi + + #Check if repo is accessible +-echo "Checking for access to updates repository..." +-repourl=$(yum repolist all -v | awk '{if ($1 ~ "baseurl" && $3 ~ "updates") print $3}' | head -1) ++echo "Checking for access to updates repository/mirrorlist..." ++repourl=$(yum repolist all -v | awk '{if ($1 ~ "baseurl|mirrors" && $3 ~ "updates") print $3}' | head -1) + if urlaccesscheck check "$repourl" ; then + UPDATE_ISSUES=0 + else diff --git a/build/patch-repos/0020-isolinux.cfg.patch b/build/patch-repos/0020-isolinux.cfg.patch new file mode 100644 index 000000000..f09be2a7f --- /dev/null +++ b/build/patch-repos/0020-isolinux.cfg.patch @@ -0,0 +1,21 @@ +From: Fuel OPNFV <fuel@opnfv.org> +Date: Mon, 13 Jun 2016 22:23:57 +0200 +Subject: OPNFV: showmenu=yes in isolinux.cfg + +--- +diff --git a/iso/isolinux/isolinux.cfg b/iso/isolinux/isolinux.cfg +index c6b1ed9..77a4b18 100644 +--- a/iso/isolinux/isolinux.cfg ++++ b/iso/isolinux/isolinux.cfg +@@ -19,9 +19,9 @@ label nailgun + menu label ^1. Fuel Install (Static IP) + menu default + kernel vmlinuz +- append initrd=initrd.img net.ifnames=0 biosdevname=0 inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:eth0:off::: nameserver=10.20.0.1 ++ append initrd=initrd.img net.ifnames=0 biosdevname=0 inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:eth0:off::: nameserver=10.20.0.1 showmenu=yes + + label nailgunifname + menu label ^2. Fuel Advanced Install (Static IP) + kernel vmlinuz +- append initrd=initrd.img inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:adminif:off::: nameserver=10.20.0.1 ifname=adminif:XX:XX:XX:XX:XX:XX ++ append initrd=initrd.img inst.repo=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ inst.ks=cdrom:LABEL=will_be_substituted_with_ISO_VOLUME_ID:/ks.cfg ip=10.20.0.2::10.20.0.1:255.255.255.0:fuel.domain.tld:adminif:off::: nameserver=10.20.0.1 ifname=adminif:XX:XX:XX:XX:XX:XX showmenu=yes diff --git a/build/patch-repos/0030-repo-multi-arch-local-mirrors.patch b/build/patch-repos/0030-repo-multi-arch-local-mirrors.patch new file mode 100644 index 000000000..d40428743 --- /dev/null +++ b/build/patch-repos/0030-repo-multi-arch-local-mirrors.patch @@ -0,0 +1,59 @@ +From: Stanislaw Kardach <stanislaw.kardach@cavium.com> +Date: Thu, 25 Feb 2016 13:31:19 +0100 +Subject: repo mirror: Allow multi-arch local mirrors + +This patch allows specifying multiple architectures via UBUNTU_ARCH in +form of a list of space separated architectures. The first architecture +in the list is considered primary and will be used for building all the +deb packages by fuel-main. Additional architectures are added to allow +targets of other architectures to use the mirror. +NOTE: this imposes a requirement that all packages built are arch +independent (which is true so far). + +Signed-off-by: Stanislaw Kardach <stanislaw.kardach@cavium.com> +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + mirror/ubuntu/module.mk | 2 +- + sandbox.mk | 2 +- + config.mk | 2 +- + 3 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/mirror/ubuntu/module.mk b/mirror/ubuntu/module.mk +index 7a9466e..fe1ada2 100644 +--- a/mirror/ubuntu/module.mk ++++ b/mirror/ubuntu/module.mk +@@ -81,7 +81,7 @@ $(BUILD_DIR)/mirror/ubuntu/mirror.done: + --root=$(MIRROR_MOS_UBUNTU_ROOT) \ + --dist=$(MIRROR_MOS_UBUNTU_SUITE) \ + --section=$(subst $(space),$(comma),$(MIRROR_MOS_UBUNTU_SECTION)) \ +- --arch=$(UBUNTU_ARCH) \ ++ --arch=$(shell echo $(UBUNTU_ARCH) | tr ' ' ',') \ + $(LOCAL_MIRROR_UBUNTU)/ + rm -rf $(LOCAL_MIRROR_UBUNTU)/.temp $(LOCAL_MIRROR_UBUNTU)/project + $(ACTION.TOUCH) +diff --git a/sandbox.mk b/sandbox.mk +index 4bc3962..5ffddbd 100644 +--- a/sandbox.mk ++++ b/sandbox.mk +@@ -188,7 +188,7 @@ touch $(SANDBOX_UBUNTU)/etc/init.d/.legacy-bootordering + mkdir -p $(SANDBOX_UBUNTU)/usr/sbin + cp -a $(BUILD_DIR)/policy-rc.d $(SANDBOX_UBUNTU)/usr/sbin + echo "Running debootstrap" +-sudo debootstrap --no-check-gpg --include=ca-certificates --arch=$(UBUNTU_ARCH) $(MIRROR_UBUNTU_SUITE) $(SANDBOX_UBUNTU) $(MIRROR_UBUNTU_METHOD)://$(MIRROR_UBUNTU)$(MIRROR_UBUNTU_ROOT) ++sudo debootstrap --no-check-gpg --include=ca-certificates --arch=$(word 1,$(UBUNTU_ARCH)) $(MIRROR_UBUNTU_SUITE) $(SANDBOX_UBUNTU) $(MIRROR_UBUNTU_METHOD)://$(MIRROR_UBUNTU)$(MIRROR_UBUNTU_ROOT) + if [ -e $(SANDBOX_UBUNTU)/etc/resolv.conf ]; then sudo cp -a $(SANDBOX_UBUNTU)/etc/resolv.conf $(SANDBOX_UBUNTU)/etc/resolv.conf.orig; fi + sudo cp /etc/resolv.conf $(SANDBOX_UBUNTU)/etc/resolv.conf + if [ -e $(SANDBOX_UBUNTU)/etc/hosts ]; then sudo cp -a $(SANDBOX_UBUNTU)/etc/hosts $(SANDBOX_UBUNTU)/etc/hosts.orig; fi +diff --git a/config.mk b/config.mk +index 74ee039..45a3b30 100644 +--- a/config.mk ++++ b/config.mk +@@ -49,7 +49,7 @@ UBUNTU_MINOR:=04 + UBUNTU_RELEASE_NUMBER:=$(UBUNTU_MAJOR).$(UBUNTU_MINOR) + UBUNTU_KERNEL_FLAVOR?=lts-trusty + UBUNTU_NETBOOT_FLAVOR?=netboot +-UBUNTU_ARCH:=amd64 ++UBUNTU_ARCH?=amd64 + UBUNTU_IMAGE_RELEASE:=$(UBUNTU_MAJOR)$(UBUNTU_MINOR) + SEPARATE_IMAGES?=/boot,ext2 /,ext4 + diff --git a/build/patch-repos/0040-xorriso-fails-to-add-files-with-path-longer-then-240.patch b/build/patch-repos/0040-xorriso-fails-to-add-files-with-path-longer-then-240.patch new file mode 100644 index 000000000..734523040 --- /dev/null +++ b/build/patch-repos/0040-xorriso-fails-to-add-files-with-path-longer-then-240.patch @@ -0,0 +1,24 @@ +From: Artem Silenkov <asilenkov@mirantis.com> +Date: Tue, 22 Sep 2015 21:54:44 +0300 +Subject: [PATCH] xorriso fails to add files with path longer then 240 + +joliet-long option added to xorriso command line + +Change-Id: I97016cf3e6a554f44d839740ba7d210337435cdc +Related-Bug: #1498619 +--- + iso/module.mk | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/iso/module.mk b/iso/module.mk +index 8995378..0073068 100644 +--- a/iso/module.mk ++++ b/iso/module.mk +@@ -174,6 +174,7 @@ $(ISO_PATH): $(BUILD_DIR)/iso/isoroot.done + -V $(ISO_VOLUME_ID) -p $(ISO_VOLUME_PREP) \ + -J -R \ + -graft-points \ ++ -joliet-long \ + -b isolinux/isolinux.bin -no-emul-boot -boot-load-size 4 -boot-info-table \ + -isohybrid-mbr /usr/lib/syslinux/isohdpfx.bin \ + -eltorito-alt-boot -e images/efiboot.img -no-emul-boot \ diff --git a/build/patch-repos/build/repos/fuel-library/0001-Disable-token-revoke-to-increase-keystone-performanc.patch b/build/patch-repos/build/repos/fuel-library/0001-Disable-token-revoke-to-increase-keystone-performanc.patch new file mode 100644 index 000000000..4d5be493b --- /dev/null +++ b/build/patch-repos/build/repos/fuel-library/0001-Disable-token-revoke-to-increase-keystone-performanc.patch @@ -0,0 +1,44 @@ +From ba542dde34743b276547b7ca3c6591feb9000877 Mon Sep 17 00:00:00 2001 +From: iberezovskiy <iberezovskiy@mirantis.com> +Date: Mon, 19 Sep 2016 12:35:05 +0300 +Subject: [PATCH] Disable token revoke to increase keystone performance + +Change-Id: I4df816369093908ad1eac372f38c92155efbe8e0 +Closes-bug: #1625077 +--- + .../openstack_tasks/manifests/keystone/keystone.pp | 2 ++ + tests/noop/spec/hosts/keystone/keystone_spec.rb | 5 +++++ + 2 files changed, 7 insertions(+) + +diff --git a/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp b/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp +index 3162287..ba9d7df 100644 +--- a/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp ++++ b/deployment/puppet/openstack_tasks/manifests/keystone/keystone.pp +@@ -327,6 +327,8 @@ class openstack_tasks::keystone::keystone { + memcache_pool_unused_timeout => '60', + cache_memcache_servers => $memcache_servers, + policy_driver => 'keystone.policy.backends.sql.Policy', ++ # Set revoke_by_id to false according to LP #1625077 ++ revoke_by_id => false, + } + + Package<| title == 'keystone'|> ~> Service<| title == 'keystone'|> +diff --git a/tests/noop/spec/hosts/keystone/keystone_spec.rb b/tests/noop/spec/hosts/keystone/keystone_spec.rb +index b29e691..3c0f847 100644 +--- a/tests/noop/spec/hosts/keystone/keystone_spec.rb ++++ b/tests/noop/spec/hosts/keystone/keystone_spec.rb +@@ -176,6 +176,11 @@ describe manifest do + should contain_class('keystone').with('sync_db' => primary_controller) + end + ++ it 'should declare keystone class with revoke_by_id set to false' do ++ # Set revoke_by_id to false according to LP #1625077 ++ should contain_class('keystone').with('revoke_by_id' => false) ++ end ++ + it 'should configure keystone with paramters' do + should contain_keystone_config('token/caching').with(:value => 'false') + should contain_keystone_config('cache/enabled').with(:value => 'true') +-- +1.7.9.5 + diff --git a/build/patch-repos/build/repos/fuel-library/0001-Reduce-ceilometer-memory-usage.patch b/build/patch-repos/build/repos/fuel-library/0001-Reduce-ceilometer-memory-usage.patch new file mode 100644 index 000000000..f7fe06884 --- /dev/null +++ b/build/patch-repos/build/repos/fuel-library/0001-Reduce-ceilometer-memory-usage.patch @@ -0,0 +1,30 @@ +From 38315ee3945a5444de13918eb29e7771eb9927c7 Mon Sep 17 00:00:00 2001 +From: Michael Polenchuk <mpolenchuk@mirantis.com> +Date: Thu, 8 Sep 2016 19:25:14 +0300 +Subject: [PATCH 1/1] Reduce ceilometer memory usage + +Change-Id: I8b1a97bd710c859a3543d1aed8226313f35f4f10 +--- + .../manifests/ceilometer/controller.pp | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp +index 2c63ff0..1dda109 100644 +--- a/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp ++++ b/deployment/puppet/openstack_tasks/manifests/ceilometer/controller.pp +@@ -234,5 +234,12 @@ class openstack_tasks::ceilometer::controller { + Ceilometer_config<| title == 'oslo_messaging_rabbit/kombu_compression' |> { value => $kombu_compression } + } + } ++ ++ # Reduce memory usage ++ ceilometer_config { ++ 'DEFAULT/executor_thread_pool_size': value => ceiling($service_workers*0.5); ++ 'notification/pipeline_processing_queues': value => ceiling($service_workers*1.5); ++ 'oslo_messaging_rabbit/rabbit_qos_prefetch_count': value => 5; ++ } + } + } +-- +1.7.9.5 + diff --git a/build/patch-repos/build/repos/fuel-nailgun/0010-Mark-Intel-82599-10-Gigabit-NIC-as-DPDK-capable.patch b/build/patch-repos/build/repos/fuel-nailgun/0010-Mark-Intel-82599-10-Gigabit-NIC-as-DPDK-capable.patch new file mode 100644 index 000000000..d486f2354 --- /dev/null +++ b/build/patch-repos/build/repos/fuel-nailgun/0010-Mark-Intel-82599-10-Gigabit-NIC-as-DPDK-capable.patch @@ -0,0 +1,22 @@ +From: Fuel OPNFV <fuel@opnfv.org> +Date: Wed, 3 Aug 2016 09:41:21 +0200 +Subject: [PATCH] Mark Intel 82599 10 Gigabit NIC as DPDK capable + +This NIC is used on Ericsson POD2 but Fuel doesn't allow to activate +DPDK on it. +--- + nailgun/nailgun/fixtures/openstack.yaml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/nailgun/nailgun/fixtures/openstack.yaml b/nailgun/nailgun/fixtures/openstack.yaml +index 74fa509..d08a4fc 100644 +--- a/nailgun/nailgun/fixtures/openstack.yaml ++++ b/nailgun/nailgun/fixtures/openstack.yaml +@@ -2215,3 +2215,3 @@ + "14e4:163f", "14e4:168d", "14e4:16a1", "14e4:16a2", "14e4:16ad", +- "14e4:16ae", "14e4:163e", "14e4:16a4" ++ "14e4:16ae", "14e4:163e", "14e4:16a4", "8086:10f8" + ] +-- +1.9.1 + diff --git a/build/repo_info.sh b/build/repo_info.sh new file mode 100755 index 000000000..347ede7f7 --- /dev/null +++ b/build/repo_info.sh @@ -0,0 +1,63 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +usage() { + cat <<EOF +Usage: `basename $0` [-r] <path> + + -r + Recursively list all repos found starting at <path> + -h + Display help text +EOF +} + +repoinfo() { + repotop=$(git -C $1 rev-parse --show-toplevel) + origin=$(git -C $repotop config --get remote.origin.url) + sha1=$(git -C $repotop rev-parse HEAD) + echo "$origin: $sha1" +} + + +if [ $# -eq 2 ]; then + case $1 in + -r) + RECURSE=1 + shift + ;; + -h) + usage + exit 0 + ;; + *) + echo "Error, argument $1 not known" >&2 + usage + exit 1 + esac +fi + +if [ $# -gt 1 ]; then + echo "Error, too many arguments" >&2 + usage + exit 1 +fi + +abspath=$(readlink -f $1) + +if [ -n "$RECURSE" ]; then + for dir in $(find $abspath -type d -name .git) + do + repoinfo $(readlink -f $dir/..) + done +else + repoinfo $abspath +fi diff --git a/build/select_closest_fuel_mirror.py b/build/select_closest_fuel_mirror.py new file mode 100755 index 000000000..0fdc69eac --- /dev/null +++ b/build/select_closest_fuel_mirror.py @@ -0,0 +1,35 @@ +#!/usr/bin/python +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Select closest fuel mirror based on latency measured with ping. +# Failsafe: The us1 mirror + +from subprocess import Popen,PIPE +import re +from operator import itemgetter + +mirrors = [ "us1", "cz1" ] +FNULL = open('/dev/null', 'w') +try: + re_avg = re.compile(r'.* = [^/]*/([^/]*).*') + + pingtime = {} + for mirror in mirrors: + fqdn = "mirror.seed-"+mirror+".fuel-infra.org" + pingtime[fqdn] = 0 + pipe = Popen("ping -c 3 " + fqdn + " | tail -1",shell = True, stdout=PIPE, stderr=FNULL) + avg = pipe.communicate()[0] + pipe.stdout.close() + pingtime[fqdn] = float(re_avg.split(avg)[1]) + + print sorted(pingtime.items(), key=itemgetter(1))[0][0] +except: + print "mirror.seed-"+mirrors[0]+".fuel-infra.org" |