diff options
Diffstat (limited to 'build/f_isoroot/f_repobuild')
| -rw-r--r-- | build/f_isoroot/f_repobuild/Makefile | 91 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/config.mk | 13 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml | 75 | ||||
| -rwxr-xr-x | build/f_isoroot/f_repobuild/opnfv_mirror_conf.py | 57 | ||||
| -rwxr-xr-x | build/f_isoroot/f_repobuild/select_ubuntu_repo.sh | 40 |
5 files changed, 276 insertions, 0 deletions
diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile new file mode 100644 index 000000000..7c6b16304 --- /dev/null +++ b/build/f_isoroot/f_repobuild/Makefile @@ -0,0 +1,91 @@ +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +SHELL := /bin/bash +TOP := $(shell pwd) +TMP_ROOT_DIR := $(shell echo "$(MIRROR_UBUNTU_ROOT)" | cut -d "/" -f2) + +include ../../config.mk +include config.mk + +export MOS_VERSION +export OPENSTACK_VERSION + +.PHONY: all +all: nailgun + +nailgun: + sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml + rm -Rf nailgun + sudo mkdir -p /var/www/nailgun + git clone $(FUEL_MIRROR_REPO) + cd fuel-mirror && git checkout -q $(FUEL_MIRROR_COMMIT) + sudo pip install -U -r ./fuel-mirror/requirements.txt + sudo pip install ./fuel-mirror + sudo pip install ./fuel-mirror/contrib/fuel_mirror + ./opnfv_mirror_conf.py + sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group ubuntu --pattern=ubuntu + sudo chmod -R 755 /var/www/nailgun + cp -Rp /var/www/nailgun . + # On the end we want to have ubuntu repository in mirrors/ubuntu directory + -if [ "$(MIRROR_UBUNTU_ROOT)" != "/ubuntu/" ]; then \ + mkdir -p nailgun/mirrors/ubuntu;\ + mv nailgun/mirrors$(MIRROR_UBUNTU_ROOT)* nailgun/mirrors/ubuntu;\ + [ "$(MIRROR_UBUNTU_ROOT)" != "/" ] && rm -rf nailgun/mirrors/$(TMP_ROOT_DIR);\ + fi + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + +.PHONY: clean +clean: + @rm -rf ../release/opnfv/nailgun nailgun fuel-mirror opnfv-config.yaml ubuntu.yaml + +.PHONY: release +release:nailgun + @rm -Rf ../release/opnfv/nailgun + @mkdir -p ../release/opnfv + @cp -Rp nailgun ../release/opnfv/nailgun + @cp fuel_bootstrap_cli.yaml ../release/opnfv/ + +############################################################################# +# Cache operations - only used when building through ci/build.sh +############################################################################# + +# Create a unique hash to be used for getting and putting cache, based on: +# - Year and week (causing the cache to be rebuilt weekly) +# - The contents of this Makefile +.cacheid: + date +"Repocache %G%V" > .cachedata + sha1sum Makefile >> .cachedata + sha1sum config.mk >> .cachedata + $(CACHETOOL) packages >> .cachedata + echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + cat .cachedata | $(CACHETOOL) getid > .cacheid + +# Clean local data related to caching - called prior to ordinary build +.PHONY: clean-cache +clean-cache: clean + rm -f .cachedata .cacheid + +# Try to download cache - called prior to ordinary build +.PHONY: get-cache +get-cache: .cacheid + @if $(CACHETOOL) check $(shell cat .cacheid); then \ + $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ + else \ + echo "No cache item found for $(shell cat .cacheid)" ;\ + exit 0;\ + fi + +# Store cache if not already stored - called after ordinary build +.PHONY: put-cache +put-cache: .cacheid + @tar cf - nailgun | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_repobuild/config.mk b/build/f_isoroot/f_repobuild/config.mk new file mode 100644 index 000000000..d81732849 --- /dev/null +++ b/build/f_isoroot/f_repobuild/config.mk @@ -0,0 +1,13 @@ +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# mskalski@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +FUEL_MIRROR_REPO?=https://github.com/openstack/fuel-mirror +# Point to the commit where 302 redirects are handled +# https://bugs.launchpad.net/fuel/mitaka/+bug/1593674 +FUEL_MIRROR_COMMIT?=192a3d9f8f993afb12c5108dd9339c6688c23e11 diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml new file mode 100644 index 000000000..fcf4257f1 --- /dev/null +++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml @@ -0,0 +1,75 @@ +############################################################################## +# Copyright (c) 2015,2016 Ericsson AB and others. +# mskalski@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- + root_ssh_authorized_file: /root/.ssh/id_rsa.pub + extend_kopts: "biosdevname=0 net.ifnames=1 debug ignore_loglevel log_buf_len=10M print_fatal_signals=1 LOGLEVEL=8" + ubuntu_release: trusty + extra_dirs: + - /usr/share/fuel_bootstrap_cli/files/trusty + output_dir: /tmp/ + kernel_flavor: linux-image-generic-lts-trusty + packages: + - fuel-agent + - hwloc + - hpsa-dkms + - i40e-dkms + - linux-firmware + - linux-firmware-nonfree + - linux-headers-generic + - live-boot + - live-boot-initramfs-tools + - mc + - mcollective + - msmtp-mta + - multipath-tools + - multipath-tools-boot + - nailgun-agent + - nailgun-mcagents + - network-checker + - ntp + - openssh-client + - openssh-server + - squashfs-tools + - ubuntu-minimal + - vim + - wget + - xz-utils + bootstrap_images_dir: /var/www/nailgun/bootstraps + active_bootstrap_symlink: /var/www/nailgun/bootstraps/active_bootstrap + flavor: ubuntu + http_proxy: "" + https_proxy: "" + repos: + - name: ubuntu + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty + type: deb + - name: ubuntu-updates + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty-updates + type: deb + - name: ubuntu-security + section: "main universe multiverse" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: + suite: trusty-security + type: deb + - name: mos + section: "main restricted" + uri: "http://127.0.0.1:8080/ubuntu/x86_64" + priority: 1050 + suite: mos9.0 + type: deb + skip_default_img_build: false + direct_repo_addresses: + - "127.0.0.1" diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py new file mode 100755 index 000000000..1c7eb591a --- /dev/null +++ b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# mskalski@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import os +import yaml + +current_snapshot = os.environ["LATEST_TARGET_UBUNTU"] +mos_version = os.environ['MOS_VERSION'] +openstack_version = os.environ['OPENSTACK_VERSION'] +mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU'] +mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT'] +mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL'] +if os.environ.get('BUILD_FUEL_PLUGINS'): + plugins = os.environ['BUILD_FUEL_PLUGINS'] +else: + plugins = os.environ['PLUGINS'] + + +configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read() +conf = yaml.load(configuration_file) +conf['pattern_dir'] = '.' +conf['openstack_version'] = openstack_version +conf['mos_version'] = mos_version + +with open('opnfv-config.yaml', 'w') as outfile: + outfile.write( yaml.dump(conf, default_flow_style=False) ) + +pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read() +pattern = yaml.load(pattern_file) +pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) +pattern['ubuntu_baseurl'] = mirror_ubuntu +for group in pattern['groups']['mos']: + group['uri'] = pattern['mos_baseurl'] +for group in pattern['groups']['ubuntu']: + group['uri'] = pattern['ubuntu_baseurl'] + +for plugin in plugins.split(): + path = "../{}/packages.yaml".format(plugin) + if os.path.isfile(path): + f = open(path).read() + plugin_yaml = yaml.load(f) + plugin_set = set(plugin_yaml['packages']) + main_set = set(pattern['packages']) + new_packages = plugin_set - main_set + print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages)) + pattern['packages'] = pattern['packages'] + list(new_packages) + +pattern['requirements']['ubuntu'] = pattern['packages'] + +with open('ubuntu.yaml', 'w') as outfile: + outfile.write( yaml.safe_dump(pattern, default_flow_style=False) ) diff --git a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh new file mode 100755 index 000000000..20b3485a1 --- /dev/null +++ b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh @@ -0,0 +1,40 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015,2016 Ericsson AB and others. +# mskalski@mirantis.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Try to choose close ubuntu mirror from mirrors.txt, but "whitewash" this +# against the full repo list to removed mirrors not up-to-date. + +# Some Ubuntu mirrors seem less reliable for this type of mirroring - +# as they are discoved they can be added to the blacklist below in order +# for them not to be considered. +BLACKLIST="mirror.clibre.uqam.ca" + +#NOTE: For now the mirror selection is disabled due to issues not yet +# understood/resolved. +#for url in $((curl -s https://launchpad.net/ubuntu/+archivemirrors | \ +# grep -P -B8 "statusUP|statusSIX" | \ +# grep -o -P "(f|ht)tp.*\"" | \ +# sed 's/"$//' | sort | uniq; \ +# curl -s http://mirrors.ubuntu.com/mirrors.txt | sort | uniq) | \ +# sort | uniq -d) +#do +# host=$(echo $url | cut -d'/' -f3) +# echo ${BLACKLIST} | grep -q ${host} && continue +# if curl -s -o /dev/null --head --fail "$url"; then +# echo $url +# exit 0 +# else +# continue +# fi +#done + +# If no suitable local mirror can be found, +# the default archive is returned instead. +echo "http://archive.ubuntu.com/ubuntu/" |