diff options
Diffstat (limited to 'build')
| -rw-r--r-- | build/config.mk | 4 | ||||
| -rw-r--r-- | build/f_isoroot/Makefile | 8 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/.gitignore | 4 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/Makefile | 43 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/config.mk | 23 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml | 41 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/opnfv_config.yaml | 166 | ||||
| -rwxr-xr-x | build/f_isoroot/f_repobuild/opnfv_mirror_conf.py | 57 | ||||
| -rwxr-xr-x | build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py | 279 | ||||
| -rwxr-xr-x | build/f_isoroot/f_repobuild/select_ubuntu_repo.sh | 41 | ||||
| -rw-r--r-- | build/install/apt-ftparchive-udeb.conf | 4 | ||||
| -rwxr-xr-x | build/install/install.sh | 14 | ||||
| -rwxr-xr-x | build/install/uninstall.sh | 16 |
13 files changed, 557 insertions, 143 deletions
diff --git a/build/config.mk b/build/config.mk index 56317cb78..5448c5eb4 100644 --- a/build/config.mk +++ b/build/config.mk @@ -17,6 +17,10 @@ FUEL_MAIN_TAG = master MOS_VERSION = 10.0 OPENSTACK_VERSION = newton-10.0 +# List of space-separated Ubuntu architectures supported with current build +# Format: same as `dpkg-architecture -qDEB_HOST_ARCH` +# NOTE: Currently only amd64 is supported by Fuel@OPNFV. Armband adds arm64. +export UBUNTU_ARCH ?= amd64 ############################################################################## # Fuel components pinning / remote tracking; use submodules from f_repos diff --git a/build/f_isoroot/Makefile b/build/f_isoroot/Makefile index 9feddba5b..617e36515 100644 --- a/build/f_isoroot/Makefile +++ b/build/f_isoroot/Makefile @@ -17,16 +17,12 @@ NON_10-0_REBASED_PLUGINS = f_onosfwpluginbuild f_ovsnfv-dpdk-pluginbuild f_vsper # If the BUILD_FUEL_PLUGINS environment variable is set, only build the plugins # indicated therein. -# Temporarilu disabling f_repobuild that currently have some issues on -# Fuel Newton. ifdef BUILD_FUEL_PLUGINS export BUILD_FUEL_PLUGINS -#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS) -SUBDIRS = f_kscfg f_bootstrap f_isolinux $(BUILD_FUEL_PLUGINS) +SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS) $(warning Overriding plugin build selection to $(BUILD_FUEL_PLUGINS)) else -#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS) -SUBDIRS = f_kscfg f_bootstrap f_isolinux $(PLUGINS) +SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS) endif SUBCLEAN = $(addsuffix .clean,$(SUBDIRS)) diff --git a/build/f_isoroot/f_repobuild/.gitignore b/build/f_isoroot/f_repobuild/.gitignore new file mode 100644 index 000000000..09baca85d --- /dev/null +++ b/build/f_isoroot/f_repobuild/.gitignore @@ -0,0 +1,4 @@ +packetary +fuel-web +nailgun +opnfv_config diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile index 8beb8824f..891712b9a 100644 --- a/build/f_isoroot/f_repobuild/Makefile +++ b/build/f_isoroot/f_repobuild/Makefile @@ -1,7 +1,8 @@ ############################################################################## -# Copyright (c) 2015 Ericsson AB and others. +# Copyright (c) 2015,2016 Ericsson AB, Enea AB and others. # stefan.k.berg@ericsson.com # jonas.bjurel@ericsson.com +# Alexandru.Avadanii@enea.com # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -10,41 +11,37 @@ SHELL := /bin/bash TOP := $(shell pwd) -TMP_ROOT_DIR := $(shell echo "$(MIRROR_UBUNTU_ROOT)" | cut -d "/" -f2) include ../../config.mk +include config.mk export MOS_VERSION -export OPENSTACK_VERSION +export MIRROR_UBUNTU_OPNFV_PATH:=$(TOP)/nailgun/mirrors/ubuntu .PHONY: all all: nailgun nailgun: - sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml - rm -Rf nailgun - sudo mkdir -p /var/www/nailgun - ln -sf ${F_SUBMOD_DIR}/fuel-mirror fuel-mirror - sudo pip install -U -r ./fuel-mirror/requirements.txt - sudo pip install ./fuel-mirror - sudo pip install ./fuel-mirror/contrib/fuel_mirror - ./opnfv_mirror_conf.py - sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group ubuntu --pattern=ubuntu - sudo chmod -R 755 /var/www/nailgun - cp -Rp /var/www/nailgun . - # In the end we want to have ubuntu repository in mirrors/ubuntu directory - -if [ "$(MIRROR_UBUNTU_ROOT)" != "/ubuntu/" ]; then \ - mkdir -p nailgun/mirrors/ubuntu;\ - mv nailgun/mirrors$(MIRROR_UBUNTU_ROOT)* nailgun/mirrors/ubuntu;\ - [ "$(MIRROR_UBUNTU_ROOT)" != "/" ] && rm -rf nailgun/mirrors/$(TMP_ROOT_DIR);\ + sudo apt-get install -y createrepo git libxml2-dev libxslt1-dev \ + python-dev zlib1g-dev + rm -Rf nailgun packetary opnfv_config && mkdir opnfv_config + # We will analyze fuel-web's fixture files for package lists + ln -sf ${F_SUBMOD_DIR}/fuel-web fuel-web + git clone --quiet $(PACKETARY_REPO) + if [ -n $(PACKETARY_COMMIT) ]; then \ + git -C packetary checkout $(PACKETARY_COMMIT); \ fi + sudo pip install -U -r ./packetary/requirements.txt + sudo pip install -U ./packetary + # Handle config and mirror build in one place + ./opnfv_mirror_ubuntu.py # Store artifact in cache straight away if caching is enabled # (no .cacheid will be present unless this is a cached build) test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 .PHONY: clean clean: - @rm -rf ../release/opnfv/nailgun nailgun fuel-mirror opnfv-config.yaml ubuntu.yaml + @rm -rf ../release/opnfv/nailgun nailgun packetary fuel-web opnfv_config .PHONY: release release:nailgun @@ -59,10 +56,12 @@ release:nailgun # Create a unique hash to be used for getting and putting cache, based on: # - Year and week (causing the cache to be rebuilt weekly) -# - The contents of this Makefile +# - The contents of this Makefile + all sh,mk,py,yaml files in CWD +# - repo packages fingerprint +# - repo arch list .cacheid: date +"Repocache %G%V" > .cachedata - sha1sum Makefile >> .cachedata + sha1sum Makefile *.{sh,mk,py,yaml} >> .cachedata $(CACHETOOL) packages >> .cachedata echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata cat .cachedata | $(CACHETOOL) getid > .cacheid diff --git a/build/f_isoroot/f_repobuild/config.mk b/build/f_isoroot/f_repobuild/config.mk new file mode 100644 index 000000000..986ea9e8e --- /dev/null +++ b/build/f_isoroot/f_repobuild/config.mk @@ -0,0 +1,23 @@ +############################################################################## +# Copyright (c) 2016 Ericsson AB, Enea AB and others. +# stefan.k.berg@ericsson.com +# jonas.bjurel@ericsson.com +# Alexandru.Avadanii@enea.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Use a recent master commit, since tags/branches are not yet mature +export PACKETARY_REPO?=https://github.com/openstack/packetary +export PACKETARY_COMMIT?=c3c2069e2ec46b35eb342386efbf366bb6340027 + +# arm64 Ubuntu mirror is separated from archive.ubuntu.com +export MIRROR_UBUNTU_URL_arm64=http://ports.ubuntu.com/ubuntu-ports/ +export MIRROR_UBUNTU_ROOT_arm64=ubuntu-ports + +# Merge all local mirror repo components/section into single "main" +# NOTE: When changing this, make sure to also update all consumer config, like: +# - fuel_bootstrap_cli.yaml +export MIRROR_UBUNTU_MERGE=true diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml index fcf4257f1..81ca6ebb0 100644 --- a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml +++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml @@ -8,19 +8,18 @@ ############################################################################## --- root_ssh_authorized_file: /root/.ssh/id_rsa.pub - extend_kopts: "biosdevname=0 net.ifnames=1 debug ignore_loglevel log_buf_len=10M print_fatal_signals=1 LOGLEVEL=8" - ubuntu_release: trusty + extend_kopts: "biosdevname=0 net.ifnames=1" + ubuntu_release: xenial extra_dirs: - - /usr/share/fuel_bootstrap_cli/files/trusty + - /usr/share/fuel_bootstrap_cli/files/xenial output_dir: /tmp/ - kernel_flavor: linux-image-generic-lts-trusty + kernel_flavor: linux-image-generic-lts-xenial packages: + - daemonize - fuel-agent - hwloc - - hpsa-dkms - i40e-dkms - linux-firmware - - linux-firmware-nonfree - linux-headers-generic - live-boot - live-boot-initramfs-tools @@ -33,13 +32,23 @@ - nailgun-mcagents - network-checker - ntp + - ntpdate - openssh-client - openssh-server + - puppet - squashfs-tools - ubuntu-minimal - vim - wget - xz-utils + # NOTE(el): Packages required for new generation + # network checker to be run without an access + # to repositories. + - sysfsutils + - bridge-utils + - ifenslave + - irqbalance + - iputils-arping bootstrap_images_dir: /var/www/nailgun/bootstraps active_bootstrap_symlink: /var/www/nailgun/bootstraps/active_bootstrap flavor: ubuntu @@ -47,29 +56,19 @@ https_proxy: "" repos: - name: ubuntu - section: "main universe multiverse" + section: "main" uri: "http://127.0.0.1:8080/mirrors/ubuntu" priority: - suite: trusty - type: deb - - name: ubuntu-updates - section: "main universe multiverse" - uri: "http://127.0.0.1:8080/mirrors/ubuntu" - priority: - suite: trusty-updates - type: deb - - name: ubuntu-security - section: "main universe multiverse" - uri: "http://127.0.0.1:8080/mirrors/ubuntu" - priority: - suite: trusty-security + suite: xenial type: deb - name: mos section: "main restricted" uri: "http://127.0.0.1:8080/ubuntu/x86_64" priority: 1050 - suite: mos9.0 + suite: mos10.0 type: deb skip_default_img_build: false direct_repo_addresses: - "127.0.0.1" + # User can provide default hashed root password for bootstrap image + # hashed_root_password: "$6$IInX3Cqo$5xytL1VZbZTusOewFnG6couuF0Ia61yS3rbC6P5YbZP2TYclwHqMq9e3Tg8rvQxhxSlBXP1DZhdUamxdOBXK0." diff --git a/build/f_isoroot/f_repobuild/opnfv_config.yaml b/build/f_isoroot/f_repobuild/opnfv_config.yaml new file mode 100644 index 000000000..b8d344196 --- /dev/null +++ b/build/f_isoroot/f_repobuild/opnfv_config.yaml @@ -0,0 +1,166 @@ +############################################################################## +# Copyright (c) 2016 Enea AB and others. +# Alexandru.Avadanii@enea.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# REPO definitions + +# Based on old fuel-mirror data [1], adapted for OPNFV and packetary. +# NOTE: 'uri' field will be added by opnfv_mirror_ubuntu.py from ENV. +# [1] https://github.com/openstack/fuel-mirror/blob/stable/mitaka/ +# contrib/fuel_mirror/data/ubuntu.yaml + +# Main is a required parameter which defines what repository will be used +# for images creation and that mirror should contain all packages for minimal +# system creation. +groups: + ubuntu: + - name: "ubuntu" + main: true + suite: "xenial" + section: + - "main" + - "multiverse" + - "restricted" + - "universe" + type: "deb" + priority: null + + - name: "ubuntu-updates" + suite: "xenial-updates" + section: + - "main" + - "multiverse" + - "restricted" + - "universe" + type: "deb" + priority: null + + - name: "ubuntu-security" + suite: "xenial-security" + section: + - "main" + - "multiverse" + - "restricted" + - "universe" + type: "deb" + priority: null + + mos: + - name: "mos" + suite: "mos$mos_version" + section: + - "main" + - "restricted" + type: "deb" + priority: 1000 + + - name: "mos-updates" + suite: "mos$mos_version-updates" + section: + - "main" + - "restricted" + type: "deb" + priority: 1000 + + - name: "mos-security" + suite: "mos$mos_version-security" + section: + - "main" + - "restricted" + type: "deb" + priority: 1000 + + - name: "mos-holdback" + suite: "mos$mos_version-holdback" + section: + - "main" + - "restricted" + type: "deb" + priority: 1000 + +# PACKAGES + +# This section lists packages that should be present in the local Ubuntu mirror, +# but are not direct dependencies of any other packages from MOS or Ubuntu. +# e.g.: additional kernels, bootloaders etc. + +packages: + - name: "vgabios" +# Packages are required to build bootstrap images for a system. +# The mirror should contiain such packages in addition to local mirror. + - name: "acpi-support" + - name: "anacron" + - name: "aptitude" + - name: "atop" + - name: "acct" + - name: "bash-completion" + - name: "bc" + - name: "build-essential" + - name: "cloud-init" + - name: "conntrackd" + - name: "cpu-checker" + - name: "cpufrequtils" + - name: "debconf-utils" + - name: "devscripts" + - name: "fping" + - name: "git" + - name: "grub-pc" + - name: "htop" + - name: "hwloc" + - name: "ifenslave" + - name: "iperf" + - name: "iptables-persistent" + - name: "irqbalance" + - name: "language-pack-en" + - name: "libapache2-mod-fastcgi" + - name: "libnss3-tools" + - name: "linux-headers-generic-lts-xenial" + - name: "linux-image-generic-lts-xenial" + - name: "live-boot" + - name: "livecd-rootfs" + - name: "mc" + - name: "memcached" + - name: "monit" + - name: "msmtp-mta" + - name: "multipath-tools" + - name: "multipath-tools-boot" + - name: "nginx" + - name: "ntp" + - name: "openssh-server" + - name: "percona-toolkit" + - name: "percona-xtrabackup" + - name: "pm-utils" + - name: "puppet" + - name: "python-lesscpy" + - name: "python-pip" + - name: "rsyslog-gnutls" + - name: "rsyslog-relp" + - name: "screen" + - name: "squashfs-tools" + - name: "swift-plugin-s3" + - name: "sysfsutils" + - name: "sysstat" + - name: "telnet" + - name: "tmux" + - name: "traceroute" + - name: "ubuntu-standard" + - name: "vim" + - name: "virt-what" + - name: "xinetd" + - name: "xmlstarlet" + - name: "tftpd-hpa" + - name: "syslinux" + +# OPNFV BLACKLIST + +# Packetary's dependency solving mechanism brings in more than one alternative +# For example, "Depends: upstart | systemd-sysv" brings both packages, which +# leads to debootstrap using "upstart" (1st option available in local repo). + +opnfv_blacklist: + - name: "upstart" diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py deleted file mode 100755 index 1c7eb591a..000000000 --- a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# Copyright (c) 2015 Ericsson AB and others. -# mskalski@mirantis.com -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import os -import yaml - -current_snapshot = os.environ["LATEST_TARGET_UBUNTU"] -mos_version = os.environ['MOS_VERSION'] -openstack_version = os.environ['OPENSTACK_VERSION'] -mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU'] -mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT'] -mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL'] -if os.environ.get('BUILD_FUEL_PLUGINS'): - plugins = os.environ['BUILD_FUEL_PLUGINS'] -else: - plugins = os.environ['PLUGINS'] - - -configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read() -conf = yaml.load(configuration_file) -conf['pattern_dir'] = '.' -conf['openstack_version'] = openstack_version -conf['mos_version'] = mos_version - -with open('opnfv-config.yaml', 'w') as outfile: - outfile.write( yaml.dump(conf, default_flow_style=False) ) - -pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read() -pattern = yaml.load(pattern_file) -pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) -pattern['ubuntu_baseurl'] = mirror_ubuntu -for group in pattern['groups']['mos']: - group['uri'] = pattern['mos_baseurl'] -for group in pattern['groups']['ubuntu']: - group['uri'] = pattern['ubuntu_baseurl'] - -for plugin in plugins.split(): - path = "../{}/packages.yaml".format(plugin) - if os.path.isfile(path): - f = open(path).read() - plugin_yaml = yaml.load(f) - plugin_set = set(plugin_yaml['packages']) - main_set = set(pattern['packages']) - new_packages = plugin_set - main_set - print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages)) - pattern['packages'] = pattern['packages'] + list(new_packages) - -pattern['requirements']['ubuntu'] = pattern['packages'] - -with open('ubuntu.yaml', 'w') as outfile: - outfile.write( yaml.safe_dump(pattern, default_flow_style=False) ) diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py new file mode 100755 index 000000000..1a603ee79 --- /dev/null +++ b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python +############################################################################## +# Copyright (c) 2015,2016 Ericsson AB, Mirantis Inc., Enea AB and others. +# mskalski@mirantis.com +# Alexandru.Avadanii@enea.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +"""Build multiarch partial local Ubuntu mirror using packetary""" + +############################################################################## +# Design quirks / workarounds: +# 1. Fuel-agent uses `debootstrap` to build bootstrap and target chroots from +# the local mirror; which only uses the "main" component from the first +# repository, i.e. does not include "updates"/"security". +# In order to fullfill all debootstrap dependencies in "main" repo, we will +# do an extra packetary run using a reduced scope: +# - only "main" component of the first mirror; +# - reduced package dependency list (without MOS/OPNFV plugin deps). +# 2. If repo structure is not mandatory to be in sync with official mirrors, +# we can mitigate the issue by "merging" all repo-components into a single +# "main". +############################################################################## +# Mirror build steps (for EACH architecture in UBUNTU_ARCH): +# 1. Collect bootstrap package deps from <fuel_bootstrap_cli.yaml>; +# 2. Collect all fixture release packages from fuel-web's <openstack.yaml>; +# 3. Parse new "opnfv_config.yaml" list of packages (from old fuel-mirror); +# 4. Inherit enviroment variable(s) for mirror URLs, paths etc. +# - Allow arch-specific overrides for each env var; +# 5. Mirror config is defined based on common config + OPNFV overrides; +# - Convert old configuration format to packetary style where needed; +# 6. Package lists are defined based on common config + OPNFV deps; +# - Keep track of "main" packages separately, required by debootstrap; +# 7. Clone/update all mirror components; +# 8. IF mirror merging is disabled: +# - Clone/update "main" mirror component (fix missing debootstrap deps); +# 9. IF mirror merging is enabled: +# - Use `dpkg-scanpackages` to filter out old versions of duplicate pkgs; +# - Run `packetary create` on the set of downloaded packages, merging +# them on the fly into a single-component mirror; +############################################################################## + +from copy import deepcopy +import os +import shutil +import sys +import yaml +from contextlib import contextmanager +from cStringIO import StringIO +from packetary.cli.app import main + +@contextmanager +def capture_stdout(output): + """Context manager for capturing stdout""" + stdout = sys.stdout + sys.stdout = output + yield + sys.stdout = stdout + +# FIXME: Find a better approach for eliminating duplicate logs than this +def force_logger_reload(): + """Force logger reload (ugly hack to prevent log duplication)""" + for mod in sys.modules.keys(): + if mod.startswith('logging'): + try: + reload(sys.modules[mod]) + except: + pass + +def get_unres_pkgs(architecture, cfg_mirror): + """Determine missing package dependecies for a mirror defition""" + unresolved_pkgs = list() + packetary_output = StringIO() + with capture_stdout(packetary_output): + main('unresolved -a {0} -r {1} -c name version --sep ;' + .format(_ARCH[architecture], cfg_mirror).split(' ')) + for dep_pkg in packetary_output.getvalue().splitlines(): + if dep_pkg.startswith('#'): + continue + dep = dep_pkg.split(';') + unresolved_pkgs += [{'name': dep[0], 'version': dep[1]}] + force_logger_reload() + return unresolved_pkgs + +def from_legacy_pkglist(legacy_pkglist): + """Package list conversion from `old fuel-mirror` to `packetary` style""" + pkglist = list() + for pkg in legacy_pkglist: + pkglist += [{'name': pkg}] + return pkglist + +def to_legacy_pkglist(pkglist): + """Package list conversion from `packetary` style to `old fuel-mirror`""" + legacy_pkglist = list() + for pkg in pkglist: + legacy_pkglist.append(pkg['name']) + return legacy_pkglist + +def legacy_diff(base_pkglist, new_pkglist, requester, architecture): + """Package list diff (old format)""" + diff_set = set(new_pkglist) + if base_pkglist: + diff_set -= set(base_pkglist) + if diff_set: + print(' * {0} requires new packages for architecture [{1}]: {2}' + .format(requester, architecture, ', '.join(diff_set))) + return list(diff_set) + +def do_local_repo(architecture, cfg_repo, cfg_packages_paths): + """Create single-component local repo (one architecture per call)""" + # Packetary does not use a global config file, so pass old settings here. + main('create -t deb -a {0} --repository {1} --package-files {2}' + ' --ignore-errors-num 2 --retries-num 3 --threads-num 10' + .format(_ARCH[architecture], cfg_repo, cfg_packages_paths).split(' ')) + force_logger_reload() + +def do_partial_mirror(architecture, cfg_mirror, cfg_packages): + """Clone partial local mirror (one architecture per call)""" + # Note: '-d .' is ignored, as each mirror defines its own path. + main('clone -t deb -a {0} -r {1} -R {2} -d .' + ' --ignore-errors-num 2 --retries-num 3 --threads-num 10' + .format(_ARCH[architecture], cfg_mirror, cfg_packages).split(' ')) + force_logger_reload() + +def write_cfg_file(cfg_mirror, data): + """Write configuration (yaml) file (package list / mirror defition)""" + with open(cfg_mirror, 'w') as outfile: + outfile.write(yaml.safe_dump(data, default_flow_style=False)) + +def get_env(env_var, architecture=None): + """Evaluate architecture-specific overrides of env vars""" + if architecture: + env_var_arch = '{0}_{1}'.format(env_var, architecture) + if os.environ.get(env_var_arch): + return os.environ[env_var_arch] + if os.environ.get(env_var): + return os.environ[env_var] + return None + +# Architecture name mapping (dpkg:packetary) for packetary CLI invocation +_ARCH = { + "i386": "i386", + "amd64": "x86_64", + "arm64": "aarch64", +} + +# Arch-indepedent configuration (old fuel-mirror + OPNFV extra packages) +CFG_D = 'opnfv_config' +CFG_OPNFV = 'opnfv_config.yaml' +MOS_VERSION = get_env('MOS_VERSION') +UBUNTU_ARCH = get_env('UBUNTU_ARCH') +MIRROR_UBUNTU_PATH = get_env('MIRROR_UBUNTU_OPNFV_PATH') +MIRROR_UBUNTU_TMP_PATH = '{0}.tmp'.format(MIRROR_UBUNTU_PATH) +MIRROR_UBUNTU_MERGE = get_env('MIRROR_UBUNTU_MERGE') +CFG_MM_UBUNTU = '{0}/ubuntu_mirror_local.yaml'.format(CFG_D) +FUEL_BOOTSTRAP_CLI_FILE = open('fuel_bootstrap_cli.yaml').read() +FUEL_BOOTSTRAP_CLI = yaml.load(FUEL_BOOTSTRAP_CLI_FILE) +FIXTURE_FILE = open('fuel-web/nailgun/nailgun/fixtures/openstack.yaml').read() +FIXTURE = yaml.load(FIXTURE_FILE) +OPNFV_CFG_YAML = open(CFG_OPNFV).read() +OPNFV_CFG = yaml.load(OPNFV_CFG_YAML) + +# Create local partial mirror using packetary, one arch at a time +for arch in UBUNTU_ARCH.split(' '): + # Mirror / Package env vars, arch-overrideable + mos_ubuntu = get_env('MIRROR_MOS_UBUNTU', arch) + mos_ubuntu_root = get_env('MIRROR_MOS_UBUNTU_ROOT', arch) + mirror_ubuntu = get_env('MIRROR_UBUNTU_URL', arch) + plugins = get_env('BUILD_FUEL_PLUGINS', arch) + if plugins is None: + plugins = get_env('PLUGINS', arch) + + # Mirror / Package list configuration files (arch-specific) + cfg_m_mos = '{0}/mos_{1}_mirror.yaml'.format(CFG_D, arch) + cfg_m_ubuntu = '{0}/ubuntu_{1}_mirror.yaml'.format(CFG_D, arch) + cfg_p_ubuntu = '{0}/ubuntu_{1}_packages.yaml'.format(CFG_D, arch) + cfg_m_ubuntu_main = '{0}/ubuntu_{1}_mirror_main.yaml'.format(CFG_D, arch) + cfg_p_ubuntu_main = '{0}/ubuntu_{1}_packages_main.yaml'.format(CFG_D, arch) + + # Mirror config fork before customizing (arch-specific) + arch_mos = 'mos_{0}'.format(arch) + arch_ubuntu = 'ubuntu_{0}'.format(arch) + arch_packages = 'packages_{0}'.format(arch) + OPNFV_CFG['groups'][arch_mos] = deepcopy(OPNFV_CFG['groups']['mos']) + OPNFV_CFG['groups'][arch_ubuntu] = deepcopy(OPNFV_CFG['groups']['ubuntu']) + OPNFV_CFG[arch_packages] = OPNFV_CFG['packages'] + + # Mirror config update & conversion to packetary input + group_main_ubuntu = dict() + for group in OPNFV_CFG['groups'][arch_mos]: + group['uri'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) + group['suite'] = group['suite'].replace('$mos_version', MOS_VERSION) + for group in OPNFV_CFG['groups'][arch_ubuntu]: + group['uri'] = mirror_ubuntu + # FIXME: At `create`, packetary insists on copying all pkgs to dest dir, + # so configure it for another dir, which will replace the orig. + group['path'] = MIRROR_UBUNTU_TMP_PATH + if not group_main_ubuntu and 'main' in group: + group_main_ubuntu = [deepcopy(group)] + group_main_ubuntu[0]['section'] = ['main'] + + # Mirror config dump: MOS (for dep resolution), Ubuntu, Ubuntu[main] + write_cfg_file(cfg_m_mos, OPNFV_CFG['groups'][arch_mos]) + write_cfg_file(cfg_m_ubuntu, OPNFV_CFG['groups'][arch_ubuntu]) + if MIRROR_UBUNTU_MERGE is None: + write_cfg_file(cfg_m_ubuntu_main, group_main_ubuntu) + else: + # FIXME: For multiarch, only one dump would be enough + group_main_ubuntu[0]['origin'] = 'Ubuntu' + group_main_ubuntu[0]['path'] = MIRROR_UBUNTU_PATH + group_main_ubuntu[0]['uri'] = MIRROR_UBUNTU_PATH + write_cfg_file(CFG_MM_UBUNTU, group_main_ubuntu[0]) + + # Collect package dependencies from: + ## 1. fuel_bootstrap_cli.yaml (bootstrap image additional packages) + legacy_unresolved = legacy_diff(None, FUEL_BOOTSTRAP_CLI['packages'] + [ + FUEL_BOOTSTRAP_CLI['kernel_flavor'], + FUEL_BOOTSTRAP_CLI['kernel_flavor'].replace('image', 'headers')], + 'Bootstrap', arch) + ## 2. openstack.yaml FIXTURE definition (default target image packages) + for release in FIXTURE: + editable = release['fields']['attributes_metadata']['editable'] + if 'provision' in editable and 'packages' in editable['provision']: + release_pkgs = editable['provision']['packages']['value'].split() + legacy_unresolved += legacy_diff(legacy_unresolved, release_pkgs, + 'Release {0}'.format(release['fields']['name']), arch) + ## 3. OPNFV additional packages (includes old fuel-mirror ubuntu.yaml pkgs) + unresolved = dict() + unresolved['mandatory'] = 'exact' + unresolved['packages'] = from_legacy_pkglist(legacy_unresolved) + if 'packages' in OPNFV_CFG: + legacy_diff(legacy_unresolved, to_legacy_pkglist(OPNFV_CFG['packages']), + 'OPNFV config', arch) + unresolved['packages'] += OPNFV_CFG['packages'] + + # OPNFV plugins dependency resolution + if plugins: + for plugin in plugins.split(): + path = "../{}/packages.yaml".format(plugin) + if os.path.isfile(path): + f = open(path).read() + plugin_yaml = yaml.load(f) + new_pkgs = legacy_diff( + to_legacy_pkglist(unresolved['packages']), + plugin_yaml['packages'], 'Plugin {0}'.format(plugin), arch) + unresolved['packages'] += from_legacy_pkglist(new_pkgs) + + # Package list (reduced, i.e. no MOS deps, but with OPNFV plugin deps) + if MIRROR_UBUNTU_MERGE is None: + write_cfg_file(cfg_p_ubuntu_main, unresolved) + + # Mirror package list (full, including MOS/OPNFV plugin deps) + unresolved['packages'] += get_unres_pkgs(arch, cfg_m_mos) + write_cfg_file(cfg_p_ubuntu, unresolved) + do_partial_mirror(arch, cfg_m_ubuntu, cfg_p_ubuntu) + if MIRROR_UBUNTU_MERGE is None: + # Ubuntu[main] must be evaluated after Ubuntu + do_partial_mirror(arch, cfg_m_ubuntu_main, cfg_p_ubuntu_main) + +if MIRROR_UBUNTU_MERGE is None: + shutil.move(MIRROR_UBUNTU_TMP_PATH, MIRROR_UBUNTU_PATH) +else: + # Construct single-component mirror from all components + for arch in UBUNTU_ARCH.split(' '): + cfg_pp_ubuntu = '{0}/ubuntu_{1}_packages_paths.yaml'.format(CFG_D, arch) + # OPNFV blacklist + opnfv_blacklist = to_legacy_pkglist(OPNFV_CFG['opnfv_blacklist']) + # FIXME: We need scanpackages to omit older DEBs + # Inspired from http://askubuntu.com/questions/198474/ + os.system('dpkg-scanpackages -a {0} {1} 2>/dev/null | ' + 'grep -e "^Filename:" | sed "s|Filename: |- file://|g" | ' + 'grep -v -E "\/({2})_" > {3}' + .format(arch, MIRROR_UBUNTU_TMP_PATH, + '|'.join(opnfv_blacklist), cfg_pp_ubuntu)) + do_local_repo(arch, CFG_MM_UBUNTU, cfg_pp_ubuntu) + shutil.rmtree(MIRROR_UBUNTU_TMP_PATH) diff --git a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh index 5618eb79b..c8c86db53 100755 --- a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh +++ b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh @@ -1,5 +1,6 @@ #!/bin/bash +UBUNTU_DISTRO="xenial" BLACKLIST="http://mirrors.se.eu.kernel.org/ubuntu/" #BLACKLIST+=" http://foo.bar" @@ -28,26 +29,26 @@ blacklisted () { check_mirror () { mirror=$1 status=0 - for packdir in dists/trusty-updates/main/binary-amd64 \ - dists/trusty-updates/restricted/binary-amd64 \ - dists/trusty-updates/universe/binary-amd64 \ - dists/trusty-updates/multiverse/binary-amd64 \ - dists/trusty-security/main/binary-amd64 \ - dists/trusty-security/restricted/binary-amd64 \ - dists/trusty-security/universe/binary-amd64 \ - dists/trusty-security/multiverse/binary-amd64 \ - dists/trusty-proposed/main/binary-amd64 \ - dists/trusty-proposed/restricted/binary-amd64 \ - dists/trusty-proposed/universe/binary-amd64 \ - dists/trusty-proposed/multiverse/binary-amd64 \ - dists/trusty/main/binary-amd64 \ - dists/trusty/restricted/binary-amd64 \ - dists/trusty/universe/binary-amd64 \ - dists/trusty/multiverse/binary-amd64 \ - dists/trusty-backports/main/binary-amd64 \ - dists/trusty-backports/restricted/binary-amd64 \ - dists/trusty-backports/universe/binary-amd64 \ - dists/trusty-backports/multiverse/binary-amd64 + for packdir in dists/${UBUNTU_DISTRO}-updates/main/binary-amd64 \ + dists/${UBUNTU_DISTRO}-updates/restricted/binary-amd64 \ + dists/${UBUNTU_DISTRO}-updates/universe/binary-amd64 \ + dists/${UBUNTU_DISTRO}-updates/multiverse/binary-amd64 \ + dists/${UBUNTU_DISTRO}-security/main/binary-amd64 \ + dists/${UBUNTU_DISTRO}-security/restricted/binary-amd64 \ + dists/${UBUNTU_DISTRO}-security/universe/binary-amd64 \ + dists/${UBUNTU_DISTRO}-security/multiverse/binary-amd64 \ + dists/${UBUNTU_DISTRO}-proposed/main/binary-amd64 \ + dists/${UBUNTU_DISTRO}-proposed/restricted/binary-amd64 \ + dists/${UBUNTU_DISTRO}-proposed/universe/binary-amd64 \ + dists/${UBUNTU_DISTRO}-proposed/multiverse/binary-amd64 \ + dists/${UBUNTU_DISTRO}/main/binary-amd64 \ + dists/${UBUNTU_DISTRO}/restricted/binary-amd64 \ + dists/${UBUNTU_DISTRO}/universe/binary-amd64 \ + dists/${UBUNTU_DISTRO}/multiverse/binary-amd64 \ + dists/${UBUNTU_DISTRO}-backports/main/binary-amd64 \ + dists/${UBUNTU_DISTRO}-backports/restricted/binary-amd64 \ + dists/${UBUNTU_DISTRO}-backports/universe/binary-amd64 \ + dists/${UBUNTU_DISTRO}-backports/multiverse/binary-amd64 do for packfile in Release Packages.gz do diff --git a/build/install/apt-ftparchive-udeb.conf b/build/install/apt-ftparchive-udeb.conf index 3b5b239a6..0cabe9425 100644 --- a/build/install/apt-ftparchive-udeb.conf +++ b/build/install/apt-ftparchive-udeb.conf @@ -17,8 +17,8 @@ TreeDefault { }; BinDirectory "pool/debian-installer" { - Packages "dists/trusty/main/debian-installer/binary-amd64/Packages"; - BinOverride "./indices/override.trusty.main.debian-installer"; + Packages "dists/xenial/main/debian-installer/binary-amd64/Packages"; + BinOverride "./indices/override.xenial.main.debian-installer"; }; Default { diff --git a/build/install/install.sh b/build/install/install.sh index 198f26cb5..b2f6ccc3c 100755 --- a/build/install/install.sh +++ b/build/install/install.sh @@ -122,17 +122,17 @@ prep_make_live() { ssh-copy-id root@$FUELHOST sshfs root@1${FUELHOST}:/ $TMP_HOSTMOUNT - if [ -f $REPO/dists/trusty/main/binary-amd64/Packages.backup ]; then + if [ -f $REPO/dists/xenial/main/binary-amd64/Packages.backup ]; then echo "Error - found backup file for Packages!" exit 1 fi - if [ -f $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup ]; then + if [ -f $REPO/dists/xenial/main/binary-amd64/Packages.gz.backup ]; then echo "Error - found backup file for Packages.gz!" exit 1 fi - if [ -f $REPO/dists/trusty/Release.backup ]; then + if [ -f $REPO/dists/xenial/Release.backup ]; then echo "Error - found backup file for Release!" exit 1 fi @@ -142,9 +142,9 @@ prep_make_live() { exit 1 fi - cp $REPO/dists/trusty/main/binary-amd64/Packages $REPO/dists/trusty/main/binary-amd64/Packages.backup - cp $REPO/dists/trusty/main/binary-amd64/Packages.gz $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup - cp $REPO/dists/trusty/Release $REPO/dists/trusty/Release.backup + cp $REPO/dists/xenial/main/binary-amd64/Packages $REPO/dists/xenial/main/binary-amd64/Packages.backup + cp $REPO/dists/xenial/main/binary-amd64/Packages.gz $REPO/dists/xenial/main/binary-amd64/Packages.gz.backup + cp $REPO/dists/xenial/Release $REPO/dists/xenial/Release.backup cp -Rvp $DEST/etc/puppet $DEST/etc/puppet.backup } @@ -443,7 +443,7 @@ copy_packages() { echo Not running apt-ftparchive generate "${APT_UDEB_CONF}" # Fuel also needs this index file - # cat dists/trusty/main/binary-amd64/Packages | \ + # cat dists/xenial/main/binary-amd64/Packages | \ # awk '/^Package:/{pkg=$2} # /^Version:/{print pkg ": \"" $2 "\""}' > ubuntu-versions.yaml # cp ubuntu-versions.yaml $DEST diff --git a/build/install/uninstall.sh b/build/install/uninstall.sh index a9e74bc39..5252c3093 100755 --- a/build/install/uninstall.sh +++ b/build/install/uninstall.sh @@ -31,17 +31,17 @@ DEST=$MOUNT REPO=$DEST/var/www/nailgun/ubuntu/fuelweb/x86_64 cd $REPO -if [ ! -f $REPO/dists/trusty/main/binary-amd64/Packages.backup ]; then +if [ ! -f $REPO/dists/xenial/main/binary-amd64/Packages.backup ]; then echo "Error - didn't find backup file for Packages!" exit 1 fi -if [ ! -f $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup ]; then +if [ ! -f $REPO/dists/xenial/main/binary-amd64/Packages.gz.backup ]; then echo "Error - didn't find backup file for Packages.gz!" exit 1 fi -if [ ! -f $REPO/dists/trusty/Release.backup ]; then +if [ ! -f $REPO/dists/xenial/Release.backup ]; then echo "Error - didn't find backup file for Release!" exit 1 fi @@ -71,9 +71,9 @@ cd $REPO echo "Restoring backups of datafiles" -rm -f $REPO/dists/trusty/main/binary-amd64/Packages $REPO/dists/trusty/main/binary-amd64/Packages.gz -rm -f $REPO/dists/trusty/Release $DEST/etc/puppet/manifests/site.pp -mv $REPO/dists/trusty/main/binary-amd64/Packages.backup $REPO/dists/trusty/main/binary-amd64/Packages -mv $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup $REPO/dists/trusty/main/binary-amd64/Packages.gz -mv $REPO/dists/trusty/Release.backup $REPO/dists/trusty/Release +rm -f $REPO/dists/xenial/main/binary-amd64/Packages $REPO/dists/xenial/main/binary-amd64/Packages.gz +rm -f $REPO/dists/xenial/Release $DEST/etc/puppet/manifests/site.pp +mv $REPO/dists/xenial/main/binary-amd64/Packages.backup $REPO/dists/xenial/main/binary-amd64/Packages +mv $REPO/dists/xenial/main/binary-amd64/Packages.gz.backup $REPO/dists/xenial/main/binary-amd64/Packages.gz +mv $REPO/dists/xenial/Release.backup $REPO/dists/xenial/Release mv $DEST/etc/puppet/manifests/site.pp.backup $DEST/etc/puppet/manifests/site.pp |