aboutsummaryrefslogtreecommitdiffstats
path: root/build/f_isoroot
diff options
context:
space:
mode:
Diffstat (limited to 'build/f_isoroot')
-rw-r--r--build/f_isoroot/Makefile8
-rw-r--r--build/f_isoroot/f_repobuild/.gitignore4
-rw-r--r--build/f_isoroot/f_repobuild/Makefile43
-rw-r--r--build/f_isoroot/f_repobuild/config.mk23
-rw-r--r--build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml41
-rw-r--r--build/f_isoroot/f_repobuild/opnfv_config.yaml166
-rwxr-xr-xbuild/f_isoroot/f_repobuild/opnfv_mirror_conf.py57
-rwxr-xr-xbuild/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py279
-rwxr-xr-xbuild/f_isoroot/f_repobuild/select_ubuntu_repo.sh41
9 files changed, 536 insertions, 126 deletions
diff --git a/build/f_isoroot/Makefile b/build/f_isoroot/Makefile
index c21546ad3..220bc9dbe 100644
--- a/build/f_isoroot/Makefile
+++ b/build/f_isoroot/Makefile
@@ -17,16 +17,12 @@ NON_10-0_REBASED_PLUGINS = f_odlpluginbuild f_onosfwpluginbuild f_ovsnfv-dpdk-pl
# If the BUILD_FUEL_PLUGINS environment variable is set, only build the plugins
# indicated therein.
-# Temporarilu disabling f_repobuild that currently have some issues on
-# Fuel Newton.
ifdef BUILD_FUEL_PLUGINS
export BUILD_FUEL_PLUGINS
-#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS)
-SUBDIRS = f_kscfg f_bootstrap f_isolinux $(BUILD_FUEL_PLUGINS)
+SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS)
$(warning Overriding plugin build selection to $(BUILD_FUEL_PLUGINS))
else
-#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS)
-SUBDIRS = f_kscfg f_bootstrap f_isolinux $(PLUGINS)
+SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS)
endif
SUBCLEAN = $(addsuffix .clean,$(SUBDIRS))
diff --git a/build/f_isoroot/f_repobuild/.gitignore b/build/f_isoroot/f_repobuild/.gitignore
new file mode 100644
index 000000000..09baca85d
--- /dev/null
+++ b/build/f_isoroot/f_repobuild/.gitignore
@@ -0,0 +1,4 @@
+packetary
+fuel-web
+nailgun
+opnfv_config
diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile
index 8beb8824f..891712b9a 100644
--- a/build/f_isoroot/f_repobuild/Makefile
+++ b/build/f_isoroot/f_repobuild/Makefile
@@ -1,7 +1,8 @@
##############################################################################
-# Copyright (c) 2015 Ericsson AB and others.
+# Copyright (c) 2015,2016 Ericsson AB, Enea AB and others.
# stefan.k.berg@ericsson.com
# jonas.bjurel@ericsson.com
+# Alexandru.Avadanii@enea.com
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
@@ -10,41 +11,37 @@
SHELL := /bin/bash
TOP := $(shell pwd)
-TMP_ROOT_DIR := $(shell echo "$(MIRROR_UBUNTU_ROOT)" | cut -d "/" -f2)
include ../../config.mk
+include config.mk
export MOS_VERSION
-export OPENSTACK_VERSION
+export MIRROR_UBUNTU_OPNFV_PATH:=$(TOP)/nailgun/mirrors/ubuntu
.PHONY: all
all: nailgun
nailgun:
- sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml
- rm -Rf nailgun
- sudo mkdir -p /var/www/nailgun
- ln -sf ${F_SUBMOD_DIR}/fuel-mirror fuel-mirror
- sudo pip install -U -r ./fuel-mirror/requirements.txt
- sudo pip install ./fuel-mirror
- sudo pip install ./fuel-mirror/contrib/fuel_mirror
- ./opnfv_mirror_conf.py
- sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group ubuntu --pattern=ubuntu
- sudo chmod -R 755 /var/www/nailgun
- cp -Rp /var/www/nailgun .
- # In the end we want to have ubuntu repository in mirrors/ubuntu directory
- -if [ "$(MIRROR_UBUNTU_ROOT)" != "/ubuntu/" ]; then \
- mkdir -p nailgun/mirrors/ubuntu;\
- mv nailgun/mirrors$(MIRROR_UBUNTU_ROOT)* nailgun/mirrors/ubuntu;\
- [ "$(MIRROR_UBUNTU_ROOT)" != "/" ] && rm -rf nailgun/mirrors/$(TMP_ROOT_DIR);\
+ sudo apt-get install -y createrepo git libxml2-dev libxslt1-dev \
+ python-dev zlib1g-dev
+ rm -Rf nailgun packetary opnfv_config && mkdir opnfv_config
+ # We will analyze fuel-web's fixture files for package lists
+ ln -sf ${F_SUBMOD_DIR}/fuel-web fuel-web
+ git clone --quiet $(PACKETARY_REPO)
+ if [ -n $(PACKETARY_COMMIT) ]; then \
+ git -C packetary checkout $(PACKETARY_COMMIT); \
fi
+ sudo pip install -U -r ./packetary/requirements.txt
+ sudo pip install -U ./packetary
+ # Handle config and mirror build in one place
+ ./opnfv_mirror_ubuntu.py
# Store artifact in cache straight away if caching is enabled
# (no .cacheid will be present unless this is a cached build)
test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0
.PHONY: clean
clean:
- @rm -rf ../release/opnfv/nailgun nailgun fuel-mirror opnfv-config.yaml ubuntu.yaml
+ @rm -rf ../release/opnfv/nailgun nailgun packetary fuel-web opnfv_config
.PHONY: release
release:nailgun
@@ -59,10 +56,12 @@ release:nailgun
# Create a unique hash to be used for getting and putting cache, based on:
# - Year and week (causing the cache to be rebuilt weekly)
-# - The contents of this Makefile
+# - The contents of this Makefile + all sh,mk,py,yaml files in CWD
+# - repo packages fingerprint
+# - repo arch list
.cacheid:
date +"Repocache %G%V" > .cachedata
- sha1sum Makefile >> .cachedata
+ sha1sum Makefile *.{sh,mk,py,yaml} >> .cachedata
$(CACHETOOL) packages >> .cachedata
echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata
cat .cachedata | $(CACHETOOL) getid > .cacheid
diff --git a/build/f_isoroot/f_repobuild/config.mk b/build/f_isoroot/f_repobuild/config.mk
new file mode 100644
index 000000000..986ea9e8e
--- /dev/null
+++ b/build/f_isoroot/f_repobuild/config.mk
@@ -0,0 +1,23 @@
+##############################################################################
+# Copyright (c) 2016 Ericsson AB, Enea AB and others.
+# stefan.k.berg@ericsson.com
+# jonas.bjurel@ericsson.com
+# Alexandru.Avadanii@enea.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# Use a recent master commit, since tags/branches are not yet mature
+export PACKETARY_REPO?=https://github.com/openstack/packetary
+export PACKETARY_COMMIT?=c3c2069e2ec46b35eb342386efbf366bb6340027
+
+# arm64 Ubuntu mirror is separated from archive.ubuntu.com
+export MIRROR_UBUNTU_URL_arm64=http://ports.ubuntu.com/ubuntu-ports/
+export MIRROR_UBUNTU_ROOT_arm64=ubuntu-ports
+
+# Merge all local mirror repo components/section into single "main"
+# NOTE: When changing this, make sure to also update all consumer config, like:
+# - fuel_bootstrap_cli.yaml
+export MIRROR_UBUNTU_MERGE=true
diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
index fcf4257f1..81ca6ebb0 100644
--- a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
+++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
@@ -8,19 +8,18 @@
##############################################################################
---
root_ssh_authorized_file: /root/.ssh/id_rsa.pub
- extend_kopts: "biosdevname=0 net.ifnames=1 debug ignore_loglevel log_buf_len=10M print_fatal_signals=1 LOGLEVEL=8"
- ubuntu_release: trusty
+ extend_kopts: "biosdevname=0 net.ifnames=1"
+ ubuntu_release: xenial
extra_dirs:
- - /usr/share/fuel_bootstrap_cli/files/trusty
+ - /usr/share/fuel_bootstrap_cli/files/xenial
output_dir: /tmp/
- kernel_flavor: linux-image-generic-lts-trusty
+ kernel_flavor: linux-image-generic-lts-xenial
packages:
+ - daemonize
- fuel-agent
- hwloc
- - hpsa-dkms
- i40e-dkms
- linux-firmware
- - linux-firmware-nonfree
- linux-headers-generic
- live-boot
- live-boot-initramfs-tools
@@ -33,13 +32,23 @@
- nailgun-mcagents
- network-checker
- ntp
+ - ntpdate
- openssh-client
- openssh-server
+ - puppet
- squashfs-tools
- ubuntu-minimal
- vim
- wget
- xz-utils
+ # NOTE(el): Packages required for new generation
+ # network checker to be run without an access
+ # to repositories.
+ - sysfsutils
+ - bridge-utils
+ - ifenslave
+ - irqbalance
+ - iputils-arping
bootstrap_images_dir: /var/www/nailgun/bootstraps
active_bootstrap_symlink: /var/www/nailgun/bootstraps/active_bootstrap
flavor: ubuntu
@@ -47,29 +56,19 @@
https_proxy: ""
repos:
- name: ubuntu
- section: "main universe multiverse"
+ section: "main"
uri: "http://127.0.0.1:8080/mirrors/ubuntu"
priority:
- suite: trusty
- type: deb
- - name: ubuntu-updates
- section: "main universe multiverse"
- uri: "http://127.0.0.1:8080/mirrors/ubuntu"
- priority:
- suite: trusty-updates
- type: deb
- - name: ubuntu-security
- section: "main universe multiverse"
- uri: "http://127.0.0.1:8080/mirrors/ubuntu"
- priority:
- suite: trusty-security
+ suite: xenial
type: deb
- name: mos
section: "main restricted"
uri: "http://127.0.0.1:8080/ubuntu/x86_64"
priority: 1050
- suite: mos9.0
+ suite: mos10.0
type: deb
skip_default_img_build: false
direct_repo_addresses:
- "127.0.0.1"
+ # User can provide default hashed root password for bootstrap image
+ # hashed_root_password: "$6$IInX3Cqo$5xytL1VZbZTusOewFnG6couuF0Ia61yS3rbC6P5YbZP2TYclwHqMq9e3Tg8rvQxhxSlBXP1DZhdUamxdOBXK0."
diff --git a/build/f_isoroot/f_repobuild/opnfv_config.yaml b/build/f_isoroot/f_repobuild/opnfv_config.yaml
new file mode 100644
index 000000000..b8d344196
--- /dev/null
+++ b/build/f_isoroot/f_repobuild/opnfv_config.yaml
@@ -0,0 +1,166 @@
+##############################################################################
+# Copyright (c) 2016 Enea AB and others.
+# Alexandru.Avadanii@enea.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+# REPO definitions
+
+# Based on old fuel-mirror data [1], adapted for OPNFV and packetary.
+# NOTE: 'uri' field will be added by opnfv_mirror_ubuntu.py from ENV.
+# [1] https://github.com/openstack/fuel-mirror/blob/stable/mitaka/
+# contrib/fuel_mirror/data/ubuntu.yaml
+
+# Main is a required parameter which defines what repository will be used
+# for images creation and that mirror should contain all packages for minimal
+# system creation.
+groups:
+ ubuntu:
+ - name: "ubuntu"
+ main: true
+ suite: "xenial"
+ section:
+ - "main"
+ - "multiverse"
+ - "restricted"
+ - "universe"
+ type: "deb"
+ priority: null
+
+ - name: "ubuntu-updates"
+ suite: "xenial-updates"
+ section:
+ - "main"
+ - "multiverse"
+ - "restricted"
+ - "universe"
+ type: "deb"
+ priority: null
+
+ - name: "ubuntu-security"
+ suite: "xenial-security"
+ section:
+ - "main"
+ - "multiverse"
+ - "restricted"
+ - "universe"
+ type: "deb"
+ priority: null
+
+ mos:
+ - name: "mos"
+ suite: "mos$mos_version"
+ section:
+ - "main"
+ - "restricted"
+ type: "deb"
+ priority: 1000
+
+ - name: "mos-updates"
+ suite: "mos$mos_version-updates"
+ section:
+ - "main"
+ - "restricted"
+ type: "deb"
+ priority: 1000
+
+ - name: "mos-security"
+ suite: "mos$mos_version-security"
+ section:
+ - "main"
+ - "restricted"
+ type: "deb"
+ priority: 1000
+
+ - name: "mos-holdback"
+ suite: "mos$mos_version-holdback"
+ section:
+ - "main"
+ - "restricted"
+ type: "deb"
+ priority: 1000
+
+# PACKAGES
+
+# This section lists packages that should be present in the local Ubuntu mirror,
+# but are not direct dependencies of any other packages from MOS or Ubuntu.
+# e.g.: additional kernels, bootloaders etc.
+
+packages:
+ - name: "vgabios"
+# Packages are required to build bootstrap images for a system.
+# The mirror should contiain such packages in addition to local mirror.
+ - name: "acpi-support"
+ - name: "anacron"
+ - name: "aptitude"
+ - name: "atop"
+ - name: "acct"
+ - name: "bash-completion"
+ - name: "bc"
+ - name: "build-essential"
+ - name: "cloud-init"
+ - name: "conntrackd"
+ - name: "cpu-checker"
+ - name: "cpufrequtils"
+ - name: "debconf-utils"
+ - name: "devscripts"
+ - name: "fping"
+ - name: "git"
+ - name: "grub-pc"
+ - name: "htop"
+ - name: "hwloc"
+ - name: "ifenslave"
+ - name: "iperf"
+ - name: "iptables-persistent"
+ - name: "irqbalance"
+ - name: "language-pack-en"
+ - name: "libapache2-mod-fastcgi"
+ - name: "libnss3-tools"
+ - name: "linux-headers-generic-lts-xenial"
+ - name: "linux-image-generic-lts-xenial"
+ - name: "live-boot"
+ - name: "livecd-rootfs"
+ - name: "mc"
+ - name: "memcached"
+ - name: "monit"
+ - name: "msmtp-mta"
+ - name: "multipath-tools"
+ - name: "multipath-tools-boot"
+ - name: "nginx"
+ - name: "ntp"
+ - name: "openssh-server"
+ - name: "percona-toolkit"
+ - name: "percona-xtrabackup"
+ - name: "pm-utils"
+ - name: "puppet"
+ - name: "python-lesscpy"
+ - name: "python-pip"
+ - name: "rsyslog-gnutls"
+ - name: "rsyslog-relp"
+ - name: "screen"
+ - name: "squashfs-tools"
+ - name: "swift-plugin-s3"
+ - name: "sysfsutils"
+ - name: "sysstat"
+ - name: "telnet"
+ - name: "tmux"
+ - name: "traceroute"
+ - name: "ubuntu-standard"
+ - name: "vim"
+ - name: "virt-what"
+ - name: "xinetd"
+ - name: "xmlstarlet"
+ - name: "tftpd-hpa"
+ - name: "syslinux"
+
+# OPNFV BLACKLIST
+
+# Packetary's dependency solving mechanism brings in more than one alternative
+# For example, "Depends: upstart | systemd-sysv" brings both packages, which
+# leads to debootstrap using "upstart" (1st option available in local repo).
+
+opnfv_blacklist:
+ - name: "upstart"
diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
deleted file mode 100755
index 1c7eb591a..000000000
--- a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-##############################################################################
-# Copyright (c) 2015 Ericsson AB and others.
-# mskalski@mirantis.com
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-import os
-import yaml
-
-current_snapshot = os.environ["LATEST_TARGET_UBUNTU"]
-mos_version = os.environ['MOS_VERSION']
-openstack_version = os.environ['OPENSTACK_VERSION']
-mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU']
-mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT']
-mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL']
-if os.environ.get('BUILD_FUEL_PLUGINS'):
- plugins = os.environ['BUILD_FUEL_PLUGINS']
-else:
- plugins = os.environ['PLUGINS']
-
-
-configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read()
-conf = yaml.load(configuration_file)
-conf['pattern_dir'] = '.'
-conf['openstack_version'] = openstack_version
-conf['mos_version'] = mos_version
-
-with open('opnfv-config.yaml', 'w') as outfile:
- outfile.write( yaml.dump(conf, default_flow_style=False) )
-
-pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read()
-pattern = yaml.load(pattern_file)
-pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root)
-pattern['ubuntu_baseurl'] = mirror_ubuntu
-for group in pattern['groups']['mos']:
- group['uri'] = pattern['mos_baseurl']
-for group in pattern['groups']['ubuntu']:
- group['uri'] = pattern['ubuntu_baseurl']
-
-for plugin in plugins.split():
- path = "../{}/packages.yaml".format(plugin)
- if os.path.isfile(path):
- f = open(path).read()
- plugin_yaml = yaml.load(f)
- plugin_set = set(plugin_yaml['packages'])
- main_set = set(pattern['packages'])
- new_packages = plugin_set - main_set
- print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages))
- pattern['packages'] = pattern['packages'] + list(new_packages)
-
-pattern['requirements']['ubuntu'] = pattern['packages']
-
-with open('ubuntu.yaml', 'w') as outfile:
- outfile.write( yaml.safe_dump(pattern, default_flow_style=False) )
diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py
new file mode 100755
index 000000000..1a603ee79
--- /dev/null
+++ b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py
@@ -0,0 +1,279 @@
+#!/usr/bin/env python
+##############################################################################
+# Copyright (c) 2015,2016 Ericsson AB, Mirantis Inc., Enea AB and others.
+# mskalski@mirantis.com
+# Alexandru.Avadanii@enea.com
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+"""Build multiarch partial local Ubuntu mirror using packetary"""
+
+##############################################################################
+# Design quirks / workarounds:
+# 1. Fuel-agent uses `debootstrap` to build bootstrap and target chroots from
+# the local mirror; which only uses the "main" component from the first
+# repository, i.e. does not include "updates"/"security".
+# In order to fullfill all debootstrap dependencies in "main" repo, we will
+# do an extra packetary run using a reduced scope:
+# - only "main" component of the first mirror;
+# - reduced package dependency list (without MOS/OPNFV plugin deps).
+# 2. If repo structure is not mandatory to be in sync with official mirrors,
+# we can mitigate the issue by "merging" all repo-components into a single
+# "main".
+##############################################################################
+# Mirror build steps (for EACH architecture in UBUNTU_ARCH):
+# 1. Collect bootstrap package deps from <fuel_bootstrap_cli.yaml>;
+# 2. Collect all fixture release packages from fuel-web's <openstack.yaml>;
+# 3. Parse new "opnfv_config.yaml" list of packages (from old fuel-mirror);
+# 4. Inherit enviroment variable(s) for mirror URLs, paths etc.
+# - Allow arch-specific overrides for each env var;
+# 5. Mirror config is defined based on common config + OPNFV overrides;
+# - Convert old configuration format to packetary style where needed;
+# 6. Package lists are defined based on common config + OPNFV deps;
+# - Keep track of "main" packages separately, required by debootstrap;
+# 7. Clone/update all mirror components;
+# 8. IF mirror merging is disabled:
+# - Clone/update "main" mirror component (fix missing debootstrap deps);
+# 9. IF mirror merging is enabled:
+# - Use `dpkg-scanpackages` to filter out old versions of duplicate pkgs;
+# - Run `packetary create` on the set of downloaded packages, merging
+# them on the fly into a single-component mirror;
+##############################################################################
+
+from copy import deepcopy
+import os
+import shutil
+import sys
+import yaml
+from contextlib import contextmanager
+from cStringIO import StringIO
+from packetary.cli.app import main
+
+@contextmanager
+def capture_stdout(output):
+ """Context manager for capturing stdout"""
+ stdout = sys.stdout
+ sys.stdout = output
+ yield
+ sys.stdout = stdout
+
+# FIXME: Find a better approach for eliminating duplicate logs than this
+def force_logger_reload():
+ """Force logger reload (ugly hack to prevent log duplication)"""
+ for mod in sys.modules.keys():
+ if mod.startswith('logging'):
+ try:
+ reload(sys.modules[mod])
+ except:
+ pass
+
+def get_unres_pkgs(architecture, cfg_mirror):
+ """Determine missing package dependecies for a mirror defition"""
+ unresolved_pkgs = list()
+ packetary_output = StringIO()
+ with capture_stdout(packetary_output):
+ main('unresolved -a {0} -r {1} -c name version --sep ;'
+ .format(_ARCH[architecture], cfg_mirror).split(' '))
+ for dep_pkg in packetary_output.getvalue().splitlines():
+ if dep_pkg.startswith('#'):
+ continue
+ dep = dep_pkg.split(';')
+ unresolved_pkgs += [{'name': dep[0], 'version': dep[1]}]
+ force_logger_reload()
+ return unresolved_pkgs
+
+def from_legacy_pkglist(legacy_pkglist):
+ """Package list conversion from `old fuel-mirror` to `packetary` style"""
+ pkglist = list()
+ for pkg in legacy_pkglist:
+ pkglist += [{'name': pkg}]
+ return pkglist
+
+def to_legacy_pkglist(pkglist):
+ """Package list conversion from `packetary` style to `old fuel-mirror`"""
+ legacy_pkglist = list()
+ for pkg in pkglist:
+ legacy_pkglist.append(pkg['name'])
+ return legacy_pkglist
+
+def legacy_diff(base_pkglist, new_pkglist, requester, architecture):
+ """Package list diff (old format)"""
+ diff_set = set(new_pkglist)
+ if base_pkglist:
+ diff_set -= set(base_pkglist)
+ if diff_set:
+ print(' * {0} requires new packages for architecture [{1}]: {2}'
+ .format(requester, architecture, ', '.join(diff_set)))
+ return list(diff_set)
+
+def do_local_repo(architecture, cfg_repo, cfg_packages_paths):
+ """Create single-component local repo (one architecture per call)"""
+ # Packetary does not use a global config file, so pass old settings here.
+ main('create -t deb -a {0} --repository {1} --package-files {2}'
+ ' --ignore-errors-num 2 --retries-num 3 --threads-num 10'
+ .format(_ARCH[architecture], cfg_repo, cfg_packages_paths).split(' '))
+ force_logger_reload()
+
+def do_partial_mirror(architecture, cfg_mirror, cfg_packages):
+ """Clone partial local mirror (one architecture per call)"""
+ # Note: '-d .' is ignored, as each mirror defines its own path.
+ main('clone -t deb -a {0} -r {1} -R {2} -d .'
+ ' --ignore-errors-num 2 --retries-num 3 --threads-num 10'
+ .format(_ARCH[architecture], cfg_mirror, cfg_packages).split(' '))
+ force_logger_reload()
+
+def write_cfg_file(cfg_mirror, data):
+ """Write configuration (yaml) file (package list / mirror defition)"""
+ with open(cfg_mirror, 'w') as outfile:
+ outfile.write(yaml.safe_dump(data, default_flow_style=False))
+
+def get_env(env_var, architecture=None):
+ """Evaluate architecture-specific overrides of env vars"""
+ if architecture:
+ env_var_arch = '{0}_{1}'.format(env_var, architecture)
+ if os.environ.get(env_var_arch):
+ return os.environ[env_var_arch]
+ if os.environ.get(env_var):
+ return os.environ[env_var]
+ return None
+
+# Architecture name mapping (dpkg:packetary) for packetary CLI invocation
+_ARCH = {
+ "i386": "i386",
+ "amd64": "x86_64",
+ "arm64": "aarch64",
+}
+
+# Arch-indepedent configuration (old fuel-mirror + OPNFV extra packages)
+CFG_D = 'opnfv_config'
+CFG_OPNFV = 'opnfv_config.yaml'
+MOS_VERSION = get_env('MOS_VERSION')
+UBUNTU_ARCH = get_env('UBUNTU_ARCH')
+MIRROR_UBUNTU_PATH = get_env('MIRROR_UBUNTU_OPNFV_PATH')
+MIRROR_UBUNTU_TMP_PATH = '{0}.tmp'.format(MIRROR_UBUNTU_PATH)
+MIRROR_UBUNTU_MERGE = get_env('MIRROR_UBUNTU_MERGE')
+CFG_MM_UBUNTU = '{0}/ubuntu_mirror_local.yaml'.format(CFG_D)
+FUEL_BOOTSTRAP_CLI_FILE = open('fuel_bootstrap_cli.yaml').read()
+FUEL_BOOTSTRAP_CLI = yaml.load(FUEL_BOOTSTRAP_CLI_FILE)
+FIXTURE_FILE = open('fuel-web/nailgun/nailgun/fixtures/openstack.yaml').read()
+FIXTURE = yaml.load(FIXTURE_FILE)
+OPNFV_CFG_YAML = open(CFG_OPNFV).read()
+OPNFV_CFG = yaml.load(OPNFV_CFG_YAML)
+
+# Create local partial mirror using packetary, one arch at a time
+for arch in UBUNTU_ARCH.split(' '):
+ # Mirror / Package env vars, arch-overrideable
+ mos_ubuntu = get_env('MIRROR_MOS_UBUNTU', arch)
+ mos_ubuntu_root = get_env('MIRROR_MOS_UBUNTU_ROOT', arch)
+ mirror_ubuntu = get_env('MIRROR_UBUNTU_URL', arch)
+ plugins = get_env('BUILD_FUEL_PLUGINS', arch)
+ if plugins is None:
+ plugins = get_env('PLUGINS', arch)
+
+ # Mirror / Package list configuration files (arch-specific)
+ cfg_m_mos = '{0}/mos_{1}_mirror.yaml'.format(CFG_D, arch)
+ cfg_m_ubuntu = '{0}/ubuntu_{1}_mirror.yaml'.format(CFG_D, arch)
+ cfg_p_ubuntu = '{0}/ubuntu_{1}_packages.yaml'.format(CFG_D, arch)
+ cfg_m_ubuntu_main = '{0}/ubuntu_{1}_mirror_main.yaml'.format(CFG_D, arch)
+ cfg_p_ubuntu_main = '{0}/ubuntu_{1}_packages_main.yaml'.format(CFG_D, arch)
+
+ # Mirror config fork before customizing (arch-specific)
+ arch_mos = 'mos_{0}'.format(arch)
+ arch_ubuntu = 'ubuntu_{0}'.format(arch)
+ arch_packages = 'packages_{0}'.format(arch)
+ OPNFV_CFG['groups'][arch_mos] = deepcopy(OPNFV_CFG['groups']['mos'])
+ OPNFV_CFG['groups'][arch_ubuntu] = deepcopy(OPNFV_CFG['groups']['ubuntu'])
+ OPNFV_CFG[arch_packages] = OPNFV_CFG['packages']
+
+ # Mirror config update & conversion to packetary input
+ group_main_ubuntu = dict()
+ for group in OPNFV_CFG['groups'][arch_mos]:
+ group['uri'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root)
+ group['suite'] = group['suite'].replace('$mos_version', MOS_VERSION)
+ for group in OPNFV_CFG['groups'][arch_ubuntu]:
+ group['uri'] = mirror_ubuntu
+ # FIXME: At `create`, packetary insists on copying all pkgs to dest dir,
+ # so configure it for another dir, which will replace the orig.
+ group['path'] = MIRROR_UBUNTU_TMP_PATH
+ if not group_main_ubuntu and 'main' in group:
+ group_main_ubuntu = [deepcopy(group)]
+ group_main_ubuntu[0]['section'] = ['main']
+
+ # Mirror config dump: MOS (for dep resolution), Ubuntu, Ubuntu[main]
+ write_cfg_file(cfg_m_mos, OPNFV_CFG['groups'][arch_mos])
+ write_cfg_file(cfg_m_ubuntu, OPNFV_CFG['groups'][arch_ubuntu])
+ if MIRROR_UBUNTU_MERGE is None:
+ write_cfg_file(cfg_m_ubuntu_main, group_main_ubuntu)
+ else:
+ # FIXME: For multiarch, only one dump would be enough
+ group_main_ubuntu[0]['origin'] = 'Ubuntu'
+ group_main_ubuntu[0]['path'] = MIRROR_UBUNTU_PATH
+ group_main_ubuntu[0]['uri'] = MIRROR_UBUNTU_PATH
+ write_cfg_file(CFG_MM_UBUNTU, group_main_ubuntu[0])
+
+ # Collect package dependencies from:
+ ## 1. fuel_bootstrap_cli.yaml (bootstrap image additional packages)
+ legacy_unresolved = legacy_diff(None, FUEL_BOOTSTRAP_CLI['packages'] + [
+ FUEL_BOOTSTRAP_CLI['kernel_flavor'],
+ FUEL_BOOTSTRAP_CLI['kernel_flavor'].replace('image', 'headers')],
+ 'Bootstrap', arch)
+ ## 2. openstack.yaml FIXTURE definition (default target image packages)
+ for release in FIXTURE:
+ editable = release['fields']['attributes_metadata']['editable']
+ if 'provision' in editable and 'packages' in editable['provision']:
+ release_pkgs = editable['provision']['packages']['value'].split()
+ legacy_unresolved += legacy_diff(legacy_unresolved, release_pkgs,
+ 'Release {0}'.format(release['fields']['name']), arch)
+ ## 3. OPNFV additional packages (includes old fuel-mirror ubuntu.yaml pkgs)
+ unresolved = dict()
+ unresolved['mandatory'] = 'exact'
+ unresolved['packages'] = from_legacy_pkglist(legacy_unresolved)
+ if 'packages' in OPNFV_CFG:
+ legacy_diff(legacy_unresolved, to_legacy_pkglist(OPNFV_CFG['packages']),
+ 'OPNFV config', arch)
+ unresolved['packages'] += OPNFV_CFG['packages']
+
+ # OPNFV plugins dependency resolution
+ if plugins:
+ for plugin in plugins.split():
+ path = "../{}/packages.yaml".format(plugin)
+ if os.path.isfile(path):
+ f = open(path).read()
+ plugin_yaml = yaml.load(f)
+ new_pkgs = legacy_diff(
+ to_legacy_pkglist(unresolved['packages']),
+ plugin_yaml['packages'], 'Plugin {0}'.format(plugin), arch)
+ unresolved['packages'] += from_legacy_pkglist(new_pkgs)
+
+ # Package list (reduced, i.e. no MOS deps, but with OPNFV plugin deps)
+ if MIRROR_UBUNTU_MERGE is None:
+ write_cfg_file(cfg_p_ubuntu_main, unresolved)
+
+ # Mirror package list (full, including MOS/OPNFV plugin deps)
+ unresolved['packages'] += get_unres_pkgs(arch, cfg_m_mos)
+ write_cfg_file(cfg_p_ubuntu, unresolved)
+ do_partial_mirror(arch, cfg_m_ubuntu, cfg_p_ubuntu)
+ if MIRROR_UBUNTU_MERGE is None:
+ # Ubuntu[main] must be evaluated after Ubuntu
+ do_partial_mirror(arch, cfg_m_ubuntu_main, cfg_p_ubuntu_main)
+
+if MIRROR_UBUNTU_MERGE is None:
+ shutil.move(MIRROR_UBUNTU_TMP_PATH, MIRROR_UBUNTU_PATH)
+else:
+ # Construct single-component mirror from all components
+ for arch in UBUNTU_ARCH.split(' '):
+ cfg_pp_ubuntu = '{0}/ubuntu_{1}_packages_paths.yaml'.format(CFG_D, arch)
+ # OPNFV blacklist
+ opnfv_blacklist = to_legacy_pkglist(OPNFV_CFG['opnfv_blacklist'])
+ # FIXME: We need scanpackages to omit older DEBs
+ # Inspired from http://askubuntu.com/questions/198474/
+ os.system('dpkg-scanpackages -a {0} {1} 2>/dev/null | '
+ 'grep -e "^Filename:" | sed "s|Filename: |- file://|g" | '
+ 'grep -v -E "\/({2})_" > {3}'
+ .format(arch, MIRROR_UBUNTU_TMP_PATH,
+ '|'.join(opnfv_blacklist), cfg_pp_ubuntu))
+ do_local_repo(arch, CFG_MM_UBUNTU, cfg_pp_ubuntu)
+ shutil.rmtree(MIRROR_UBUNTU_TMP_PATH)
diff --git a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh
index 5618eb79b..c8c86db53 100755
--- a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh
+++ b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh
@@ -1,5 +1,6 @@
#!/bin/bash
+UBUNTU_DISTRO="xenial"
BLACKLIST="http://mirrors.se.eu.kernel.org/ubuntu/"
#BLACKLIST+=" http://foo.bar"
@@ -28,26 +29,26 @@ blacklisted () {
check_mirror () {
mirror=$1
status=0
- for packdir in dists/trusty-updates/main/binary-amd64 \
- dists/trusty-updates/restricted/binary-amd64 \
- dists/trusty-updates/universe/binary-amd64 \
- dists/trusty-updates/multiverse/binary-amd64 \
- dists/trusty-security/main/binary-amd64 \
- dists/trusty-security/restricted/binary-amd64 \
- dists/trusty-security/universe/binary-amd64 \
- dists/trusty-security/multiverse/binary-amd64 \
- dists/trusty-proposed/main/binary-amd64 \
- dists/trusty-proposed/restricted/binary-amd64 \
- dists/trusty-proposed/universe/binary-amd64 \
- dists/trusty-proposed/multiverse/binary-amd64 \
- dists/trusty/main/binary-amd64 \
- dists/trusty/restricted/binary-amd64 \
- dists/trusty/universe/binary-amd64 \
- dists/trusty/multiverse/binary-amd64 \
- dists/trusty-backports/main/binary-amd64 \
- dists/trusty-backports/restricted/binary-amd64 \
- dists/trusty-backports/universe/binary-amd64 \
- dists/trusty-backports/multiverse/binary-amd64
+ for packdir in dists/${UBUNTU_DISTRO}-updates/main/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-updates/restricted/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-updates/universe/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-updates/multiverse/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-security/main/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-security/restricted/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-security/universe/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-security/multiverse/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-proposed/main/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-proposed/restricted/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-proposed/universe/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-proposed/multiverse/binary-amd64 \
+ dists/${UBUNTU_DISTRO}/main/binary-amd64 \
+ dists/${UBUNTU_DISTRO}/restricted/binary-amd64 \
+ dists/${UBUNTU_DISTRO}/universe/binary-amd64 \
+ dists/${UBUNTU_DISTRO}/multiverse/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-backports/main/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-backports/restricted/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-backports/universe/binary-amd64 \
+ dists/${UBUNTU_DISTRO}-backports/multiverse/binary-amd64
do
for packfile in Release Packages.gz
do