Merge "AArch64: Ubuntu local partial mirror: Add arm64"

8 files changed, 800 insertions, 48 deletions

diff --git a/.gitmodules b/.gitmodules
index b49fa711..684867af 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -18,3 +18,8 @@
 	url = https://gerrit.opnfv.org/gerrit/vswitchperf.git
 	branch = master
 	ignore = all
+[submodule "packetary"]
+	path = upstream/packetary
+	url = https://github.com/openstack/packetary.git
+	branch = master
+	ignore = all
diff --git a/armband-fuel-config.mk b/armband-fuel-config.mk
index 7aec79ea..8ccb29cf 100644
--- a/armband-fuel-config.mk
+++ b/armband-fuel-config.mk
@@ -46,6 +46,9 @@ export VSPERF_REPO   := ${ARMBAND_BASE}/upstream/vswitchperf
 export VSPERF_BRANCH := opnfv-armband
 export VSPERF_CHANGE := ${A_OPNFV_TAG}
 
+export PACKETARY_REPO   := ${ARMBAND_BASE}/upstream/packetary
+export PACKETARY_COMMIT := ${A_OPNFV_TAG}
+
 # for the patches applying purposes (empty git config in docker build container)
 export GIT_COMMITTER_NAME?=Fuel OPNFV
 export GIT_COMMITTER_EMAIL?=fuel@opnfv.org
diff --git a/patches/opnfv-fuel/cross-bootstrap/0003-bootstrap-Use-public-Ubuntu-ports-mirrors.patch b/patches/opnfv-fuel/cross-bootstrap/0003-bootstrap-Use-public-Ubuntu-ports-mirrors.patch
deleted file mode 100644
index 157306b2..00000000
--- a/patches/opnfv-fuel/cross-bootstrap/0003-bootstrap-Use-public-Ubuntu-ports-mirrors.patch
+++ /dev/null
@@ -1,48 +0,0 @@
-From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
-Date: Thu, 3 Mar 2016 19:18:27 +0100
-Subject: [PATCH] bootstrap: Use public Ubuntu-ports mirrors.
-
-Instead of relying on 127.0.0.1 partial Ubuntu mirror, use
-public archives.
-While doing so, switch to Ubuntu-ports, enabling  building arm64
-bootstraps.
-
-NOTE: This disables the abillity to build amd64 bootstraps with
-current config.
----
- build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml | 7 ++++---
- 1 file changed, 4 insertions(+), 3 deletions(-)
-
-diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
-index 264ae0e..17e161f 100644
---- a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
-+++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
-@@ -49,19 +49,19 @@
-   repos:
-     - name: ubuntu
-       section: "main universe multiverse"
--      uri: "http://127.0.0.1:8080/mirrors/ubuntu"
-+      uri: "http://ports.ubuntu.com/ubuntu-ports"
-       priority:
-       suite: trusty
-       type: deb
-     - name: ubuntu-updates
-       section: "main universe multiverse"
--      uri: "http://127.0.0.1:8080/mirrors/ubuntu"
-+      uri: "http://ports.ubuntu.com/ubuntu-ports"
-       priority:
-       suite: trusty-updates
-       type: deb
-     - name: ubuntu-security
-       section: "main universe multiverse"
--      uri: "http://127.0.0.1:8080/mirrors/ubuntu"
-+      uri: "http://ports.ubuntu.com/ubuntu-ports"
-       priority:
-       suite: trusty-security
-       type: deb
-@@ -73,4 +73,5 @@
-       type: deb
-   skip_default_img_build: false
-   direct_repo_addresses:
-+    - "ports.ubuntu.com"
-     - "127.0.0.1"
diff --git a/patches/opnfv-fuel/upstream-backports/0003-f_repobuild-Use-packetary-to-build-partial-mirror.patch b/patches/opnfv-fuel/upstream-backports/0003-f_repobuild-Use-packetary-to-build-partial-mirror.patch
new file mode 100644
index 00000000..cd3e8e2e
--- /dev/null
+++ b/patches/opnfv-fuel/upstream-backports/0003-f_repobuild-Use-packetary-to-build-partial-mirror.patch
@@ -0,0 +1,643 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Wed, 6 Jul 2016 17:03:49 +0200
+Subject: [PATCH] f_repobuild: Use packetary to build partial mirror
+
+FIXME: DO NOT MERGE until [2, 3] are merged upstream & included here!
+TODO: drop ALL fuel-mirror dependencies, for Fuel Newton compatibility.
+
+Packetary [1] will replace fuel-mirror in upstream Fuel.
+fuel-mirror is using packetary under the hood already, so the
+transition should not be too hard.
+
+Adapt OPNFV Ubuntu partial mirror build:
+- obsolete opnfv-config.yaml (packetary uses CLI args for this);
+- split old ubuntu.yaml into:
+  * mos_<arch>_mirror.yaml - MOS mirror cfg, only for deps analysis;
+  * ubuntu_<arch>_mirror.yaml - Ubuntu mirror cfg, for pkgs download;
+  * ubuntu_<arch>_packages.yaml - Ubuntu packages to download;
+
+While the consumers of our mirror still use `debootstrap`, which only
+supports the "main" repo component, make sure the "main" package set
+is complete, in the scope of:
+  * ubuntu_<arch>_mirror_main.yaml - Ubuntu [main only] mirror;
+  * ubuntu_<arch>_packages_main.yaml - Ubuntu packages for [main];
+
+New workflow (executed for EACH architecture):
+1. Parse "fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml"
+   and write correponding sections to new yaml files described above;
+
+   NOTE: Since fuel-mirror is going to be made obsolete, there are a
+   number of new ways we can gather the data currently passed via
+   fuel-mirror's "ubuntu.yaml" (e.g. use "mos-repos" from Fuel ISO
+   to determine MOS dependencies).
+
+2. Add OPNFV extra package dependencies (NEW)
+   Currently, this is empty / not used, but might be used to add
+   extra packages to the local mirror, like additional kernels,
+   bootloaders etc.
+
+   NOTE: Dependencies up to and including this group should also
+   be satisfiable from Ubuntu "main" repo component only.
+
+3. Append plugin dependencies to the package list;
+
+4. Run `packetary unresolved` for MOS repos, gathering MOS deps that
+   should be fetched from Ubuntu. The new packages will be appended
+   to the predefined list from old "ubuntu.yaml";
+
+5. Run `packetary clone` to download all required pkgs for the
+   partial Ubuntu mirror.
+
+6. IF repo component merging is disabled, run `packetary clone` only
+   on Ubuntu[main] repo components to download any (still) missing
+   dependencies for `debootstrap`, which only looks at this component,
+   and not the whole mirror.
+
+7. IF repo component merging is enabled, run first `dpkg-scanpackages`
+   to filter out old duplicate versions of packages, then run
+   `packetary create` to merge all repo comps into a single "main".
+
+v1 -> v2:
+- multiarch support (activated via UBUNTU_ARCH envvar);
+- fixed debootstrap missing deps in "main" component of Ubuntu mirror;
+- factored out some hardcodes into variables/functions;
+- add .gitignore file;
+- move generated config files to "opnfv_config" subdir;
+- added arm64 Ubuntu repo URL (ports.ubuntu.com is separated from the
+  main Ubuntu mirror, archive.ubuntu.com, which only holds x86 pkgs);
+
+v3 -> v4:
+- introduce env var "MIRROR_UBUNTU_MERGE" to control local mirror
+  repo-component merging into a single "main";
+- enable mirror repo-component merging by default, edit
+  fuel_bootstrap_cli.yaml accordingly;
+
+v4 -> v6:
+- edit dea_base to use only main mirror;
+- fix duplicated logs;
+- remove `-q` flag for `create`;
+
+NOTE: Without filtering out old package verisons using
+`dpkg-scanpackages`, bootstrap build cannot solve all deps.
+
+FIXME: Packetary solves missing dependecies by also accepting
+different packages that provide the same package name (e.g. "ifupdown"
+is satisfied by "netscript" package).
+
+FIXME: Repo component merging is sort of slow, since packetary insists
+on copying the source files to the destination dir even if they are
+pointing to the same repo.
+
+FIXME: Packetary `create` uses a different directory scheme for the
+created mirror than the upstream Ubuntu, e.g.:
+[p] mirrors/ubuntu/pool/main/l/lvm2_2.02.98-6ubuntu2_amd64.deb
+[U] mirrors/ubuntu/pool/main/l/lvm2/lvm2_2.02.98-6ubuntu2_amd64.deb
+
+This disencourages creating the "merged" mirror in the same location
+as the source partial mirror.
+
+[1] https://wiki.openstack.org/wiki/Packetary
+[2] https://review.openstack.org/#/c/392936/
+[3] https://review.openstack.org/#/c/392937/
+
+JIRA: FUEL-218
+JIRA: FUEL-223
+
+Change-Id: If2ee86f348b7683c83bfaf686baba4f1b8f555f0
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ build/config.mk                                    |   5 +
+ build/f_isoroot/f_repobuild/.gitignore             |   4 +
+ build/f_isoroot/f_repobuild/Makefile               |  34 ++-
+ build/f_isoroot/f_repobuild/config.mk              |  26 +++
+ .../f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml  |  14 +-
+ build/f_isoroot/f_repobuild/opnfv_mirror_conf.py   |  57 -----
+ build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py | 246 +++++++++++++++++++++
+ build/f_isoroot/f_repobuild/opnfv_packages.yaml    |  14 ++
+ deploy/config/dea_base.yaml                        |  14 +-
+ 9 files changed, 312 insertions(+), 102 deletions(-)
+ create mode 100644 build/f_isoroot/f_repobuild/.gitignore
+ create mode 100644 build/f_isoroot/f_repobuild/config.mk
+ delete mode 100755 build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
+ create mode 100755 build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py
+ create mode 100644 build/f_isoroot/f_repobuild/opnfv_packages.yaml
+
+diff --git a/build/config.mk b/build/config.mk
+index ab27858..a62b7e9 100644
+--- a/build/config.mk
++++ b/build/config.mk
+@@ -17,6 +17,11 @@ FUEL_MAIN_TAG = 9.0.1
+ MOS_VERSION   = 9.0
+ OPENSTACK_VERSION = mitaka-9.0
+ 
++# List of space-separated Ubuntu architectures supported with current build
++# Format: same as `dpkg-architecture -qDEB_HOST_ARCH1`
++# NOTE: Currently only amd64 is supported by Fuel@OPNFV. Armband adds arm64.
++export UBUNTU_ARCH ?= amd64
++
+ # FIXME(alav): Disable remote tracking for now, stick to submodule commits
+ FUEL_TRACK_REMOTES =
+ 
+diff --git a/build/f_isoroot/f_repobuild/.gitignore b/build/f_isoroot/f_repobuild/.gitignore
+new file mode 100644
+index 0000000..cfedefb
+--- /dev/null
++++ b/build/f_isoroot/f_repobuild/.gitignore
+@@ -0,0 +1,4 @@
++packetary
++fuel-mirror
++nailgun
++opnfv_config
+diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile
+index 8beb882..a47441c 100644
+--- a/build/f_isoroot/f_repobuild/Makefile
++++ b/build/f_isoroot/f_repobuild/Makefile
+@@ -1,7 +1,8 @@
+ ##############################################################################
+-# Copyright (c) 2015 Ericsson AB and others.
++# Copyright (c) 2015,2016 Ericsson AB, Enea AB and others.
+ # stefan.k.berg@ericsson.com
+ # jonas.bjurel@ericsson.com
++# Alexandru.Avadanii@enea.com
+ # All rights reserved. This program and the accompanying materials
+ # are made available under the terms of the Apache License, Version 2.0
+ # which accompanies this distribution, and is available at
+@@ -10,41 +11,36 @@
+ 
+ SHELL := /bin/bash
+ TOP := $(shell pwd)
+-TMP_ROOT_DIR := $(shell echo "$(MIRROR_UBUNTU_ROOT)" | cut -d "/" -f2)
+ 
+ include ../../config.mk
++include config.mk
+ 
+ export MOS_VERSION
+-export OPENSTACK_VERSION
++export MIRROR_UBUNTU_OPNFV_PATH:=$(TOP)/nailgun/mirrors/ubuntu
+ 
+ .PHONY: all
+ all: nailgun
+ 
+ nailgun:
+-	sudo apt-get install -y git libxml2-dev libxslt-dev python-dev  python-pip libz-dev libyaml-dev createrepo python-yaml
+-	rm -Rf nailgun
+-	sudo mkdir -p /var/www/nailgun
++	sudo apt-get install -y createrepo git libxml2-dev libxslt1-dev \
++		python-dev zlib1g-dev
++	rm -Rf nailgun packetary opnfv_config && mkdir opnfv_config
+ 	ln -sf ${F_SUBMOD_DIR}/fuel-mirror fuel-mirror
+-	sudo pip install -U -r ./fuel-mirror/requirements.txt
+-	sudo pip install ./fuel-mirror
+-	sudo pip install ./fuel-mirror/contrib/fuel_mirror
+-	./opnfv_mirror_conf.py
+-	sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group ubuntu --pattern=ubuntu
+-	sudo chmod -R 755 /var/www/nailgun
+-	cp -Rp /var/www/nailgun .
+-	# In the end we want to have ubuntu repository in mirrors/ubuntu directory
+-	-if [ "$(MIRROR_UBUNTU_ROOT)" != "/ubuntu/" ]; then \
+-	  mkdir -p nailgun/mirrors/ubuntu;\
+-	  mv nailgun/mirrors$(MIRROR_UBUNTU_ROOT)* nailgun/mirrors/ubuntu;\
+-	  [ "$(MIRROR_UBUNTU_ROOT)" != "/" ] && rm -rf nailgun/mirrors/$(TMP_ROOT_DIR);\
++	git clone --quiet $(PACKETARY_REPO)
++	if [ -n $(PACKETARY_COMMIT) ]; then \
++		git -C packetary checkout $(PACKETARY_COMMIT); \
+ 	fi
++	sudo pip install -U -r ./packetary/requirements.txt
++	sudo pip install ./packetary
++	# Handle config and mirror build in one place
++	./opnfv_mirror_ubuntu.py
+ 	# Store artifact in cache straight away if caching is enabled
+ 	# (no .cacheid will be present unless this is a cached build)
+ 	test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0
+ 
+ .PHONY: clean
+ clean:
+-	@rm -rf ../release/opnfv/nailgun nailgun fuel-mirror opnfv-config.yaml ubuntu.yaml
++	@rm -rf ../release/opnfv/nailgun nailgun packetary fuel-mirror opnfv_config
+ 
+ .PHONY: release
+ release:nailgun
+diff --git a/build/f_isoroot/f_repobuild/config.mk b/build/f_isoroot/f_repobuild/config.mk
+new file mode 100644
+index 0000000..79e7d1a
+--- /dev/null
++++ b/build/f_isoroot/f_repobuild/config.mk
+@@ -0,0 +1,26 @@
++##############################################################################
++# Copyright (c) 2016 Ericsson AB, Enea AB and others.
++# stefan.k.berg@ericsson.com
++# jonas.bjurel@ericsson.com
++# Alexandru.Avadanii@enea.com
++# All rights reserved. This program and the accompanying materials
++# are made available under the terms of the Apache License, Version 2.0
++# which accompanies this distribution, and is available at
++# http://www.apache.org/licenses/LICENSE-2.0
++##############################################################################
++
++# Use a recent master commit, since tags/branches are not yet mature
++# FIXME(armband): Update upstream commit ref once [1, 2] are merged
++# [1] https://review.openstack.org/#/c/392937/
++# [2] https://review.openstack.org/#/c/392936/
++export PACKETARY_REPO?=https://github.com/openstack/packetary
++export PACKETARY_COMMIT?=c46465c3255a9f5e59a05b8701e06054df39f32f
++
++# arm64 Ubuntu mirror is separated from archive.ubuntu.com
++export MIRROR_UBUNTU_URL_arm64=http://ports.ubuntu.com/ubuntu-ports/
++export MIRROR_UBUNTU_ROOT_arm64=ubuntu-ports
++
++# Merge all local mirror repo components/section into single "main"
++# NOTE: When changing this, make sure to also update all consumer config, like:
++# - fuel_bootstrap_cli.yaml
++export MIRROR_UBUNTU_MERGE=true
+diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
+index fcf4257..647dcb6 100644
+--- a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
++++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml
+@@ -47,23 +47,11 @@
+   https_proxy: ""
+   repos:
+     - name: ubuntu
+-      section: "main universe multiverse"
++      section: "main"
+       uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+       priority:
+       suite: trusty
+       type: deb
+-    - name: ubuntu-updates
+-      section: "main universe multiverse"
+-      uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+-      priority:
+-      suite: trusty-updates
+-      type: deb
+-    - name: ubuntu-security
+-      section: "main universe multiverse"
+-      uri: "http://127.0.0.1:8080/mirrors/ubuntu"
+-      priority:
+-      suite: trusty-security
+-      type: deb
+     - name: mos
+       section: "main restricted"
+       uri: "http://127.0.0.1:8080/ubuntu/x86_64"
+diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
+deleted file mode 100755
+index 1c7eb59..0000000
+--- a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py
++++ /dev/null
+@@ -1,57 +0,0 @@
+-#!/usr/bin/env python
+-##############################################################################
+-# Copyright (c) 2015 Ericsson AB and others.
+-# mskalski@mirantis.com
+-# All rights reserved. This program and the accompanying materials
+-# are made available under the terms of the Apache License, Version 2.0
+-# which accompanies this distribution, and is available at
+-# http://www.apache.org/licenses/LICENSE-2.0
+-##############################################################################
+-import os
+-import yaml
+-
+-current_snapshot =  os.environ["LATEST_TARGET_UBUNTU"]
+-mos_version = os.environ['MOS_VERSION']
+-openstack_version = os.environ['OPENSTACK_VERSION']
+-mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU']
+-mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT']
+-mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL']
+-if os.environ.get('BUILD_FUEL_PLUGINS'):
+-  plugins = os.environ['BUILD_FUEL_PLUGINS']
+-else:
+-  plugins = os.environ['PLUGINS']
+-
+-
+-configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read()
+-conf = yaml.load(configuration_file)
+-conf['pattern_dir'] = '.'
+-conf['openstack_version'] = openstack_version
+-conf['mos_version'] = mos_version
+-
+-with open('opnfv-config.yaml', 'w') as outfile:
+-  outfile.write( yaml.dump(conf, default_flow_style=False) )
+-
+-pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read()
+-pattern = yaml.load(pattern_file)
+-pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root)
+-pattern['ubuntu_baseurl'] = mirror_ubuntu
+-for group in pattern['groups']['mos']:
+-  group['uri'] = pattern['mos_baseurl']
+-for group in pattern['groups']['ubuntu']:
+-  group['uri'] = pattern['ubuntu_baseurl']
+-
+-for plugin in plugins.split():
+-  path = "../{}/packages.yaml".format(plugin)
+-  if os.path.isfile(path):
+-    f = open(path).read()
+-    plugin_yaml = yaml.load(f)
+-    plugin_set = set(plugin_yaml['packages'])
+-    main_set = set(pattern['packages'])
+-    new_packages = plugin_set - main_set
+-    print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages))
+-    pattern['packages'] = pattern['packages'] + list(new_packages)
+-
+-pattern['requirements']['ubuntu'] = pattern['packages']
+-
+-with open('ubuntu.yaml', 'w') as outfile:
+-  outfile.write( yaml.safe_dump(pattern, default_flow_style=False) )
+diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py
+new file mode 100755
+index 0000000..89ec938
+--- /dev/null
++++ b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py
+@@ -0,0 +1,246 @@
++#!/usr/bin/env python
++##############################################################################
++# Copyright (c) 2015,2016 Ericsson AB, Mirantis Inc., Enea AB and others.
++# mskalski@mirantis.com
++# Alexandru.Avadanii@enea.com
++# All rights reserved. This program and the accompanying materials
++# are made available under the terms of the Apache License, Version 2.0
++# which accompanies this distribution, and is available at
++# http://www.apache.org/licenses/LICENSE-2.0
++##############################################################################
++
++##############################################################################
++# Build multiarch partial local Ubuntu mirror
++##############################################################################
++# Design quirks / workarounds:
++# 1. Fuel-agent uses `debootstrap` to build bootstrap and target chroots from
++#    the local mirror; which only uses the "main" component from the first
++#    repository, i.e. does not include "updates"/"security".
++#    In order to fullfill all debootstrap dependencies in "main" repo, we will
++#    do an extra packetary run using a reduced scope:
++#    - only "main" component of the first mirror;
++#    - reduced package dependency list (without MOS/OPNFV plugin deps).
++# 2. If repo structure is not mandatory to be in sync with official mirrors,
++#    we can mitigate the issue by "merging" all repo-components into a single
++#    "main".
++##############################################################################
++# Mirror build steps:
++# 1. Parse old <fuel-mirror> package list ("ubuntu.yaml");
++# 2. Parse new "opnfv_packages.yaml" list of additional packages;
++# 3. Inherit enviroment variable(s) for mirror URLs, paths etc.
++#    - Allow arch-specific overrides for each env var;
++# 4. For each architecture in UBUNTU_ARCH:
++# 4.1. Mirror config is defined based on common config + OPNFV overrides;
++#      - Convert old configuration format to packetary style where needed;
++# 4.2. Package lists are defined based on common config + OPNFV deps;
++#      - Keep track of "main" packages separately, required by debootstrap;
++# 4.3. Clone/update all mirror components;
++# 4.4. IF mirror merging is disabled OR workaround for ifupdown (see below):
++#      - Clone/update "main" mirror component (fix missing debootstrap deps);
++# 5. IF mirror merging is enabled:
++#    - Use `dpkg-scanpackages` to filter out old versions of duplicate pkgs;
++#    - Run `packetary create` on the set of downloaded packages, merging
++#      them on the fly into a single-component mirror;
++##############################################################################
++
++import copy
++import os
++import shutil
++import sys
++import yaml
++from contextlib import contextmanager
++from cStringIO import StringIO
++from packetary.cli.app import main
++
++@contextmanager
++def captureStdOut(output):
++    stdout = sys.stdout
++    sys.stdout = output
++    yield
++    sys.stdout = stdout
++
++# FIXME: Find a better approach for eliminating duplicate logs than this
++def force_logger_reload():
++  for mod in sys.modules.keys():
++    if mod.startswith('logging'):
++      try:
++        reload(sys.modules[mod])
++      except:
++        pass
++
++# Determine missing package dependecies for a mirror defition
++def get_unres_pkgs(arch, cfg_mirror):
++  unresolved = dict()
++  unresolved['packages'] = list()
++  packetary_output = StringIO()
++  with captureStdOut(packetary_output):
++    main('unresolved -a {0} -r {1} -c name version --sep ;'
++      .format(_ARCHITECTURES[arch], cfg_mirror).split(' '))
++  for dep_pkg in packetary_output.getvalue().splitlines():
++    if dep_pkg.startswith('#'):
++      continue
++    dep = dep_pkg.split(';')
++    unresolved['packages'] += [ {'name': dep[0], 'version': dep[1]} ]
++  force_logger_reload()
++  return unresolved
++
++# Create single-component local repo (one arch per call)
++def do_local_repo(arch, cfg_repo, cfg_packages_paths):
++  # Packetary does not use a global config file, so pass old settings here.
++  main('create -t deb -a {0} --repository {1} --package-files {2}'
++    ' --ignore-errors-num 2 --retries-num 3 --threads-num 10'
++    .format(_ARCHITECTURES[arch], cfg_repo, cfg_packages_paths).split(' '))
++  force_logger_reload()
++
++# Clone partial local mirror (one arch per call)
++def do_partial_mirror(arch, cfg_mirror, cfg_packages):
++  # Note: '-d .' is ignored, as each mirror defines its own path.
++  main('clone -t deb -a {0} -r {1} -R {2} -d .'
++    ' --ignore-errors-num 2 --retries-num 3 --threads-num 10'
++    .format(_ARCHITECTURES[arch], cfg_mirror, cfg_packages).split(' '))
++  force_logger_reload()
++
++# Write configuration (yaml) file (package list / mirror defition)
++def write_cfg_file(cfg_mirror, data):
++  with open(cfg_mirror, 'w') as outfile:
++    outfile.write( yaml.safe_dump(data, default_flow_style=False) )
++
++# Allow arch-specific overrides of env vars
++def get_env(env_var, arch=None):
++  if arch:
++    env_var_arch = '{0}_{1}'.format(env_var, arch)
++    if os.environ.get(env_var_arch):
++      return os.environ[env_var_arch]
++  if os.environ.get(env_var):
++    return os.environ[env_var]
++  return None
++
++# Architecture name mapping (dpkg:packetary) for packetary CLI invocation
++_ARCHITECTURES = {
++    "i386": "i386",
++    "amd64": "x86_64",
++    "arm64": "aarch64",
++}
++
++# Arch-indepedent configuration (old fuel-mirror + OPNFV extra packages)
++cfg_dir = 'opnfv_config'
++cfg_p_opnfv = 'opnfv_packages.yaml'
++mos_version = get_env('MOS_VERSION')
++ubuntu_arch = get_env('UBUNTU_ARCH')
++mirror_ubuntu_path = get_env('MIRROR_UBUNTU_OPNFV_PATH')
++mirror_ubuntu_tmp_path = '{0}.tmp'.format(mirror_ubuntu_path)
++mirror_ubuntu_merge = get_env('MIRROR_UBUNTU_MERGE')
++cfg_mm_ubuntu = '{0}/ubuntu_mirror_local.yaml'.format(cfg_dir)
++pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read()
++pattern = yaml.load(pattern_file)
++opnfv_pkgs_yaml = open(cfg_p_opnfv).read()
++opnfv_pkgs = yaml.load(opnfv_pkgs_yaml)
++
++# FIXME: Packetary solves missing dependecies by also accepting
++# different packages that provide the same name (e.g. "ifupdown" dependency
++# is satisfied by "netscript" package from "universe" repo-component).
++# Work around this by resolving all deps in "main" repo-component,
++# then scan and keep only latest debs for the whole <merged> repo.
++mirror_ubuntu_resolve_main_deps = True
++
++# Create local partial mirror using packetary, one arch at a time
++for arch in ubuntu_arch.split(' '):
++  # Mirror / Package env vars, arch-overrideable
++  mos_ubuntu = get_env('MIRROR_MOS_UBUNTU', arch)
++  mos_ubuntu_root = get_env('MIRROR_MOS_UBUNTU_ROOT', arch)
++  mirror_ubuntu = get_env('MIRROR_UBUNTU_URL', arch)
++  plugins = get_env('BUILD_FUEL_PLUGINS', arch)
++  if plugins is None:
++    plugins = get_env('PLUGINS', arch)
++
++  # Mirror / Package list configuration files (arch-specific)
++  cfg_m_mos = '{0}/mos_{1}_mirror.yaml'.format(cfg_dir, arch)
++  cfg_m_ubuntu = '{0}/ubuntu_{1}_mirror.yaml'.format(cfg_dir, arch)
++  cfg_p_ubuntu = '{0}/ubuntu_{1}_packages.yaml'.format(cfg_dir, arch)
++  cfg_m_ubuntu_main = '{0}/ubuntu_{1}_mirror_main.yaml'.format(cfg_dir, arch)
++  cfg_p_ubuntu_main = '{0}/ubuntu_{1}_packages_main.yaml'.format(cfg_dir, arch)
++
++  # Mirror config fork before customizing (arch-specific)
++  arch_group_mos = 'mos_{0}'.format(arch)
++  arch_group_ubuntu = 'ubuntu_{0}'.format(arch)
++  arch_packages = 'packages_{0}'.format(arch)
++  pattern['groups'][arch_group_mos] = copy.deepcopy(pattern['groups']['mos'])
++  pattern['groups'][arch_group_ubuntu] = copy.deepcopy(pattern['groups']['ubuntu'])
++  pattern[arch_packages] = pattern['packages']
++
++  # Mirror config update & conversion to packetary input
++  group_main_ubuntu = dict()
++  for group in pattern['groups'][arch_group_mos]:
++    group['uri'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root)
++    group['suite'] = group['suite'].replace('$mos_version', mos_version)
++    group['section'] = group['section'].split()
++  for group in pattern['groups'][arch_group_ubuntu]:
++    group['uri'] = mirror_ubuntu
++    # FIXME: At `create`, packetary insists on copying all pkgs to dest dir,
++    # so configure it for another dir, which will replace the orig at the end.
++    group['path'] = mirror_ubuntu_tmp_path
++    group['section'] = group['section'].split()
++    if not group_main_ubuntu and 'main' in group:
++      group_main_ubuntu = [ copy.deepcopy(group) ]
++      group_main_ubuntu[0]['section'] = [ 'main' ]
++
++  # Mirror config dump: MOS (for dep resolution), Ubuntu, Ubuntu[main]
++  write_cfg_file(cfg_m_mos, pattern['groups'][arch_group_mos])
++  write_cfg_file(cfg_m_ubuntu, pattern['groups'][arch_group_ubuntu])
++  if mirror_ubuntu_resolve_main_deps or mirror_ubuntu_merge is None:
++    write_cfg_file(cfg_m_ubuntu_main, group_main_ubuntu)
++  if mirror_ubuntu_merge is not None:
++    # FIXME: For multiarch, only one dump would be enough
++    group_main_ubuntu[0]['origin'] = 'Ubuntu'
++    group_main_ubuntu[0]['path'] = mirror_ubuntu_path
++    group_main_ubuntu[0]['uri'] = mirror_ubuntu_path
++    write_cfg_file(cfg_mm_ubuntu, group_main_ubuntu[0])
++
++  # Package list conversion from `old fuel-mirror` to `packetary` style + OPNFV
++  unresolved_pkgs = dict()
++  unresolved_pkgs['packages'] = list()
++  unresolved_pkgs['mandatory'] = 'exact'
++  if opnfv_pkgs['packages'] is not None:
++    unresolved_pkgs['packages'] = opnfv_pkgs['packages']
++  for pkg in pattern['packages']:
++    unresolved_pkgs['packages'] += [ {'name': pkg} ]
++
++  # Package list (reduced, i.e. no MOS/OPNFV plugin deps)
++  if mirror_ubuntu_resolve_main_deps or mirror_ubuntu_merge is None:
++    write_cfg_file(cfg_p_ubuntu_main, unresolved_pkgs)
++
++  # OPNFV plugins dependency resolution
++  for plugin in plugins.split():
++    path = "../{}/packages.yaml".format(plugin)
++    if os.path.isfile(path):
++      f = open(path).read()
++      plugin_yaml = yaml.load(f)
++      plugin_set = set(plugin_yaml['packages'])
++      main_set = set(pattern['packages'])
++      new_packages = plugin_set - main_set
++      print('Plugin {0} requires new packages for arch [{1}]: {2}'
++            .format(plugin, arch, ', '.join(new_packages)))
++      for pkg in new_packages:
++        unresolved_pkgs['packages'] += [ {'name': pkg} ]
++
++  # Mirror package list (full, including MOS/OPNFV plugin deps)
++  unresolved_pkgs['packages'] += get_unres_pkgs(arch, cfg_m_mos)['packages']
++  write_cfg_file(cfg_p_ubuntu, unresolved_pkgs)
++  do_partial_mirror(arch, cfg_m_ubuntu, cfg_p_ubuntu)
++  if mirror_ubuntu_resolve_main_deps or mirror_ubuntu_merge is None:
++    # Ubuntu[main] must be evaluated after Ubuntu
++    do_partial_mirror(arch, cfg_m_ubuntu_main, cfg_p_ubuntu_main)
++
++if mirror_ubuntu_merge is None:
++  shutil.move(mirror_ubuntu_tmp_path, mirror_ubuntu_path)
++else:
++  # Construct single-component mirror from all components
++  for arch in ubuntu_arch.split(' '):
++    cfg_pp_ubuntu = '{0}/ubuntu_{1}_packages_paths.yaml'.format(cfg_dir, arch)
++    # FIXME: We need scanpackages to omit older DEBs
++    # Inspired from http://askubuntu.com/questions/198474/
++    os.system('dpkg-scanpackages -a {0} {1} 2>/dev/null | '
++      'grep -e "^Filename:" | sed "s|Filename: |- file://|g" > {2}'
++        .format(arch, mirror_ubuntu_tmp_path, cfg_pp_ubuntu))
++    do_local_repo(arch, cfg_mm_ubuntu, cfg_pp_ubuntu)
++  shutil.rmtree(mirror_ubuntu_tmp_path)
+diff --git a/build/f_isoroot/f_repobuild/opnfv_packages.yaml b/build/f_isoroot/f_repobuild/opnfv_packages.yaml
+new file mode 100644
+index 0000000..3f5c59f
+--- /dev/null
++++ b/build/f_isoroot/f_repobuild/opnfv_packages.yaml
+@@ -0,0 +1,14 @@
++##############################################################################
++# Copyright (c) 2016 Enea AB and others.
++# Alexandru.Avadanii@enea.com
++# All rights reserved. This program and the accompanying materials
++# are made available under the terms of the Apache License, Version 2.0
++# which accompanies this distribution, and is available at
++# http://www.apache.org/licenses/LICENSE-2.0
++##############################################################################
++
++# This file lists packages that should be present in the local Ubuntu mirror,
++# but are not direct dependencies of any other packages from  MOS or Ubuntu.
++# e.g.: additional kernels, bootloaders etc.
++
++packages:
+diff --git a/deploy/config/dea_base.yaml b/deploy/config/dea_base.yaml
+index c1a0606..786c231 100644
+--- a/deploy/config/dea_base.yaml
++++ b/deploy/config/dea_base.yaml
+@@ -633,22 +633,10 @@ settings:
+         value:
+         - name: ubuntu
+           priority: null
+-          section: main universe multiverse
++          section: main
+           suite: trusty
+           type: deb
+           uri: http://10.20.0.2:8080/mirrors/ubuntu/
+-        - name: ubuntu-updates
+-          priority: null
+-          section: main universe multiverse
+-          suite: trusty-updates
+-          type: deb
+-          uri: http://10.20.0.2:8080/mirrors/ubuntu/
+-        - name: ubuntu-security
+-          priority: null
+-          section: main universe multiverse
+-          suite: trusty-security
+-          type: deb
+-          uri: http://10.20.0.2:8080/mirrors/ubuntu/
+         - name: mos
+           priority: 1050
+           section: main restricted
diff --git a/patches/packetary/0001-deb_driver-Translate-repository.architecture.patch b/patches/packetary/0001-deb_driver-Translate-repository.architecture.patch
new file mode 100644
index 00000000..18f20015
--- /dev/null
+++ b/patches/packetary/0001-deb_driver-Translate-repository.architecture.patch
@@ -0,0 +1,30 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Thu, 27 Oct 2016 22:24:28 +0200
+Subject: [PATCH] deb_driver: Translate repository.architecture
+
+When generating repository metadata, the Release file gets the
+architecture list in an untranslated format, ending up with
+'x86_64' instead of 'amd64'.
+
+Closes-bug: 1638628
+
+Change-Id: I8471db6d54157175ff275c969ed5195d4616fa0f
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ packetary/drivers/deb_driver.py | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/packetary/drivers/deb_driver.py b/packetary/drivers/deb_driver.py
+index ece9fa1..929bdca 100644
+--- a/packetary/drivers/deb_driver.py
++++ b/packetary/drivers/deb_driver.py
+@@ -409,7 +409,8 @@ class DebRepositoryDriver(RepositoryDriverBase):
+         release.setdefault("Description", "The packages repository.")
+
+         keys = ("Architectures", "Components")
+-        values = (repository.architecture, repository.section[1])
++        values = (_ARCHITECTURES[repository.architecture],
++                  repository.section[1])
+         for key, value in six.moves.zip(keys, values):
+             if key in release:
+                 release[key] = utils.append_token_to_string(
diff --git a/patches/packetary/0002-clone-Create-metadata-for-empty-components.patch b/patches/packetary/0002-clone-Create-metadata-for-empty-components.patch
new file mode 100644
index 00000000..70d75fee
--- /dev/null
+++ b/patches/packetary/0002-clone-Create-metadata-for-empty-components.patch
@@ -0,0 +1,43 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Tue, 1 Nov 2016 23:01:58 +0100
+Subject: [PATCH] clone: Create metadata for empty components
+
+In certain scenarios, mirror components (e.g. trusty-security)
+are present in both packetary configuration input, and in target
+system's apt source definitions, but contain no packages of interest.
+
+For such repository/requirements combinations, packetary currently
+skips creating metadata (Release, Packages) for those components,
+which leads to the partial mirror missing some critical files for apt,
+(an empty file would be enough).
+
+e.g.: Using packetary to create a partial Ubuntu mirror, then trying
+to build a bootstrap image from the new mirror leads to:
+
+W: Failed to fetch http://127.0.0.1:8080/mirrors/ubuntu/dists/\
+   trusty-security/multiverse/binary-amd64/Packages  404
+
+Closes-bug: 1638631
+
+Change-Id: I850b43d5b4d8742d99e9a5702cc9ad4de881a401
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ packetary/api/repositories.py | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/packetary/api/repositories.py b/packetary/api/repositories.py
+index ec8c54c..2c72e29 100644
+--- a/packetary/api/repositories.py
++++ b/packetary/api/repositories.py
+@@ -121,6 +121,11 @@ class RepositoryApi(object):
+         for pkg in all_packages:
+             package_groups[pkg.repository].add(pkg)
+
++        # Make sure we create metadata for all repos, even if empty
++        for repo in repositories:
++            if repo not in package_groups:
++                package_groups[repo] = set()
++
+         stat = CopyStatistics()
+         mirrors = defaultdict(set)
+         options = options or self.CopyOptions()
diff --git a/patches/packetary/0003-AArch64-support-api-cli-controllers-drivers.patch b/patches/packetary/0003-AArch64-support-api-cli-controllers-drivers.patch
new file mode 100644
index 00000000..da01bc33
--- /dev/null
+++ b/patches/packetary/0003-AArch64-support-api-cli-controllers-drivers.patch
@@ -0,0 +1,76 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Thu, 27 Oct 2016 21:51:10 +0200
+Subject: [PATCH] AArch64 support: api, cli, controllers, drivers
+
+Enable new architecture 'aarch64' ('arm64' on Ubuntu).
+Tested with DEB driver.
+
+Closes-bug: 1638635
+
+Change-Id: I9761322020837186c109c18e849128791ab909d8
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ packetary/api/repositories.py       | 3 ++-
+ packetary/cli/commands/base.py      | 2 +-
+ packetary/controllers/repository.py | 3 ++-
+ packetary/drivers/deb_driver.py     | 2 ++
+ 4 files changed, 7 insertions(+), 3 deletions(-)
+
+diff --git a/packetary/api/repositories.py b/packetary/api/repositories.py
+index ec8c54c..d49d8d0 100644
+--- a/packetary/api/repositories.py
++++ b/packetary/api/repositories.py
+@@ -73,7 +73,8 @@ class RepositoryApi(object):
+
+         :param config: the configuration
+         :param repotype: the kind of repository(deb, yum, etc)
+-        :param repoarch: the architecture of repository (x86_64 or i386)
++        :param repoarch: the architecture of repository
++                         (x86_64, i386 or aarch64)
+         """
+         context = config if isinstance(config, Context) else Context(config)
+         return cls(RepositoryController.load(context, repotype, repoarch))
+diff --git a/packetary/cli/commands/base.py b/packetary/cli/commands/base.py
+index 83c2f38..c2cc497 100644
+--- a/packetary/cli/commands/base.py
++++ b/packetary/cli/commands/base.py
+@@ -56,7 +56,7 @@ class BaseRepoCommand(BaseCommand):
+             '-a',
+             '--arch',
+             type=str,
+-            choices=["x86_64", "i386"],
++            choices=["x86_64", "i386", "aarch64"],
+             metavar='ARCHITECTURE',
+             default="x86_64",
+             help='The target architecture.')
+diff --git a/packetary/controllers/repository.py b/packetary/controllers/repository.py
+index 921a5c1..e6cbc9b 100644
+--- a/packetary/controllers/repository.py
++++ b/packetary/controllers/repository.py
+@@ -45,7 +45,8 @@ class RepositoryController(object):
+
+         :param context: the context
+         :param driver_name: the name of required driver
+-        :param repoarch: the architecture of repository (x86_64 or i386)
++        :param repoarch: the architecture of repository
++                         (x86_64, i386 or aarch64)
+         """
+         if cls._drivers is None:
+             cls._drivers = stevedore.ExtensionManager(
+diff --git a/packetary/drivers/deb_driver.py b/packetary/drivers/deb_driver.py
+index ece9fa1..0df1aa4 100644
+--- a/packetary/drivers/deb_driver.py
++++ b/packetary/drivers/deb_driver.py
+@@ -48,10 +48,12 @@ _OPERATORS_MAPPING = {
+ }
+
+ _ARCHITECTURES = {
++    "aarch64": "arm64",
+     "x86_64": "amd64",
+     "i386": "i386",
+     "source": "Source",
+     "amd64": "x86_64",
++    "arm64": "aarch64",
+ }
+
+ _PRIORITIES = {
diff --git a/upstream/packetary b/upstream/packetary
new file mode 160000
+Subproject c46465c3255a9f5e59a05b8701e06054df39f32