diff options
| author |   Alexandru Avadanii <Alexandru.Avadanii@enea.com> | 2016-12-17 19:05:18 +0100 |
|---|---|---|
| committer | Alexandru Avadanii <Alexandru.Avadanii@enea.com> | 2016-12-17 19:34:30 +0100 |
| commit | 7fa6312de8008d535419795c7c8805ff34970645 (patch) | |
| tree | 779e58377c0f3df5ac0f218460f3fa7a4874208e | |
| parent | 03854a0311149a140fb751a47886ff1c05022231 (diff) | |
sub: packetary: remove, patches upstream
Backported from [1] and adapted for Trusty, since Armband
did not switch to Ubuntu Xenial / Openstack Newton yet.
[1] https://gerrit.opnfv.org/gerrit/#/c/23999/7/
JIRA: ARMBAND-185
Change-Id: I1dd2cec09aa1077a5779573bc54c01db93339ac9
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
7 files changed, 313 insertions, 291 deletions
diff --git a/.gitmodules b/.gitmodules index 684867af..b49fa711 100644 --- a/.gitmodules +++ b/.gitmodules @@ -18,8 +18,3 @@ url = https://gerrit.opnfv.org/gerrit/vswitchperf.git branch = master ignore = all -[submodule "packetary"] - path = upstream/packetary - url = https://github.com/openstack/packetary.git - branch = master - ignore = all diff --git a/armband-fuel-config.mk b/armband-fuel-config.mk index 42ff7aa4..591a23d9 100644 --- a/armband-fuel-config.mk +++ b/armband-fuel-config.mk @@ -47,9 +47,6 @@ export VSPERF_REPO := ${ARMBAND_BASE}/upstream/vswitchperf export VSPERF_BRANCH := opnfv-armband export VSPERF_CHANGE := ${A_OPNFV_TAG} -export PACKETARY_REPO := ${ARMBAND_BASE}/upstream/packetary -export PACKETARY_COMMIT := ${A_OPNFV_TAG} - # for the patches applying purposes (empty git config in docker build container) export GIT_COMMITTER_NAME?=Fuel OPNFV export GIT_COMMITTER_EMAIL?=fuel@opnfv.org diff --git a/patches/opnfv-fuel/upstream-backports/0003-f_repobuild-Use-packetary-to-build-partial-mirror.patch b/patches/opnfv-fuel/upstream-backports/0003-f_repobuild-Use-packetary-to-build-partial-mirror.patch index 4b630e0e..e7036e16 100644 --- a/patches/opnfv-fuel/upstream-backports/0003-f_repobuild-Use-packetary-to-build-partial-mirror.patch +++ b/patches/opnfv-fuel/upstream-backports/0003-f_repobuild-Use-packetary-to-build-partial-mirror.patch @@ -2,16 +2,14 @@ From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> Date: Wed, 6 Jul 2016 17:03:49 +0200 Subject: [PATCH] f_repobuild: Use packetary to build partial mirror -FIXME: DO NOT MERGE until [2, 3] are merged upstream & included here! -TODO: drop ALL fuel-mirror dependencies, for Fuel Newton compatibility. - Packetary [1] will replace fuel-mirror in upstream Fuel. fuel-mirror is using packetary under the hood already, so the transition should not be too hard. Adapt OPNFV Ubuntu partial mirror build: - obsolete opnfv-config.yaml (packetary uses CLI args for this); -- split old ubuntu.yaml into: +- new opnfv_config.yaml will hold old ubuntu.yaml data, split at + build runtime into: * mos_<arch>_mirror.yaml - MOS mirror cfg, only for deps analysis; * ubuntu_<arch>_mirror.yaml - Ubuntu mirror cfg, for pkgs download; * ubuntu_<arch>_packages.yaml - Ubuntu packages to download; @@ -23,37 +21,38 @@ is complete, in the scope of: * ubuntu_<arch>_packages_main.yaml - Ubuntu packages for [main]; New workflow (executed for EACH architecture): -1. Parse "fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml" - and write correponding sections to new yaml files described above; - - NOTE: Since fuel-mirror is going to be made obsolete, there are a - number of new ways we can gather the data currently passed via - fuel-mirror's "ubuntu.yaml" (e.g. use "mos-repos" from Fuel ISO - to determine MOS dependencies). - -2. Add OPNFV extra package dependencies (NEW) - Currently, this is empty / not used, but might be used to add - extra packages to the local mirror, like additional kernels, - bootloaders etc. +1. Collect bootstrap package deps from <fuel_bootstrap_cli.yaml>; +2. Collect all fixture release pkgs from fuel-web's <openstack.yaml>; +3. Add OPNFV extra pkg deps from "opnfv_config.yaml" + (based on old fuel-mirror) + + Currently, this is based on old fuel-mirror's + <fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml>, holding a + hardcoded list of package deps; plus eventual extra OPNFV packages, + like additional kernels, bootloaders etc. + Since we now inspect MOS repos for deps with packetary, this old + list should be easy to throw away as soon as we confirm each item + is directly required by another, already existing package in the + mirror. For now, keep the old list as safety net for missing deps. NOTE: Dependencies up to and including this group should also be satisfiable from Ubuntu "main" repo component only. -3. Append plugin dependencies to the package list; +4. Append plugin dependencies to the package list; -4. Run `packetary unresolved` for MOS repos, gathering MOS deps that +5. Run `packetary unresolved` for MOS repos, gathering MOS deps that should be fetched from Ubuntu. The new packages will be appended to the predefined list from old "ubuntu.yaml"; -5. Run `packetary clone` to download all required pkgs for the +6. Run `packetary clone` to download all required pkgs for the partial Ubuntu mirror. -6. IF repo component merging is disabled, run `packetary clone` only +7. IF repo component merging is disabled, run `packetary clone` only on Ubuntu[main] repo components to download any (still) missing dependencies for `debootstrap`, which only looks at this component, and not the whole mirror. -7. IF repo component merging is enabled, run first `dpkg-scanpackages` +8. IF repo component merging is enabled, run first `dpkg-scanpackages` to filter out old duplicate versions of packages, then run `packetary create` to merge all repo comps into a single "main". @@ -77,6 +76,14 @@ v4 -> v6: - fix duplicated logs; - remove `-q` flag for `create`; +v6 -> v7: +- rebase for Newton fuel_bootstrap_cli.yaml (s/trusty/xenial/); +- include OPNFV plugin deps in local Ubuntu mirror; +- add `vgabios` dependency, required for arm64 guest firmware; +- remove all fuel-mirror references; +- merge old fuel-mirror pkglist to OPNFV pkglist; +- update packetary commit to include upstream bugfixes [2, 3]; + NOTE: Without filtering out old package verisons using `dpkg-scanpackages`, bootstrap build cannot solve all deps. @@ -103,54 +110,53 @@ as the source partial mirror. JIRA: FUEL-218 JIRA: FUEL-223 JIRA: ARMBAND-169 +JIRA: ARMBAND-185 Change-Id: If2ee86f348b7683c83bfaf686baba4f1b8f555f0 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> --- - build/config.mk | 5 + + build/config.mk | 4 + build/f_isoroot/f_repobuild/.gitignore | 4 + - build/f_isoroot/f_repobuild/Makefile | 34 ++- - build/f_isoroot/f_repobuild/config.mk | 26 +++ - .../f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml | 14 +- + build/f_isoroot/f_repobuild/Makefile | 37 ++- + build/f_isoroot/f_repobuild/config.mk | 23 ++ + .../f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml | 22 +- + build/f_isoroot/f_repobuild/opnfv_config.yaml | 160 ++++++++++++ build/f_isoroot/f_repobuild/opnfv_mirror_conf.py | 57 ----- - build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py | 247 +++++++++++++++++++++ - build/f_isoroot/f_repobuild/opnfv_packages.yaml | 14 ++ - deploy/config/dea_base.yaml | 14 +- - 9 files changed, 313 insertions(+), 102 deletions(-) + build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py | 277 +++++++++++++++++++++ + deploy/config/dea_base.yaml | 16 +- + 9 files changed, 492 insertions(+), 108 deletions(-) create mode 100644 build/f_isoroot/f_repobuild/.gitignore create mode 100644 build/f_isoroot/f_repobuild/config.mk + create mode 100644 build/f_isoroot/f_repobuild/opnfv_config.yaml delete mode 100755 build/f_isoroot/f_repobuild/opnfv_mirror_conf.py create mode 100755 build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py - create mode 100644 build/f_isoroot/f_repobuild/opnfv_packages.yaml diff --git a/build/config.mk b/build/config.mk -index ab27858..a62b7e9 100644 +index 56317cb..5448c5e 100644 --- a/build/config.mk +++ b/build/config.mk -@@ -17,6 +17,11 @@ FUEL_MAIN_TAG = 9.0.1 - MOS_VERSION = 9.0 - OPENSTACK_VERSION = mitaka-9.0 - +@@ -17,4 +17,9 @@ FUEL_MAIN_TAG = master + FUEL_TRACK_REMOTES = ++ +# List of space-separated Ubuntu architectures supported with current build -+# Format: same as `dpkg-architecture -qDEB_HOST_ARCH1` ++# Format: same as `dpkg-architecture -qDEB_HOST_ARCH` +# NOTE: Currently only amd64 is supported by Fuel@OPNFV. Armband adds arm64. +export UBUNTU_ARCH ?= amd64 -+ - # FIXME(alav): Disable remote tracking for now, stick to submodule commits - FUEL_TRACK_REMOTES = + ############################################################################## + # Fuel components pinning / remote tracking; use submodules from f_repos diff --git a/build/f_isoroot/f_repobuild/.gitignore b/build/f_isoroot/f_repobuild/.gitignore new file mode 100644 -index 0000000..cfedefb +index 0000000..09baca8 --- /dev/null +++ b/build/f_isoroot/f_repobuild/.gitignore @@ -0,0 +1,4 @@ +packetary -+fuel-mirror ++fuel-web +nailgun +opnfv_config diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile -index ad1a934..5ab2650 100644 +index 8beb882..9784edf 100644 --- a/build/f_isoroot/f_repobuild/Makefile +++ b/build/f_isoroot/f_repobuild/Makefile @@ -1,7 +1,8 @@ @@ -163,7 +169,7 @@ index ad1a934..5ab2650 100644 # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at -@@ -10,41 +11,36 @@ +@@ -10,41 +11,37 @@ SHELL := /bin/bash TOP := $(shell pwd) @@ -183,10 +189,7 @@ index ad1a934..5ab2650 100644 - sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml - rm -Rf nailgun - sudo mkdir -p /var/www/nailgun -+ sudo apt-get install -y createrepo git libxml2-dev libxslt1-dev \ -+ python-dev zlib1g-dev -+ rm -Rf nailgun packetary opnfv_config && mkdir opnfv_config - ln -sf ${F_SUBMOD_DIR}/fuel-mirror fuel-mirror +- ln -sf ${F_SUBMOD_DIR}/fuel-mirror fuel-mirror - sudo pip install -U -r ./fuel-mirror/requirements.txt - sudo pip install ./fuel-mirror - sudo pip install ./fuel-mirror/contrib/fuel_mirror @@ -199,6 +202,11 @@ index ad1a934..5ab2650 100644 - mkdir -p nailgun/mirrors/ubuntu;\ - mv nailgun/mirrors$(MIRROR_UBUNTU_ROOT)* nailgun/mirrors/ubuntu;\ - [ "$(MIRROR_UBUNTU_ROOT)" != "/" ] && rm -rf nailgun/mirrors/$(TMP_ROOT_DIR);\ ++ sudo apt-get install -y createrepo git libxml2-dev libxslt1-dev \ ++ python-dev zlib1g-dev ++ rm -Rf nailgun packetary opnfv_config && mkdir opnfv_config ++ # We will analyze fuel-web's fixture files for package lists ++ ln -sf ${F_SUBMOD_DIR}/fuel-web fuel-web + git clone --quiet $(PACKETARY_REPO) + if [ -n $(PACKETARY_COMMIT) ]; then \ + git -C packetary checkout $(PACKETARY_COMMIT); \ @@ -220,10 +228,10 @@ index ad1a934..5ab2650 100644 release:nailgun diff --git a/build/f_isoroot/f_repobuild/config.mk b/build/f_isoroot/f_repobuild/config.mk new file mode 100644 -index 0000000..79e7d1a +index 0000000..9852093 --- /dev/null +++ b/build/f_isoroot/f_repobuild/config.mk -@@ -0,0 +1,26 @@ +@@ -0,0 +1,23 @@ +############################################################################## +# Copyright (c) 2016 Ericsson AB, Enea AB and others. +# stefan.k.berg@ericsson.com @@ -236,11 +244,8 @@ index 0000000..79e7d1a +############################################################################## + +# Use a recent master commit, since tags/branches are not yet mature -+# FIXME(armband): Update upstream commit ref once [1, 2] are merged -+# [1] https://review.openstack.org/#/c/392937/ -+# [2] https://review.openstack.org/#/c/392936/ +export PACKETARY_REPO?=https://github.com/openstack/packetary -+export PACKETARY_COMMIT?=c46465c3255a9f5e59a05b8701e06054df39f32f ++export PACKETARY_COMMIT?=3021c001561b4baef352bf0b881d064ac687cc20 + +# arm64 Ubuntu mirror is separated from archive.ubuntu.com +export MIRROR_UBUNTU_URL_arm64=http://ports.ubuntu.com/ubuntu-ports/ @@ -251,10 +256,10 @@ index 0000000..79e7d1a +# - fuel_bootstrap_cli.yaml +export MIRROR_UBUNTU_MERGE=true diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml -index 19dea1c..52786fb 100644 +index fcf4257..7cbea29 100644 --- a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml +++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml -@@ -48,23 +48,11 @@ +@@ -47,22 +47,10 @@ https_proxy: "" repos: - name: ubuntu @@ -263,7 +268,7 @@ index 19dea1c..52786fb 100644 uri: "http://127.0.0.1:8080/mirrors/ubuntu" priority: suite: trusty - type: deb +- type: deb - - name: ubuntu-updates - section: "main universe multiverse" - uri: "http://127.0.0.1:8080/mirrors/ubuntu" @@ -275,10 +280,175 @@ index 19dea1c..52786fb 100644 - uri: "http://127.0.0.1:8080/mirrors/ubuntu" - priority: - suite: trusty-security -- type: deb + type: deb - name: mos section: "main restricted" - uri: "http://127.0.0.1:8080/ubuntu/x86_64" +diff --git a/build/f_isoroot/f_repobuild/opnfv_config.yaml b/build/f_isoroot/f_repobuild/opnfv_config.yaml +new file mode 100644 +index 0000000..bb78747 +--- /dev/null ++++ b/build/f_isoroot/f_repobuild/opnfv_config.yaml +@@ -0,0 +1,160 @@ ++############################################################################## ++# Copyright (c) 2016 Enea AB and others. ++# Alexandru.Avadanii@enea.com ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++ ++# REPO definitions ++ ++# Based on old fuel-mirror data [1], adapted for OPNFV and packetary. ++# NOTE: 'uri' field will be added by opnfv_mirror_ubuntu.py from ENV. ++# [1] https://github.com/openstack/fuel-mirror/blob/stable/mitaka/ ++# contrib/fuel_mirror/data/ubuntu.yaml ++ ++# Main is a required parameter which defines what repository will be used ++# for images creation and that mirror should contain all packages for minimal ++# system creation. ++groups: ++ ubuntu: ++ - name: "ubuntu" ++ main: true ++ suite: "trusty" ++ section: ++ - "main" ++ - "multiverse" ++ - "restricted" ++ - "universe" ++ type: "deb" ++ priority: null ++ ++ - name: "ubuntu-updates" ++ suite: "trusty-updates" ++ section: ++ - "main" ++ - "multiverse" ++ - "restricted" ++ - "universe" ++ type: "deb" ++ priority: null ++ ++ - name: "ubuntu-security" ++ suite: "trusty-security" ++ section: ++ - "main" ++ - "multiverse" ++ - "restricted" ++ - "universe" ++ type: "deb" ++ priority: null ++ ++ mos: ++ - name: "mos" ++ suite: "mos$mos_version" ++ section: ++ - "main" ++ - "restricted" ++ type: "deb" ++ priority: 1000 ++ ++ - name: "mos-updates" ++ suite: "mos$mos_version-updates" ++ section: ++ - "main" ++ - "restricted" ++ type: "deb" ++ priority: 1000 ++ ++ - name: "mos-security" ++ suite: "mos$mos_version-security" ++ section: ++ - "main" ++ - "restricted" ++ type: "deb" ++ priority: 1000 ++ ++ - name: "mos-holdback" ++ suite: "mos$mos_version-holdback" ++ section: ++ - "main" ++ - "restricted" ++ type: "deb" ++ priority: 1000 ++ ++# PACKAGES ++ ++# This section lists packages that should be present in the local Ubuntu mirror, ++# but are not direct dependencies of any other packages from MOS or Ubuntu. ++# e.g.: additional kernels, bootloaders etc. ++ ++packages: ++ - name: "vgabios" ++# Packages are required to build bootstrap images for a system. ++# The mirror should contiain such packages in addition to local mirror. ++ - name: "acpi-support" ++ - name: "anacron" ++ - name: "aptitude" ++ - name: "atop" ++ - name: "acct" ++ - name: "bash-completion" ++ - name: "bc" ++ - name: "build-essential" ++ - name: "cloud-init" ++ - name: "conntrackd" ++ - name: "cpu-checker" ++ - name: "cpufrequtils" ++ - name: "debconf-utils" ++ - name: "devscripts" ++ - name: "fping" ++ - name: "git" ++ - name: "grub-pc" ++ - name: "htop" ++ - name: "hwloc" ++ - name: "ifenslave" ++ - name: "iperf" ++ - name: "iptables-persistent" ++ - name: "irqbalance" ++ - name: "language-pack-en" ++ - name: "libapache2-mod-fastcgi" ++ - name: "libnss3-tools" ++ - name: "linux-firmware-nonfree" ++ - name: "linux-headers-generic-lts-trusty" ++ - name: "linux-headers-generic-lts-xenial" ++ - name: "linux-image-generic-lts-trusty" ++ - name: "linux-image-generic-lts-xenial" ++ - name: "live-boot" ++ - name: "livecd-rootfs" ++ - name: "mc" ++ - name: "memcached" ++ - name: "monit" ++ - name: "msmtp-mta" ++ - name: "multipath-tools" ++ - name: "multipath-tools-boot" ++ - name: "nginx" ++ - name: "ntp" ++ - name: "openssh-server" ++ - name: "percona-toolkit" ++ - name: "percona-xtrabackup" ++ - name: "pm-utils" ++ - name: "puppet" ++ - name: "python-lesscpy" ++ - name: "python-pip" ++ - name: "rsyslog-gnutls" ++ - name: "rsyslog-relp" ++ - name: "screen" ++ - name: "squashfs-tools" ++ - name: "swift-plugin-s3" ++ - name: "sysfsutils" ++ - name: "sysstat" ++ - name: "telnet" ++ - name: "tmux" ++ - name: "traceroute" ++ - name: "ubuntu-standard" ++ - name: "vim" ++ - name: "virt-what" ++ - name: "xinetd" ++ - name: "xmlstarlet" ++ - name: "tftpd-hpa" ++ - name: "syslinux" diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py deleted file mode 100755 index 1c7eb59..0000000 @@ -344,10 +514,10 @@ index 1c7eb59..0000000 - outfile.write( yaml.safe_dump(pattern, default_flow_style=False) ) diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py new file mode 100755 -index 0000000..da88445 +index 0000000..5b59182 --- /dev/null +++ b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py -@@ -0,0 +1,247 @@ +@@ -0,0 +1,277 @@ +#!/usr/bin/env python +############################################################################## +# Copyright (c) 2015,2016 Ericsson AB, Mirantis Inc., Enea AB and others. @@ -374,20 +544,20 @@ index 0000000..da88445 +# we can mitigate the issue by "merging" all repo-components into a single +# "main". +############################################################################## -+# Mirror build steps: -+# 1. Parse old <fuel-mirror> package list ("ubuntu.yaml"); -+# 2. Parse new "opnfv_packages.yaml" list of additional packages; -+# 3. Inherit enviroment variable(s) for mirror URLs, paths etc. ++# Mirror build steps (for EACH architecture in UBUNTU_ARCH): ++# 1. Collect bootstrap package deps from <fuel_bootstrap_cli.yaml>; ++# 2. Collect all fixture release packages from fuel-web's <openstack.yaml>; ++# 3. Parse new "opnfv_config.yaml" list of packages (from old fuel-mirror); ++# 4. Inherit enviroment variable(s) for mirror URLs, paths etc. +# - Allow arch-specific overrides for each env var; -+# 4. For each architecture in UBUNTU_ARCH: -+# 4.1. Mirror config is defined based on common config + OPNFV overrides; -+# - Convert old configuration format to packetary style where needed; -+# 4.2. Package lists are defined based on common config + OPNFV deps; -+# - Keep track of "main" packages separately, required by debootstrap; -+# 4.3. Clone/update all mirror components; -+# 4.4. IF mirror merging is disabled OR workaround for ifupdown (see below): -+# - Clone/update "main" mirror component (fix missing debootstrap deps); -+# 5. IF mirror merging is enabled: ++# 5. Mirror config is defined based on common config + OPNFV overrides; ++# - Convert old configuration format to packetary style where needed; ++# 6. Package lists are defined based on common config + OPNFV deps; ++# - Keep track of "main" packages separately, required by debootstrap; ++# 7. Clone/update all mirror components; ++# 8. IF mirror merging is disabled OR workaround for ifupdown (see below): ++# - Clone/update "main" mirror component (fix missing debootstrap deps); ++# 9. IF mirror merging is enabled: +# - Use `dpkg-scanpackages` to filter out old versions of duplicate pkgs; +# - Run `packetary create` on the set of downloaded packages, merging +# them on the fly into a single-component mirror; @@ -420,8 +590,7 @@ index 0000000..da88445 + +# Determine missing package dependecies for a mirror defition +def get_unres_pkgs(arch, cfg_mirror): -+ unresolved = dict() -+ unresolved['packages'] = list() ++ unresolved_pkgs = list() + packetary_output = StringIO() + with captureStdOut(packetary_output): + main('unresolved -a {0} -r {1} -c name version --sep ;' @@ -430,9 +599,31 @@ index 0000000..da88445 + if dep_pkg.startswith('#'): + continue + dep = dep_pkg.split(';') -+ unresolved['packages'] += [ {'name': dep[0], 'version': dep[1]} ] ++ unresolved_pkgs += [ {'name': dep[0], 'version': dep[1]} ] + force_logger_reload() -+ return unresolved ++ return unresolved_pkgs ++ ++# Package list conversion from `old fuel-mirror` to `packetary` style ++def from_legacy_pkglist(legacy_pkglist): ++ pkglist = list() ++ for pkg in legacy_pkglist: ++ pkglist += [ {'name': pkg} ] ++ return pkglist ++ ++def to_legacy_pkglist(pkglist): ++ legacy_pkglist = list() ++ for pkg in pkglist: ++ legacy_pkglist.append(pkg['name']) ++ return legacy_pkglist ++ ++def legacy_diff(base_pkglist, new_pkglist, requester, arch): ++ diff_set = set(new_pkglist) ++ if base_pkglist: ++ diff_set -= set(base_pkglist) ++ if diff_set: ++ print(' * {0} requires new packages for arch [{1}]: {2}' ++ .format(requester, arch, ', '.join(diff_set))) ++ return list(diff_set) + +# Create single-component local repo (one arch per call) +def do_local_repo(arch, cfg_repo, cfg_packages_paths): @@ -474,17 +665,19 @@ index 0000000..da88445 + +# Arch-indepedent configuration (old fuel-mirror + OPNFV extra packages) +cfg_dir = 'opnfv_config' -+cfg_p_opnfv = 'opnfv_packages.yaml' ++cfg_p_opnfv = 'opnfv_config.yaml' +mos_version = get_env('MOS_VERSION') +ubuntu_arch = get_env('UBUNTU_ARCH') +mirror_ubuntu_path = get_env('MIRROR_UBUNTU_OPNFV_PATH') +mirror_ubuntu_tmp_path = '{0}.tmp'.format(mirror_ubuntu_path) +mirror_ubuntu_merge = get_env('MIRROR_UBUNTU_MERGE') +cfg_mm_ubuntu = '{0}/ubuntu_mirror_local.yaml'.format(cfg_dir) -+pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read() -+pattern = yaml.load(pattern_file) -+opnfv_pkgs_yaml = open(cfg_p_opnfv).read() -+opnfv_pkgs = yaml.load(opnfv_pkgs_yaml) ++fuel_bootstrap_cli_file = open('fuel_bootstrap_cli.yaml').read() ++fuel_bootstrap_cli = yaml.load(fuel_bootstrap_cli_file) ++fixture_file = open('fuel-web/nailgun/nailgun/fixtures/openstack.yaml').read() ++fixture = yaml.load(fixture_file) ++opnfv_cfg_yaml = open(cfg_p_opnfv).read() ++opnfv_cfg = yaml.load(opnfv_cfg_yaml) + +# FIXME: Packetary solves missing dependecies by also accepting +# different packages that provide the same name (e.g. "ifupdown" dependency @@ -514,29 +707,27 @@ index 0000000..da88445 + arch_group_mos = 'mos_{0}'.format(arch) + arch_group_ubuntu = 'ubuntu_{0}'.format(arch) + arch_packages = 'packages_{0}'.format(arch) -+ pattern['groups'][arch_group_mos] = copy.deepcopy(pattern['groups']['mos']) -+ pattern['groups'][arch_group_ubuntu] = copy.deepcopy(pattern['groups']['ubuntu']) -+ pattern[arch_packages] = pattern['packages'] ++ opnfv_cfg['groups'][arch_group_mos] = copy.deepcopy(opnfv_cfg['groups']['mos']) ++ opnfv_cfg['groups'][arch_group_ubuntu] = copy.deepcopy(opnfv_cfg['groups']['ubuntu']) ++ opnfv_cfg[arch_packages] = opnfv_cfg['packages'] + + # Mirror config update & conversion to packetary input + group_main_ubuntu = dict() -+ for group in pattern['groups'][arch_group_mos]: ++ for group in opnfv_cfg['groups'][arch_group_mos]: + group['uri'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) + group['suite'] = group['suite'].replace('$mos_version', mos_version) -+ group['section'] = group['section'].split() -+ for group in pattern['groups'][arch_group_ubuntu]: ++ for group in opnfv_cfg['groups'][arch_group_ubuntu]: + group['uri'] = mirror_ubuntu + # FIXME: At `create`, packetary insists on copying all pkgs to dest dir, + # so configure it for another dir, which will replace the orig at the end. + group['path'] = mirror_ubuntu_tmp_path -+ group['section'] = group['section'].split() + if not group_main_ubuntu and 'main' in group: + group_main_ubuntu = [ copy.deepcopy(group) ] + group_main_ubuntu[0]['section'] = [ 'main' ] + + # Mirror config dump: MOS (for dep resolution), Ubuntu, Ubuntu[main] -+ write_cfg_file(cfg_m_mos, pattern['groups'][arch_group_mos]) -+ write_cfg_file(cfg_m_ubuntu, pattern['groups'][arch_group_ubuntu]) ++ write_cfg_file(cfg_m_mos, opnfv_cfg['groups'][arch_group_mos]) ++ write_cfg_file(cfg_m_ubuntu, opnfv_cfg['groups'][arch_group_ubuntu]) + if mirror_ubuntu_resolve_main_deps or mirror_ubuntu_merge is None: + write_cfg_file(cfg_m_ubuntu_main, group_main_ubuntu) + if mirror_ubuntu_merge is not None: @@ -546,14 +737,27 @@ index 0000000..da88445 + group_main_ubuntu[0]['uri'] = mirror_ubuntu_path + write_cfg_file(cfg_mm_ubuntu, group_main_ubuntu[0]) + -+ # Package list conversion from `old fuel-mirror` to `packetary` style + OPNFV -+ unresolved_pkgs = dict() -+ unresolved_pkgs['packages'] = list() -+ unresolved_pkgs['mandatory'] = 'exact' -+ if opnfv_pkgs['packages'] is not None: -+ unresolved_pkgs['packages'] = opnfv_pkgs['packages'] -+ for pkg in pattern['packages']: -+ unresolved_pkgs['packages'] += [ {'name': pkg} ] ++ # Collect package dependencies from: ++ ## 1. fuel_bootstrap_cli.yaml (bootstrap image additional packages) ++ legacy_unresolved = legacy_diff(None, fuel_bootstrap_cli['packages'] + [ ++ fuel_bootstrap_cli['kernel_flavor'], ++ fuel_bootstrap_cli['kernel_flavor'].replace('image', 'headers')], ++ 'Bootstrap', arch) ++ ## 2. openstack.yaml fixture definition (default target image packages) ++ for release in fixture: ++ editable = release['fields']['attributes_metadata']['editable'] ++ if 'provision' in editable and 'packages' in editable['provision']: ++ release_pkgs = editable['provision']['packages']['value'].split() ++ legacy_unresolved += legacy_diff(legacy_unresolved, ++ release_pkgs, 'Release {0}'.format(release['fields']['name']), arch) ++ ## 3. OPNFV additional packages (includes old fuel-mirror ubuntu.yaml pkgs) ++ unresolved = dict() ++ unresolved['mandatory'] = 'exact' ++ unresolved['packages'] = from_legacy_pkglist(legacy_unresolved) ++ if 'packages' in opnfv_cfg: ++ legacy_diff(legacy_unresolved, to_legacy_pkglist(opnfv_cfg['packages']), ++ 'OPNFV config', arch) ++ unresolved['packages'] += opnfv_cfg['packages'] + + # OPNFV plugins dependency resolution + for plugin in plugins.split(): @@ -561,22 +765,18 @@ index 0000000..da88445 + if os.path.isfile(path): + f = open(path).read() + plugin_yaml = yaml.load(f) -+ plugin_set = set(plugin_yaml['packages']) -+ main_set = set(pattern['packages']) -+ new_packages = plugin_set - main_set -+ print('Plugin {0} requires new packages for arch [{1}]: {2}' -+ .format(plugin, arch, ', '.join(new_packages))) -+ for pkg in new_packages: -+ unresolved_pkgs['packages'] += [ {'name': pkg} ] ++ new_packages = legacy_diff(to_legacy_pkglist(unresolved['packages']), ++ plugin_yaml['packages'], 'Plugin {0}'.format(plugin), arch) ++ unresolved['packages'] += from_legacy_pkglist(new_packages) + + # Package list (reduced, i.e. no MOS deps, but with OPNFV plugin deps) + # FIXME: This helps work around packetary solving main deps from universe + if mirror_ubuntu_resolve_main_deps or mirror_ubuntu_merge is None: -+ write_cfg_file(cfg_p_ubuntu_main, unresolved_pkgs) ++ write_cfg_file(cfg_p_ubuntu_main, unresolved) + + # Mirror package list (full, including MOS/OPNFV plugin deps) -+ unresolved_pkgs['packages'] += get_unres_pkgs(arch, cfg_m_mos)['packages'] -+ write_cfg_file(cfg_p_ubuntu, unresolved_pkgs) ++ unresolved['packages'] += get_unres_pkgs(arch, cfg_m_mos) ++ write_cfg_file(cfg_p_ubuntu, unresolved) + do_partial_mirror(arch, cfg_m_ubuntu, cfg_p_ubuntu) + if mirror_ubuntu_resolve_main_deps or mirror_ubuntu_merge is None: + # Ubuntu[main] must be evaluated after Ubuntu @@ -595,32 +795,11 @@ index 0000000..da88445 + .format(arch, mirror_ubuntu_tmp_path, cfg_pp_ubuntu)) + do_local_repo(arch, cfg_mm_ubuntu, cfg_pp_ubuntu) + shutil.rmtree(mirror_ubuntu_tmp_path) -diff --git a/build/f_isoroot/f_repobuild/opnfv_packages.yaml b/build/f_isoroot/f_repobuild/opnfv_packages.yaml -new file mode 100644 -index 0000000..3f5c59f ---- /dev/null -+++ b/build/f_isoroot/f_repobuild/opnfv_packages.yaml -@@ -0,0 +1,15 @@ -+############################################################################## -+# Copyright (c) 2016 Enea AB and others. -+# Alexandru.Avadanii@enea.com -+# All rights reserved. This program and the accompanying materials -+# are made available under the terms of the Apache License, Version 2.0 -+# which accompanies this distribution, and is available at -+# http://www.apache.org/licenses/LICENSE-2.0 -+############################################################################## -+ -+# This file lists packages that should be present in the local Ubuntu mirror, -+# but are not direct dependencies of any other packages from MOS or Ubuntu. -+# e.g.: additional kernels, bootloaders etc. -+ -+packages: -+ - name: vgabios diff --git a/deploy/config/dea_base.yaml b/deploy/config/dea_base.yaml -index f5a03e2..370a75a 100644 +index ce81994..2520cf5 100644 --- a/deploy/config/dea_base.yaml +++ b/deploy/config/dea_base.yaml -@@ -633,22 +633,10 @@ settings: +@@ -622,22 +622,10 @@ settings: value: - name: ubuntu priority: null @@ -628,7 +807,7 @@ index f5a03e2..370a75a 100644 + section: main suite: trusty type: deb - uri: http://10.20.0.2:8080/mirrors/ubuntu/ +- uri: http://10.20.0.2:8080/mirrors/ubuntu/ - - name: ubuntu-updates - priority: null - section: main universe multiverse @@ -640,7 +819,7 @@ index f5a03e2..370a75a 100644 - section: main universe multiverse - suite: trusty-security - type: deb -- uri: http://10.20.0.2:8080/mirrors/ubuntu/ + uri: http://10.20.0.2:8080/mirrors/ubuntu/ - name: mos priority: 1050 section: main restricted diff --git a/patches/packetary/0001-deb_driver-Translate-repository.architecture.patch b/patches/packetary/0001-deb_driver-Translate-repository.architecture.patch deleted file mode 100644 index 18f20015..00000000 --- a/patches/packetary/0001-deb_driver-Translate-repository.architecture.patch +++ /dev/null @@ -1,30 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Thu, 27 Oct 2016 22:24:28 +0200 -Subject: [PATCH] deb_driver: Translate repository.architecture - -When generating repository metadata, the Release file gets the -architecture list in an untranslated format, ending up with -'x86_64' instead of 'amd64'. - -Closes-bug: 1638628 - -Change-Id: I8471db6d54157175ff275c969ed5195d4616fa0f -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - packetary/drivers/deb_driver.py | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/packetary/drivers/deb_driver.py b/packetary/drivers/deb_driver.py -index ece9fa1..929bdca 100644 ---- a/packetary/drivers/deb_driver.py -+++ b/packetary/drivers/deb_driver.py -@@ -409,7 +409,8 @@ class DebRepositoryDriver(RepositoryDriverBase): - release.setdefault("Description", "The packages repository.") - - keys = ("Architectures", "Components") -- values = (repository.architecture, repository.section[1]) -+ values = (_ARCHITECTURES[repository.architecture], -+ repository.section[1]) - for key, value in six.moves.zip(keys, values): - if key in release: - release[key] = utils.append_token_to_string( diff --git a/patches/packetary/0002-clone-Create-metadata-for-empty-components.patch b/patches/packetary/0002-clone-Create-metadata-for-empty-components.patch deleted file mode 100644 index 70d75fee..00000000 --- a/patches/packetary/0002-clone-Create-metadata-for-empty-components.patch +++ /dev/null @@ -1,43 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Tue, 1 Nov 2016 23:01:58 +0100 -Subject: [PATCH] clone: Create metadata for empty components - -In certain scenarios, mirror components (e.g. trusty-security) -are present in both packetary configuration input, and in target -system's apt source definitions, but contain no packages of interest. - -For such repository/requirements combinations, packetary currently -skips creating metadata (Release, Packages) for those components, -which leads to the partial mirror missing some critical files for apt, -(an empty file would be enough). - -e.g.: Using packetary to create a partial Ubuntu mirror, then trying -to build a bootstrap image from the new mirror leads to: - -W: Failed to fetch http://127.0.0.1:8080/mirrors/ubuntu/dists/\ - trusty-security/multiverse/binary-amd64/Packages 404 - -Closes-bug: 1638631 - -Change-Id: I850b43d5b4d8742d99e9a5702cc9ad4de881a401 -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - packetary/api/repositories.py | 5 +++++ - 1 file changed, 5 insertions(+) - -diff --git a/packetary/api/repositories.py b/packetary/api/repositories.py -index ec8c54c..2c72e29 100644 ---- a/packetary/api/repositories.py -+++ b/packetary/api/repositories.py -@@ -121,6 +121,11 @@ class RepositoryApi(object): - for pkg in all_packages: - package_groups[pkg.repository].add(pkg) - -+ # Make sure we create metadata for all repos, even if empty -+ for repo in repositories: -+ if repo not in package_groups: -+ package_groups[repo] = set() -+ - stat = CopyStatistics() - mirrors = defaultdict(set) - options = options or self.CopyOptions() diff --git a/patches/packetary/0003-AArch64-support-api-cli-controllers-drivers.patch b/patches/packetary/0003-AArch64-support-api-cli-controllers-drivers.patch deleted file mode 100644 index da01bc33..00000000 --- a/patches/packetary/0003-AArch64-support-api-cli-controllers-drivers.patch +++ /dev/null @@ -1,76 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Thu, 27 Oct 2016 21:51:10 +0200 -Subject: [PATCH] AArch64 support: api, cli, controllers, drivers - -Enable new architecture 'aarch64' ('arm64' on Ubuntu). -Tested with DEB driver. - -Closes-bug: 1638635 - -Change-Id: I9761322020837186c109c18e849128791ab909d8 -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - packetary/api/repositories.py | 3 ++- - packetary/cli/commands/base.py | 2 +- - packetary/controllers/repository.py | 3 ++- - packetary/drivers/deb_driver.py | 2 ++ - 4 files changed, 7 insertions(+), 3 deletions(-) - -diff --git a/packetary/api/repositories.py b/packetary/api/repositories.py -index ec8c54c..d49d8d0 100644 ---- a/packetary/api/repositories.py -+++ b/packetary/api/repositories.py -@@ -73,7 +73,8 @@ class RepositoryApi(object): - - :param config: the configuration - :param repotype: the kind of repository(deb, yum, etc) -- :param repoarch: the architecture of repository (x86_64 or i386) -+ :param repoarch: the architecture of repository -+ (x86_64, i386 or aarch64) - """ - context = config if isinstance(config, Context) else Context(config) - return cls(RepositoryController.load(context, repotype, repoarch)) -diff --git a/packetary/cli/commands/base.py b/packetary/cli/commands/base.py -index 83c2f38..c2cc497 100644 ---- a/packetary/cli/commands/base.py -+++ b/packetary/cli/commands/base.py -@@ -56,7 +56,7 @@ class BaseRepoCommand(BaseCommand): - '-a', - '--arch', - type=str, -- choices=["x86_64", "i386"], -+ choices=["x86_64", "i386", "aarch64"], - metavar='ARCHITECTURE', - default="x86_64", - help='The target architecture.') -diff --git a/packetary/controllers/repository.py b/packetary/controllers/repository.py -index 921a5c1..e6cbc9b 100644 ---- a/packetary/controllers/repository.py -+++ b/packetary/controllers/repository.py -@@ -45,7 +45,8 @@ class RepositoryController(object): - - :param context: the context - :param driver_name: the name of required driver -- :param repoarch: the architecture of repository (x86_64 or i386) -+ :param repoarch: the architecture of repository -+ (x86_64, i386 or aarch64) - """ - if cls._drivers is None: - cls._drivers = stevedore.ExtensionManager( -diff --git a/packetary/drivers/deb_driver.py b/packetary/drivers/deb_driver.py -index ece9fa1..0df1aa4 100644 ---- a/packetary/drivers/deb_driver.py -+++ b/packetary/drivers/deb_driver.py -@@ -48,10 +48,12 @@ _OPERATORS_MAPPING = { - } - - _ARCHITECTURES = { -+ "aarch64": "arm64", - "x86_64": "amd64", - "i386": "i386", - "source": "Source", - "amd64": "x86_64", -+ "arm64": "aarch64", - } - - _PRIORITIES = { diff --git a/upstream/packetary b/upstream/packetary deleted file mode 160000 -Subproject c46465c3255a9f5e59a05b8701e06054df39f32 |