diff options
Diffstat (limited to 'build/f_isoroot/f_repobuild')
| -rw-r--r-- | build/f_isoroot/f_repobuild/.gitignore | 4 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/Makefile | 88 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/config.mk | 23 | ||||
| -rw-r--r-- | build/f_isoroot/f_repobuild/opnfv_config.yaml | 174 | ||||
| -rwxr-xr-x | build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py | 280 | ||||
| -rwxr-xr-x | build/f_isoroot/f_repobuild/select_ubuntu_repo.sh | 103 |
6 files changed, 0 insertions, 672 deletions
diff --git a/build/f_isoroot/f_repobuild/.gitignore b/build/f_isoroot/f_repobuild/.gitignore deleted file mode 100644 index 09baca85d..000000000 --- a/build/f_isoroot/f_repobuild/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -packetary -fuel-web -nailgun -opnfv_config diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile deleted file mode 100644 index c61d1adf5..000000000 --- a/build/f_isoroot/f_repobuild/Makefile +++ /dev/null @@ -1,88 +0,0 @@ -############################################################################## -# Copyright (c) 2015,2016 Ericsson AB, Enea AB and others. -# stefan.k.berg@ericsson.com -# jonas.bjurel@ericsson.com -# Alexandru.Avadanii@enea.com -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -SHELL := /bin/bash -TOP := $(shell pwd) - -include ../../config.mk -include config.mk - -export MOS_VERSION -export MIRROR_UBUNTU_OPNFV_PATH:=$(TOP)/nailgun/mirrors/ubuntu - -.PHONY: all -all: nailgun - -nailgun: - sudo apt-get install -y createrepo git libxml2-dev libxslt1-dev \ - python-dev zlib1g-dev - rm -Rf nailgun packetary opnfv_config && mkdir opnfv_config - # We will analyze fuel-web's fixture files for package lists - ln -sf ${F_SUBMOD_DIR}/fuel-web fuel-web - # Same for fuel-agent's bootstrap package list - ln -sf ${F_SUBMOD_DIR}/fuel-agent fuel-agent - git clone --quiet $(PACKETARY_REPO) - if [ -n $(PACKETARY_COMMIT) ]; then \ - git -C packetary checkout $(PACKETARY_COMMIT); \ - fi - sudo pip install -U -r ./packetary/requirements.txt - sudo pip install -U ./packetary - # Handle config and mirror build in one place - ./opnfv_mirror_ubuntu.py - # Store artifact in cache straight away if caching is enabled - # (no .cacheid will be present unless this is a cached build) - test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 - -.PHONY: clean -clean: - @rm -rf ../release/opnfv/nailgun nailgun packetary fuel-web opnfv_config - -.PHONY: release -release:nailgun - @rm -Rf ../release/opnfv/nailgun - @mkdir -p ../release/opnfv - @cp -Rp nailgun ../release/opnfv/nailgun - -############################################################################ -# Cache operations - only used when building through ci/build.sh -############################################################################ - -# Create a unique hash to be used for getting and putting cache, based on: -# - Year and week (causing the cache to be rebuilt weekly) -# - The contents of this Makefile + all sh,mk,py,yaml files in CWD -# - repo packages fingerprint -# - repo arch list -.cacheid: - date +"Repocache %G%V" > .cachedata - sha1sum Makefile *.{sh,mk,py,yaml} >> .cachedata - $(CACHETOOL) packages >> .cachedata - echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata - cat .cachedata | $(CACHETOOL) getid > .cacheid - -# Clean local data related to caching - called prior to ordinary build -.PHONY: clean-cache -clean-cache: clean - rm -f .cachedata .cacheid - -# Try to download cache - called prior to ordinary build -.PHONY: get-cache -get-cache: .cacheid - @if $(CACHETOOL) check $(shell cat .cacheid); then \ - $(CACHETOOL) get $(shell cat .cacheid) | tar xf -;\ - else \ - echo "No cache item found for $(shell cat .cacheid)" ;\ - exit 0;\ - fi - -# Store cache if not already stored - called after ordinary build -.PHONY: put-cache -put-cache: .cacheid - @tar cf - nailgun | $(CACHETOOL) put $(shell cat .cacheid) diff --git a/build/f_isoroot/f_repobuild/config.mk b/build/f_isoroot/f_repobuild/config.mk deleted file mode 100644 index 68101d6ec..000000000 --- a/build/f_isoroot/f_repobuild/config.mk +++ /dev/null @@ -1,23 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Ericsson AB, Enea AB and others. -# stefan.k.berg@ericsson.com -# jonas.bjurel@ericsson.com -# Alexandru.Avadanii@enea.com -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -# Use a recent master commit, since tags/branches are not yet mature -export PACKETARY_REPO?=https://github.com/openstack/packetary -export PACKETARY_COMMIT?=c3c2069e2ec46b35eb342386efbf366bb6340027 - -# arm64 Ubuntu mirror is separated from archive.ubuntu.com -export MIRROR_UBUNTU_URL_arm64=http://ports.ubuntu.com/ubuntu-ports/ -export MIRROR_UBUNTU_ROOT_arm64=ubuntu-ports - -# Merge all local mirror repo components/section into single "main" -# NOTE: When changing this, make sure to also update all consumer config, like: -# - fuel-menu/fuelmenu/settings.yaml -export MIRROR_UBUNTU_MERGE=true diff --git a/build/f_isoroot/f_repobuild/opnfv_config.yaml b/build/f_isoroot/f_repobuild/opnfv_config.yaml deleted file mode 100644 index bf3913cc4..000000000 --- a/build/f_isoroot/f_repobuild/opnfv_config.yaml +++ /dev/null @@ -1,174 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Enea AB and others. -# Alexandru.Avadanii@enea.com -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -# REPO definitions - -# Based on old fuel-mirror data [1], adapted for OPNFV and packetary. -# NOTE: 'uri' field will be added by opnfv_mirror_ubuntu.py from ENV. -# [1] https://github.com/openstack/fuel-mirror/blob/stable/mitaka/ -# contrib/fuel_mirror/data/ubuntu.yaml - -# Main is a required parameter which defines what repository will be used -# for images creation and that mirror should contain all packages for minimal -# system creation. -groups: - ubuntu: - - name: "ubuntu" - main: true - suite: "xenial" - section: - - "main" - - "multiverse" - - "restricted" - - "universe" - type: "deb" - priority: null - - - name: "ubuntu-updates" - suite: "xenial-updates" - section: - - "main" - - "multiverse" - - "restricted" - - "universe" - type: "deb" - priority: null - - - name: "ubuntu-security" - suite: "xenial-security" - section: - - "main" - - "multiverse" - - "restricted" - - "universe" - type: "deb" - priority: null - - mos: - - name: "mos" - suite: "mos$mos_version" - section: - - "main" - - "restricted" - type: "deb" - priority: 1000 - - - name: "mos-updates" - suite: "mos$mos_version-updates" - section: - - "main" - - "restricted" - type: "deb" - priority: 1000 - - - name: "mos-security" - suite: "mos$mos_version-security" - section: - - "main" - - "restricted" - type: "deb" - priority: 1000 - - - name: "mos-holdback" - suite: "mos$mos_version-holdback" - section: - - "main" - - "restricted" - type: "deb" - priority: 1000 - -# PACKAGES - -# This section lists packages that should be present in the local Ubuntu mirror, -# but are not direct dependencies of any other packages from MOS or Ubuntu. -# e.g.: additional kernels, bootloaders etc. - -packages: - - name: "vgabios" -# Packages are required to build bootstrap images for a system. -# The mirror should contiain such packages in addition to local mirror. - - name: "acpi-support" - - name: "anacron" - - name: "aptitude" - - name: "atop" - - name: "acct" - - name: "bash-completion" - - name: "bc" - - name: "build-essential" - - name: "ceph" - - name: "cloud-init" - - name: "conntrackd" - - name: "cpu-checker" - - name: "cpufrequtils" - - name: "debconf-utils" - - name: "devscripts" - - name: "fping" - - name: "galera-3" - - name: "git" - - name: "grub-pc" - - name: "htop" - - name: "hwloc" - - name: "ifenslave" - - name: "iperf" - - name: "iptables-persistent" - - name: "irqbalance" - - name: "language-pack-en" - - name: "libapache2-mod-fastcgi" - - name: "libnss3-tools" - - name: "linux-headers-generic-lts-xenial" - - name: "linux-image-generic-lts-xenial" - - name: "live-boot" - - name: "livecd-rootfs" - - name: "mc" - - name: "memcached" - - name: "mongodb-server" - - name: "monit" - - name: "msmtp-mta" - - name: "multipath-tools" - - name: "multipath-tools-boot" - - name: "nginx" - - name: "ntp" - - name: "openssh-server" - - name: "pcs" - - name: "percona-toolkit" - - name: "percona-xtrabackup" - - name: "pm-utils" - - name: "postfix" - - name: "puppet" - - name: "python-lesscpy" - - name: "python-mysqldb" - - name: "python-pip" - - name: "radosgw" - - name: "rbd-fuse" - - name: "rsyslog-gnutls" - - name: "rsyslog-relp" - - name: "screen" - - name: "squashfs-tools" - - name: "swift-plugin-s3" - - name: "sysfsutils" - - name: "sysstat" - - name: "telnet" - - name: "tmux" - - name: "traceroute" - - name: "ubuntu-standard" - - name: "vim" - - name: "virt-what" - - name: "xinetd" - - name: "xmlstarlet" - - name: "tftpd-hpa" - - name: "syslinux" - -# OPNFV BLACKLIST - -# Packetary's dependency solving mechanism brings in more than one alternative -# For example, "Depends: upstart | systemd-sysv" brings both packages, which -# leads to debootstrap using "upstart" (1st option available in local repo). - -opnfv_blacklist: - - name: "upstart" diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py deleted file mode 100755 index 7c383639e..000000000 --- a/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py +++ /dev/null @@ -1,280 +0,0 @@ -#!/usr/bin/env python -############################################################################## -# Copyright (c) 2015,2016 Ericsson AB, Mirantis Inc., Enea AB and others. -# mskalski@mirantis.com -# Alexandru.Avadanii@enea.com -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -"""Build multiarch partial local Ubuntu mirror using packetary""" - -############################################################################## -# Design quirks / workarounds: -# 1. Fuel-agent uses `debootstrap` to build bootstrap and target chroots from -# the local mirror; which only uses the "main" component from the first -# repository, i.e. does not include "updates"/"security". -# In order to fullfill all debootstrap dependencies in "main" repo, we will -# do an extra packetary run using a reduced scope: -# - only "main" component of the first mirror; -# - reduced package dependency list (without MOS/OPNFV plugin deps). -# 2. If repo structure is not mandatory to be in sync with official mirrors, -# we can mitigate the issue by "merging" all repo-components into a single -# "main". -############################################################################## -# Mirror build steps (for EACH architecture in UBUNTU_ARCH): -# 1. Collect bootstrap package deps from fuel-agent's <settings.yaml.sample>; -# 2. Collect all fixture release packages from fuel-web's <openstack.yaml>; -# 3. Parse new "opnfv_config.yaml" list of packages (from old fuel-mirror); -# 4. Inherit enviroment variable(s) for mirror URLs, paths etc. -# - Allow arch-specific overrides for each env var; -# 5. Mirror config is defined based on common config + OPNFV overrides; -# - Convert old configuration format to packetary style where needed; -# 6. Package lists are defined based on common config + OPNFV deps; -# - Keep track of "main" packages separately, required by debootstrap; -# 7. Clone/update all mirror components; -# 8. IF mirror merging is disabled: -# - Clone/update "main" mirror component (fix missing debootstrap deps); -# 9. IF mirror merging is enabled: -# - Use `dpkg-scanpackages` to filter out old versions of duplicate pkgs; -# - Run `packetary create` on the set of downloaded packages, merging -# them on the fly into a single-component mirror; -############################################################################## - -from copy import deepcopy -import os -import shutil -import sys -import yaml -from contextlib import contextmanager -from cStringIO import StringIO -from packetary.cli.app import main - -@contextmanager -def capture_stdout(output): - """Context manager for capturing stdout""" - stdout = sys.stdout - sys.stdout = output - yield - sys.stdout = stdout - -# FIXME: Find a better approach for eliminating duplicate logs than this -def force_logger_reload(): - """Force logger reload (ugly hack to prevent log duplication)""" - for mod in sys.modules.keys(): - if mod.startswith('logging'): - try: - reload(sys.modules[mod]) - except: - pass - -def get_unres_pkgs(architecture, cfg_mirror): - """Determine missing package dependecies for a mirror defition""" - unresolved_pkgs = list() - packetary_output = StringIO() - with capture_stdout(packetary_output): - main('unresolved -a {0} -r {1} -c name version --sep ;' - .format(_ARCH[architecture], cfg_mirror).split(' ')) - for dep_pkg in packetary_output.getvalue().splitlines(): - if dep_pkg.startswith('#'): - continue - dep = dep_pkg.split(';') - unresolved_pkgs += [{'name': dep[0], 'version': dep[1]}] - force_logger_reload() - return unresolved_pkgs - -def from_legacy_pkglist(legacy_pkglist): - """Package list conversion from `old fuel-mirror` to `packetary` style""" - pkglist = list() - for pkg in legacy_pkglist: - pkglist += [{'name': pkg}] - return pkglist - -def to_legacy_pkglist(pkglist): - """Package list conversion from `packetary` style to `old fuel-mirror`""" - legacy_pkglist = list() - for pkg in pkglist: - legacy_pkglist.append(pkg['name']) - return legacy_pkglist - -def legacy_diff(base_pkglist, new_pkglist, requester, architecture): - """Package list diff (old format)""" - diff_set = set(new_pkglist) - if base_pkglist: - diff_set -= set(base_pkglist) - if diff_set: - print(' * {0} requires new packages for architecture [{1}]: {2}' - .format(requester, architecture, ', '.join(diff_set))) - return list(diff_set) - -def do_local_repo(architecture, cfg_repo, cfg_packages_paths): - """Create single-component local repo (one architecture per call)""" - # Packetary does not use a global config file, so pass old settings here. - main('create -t deb -a {0} --repository {1} --package-files {2}' - ' --ignore-errors-num 2 --retries-num 3 --threads-num 10' - .format(_ARCH[architecture], cfg_repo, cfg_packages_paths).split(' ')) - force_logger_reload() - -def do_partial_mirror(architecture, cfg_mirror, cfg_packages): - """Clone partial local mirror (one architecture per call)""" - # Note: '-d .' is ignored, as each mirror defines its own path. - main('clone -t deb -a {0} -r {1} -R {2} -d .' - ' --ignore-errors-num 2 --retries-num 3 --threads-num 10' - .format(_ARCH[architecture], cfg_mirror, cfg_packages).split(' ')) - force_logger_reload() - -def write_cfg_file(cfg_mirror, data): - """Write configuration (yaml) file (package list / mirror defition)""" - with open(cfg_mirror, 'w') as outfile: - outfile.write(yaml.safe_dump(data, default_flow_style=False)) - -def get_env(env_var, architecture=None): - """Evaluate architecture-specific overrides of env vars""" - if architecture: - env_var_arch = '{0}_{1}'.format(env_var, architecture) - if os.environ.get(env_var_arch): - return os.environ[env_var_arch] - if os.environ.get(env_var): - return os.environ[env_var] - return None - -# Architecture name mapping (dpkg:packetary) for packetary CLI invocation -_ARCH = { - "i386": "i386", - "amd64": "x86_64", - "arm64": "aarch64", -} - -# Arch-indepedent configuration (old fuel-mirror + OPNFV extra packages) -CFG_D = 'opnfv_config' -CFG_OPNFV = 'opnfv_config.yaml' -MOS_VERSION = get_env('MOS_VERSION') -UBUNTU_ARCH = get_env('UBUNTU_ARCH') -MIRROR_UBUNTU_PATH = get_env('MIRROR_UBUNTU_OPNFV_PATH') -MIRROR_UBUNTU_TMP_PATH = '{0}.tmp'.format(MIRROR_UBUNTU_PATH) -MIRROR_UBUNTU_MERGE = get_env('MIRROR_UBUNTU_MERGE') -CFG_MM_UBUNTU = '{0}/ubuntu_mirror_local.yaml'.format(CFG_D) -FUEL_BOOTSTRAP_CLI_FILE = open('fuel-agent/contrib/fuel_bootstrap/' - 'fuel_bootstrap_cli/fuel_bootstrap/settings.yaml.sample').read() -FUEL_BOOTSTRAP_CLI = yaml.load(FUEL_BOOTSTRAP_CLI_FILE) -FIXTURE_FILE = open('fuel-web/nailgun/nailgun/fixtures/openstack.yaml').read() -FIXTURE = yaml.load(FIXTURE_FILE) -OPNFV_CFG_YAML = open(CFG_OPNFV).read() -OPNFV_CFG = yaml.load(OPNFV_CFG_YAML) - -# Create local partial mirror using packetary, one arch at a time -for arch in UBUNTU_ARCH.split(' '): - # Mirror / Package env vars, arch-overrideable - mos_ubuntu = get_env('MIRROR_MOS_UBUNTU', arch) - mos_ubuntu_root = get_env('MIRROR_MOS_UBUNTU_ROOT', arch) - mirror_ubuntu = get_env('MIRROR_UBUNTU_URL', arch) - plugins = get_env('BUILD_FUEL_PLUGINS', arch) - if plugins is None: - plugins = get_env('PLUGINS', arch) - - # Mirror / Package list configuration files (arch-specific) - cfg_m_mos = '{0}/mos_{1}_mirror.yaml'.format(CFG_D, arch) - cfg_m_ubuntu = '{0}/ubuntu_{1}_mirror.yaml'.format(CFG_D, arch) - cfg_p_ubuntu = '{0}/ubuntu_{1}_packages.yaml'.format(CFG_D, arch) - cfg_m_ubuntu_main = '{0}/ubuntu_{1}_mirror_main.yaml'.format(CFG_D, arch) - cfg_p_ubuntu_main = '{0}/ubuntu_{1}_packages_main.yaml'.format(CFG_D, arch) - - # Mirror config fork before customizing (arch-specific) - arch_mos = 'mos_{0}'.format(arch) - arch_ubuntu = 'ubuntu_{0}'.format(arch) - arch_packages = 'packages_{0}'.format(arch) - OPNFV_CFG['groups'][arch_mos] = deepcopy(OPNFV_CFG['groups']['mos']) - OPNFV_CFG['groups'][arch_ubuntu] = deepcopy(OPNFV_CFG['groups']['ubuntu']) - OPNFV_CFG[arch_packages] = OPNFV_CFG['packages'] - - # Mirror config update & conversion to packetary input - group_main_ubuntu = dict() - for group in OPNFV_CFG['groups'][arch_mos]: - group['uri'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) - group['suite'] = group['suite'].replace('$mos_version', MOS_VERSION) - for group in OPNFV_CFG['groups'][arch_ubuntu]: - group['uri'] = mirror_ubuntu - # FIXME: At `create`, packetary insists on copying all pkgs to dest dir, - # so configure it for another dir, which will replace the orig. - group['path'] = MIRROR_UBUNTU_TMP_PATH - if not group_main_ubuntu and 'main' in group: - group_main_ubuntu = [deepcopy(group)] - group_main_ubuntu[0]['section'] = ['main'] - - # Mirror config dump: MOS (for dep resolution), Ubuntu, Ubuntu[main] - write_cfg_file(cfg_m_mos, OPNFV_CFG['groups'][arch_mos]) - write_cfg_file(cfg_m_ubuntu, OPNFV_CFG['groups'][arch_ubuntu]) - if MIRROR_UBUNTU_MERGE is None: - write_cfg_file(cfg_m_ubuntu_main, group_main_ubuntu) - else: - # FIXME: For multiarch, only one dump would be enough - group_main_ubuntu[0]['origin'] = 'Ubuntu' - group_main_ubuntu[0]['path'] = MIRROR_UBUNTU_PATH - group_main_ubuntu[0]['uri'] = MIRROR_UBUNTU_PATH - write_cfg_file(CFG_MM_UBUNTU, group_main_ubuntu[0]) - - # Collect package dependencies from: - ## 1. fuel_bootstrap_cli (bootstrap image additional packages) - legacy_unresolved = legacy_diff(None, FUEL_BOOTSTRAP_CLI['packages'] + [ - FUEL_BOOTSTRAP_CLI['kernel_flavor'], - FUEL_BOOTSTRAP_CLI['kernel_flavor'].replace('image', 'headers')], - 'Bootstrap', arch) - ## 2. openstack.yaml FIXTURE definition (default target image packages) - for release in FIXTURE: - editable = release['fields']['attributes_metadata']['editable'] - if 'provision' in editable and 'packages' in editable['provision']: - release_pkgs = editable['provision']['packages']['value'].split() - legacy_unresolved += legacy_diff(legacy_unresolved, release_pkgs, - 'Release {0}'.format(release['fields']['name']), arch) - ## 3. OPNFV additional packages (includes old fuel-mirror ubuntu.yaml pkgs) - unresolved = dict() - unresolved['mandatory'] = 'exact' - unresolved['packages'] = from_legacy_pkglist(legacy_unresolved) - if 'packages' in OPNFV_CFG: - legacy_diff(legacy_unresolved, to_legacy_pkglist(OPNFV_CFG['packages']), - 'OPNFV config', arch) - unresolved['packages'] += OPNFV_CFG['packages'] - - # OPNFV plugins dependency resolution - if plugins: - for plugin in plugins.split(): - path = "../{}/packages.yaml".format(plugin) - if os.path.isfile(path): - f = open(path).read() - plugin_yaml = yaml.load(f) - new_pkgs = legacy_diff( - to_legacy_pkglist(unresolved['packages']), - plugin_yaml['packages'], 'Plugin {0}'.format(plugin), arch) - unresolved['packages'] += from_legacy_pkglist(new_pkgs) - - # Package list (reduced, i.e. no MOS deps, but with OPNFV plugin deps) - if MIRROR_UBUNTU_MERGE is None: - write_cfg_file(cfg_p_ubuntu_main, unresolved) - - # Mirror package list (full, including MOS/OPNFV plugin deps) - unresolved['packages'] += get_unres_pkgs(arch, cfg_m_mos) - write_cfg_file(cfg_p_ubuntu, unresolved) - do_partial_mirror(arch, cfg_m_ubuntu, cfg_p_ubuntu) - if MIRROR_UBUNTU_MERGE is None: - # Ubuntu[main] must be evaluated after Ubuntu - do_partial_mirror(arch, cfg_m_ubuntu_main, cfg_p_ubuntu_main) - -if MIRROR_UBUNTU_MERGE is None: - shutil.move(MIRROR_UBUNTU_TMP_PATH, MIRROR_UBUNTU_PATH) -else: - # Construct single-component mirror from all components - for arch in UBUNTU_ARCH.split(' '): - cfg_pp_ubuntu = '{0}/ubuntu_{1}_packages_paths.yaml'.format(CFG_D, arch) - # OPNFV blacklist - opnfv_blacklist = to_legacy_pkglist(OPNFV_CFG['opnfv_blacklist']) - # FIXME: We need scanpackages to omit older DEBs - # Inspired from http://askubuntu.com/questions/198474/ - os.system('dpkg-scanpackages -a {0} {1} 2>/dev/null | ' - 'grep -e "^Filename:" | sed "s|Filename: |- file://|g" | ' - 'grep -v -E "\/({2})_" > {3}' - .format(arch, MIRROR_UBUNTU_TMP_PATH, - '|'.join(opnfv_blacklist), cfg_pp_ubuntu)) - do_local_repo(arch, CFG_MM_UBUNTU, cfg_pp_ubuntu) - shutil.rmtree(MIRROR_UBUNTU_TMP_PATH) diff --git a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh deleted file mode 100755 index c8c86db53..000000000 --- a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -UBUNTU_DISTRO="xenial" -BLACKLIST="http://mirrors.se.eu.kernel.org/ubuntu/" -#BLACKLIST+=" http://foo.bar" - -cleanup() { - rm -f $TMPFILE -} - -debugmsg() { - test -n "$DEBUG" && echo "$@" >&2 -} - - -# Check if url is blacklisted in this script -blacklisted () { - for blackurl in $BLACKLIST - do - if [ "$1" == "$blackurl" ]; then - return 0 - fi - done - return 1 -} - - -# Check mirror's integrity -check_mirror () { - mirror=$1 - status=0 - for packdir in dists/${UBUNTU_DISTRO}-updates/main/binary-amd64 \ - dists/${UBUNTU_DISTRO}-updates/restricted/binary-amd64 \ - dists/${UBUNTU_DISTRO}-updates/universe/binary-amd64 \ - dists/${UBUNTU_DISTRO}-updates/multiverse/binary-amd64 \ - dists/${UBUNTU_DISTRO}-security/main/binary-amd64 \ - dists/${UBUNTU_DISTRO}-security/restricted/binary-amd64 \ - dists/${UBUNTU_DISTRO}-security/universe/binary-amd64 \ - dists/${UBUNTU_DISTRO}-security/multiverse/binary-amd64 \ - dists/${UBUNTU_DISTRO}-proposed/main/binary-amd64 \ - dists/${UBUNTU_DISTRO}-proposed/restricted/binary-amd64 \ - dists/${UBUNTU_DISTRO}-proposed/universe/binary-amd64 \ - dists/${UBUNTU_DISTRO}-proposed/multiverse/binary-amd64 \ - dists/${UBUNTU_DISTRO}/main/binary-amd64 \ - dists/${UBUNTU_DISTRO}/restricted/binary-amd64 \ - dists/${UBUNTU_DISTRO}/universe/binary-amd64 \ - dists/${UBUNTU_DISTRO}/multiverse/binary-amd64 \ - dists/${UBUNTU_DISTRO}-backports/main/binary-amd64 \ - dists/${UBUNTU_DISTRO}-backports/restricted/binary-amd64 \ - dists/${UBUNTU_DISTRO}-backports/universe/binary-amd64 \ - dists/${UBUNTU_DISTRO}-backports/multiverse/binary-amd64 - do - for packfile in Release Packages.gz - do - if [ $status -ne 1 ]; then - curl --output /dev/null --silent --head --fail \ - $mirror/$packdir/$packfile - if [ $? -ne 0 ]; then - debugmsg "$mirror: Faulty (at least missing $packdir/$packfile)" - status=1 - fi - fi - done - done - return $status -} - -if [ "$1" == "-d" ]; then - DEBUG=1 -fi - -# Hardcode for testing purposes -# DEBUG=1 - -TMPFILE=$(mktemp /tmp/mirrorsXXXXX)A -trap cleanup exit - -# Generate a list of mirrors considered as "up" -curl -s https://launchpad.net/ubuntu/+archivemirrors | \ - grep -P -B8 "statusUP|statusONE|statusSIX" | \ - grep -o -P "(f|ht)tp.*\"" | \ - sed 's/"$//' | sort | uniq > $TMPFILE - -# Iterate over "close" mirror, check that they are considered up -# and sane. -for url in $(curl -s http://mirrors.ubuntu.com/mirrors.txt) -do - if ! grep -q $url $TMPFILE; then - debugmsg "$url Faulty (detected by Ubuntu)" - elif blacklisted $url; then - debugmsg "$url blacklisted" - elif [ -z $BESTURL ]; then - if grep -q $url $TMPFILE && check_mirror $url; then - debugmsg "$url: OK (setting as primary URL)" - BESTURL=$url - test -z "$DEBUG" && break - fi - else - grep -q $url $TMPFILE && check_mirror $url && debugmsg "$url: OK" - fi -done - -echo "$BESTURL" |