From 42f8585ebb8fffad19a89314659ab9129176c3e9 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Sat, 1 Oct 2016 13:18:11 +0200 Subject: build: Rework patch mechanism for Fuel submodules While refactoring the patching mechanism, take care of: - Sync submodule handling with Fuel@OPFNV; - build: Investigate/prepare for moving patches to Fuel@OPNFV; - build: Investigate divergent fuel-mirror; - ISO build: cacheid for Fuel comps should not depend on Armband git commit; CHANGE: Rename/shuffle patches while grouping them in "features", preparing for upstreaming them to Fuel@OPNFV and beyond. CHANGE: Allow linking patches for better representing the dependency between one patch and different features. e.g. 0001-Add-arch-to-nailgun-release-and-target-image.patch: - part of `multiarch-fuel`, because it extends Fuel; - part of `direct-kernel-boot`, as arch is required for that; - part of `cross-bootstrap`, target image is arch-dependent; NOTE: Patch links are not staged to Fuel@OPNFV, they only serve as markers that a specific patch is part of a feature. CHANGE: Kill all Fuel component submodules, now handled in Fuel@OPNFV: - fuel-agent - fuel-astute - fuel-library - fuel-mirror - fuel-nailgun-agent - fuel-web CHANGE: Move armband-fuel-config.mk to armband git root. FIXME: m1.micro-Increase-profile-RAM-size-to-128MB.patch is NOT part of `cross-bootstrap` feature, but patch context says so ... FIXME: 0001-Add-arm64-deb-repositories-setup.patch is broken at `make patches-export` by removing spaces at EOL. v2 -> v3: * Phony patch support (links to show a patch belongs to a feature); * Updated README.md v3 -> v7: * Re-export Fuel submodules & plugins patches (update patch context); * Update Cavium mail addresses (s/caviumnetworks.com/cavium.com/); * Ignore submodule changes; * Add armband git repo info to gitinfo_fuel.txt at build time; Implements: ARMBAND-136 Closes-bug: ARMBAND-95 Closes-bug: ARMBAND-93 Closes-bug: ARMBAND-92 Change-Id: I1a236d9f43b2e6dca22055911f696b43c22b5973 Signed-off-by: Alexandru Avadanii --- ...1-Use-qemu-debootstrap-for-image-creation.patch | 154 -------------------- .../0002-Add-FLASH_KERNEL_SKIP-true.patch | 21 --- .../0003-Fix-qemu-user-static-replacement.patch | 55 -------- ...common-cross-debootstrap-newaliases-issue.patch | 101 ------------- .../fuel-agent/0005-Add-esp-partition-flag.patch | 48 ------- .../fuel-agent/0007-Add-fs-for-efi-partition.patch | 66 --------- ...XME-Add-force-yes-to-apt-get-dist-upgrade.patch | 31 ---- ...date-bootstrap-target-build-time-estimate.patch | 33 ----- ...nel-flavor-linux-image-generic-lts-xenial.patch | 28 ---- ...1-Use-qemu-debootstrap-for-image-creation.patch | 157 +++++++++++++++++++++ .../0002-Add-FLASH_KERNEL_SKIP-true.patch | 21 +++ .../0003-Fix-qemu-user-static-replacement.patch | 55 ++++++++ ...common-cross-debootstrap-newaliases-issue.patch | 102 +++++++++++++ ...XME-Add-force-yes-to-apt-get-dist-upgrade.patch | 31 ++++ ...date-bootstrap-target-build-time-estimate.patch | 33 +++++ ...nel-flavor-linux-image-generic-lts-xenial.patch | 28 ++++ .../0001-Add-esp-partition-flag.patch | 48 +++++++ .../0002-Add-fs-for-efi-partition.patch | 66 +++++++++ 18 files changed, 541 insertions(+), 537 deletions(-) delete mode 100644 patches/fuel-agent/0001-Use-qemu-debootstrap-for-image-creation.patch delete mode 100644 patches/fuel-agent/0002-Add-FLASH_KERNEL_SKIP-true.patch delete mode 100644 patches/fuel-agent/0003-Fix-qemu-user-static-replacement.patch delete mode 100644 patches/fuel-agent/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch delete mode 100644 patches/fuel-agent/0005-Add-esp-partition-flag.patch delete mode 100644 patches/fuel-agent/0007-Add-fs-for-efi-partition.patch delete mode 100644 patches/fuel-agent/0009-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch delete mode 100644 patches/fuel-agent/0010-UX-Update-bootstrap-target-build-time-estimate.patch delete mode 100644 patches/fuel-agent/0011-kernel-flavor-linux-image-generic-lts-xenial.patch create mode 100644 patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch create mode 100644 patches/fuel-agent/cross-bootstrap/0002-Add-FLASH_KERNEL_SKIP-true.patch create mode 100644 patches/fuel-agent/cross-bootstrap/0003-Fix-qemu-user-static-replacement.patch create mode 100644 patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch create mode 100644 patches/fuel-agent/cross-bootstrap/0005-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch create mode 100644 patches/fuel-agent/cross-bootstrap/0006-UX-Update-bootstrap-target-build-time-estimate.patch create mode 100644 patches/fuel-agent/kernel-bump/0001-kernel-flavor-linux-image-generic-lts-xenial.patch create mode 100644 patches/fuel-agent/multiarch-efi/0001-Add-esp-partition-flag.patch create mode 100644 patches/fuel-agent/multiarch-efi/0002-Add-fs-for-efi-partition.patch (limited to 'patches/fuel-agent') diff --git a/patches/fuel-agent/0001-Use-qemu-debootstrap-for-image-creation.patch b/patches/fuel-agent/0001-Use-qemu-debootstrap-for-image-creation.patch deleted file mode 100644 index 6f28d8a3..00000000 --- a/patches/fuel-agent/0001-Use-qemu-debootstrap-for-image-creation.patch +++ /dev/null @@ -1,154 +0,0 @@ -From: Stanislaw Kardach -Date: Thu, 25 Feb 2016 13:38:14 +0100 -Subject: [PATCH] Use qemu-debootstrap for image creation - -This commit adds qemu-debootstrap support and a command line argument -`target_arch` which can be used for choosing the architecture to build -the image for. The architecture detection from environment settings is -not yet implemented. ---- - .../fuel_bootstrap_cli/fuel_bootstrap/commands/build.py | 8 ++++++++ - .../fuel_bootstrap/utils/bootstrap_image.py | 3 ++- - debian/control | 2 ++ - fuel_agent/manager.py | 13 ++++++++++++- - fuel_agent/tests/test_build_utils.py | 4 ++-- - fuel_agent/utils/build.py | 2 +- - specs/fuel-agent.spec | 2 ++ - 7 files changed, 29 insertions(+), 5 deletions(-) - -diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py -index ca2d3e1..b0d410d 100644 ---- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py -+++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py -@@ -166,4 +166,12 @@ class BuildCommand(command.Command): - ) -+ parser.add_argument( -+ '--target_arch', -+ type=str, -+ choices=['arm64', 'amd64'], -+ help="Choose the target architecture for which image is built", -+ default="amd64" -+ ) -+ - return parser - - def take_action(self, parsed_args): -diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py -index 0219961..6e60fb8 100644 ---- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py -+++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py -@@ -172,7 +172,8 @@ def make_bootstrap(data): - - LOG.info("Try to build image with data:\n%s", yaml.safe_dump(bootdata)) - -- opts = ['--data_driver', 'bootstrap_build_image'] -+ opts = ['--data_driver', 'bootstrap_build_image', -+ '--target_arch', data['target_arch']] - if data.get('image_build_dir'): - opts.extend(['--image_build_dir', data['image_build_dir']]) - -diff --git a/debian/control b/debian/control -index 0a6f947..189dc00 100644 ---- a/debian/control -+++ b/debian/control -@@ -36,6 +36,8 @@ Pre-Depends: dpkg (>= 1.15.6~) - Depends: bzip2, - cloud-utils, - debootstrap, -+ qemu-user-static, -+ binfmt-support, - dmidecode, - ethtool, - gdisk, -diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py -index 15cc5d8..f613aef 100644 ---- a/fuel_agent/manager.py -+++ b/fuel_agent/manager.py -@@ -18,5 +18,6 @@ import shutil - - from oslo_config import cfg -+from oslo_config import types - import six - import yaml - -@@ -33,6 +34,8 @@ from fuel_agent.utils import md as mu - from fuel_agent.utils import partition as pu - from fuel_agent.utils import utils - -+ArchType = types.String(choices=['amd64', 'arm64']) -+ - opts = [ - cfg.StrOpt( - 'nc_template_path', -@@ -153,6 +156,13 @@ cli_opts = [ - default='/tmp', - help='Directory where the image is supposed to be built', - ), -+ cfg.Opt( -+ 'target_arch', -+ default='amd64', -+ type=ArchType, -+ help='Architecture for which image will be built using ' -+ 'debootstrap', -+ ), - ] - - CONF = cfg.CONF -@@ -551,7 +561,8 @@ class Manager(object): - LOG.debug('Preventing services from being get started') - bu.suppress_services_start(chroot) - LOG.debug('Installing base operating system using debootstrap') -- bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot, -+ bu.run_debootstrap(uri=uri, suite=suite, arch=CONF.target_arch, -+ chroot=chroot, - attempts=CONF.fetch_packages_attempts, - proxies=proxies.proxies, - direct_repo_addr=proxies.direct_repo_addr_list) -diff --git a/fuel_agent/tests/test_build_utils.py b/fuel_agent/tests/test_build_utils.py -index 82c3462..ac5ae82 100644 ---- a/fuel_agent/tests/test_build_utils.py -+++ b/fuel_agent/tests/test_build_utils.py -@@ -42,7 +42,7 @@ class BuildUtilsTestCase(unittest2.TestCase): - def test_run_debootstrap(self, mock_exec, mock_environ): - bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', attempts=2) - mock_exec.assert_called_once_with( -- 'debootstrap', '--include={0}' -+ 'qemu-debootstrap', '--include={0}' - .format(','.join(bu.ADDITIONAL_DEBOOTSTRAP_PACKAGES)), - '--verbose', '--no-check-gpg', '--arch=arch', - 'suite', 'chroot', 'uri', attempts=2, env_variables={}) -@@ -53,7 +53,7 @@ class BuildUtilsTestCase(unittest2.TestCase): - bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', eatmydata=True, - attempts=2) - mock_exec.assert_called_once_with( -- 'debootstrap', '--include={0}' -+ 'qemu-debootstrap', '--include={0}' - .format(','.join(bu.ADDITIONAL_DEBOOTSTRAP_PACKAGES)), - '--verbose', '--no-check-gpg', '--arch=arch', - '--include=eatmydata', 'suite', -diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py -index 7247965..af41b2b 100644 ---- a/fuel_agent/utils/build.py -+++ b/fuel_agent/utils/build.py -@@ -80,7 +80,7 @@ def run_debootstrap(uri, suite, chroot, arch='amd64', eatmydata=False, - env_vars['no_proxy'] = ','.join(direct_repo_addr) - LOG.debug('Setting no_proxy for: {0}'.format(env_vars['no_proxy'])) - -- cmds = ['debootstrap', -+ cmds = ['qemu-debootstrap', - '--include={0}'.format(",".join(ADDITIONAL_DEBOOTSTRAP_PACKAGES)), - '--verbose', '--no-check-gpg', - '--arch={0}'.format(arch)] -diff --git a/specs/fuel-agent.spec b/specs/fuel-agent.spec -index 5c37600..a43f693 100644 ---- a/specs/fuel-agent.spec -+++ b/specs/fuel-agent.spec -@@ -50,6 +50,8 @@ Requires: xfsprogs - Requires: pciutils - Requires: ethtool - Requires: debootstrap -+Requires: dpkg -+Requires: qemu-user-static - Requires: xz - Requires: coreutils - Requires: psmisc diff --git a/patches/fuel-agent/0002-Add-FLASH_KERNEL_SKIP-true.patch b/patches/fuel-agent/0002-Add-FLASH_KERNEL_SKIP-true.patch deleted file mode 100644 index 0c2f2ac9..00000000 --- a/patches/fuel-agent/0002-Add-FLASH_KERNEL_SKIP-true.patch +++ /dev/null @@ -1,21 +0,0 @@ -From: Alexandru Avadanii -Date: Thu, 25 Feb 2016 16:01:18 +0100 -Subject: [PATCH] Add FLASH_KERNEL_SKIP=true. - -FIXME: Add nice description of the issue at hand. ---- - fuel_agent/utils/build.py | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py -index af41b2b..6f31732 100644 ---- a/fuel_agent/utils/build.py -+++ b/fuel_agent/utils/build.py -@@ -97,6 +97,7 @@ def set_apt_get_env(): - # NOTE(agordeev): disable any confirmations/questions from apt-get side - os.environ['DEBIAN_FRONTEND'] = 'noninteractive' - os.environ['DEBCONF_NONINTERACTIVE_SEEN'] = 'true' -+ os.environ['FLASH_KERNEL_SKIP'] = 'true' - os.environ['LC_ALL'] = os.environ['LANG'] = os.environ['LANGUAGE'] = 'C' - - diff --git a/patches/fuel-agent/0003-Fix-qemu-user-static-replacement.patch b/patches/fuel-agent/0003-Fix-qemu-user-static-replacement.patch deleted file mode 100644 index a4f65d68..00000000 --- a/patches/fuel-agent/0003-Fix-qemu-user-static-replacement.patch +++ /dev/null @@ -1,55 +0,0 @@ -From: Stanislaw Kardach -Date: Sun, 6 Mar 2016 16:09:39 +0100 -Subject: [PATCH] Fix qemu-user-static replacement - ---- - fuel_agent/manager.py | 6 ++++++ - fuel_agent/utils/build.py | 18 ++++++++++++++++++ - 2 files changed, 24 insertions(+) - -diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py -index f613aef..6322a10 100644 ---- a/fuel_agent/manager.py -+++ b/fuel_agent/manager.py -@@ -587,6 +587,12 @@ class Manager(object): - fu.mount_bind(chroot, '/proc') - bu.populate_basic_dev(chroot) - -+ # we need to make sure that qemu-{target_arch}-static binary is -+ # not replaced inside chroot because we're possibly using it -+ # through qemu-debootstrap -+ LOG.debug('Preventing qemu-user-static replacement inside chroot') -+ bu.prevent_qemu_replacement(chroot, CONF.target_arch) -+ - def destroy_chroot(self, chroot): - # Umount chroot tree and remove images tmp files - if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM): -diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py -index 6f31732..abd762e 100644 ---- a/fuel_agent/utils/build.py -+++ b/fuel_agent/utils/build.py -@@ -312,6 +312,24 @@ def populate_basic_dev(chroot): - utils.execute('chroot', chroot, - 'ln', '-s', '/proc/self/fd', '/dev/fd') - -+def prevent_qemu_replacement(chroot, arch): -+ """Prevents qemu-user-static replacement inside chroot. -+ -+ Use dpkg-divert to prevent replacing qemu-user-static binary inside -+ chroot which is necessary for host-side qemu-debootstrap to work -+ properly.""" -+ if arch == 'arm64': -+ qemu = 'qemu-aarch64-static' -+ elif arch == 'amd64': -+ qemu = 'qemu-x86_64-static' -+ utils.execute('chroot', chroot, -+ 'dpkg-divert', '--divert', '/usr/bin/%s.orig' % qemu, -+ '/usr/bin/%s' % qemu) -+ utils.execute('chroot', chroot, -+ 'dpkg-divert', '--divert', '/usr/sbin/update-binfmts.orig', -+ '--rename', '/usr/sbin/update-binfmts') -+ utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true', -+ '/usr/sbin/update-binfmts') - - def create_sparse_tmp_file(dir, suffix, size=8192): - """Creates sparse file. diff --git a/patches/fuel-agent/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch b/patches/fuel-agent/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch deleted file mode 100644 index 37017e1a..00000000 --- a/patches/fuel-agent/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch +++ /dev/null @@ -1,101 +0,0 @@ -From: Alexandru Avadanii -Date: Tue, 8 Mar 2016 00:44:05 +0100 -Subject: [PATCH] Prevent common cross-debootstrap newaliases issue. - -While building target images for a different arch (using qemu-user-static), -the following issue prevents the succesful image build: -newaliases: fatal: inet_addr_local[getifaddrs]: -getifaddrs: Address family not supported by protocol - -For more information, see [1]. - -[1] https://bugs.launchpad.net/ubuntu/+source/postfix/+bug/1531299 ---- - fuel_agent/manager.py | 14 ++++++++++++++ - fuel_agent/utils/build.py | 24 ++++++++++++++++++++++++ - 2 files changed, 38 insertions(+) - -diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py -index 6322a10..843c40b 100644 ---- a/fuel_agent/manager.py -+++ b/fuel_agent/manager.py -@@ -836,10 +836,15 @@ class Manager(object): - direct_repo_addrs=driver_os.proxies.direct_repo_addr_list) - self._update_metadata_with_repos( - metadata, driver_os.repos) -+ # Prevent common cross-debootstraping problem w/ newaliases & qemu -+ # inet_addr_local[getifaddrs]: getifaddrs: Address family not supp -+ LOG.debug('Preventing newaliases from running inside chroot') -+ bu.prevent_qemu_newaliases(chroot) - LOG.debug('Installing packages using apt-get: %s', - ' '.join(packages)) - # disable hosts/resolv files - bu.propagate_host_resolv_conf(chroot) -+ - if hasattr(bs_scheme, 'certs') and bs_scheme.certs: - bu.copy_update_certs(bs_scheme.certs, chroot) - bu.run_apt_get(chroot, packages=packages, -@@ -873,5 +878,6 @@ class Manager(object): - # restore disabled hosts/resolv files - bu.restore_resolv_conf(chroot) -+ bu.restore_newaliases(chroot) - metadata['all_packages'] = bu.get_installed_packages(chroot) - # We need to recompress initramfs with new compression: - bu.recompress_initramfs( -@@ -964,6 +970,11 @@ class Manager(object): - self._update_metadata_with_repos( - metadata, driver_os.repos) - -+ # Prevent common cross-debootstraping problem w/ newaliases & qemu -+ # inet_addr_local[getifaddrs]: getifaddrs: Address family not supp -+ LOG.debug('Preventing newaliases from running inside chroot') -+ bu.prevent_qemu_newaliases(chroot) -+ - LOG.debug('Installing packages using apt-get: %s', - ' '.join(packages)) - bu.run_apt_get(chroot, packages=packages, -@@ -974,6 +985,9 @@ class Manager(object): - allow_unsigned_file=CONF.allow_unsigned_file, - force_ipv4_file=CONF.force_ipv4_file) - -+ LOG.debug('Restoring newaliases command inside chroot') -+ bu.restore_newaliases(chroot) -+ - LOG.debug('Making sure there are no running processes ' - 'inside chroot before trying to umount chroot') - if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM): -diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py -index abd762e..e11ceba 100644 ---- a/fuel_agent/utils/build.py -+++ b/fuel_agent/utils/build.py -@@ -331,6 +331,30 @@ def prevent_qemu_replacement(chroot, arch): - utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true', - '/usr/sbin/update-binfmts') - -+def prevent_qemu_newaliases(chroot): -+ """Prevents running newaliases under qemu-user-static inside chroot. -+ -+ Use dpkg-divert to prevent running newaliases binary through qemu inside -+ chroot which is necessary to avoid a dpkg --configure error: -+ inet_addr_local[getifaddrs]: getifaddrs: Address family not supported ... -+ """ -+ utils.execute('chroot', chroot, -+ 'dpkg-divert', '--local', '--rename', '--add', -+ '/usr/bin/newaliases') -+ utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true', -+ '/usr/bin/newaliases') -+ -+def restore_newaliases(chroot): -+ """Restores newaliases inside chroot after previous dpkg-divert. -+ -+ opposite to prevent_qemu_newaliases -+ """ -+ utils.execute('chroot', chroot, -+ 'rm', '-f', '/usr/bin/newaliases') -+ utils.execute('chroot', chroot, -+ 'dpkg-divert', '--local', '--rename', '--remove', -+ '/usr/bin/newaliases') -+ - def create_sparse_tmp_file(dir, suffix, size=8192): - """Creates sparse file. - diff --git a/patches/fuel-agent/0005-Add-esp-partition-flag.patch b/patches/fuel-agent/0005-Add-esp-partition-flag.patch deleted file mode 100644 index 2ecec416..00000000 --- a/patches/fuel-agent/0005-Add-esp-partition-flag.patch +++ /dev/null @@ -1,48 +0,0 @@ -From: Alexandru Avadanii -Date: Tue, 8 Mar 2016 16:29:39 +0100 -Subject: [PATCH] Add partition flag. - -Parted flag (see [1]) will mark EFI system partition, allowing us -to properly format and mount it during do_partitioning deployment phase. - -[1] https://www.gnu.org/software/parted/manual/html_node/set.html ---- - fuel_agent/drivers/nailgun.py | 2 +- - fuel_agent/utils/partition.py | 4 ++-- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py -index c2fef69..0ae81f9 100644 ---- a/fuel_agent/drivers/nailgun.py -+++ b/fuel_agent/drivers/nailgun.py -@@ -324,7 +324,7 @@ class Nailgun(BaseDataDriver): - # uefi partition (for future use) - LOG.debug('Adding UEFI partition on disk %s: size=200' % - disk['name']) -- parted.add_partition(size=200) -+ parted.add_partition(size=200, flags=['esp']) - - LOG.debug('Looping over all volumes on disk %s' % disk['name']) - for volume in disk['volumes']: -diff --git a/fuel_agent/utils/partition.py b/fuel_agent/utils/partition.py -index acdd0b3..86349d2 100644 ---- a/fuel_agent/utils/partition.py -+++ b/fuel_agent/utils/partition.py -@@ -93,7 +93,7 @@ def set_partition_flag(dev, num, flag, state='on'): - :param dev: A device file, e.g. /dev/sda. - :param num: Partition number - :param flag: Flag name. Must be one of 'bios_grub', 'legacy_boot', -- 'boot', 'raid', 'lvm' -+ 'boot', 'raid', 'lvm', 'esp' - :param state: Desiable flag state. 'on' or 'off'. Default is 'on'. - - :returns: None -@@ -103,7 +103,7 @@ def set_partition_flag(dev, num, flag, state='on'): - # parted supports more flags but we are interested in - # setting only this subset of them. - # not all of these flags are compatible with one another. -- if flag not in ('bios_grub', 'legacy_boot', 'boot', 'raid', 'lvm'): -+ if flag not in ('bios_grub', 'legacy_boot', 'boot', 'raid', 'lvm', 'esp'): - raise errors.WrongPartitionSchemeError( - 'Unsupported partition flag: %s' % flag) - if state not in ('on', 'off'): diff --git a/patches/fuel-agent/0007-Add-fs-for-efi-partition.patch b/patches/fuel-agent/0007-Add-fs-for-efi-partition.patch deleted file mode 100644 index 0fb94215..00000000 --- a/patches/fuel-agent/0007-Add-fs-for-efi-partition.patch +++ /dev/null @@ -1,66 +0,0 @@ -From: Stanislaw Kardach -Date: Tue, 8 Mar 2016 21:08:55 +0100 -Subject: [PATCH] Add fs for efi partition - ---- - debian/control | 1 + - fuel_agent/drivers/nailgun.py | 13 ++++++++++--- - specs/fuel-agent.spec | 1 + - 3 files changed, 12 insertions(+), 3 deletions(-) - -diff --git a/debian/control b/debian/control -index 189dc00..acfda35 100644 ---- a/debian/control -+++ b/debian/control -@@ -39,6 +39,7 @@ Depends: bzip2, - qemu-user-static, - binfmt-support, - dmidecode, -+ dosfstools, - ethtool, - gdisk, - genisoimage, -diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py -index bc532b4..f092865 100644 ---- a/fuel_agent/drivers/nailgun.py -+++ b/fuel_agent/drivers/nailgun.py -@@ -81,6 +81,9 @@ class Nailgun(BaseDataDriver): - # was already allocated on first matching volume - # or not - self._boot_partition_done = False -+ # this var states whether ESP partition was already -+ # allocated on the first matching volume or not -+ self._esp_partition_done = False - # this var is used as a flag that /boot fs - # has already been added. we need this to - # get rid of md over all disks for /boot partition. -@@ -321,10 +324,14 @@ class Nailgun(BaseDataDriver): - LOG.debug('Adding bios_grub partition on disk %s: size=24' % - disk['name']) - parted.add_partition(size=24, flags=['bios_grub']) -- # uefi partition (for future use) -- LOG.debug('Adding UEFI partition on disk %s: size=200' % -+ # uefi partition - added only once. -+ if self._is_boot_disk(disk) and not self._esp_partition_done: -+ LOG.debug('Adding UEFI partition on disk %s: size=200' % - disk['name']) -- parted.add_partition(size=200, flags=['esp']) -+ prt = parted.add_partition(size=200, flags=['esp']) -+ partition_scheme.add_fs(device=prt.name, mount='/boot/efi', -+ fs_type='vfat') -+ self._esp_partition_done = True - - LOG.debug('Looping over all volumes on disk %s' % disk['name']) - for volume in disk['volumes']: -diff --git a/specs/fuel-agent.spec b/specs/fuel-agent.spec -index a43f693..c2b9b51 100644 ---- a/specs/fuel-agent.spec -+++ b/specs/fuel-agent.spec -@@ -50,6 +50,7 @@ Requires: xfsprogs - Requires: pciutils - Requires: ethtool - Requires: debootstrap -+Requires: dosfstools - Requires: dpkg - Requires: qemu-user-static - Requires: xz diff --git a/patches/fuel-agent/0009-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch b/patches/fuel-agent/0009-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch deleted file mode 100644 index 90428c57..00000000 --- a/patches/fuel-agent/0009-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch +++ /dev/null @@ -1,31 +0,0 @@ -From: Alexandru Avadanii -Date: Fri, 8 Apr 2016 15:28:43 +0200 -Subject: [PATCH] FIXME: Add --force-yes to apt-get dist-upgrade. - -arm64 udev in Ubuntu Trusty is broken, so we had to provide our own -patched udev package in armband MOS repos. - -Due to dpkg version comparison algorithm, our MOS version of udev -is considered a downgrade, which requires --force-yes for -apt-get dist-upgrade to work and pick up this version, otherwise -bootstrap/target image build would fail with apt-get error code 100. - -This change can be dropped later, if other packages do not manifest -the same behavior. ---- - fuel_agent/utils/build.py | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py -index e11ceba..8cca26d 100644 ---- a/fuel_agent/utils/build.py -+++ b/fuel_agent/utils/build.py -@@ -111,7 +111,7 @@ def run_apt_get(chroot, packages, eatmydata=False, attempts=10): - time in X times. - """ - for action in ('update', 'dist-upgrade'): -- cmds = ['chroot', chroot, 'apt-get', '-y', action] -+ cmds = ['chroot', chroot, 'apt-get', '-y', '--force-yes', action] - stdout, stderr = utils.execute(*cmds, attempts=attempts) - LOG.debug('Running apt-get %s completed.\nstdout: %s\nstderr: %s', - action, stdout, stderr) diff --git a/patches/fuel-agent/0010-UX-Update-bootstrap-target-build-time-estimate.patch b/patches/fuel-agent/0010-UX-Update-bootstrap-target-build-time-estimate.patch deleted file mode 100644 index d8a7588f..00000000 --- a/patches/fuel-agent/0010-UX-Update-bootstrap-target-build-time-estimate.patch +++ /dev/null @@ -1,33 +0,0 @@ -From: Alexandru Avadanii -Date: Mon, 9 May 2016 17:08:06 +0200 -Subject: [PATCH] UX: Update bootstrap/target build time estimate. - -While building for a different architecture (e.g. AArch64 on x86_64), -the bootstrap/target image build may take longer, due to latency -introduced by using qemu-user-static. - -Signed-off-by: Alexandru Avadanii ---- - .../fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py | 9 ++++++--- - 1 file changed, 6 insertions(+), 3 deletions(-) - -diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py -index 6e60fb8..95bc08a 100644 ---- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py -+++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py -@@ -180,9 +180,12 @@ def make_bootstrap(data): - OSLO_CONF = cfg.CONF - OSLO_CONF(opts, project='fuel-agent') - mngr = manager.Manager(bootdata) -- LOG.info("Build process is in progress. Usually it takes 15-20 minutes." -- " It depends on your internet connection and hardware" -- " performance.") -+ LOG.info("Build process is in progress. Usually it takes 15-20 minutes for" -+ " a native build (x86_64) and/or 30-45 minutes for each" -+ " cross-build (e.g. AArch64)." -+ " It depends on your internet connection, hardware performance" -+ " and selected bootstrap architecture(s)." -+ " This ISO supports AArch64 only.") - mngr.do_mkbootstrap() - - return bootdata['bootstrap']['uuid'], bootdata['output'] diff --git a/patches/fuel-agent/0011-kernel-flavor-linux-image-generic-lts-xenial.patch b/patches/fuel-agent/0011-kernel-flavor-linux-image-generic-lts-xenial.patch deleted file mode 100644 index c94bc4aa..00000000 --- a/patches/fuel-agent/0011-kernel-flavor-linux-image-generic-lts-xenial.patch +++ /dev/null @@ -1,28 +0,0 @@ -From: Alexandru Avadanii -Date: Thu, 4 Aug 2016 12:44:53 +0200 -Subject: [PATCH] kernel-flavor: linux-image-generic-lts-xenial - -Ubuntu arm64: switch from using kernel 4.2 for Trusty -(linux-image-generic-lts-trusty) to kernel 4.4 backported from Xenial -(linux-image-generic-lts-xenial). - -Signed-off-by: Alexandru Avadanii ---- - fuel_agent/drivers/nailgun.py | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py -index 5ff1cf6..1463f22 100644 ---- a/fuel_agent/drivers/nailgun.py -+++ b/fuel_agent/drivers/nailgun.py -@@ -734,8 +734,8 @@ class NailgunBuildImage(BaseDataDriver): - "i40e-dkms", - "linux-firmware", - "linux-firmware-nonfree", -- "linux-headers-generic-lts-trusty", -- "linux-image-generic-lts-trusty", -+ "linux-headers-generic-lts-xenial", -+ "linux-image-generic-lts-xenial", - "lvm2", - "mcollective", - "mdadm", diff --git a/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch b/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch new file mode 100644 index 00000000..54e8027e --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch @@ -0,0 +1,157 @@ +From: Stanislaw Kardach +Date: Thu, 25 Feb 2016 13:38:14 +0100 +Subject: [PATCH] Use qemu-debootstrap for image creation + +This commit adds qemu-debootstrap support and a command line argument +`target_arch` which can be used for choosing the architecture to build +the image for. The architecture detection from environment settings is +not yet implemented. +--- + .../fuel_bootstrap_cli/fuel_bootstrap/commands/build.py | 8 ++++++++ + .../fuel_bootstrap/utils/bootstrap_image.py | 3 ++- + debian/control | 2 ++ + fuel_agent/manager.py | 13 ++++++++++++- + fuel_agent/tests/test_build_utils.py | 4 ++-- + fuel_agent/utils/build.py | 2 +- + specs/fuel-agent.spec | 2 ++ + 7 files changed, 29 insertions(+), 5 deletions(-) + +diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py +index b4e9a05..a22d319 100644 +--- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py ++++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py +@@ -171,6 +171,14 @@ class BuildCommand(command.Command): + " by ssh still rejected by default! This password actual" + " only for tty login!"), + ) ++ parser.add_argument( ++ '--target_arch', ++ type=str, ++ choices=['arm64', 'amd64'], ++ help="Choose the target architecture for which image is built", ++ default="amd64" ++ ) ++ + return parser + + def take_action(self, parsed_args): +diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py +index c09d421..a535075 100644 +--- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py ++++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py +@@ -161,7 +161,8 @@ def make_bootstrap(data): + + LOG.info("Try to build image with data:\n%s", yaml.safe_dump(bootdata)) + +- opts = ['--data_driver', 'bootstrap_build_image'] ++ opts = ['--data_driver', 'bootstrap_build_image', ++ '--target_arch', data['target_arch']] + if data.get('image_build_dir'): + opts.extend(['--image_build_dir', data['image_build_dir']]) + +diff --git a/debian/control b/debian/control +index d24450c..e843980 100644 +--- a/debian/control ++++ b/debian/control +@@ -37,6 +37,8 @@ Pre-Depends: dpkg (>= 1.15.6~) + Depends: bzip2, + cloud-utils, + debootstrap, ++ qemu-user-static, ++ binfmt-support, + dmidecode, + ethtool, + gdisk, +diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py +index df54f65..ba1ab78 100644 +--- a/fuel_agent/manager.py ++++ b/fuel_agent/manager.py +@@ -19,6 +19,7 @@ import signal + import tempfile + + from oslo_config import cfg ++from oslo_config import types + import six + import yaml + +@@ -34,6 +35,8 @@ from fuel_agent.utils import md as mu + from fuel_agent.utils import partition as pu + from fuel_agent.utils import utils + ++ArchType = types.String(choices=['amd64', 'arm64']) ++ + opts = [ + cfg.StrOpt( + 'nc_template_path', +@@ -192,6 +195,13 @@ cli_opts = [ + default='/tmp', + help='Directory where the image is supposed to be built', + ), ++ cfg.Opt( ++ 'target_arch', ++ default='amd64', ++ type=ArchType, ++ help='Architecture for which image will be built using ' ++ 'debootstrap', ++ ), + ] + + CONF = cfg.CONF +@@ -724,7 +734,8 @@ class Manager(object): + LOG.debug('Preventing services from being get started') + bu.suppress_services_start(chroot) + LOG.debug('Installing base operating system using debootstrap') +- bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot, ++ bu.run_debootstrap(uri=uri, suite=suite, arch=CONF.target_arch, ++ chroot=chroot, + attempts=CONF.fetch_packages_attempts, + proxies=proxies.proxies, + direct_repo_addr=proxies.direct_repo_addr_list) +diff --git a/fuel_agent/tests/test_build_utils.py b/fuel_agent/tests/test_build_utils.py +index 54f79f9..0ec466f 100644 +--- a/fuel_agent/tests/test_build_utils.py ++++ b/fuel_agent/tests/test_build_utils.py +@@ -43,7 +43,7 @@ class BuildUtilsTestCase(unittest2.TestCase): + def test_run_debootstrap(self, mock_exec, mock_environ): + bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', attempts=2) + mock_exec.assert_called_once_with( +- 'debootstrap', '--include={0}' ++ 'qemu-debootstrap', '--include={0}' + .format(','.join(bu.ADDITIONAL_DEBOOTSTRAP_PACKAGES)), + '--verbose', '--no-check-gpg', '--arch=arch', + 'suite', 'chroot', 'uri', attempts=2, env_variables={}) +@@ -54,7 +54,7 @@ class BuildUtilsTestCase(unittest2.TestCase): + bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', eatmydata=True, + attempts=2) + mock_exec.assert_called_once_with( +- 'debootstrap', '--include={0}' ++ 'qemu-debootstrap', '--include={0}' + .format(','.join(bu.ADDITIONAL_DEBOOTSTRAP_PACKAGES)), + '--verbose', '--no-check-gpg', '--arch=arch', + '--include=eatmydata', 'suite', +diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py +index b1ecc0f..2950ad8 100644 +--- a/fuel_agent/utils/build.py ++++ b/fuel_agent/utils/build.py +@@ -76,7 +76,7 @@ def run_debootstrap(uri, suite, chroot, arch='amd64', eatmydata=False, + env_vars['no_proxy'] = ','.join(direct_repo_addr) + LOG.debug('Setting no_proxy for: {0}'.format(env_vars['no_proxy'])) + +- cmds = ['debootstrap', ++ cmds = ['qemu-debootstrap', + '--include={0}'.format(",".join(ADDITIONAL_DEBOOTSTRAP_PACKAGES)), + '--verbose', '--no-check-gpg', + '--arch={0}'.format(arch)] +diff --git a/specs/fuel-agent.spec b/specs/fuel-agent.spec +index 72cd6a1..18af4b9 100644 +--- a/specs/fuel-agent.spec ++++ b/specs/fuel-agent.spec +@@ -50,6 +50,8 @@ Requires: xfsprogs + Requires: pciutils + Requires: ethtool + Requires: debootstrap ++Requires: dpkg ++Requires: qemu-user-static + Requires: xz + Requires: coreutils + Requires: psmisc diff --git a/patches/fuel-agent/cross-bootstrap/0002-Add-FLASH_KERNEL_SKIP-true.patch b/patches/fuel-agent/cross-bootstrap/0002-Add-FLASH_KERNEL_SKIP-true.patch new file mode 100644 index 00000000..91483ac1 --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0002-Add-FLASH_KERNEL_SKIP-true.patch @@ -0,0 +1,21 @@ +From: Alexandru Avadanii +Date: Thu, 25 Feb 2016 16:01:18 +0100 +Subject: [PATCH] Add FLASH_KERNEL_SKIP=true. + +FIXME: Add nice description of the issue at hand. +--- + fuel_agent/utils/build.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py +index 2950ad8..74cbfd8 100644 +--- a/fuel_agent/utils/build.py ++++ b/fuel_agent/utils/build.py +@@ -93,6 +93,7 @@ def set_apt_get_env(): + # NOTE(agordeev): disable any confirmations/questions from apt-get side + os.environ['DEBIAN_FRONTEND'] = 'noninteractive' + os.environ['DEBCONF_NONINTERACTIVE_SEEN'] = 'true' ++ os.environ['FLASH_KERNEL_SKIP'] = 'true' + os.environ['LC_ALL'] = os.environ['LANG'] = os.environ['LANGUAGE'] = 'C' + + diff --git a/patches/fuel-agent/cross-bootstrap/0003-Fix-qemu-user-static-replacement.patch b/patches/fuel-agent/cross-bootstrap/0003-Fix-qemu-user-static-replacement.patch new file mode 100644 index 00000000..3c849591 --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0003-Fix-qemu-user-static-replacement.patch @@ -0,0 +1,55 @@ +From: Stanislaw Kardach +Date: Sun, 6 Mar 2016 16:09:39 +0100 +Subject: [PATCH] Fix qemu-user-static replacement + +--- + fuel_agent/manager.py | 6 ++++++ + fuel_agent/utils/build.py | 18 ++++++++++++++++++ + 2 files changed, 24 insertions(+) + +diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py +index ba1ab78..637c99a 100644 +--- a/fuel_agent/manager.py ++++ b/fuel_agent/manager.py +@@ -760,6 +760,12 @@ class Manager(object): + fu.mount_bind(chroot, '/proc') + bu.populate_basic_dev(chroot) + ++ # we need to make sure that qemu-{target_arch}-static binary is ++ # not replaced inside chroot because we're possibly using it ++ # through qemu-debootstrap ++ LOG.debug('Preventing qemu-user-static replacement inside chroot') ++ bu.prevent_qemu_replacement(chroot, CONF.target_arch) ++ + def destroy_chroot(self, chroot): + # Umount chroot tree and remove images tmp files + if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM): +diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py +index 74cbfd8..1bc0a5f 100644 +--- a/fuel_agent/utils/build.py ++++ b/fuel_agent/utils/build.py +@@ -320,6 +320,24 @@ def populate_basic_dev(chroot): + utils.execute('chroot', chroot, + 'ln', '-s', '/proc/self/fd', '/dev/fd') + ++def prevent_qemu_replacement(chroot, arch): ++ """Prevents qemu-user-static replacement inside chroot. ++ ++ Use dpkg-divert to prevent replacing qemu-user-static binary inside ++ chroot which is necessary for host-side qemu-debootstrap to work ++ properly.""" ++ if arch == 'arm64': ++ qemu = 'qemu-aarch64-static' ++ elif arch == 'amd64': ++ qemu = 'qemu-x86_64-static' ++ utils.execute('chroot', chroot, ++ 'dpkg-divert', '--divert', '/usr/bin/%s.orig' % qemu, ++ '/usr/bin/%s' % qemu) ++ utils.execute('chroot', chroot, ++ 'dpkg-divert', '--divert', '/usr/sbin/update-binfmts.orig', ++ '--rename', '/usr/sbin/update-binfmts') ++ utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true', ++ '/usr/sbin/update-binfmts') + + def create_sparse_tmp_file(dir, suffix, size=8192): + """Creates sparse file. diff --git a/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch b/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch new file mode 100644 index 00000000..75602951 --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch @@ -0,0 +1,102 @@ +From: Alexandru Avadanii +Date: Tue, 8 Mar 2016 00:44:05 +0100 +Subject: [PATCH] Prevent common cross-debootstrap newaliases issue. + +While building target images for a different arch (using qemu-user-static), +the following issue prevents the succesful image build: +newaliases: fatal: inet_addr_local[getifaddrs]: +getifaddrs: Address family not supported by protocol + +For more information, see [1]. + +[1] https://bugs.launchpad.net/ubuntu/+source/postfix/+bug/1531299 +--- + fuel_agent/manager.py | 14 ++++++++++++++ + fuel_agent/utils/build.py | 24 ++++++++++++++++++++++++ + 2 files changed, 38 insertions(+) + +diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py +index 637c99a..86f76b9 100644 +--- a/fuel_agent/manager.py ++++ b/fuel_agent/manager.py +@@ -1037,10 +1037,15 @@ class Manager(object): + direct_repo_addrs=driver_os.proxies.direct_repo_addr_list) + self._update_metadata_with_repos( + metadata, driver_os.repos) ++ # Prevent common cross-debootstraping problem w/ newaliases & qemu ++ # inet_addr_local[getifaddrs]: getifaddrs: Address family not supp ++ LOG.debug('Preventing newaliases from running inside chroot') ++ bu.prevent_qemu_newaliases(chroot) + LOG.debug('Installing packages using apt-get: %s', + ' '.join(packages)) + # disable hosts/resolv files + bu.propagate_host_resolv_conf(chroot) ++ + if hasattr(bs_scheme, 'certs') and bs_scheme.certs: + bu.copy_update_certs(bs_scheme.certs, chroot) + bu.run_apt_get(chroot, packages=packages, +@@ -1084,6 +1089,7 @@ class Manager(object): + add_multipath_conf=False) + # restore disabled hosts/resolv files + bu.restore_resolv_conf(chroot) ++ bu.restore_newaliases(chroot) + metadata['all_packages'] = bu.get_installed_packages(chroot) + # We need to recompress initramfs with new compression: + bu.recompress_initramfs( +@@ -1175,6 +1181,11 @@ class Manager(object): + self._update_metadata_with_repos( + metadata, driver_os.repos) + ++ # Prevent common cross-debootstraping problem w/ newaliases & qemu ++ # inet_addr_local[getifaddrs]: getifaddrs: Address family not supp ++ LOG.debug('Preventing newaliases from running inside chroot') ++ bu.prevent_qemu_newaliases(chroot) ++ + LOG.debug('Installing packages using apt-get: %s', + ' '.join(packages)) + bu.run_apt_get(chroot, packages=packages, +@@ -1187,6 +1198,9 @@ class Manager(object): + allow_unsigned_file=CONF.allow_unsigned_file, + force_ipv4_file=CONF.force_ipv4_file) + ++ LOG.debug('Restoring newaliases command inside chroot') ++ bu.restore_newaliases(chroot) ++ + LOG.debug('Making sure there are no running processes ' + 'inside chroot before trying to umount chroot') + if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM): +diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py +index 1bc0a5f..5761cc5 100644 +--- a/fuel_agent/utils/build.py ++++ b/fuel_agent/utils/build.py +@@ -339,6 +339,30 @@ def prevent_qemu_replacement(chroot, arch): + utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true', + '/usr/sbin/update-binfmts') + ++def prevent_qemu_newaliases(chroot): ++ """Prevents running newaliases under qemu-user-static inside chroot. ++ ++ Use dpkg-divert to prevent running newaliases binary through qemu inside ++ chroot which is necessary to avoid a dpkg --configure error: ++ inet_addr_local[getifaddrs]: getifaddrs: Address family not supported ... ++ """ ++ utils.execute('chroot', chroot, ++ 'dpkg-divert', '--local', '--rename', '--add', ++ '/usr/bin/newaliases') ++ utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true', ++ '/usr/bin/newaliases') ++ ++def restore_newaliases(chroot): ++ """Restores newaliases inside chroot after previous dpkg-divert. ++ ++ opposite to prevent_qemu_newaliases ++ """ ++ utils.execute('chroot', chroot, ++ 'rm', '-f', '/usr/bin/newaliases') ++ utils.execute('chroot', chroot, ++ 'dpkg-divert', '--local', '--rename', '--remove', ++ '/usr/bin/newaliases') ++ + def create_sparse_tmp_file(dir, suffix, size=8192): + """Creates sparse file. + diff --git a/patches/fuel-agent/cross-bootstrap/0005-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch b/patches/fuel-agent/cross-bootstrap/0005-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch new file mode 100644 index 00000000..654ddeab --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0005-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch @@ -0,0 +1,31 @@ +From: Alexandru Avadanii +Date: Fri, 8 Apr 2016 15:28:43 +0200 +Subject: [PATCH] FIXME: Add --force-yes to apt-get dist-upgrade. + +arm64 udev in Ubuntu Trusty is broken, so we had to provide our own +patched udev package in armband MOS repos. + +Due to dpkg version comparison algorithm, our MOS version of udev +is considered a downgrade, which requires --force-yes for +apt-get dist-upgrade to work and pick up this version, otherwise +bootstrap/target image build would fail with apt-get error code 100. + +This change can be dropped later, if other packages do not manifest +the same behavior. +--- + fuel_agent/utils/build.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py +index 5761cc5..5557810 100644 +--- a/fuel_agent/utils/build.py ++++ b/fuel_agent/utils/build.py +@@ -107,7 +107,7 @@ def run_apt_get(chroot, packages, eatmydata=False, attempts=10): + time in X times. + """ + for action in ('update', 'dist-upgrade'): +- cmds = ['chroot', chroot, 'apt-get', '-y', action] ++ cmds = ['chroot', chroot, 'apt-get', '-y', '--force-yes', action] + stdout, stderr = utils.execute(*cmds, attempts=attempts) + LOG.debug('Running apt-get %s completed.\nstdout: %s\nstderr: %s', + action, stdout, stderr) diff --git a/patches/fuel-agent/cross-bootstrap/0006-UX-Update-bootstrap-target-build-time-estimate.patch b/patches/fuel-agent/cross-bootstrap/0006-UX-Update-bootstrap-target-build-time-estimate.patch new file mode 100644 index 00000000..3c68e5ad --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0006-UX-Update-bootstrap-target-build-time-estimate.patch @@ -0,0 +1,33 @@ +From: Alexandru Avadanii +Date: Mon, 9 May 2016 17:08:06 +0200 +Subject: [PATCH] UX: Update bootstrap/target build time estimate. + +While building for a different architecture (e.g. AArch64 on x86_64), +the bootstrap/target image build may take longer, due to latency +introduced by using qemu-user-static. + +Signed-off-by: Alexandru Avadanii +--- + .../fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py | 9 ++++++--- + 1 file changed, 6 insertions(+), 3 deletions(-) + +diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py +index a535075..c7d67e9 100644 +--- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py ++++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py +@@ -169,9 +169,12 @@ def make_bootstrap(data): + OSLO_CONF = cfg.CONF + OSLO_CONF(opts, project='fuel-agent') + mngr = manager.Manager(bootdata) +- LOG.info("Build process is in progress. Usually it takes 15-20 minutes." +- " It depends on your internet connection and hardware" +- " performance.") ++ LOG.info("Build process is in progress. Usually it takes 15-20 minutes for" ++ " a native build (x86_64) and/or 30-45 minutes for each" ++ " cross-build (e.g. AArch64)." ++ " It depends on your internet connection, hardware performance" ++ " and selected bootstrap architecture(s)." ++ " This ISO supports AArch64 only.") + mngr.do_mkbootstrap() + + return bootdata['bootstrap']['uuid'], bootdata['output'] diff --git a/patches/fuel-agent/kernel-bump/0001-kernel-flavor-linux-image-generic-lts-xenial.patch b/patches/fuel-agent/kernel-bump/0001-kernel-flavor-linux-image-generic-lts-xenial.patch new file mode 100644 index 00000000..7c29c5c5 --- /dev/null +++ b/patches/fuel-agent/kernel-bump/0001-kernel-flavor-linux-image-generic-lts-xenial.patch @@ -0,0 +1,28 @@ +From: Alexandru Avadanii +Date: Thu, 4 Aug 2016 12:44:53 +0200 +Subject: [PATCH] kernel-flavor: linux-image-generic-lts-xenial + +Ubuntu arm64: switch from using kernel 4.2 for Trusty +(linux-image-generic-lts-trusty) to kernel 4.4 backported from Xenial +(linux-image-generic-lts-xenial). + +Signed-off-by: Alexandru Avadanii +--- + fuel_agent/drivers/nailgun.py | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py +index f70d165..ef0e79a 100644 +--- a/fuel_agent/drivers/nailgun.py ++++ b/fuel_agent/drivers/nailgun.py +@@ -727,8 +727,8 @@ class NailgunBuildImage(BaseDataDriver): + "i40e-dkms", + "linux-firmware", + "linux-firmware-nonfree", +- "linux-headers-generic-lts-trusty", +- "linux-image-generic-lts-trusty", ++ "linux-headers-generic-lts-xenial", ++ "linux-image-generic-lts-xenial", + "lvm2", + "mcollective", + "mdadm", diff --git a/patches/fuel-agent/multiarch-efi/0001-Add-esp-partition-flag.patch b/patches/fuel-agent/multiarch-efi/0001-Add-esp-partition-flag.patch new file mode 100644 index 00000000..1b067b10 --- /dev/null +++ b/patches/fuel-agent/multiarch-efi/0001-Add-esp-partition-flag.patch @@ -0,0 +1,48 @@ +From: Alexandru Avadanii +Date: Tue, 8 Mar 2016 16:29:39 +0100 +Subject: [PATCH] Add partition flag. + +Parted flag (see [1]) will mark EFI system partition, allowing us +to properly format and mount it during do_partitioning deployment phase. + +[1] https://www.gnu.org/software/parted/manual/html_node/set.html +--- + fuel_agent/drivers/nailgun.py | 2 +- + fuel_agent/utils/partition.py | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py +index ef0e79a..94280e8 100644 +--- a/fuel_agent/drivers/nailgun.py ++++ b/fuel_agent/drivers/nailgun.py +@@ -354,7 +354,7 @@ class Nailgun(BaseDataDriver): + # uefi partition (for future use) + LOG.debug('Adding UEFI partition on disk %s: size=200' % + disk['name']) +- parted.add_partition(size=200) ++ parted.add_partition(size=200, flags=['esp']) + + LOG.debug('Looping over all volumes on disk %s' % disk['name']) + for volume in disk['volumes']: +diff --git a/fuel_agent/utils/partition.py b/fuel_agent/utils/partition.py +index f7126b1..b811511 100644 +--- a/fuel_agent/utils/partition.py ++++ b/fuel_agent/utils/partition.py +@@ -94,7 +94,7 @@ def set_partition_flag(dev, num, flag, state='on'): + :param dev: A device file, e.g. /dev/sda. + :param num: Partition number + :param flag: Flag name. Must be one of 'bios_grub', 'legacy_boot', +- 'boot', 'raid', 'lvm' ++ 'boot', 'raid', 'lvm', 'esp' + :param state: Desiable flag state. 'on' or 'off'. Default is 'on'. + + :returns: None +@@ -104,7 +104,7 @@ def set_partition_flag(dev, num, flag, state='on'): + # parted supports more flags but we are interested in + # setting only this subset of them. + # not all of these flags are compatible with one another. +- if flag not in ('bios_grub', 'legacy_boot', 'boot', 'raid', 'lvm'): ++ if flag not in ('bios_grub', 'legacy_boot', 'boot', 'raid', 'lvm', 'esp'): + raise errors.WrongPartitionSchemeError( + 'Unsupported partition flag: %s' % flag) + if state not in ('on', 'off'): diff --git a/patches/fuel-agent/multiarch-efi/0002-Add-fs-for-efi-partition.patch b/patches/fuel-agent/multiarch-efi/0002-Add-fs-for-efi-partition.patch new file mode 100644 index 00000000..c18c7093 --- /dev/null +++ b/patches/fuel-agent/multiarch-efi/0002-Add-fs-for-efi-partition.patch @@ -0,0 +1,66 @@ +From: Stanislaw Kardach +Date: Tue, 8 Mar 2016 21:08:55 +0100 +Subject: [PATCH] Add fs for efi partition + +--- + debian/control | 1 + + fuel_agent/drivers/nailgun.py | 13 ++++++++++--- + specs/fuel-agent.spec | 1 + + 3 files changed, 12 insertions(+), 3 deletions(-) + +diff --git a/debian/control b/debian/control +index e843980..c763ab9 100644 +--- a/debian/control ++++ b/debian/control +@@ -40,6 +40,7 @@ Depends: bzip2, + qemu-user-static, + binfmt-support, + dmidecode, ++ dosfstools, + ethtool, + gdisk, + genisoimage, +diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py +index 94280e8..1463f22 100644 +--- a/fuel_agent/drivers/nailgun.py ++++ b/fuel_agent/drivers/nailgun.py +@@ -82,6 +82,9 @@ class Nailgun(BaseDataDriver): + # was already allocated on first matching volume + # or not + self._boot_partition_done = False ++ # this var states whether ESP partition was already ++ # allocated on the first matching volume or not ++ self._esp_partition_done = False + # this var is used as a flag that /boot fs + # has already been added. we need this to + # get rid of md over all disks for /boot partition. +@@ -351,10 +354,14 @@ class Nailgun(BaseDataDriver): + LOG.debug('Adding bios_grub partition on disk %s: size=24' % + disk['name']) + parted.add_partition(size=24, flags=['bios_grub']) +- # uefi partition (for future use) +- LOG.debug('Adding UEFI partition on disk %s: size=200' % ++ # uefi partition - added only once. ++ if self._is_boot_disk(disk) and not self._esp_partition_done: ++ LOG.debug('Adding UEFI partition on disk %s: size=200' % + disk['name']) +- parted.add_partition(size=200, flags=['esp']) ++ prt = parted.add_partition(size=200, flags=['esp']) ++ partition_scheme.add_fs(device=prt.name, mount='/boot/efi', ++ fs_type='vfat') ++ self._esp_partition_done = True + + LOG.debug('Looping over all volumes on disk %s' % disk['name']) + for volume in disk['volumes']: +diff --git a/specs/fuel-agent.spec b/specs/fuel-agent.spec +index 18af4b9..8604e57 100644 +--- a/specs/fuel-agent.spec ++++ b/specs/fuel-agent.spec +@@ -50,6 +50,7 @@ Requires: xfsprogs + Requires: pciutils + Requires: ethtool + Requires: debootstrap ++Requires: dosfstools + Requires: dpkg + Requires: qemu-user-static + Requires: xz -- cgit 1.2.3-korg