diff options
Diffstat (limited to 'patches/fuel-agent/cross-bootstrap')
6 files changed, 399 insertions, 0 deletions
diff --git a/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch b/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch new file mode 100644 index 00000000..54e8027e --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch @@ -0,0 +1,157 @@ +From: Stanislaw Kardach <stanislaw.kardach@cavium.com> +Date: Thu, 25 Feb 2016 13:38:14 +0100 +Subject: [PATCH] Use qemu-debootstrap for image creation + +This commit adds qemu-debootstrap support and a command line argument +`target_arch` which can be used for choosing the architecture to build +the image for. The architecture detection from environment settings is +not yet implemented. +--- + .../fuel_bootstrap_cli/fuel_bootstrap/commands/build.py | 8 ++++++++ + .../fuel_bootstrap/utils/bootstrap_image.py | 3 ++- + debian/control | 2 ++ + fuel_agent/manager.py | 13 ++++++++++++- + fuel_agent/tests/test_build_utils.py | 4 ++-- + fuel_agent/utils/build.py | 2 +- + specs/fuel-agent.spec | 2 ++ + 7 files changed, 29 insertions(+), 5 deletions(-) + +diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py +index b4e9a05..a22d319 100644 +--- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py ++++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py +@@ -171,6 +171,14 @@ class BuildCommand(command.Command): + " by ssh still rejected by default! This password actual" + " only for tty login!"), + ) ++ parser.add_argument( ++ '--target_arch', ++ type=str, ++ choices=['arm64', 'amd64'], ++ help="Choose the target architecture for which image is built", ++ default="amd64" ++ ) ++ + return parser + + def take_action(self, parsed_args): +diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py +index c09d421..a535075 100644 +--- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py ++++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py +@@ -161,7 +161,8 @@ def make_bootstrap(data): + + LOG.info("Try to build image with data:\n%s", yaml.safe_dump(bootdata)) + +- opts = ['--data_driver', 'bootstrap_build_image'] ++ opts = ['--data_driver', 'bootstrap_build_image', ++ '--target_arch', data['target_arch']] + if data.get('image_build_dir'): + opts.extend(['--image_build_dir', data['image_build_dir']]) + +diff --git a/debian/control b/debian/control +index d24450c..e843980 100644 +--- a/debian/control ++++ b/debian/control +@@ -37,6 +37,8 @@ Pre-Depends: dpkg (>= 1.15.6~) + Depends: bzip2, + cloud-utils, + debootstrap, ++ qemu-user-static, ++ binfmt-support, + dmidecode, + ethtool, + gdisk, +diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py +index df54f65..ba1ab78 100644 +--- a/fuel_agent/manager.py ++++ b/fuel_agent/manager.py +@@ -19,6 +19,7 @@ import signal + import tempfile + + from oslo_config import cfg ++from oslo_config import types + import six + import yaml + +@@ -34,6 +35,8 @@ from fuel_agent.utils import md as mu + from fuel_agent.utils import partition as pu + from fuel_agent.utils import utils + ++ArchType = types.String(choices=['amd64', 'arm64']) ++ + opts = [ + cfg.StrOpt( + 'nc_template_path', +@@ -192,6 +195,13 @@ cli_opts = [ + default='/tmp', + help='Directory where the image is supposed to be built', + ), ++ cfg.Opt( ++ 'target_arch', ++ default='amd64', ++ type=ArchType, ++ help='Architecture for which image will be built using ' ++ 'debootstrap', ++ ), + ] + + CONF = cfg.CONF +@@ -724,7 +734,8 @@ class Manager(object): + LOG.debug('Preventing services from being get started') + bu.suppress_services_start(chroot) + LOG.debug('Installing base operating system using debootstrap') +- bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot, ++ bu.run_debootstrap(uri=uri, suite=suite, arch=CONF.target_arch, ++ chroot=chroot, + attempts=CONF.fetch_packages_attempts, + proxies=proxies.proxies, + direct_repo_addr=proxies.direct_repo_addr_list) +diff --git a/fuel_agent/tests/test_build_utils.py b/fuel_agent/tests/test_build_utils.py +index 54f79f9..0ec466f 100644 +--- a/fuel_agent/tests/test_build_utils.py ++++ b/fuel_agent/tests/test_build_utils.py +@@ -43,7 +43,7 @@ class BuildUtilsTestCase(unittest2.TestCase): + def test_run_debootstrap(self, mock_exec, mock_environ): + bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', attempts=2) + mock_exec.assert_called_once_with( +- 'debootstrap', '--include={0}' ++ 'qemu-debootstrap', '--include={0}' + .format(','.join(bu.ADDITIONAL_DEBOOTSTRAP_PACKAGES)), + '--verbose', '--no-check-gpg', '--arch=arch', + 'suite', 'chroot', 'uri', attempts=2, env_variables={}) +@@ -54,7 +54,7 @@ class BuildUtilsTestCase(unittest2.TestCase): + bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', eatmydata=True, + attempts=2) + mock_exec.assert_called_once_with( +- 'debootstrap', '--include={0}' ++ 'qemu-debootstrap', '--include={0}' + .format(','.join(bu.ADDITIONAL_DEBOOTSTRAP_PACKAGES)), + '--verbose', '--no-check-gpg', '--arch=arch', + '--include=eatmydata', 'suite', +diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py +index b1ecc0f..2950ad8 100644 +--- a/fuel_agent/utils/build.py ++++ b/fuel_agent/utils/build.py +@@ -76,7 +76,7 @@ def run_debootstrap(uri, suite, chroot, arch='amd64', eatmydata=False, + env_vars['no_proxy'] = ','.join(direct_repo_addr) + LOG.debug('Setting no_proxy for: {0}'.format(env_vars['no_proxy'])) + +- cmds = ['debootstrap', ++ cmds = ['qemu-debootstrap', + '--include={0}'.format(",".join(ADDITIONAL_DEBOOTSTRAP_PACKAGES)), + '--verbose', '--no-check-gpg', + '--arch={0}'.format(arch)] +diff --git a/specs/fuel-agent.spec b/specs/fuel-agent.spec +index 72cd6a1..18af4b9 100644 +--- a/specs/fuel-agent.spec ++++ b/specs/fuel-agent.spec +@@ -50,6 +50,8 @@ Requires: xfsprogs + Requires: pciutils + Requires: ethtool + Requires: debootstrap ++Requires: dpkg ++Requires: qemu-user-static + Requires: xz + Requires: coreutils + Requires: psmisc diff --git a/patches/fuel-agent/cross-bootstrap/0002-Add-FLASH_KERNEL_SKIP-true.patch b/patches/fuel-agent/cross-bootstrap/0002-Add-FLASH_KERNEL_SKIP-true.patch new file mode 100644 index 00000000..91483ac1 --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0002-Add-FLASH_KERNEL_SKIP-true.patch @@ -0,0 +1,21 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Thu, 25 Feb 2016 16:01:18 +0100 +Subject: [PATCH] Add FLASH_KERNEL_SKIP=true. + +FIXME: Add nice description of the issue at hand. +--- + fuel_agent/utils/build.py | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py +index 2950ad8..74cbfd8 100644 +--- a/fuel_agent/utils/build.py ++++ b/fuel_agent/utils/build.py +@@ -93,6 +93,7 @@ def set_apt_get_env(): + # NOTE(agordeev): disable any confirmations/questions from apt-get side + os.environ['DEBIAN_FRONTEND'] = 'noninteractive' + os.environ['DEBCONF_NONINTERACTIVE_SEEN'] = 'true' ++ os.environ['FLASH_KERNEL_SKIP'] = 'true' + os.environ['LC_ALL'] = os.environ['LANG'] = os.environ['LANGUAGE'] = 'C' + + diff --git a/patches/fuel-agent/cross-bootstrap/0003-Fix-qemu-user-static-replacement.patch b/patches/fuel-agent/cross-bootstrap/0003-Fix-qemu-user-static-replacement.patch new file mode 100644 index 00000000..3c849591 --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0003-Fix-qemu-user-static-replacement.patch @@ -0,0 +1,55 @@ +From: Stanislaw Kardach <stanislaw.kardach@cavium.com> +Date: Sun, 6 Mar 2016 16:09:39 +0100 +Subject: [PATCH] Fix qemu-user-static replacement + +--- + fuel_agent/manager.py | 6 ++++++ + fuel_agent/utils/build.py | 18 ++++++++++++++++++ + 2 files changed, 24 insertions(+) + +diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py +index ba1ab78..637c99a 100644 +--- a/fuel_agent/manager.py ++++ b/fuel_agent/manager.py +@@ -760,6 +760,12 @@ class Manager(object): + fu.mount_bind(chroot, '/proc') + bu.populate_basic_dev(chroot) + ++ # we need to make sure that qemu-{target_arch}-static binary is ++ # not replaced inside chroot because we're possibly using it ++ # through qemu-debootstrap ++ LOG.debug('Preventing qemu-user-static replacement inside chroot') ++ bu.prevent_qemu_replacement(chroot, CONF.target_arch) ++ + def destroy_chroot(self, chroot): + # Umount chroot tree and remove images tmp files + if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM): +diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py +index 74cbfd8..1bc0a5f 100644 +--- a/fuel_agent/utils/build.py ++++ b/fuel_agent/utils/build.py +@@ -320,6 +320,24 @@ def populate_basic_dev(chroot): + utils.execute('chroot', chroot, + 'ln', '-s', '/proc/self/fd', '/dev/fd') + ++def prevent_qemu_replacement(chroot, arch): ++ """Prevents qemu-user-static replacement inside chroot. ++ ++ Use dpkg-divert to prevent replacing qemu-user-static binary inside ++ chroot which is necessary for host-side qemu-debootstrap to work ++ properly.""" ++ if arch == 'arm64': ++ qemu = 'qemu-aarch64-static' ++ elif arch == 'amd64': ++ qemu = 'qemu-x86_64-static' ++ utils.execute('chroot', chroot, ++ 'dpkg-divert', '--divert', '/usr/bin/%s.orig' % qemu, ++ '/usr/bin/%s' % qemu) ++ utils.execute('chroot', chroot, ++ 'dpkg-divert', '--divert', '/usr/sbin/update-binfmts.orig', ++ '--rename', '/usr/sbin/update-binfmts') ++ utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true', ++ '/usr/sbin/update-binfmts') + + def create_sparse_tmp_file(dir, suffix, size=8192): + """Creates sparse file. diff --git a/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch b/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch new file mode 100644 index 00000000..75602951 --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch @@ -0,0 +1,102 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Tue, 8 Mar 2016 00:44:05 +0100 +Subject: [PATCH] Prevent common cross-debootstrap newaliases issue. + +While building target images for a different arch (using qemu-user-static), +the following issue prevents the succesful image build: +newaliases: fatal: inet_addr_local[getifaddrs]: +getifaddrs: Address family not supported by protocol + +For more information, see [1]. + +[1] https://bugs.launchpad.net/ubuntu/+source/postfix/+bug/1531299 +--- + fuel_agent/manager.py | 14 ++++++++++++++ + fuel_agent/utils/build.py | 24 ++++++++++++++++++++++++ + 2 files changed, 38 insertions(+) + +diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py +index 637c99a..86f76b9 100644 +--- a/fuel_agent/manager.py ++++ b/fuel_agent/manager.py +@@ -1037,10 +1037,15 @@ class Manager(object): + direct_repo_addrs=driver_os.proxies.direct_repo_addr_list) + self._update_metadata_with_repos( + metadata, driver_os.repos) ++ # Prevent common cross-debootstraping problem w/ newaliases & qemu ++ # inet_addr_local[getifaddrs]: getifaddrs: Address family not supp ++ LOG.debug('Preventing newaliases from running inside chroot') ++ bu.prevent_qemu_newaliases(chroot) + LOG.debug('Installing packages using apt-get: %s', + ' '.join(packages)) + # disable hosts/resolv files + bu.propagate_host_resolv_conf(chroot) ++ + if hasattr(bs_scheme, 'certs') and bs_scheme.certs: + bu.copy_update_certs(bs_scheme.certs, chroot) + bu.run_apt_get(chroot, packages=packages, +@@ -1084,6 +1089,7 @@ class Manager(object): + add_multipath_conf=False) + # restore disabled hosts/resolv files + bu.restore_resolv_conf(chroot) ++ bu.restore_newaliases(chroot) + metadata['all_packages'] = bu.get_installed_packages(chroot) + # We need to recompress initramfs with new compression: + bu.recompress_initramfs( +@@ -1175,6 +1181,11 @@ class Manager(object): + self._update_metadata_with_repos( + metadata, driver_os.repos) + ++ # Prevent common cross-debootstraping problem w/ newaliases & qemu ++ # inet_addr_local[getifaddrs]: getifaddrs: Address family not supp ++ LOG.debug('Preventing newaliases from running inside chroot') ++ bu.prevent_qemu_newaliases(chroot) ++ + LOG.debug('Installing packages using apt-get: %s', + ' '.join(packages)) + bu.run_apt_get(chroot, packages=packages, +@@ -1187,6 +1198,9 @@ class Manager(object): + allow_unsigned_file=CONF.allow_unsigned_file, + force_ipv4_file=CONF.force_ipv4_file) + ++ LOG.debug('Restoring newaliases command inside chroot') ++ bu.restore_newaliases(chroot) ++ + LOG.debug('Making sure there are no running processes ' + 'inside chroot before trying to umount chroot') + if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM): +diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py +index 1bc0a5f..5761cc5 100644 +--- a/fuel_agent/utils/build.py ++++ b/fuel_agent/utils/build.py +@@ -339,6 +339,30 @@ def prevent_qemu_replacement(chroot, arch): + utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true', + '/usr/sbin/update-binfmts') + ++def prevent_qemu_newaliases(chroot): ++ """Prevents running newaliases under qemu-user-static inside chroot. ++ ++ Use dpkg-divert to prevent running newaliases binary through qemu inside ++ chroot which is necessary to avoid a dpkg --configure error: ++ inet_addr_local[getifaddrs]: getifaddrs: Address family not supported ... ++ """ ++ utils.execute('chroot', chroot, ++ 'dpkg-divert', '--local', '--rename', '--add', ++ '/usr/bin/newaliases') ++ utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true', ++ '/usr/bin/newaliases') ++ ++def restore_newaliases(chroot): ++ """Restores newaliases inside chroot after previous dpkg-divert. ++ ++ opposite to prevent_qemu_newaliases ++ """ ++ utils.execute('chroot', chroot, ++ 'rm', '-f', '/usr/bin/newaliases') ++ utils.execute('chroot', chroot, ++ 'dpkg-divert', '--local', '--rename', '--remove', ++ '/usr/bin/newaliases') ++ + def create_sparse_tmp_file(dir, suffix, size=8192): + """Creates sparse file. + diff --git a/patches/fuel-agent/cross-bootstrap/0005-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch b/patches/fuel-agent/cross-bootstrap/0005-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch new file mode 100644 index 00000000..654ddeab --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0005-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch @@ -0,0 +1,31 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Fri, 8 Apr 2016 15:28:43 +0200 +Subject: [PATCH] FIXME: Add --force-yes to apt-get dist-upgrade. + +arm64 udev in Ubuntu Trusty is broken, so we had to provide our own +patched udev package in armband MOS repos. + +Due to dpkg version comparison algorithm, our MOS version of udev +is considered a downgrade, which requires --force-yes for +apt-get dist-upgrade to work and pick up this version, otherwise +bootstrap/target image build would fail with apt-get error code 100. + +This change can be dropped later, if other packages do not manifest +the same behavior. +--- + fuel_agent/utils/build.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py +index 5761cc5..5557810 100644 +--- a/fuel_agent/utils/build.py ++++ b/fuel_agent/utils/build.py +@@ -107,7 +107,7 @@ def run_apt_get(chroot, packages, eatmydata=False, attempts=10): + time in X times. + """ + for action in ('update', 'dist-upgrade'): +- cmds = ['chroot', chroot, 'apt-get', '-y', action] ++ cmds = ['chroot', chroot, 'apt-get', '-y', '--force-yes', action] + stdout, stderr = utils.execute(*cmds, attempts=attempts) + LOG.debug('Running apt-get %s completed.\nstdout: %s\nstderr: %s', + action, stdout, stderr) diff --git a/patches/fuel-agent/cross-bootstrap/0006-UX-Update-bootstrap-target-build-time-estimate.patch b/patches/fuel-agent/cross-bootstrap/0006-UX-Update-bootstrap-target-build-time-estimate.patch new file mode 100644 index 00000000..3c68e5ad --- /dev/null +++ b/patches/fuel-agent/cross-bootstrap/0006-UX-Update-bootstrap-target-build-time-estimate.patch @@ -0,0 +1,33 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Mon, 9 May 2016 17:08:06 +0200 +Subject: [PATCH] UX: Update bootstrap/target build time estimate. + +While building for a different architecture (e.g. AArch64 on x86_64), +the bootstrap/target image build may take longer, due to latency +introduced by using qemu-user-static. + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + .../fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py | 9 ++++++--- + 1 file changed, 6 insertions(+), 3 deletions(-) + +diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py +index a535075..c7d67e9 100644 +--- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py ++++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py +@@ -169,9 +169,12 @@ def make_bootstrap(data): + OSLO_CONF = cfg.CONF + OSLO_CONF(opts, project='fuel-agent') + mngr = manager.Manager(bootdata) +- LOG.info("Build process is in progress. Usually it takes 15-20 minutes." +- " It depends on your internet connection and hardware" +- " performance.") ++ LOG.info("Build process is in progress. Usually it takes 15-20 minutes for" ++ " a native build (x86_64) and/or 30-45 minutes for each" ++ " cross-build (e.g. AArch64)." ++ " It depends on your internet connection, hardware performance" ++ " and selected bootstrap architecture(s)." ++ " This ISO supports AArch64 only.") + mngr.do_mkbootstrap() + + return bootdata['bootstrap']['uuid'], bootdata['output'] |