summaryrefslogtreecommitdiffstats
path: root/patches/fuel-agent/cross-bootstrap
diff options
context:
space:
mode:
Diffstat (limited to 'patches/fuel-agent/cross-bootstrap')
-rw-r--r--patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch157
-rw-r--r--patches/fuel-agent/cross-bootstrap/0002-Add-FLASH_KERNEL_SKIP-true.patch21
-rw-r--r--patches/fuel-agent/cross-bootstrap/0003-Fix-qemu-user-static-replacement.patch55
-rw-r--r--patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch102
-rw-r--r--patches/fuel-agent/cross-bootstrap/0005-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch31
-rw-r--r--patches/fuel-agent/cross-bootstrap/0006-UX-Update-bootstrap-target-build-time-estimate.patch33
6 files changed, 399 insertions, 0 deletions
diff --git a/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch b/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch
new file mode 100644
index 00000000..54e8027e
--- /dev/null
+++ b/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch
@@ -0,0 +1,157 @@
+From: Stanislaw Kardach <stanislaw.kardach@cavium.com>
+Date: Thu, 25 Feb 2016 13:38:14 +0100
+Subject: [PATCH] Use qemu-debootstrap for image creation
+
+This commit adds qemu-debootstrap support and a command line argument
+`target_arch` which can be used for choosing the architecture to build
+the image for. The architecture detection from environment settings is
+not yet implemented.
+---
+ .../fuel_bootstrap_cli/fuel_bootstrap/commands/build.py | 8 ++++++++
+ .../fuel_bootstrap/utils/bootstrap_image.py | 3 ++-
+ debian/control | 2 ++
+ fuel_agent/manager.py | 13 ++++++++++++-
+ fuel_agent/tests/test_build_utils.py | 4 ++--
+ fuel_agent/utils/build.py | 2 +-
+ specs/fuel-agent.spec | 2 ++
+ 7 files changed, 29 insertions(+), 5 deletions(-)
+
+diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py
+index b4e9a05..a22d319 100644
+--- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py
++++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/commands/build.py
+@@ -171,6 +171,14 @@ class BuildCommand(command.Command):
+ " by ssh still rejected by default! This password actual"
+ " only for tty login!"),
+ )
++ parser.add_argument(
++ '--target_arch',
++ type=str,
++ choices=['arm64', 'amd64'],
++ help="Choose the target architecture for which image is built",
++ default="amd64"
++ )
++
+ return parser
+
+ def take_action(self, parsed_args):
+diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py
+index c09d421..a535075 100644
+--- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py
++++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py
+@@ -161,7 +161,8 @@ def make_bootstrap(data):
+
+ LOG.info("Try to build image with data:\n%s", yaml.safe_dump(bootdata))
+
+- opts = ['--data_driver', 'bootstrap_build_image']
++ opts = ['--data_driver', 'bootstrap_build_image',
++ '--target_arch', data['target_arch']]
+ if data.get('image_build_dir'):
+ opts.extend(['--image_build_dir', data['image_build_dir']])
+
+diff --git a/debian/control b/debian/control
+index d24450c..e843980 100644
+--- a/debian/control
++++ b/debian/control
+@@ -37,6 +37,8 @@ Pre-Depends: dpkg (>= 1.15.6~)
+ Depends: bzip2,
+ cloud-utils,
+ debootstrap,
++ qemu-user-static,
++ binfmt-support,
+ dmidecode,
+ ethtool,
+ gdisk,
+diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py
+index df54f65..ba1ab78 100644
+--- a/fuel_agent/manager.py
++++ b/fuel_agent/manager.py
+@@ -19,6 +19,7 @@ import signal
+ import tempfile
+
+ from oslo_config import cfg
++from oslo_config import types
+ import six
+ import yaml
+
+@@ -34,6 +35,8 @@ from fuel_agent.utils import md as mu
+ from fuel_agent.utils import partition as pu
+ from fuel_agent.utils import utils
+
++ArchType = types.String(choices=['amd64', 'arm64'])
++
+ opts = [
+ cfg.StrOpt(
+ 'nc_template_path',
+@@ -192,6 +195,13 @@ cli_opts = [
+ default='/tmp',
+ help='Directory where the image is supposed to be built',
+ ),
++ cfg.Opt(
++ 'target_arch',
++ default='amd64',
++ type=ArchType,
++ help='Architecture for which image will be built using '
++ 'debootstrap',
++ ),
+ ]
+
+ CONF = cfg.CONF
+@@ -724,7 +734,8 @@ class Manager(object):
+ LOG.debug('Preventing services from being get started')
+ bu.suppress_services_start(chroot)
+ LOG.debug('Installing base operating system using debootstrap')
+- bu.run_debootstrap(uri=uri, suite=suite, chroot=chroot,
++ bu.run_debootstrap(uri=uri, suite=suite, arch=CONF.target_arch,
++ chroot=chroot,
+ attempts=CONF.fetch_packages_attempts,
+ proxies=proxies.proxies,
+ direct_repo_addr=proxies.direct_repo_addr_list)
+diff --git a/fuel_agent/tests/test_build_utils.py b/fuel_agent/tests/test_build_utils.py
+index 54f79f9..0ec466f 100644
+--- a/fuel_agent/tests/test_build_utils.py
++++ b/fuel_agent/tests/test_build_utils.py
+@@ -43,7 +43,7 @@ class BuildUtilsTestCase(unittest2.TestCase):
+ def test_run_debootstrap(self, mock_exec, mock_environ):
+ bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', attempts=2)
+ mock_exec.assert_called_once_with(
+- 'debootstrap', '--include={0}'
++ 'qemu-debootstrap', '--include={0}'
+ .format(','.join(bu.ADDITIONAL_DEBOOTSTRAP_PACKAGES)),
+ '--verbose', '--no-check-gpg', '--arch=arch',
+ 'suite', 'chroot', 'uri', attempts=2, env_variables={})
+@@ -54,7 +54,7 @@ class BuildUtilsTestCase(unittest2.TestCase):
+ bu.run_debootstrap('uri', 'suite', 'chroot', 'arch', eatmydata=True,
+ attempts=2)
+ mock_exec.assert_called_once_with(
+- 'debootstrap', '--include={0}'
++ 'qemu-debootstrap', '--include={0}'
+ .format(','.join(bu.ADDITIONAL_DEBOOTSTRAP_PACKAGES)),
+ '--verbose', '--no-check-gpg', '--arch=arch',
+ '--include=eatmydata', 'suite',
+diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py
+index b1ecc0f..2950ad8 100644
+--- a/fuel_agent/utils/build.py
++++ b/fuel_agent/utils/build.py
+@@ -76,7 +76,7 @@ def run_debootstrap(uri, suite, chroot, arch='amd64', eatmydata=False,
+ env_vars['no_proxy'] = ','.join(direct_repo_addr)
+ LOG.debug('Setting no_proxy for: {0}'.format(env_vars['no_proxy']))
+
+- cmds = ['debootstrap',
++ cmds = ['qemu-debootstrap',
+ '--include={0}'.format(",".join(ADDITIONAL_DEBOOTSTRAP_PACKAGES)),
+ '--verbose', '--no-check-gpg',
+ '--arch={0}'.format(arch)]
+diff --git a/specs/fuel-agent.spec b/specs/fuel-agent.spec
+index 72cd6a1..18af4b9 100644
+--- a/specs/fuel-agent.spec
++++ b/specs/fuel-agent.spec
+@@ -50,6 +50,8 @@ Requires: xfsprogs
+ Requires: pciutils
+ Requires: ethtool
+ Requires: debootstrap
++Requires: dpkg
++Requires: qemu-user-static
+ Requires: xz
+ Requires: coreutils
+ Requires: psmisc
diff --git a/patches/fuel-agent/cross-bootstrap/0002-Add-FLASH_KERNEL_SKIP-true.patch b/patches/fuel-agent/cross-bootstrap/0002-Add-FLASH_KERNEL_SKIP-true.patch
new file mode 100644
index 00000000..91483ac1
--- /dev/null
+++ b/patches/fuel-agent/cross-bootstrap/0002-Add-FLASH_KERNEL_SKIP-true.patch
@@ -0,0 +1,21 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Thu, 25 Feb 2016 16:01:18 +0100
+Subject: [PATCH] Add FLASH_KERNEL_SKIP=true.
+
+FIXME: Add nice description of the issue at hand.
+---
+ fuel_agent/utils/build.py | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py
+index 2950ad8..74cbfd8 100644
+--- a/fuel_agent/utils/build.py
++++ b/fuel_agent/utils/build.py
+@@ -93,6 +93,7 @@ def set_apt_get_env():
+ # NOTE(agordeev): disable any confirmations/questions from apt-get side
+ os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
+ os.environ['DEBCONF_NONINTERACTIVE_SEEN'] = 'true'
++ os.environ['FLASH_KERNEL_SKIP'] = 'true'
+ os.environ['LC_ALL'] = os.environ['LANG'] = os.environ['LANGUAGE'] = 'C'
+
+
diff --git a/patches/fuel-agent/cross-bootstrap/0003-Fix-qemu-user-static-replacement.patch b/patches/fuel-agent/cross-bootstrap/0003-Fix-qemu-user-static-replacement.patch
new file mode 100644
index 00000000..3c849591
--- /dev/null
+++ b/patches/fuel-agent/cross-bootstrap/0003-Fix-qemu-user-static-replacement.patch
@@ -0,0 +1,55 @@
+From: Stanislaw Kardach <stanislaw.kardach@cavium.com>
+Date: Sun, 6 Mar 2016 16:09:39 +0100
+Subject: [PATCH] Fix qemu-user-static replacement
+
+---
+ fuel_agent/manager.py | 6 ++++++
+ fuel_agent/utils/build.py | 18 ++++++++++++++++++
+ 2 files changed, 24 insertions(+)
+
+diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py
+index ba1ab78..637c99a 100644
+--- a/fuel_agent/manager.py
++++ b/fuel_agent/manager.py
+@@ -760,6 +760,12 @@ class Manager(object):
+ fu.mount_bind(chroot, '/proc')
+ bu.populate_basic_dev(chroot)
+
++ # we need to make sure that qemu-{target_arch}-static binary is
++ # not replaced inside chroot because we're possibly using it
++ # through qemu-debootstrap
++ LOG.debug('Preventing qemu-user-static replacement inside chroot')
++ bu.prevent_qemu_replacement(chroot, CONF.target_arch)
++
+ def destroy_chroot(self, chroot):
+ # Umount chroot tree and remove images tmp files
+ if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
+diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py
+index 74cbfd8..1bc0a5f 100644
+--- a/fuel_agent/utils/build.py
++++ b/fuel_agent/utils/build.py
+@@ -320,6 +320,24 @@ def populate_basic_dev(chroot):
+ utils.execute('chroot', chroot,
+ 'ln', '-s', '/proc/self/fd', '/dev/fd')
+
++def prevent_qemu_replacement(chroot, arch):
++ """Prevents qemu-user-static replacement inside chroot.
++
++ Use dpkg-divert to prevent replacing qemu-user-static binary inside
++ chroot which is necessary for host-side qemu-debootstrap to work
++ properly."""
++ if arch == 'arm64':
++ qemu = 'qemu-aarch64-static'
++ elif arch == 'amd64':
++ qemu = 'qemu-x86_64-static'
++ utils.execute('chroot', chroot,
++ 'dpkg-divert', '--divert', '/usr/bin/%s.orig' % qemu,
++ '/usr/bin/%s' % qemu)
++ utils.execute('chroot', chroot,
++ 'dpkg-divert', '--divert', '/usr/sbin/update-binfmts.orig',
++ '--rename', '/usr/sbin/update-binfmts')
++ utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true',
++ '/usr/sbin/update-binfmts')
+
+ def create_sparse_tmp_file(dir, suffix, size=8192):
+ """Creates sparse file.
diff --git a/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch b/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch
new file mode 100644
index 00000000..75602951
--- /dev/null
+++ b/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch
@@ -0,0 +1,102 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Tue, 8 Mar 2016 00:44:05 +0100
+Subject: [PATCH] Prevent common cross-debootstrap newaliases issue.
+
+While building target images for a different arch (using qemu-user-static),
+the following issue prevents the succesful image build:
+newaliases: fatal: inet_addr_local[getifaddrs]:
+getifaddrs: Address family not supported by protocol
+
+For more information, see [1].
+
+[1] https://bugs.launchpad.net/ubuntu/+source/postfix/+bug/1531299
+---
+ fuel_agent/manager.py | 14 ++++++++++++++
+ fuel_agent/utils/build.py | 24 ++++++++++++++++++++++++
+ 2 files changed, 38 insertions(+)
+
+diff --git a/fuel_agent/manager.py b/fuel_agent/manager.py
+index 637c99a..86f76b9 100644
+--- a/fuel_agent/manager.py
++++ b/fuel_agent/manager.py
+@@ -1037,10 +1037,15 @@ class Manager(object):
+ direct_repo_addrs=driver_os.proxies.direct_repo_addr_list)
+ self._update_metadata_with_repos(
+ metadata, driver_os.repos)
++ # Prevent common cross-debootstraping problem w/ newaliases & qemu
++ # inet_addr_local[getifaddrs]: getifaddrs: Address family not supp
++ LOG.debug('Preventing newaliases from running inside chroot')
++ bu.prevent_qemu_newaliases(chroot)
+ LOG.debug('Installing packages using apt-get: %s',
+ ' '.join(packages))
+ # disable hosts/resolv files
+ bu.propagate_host_resolv_conf(chroot)
++
+ if hasattr(bs_scheme, 'certs') and bs_scheme.certs:
+ bu.copy_update_certs(bs_scheme.certs, chroot)
+ bu.run_apt_get(chroot, packages=packages,
+@@ -1084,6 +1089,7 @@ class Manager(object):
+ add_multipath_conf=False)
+ # restore disabled hosts/resolv files
+ bu.restore_resolv_conf(chroot)
++ bu.restore_newaliases(chroot)
+ metadata['all_packages'] = bu.get_installed_packages(chroot)
+ # We need to recompress initramfs with new compression:
+ bu.recompress_initramfs(
+@@ -1175,6 +1181,11 @@ class Manager(object):
+ self._update_metadata_with_repos(
+ metadata, driver_os.repos)
+
++ # Prevent common cross-debootstraping problem w/ newaliases & qemu
++ # inet_addr_local[getifaddrs]: getifaddrs: Address family not supp
++ LOG.debug('Preventing newaliases from running inside chroot')
++ bu.prevent_qemu_newaliases(chroot)
++
+ LOG.debug('Installing packages using apt-get: %s',
+ ' '.join(packages))
+ bu.run_apt_get(chroot, packages=packages,
+@@ -1187,6 +1198,9 @@ class Manager(object):
+ allow_unsigned_file=CONF.allow_unsigned_file,
+ force_ipv4_file=CONF.force_ipv4_file)
+
++ LOG.debug('Restoring newaliases command inside chroot')
++ bu.restore_newaliases(chroot)
++
+ LOG.debug('Making sure there are no running processes '
+ 'inside chroot before trying to umount chroot')
+ if not bu.stop_chrooted_processes(chroot, signal=signal.SIGTERM):
+diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py
+index 1bc0a5f..5761cc5 100644
+--- a/fuel_agent/utils/build.py
++++ b/fuel_agent/utils/build.py
+@@ -339,6 +339,30 @@ def prevent_qemu_replacement(chroot, arch):
+ utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true',
+ '/usr/sbin/update-binfmts')
+
++def prevent_qemu_newaliases(chroot):
++ """Prevents running newaliases under qemu-user-static inside chroot.
++
++ Use dpkg-divert to prevent running newaliases binary through qemu inside
++ chroot which is necessary to avoid a dpkg --configure error:
++ inet_addr_local[getifaddrs]: getifaddrs: Address family not supported ...
++ """
++ utils.execute('chroot', chroot,
++ 'dpkg-divert', '--local', '--rename', '--add',
++ '/usr/bin/newaliases')
++ utils.execute('chroot', chroot, 'ln', '-sf', '/bin/true',
++ '/usr/bin/newaliases')
++
++def restore_newaliases(chroot):
++ """Restores newaliases inside chroot after previous dpkg-divert.
++
++ opposite to prevent_qemu_newaliases
++ """
++ utils.execute('chroot', chroot,
++ 'rm', '-f', '/usr/bin/newaliases')
++ utils.execute('chroot', chroot,
++ 'dpkg-divert', '--local', '--rename', '--remove',
++ '/usr/bin/newaliases')
++
+ def create_sparse_tmp_file(dir, suffix, size=8192):
+ """Creates sparse file.
+
diff --git a/patches/fuel-agent/cross-bootstrap/0005-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch b/patches/fuel-agent/cross-bootstrap/0005-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch
new file mode 100644
index 00000000..654ddeab
--- /dev/null
+++ b/patches/fuel-agent/cross-bootstrap/0005-FIXME-Add-force-yes-to-apt-get-dist-upgrade.patch
@@ -0,0 +1,31 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Fri, 8 Apr 2016 15:28:43 +0200
+Subject: [PATCH] FIXME: Add --force-yes to apt-get dist-upgrade.
+
+arm64 udev in Ubuntu Trusty is broken, so we had to provide our own
+patched udev package in armband MOS repos.
+
+Due to dpkg version comparison algorithm, our MOS version of udev
+is considered a downgrade, which requires --force-yes for
+apt-get dist-upgrade to work and pick up this version, otherwise
+bootstrap/target image build would fail with apt-get error code 100.
+
+This change can be dropped later, if other packages do not manifest
+the same behavior.
+---
+ fuel_agent/utils/build.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/fuel_agent/utils/build.py b/fuel_agent/utils/build.py
+index 5761cc5..5557810 100644
+--- a/fuel_agent/utils/build.py
++++ b/fuel_agent/utils/build.py
+@@ -107,7 +107,7 @@ def run_apt_get(chroot, packages, eatmydata=False, attempts=10):
+ time in X times.
+ """
+ for action in ('update', 'dist-upgrade'):
+- cmds = ['chroot', chroot, 'apt-get', '-y', action]
++ cmds = ['chroot', chroot, 'apt-get', '-y', '--force-yes', action]
+ stdout, stderr = utils.execute(*cmds, attempts=attempts)
+ LOG.debug('Running apt-get %s completed.\nstdout: %s\nstderr: %s',
+ action, stdout, stderr)
diff --git a/patches/fuel-agent/cross-bootstrap/0006-UX-Update-bootstrap-target-build-time-estimate.patch b/patches/fuel-agent/cross-bootstrap/0006-UX-Update-bootstrap-target-build-time-estimate.patch
new file mode 100644
index 00000000..3c68e5ad
--- /dev/null
+++ b/patches/fuel-agent/cross-bootstrap/0006-UX-Update-bootstrap-target-build-time-estimate.patch
@@ -0,0 +1,33 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Mon, 9 May 2016 17:08:06 +0200
+Subject: [PATCH] UX: Update bootstrap/target build time estimate.
+
+While building for a different architecture (e.g. AArch64 on x86_64),
+the bootstrap/target image build may take longer, due to latency
+introduced by using qemu-user-static.
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ .../fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py | 9 ++++++---
+ 1 file changed, 6 insertions(+), 3 deletions(-)
+
+diff --git a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py
+index a535075..c7d67e9 100644
+--- a/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py
++++ b/contrib/fuel_bootstrap/fuel_bootstrap_cli/fuel_bootstrap/utils/bootstrap_image.py
+@@ -169,9 +169,12 @@ def make_bootstrap(data):
+ OSLO_CONF = cfg.CONF
+ OSLO_CONF(opts, project='fuel-agent')
+ mngr = manager.Manager(bootdata)
+- LOG.info("Build process is in progress. Usually it takes 15-20 minutes."
+- " It depends on your internet connection and hardware"
+- " performance.")
++ LOG.info("Build process is in progress. Usually it takes 15-20 minutes for"
++ " a native build (x86_64) and/or 30-45 minutes for each"
++ " cross-build (e.g. AArch64)."
++ " It depends on your internet connection, hardware performance"
++ " and selected bootstrap architecture(s)."
++ " This ISO supports AArch64 only.")
+ mngr.do_mkbootstrap()
+
+ return bootdata['bootstrap']['uuid'], bootdata['output']