diff options
| author |   Alexandru Avadanii <Alexandru.Avadanii@enea.com> | 2016-12-18 16:53:26 +0100 |
|---|---|---|
| committer | Alexandru Avadanii <Alexandru.Avadanii@enea.com> | 2017-01-21 18:07:28 +0100 |
| commit | e42a9b3011f96ad26f4a19db77ac44cad31a4290 (patch) | |
| tree | b6a1c7f918138dd3e4705d170dde85eb2e5aaa6a | |
| parent | 5c258a9ae96fd2a0c6fd56e41c4368467e931782 (diff) | |
Uplift Armband to Fuel Newton
[ Dan Andresan ]
- puppet: database: Fix Percona XtraBackup sync
[ Alexandru Avadanii ]
- disable all plugins;
- re-enable remote tracking;
- remove "Revert: Point to specific snapshot ...";
- patch context adjustments;
- obsolete "kernel-bump", linux-image-lts-xenial is now the default;
- network-checker iface state check now fixed upstream [1];
- fuel-nailgun-agent hugepage size should also check /proc/meminfo;
- fuel-nailgun-agent CPU detection for AArch64;
- nova AArch64 hugepage support is now upstream;
- obsolete Cirros Test VM direct kernel boot (by switch to AAVMF):
* f-l/0001-upload_cirros-Add-direct-kernel-boot-support.patch
* f-w/0001-direct-kernel-boot-for-cirros.patch
- rework m1.micro RAM size patch after puppet manifest split upstream;
- re-enable arch-agnostic plugins which were rebased in Fuel@OPNFV:
* f_yardstick-pluginbuild
* f_congress-pluginbuild
- do NOT retire MySQL SST provider patch series (nack: ARMBAND-186),
rebase (and keep for now) MySQL SST provider patches, as trying to
use xtrabackup-v2 revelead a regression since Colorado.3.0, and
these patches simplify troubleshooting a lot;
- AArch64: nova: libvirt: Use host-model cpu (ARMBAND-193);
- AArch64: nova: libvirt: Use pointer_model instead of use_usb_tablet;
- m1.micro RAM size insufficient for TestVM with AAVMF (s/128/256/)
- switch Cirros TestVM to AAVMF from direct kernel boot;
- backport nova libvirt driver fix for deleting instances booted with
AAVMF firmware from [2];
TODO (later):
- Include ISO build time fixes for cirros_testvm in Armband package;
TODO (ODL, later):
- test & revise leveldb patching;
- bring back Qugga patching for arm64;
- configure systemd service to automatically respawn;
[1] https://review.openstack.org/#/c/417373/
[2] https://review.openstack.org/#/c/357190/
JIRA: ARMBAND-29
JIRA: ARMBAND-32
JIRA: ARMBAND-63
JIRA: ARMBAND-88
JIRA: ARMBAND-116
JIRA: ARMBAND-118
JIRA: ARMBAND-186
JIRA: ARMBAND-193
JIRA: ARMBAND-194
JIRA: ARMBAND-195
JIRA: ARMBAND-196
JIRA: ARMBAND-197
Change-Id: Ia99022e364e61245d109cabab9d0ed7157b4d2f5
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
Signed-off-by: Dan Andresan <dan.andresan@enea.com>
60 files changed, 1983 insertions, 2423 deletions
@@ -1,7 +1,7 @@ Armband Fuel@OPNFV ================== -This repository holds build scripts for Fuel 9.0 OPNFV installer +This repository holds build scripts for Fuel 10.0 OPNFV installer for AArch64 machines. Workflow diff --git a/armband-fuel-config.mk b/armband-fuel-config.mk index 591a23d9..aceafd3a 100644 --- a/armband-fuel-config.mk +++ b/armband-fuel-config.mk @@ -19,14 +19,16 @@ F_SUB_DIR := ${F_REPOS_DIR}/sub # To enable remote tracking, set the following var to any non-empty string. # Leaving this var empty will bind each git submodule to its saved commit. -ARMBAND_TRACK_REMOTES ?= +ARMBAND_TRACK_REMOTES ?= yes # Armband plugins, supported archs & specific info -export PLUGINS := f_odlpluginbuild f_bgpvpn-pluginbuild f_ovs-nsh-dpdk-pluginbuild f_tacker-pluginbuild f_yardstick-pluginbuild +# FIXME(alav): re-enable for D release after rebasing to Newton +#export PLUGINS := f_odlpluginbuild f_bgpvpn-pluginbuild f_ovs-nsh-dpdk-pluginbuild f_tacker-pluginbuild f_yardstick-pluginbuild +export PLUGINS := f_yardstick-pluginbuild f_congress-pluginbuild export UBUNTU_ARCH := amd64 arm64 export PRODNO := OPNFV_A_FUEL export MIRROR_MOS_UBUNTU := linux.enea.com -export EXTRA_RPM_REPOS := armband,http://linux.enea.com/mos-repos/centos/mos9.0-centos7/armband/x86_64,10 +export EXTRA_RPM_REPOS := armband,http://linux.enea.com/mos-repos/centos/mos10.0-centos7/armband/x86_64,10 # Temporary fuel-plugin-builder repo info for runtime patching export FPB_REPO := https://github.com/openstack/fuel-plugins @@ -37,6 +39,7 @@ export FPB_CHANGE := refs/changes/31/311031/2 export FUEL_PLUGIN_ODL_REPO := ${ARMBAND_BASE}/upstream/fuel-plugin-opendaylight export FUEL_PLUGIN_ODL_BRANCH := opnfv-armband export FUEL_PLUGIN_ODL_CHANGE := ${A_OPNFV_TAG} +# FIXME(alav): rebase to Newton and update ref export OPNFV_QUAGGE_PACKAGING_REPO := https://github.com/alexandruavadanii/opnfv-quagga-packaging export OVS_NSH_DPDK_REPO := ${ARMBAND_BASE}/upstream/fuel-plugin-ovs diff --git a/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch b/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch index 54e8027e..b68ca4e4 100644 --- a/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch +++ b/patches/fuel-agent/cross-bootstrap/0001-Use-qemu-debootstrap-for-image-creation.patch @@ -70,12 +70,11 @@ index df54f65..ba1ab78 100644 import tempfile from oslo_config import cfg + from oslo_log import log as logging +from oslo_config import types import six import yaml - -@@ -34,6 +35,8 @@ from fuel_agent.utils import md as mu - from fuel_agent.utils import partition as pu +@@ -34,5 +35,7 @@ from fuel_agent.utils import md as mu from fuel_agent.utils import utils +ArchType = types.String(choices=['amd64', 'arm64']) diff --git a/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch b/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch index 75602951..8a0cbcf3 100644 --- a/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch +++ b/patches/fuel-agent/cross-bootstrap/0004-Prevent-common-cross-debootstrap-newaliases-issue.patch @@ -35,8 +35,7 @@ index 637c99a..86f76b9 100644 if hasattr(bs_scheme, 'certs') and bs_scheme.certs: bu.copy_update_certs(bs_scheme.certs, chroot) bu.run_apt_get(chroot, packages=packages, -@@ -1084,6 +1089,7 @@ class Manager(object): - add_multipath_conf=False) +@@ -1084,5 +1089,6 @@ class Manager(object): # restore disabled hosts/resolv files bu.restore_resolv_conf(chroot) + bu.restore_newaliases(chroot) diff --git a/patches/fuel-agent/multiarch-efi/0002-Add-fs-for-efi-partition.patch b/patches/fuel-agent/multiarch-efi/0002-Add-fs-for-efi-partition.patch index c18c7093..7c05c69d 100644 --- a/patches/fuel-agent/multiarch-efi/0002-Add-fs-for-efi-partition.patch +++ b/patches/fuel-agent/multiarch-efi/0002-Add-fs-for-efi-partition.patch @@ -12,14 +12,13 @@ diff --git a/debian/control b/debian/control index e843980..c763ab9 100644 --- a/debian/control +++ b/debian/control -@@ -40,6 +40,7 @@ Depends: bzip2, +@@ -40,5 +40,6 @@ Depends: bzip2, qemu-user-static, binfmt-support, dmidecode, + dosfstools, ethtool, gdisk, - genisoimage, diff --git a/fuel_agent/drivers/nailgun.py b/fuel_agent/drivers/nailgun.py index 94280e8..1463f22 100644 --- a/fuel_agent/drivers/nailgun.py diff --git a/patches/fuel-library/0001-ceph-Fix-obsolete-XFS-mount-param-delaylog.patch b/patches/fuel-library/0001-ceph-Fix-obsolete-XFS-mount-param-delaylog.patch deleted file mode 100644 index facde8f2..00000000 --- a/patches/fuel-library/0001-ceph-Fix-obsolete-XFS-mount-param-delaylog.patch +++ /dev/null @@ -1,48 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Sun, 27 Mar 2016 20:32:50 +0200 -Subject: [PATCH] ceph: Fix obsolete XFS mount param "delaylog". - -According to [1], "delaylog" mount arg is now enforced by default, -and passing it will lead to mount failure. - -Trim "delaylog" from default list of xfs mount args and leave it up to -ceph-osd.pp to add this obsoleted arg only for targets running a kernel -older than 4.0. - -[1] https://www.kernel.org/doc/Documentation/filesystems/xfs.txt ---- - deployment/puppet/osnailyfacter/manifests/ceph/ceph_osd.pp | 7 +++++++ - deployment/puppet/osnailyfacter/manifests/globals/globals.pp | 2 +- - 2 files changed, 8 insertions(+), 1 deletion(-) - -diff --git a/deployment/puppet/osnailyfacter/manifests/ceph/ceph_osd.pp b/deployment/puppet/osnailyfacter/manifests/ceph/ceph_osd.pp -index 2480f18..0957010 100644 ---- a/deployment/puppet/osnailyfacter/manifests/ceph/ceph_osd.pp -+++ b/deployment/puppet/osnailyfacter/manifests/ceph/ceph_osd.pp -@@ -35,6 +35,13 @@ class osnailyfacter::ceph::ceph_osd { - } - - if $ceph_tuning_settings_hash != {} { -+ if versioncmp($::kernelmajversion, '4.0') < 0 { -+ # FIXME(armband): XFS mount opt delaylog is deprecated in kernels >=4.0. -+ $ceph_tuning_settings_hash['osd_mount_options_xfs'] = join([ -+ $ceph_tuning_settings_hash['osd_mount_options_xfs'], -+ 'delaylog' -+ ], ',') -+ } - ceph_conf { - 'global/debug_default' : value => $debug; - 'global/max_open_files' : value => $ceph_tuning_settings_hash['max_open_files']; -diff --git a/deployment/puppet/osnailyfacter/manifests/globals/globals.pp b/deployment/puppet/osnailyfacter/manifests/globals/globals.pp -index 854d297..7345d66 100644 ---- a/deployment/puppet/osnailyfacter/manifests/globals/globals.pp -+++ b/deployment/puppet/osnailyfacter/manifests/globals/globals.pp -@@ -148,7 +148,7 @@ class osnailyfacter::globals::globals { - $ceph_tuning_settings = { - 'max_open_files' => pick($storage_tuning_settings['max_open_files'], '131072'), - 'osd_mkfs_type' => pick($storage_tuning_settings['osd_mkfs_type'], 'xfs'), -- 'osd_mount_options_xfs' => pick($storage_tuning_settings['osd_mount_options_xfs'], 'rw,relatime,inode64,logbsize=256k,delaylog,allocsize=4M'), -+ 'osd_mount_options_xfs' => pick($storage_tuning_settings['osd_mount_options_xfs'], 'rw,relatime,inode64,logbsize=256k,allocsize=4M'), - 'osd_op_threads' => pick($storage_tuning_settings['osd_op_threads'], '20'), - 'filestore_queue_max_ops' => pick($storage_tuning_settings['filestore_queue_max_ops'], '500'), - 'filestore_queue_committing_max_ops' => pick($storage_tuning_settings['filestore_queue_committing_max_ops'], '5000'), diff --git a/patches/fuel-library/0001-puppet-database-Fix-Percona-XtraBackup-sync.patch b/patches/fuel-library/0001-puppet-database-Fix-Percona-XtraBackup-sync.patch new file mode 100644 index 00000000..bf09ce07 --- /dev/null +++ b/patches/fuel-library/0001-puppet-database-Fix-Percona-XtraBackup-sync.patch @@ -0,0 +1,54 @@ +From: Dan Andresan <dan.andresan@enea.com> +Date: Fri, 20 Jan 2017 17:24:40 +0200 +Subject: [PATCH] puppet: database: Fix Percona XtraBackup sync + +Recently, Percona released XtraBackup 2.4.5 which considers that SSL +is enabled if it finds ssl-ca, ssl-cert, ssl-key set up in my.cnf, +even if ssl is set to false in the same config file. Furthermore, +it complains that ssl is deprecated on the stdout instead of stderr, +corrupting xbstream in the process. + +Because ssl was set to false, this patch delete all other ssl-* keys +from the config file, avoiding xtrabackup getting confused. + +See percona xtrabackup bugs here: +https://bugs.launchpad.net/percona-xtrabackup/+bug/1646480 +https://bugs.launchpad.net/percona-xtrabackup/+bug/1647340 + +Closes-bug: https://jira.opnfv.org/browse/ARMBAND-197 + +Signed-off-by: Dan Andresan <dan.andresan@enea.com> +--- + .../puppet/osnailyfacter/manifests/database/database.pp | 14 +++++++++++++- + 1 file changed, 13 insertions(+), 1 deletion(-) + +diff --git a/deployment/puppet/osnailyfacter/manifests/database/database.pp b/deployment/puppet/osnailyfacter/manifests/database/database.pp +index 4fca959..ce406d7 100644 +--- a/deployment/puppet/osnailyfacter/manifests/database/database.pp ++++ b/deployment/puppet/osnailyfacter/manifests/database/database.pp +@@ -253,12 +253,24 @@ class osnailyfacter::database::database { + package_name => $mysql_package_name, + } + ++ # remove ssl - not used and confuse Percona XtraBackup 2.4.5 ++ # see ARMBAND-197: https://jira.opnfv.org/browse/ARMBAND-197 ++ $percona_hotfix = { ++ 'mysqld' => { ++ 'ssl' => undef, ++ 'ssl-ca' => undef, ++ 'ssl-cert' => undef, ++ 'ssl-key' => undef ++ } ++ } ++ + # build our mysql options to be configured in my.cnf + $mysql_override_options = mysql_deepmerge( + $fuel_override_options, + $ignore_db_dir_options, + $binary_logs_options, +- $syslog_options ++ $syslog_options, ++ $percona_hotfix + ) + $galera_options = mysql_deepmerge($wsrep_options, $vendor_override_options) + $override_options = mysql_deepmerge($mysql_override_options, $galera_options) diff --git a/patches/fuel-library/0002-ceilometer-Fix-libvirt-bin-group-name-for-armband.patch b/patches/fuel-library/0002-ceilometer-Fix-libvirt-bin-group-name-for-armband.patch deleted file mode 100644 index 17700a9a..00000000 --- a/patches/fuel-library/0002-ceilometer-Fix-libvirt-bin-group-name-for-armband.patch +++ /dev/null @@ -1,46 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Mon, 18 Apr 2016 18:44:27 +0200 -Subject: [PATCH] ceilometer: Fix libvirt-bin group name for armband. - -Upstream mirrors use a custom version of libvirt, ported from -Debian, while keeping the Ubuntu ceilometer package, leading to a -libvirt group name mismatch between the two. - -Fuel-library hardcodes a deb-version test for libvirt-bin, which -enforces the use of "libvirt" instead of "libvirtd" for 1.2.9 or -newer libvirt-bin [1]. - -Armband brings its own 1.3.2 libvirt-bin package, which respects -Ubuntu standard group naming ("libvirtd"), but since 1.3.2 > 1.2.9, -the old group name ("libvirt") is still used. - -This patch extends the version checking introduced in [1]: - - libvirt-bin 1.3.2 or newer will use "libvirtd" naming; - - libvirt-bin 1.2.9 ... <1.3.2 will use "libvirt". - -[1] https://review.openstack.org/#/c/200602/8/deployment/puppet/ - openstack/manifests/ceilometer.pp ---- - deployment/puppet/openstack_tasks/manifests/ceilometer/compute.pp | 8 +++++++- - 1 file changed, 7 insertions(+), 1 deletion(-) - -diff --git a/deployment/puppet/openstack_tasks/manifests/ceilometer/compute.pp b/deployment/puppet/openstack_tasks/manifests/ceilometer/compute.pp -index af4f0e5..e8dbfcf 100644 ---- a/deployment/puppet/openstack_tasks/manifests/ceilometer/compute.pp -+++ b/deployment/puppet/openstack_tasks/manifests/ceilometer/compute.pp -@@ -93,8 +93,14 @@ class openstack_tasks::ceilometer::compute { - } - - if $::operatingsystem == 'Ubuntu' and $::ceilometer::params::libvirt_group { -+ # Armband libvirt-bin deb package (1.3.2 version) creates 'libvirtd' group on Ubuntu -+ if (versioncmp($::libvirt_package_version, '1.3.2') >= 0) { -+ User<| name == 'ceilometer' |> { -+ groups => ['nova', 'libvirtd'], -+ } -+ } - # Our libvirt-bin deb package (1.2.9 version) creates 'libvirt' group on Ubuntu -- if (versioncmp($::libvirt_package_version, '1.2.9') >= 0) { -+ elsif (versioncmp($::libvirt_package_version, '1.2.9') >= 0) { - User<| name == 'ceilometer' |> { - groups => ['nova', 'libvirt'], - } diff --git a/patches/fuel-library/0003-compute-Fix-virtlogd-not-started-after-install.patch b/patches/fuel-library/0003-compute-Fix-virtlogd-not-started-after-install.patch deleted file mode 100644 index a9a49331..00000000 --- a/patches/fuel-library/0003-compute-Fix-virtlogd-not-started-after-install.patch +++ /dev/null @@ -1,33 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Fri, 15 Jul 2016 17:22:44 +0200 -Subject: [PATCH] compute: Fix virtlogd not started after install - -libvirt >= 1.3.0 installs additional services, including virtlogd, -which need to be started explicitly after install, as the Ubuntu UCA -package is currently broken. - -Fuel code already carries a fix for this, but only for UCA libvirt. -Refactor the condition for the existing fix to check package version -instead of origin. - -Closes-bug: https://jira.opnfv.org/browse/ARMBAND-37 - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - deployment/puppet/openstack_tasks/manifests/roles/compute.pp | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp -index 164bd70..211fefa 100644 ---- a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp -+++ b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp -@@ -337,7 +337,8 @@ class openstack_tasks::roles::compute { - - # TODO(aschultz): work around until https://review.openstack.org/#/c/306677/ - # lands. -- if $::os_package_type == 'ubuntu' { -+ # Armband: also apply for our libvirt-bin deb package (>=1.3.0 version) -+ if (versioncmp($::libvirt_package_version, '1.3.0') >= 0) { - ensure_resource('service', ['virtlogd','virtlockd'], { - ensure => running, - enable => true, diff --git a/patches/fuel-library/0004-puppet-cobbler-Fix-CentOS-version-detection.patch b/patches/fuel-library/0004-puppet-cobbler-Fix-CentOS-version-detection.patch deleted file mode 100644 index 1ada6159..00000000 --- a/patches/fuel-library/0004-puppet-cobbler-Fix-CentOS-version-detection.patch +++ /dev/null @@ -1,39 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Wed, 14 Dec 2016 15:40:41 +0100 -Subject: [PATCH] puppet: cobbler: Fix CentOS version detection - -Recently, CentOS released version 7.3.1611, which triggered a latent -bug in fuel-library puppet scripts related to differentiating between -6.x and 7.x major versions: 7.3.1611 is wrongly detected as 6.x, -leading to the wrong packages being attempted to be installed, -leaving nailgun server service broken on the Fuel Master node. - -This change forces the match at the beggining of the version string -(e.g. "/^6.+/: {" instead of "/6.+/: {"), so the new CentOS 7.3.1611 -no longer matches the condition for 6.x). - -Closes-bug: 1649895 - -Change-Id: I8873701d24e406e2804b9e654decb776ec3bf61c -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - deployment/puppet/cobbler/manifests/packages.pp | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) - -diff --git a/deployment/puppet/cobbler/manifests/packages.pp b/deployment/puppet/cobbler/manifests/packages.pp -index 267c18c..3c14ea1 100644 ---- a/deployment/puppet/cobbler/manifests/packages.pp -+++ b/deployment/puppet/cobbler/manifests/packages.pp -@@ -24,10 +24,10 @@ class cobbler::packages { - $openssh_package = 'openssh-clients' - $pexpect_package = 'pexpect' - case $::operatingsystemrelease { -- /6.+/: { -+ /^6.+/: { - $cobbler_additional_packages = ['xinetd', 'tftp-server', 'syslinux', 'wget', 'python-ipaddr','fence-agents', 'bind-utils'] - } -- /7.+/: { -+ /^7.+/: { - $cobbler_additional_packages = ['xinetd', 'tftp-server', 'syslinux', 'wget', 'python-ipaddr','fence-agents-all', 'bind-utils'] - } - } diff --git a/patches/fuel-library/arm64-bug-fixes/0001-nova-config-Disable-usb-tablet-for-aarch64.patch b/patches/fuel-library/arm64-bug-fixes/0001-nova-config-Disable-usb-tablet-for-aarch64.patch index f4e19229..eb611bfc 100644 --- a/patches/fuel-library/arm64-bug-fixes/0001-nova-config-Disable-usb-tablet-for-aarch64.patch +++ b/patches/fuel-library/arm64-bug-fixes/0001-nova-config-Disable-usb-tablet-for-aarch64.patch @@ -2,22 +2,29 @@ From: Stanislaw Kardach <stanislaw.kardach@cavium.com> Date: Tue, 15 Mar 2016 11:33:52 +0100 Subject: [PATCH] nova config: Disable usb tablet for aarch64 +[ Alexandru Avadanii ] +Rebased for Newton: reworked use_usb_tablet using pointer_model. + +JIRA: ARMBAND-194 + +Signed-off-by: Stanislaw Kardach <stanislaw.kardach@cavium.com> +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> --- deployment/puppet/openstack_tasks/manifests/roles/compute.pp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp -index 211fefa..4240126 100644 +index 5b247b0..0a1a1c7 100644 --- a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp +++ b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp -@@ -379,6 +379,10 @@ class openstack_tasks::roles::compute { - } - - nova_config { -+ 'libvirt/use_usb_tablet': value => $::architecture ? { -+ /(arm64|aarch64)/ => false, -+ default => true, -+ }; +@@ -391,6 +391,10 @@ class openstack_tasks::roles::compute { 'libvirt/live_migration_flag': value => 'VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_PERSIST_DEST'; 'libvirt/block_migration_flag': value => 'VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE,VIR_MIGRATE_NON_SHARED_INC'; 'DEFAULT/connection_type': value => 'libvirt'; ++ 'DEFAULT/pointer_model': value => $::architecture ? { ++ /(arm64|aarch64)/ => ps2mouse, ++ default => usbtablet, ++ }; + } + + # TODO (iberezovskiy): rework this option management once it's available in puppet-nova module diff --git a/patches/fuel-library/arm64-bug-fixes/0002-Install-vgabios-and-link-for-aarch64.patch b/patches/fuel-library/arm64-bug-fixes/0002-Install-vgabios-and-link-for-aarch64.patch index da152ead..9b59349e 100644 --- a/patches/fuel-library/arm64-bug-fixes/0002-Install-vgabios-and-link-for-aarch64.patch +++ b/patches/fuel-library/arm64-bug-fixes/0002-Install-vgabios-and-link-for-aarch64.patch @@ -17,9 +17,9 @@ index 4240126..b18fa7b 100644 --- a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp +++ b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp @@ -409,6 +409,14 @@ class openstack_tasks::roles::compute { - vncserver_listen => '0.0.0.0', - remove_unused_original_minimum_age_seconds => pick($nova_hash_real['remove_unused_original_minimum_age_seconds'], '86400'), - libvirt_service_name => $::nova::params::libvirt_service_name, + libvirt_service_name => 'libvirt-bin', + virtlock_service_name => 'virtlockd', + virtlog_service_name => 'virtlogd', + } -> + package { 'vgabios': + ensure => present; diff --git a/patches/fuel-library/arm64-bug-fixes/0003-nova-Update-vga-console-defaults-for-armv7-aarch64.patch b/patches/fuel-library/arm64-bug-fixes/0003-nova-Update-vga-console-defaults-for-armv7-aarch64.patch index 9e9c3d53..be481a78 100644 --- a/patches/fuel-library/arm64-bug-fixes/0003-nova-Update-vga-console-defaults-for-armv7-aarch64.patch +++ b/patches/fuel-library/arm64-bug-fixes/0003-nova-Update-vga-console-defaults-for-armv7-aarch64.patch @@ -75,18 +75,17 @@ index b18fa7b..2f0f2a8 100644 case $::osfamily { 'RedHat': { # From legacy libvirt.pp -@@ -290,6 +296,7 @@ class openstack_tasks::roles::compute { +@@ -290,5 +296,6 @@ class openstack_tasks::roles::compute { } $notify_on_state_change = 'vm_and_task_state' + $nova_path = '/usr/lib/python2.7/dist-packages/nova' class { '::nova': - rpc_backend => $rpc_backend_real, -@@ -311,6 +318,18 @@ class openstack_tasks::roles::compute { - memcached_servers => $memcached_addresses, - cinder_catalog_info => pick($nova_hash_real['cinder_catalog_info'], 'volumev2:cinderv2:internalURL'), - rabbit_heartbeat_timeout_threshold => $::os_service_default, +@@ -311,5 +318,17 @@ class openstack_tasks::roles::compute { + rabbit_heartbeat_timeout_threshold => $rabbit_heartbeat_timeout_threshold, + rabbit_heartbeat_rate => $rabbit_heartbeat_rate, + os_region_name => $region_name, + } -> + # FIXME(armband): Workaround for missing arm defaults in nova libvirt driver + file { "${nova_path}/libvirt-vga-console.patch": @@ -101,4 +100,3 @@ index b18fa7b..2f0f2a8 100644 + require => [Package['patch']], } - class { '::nova::availability_zone': diff --git a/patches/fuel-library/arm64-bug-fixes/0004-AArch64-nova-libvirt-Use-host-model-cpu.patch b/patches/fuel-library/arm64-bug-fixes/0004-AArch64-nova-libvirt-Use-host-model-cpu.patch new file mode 100644 index 00000000..aff89365 --- /dev/null +++ b/patches/fuel-library/arm64-bug-fixes/0004-AArch64-nova-libvirt-Use-host-model-cpu.patch @@ -0,0 +1,35 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Thu, 19 Jan 2017 19:19:28 +0100 +Subject: [PATCH] AArch64: nova: libvirt: Use host-model cpu + +Closes-bug: https://jira.opnfv.org/browse/ARMBAND-193 + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + deployment/puppet/openstack_tasks/manifests/roles/compute.pp | 11 ++++++++++- + 1 file changed, 10 insertions(+), 1 deletion(-) + +diff --git a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp +index a6f4729..f6e6698 100644 +--- a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp ++++ b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp +@@ -319,9 +319,18 @@ class openstack_tasks::roles::compute { + # guest OS by using a combination of CPU features and other parameters (such + # as CPUID level) that don't work. Until these issues are fixed, it's a good + # idea to avoid using host-model ++ # NOTE(armband): AArch64: Use host-model cpu + # http://libvirt.org/formatdomain.html#elementsCPU + # https://bugs.launchpad.net/mos/+bug/1618473 +- $libvirt_cpu_mode = 'none' ++ # https://jira.opnfv.org/browse/ARMBAND-193 ++ if str2bool($::is_virtual) { ++ $libvirt_cpu_mode = 'none' ++ } else { ++ $libvirt_cpu_mode = $::architecture ? { ++ /(arm64|aarch64)/ => 'host-model', ++ default => 'none', ++ } ++ } + + # Install / configure nova-compute + diff --git a/patches/fuel-library/arm64-bug-fixes/0004-nova-AArch64-enable-hugepage-support.patch b/patches/fuel-library/arm64-bug-fixes/0004-nova-AArch64-enable-hugepage-support.patch deleted file mode 100644 index ab0fb664..00000000 --- a/patches/fuel-library/arm64-bug-fixes/0004-nova-AArch64-enable-hugepage-support.patch +++ /dev/null @@ -1,93 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Fri, 16 Sep 2016 21:31:56 +0200 -Subject: [PATCH] nova: AArch64: enable hugepage support - -This patch adds AArch64 to the hardcoded lists of valid -architectures for hugepage support. - -Closes-bug: ARMBAND-90 - -Signed-off-by: Veena Lingadahalli <vlingadahalli@mvista.com> -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - .../nova-AArch64-enable-hugepage-support.patch | 47 ++++++++++++++++++++++ - .../openstack_tasks/manifests/roles/compute.pp | 12 ++++++ - 2 files changed, 59 insertions(+) - create mode 100644 deployment/puppet/openstack/files/nova-AArch64-enable-hugepage-support.patch - -diff --git a/deployment/puppet/openstack/files/nova-AArch64-enable-hugepage-support.patch b/deployment/puppet/openstack/files/nova-AArch64-enable-hugepage-support.patch -new file mode 100644 -index 0000000..72d6bfc ---- /dev/null -+++ b/deployment/puppet/openstack/files/nova-AArch64-enable-hugepage-support.patch -@@ -0,0 +1,47 @@ -+From: Veena Lingadahalli <vlingadahalli@mvista.com> -+Date: Fri Sep 16 17:52:02 CEST 2016 -+Subject: [PATCH] nova: AArch64: enable hugepage support -+ -+This patch adds AArch64 to the hardcoded lists of valid -+architectures for hugepage support. -+ -+Signed-off-by: Veena Lingadahalli <vlingadahalli@mvista.com> -+Acked-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -+--- -+ -+diff --git a/compute/arch.py a/compute/arch.py -+index 265f853..6a4b4a3 100644 -+--- a/compute/arch.py -++++ a/compute/arch.py -+@@ -156,6 +156,9 @@ def canonicalize(name): -+ if newname == "amd64": -+ newname = X86_64 -+ -++ if newname == "aarch64": -++ newname = AARCH64 -++ -+ if not is_valid(newname): -+ raise exception.InvalidArchitectureName(arch=name) -+ -+diff --git a/virt/libvirt/driver.py a/virt/libvirt/driver.py -+index bac17cf..0e0bcae 100644 -+--- a/virt/libvirt/driver.py -++++ a/virt/libvirt/driver.py -+@@ -5206,7 +5206,7 @@ class LibvirtDriver(driver.ComputeDriver): -+ self._bad_libvirt_numa_version_warn = True -+ return False -+ -+- support_matrix = {(arch.I686, arch.X86_64): MIN_LIBVIRT_NUMA_VERSION, -++ support_matrix = {(arch.I686, arch.X86_64, arch.AARCH64): MIN_LIBVIRT_NUMA_VERSION, -+ (arch.PPC64, -+ arch.PPC64LE): MIN_LIBVIRT_NUMA_VERSION_PPC} -+ caps = self._host.get_capabilities() -+@@ -5222,7 +5222,7 @@ class LibvirtDriver(driver.ComputeDriver): -+ def _has_hugepage_support(self): -+ # This means that the host can support multiple values for the size -+ # field in LibvirtConfigGuestMemoryBackingPage -+- supported_archs = [arch.I686, arch.X86_64] -++ supported_archs = [arch.I686, arch.X86_64, arch.AARCH64] -+ caps = self._host.get_capabilities() -+ return ((caps.host.cpu.arch in supported_archs) and -+ self._host.has_min_version(MIN_LIBVIRT_HUGEPAGE_VERSION, -diff --git a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp -index 2f0f2a8..dc61033 100644 ---- a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp -+++ b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp -@@ -330,6 +330,18 @@ class openstack_tasks::roles::compute { - unless => "patch -p1 -R -N --dry-run < ${nova_path}/libvirt-vga-console.patch", - cwd => $nova_path, - require => [Package['patch']], -+ } -> -+ # FIXME(armband): Add AArch64 to arch list for nova hugepage support -+ file { "${nova_path}/nova-AArch64-enable-hugepage-support.patch": -+ ensure => "file", -+ source => "puppet:///modules/openstack/nova-AArch64-enable-hugepage-support.patch", -+ } -> -+ exec { 'nova AArch64 enable hugepage support': -+ path => ['/usr/bin'], -+ command => "patch -p1 < ${nova_path}/nova-AArch64-enable-hugepage-support.patch", -+ unless => "patch -p1 -R -N --dry-run < ${nova_path}/nova-AArch64-enable-hugepage-support.patch", -+ cwd => $nova_path, -+ require => [Package['patch']], - } - - class { '::nova::availability_zone': diff --git a/patches/fuel-library/arm64-bug-fixes/0005-nova-libvirt-fix-delete-instance-with-nvram.patch b/patches/fuel-library/arm64-bug-fixes/0005-nova-libvirt-fix-delete-instance-with-nvram.patch new file mode 100644 index 00000000..9bf9a5c4 --- /dev/null +++ b/patches/fuel-library/arm64-bug-fixes/0005-nova-libvirt-fix-delete-instance-with-nvram.patch @@ -0,0 +1,117 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Thu, 19 Jan 2017 23:03:54 +0100 +Subject: [PATCH] nova: libvirt: fix delete instance with nvram + +Backported from [1]. + +Closes-bug: 1567807 + +[1] https://review.openstack.org/#/q/ + 539d381434ccadcdc3f5d58c2705c35558a3a065 + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + .../files/nova-libvirt-delete-with-nvram.patch | 70 ++++++++++++++++++++++ + .../openstack_tasks/manifests/roles/compute.pp | 12 ++++ + 2 files changed, 82 insertions(+) + create mode 100644 deployment/puppet/openstack/files/nova-libvirt-delete-with-nvram.patch + +diff --git a/deployment/puppet/openstack/files/nova-libvirt-delete-with-nvram.patch b/deployment/puppet/openstack/files/nova-libvirt-delete-with-nvram.patch +new file mode 100644 +index 0000000..5d4f67c +--- /dev/null ++++ b/deployment/puppet/openstack/files/nova-libvirt-delete-with-nvram.patch +@@ -0,0 +1,70 @@ ++From: Kevin Zhao <kevin.zhao@linaro.org> ++Date: Thu, 5 Jan 2017 21:32:41 +0000 ++Subject: [PATCH] libvirt: fix nova can't delete the instance with nvram ++ ++Currently libvirt needs a flag when deleting an VM with a nvram file, ++without which nova can't delete an instance booted with UEFI. Add ++deletion flag for NVRAM. Also add a test case. ++ ++[ Alexandru Avadanii ] ++Removed chunks affecting tests and adapted for OPNFV Armband. ++ ++Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ++Co-authored-by: Derek Higgins <derekh@redhat.com> ++Change-Id: I46baa952b6c3a1a4c5cf2660931f317cafb5757d ++Closes-Bug: #1567807 ++--- ++ ++diff --git a/virt/libvirt/driver.py b/virt/libvirt/driver.py ++--- a/virt/libvirt/driver.py +++++ b/virt/libvirt/driver.py ++@@ -903,7 +903,8 @@ class LibvirtDriver(driver.ComputeDriver): ++ try: ++ guest = self._host.get_guest(instance) ++ try: ++- guest.delete_configuration() +++ support_uefi = self._has_uefi_support() +++ guest.delete_configuration(support_uefi) ++ except libvirt.libvirtError as e: ++ with excutils.save_and_reraise_exception(): ++ errcode = e.get_error_code() ++@@ -1241,7 +1242,8 @@ class LibvirtDriver(driver.ComputeDriver): ++ # If any part of this block fails, the domain is ++ # re-defined regardless. ++ if guest.has_persistent_configuration(): ++- guest.delete_configuration() +++ support_uefi = self._has_uefi_support() +++ guest.delete_configuration(support_uefi) ++ ++ # Start copy with VIR_DOMAIN_REBASE_REUSE_EXT flag to ++ # allow writing to existing external volume file ++@@ -1760,7 +1762,8 @@ class LibvirtDriver(driver.ComputeDriver): ++ # If any part of this block fails, the domain is ++ # re-defined regardless. ++ if guest.has_persistent_configuration(): ++- guest.delete_configuration() +++ support_uefi = self._has_uefi_support() +++ guest.delete_configuration(support_uefi) ++ ++ # NOTE (rmk): Establish a temporary mirror of our root disk and ++ # issue an abort once we have a complete copy. ++diff --git a/virt/libvirt/guest.py b/virt/libvirt/guest.py ++--- a/virt/libvirt/guest.py +++++ b/virt/libvirt/guest.py ++@@ -262,11 +262,13 @@ class Guest(object): ++ yield VCPUInfo( ++ id=vcpu[0], cpu=vcpu[3], state=vcpu[1], time=vcpu[2]) ++ ++- def delete_configuration(self): +++ def delete_configuration(self, support_uefi=False): ++ """Undefines a domain from hypervisor.""" ++ try: ++- self._domain.undefineFlags( ++- libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE) +++ flags = libvirt.VIR_DOMAIN_UNDEFINE_MANAGED_SAVE +++ if support_uefi: +++ flags |= libvirt.VIR_DOMAIN_UNDEFINE_NVRAM +++ self._domain.undefineFlags(flags) ++ except libvirt.libvirtError: ++ LOG.debug("Error from libvirt during undefineFlags. %d" ++ "Retrying with undefine", self.id) +diff --git a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp +index f6e6698..dd0c034 100644 +--- a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp ++++ b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp +@@ -301,6 +301,18 @@ class openstack_tasks::roles::compute { + unless => "patch -p1 -R -N --dry-run < ${nova_path}/libvirt-vga-console.patch", + cwd => $nova_path, + require => [Package['patch']], ++ } -> ++ # FIXME(armband): Backport fix: nova delete instance with nvram ++ file { "${nova_path}/nova-libvirt-delete-with-nvram.patch": ++ ensure => "file", ++ source => "puppet:///modules/openstack/nova-libvirt-delete-with-nvram.patch", ++ } -> ++ exec { 'nova libvirt delete instance with nvram': ++ path => ['/usr/bin'], ++ command => "patch -p1 < ${nova_path}/nova-libvirt-delete-with-nvram.patch", ++ unless => "patch -p1 -R -N --dry-run < ${nova_path}/nova-libvirt-delete-with-nvram.patch", ++ cwd => $nova_path, ++ require => [Package['patch']], + } + + class { '::nova::cache': diff --git a/patches/fuel-library/direct-kernel-boot/0001-upload_cirros-Add-direct-kernel-boot-support.patch b/patches/fuel-library/direct-kernel-boot/0001-upload_cirros-Add-direct-kernel-boot-support.patch deleted file mode 100644 index f3eb0748..00000000 --- a/patches/fuel-library/direct-kernel-boot/0001-upload_cirros-Add-direct-kernel-boot-support.patch +++ /dev/null @@ -1,118 +0,0 @@ -From: Stanislaw Kardach <stanislaw.kardach@cavium.com> -Date: Tue, 15 Mar 2016 15:01:34 +0100 -Subject: [PATCH] upload_cirros: Add direct kernel boot support. - -AArch64 currently only supports direct kernel boot, so add the -functionality of uploading and connecting kernel and initramfs -images (disk formats `AKI` and `ARI`). - -Signed-off-by: Stanislaw Kardach <stanislaw.kardach@cavium.com> ---- - .../osnailyfacter/modular/astute/upload_cirros.rb | 55 ++++++++++++++++++++-- - 1 file changed, 51 insertions(+), 4 deletions(-) - -diff --git a/deployment/puppet/osnailyfacter/modular/astute/upload_cirros.rb b/deployment/puppet/osnailyfacter/modular/astute/upload_cirros.rb -index f0441b0..371d051 100755 ---- a/deployment/puppet/osnailyfacter/modular/astute/upload_cirros.rb -+++ b/deployment/puppet/osnailyfacter/modular/astute/upload_cirros.rb -@@ -51,12 +51,12 @@ end - def image_list - stdout = `glance --verbose image-list` - return_code = $?.exitstatus -- images = [] -+ images = Hash[] - stdout.split("\n").each do |line| - fields = line.split('|').map { |f| f.chomp.strip } - next if fields[1] == 'ID' - next unless fields[2] -- images << {fields[2] => fields[6]} -+ images[fields[2]] = { :id => fields[1], :status => fields[6] } - end - {:images => images, :exit_code => return_code} - end -@@ -78,6 +78,16 @@ EOF - [ stdout, return_code ] - end - -+# Calls glance update-image with a given property and value -+# Supported properties: 'kernel-id', 'ramdisk-id' -+def update_image(image_id, property, value) -+ command = "/usr/bin/openstack image set --#{property}=#{value} #{image_id}" -+ puts command -+ stdout = `#{command}` -+ return_code = $?.exitstatus -+ [ stdout, return_code ] -+end -+ - # check if Glance is online - # waited until the glance is started because when vCenter used as a glance - # backend launch may takes up to 1 minute. -@@ -133,7 +143,7 @@ end - # return true if image has been uploaded and active - def check_image(image) - list_of_images = image_list -- if list_of_images[:exit_code] == 0 && list_of_images[:images].include?(image['img_name'] => "active") -+ if list_of_images[:exit_code] == 0 && list_of_images[:images].select { |k,v| k == image['img_name'] and v[:status] == "active" }.count > 0 - return true - end - return false -@@ -144,7 +154,7 @@ end - # the first one - def cleanup_image(image) - list_of_images = image_list -- unless list_of_images[:images].select { |img_hash| img_hash.key?(image['img_name']) }.empty? -+ unless list_of_images[:images].select { |img_hash, v| img_hash == image['img_name'] }.count == 0 - delete_image(image['img_name']) - end - end -@@ -157,6 +167,41 @@ def delete_image(image_name) - [ stdout, return_code ] - end - -+# For each disk image try to find a kernel and initramfs images and -+# attach then to it via kernel_id and ramdisk_id glance properties. -+def connect_dependant_images(images) -+ # for each image -+ # get image id from glance -+ img_list = image_list -+ return_code = img_list[:exit_code] -+ if return_code == 0 -+ images.each do |image| -+ img_list[:images].each do |k,v| -+ if k == image['img_name'] -+ image['id'] = v[:id] -+ end -+ end -+ end -+ # for each image that is not in [aki, ari] -+ images.each do |image| -+ next if ['aki', 'ari'].include?(image['disk_format']) -+ images.each do |i| -+ # find aki/ari image whose name starts with this image's name -+ if i['img_name'].start_with?(image['img_name']) -+ ret = 0 -+ if i['disk_format'] == 'aki' -+ _, ret = update_image(image['id'], 'kernel-id', i['id']) -+ elsif i['disk_format'] == 'ari' -+ _, ret = update_image(image['id'], 'ramdisk-id', i['id']) -+ end -+ return_code += ret -+ end -+ end -+ end -+ end -+ return return_code -+end -+ - ######################## - - wait_for_glance -@@ -180,6 +225,8 @@ if errors > 0 - cleanup_image(image) - end - exit 1 -+elsif connect_dependant_images(test_vm_images) > 0 -+ exit 2 - end - - exit 0 diff --git a/patches/fuel-library/direct-kernel-boot/0002-nova-Fix-inject-for-direct-boot-with-part-table.patch b/patches/fuel-library/direct-kernel-boot/0002-nova-Fix-inject-for-direct-boot-with-part-table.patch index 7cc743af..51c66f76 100644 --- a/patches/fuel-library/direct-kernel-boot/0002-nova-Fix-inject-for-direct-boot-with-part-table.patch +++ b/patches/fuel-library/direct-kernel-boot/0002-nova-Fix-inject-for-direct-boot-with-part-table.patch @@ -71,9 +71,7 @@ diff --git a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp b/depl index dc61033..3fcfb51 100644 --- a/deployment/puppet/openstack_tasks/manifests/roles/compute.pp +++ b/deployment/puppet/openstack_tasks/manifests/roles/compute.pp -@@ -319,6 +319,18 @@ class openstack_tasks::roles::compute { - cinder_catalog_info => pick($nova_hash_real['cinder_catalog_info'], 'volumev2:cinderv2:internalURL'), - rabbit_heartbeat_timeout_threshold => $::os_service_default, +@@ -319,4 +319,16 @@ class openstack_tasks::roles::compute { } -> + # FIXME(armband): Fix inject for part image direct boot (bug LP #1469308) + file { "${nova_path}/libvirt-inject.patch": diff --git a/patches/fuel-library/multiarch-efi/0001-Add-arm64-templates-to-cobbler-and-nailgun.patch b/patches/fuel-library/multiarch-efi/0001-Add-arm64-templates-to-cobbler-and-nailgun.patch index 262bd994..bf5db54e 100644 --- a/patches/fuel-library/multiarch-efi/0001-Add-arm64-templates-to-cobbler-and-nailgun.patch +++ b/patches/fuel-library/multiarch-efi/0001-Add-arm64-templates-to-cobbler-and-nailgun.patch @@ -6,13 +6,22 @@ Add grub template import to cobbler server manifest Add arm64 templates to cobbler and nailgun -[Matei Valeanu] Add grub template to cobbler for MAC-files +[ Matei Valeanu ] +Add grub template to cobbler for MAC-files Limitation: grub2 needs to be installed directly on the root (/) of a device or partition, as the MAC-files will search for /grub/grub.cfg on the available partitions +[ Alexandru Avadanii ] +Rebased for Fuel Newton after upstream change: +"Remove deprecated cobbler parts for classic provisioning" [1]. + +[1] https://github.com/openstack/fuel-library/commit/ + e207593d7b3234cfbde6689dbccdcafe6ba67e58 + Signed-off-by: Stanislaw Kardach <stanislaw.kardach@cavium.com> Signed-off-by: Matei Valeanu <Matei.Valeanu@enea.com> +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> --- deployment/puppet/cobbler/manifests/server.pp | 36 ++++++++++++++++++++++ .../puppet/cobbler/templates/dnsmasq.template.erb | 2 ++ @@ -20,16 +29,13 @@ Signed-off-by: Matei Valeanu <Matei.Valeanu@enea.com> .../cobbler/templates/grublocal.template.erb | 2 ++ .../cobbler/templates/grubprofile.template.erb | 12 ++++++++ .../cobbler/templates/grubsystem.template.erb | 16 ++++++++++ - .../templates/preseed/ubuntu-1404.preseed.erb | 2 ++ deployment/puppet/fuel/manifests/auxiliaryrepos.pp | 12 +++++--- deployment/puppet/fuel/manifests/cobbler.pp | 36 +++++++++++++++++----- - .../puppet/fuel/manifests/cobbler/preseed.pp | 13 ++++++++ 10 files changed, 125 insertions(+), 11 deletions(-) create mode 100644 deployment/puppet/cobbler/templates/efidefault.template.erb create mode 100644 deployment/puppet/cobbler/templates/grublocal.template.erb create mode 100644 deployment/puppet/cobbler/templates/grubprofile.template.erb create mode 100644 deployment/puppet/cobbler/templates/grubsystem.template.erb - create mode 100644 deployment/puppet/fuel/manifests/cobbler/preseed.pp diff --git a/deployment/puppet/cobbler/manifests/server.pp b/deployment/puppet/cobbler/manifests/server.pp index 891dff7..52936d5 100644 @@ -149,22 +155,6 @@ index 0000000..9a191bc + initrd (tftp)$initrd_path + echo 'Booting bootstrap image ...' +} -diff --git a/deployment/puppet/cobbler/templates/preseed/ubuntu-1404.preseed.erb b/deployment/puppet/cobbler/templates/preseed/ubuntu-1404.preseed.erb -index 6762153..25f8067 100644 ---- a/deployment/puppet/cobbler/templates/preseed/ubuntu-1404.preseed.erb -+++ b/deployment/puppet/cobbler/templates/preseed/ubuntu-1404.preseed.erb -@@ -66,9 +66,11 @@ d-i apt-setup/mirror/error select Retry - # Use network mirror - d-i apt-setup/use_mirror boolean true - -+<% if @arch == 'amd64' %> - # Select architecture to amd64. That's very important to dpkg, since - # by default we have both amd64 and i386. - d-i apt-setup/multiarch string amd64 -+<% end %> - - # You can choose to install restricted and universe software, or to install - # software from the backports repository. diff --git a/deployment/puppet/fuel/manifests/auxiliaryrepos.pp b/deployment/puppet/fuel/manifests/auxiliaryrepos.pp index 493ffc6..909fc92 100644 --- a/deployment/puppet/fuel/manifests/auxiliaryrepos.pp @@ -199,61 +189,6 @@ diff --git a/deployment/puppet/fuel/manifests/cobbler.pp b/deployment/puppet/fue index b403733..c89027d 100644 --- a/deployment/puppet/fuel/manifests/cobbler.pp +++ b/deployment/puppet/fuel/manifests/cobbler.pp -@@ -114,12 +114,9 @@ class fuel::cobbler( - require => Class['::cobbler::server'], - } - -- file { '/var/lib/cobbler/kickstarts/ubuntu-amd64.preseed': -- content => template('cobbler/preseed/ubuntu-1404.preseed.erb'), -- owner => 'root', -- group => 'root', -- mode => '0644', -- require => Class['::cobbler::server'], -+ fuel::cobbler::preseed { 'ubuntu_1404_x86_64.preseed': -+ path =>'/var/lib/cobbler/kickstarts/ubuntu-amd64.preseed', -+ arch => "amd64", - } -> - - cobbler_distro { 'ubuntu_1404_x86_64': -@@ -132,6 +129,21 @@ class fuel::cobbler( - require => Class['::cobbler::server'], - } - -+ fuel::cobbler::preseed { 'ubuntu_1404_arm64.preseed': -+ path => '/var/lib/cobbler/kickstarts/ubuntu-arm64.preseed', -+ arch => "arm64", -+ } -> -+ -+ cobbler_distro { 'ubuntu_1404_arm64': -+ kernel => "${repo_root}/ubuntu/x86_64/images/linux", -+ initrd => "${repo_root}/ubuntu/x86_64/images/initrd.gz", -+ arch => 'x86_64', # Makes Cobbler choose PXE -+ breed => 'ubuntu', -+ osversion => 'trusty', -+ ksmeta => '', -+ require => Class['::cobbler::server'], -+ } -+ - cobbler_profile { 'centos-x86_64': - kickstart => '/var/lib/cobbler/kickstarts/centos-x86_64.ks', - kopts => 'biosdevname=0 sshd=1 dhcptimeout=120', -@@ -152,6 +164,16 @@ class fuel::cobbler( - require => Cobbler_distro['ubuntu_1404_x86_64'], - } - -+ cobbler_profile { 'ubuntu_1404_arm64': -+ kickstart => '/var/lib/cobbler/kickstarts/ubuntu-arm64.preseed', -+ kopts => 'console=ttyAMA0,115200 console=ttyS0,115200 net.ifnames=0 biosdevname=0 netcfg/choose_interface=eth0 netcfg/dhcp_timeout=120 netcfg/link_detection_timeout=20', -+ distro => 'ubuntu_1404_arm64', -+ ksmeta => '', -+ menu => false, -+ server => $real_server, -+ require => Cobbler_distro['ubuntu_1404_arm64'], -+ } -+ - cobbler_distro { 'ubuntu_bootstrap': - kernel => "${bootstrap_path}/vmlinuz", - initrd => "${bootstrap_path}/initrd.img", @@ -166,7 +188,7 @@ class fuel::cobbler( distro => 'ubuntu_bootstrap', menu => true, @@ -263,22 +198,3 @@ index b403733..c89027d 100644 ksmeta => '', server => $real_server, require => Cobbler_distro['ubuntu_bootstrap'], -diff --git a/deployment/puppet/fuel/manifests/cobbler/preseed.pp b/deployment/puppet/fuel/manifests/cobbler/preseed.pp -new file mode 100644 -index 0000000..7e1bee9 ---- /dev/null -+++ b/deployment/puppet/fuel/manifests/cobbler/preseed.pp -@@ -0,0 +1,13 @@ -+define fuel::cobbler::preseed( -+ $path, -+ $arch = "amd64", -+) { -+ -+ file { $path: -+ content => template('cobbler/preseed/ubuntu-1404.preseed.erb'), -+ owner => 'root', -+ group => 'root', -+ mode => '0644', -+ require => Class['::cobbler::server'], -+ } -+} diff --git a/patches/fuel-library/multiarch-efi/0002-Add-cobbler-grub-aarch64-to-cobbler-dependencies.patch b/patches/fuel-library/multiarch-efi/0002-Add-cobbler-grub-aarch64-to-cobbler-dependencies.patch index 8314663f..f4edfeae 100644 --- a/patches/fuel-library/multiarch-efi/0002-Add-cobbler-grub-aarch64-to-cobbler-dependencies.patch +++ b/patches/fuel-library/multiarch-efi/0002-Add-cobbler-grub-aarch64-to-cobbler-dependencies.patch @@ -9,27 +9,22 @@ on the Fuel master node. [Alexandru.Avadanii@enea.com] Reworked based on Florin's work for the docker container. +Rebased for Fuel Newton. Signed-off-by: Florin Dumitrascu <florin.dumitrascu@enea.com> Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> --- - deployment/puppet/cobbler/manifests/packages.pp | 4 ++-- - 1 file changed, 2 insertions(+), 2 deletions(-) + deployment/puppet/cobbler/manifests/packages.pp | 2 +- + 1 file changed, 1 insertions(+), 1 deletions(-) diff --git a/deployment/puppet/cobbler/manifests/packages.pp b/deployment/puppet/cobbler/manifests/packages.pp index 267c18c..9782c4c 100644 --- a/deployment/puppet/cobbler/manifests/packages.pp +++ b/deployment/puppet/cobbler/manifests/packages.pp -@@ -25,10 +25,10 @@ class cobbler::packages { - $pexpect_package = 'pexpect' - case $::operatingsystemrelease { - /^6.+/: { -- $cobbler_additional_packages = ['xinetd', 'tftp-server', 'syslinux', 'wget', 'python-ipaddr','fence-agents', 'bind-utils'] -+ $cobbler_additional_packages = ['xinetd', 'tftp-server', 'syslinux', 'wget', 'python-ipaddr','fence-agents', 'bind-utils', 'cobbler-grub-aarch64'] - } - /^7.+/: { -- $cobbler_additional_packages = ['xinetd', 'tftp-server', 'syslinux', 'wget', 'python-ipaddr','fence-agents-all', 'bind-utils'] -+ $cobbler_additional_packages = ['xinetd', 'tftp-server', 'syslinux', 'wget', 'python-ipaddr','fence-agents-all', 'bind-utils', 'cobbler-grub-aarch64'] - } - } +@@ -23,5 +23,5 @@ class cobbler::packages { + $django_package = 'python-django' + $openssh_package = 'openssh-clients' + $pexpect_package = 'pexpect' +- $cobbler_additional_packages = ['xinetd', 'tftp-server', 'syslinux', 'wget', 'python-ipaddr','fence-agents-all', 'bind-utils'] ++ $cobbler_additional_packages = ['xinetd', 'tftp-server', 'syslinux', 'wget', 'python-ipaddr','fence-agents-all', 'bind-utils', 'cobbler-grub-aarch64'] } diff --git a/patches/fuel-library/multiarch-fuel/0002-Make-TestVM-creation-architecture-aware.patch b/patches/fuel-library/multiarch-fuel/0002-Make-TestVM-creation-architecture-aware.patch index 325b51d4..c326c7d8 100644 --- a/patches/fuel-library/multiarch-fuel/0002-Make-TestVM-creation-architecture-aware.patch +++ b/patches/fuel-library/multiarch-fuel/0002-Make-TestVM-creation-architecture-aware.patch @@ -48,41 +48,35 @@ index 70fef32..dfefcbe 100644 --- a/deployment/puppet/osnailyfacter/manifests/generate_vms/vm_config.pp +++ b/deployment/puppet/osnailyfacter/manifests/generate_vms/vm_config.pp @@ -4,6 +4,17 @@ define osnailyfacter::generate_vms::vm_config( - $details = $name - $id = $details['id'] - + $details, + $template_dir = '/var/lib/nova', + ) { + case $::architecture { + 'arm64', 'aarch64': { + $machine_arch = 'aarch64' -+ $machine_type = 'virt' ++ $machine_type = 'virt-2.6' + } + default: { + $machine_arch = 'x86_64' -+ $machine_type = 'pc-i440fx-trusty' ++ $machine_type = 'pc' + } + } + - file { "${template_dir}/template_${id}_vm.xml": + file { "${template_dir}/template_${name}_vm.xml": owner => 'root', group => 'root', diff --git a/deployment/puppet/osnailyfacter/templates/vm_libvirt.erb b/deployment/puppet/osnailyfacter/templates/vm_libvirt.erb index ae2ba5b..64e716d 100644 --- a/deployment/puppet/osnailyfacter/templates/vm_libvirt.erb +++ b/deployment/puppet/osnailyfacter/templates/vm_libvirt.erb -@@ -3,14 +3,23 @@ +@@ -3,14 +3,17 @@ <memory unit='GiB'><%= @details['mem'] %></memory> <vcpu placement='static'><%= @details['cpu'] %></vcpu> <os> -- <type arch='x86_64' machine='pc-i440fx-trusty'>hvm</type> +- <type arch='x86_64' machine='pc'>hvm</type> + <type arch='<%= @machine_arch %>' machine='<%= @machine_type %>'>hvm</type> -+ <% if == 'aarch64' %> -+ <kernel>/var/lib/nova/<%= @details['id'] %>_vm.kernel</kernel> -+ <initrd>/var/lib/nova/<%= @details['id'] %>_vm.initramfs</initrd> -+ <cmdline>root=/dev/vda1 rw rootwait console=tty0 console=ttyS0 console=ttyAMA0</cmdline> -+ <% else %> <boot dev='network'/> <boot dev='hd'/> -+ <% end %> </os> <features> <acpi/> diff --git a/patches/fuel-library/mysql-sst-provider/0001-Allow-configuring-MySQL-WSREP-SST-provider.patch b/patches/fuel-library/mysql-sst-provider/0001-Allow-configuring-MySQL-WSREP-SST-provider.patch index 632aa8f3..f52af67a 100644 --- a/patches/fuel-library/mysql-sst-provider/0001-Allow-configuring-MySQL-WSREP-SST-provider.patch +++ b/patches/fuel-library/mysql-sst-provider/0001-Allow-configuring-MySQL-WSREP-SST-provider.patch @@ -24,17 +24,16 @@ diff --git a/deployment/puppet/osnailyfacter/manifests/database/database.pp b/de index e741ff8..be19eb5 100644 --- a/deployment/puppet/osnailyfacter/manifests/database/database.pp +++ b/deployment/puppet/osnailyfacter/manifests/database/database.pp -@@ -8,6 +8,7 @@ class osnailyfacter::database::database { - $use_syslog = hiera('use_syslog', true) - $primary_controller = hiera('primary_controller') - $mysql_hash = hiera_hash('mysql', {}) -+ $wsrep_hash = hiera_hash('mysql_wsrep', {}) - $debug = pick($mysql_hash['debug'], hiera('debug', false)) +@@ -8,5 +8,6 @@ class osnailyfacter::database::database { + $use_syslog = hiera('use_syslog', true) + $mysql_hash = hiera_hash('mysql', {}) ++ $wsrep_hash = hiera_hash('mysql_wsrep', {}) + $debug = pick($mysql_hash['debug'], hiera('debug', false)) - $mgmt_iface = get_network_role_property('mgmt/database', 'interface') + $mgmt_iface = get_network_role_property('mgmt/database', 'interface') @@ -17,6 +18,7 @@ class osnailyfacter::database::database { - $mysql_root_password = $mysql_hash['root_password'] + $deb_sysmaint_password = $mysql_hash['wsrep_password'] $enabled = pick($mysql_hash['enabled'], true) + $wsrep_sst_method = pick($wsrep_hash['wsrep_method'], 'xtrabackup-v2') diff --git a/patches/fuel-library/upstream-backports/0001-Set-net.ipv4.ip_nonlocal_bind-1-for-vrouter-namespac.patch b/patches/fuel-library/upstream-backports/0001-Set-net.ipv4.ip_nonlocal_bind-1-for-vrouter-namespac.patch deleted file mode 100644 index 281ef231..00000000 --- a/patches/fuel-library/upstream-backports/0001-Set-net.ipv4.ip_nonlocal_bind-1-for-vrouter-namespac.patch +++ /dev/null @@ -1,54 +0,0 @@ -From: Bartosz Kupidura <bkupidura@mirantis.com> -Date: Mon, 27 Jun 2016 13:12:29 +0200 -Subject: [PATCH] Set net.ipv4.ip_nonlocal_bind=1 for vrouter namespace - -Change-Id: I123af7e3b53f9a53fcd9d2818640c0bd4699e024 -Closes-Bug: #1595957 -(cherry picked from commit 244456a3b77074a6cd85fa9d33ebb03ac25decf8) ---- - files/fuel-ha-utils/ocf/ns_dns | 3 ++- - files/fuel-ha-utils/ocf/ns_vrouter | 3 ++- - 2 files changed, 4 insertions(+), 2 deletions(-) - -diff --git a/files/fuel-ha-utils/ocf/ns_dns b/files/fuel-ha-utils/ocf/ns_dns -index cdd814c..49cbd17 100644 ---- a/files/fuel-ha-utils/ocf/ns_dns -+++ b/files/fuel-ha-utils/ocf/ns_dns -@@ -140,7 +140,7 @@ exit $OCF_SUCCESS - - check_ns() { - local ns=`ip netns list | grep "$OCF_RESKEY_ns"` -- [ $ns != $OCF_RESKEY_ns ] && return $OCF_ERR_GENERIC -+ [ "$ns" != $OCF_RESKEY_ns ] && return $OCF_ERR_GENERIC - return $OCF_SUCCESS - } - -@@ -150,6 +150,7 @@ get_ns() { - - ocf_run ip netns add $OCF_RESKEY_ns - rc=$? -+ ocf_run $RUN_IN_NS /sbin/sysctl -w net.ipv4.ip_nonlocal_bind=1 - ocf_run $RUN_IN_NS ip link set up dev lo - - return $rc -diff --git a/files/fuel-ha-utils/ocf/ns_vrouter b/files/fuel-ha-utils/ocf/ns_vrouter -index a65e9cf..5cc6c98 100644 ---- a/files/fuel-ha-utils/ocf/ns_vrouter -+++ b/files/fuel-ha-utils/ocf/ns_vrouter -@@ -186,7 +186,7 @@ check_ns() { - local LH="${LL} check_ns():" - local ns=`ip netns list | grep "$OCF_RESKEY_ns"` - ocf_log debug "${LH} recieved netns list: ${ns}" -- [[ $ns != $OCF_RESKEY_ns ]] && return $OCF_ERR_GENERIC -+ [[ "$ns" != $OCF_RESKEY_ns ]] && return $OCF_ERR_GENERIC - return $OCF_SUCCESS - } - -@@ -197,6 +197,7 @@ get_ns() { - - ocf_run ip netns add $OCF_RESKEY_ns - rc=$? -+ ocf_run $RUN_IN_NS /sbin/sysctl -w net.ipv4.ip_nonlocal_bind=1 - ocf_run $RUN_IN_NS ip link set up dev lo - ocf_log debug "${LH} added netns ${OCF_RESKEY_ns} and set up lo" - diff --git a/patches/fuel-library/upstream-backports/0002-Always-wait-for-MySQL-sync-on-service-refresh.patch b/patches/fuel-library/upstream-backports/0002-Always-wait-for-MySQL-sync-on-service-refresh.patch deleted file mode 100644 index d7fac948..00000000 --- a/patches/fuel-library/upstream-backports/0002-Always-wait-for-MySQL-sync-on-service-refresh.patch +++ /dev/null @@ -1,60 +0,0 @@ -From: dmburmistrov <dmburmistrov@mirantis.com> -Date: Thu, 18 Aug 2016 21:30:26 +0300 -Subject: [PATCH] Always wait for MySQL sync on service refresh - -Always trigger Exec['wait-for-sync'] on -MySQL service refresh. - -Closes-bug: #1614647 - -Change-Id: I82ec0d46eff217daedc1bd07c67069498c515c44 ---- - deployment/puppet/cluster/manifests/mysql.pp | 6 +++--- - deployment/puppet/cluster/spec/classes/cluster_mysql_spec.rb | 6 +++++- - 2 files changed, 8 insertions(+), 4 deletions(-) - -diff --git a/deployment/puppet/cluster/manifests/mysql.pp b/deployment/puppet/cluster/manifests/mysql.pp -index c36b65d..f483cf6 100644 ---- a/deployment/puppet/cluster/manifests/mysql.pp -+++ b/deployment/puppet/cluster/manifests/mysql.pp -@@ -104,7 +104,7 @@ class cluster::mysql ( - require => Package['mysql-server'], - } ~> - -- exec { 'wait-initial-sync': -+ exec { 'wait-for-sync': - path => '/bin:/sbin:/usr/bin:/usr/sbin', - command => "mysql ${user_password_string} -Nbe \"show status like 'wsrep_local_state_comment'\" | grep -q -e Synced && sleep 10", - try_sleep => 10, -@@ -119,7 +119,7 @@ class cluster::mysql ( - } - - Exec['create-init-file'] -> -- Service<| title == $service_name |> -> -- Exec['wait-initial-sync'] -> -+ Service<| title == $service_name |> ~> -+ Exec['wait-for-sync'] -> - Exec['rm-init-file'] - } -diff --git a/deployment/puppet/cluster/spec/classes/cluster_mysql_spec.rb b/deployment/puppet/cluster/spec/classes/cluster_mysql_spec.rb -index cf2c719..72c8c2f 100644 ---- a/deployment/puppet/cluster/spec/classes/cluster_mysql_spec.rb -+++ b/deployment/puppet/cluster/spec/classes/cluster_mysql_spec.rb -@@ -34,12 +34,16 @@ describe 'cluster::mysql' do - /'username'@'localhost' IDENTIFIED BY 'password'/ - ) - should contain_exec('create-init-file').that_comes_before('Service[mysqld]') -- should contain_exec('create-init-file').that_notifies('Exec[wait-initial-sync]') -+ should contain_exec('create-init-file').that_notifies('Exec[wait-for-sync]') - end - - it 'creates exec to remove init-file' do - should contain_exec('rm-init-file') - end -+ -+ it 'creates exec to wait initial database sync' do -+ should contain_exec('wait-for-sync').that_subscribes_to('Service[mysqld]') -+ end - end - - end diff --git a/patches/fuel-library/upstream-backports/0003-Fix-MySQL-logging-dir-permissions-for-syslog.patch b/patches/fuel-library/upstream-backports/0003-Fix-MySQL-logging-dir-permissions-for-syslog.patch deleted file mode 100644 index 7dfede1b..00000000 --- a/patches/fuel-library/upstream-backports/0003-Fix-MySQL-logging-dir-permissions-for-syslog.patch +++ /dev/null @@ -1,90 +0,0 @@ -From: Maksim Malchuk <mmalchuk@mirantis.com> -Date: Wed, 24 Aug 2016 00:15:02 +0300 -Subject: [PATCH] Fix MySQL logging dir permissions for syslog - -This change fixes permissions for MySQL logging directory to enable -syslogd create and write logs files into it. - -Closes-Bug: #1615680 -Change-Id: Id29aead6619ea8ed68811d0fd80895ee79f51687 -Signed-off-by: Maksim Malchuk <mmalchuk@mirantis.com> -(cherry picked from commit 9c978278811eeebfb11ab4d2b42ebabe00c132c2) ---- - deployment/puppet/cluster/manifests/mysql.pp | 14 +++++++++++--- - .../puppet/cluster/spec/classes/cluster_mysql_spec.rb | 11 ++++++++++- - tests/noop/spec/hosts/database/database_spec.rb | 9 +++++++++ - 3 files changed, 30 insertions(+), 4 deletions(-) - -diff --git a/deployment/puppet/cluster/manifests/mysql.pp b/deployment/puppet/cluster/manifests/mysql.pp -index f483cf6..7bc8874 100644 ---- a/deployment/puppet/cluster/manifests/mysql.pp -+++ b/deployment/puppet/cluster/manifests/mysql.pp -@@ -118,8 +118,16 @@ class cluster::mysql ( - onlyif => 'test -f /tmp/wsrep-init-file', - } - -+ file { 'fix-log-dir': -+ ensure => directory, -+ path => '/var/log/mysql', -+ mode => '0770', -+ require => Package['mysql-server'], -+ } -+ - Exec['create-init-file'] -> -- Service<| title == $service_name |> ~> -- Exec['wait-for-sync'] -> -- Exec['rm-init-file'] -+ File['fix-log-dir'] -> -+ Service<| title == $service_name |> ~> -+ Exec['wait-for-sync'] -> -+ Exec['rm-init-file'] - } -diff --git a/deployment/puppet/cluster/spec/classes/cluster_mysql_spec.rb b/deployment/puppet/cluster/spec/classes/cluster_mysql_spec.rb -index 72c8c2f..ad3fc1b 100644 ---- a/deployment/puppet/cluster/spec/classes/cluster_mysql_spec.rb -+++ b/deployment/puppet/cluster/spec/classes/cluster_mysql_spec.rb -@@ -33,7 +33,7 @@ describe 'cluster::mysql' do - should contain_exec('create-init-file').with_command( - /'username'@'localhost' IDENTIFIED BY 'password'/ - ) -- should contain_exec('create-init-file').that_comes_before('Service[mysqld]') -+ should contain_exec('create-init-file').that_comes_before('File[fix-log-dir]') - should contain_exec('create-init-file').that_notifies('Exec[wait-for-sync]') - end - -@@ -41,6 +41,15 @@ describe 'cluster::mysql' do - should contain_exec('rm-init-file') - end - -+ it 'should have correct permissions for logging directory' do -+ should contain_file('fix-log-dir').with( -+ :ensure => 'directory', -+ :path => '/var/log/mysql', -+ :mode => '0770', -+ ).that_requires('Package[mysql-server]') -+ should contain_file('fix-log-dir').that_comes_before('Service[mysqld]') -+ end -+ - it 'creates exec to wait initial database sync' do - should contain_exec('wait-for-sync').that_subscribes_to('Service[mysqld]') - end -diff --git a/tests/noop/spec/hosts/database/database_spec.rb b/tests/noop/spec/hosts/database/database_spec.rb -index d4ad8d0..2c4c5c5 100644 ---- a/tests/noop/spec/hosts/database/database_spec.rb -+++ b/tests/noop/spec/hosts/database/database_spec.rb -@@ -261,6 +261,15 @@ describe manifest do - ) - end - -+ it 'should have correct permissions for logging directory' do -+ should contain_file('fix-log-dir').with( -+ :ensure => 'directory', -+ :path => '/var/log/mysql', -+ :mode => '0770', -+ ).that_requires('Package[mysql-server]') -+ should contain_file('fix-log-dir').that_comes_before('Service[mysqld]') -+ end -+ - it 'should configure galera grants service and proper flow' do - if primary_controller - should contain_class('cluster::galera_grants').with( diff --git a/patches/fuel-nailgun-agent/0003-AArch64-Add-CPU-details-detection.patch b/patches/fuel-nailgun-agent/0003-AArch64-Add-CPU-details-detection.patch new file mode 100644 index 00000000..89f4aeb4 --- /dev/null +++ b/patches/fuel-nailgun-agent/0003-AArch64-Add-CPU-details-detection.patch @@ -0,0 +1,130 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Sun, 15 Jan 2017 08:09:10 +0100 +Subject: [PATCH] AArch64: Add CPU details detection + +[ Alexandru Avadanii ] +Based on Stan's previous patch for ohai, extend fuel-nailgun-agent +to properly support AArch64 CPUs. + +[ Stanislaw Kardach ] +There is currently little human readable detail in /proc/cpuinfo on +arm64 so this patch tries to enchance this information by parsing the +DMI data (using `dmidecode`) and fail gracefully to empty strings +if no information could not be read from there. By no means this +parsing is to be taken as a standardised way of discovering an +arm64 CPU, it is just a suggestion. + +Signed-off-by: Stanislaw Kardach <stanislaw.kardach@cavium.com> +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + agent | 90 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 90 insertions(+) + +diff --git a/agent b/agent +index 0c56264..eeb4d26 100755 +--- a/agent ++++ b/agent +@@ -340,6 +340,15 @@ class NodeAgent + end + + def _get_detailed_cpuinfo ++ case `uname -m` ++ when /aarch64.*/ ++ _get_detailed_cpuinfo_for_arm64 ++ else # default to x86 ++ _get_detailed_cpuinfo_for_x86 ++ end ++ end ++ ++ def _get_detailed_cpuinfo_for_x86 + real = {} + info = {} + info[:total] = 0 +@@ -371,6 +380,87 @@ class NodeAgent + info + end + ++ def _get_detailed_cpuinfo_for_arm64 ++ cpu_implementers = { ++ "0x41" => "ARM", ++ "0x53" => "Samsung", ++ "0x51" => "Qualcom", ++ "0x43" => "Cavium", ++ "0x50" => "APM" ++ } ++ core_models = { ++ "0xd04" => "cortex-a35", ++ "0xd03" => "cortex-a53", ++ "0xd07" => "cortex-a57", ++ "0xd08" => "cortex-a72", ++ "0x001" => "exynos-m1", ++ "0x800" => "qdf24xx", ++ "0x0a1" => "thunderx", ++ "0x000" => "xgene1", ++ "0xd07.0xd03" => "cortex-a57.cortex-a53", ++ "0xd08.0xd03" => "cortex-a72.cortex-a53" ++ } ++ cpuinfo = {} ++ real_cpus = {} ++ cpu_number = -1 ++ current_cpu = nil ++ `dmidecode -t 4 -q`.each_line do |line| ++ case line ++ when /Processor Information/ ++ cpu_number +=1 ++ real_cpus[cpu_number] = {} ++ when /\s*Manufacturer:\s(.+)/ ++ real_cpus[cpu_number][:vendor_id] = $1 ++ when /\s*Family:\s(.+)/ ++ real_cpus[cpu_number][:family] = $1 ++ when /\s*Max Speed:\s(.+)\s*MHz/ ++ real_cpus[cpu_number][:mhz] = $1.strip ++ when /\s*Core Enabled:\s(.+)/ ++ real_cpus[cpu_number][:cores] = $1 ++ end ++ end ++ ++ cpu_number = 0 ++ File.open("/proc/cpuinfo").each do |line| ++ case line ++ when /processor\s+:\s(.+)/ ++ cpuinfo[$1] = {} ++ current_cpu = $1 ++ cpu_number += 1 ++ phys_id = File.read( ++ "/sys/devices/system/cpu/cpu%d/topology/physical_package_id" % ++ $1).strip ++ i = phys_id.to_i ++ cpuinfo[$1][:core_id] = ++ File.read("/sys/devices/system/cpu/cpu%d/topology/core_id" % ++ $1).strip rescue "" ++ cpuinfo[$1][:physical_id] = phys_id if not phys_id.empty? ++ if real_cpus[i].nil? ++ i = 0 ++ end ++ if not real_cpus[i].nil? ++ cpuinfo[$1][:family] = real_cpus[i][:family] rescue "" ++ cpuinfo[$1][:cores] = real_cpus[i][:cores] rescue "" ++ cpuinfo[$1][:mhz] = real_cpus[i][:mhz] rescue "" ++ end ++ when /CPU implementer\s+:\s(.+)/ ++ cpuinfo[current_cpu][:vendor_id] = cpu_implementers[$1] ++ cpuinfo[current_cpu][:vendor_id] ||= real_cpus[cpuinfo[current_cpu][:physical_id].to_i][:vendor_id] rescue nil ++ cpuinfo[current_cpu][:vendor_id] ||= real_cpus[0][:vendor_id] rescue nil ++ cpuinfo[current_cpu][:vendor_id] ||= $1 ++ when /CPU part\s+:\s(.+)/ ++ cpuinfo[current_cpu][:model] = $1 ++ cpuinfo[current_cpu][:model_name] = core_models[$1] ++ cpuinfo[current_cpu][:model_name] ||= "" ++ when /Features\s+:\s(.+)/ ++ cpuinfo[current_cpu][:flags] = $1.split(' ') ++ end ++ end ++ cpuinfo[:total] = cpu_number ++ cpuinfo[:real] = real_cpus.keys.length ++ cpuinfo ++ end ++ + def _get_blkdev_info + info = {} + if File.directory?('/sys/block/') diff --git a/patches/fuel-nailgun-agent/0004-AArch64-Read-hugepage-size-from-proc-meminfo.patch b/patches/fuel-nailgun-agent/0004-AArch64-Read-hugepage-size-from-proc-meminfo.patch new file mode 100644 index 00000000..de0ca78a --- /dev/null +++ b/patches/fuel-nailgun-agent/0004-AArch64-Read-hugepage-size-from-proc-meminfo.patch @@ -0,0 +1,30 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Sun, 15 Jan 2017 08:14:12 +0100 +Subject: [PATCH] AArch64: Read hugepage size from /proc/meminfo + +This method should be arch-indepedent, provided /proc/meminfo +reports the correct information. + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + agent | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/agent b/agent +index eeb4d26..d514de6 100755 +--- a/agent ++++ b/agent +@@ -1399,6 +1399,13 @@ class NodeAgent + def supported_hugepages + return [2048, 1048576] if _get_detailed_cpuinfo['0'][:flags].include?('pdpe1gb') + return [2048] if _get_detailed_cpuinfo['0'][:flags].include?('pse') ++ # AArch64 does not expose CPU flags, but we can rely on /proc/meminfo ++ File.open('/proc/meminfo').each do |l| ++ case l.strip ++ when /Hugepagesize:\s+(\d+)\s+kB/ ++ return [$1.to_i()] ++ end ++ end + [] + end + diff --git a/patches/fuel-plugin-opendaylight/0001-ODL-leveldb-leveldbjni-Enable-arm64-support.patch b/patches/fuel-plugin-opendaylight/0001-ODL-leveldb-leveldbjni-Enable-arm64-support.patch index 5cfa99b5..5804722e 100644 --- a/patches/fuel-plugin-opendaylight/0001-ODL-leveldb-leveldbjni-Enable-arm64-support.patch +++ b/patches/fuel-plugin-opendaylight/0001-ODL-leveldb-leveldbjni-Enable-arm64-support.patch @@ -48,22 +48,18 @@ index 69cc23a..791ceba 100644 + } } diff --git a/deployment_scripts/puppet/modules/opendaylight/manifests/install.pp b/deployment_scripts/puppet/modules/opendaylight/manifests/install.pp -index 1962609..ca98532 100644 --- a/deployment_scripts/puppet/modules/opendaylight/manifests/install.pp +++ b/deployment_scripts/puppet/modules/opendaylight/manifests/install.pp -@@ -17,6 +17,12 @@ class opendaylight::install ( - ensure => installed, - } - +@@ -15,3 +15,9 @@ class opendaylight::install ( + package { 'opendaylight': + ensure => $odl_package, ++ } ++ + if ($::osfamily == 'Debian' and $::opendaylight::arch == 'arm64') { + class { 'opendaylight::leveldbjni': -+ require => Package[$odl_package], ++ require => Package['opendaylight'], + } -+ } -+ - # quagga - class { 'opendaylight::quagga': - before => Service['opendaylight'] + } @@ -35,7 +41,8 @@ class opendaylight::install ( debug("Set odl rest api port to ${rest_port}") @@ -74,13 +70,12 @@ index 1962609..ca98532 100644 ensure => file, owner => 'odl', content => template('opendaylight/jetty.xml.erb') -@@ -71,6 +78,7 @@ class opendaylight::install ( - Package[$odl_package] -> +@@ -71,5 +78,6 @@ class opendaylight::install ( Ini_setting <||> -> Firewall <||> -> -- File <||> -> -+ File['jetty.xml'] -> -+ Class['opendaylight::leveldbjni'] -> +- File <||> ~> ++ File['jetty.xml'] ~> ++ Class['opendaylight::leveldbjni'] ~> Service['opendaylight'] } diff --git a/deployment_scripts/puppet/modules/opendaylight/manifests/leveldbjni.pp b/deployment_scripts/puppet/modules/opendaylight/manifests/leveldbjni.pp diff --git a/patches/fuel-plugin-opendaylight/0002-opnfv-quagga-Build-for-UBUNTU_ARCH.patch b/patches/fuel-plugin-opendaylight/0002-opnfv-quagga-Build-for-UBUNTU_ARCH.patch deleted file mode 100644 index e6e7716f..00000000 --- a/patches/fuel-plugin-opendaylight/0002-opnfv-quagga-Build-for-UBUNTU_ARCH.patch +++ /dev/null @@ -1,70 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Mon, 4 Jul 2016 20:44:49 +0200 -Subject: [PATCH] opnfv-quagga: Build for UBUNTU_ARCH. - -ODL itself is mostly arch-indep, but its deps (e.g. opnfv-quagga -and friends) are not. - -Introduce opnfv-quagga building/packing for multiple archs, listed -in UBUNTU_ARCH, which holds a space-separated list of target -arch(s), represented in `dpkg --print-architecture` format. -Only applicable when "BUILD_FOR=ubuntu". - -FIXME(armband): For now, only native building and prebuilt debs -are supported (no cross-building). - -Due to the design of opnfv-packaging repo, it is easier to: -- patch upstream sources to support native arm64 builds (see [1]); -- add prebuilt arm64 binaries to output dir in git repo (like amd64), -submitted as pull request upstream [2]; - -Until above OPNFV-QUAGGA changes land upstream and/or are refactored, -rely on forked repository that contains above patches [3] on -branch "stable/colorado", including prebuilt binaries (DEBs) for: -- amd64 (also available in upstream repo); -- arm64 (submitted upstream in [3]); - -[1] https://github.com/nikolas-hermanns/opnfv-quagga-packaging/pull/1 -[2] https://github.com/nikolas-hermanns/opnfv-quagga-packaging/pull/2 -[3] https://github.com/alexandruavadanii/opnfv-quagga-packaging - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - pre_build_hook | 15 +++++++++++++-- - 1 file changed, 13 insertions(+), 2 deletions(-) - -diff --git a/pre_build_hook b/pre_build_hook -index 0296821..a6d55e9 100755 ---- a/pre_build_hook -+++ b/pre_build_hook -@@ -53,9 +53,17 @@ NETWORKING_ODL_BRANCH=${NETWORKING_ODL_BRANCH:-stable/mitaka} - OPNFV_QUAGGE_PACKAGING_REPO=${OPNFV_QUAGGE_PACKAGING_REPO:-https://github.com/nikolas-hermanns/opnfv-quagga-packaging} - OPNFV_QUAGGE_PACKAGING_BRANCH=stable/colorado - --# For which systems odl package should be build -+# For which systems odl package should be built - BUILD_FOR=${BUILD_FOR:-ubuntu} - -+# For which architecture(s) package should be built -+# Only applicable when "BUILD_FOR=ubuntu" -+# ODL itself is mostly arch-indep, but its deps are not -+# All archs should be represented in `dpkg --print-architecture` format -+# UBUNTU_ARCH holds a space-separated list of target arch(s) -+# FIXME(armband): For now, only native building is supported! -+UBUNTU_ARCH=${UBUNTU_ARCH:-$(dpkg --print-architecture)} -+ - DIR="$(dirname `readlink -f $0`)" - TMP_DIR="${DIR}/tmp" - MODULES="${DIR}/deployment_scripts/puppet/modules" -@@ -86,7 +94,10 @@ function add_opnfv_quagga { - sudo apt-get -y install `cat requirements.txt` - make -j6 - fi -- cp debian_package/* $path -+ # Filter only requested archs, bail on missing DEBs -+ for ARCH in ${UBUNTU_ARCH}; do -+ cp debian_package/*_${ARCH}.deb $path -+ done - popd - popd - diff --git a/patches/fuel-plugin-opendaylight/0003-FIXME-ODL-wget-no-check-certificate-for-cache.patch b/patches/fuel-plugin-opendaylight/0003-FIXME-ODL-wget-no-check-certificate-for-cache.patch deleted file mode 100644 index 8b2d1a38..00000000 --- a/patches/fuel-plugin-opendaylight/0003-FIXME-ODL-wget-no-check-certificate-for-cache.patch +++ /dev/null @@ -1,42 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Tue, 12 Jul 2016 14:26:22 +0200 -Subject: [PATCH] FIXME: ODL wget: --no-check-certificate for cache. - -ODL tarball is normally fetched from nexus.opendaylight.org domain, -which has a very limited bandwidth, taking up to 10-15mins for our -build server to download the ODL archive at each ISO build. -This adds up to 15-25% of each ISO build time spent on downloading -ODL archive. -Therefore, we chose to set up a local HTTPS cache of nexus repos, -(just for the sake of not hardcoding the URL in our Makefile), -and just disable certificate check for wget from local URL. - -Allow using a HTTPS local cache, so we don't override the whole -URL of the ODL archive. Our local cache has HTTPS enabled, but the -certificate won't match the domain name. - -FIXME: This patch is only needed if you have a local cache of -nexus.opendaylight.org! - -Only use this option if you are otherwise convinced of the site's -authenticity, or if you really don't care about the validity of -its certificate. - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - pre_build_hook | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/pre_build_hook b/pre_build_hook -index a6d55e9..21e7a99 100755 ---- a/pre_build_hook -+++ b/pre_build_hook -@@ -76,7 +76,7 @@ function cleanup { - } - - function download { -- wget "$1" -qO $2 -+ wget --no-check-certificate "$1" -qO $2 - } - - function unpack { diff --git a/patches/fuel-plugin-opendaylight/0004-ODL-Upstart-service-Respawn-on-crash.patch b/patches/fuel-plugin-opendaylight/0004-ODL-Upstart-service-Respawn-on-crash.patch deleted file mode 100644 index 6a9150c0..00000000 --- a/patches/fuel-plugin-opendaylight/0004-ODL-Upstart-service-Respawn-on-crash.patch +++ /dev/null @@ -1,38 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Thu, 1 Dec 2016 13:12:35 +0100 -Subject: [PATCH] ODL Upstart service: Respawn on crash - -On arm64, JVM sometimes (~10%) crashes at the very start, due to -JIT issues in our openjdk8 package. - -We can't really solve the JVM on arm64 issues in the near future, -so we will try to work around the issue by configuring the ODL -service as respawn-able (see [1] for more details). - -Note: This is specific to Ubuntu Trusty (14.04), which uses upstart, -for Ubuntu Xenial (or anything else >14.x), which uses systemd, -a different, equivalent mechanism should be used. - -[1] http://upstart.ubuntu.com/cookbook/#respawn - -JIRA: ARMBAND-134 - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - odl_package/ubuntu/opendaylight | 3 +++ - 1 file changed, 3 insertions(+) - -diff --git a/odl_package/ubuntu/opendaylight b/odl_package/ubuntu/opendaylight -index fd84376..b25e3b8 100644 ---- a/odl_package/ubuntu/opendaylight -+++ b/odl_package/ubuntu/opendaylight -@@ -8,6 +8,9 @@ stop on runlevel [!2345] - setgid odl - setuid odl - -+respawn -+respawn limit 20 20 -+ - env KARAF_HOME="/opt/opendaylight" - env JAVA_OPTS="-server -Xms1g -Xmx2g -XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+UnsyncloadClass -Dcom.sun.management.jmxremote" - env OPTS="-Dkaraf.startLocalConsole=false -Dkaraf.startRemoteShell=true" diff --git a/patches/fuel-web/0001-nailgun-Add-AArch64-Openstack-Mitaka-release-s.patch b/patches/fuel-web/0001-nailgun-Add-AArch64-Openstack-Newton-release-s.patch index 1f00e834..8ac89698 100644 --- a/patches/fuel-web/0001-nailgun-Add-AArch64-Openstack-Mitaka-release-s.patch +++ b/patches/fuel-web/0001-nailgun-Add-AArch64-Openstack-Newton-release-s.patch @@ -1,39 +1,38 @@ From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Sat, 14 May 2016 00:53:36 +0200 -Subject: [PATCH] nailgun: Add AArch64 Openstack Mitaka release(s). +Date: Sun, 18 Dec 2016 21:08:42 +0100 +Subject: [PATCH] nailgun: Add AArch64 Openstack Newton release(s). Add the following Openstack releases: - -- Mitaka on Ubuntu 14.04 (aarch64) -- Mitaka on Ubuntu+UCA 14.04 (aarch64) +- Newton on Ubuntu 16.04 (aarch64) +- Newton on Ubuntu+UCA 16.04 (aarch64) Signed-off-by: Stanislaw Kardach <stanislaw.kardach@cavium.com> Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> --- - nailgun/nailgun/fixtures/openstack.yaml | 287 ++++++++++++++++++++++++++++++++ - 1 file changed, 287 insertions(+) + nailgun/nailgun/fixtures/openstack.yaml | 343 ++++++++++++++++++++++++++++++++ + 1 file changed, 343 insertions(+) diff --git a/nailgun/nailgun/fixtures/openstack.yaml b/nailgun/nailgun/fixtures/openstack.yaml -index d08a4fc..65646d4 100644 +index eb33db2..ba967d7 100644 --- a/nailgun/nailgun/fixtures/openstack.yaml +++ b/nailgun/nailgun/fixtures/openstack.yaml -@@ -2314,3 +2314,302 @@ +@@ -2511,3 +2511,346 @@ weight: 61 type: "checkbox" value: true -+- &ubuntu_aarch64_release ++- &ubuntu_aarch64_xenial + pk: 4 + extend: *base_release + fields: -+ name: "Mitaka on Ubuntu 14.04 (aarch64)" -+ version: "mitaka-9.0" ++ name: "Newton on Ubuntu 16.04 (aarch64)" ++ version: "newton-10.0" + operating_system: "Ubuntu" -+ description: "This option will install the OpenStack Mitaka packages using Ubuntu as a base operating system. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." ++ description: "This option will install the OpenStack Newton packages using Ubuntu as a base operating system. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." + attributes_metadata: + editable: + kernel_params: + kernel: -+ value: "console=ttyAMA0,115200 console=ttyS0,115200 net.ifnames=1 biosdevname=0 rootdelay=90 nomodeset" ++ value: "console=ttyAMA0,115200 console=ttyS0,115200 console=tty0 net.ifnames=1 biosdevname=0 rootdelay=90 nomodeset" + repo_setup: + metadata: + label: "Repositories" @@ -47,54 +46,53 @@ index d08a4fc..65646d4 100644 + Please note: the first repository will be considered the operating system mirror that will be used during node provisioning. + To create a local repository mirror on the Fuel master node, please follow the instructions provided by running "fuel-createmirror --help" on the Fuel master node. + Please make sure your Fuel master node has Internet access to the repository before attempting to create a mirror. -+ For more details, please refer to the documentation (https://docs.mirantis.com/openstack/fuel/fuel-9.0/operations.html#external-ubuntu-ops). + value: + - type: "deb" + name: "ubuntu-local" + uri: "http://{settings.MASTER_IP}:8080/mirrors/ubuntu" -+ suite: "trusty" ++ suite: "xenial" + section: "main" + priority: null + - type: "deb" + name: "ubuntu" + uri: "http://ports.ubuntu.com/" -+ suite: "trusty" ++ suite: "xenial" + section: "main universe multiverse" + priority: null + - type: "deb" + name: "ubuntu-updates" + uri: "http://ports.ubuntu.com/" -+ suite: "trusty-updates" ++ suite: "xenial-updates" + section: "main universe multiverse" + priority: null + - type: "deb" + name: "ubuntu-security" + uri: "http://ports.ubuntu.com/" -+ suite: "trusty-security" ++ suite: "xenial-security" + section: "main universe multiverse" + priority: null + - type: "deb" + name: "mos" + uri: "http://{settings.MASTER_IP}:8080/{cluster.release.version}/ubuntu/x86_64" -+ suite: "mos9.0" ++ suite: "mos10.0" + section: "main restricted" + priority: 1050 + - type: "deb" + name: "mos-updates" + uri: "http://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}" -+ suite: "mos9.0-updates" ++ suite: "mos10.0-updates" + section: "main restricted" + priority: 1050 + - type: "deb" + name: "mos-security" + uri: "http://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}" -+ suite: "mos9.0-security" ++ suite: "mos10.0-security" + section: "main restricted" + priority: 1050 + - type: "deb" + name: "mos-holdback" + uri: "http://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}" -+ suite: "mos9.0-holdback" ++ suite: "mos10.0-holdback" + section: "main restricted" + priority: 1100 + - type: "deb" @@ -128,13 +126,10 @@ index d08a4fc..65646d4 100644 + debconf-utils + gdisk + grub-efi-arm64 -+ hpsa-dkms + hwloc -+ i40e-dkms + linux-firmware -+ linux-firmware-nonfree -+ linux-headers-generic-lts-trusty -+ linux-image-generic-lts-trusty ++ linux-headers-generic-lts-xenial ++ linux-image-generic-lts-xenial + lvm2 + mcollective + mdadm @@ -144,6 +139,7 @@ index d08a4fc..65646d4 100644 + nailgun-mcagents + network-checker + ntp ++ ntpdate + openssh-client + openssh-server + puppet @@ -163,25 +159,18 @@ index d08a4fc..65646d4 100644 + virt-what + vlan + generated: -+ repo_setup: -+ installer_kernel: -+ remote_relative: "dists/trusty/main/installer-arm64/current/images/generic/netboot/vmlinuz" -+ local: "/var/www/nailgun/ubuntu/arm64/images/linux" -+ installer_initrd: -+ remote_relative: "dists/trusty/main/installer-arm64/current/images/generic/netboot/initrd.gz" -+ local: "/var/www/nailgun/ubuntu/arm64/images/initrd.gz" + cobbler: + profile: -+ generator_arg: "ubuntu_1404_arm64" ++ generator_arg: "ubuntu_bootstrap" + provision: -+ codename: "trusty" ++ codename: "xenial" + image_data: + /: -+ uri: "http://{settings.MASTER_IP}:8080/targetimages/env_{cluster.id}_ubuntu_1404_arm64.img.gz" ++ uri: "http://{settings.MASTER_IP}:8080/targetimages/env_{cluster.id}_ubuntu_1604_arm64.img.gz" + format: "ext4" + container: "gzip" + /boot: -+ uri: "http://{settings.MASTER_IP}:8080/targetimages/env_{cluster.id}_ubuntu_1404_arm64-boot.img.gz" ++ uri: "http://{settings.MASTER_IP}:8080/targetimages/env_{cluster.id}_ubuntu_1604_arm64-boot.img.gz" + format: "ext2" + container: "gzip" + node_attributes: @@ -200,6 +189,16 @@ index d08a4fc..65646d4 100644 + type: "number" + value: 0 + min: 0 ++ dpdk: ++ weight: 20 ++ description: "Number of CPUs for DPDK usage" ++ label: "DPDK CPU pinning" ++ type: "number" ++ value: 0 ++ min: 0 ++ restrictions: ++ - condition: "not ('experimental' in version:feature_groups)" ++ action: "hide" + hugepages: + metadata: + group: "nfv" @@ -214,12 +213,57 @@ index d08a4fc..65646d4 100644 + label: "Nova Huge Pages" + type: "custom_hugepages" + value: {} ++ dpdk: ++ weight: 20 ++ description: "DPDK Huge Pages per NUMA node in MB" ++ label: "DPDK Huge Pages" ++ type: "number" ++ value: 0 ++ min: 0 ++ restrictions: ++ - condition: "not ('experimental' in version:feature_groups)" ++ action: "hide" ++ networks_metadata: ++ dpdk_drivers: ++ # Sources: ++ # http://dpdk.org/browse/dpdk/tree/doc/guides/rel_notes/known_issues.rst?h=releases&id=7173acefc7cfdfbbb9b91fcba1c9a67adb4c07c9#n168 ++ # http://dpdk.org/browse/dpdk/tree/lib/librte_eal/common/include/rte_pci_dev_ids.h?h=releases&id=7173acefc7cfdfbbb9b91fcba1c9a67adb4c07c9 ++ igb_uio: [ ++ "8086:100e", "8086:100f", "8086:1011", "8086:1010", "8086:1012", ++ "8086:101d", "8086:105e", "8086:105f", "8086:1060", "8086:10d9", ++ "8086:10da", "8086:10a4", "8086:10d5", "8086:10a5", "8086:10bc", ++ "8086:107d", "8086:107e", "8086:107f", "8086:10b9", "8086:109a", ++ "8086:10d3", "8086:10f6", "8086:150c", "8086:10c9", "8086:10e6", ++ "8086:10e7", "8086:10e8", "8086:1526", "8086:150a", "8086:1518", ++ "8086:150d", "8086:10a7", "8086:10a9", "8086:10d6", "8086:150e", ++ "8086:150f", "8086:1510", "8086:1511", "8086:1516", "8086:1527", ++ "8086:1521", "8086:1522", "8086:1523", "8086:1524", "8086:1546", ++ "8086:1533", "8086:1534", "8086:1535", "8086:1536", "8086:1537", ++ "8086:1538", "8086:1539", "8086:1f40", "8086:1f41", "8086:1f45", ++ "8086:0438", "8086:043a", "8086:043c", "8086:0440", "8086:10b6", ++ "8086:1508", "8086:10c6", "8086:10c8", "8086:150b", "8086:10db", ++ "8086:10dd", "8086:10ec", "8086:10f1", "8086:10f4", "8086:10f7", ++ "8086:1514", "8086:1517", "8086:10f9", "8086:10fb", "8086:11a9", ++ "8086:1f72", "8086:17d0", "8086:0470", "8086:152a", "8086:1529", ++ "8086:1507", "8086:154d", "8086:154a", "8086:1558", "8086:1557", ++ "8086:10fc", "8086:151c", "8086:154f", "8086:1528", "8086:1560", ++ "8086:15ae", "8086:1563", "8086:15aa", "8086:15ab", "8086:155d", ++ "8086:1572", "8086:1574", "8086:157f", "8086:1580", "8086:1581", ++ "8086:1583", "8086:1584", "8086:1585", "8086:1586", "8086:15a4", ++ "8086:10ca", "8086:152d", "8086:1520", "8086:152f", "8086:10ed", ++ "8086:152e", "8086:1515", "8086:1530", "8086:1564", "8086:1565", ++ "8086:15a8", "8086:15a9", "8086:154c", "8086:1571", "1af4:1000", ++ "15ad:07b0", "8086:15a5", "1137:0043", "1137:0071", "14e4:168a", ++ "14e4:16a9", "14e4:164f", "14e4:168e", "14e4:16af", "14e4:163d", ++ "14e4:163f", "14e4:168d", "14e4:16a1", "14e4:16a2", "14e4:16ad", ++ "14e4:16ae", "14e4:163e", "14e4:16a4" ++ ] +- pk: 5 -+ extend: *ubuntu_aarch64_release ++ extend: *ubuntu_aarch64_xenial + fields: -+ name: "Mitaka on Ubuntu+UCA 14.04 (aarch64)" -+ version: "mitaka-9.0" -+ description: "This option will install the OpenStack Mitaka packages using Ubuntu as a base operating system, including Ubuntu Cloud Archive OpenStack packages. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." ++ name: "Newton on Ubuntu+UCA 16.04 (aarch64)" ++ version: "newton-10.0" ++ description: "This option will install the OpenStack Newton packages using Ubuntu as a base operating system, including Ubuntu Cloud Archive OpenStack packages. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." + attributes_metadata: + editable: + repo_setup: @@ -230,66 +274,65 @@ index d08a4fc..65646d4 100644 + Please note: the first repository will be considered the operating system mirror that will be used during node provisioning. + To create a local repository mirror on the Fuel master node, please follow the instructions provided by running "fuel-createmirror --help" on the Fuel master node. + Please make sure your Fuel master node has Internet access to the repository before attempting to create a mirror. -+ For more details, please refer to the documentation (https://docs.mirantis.com/openstack/fuel/fuel-9.0/operations.html#external-ubuntu-ops). + value: + - type: "deb" + name: "ubuntu-local" + uri: "http://{settings.MASTER_IP}:8080/mirrors/ubuntu" -+ suite: "trusty" ++ suite: "xenial" + section: "main" + priority: null + - type: "deb" + name: "ubuntu" + uri: "http://ports.ubuntu.com/" -+ suite: "trusty" ++ suite: "xenial" + section: "main universe multiverse" + priority: null + - type: "deb" + name: "ubuntu-updates" + uri: "http://ports.ubuntu.com/" -+ suite: "trusty-updates" ++ suite: "xenial-updates" + section: "main universe multiverse" + priority: null + - type: "deb" + name: "ubuntu-security" + uri: "http://ports.ubuntu.com/" -+ suite: "trusty-security" ++ suite: "xenial-security" + section: "main universe multiverse" + priority: null + - type: "deb" + name: "uca" + uri: "http://ubuntu-cloud.archive.canonical.com/ubuntu" -+ suite: "trusty-updates/mitaka" ++ suite: "xenial-updates/newton" + section: "main" + priority: 1080 + - type: "deb" + name: "uca-proposed" + uri: "http://ubuntu-cloud.archive.canonical.com/ubuntu" -+ suite: "trusty-proposed/mitaka" ++ suite: "xenial-proposed/newton" + section: "main" + priority: 1080 + - type: "deb" + name: "mos" + uri: "http://{settings.MASTER_IP}:8080/{cluster.release.version}/ubuntu/x86_64" -+ suite: "mos9.0" ++ suite: "mos10.0" + section: "main restricted" + priority: 1050 + - type: "deb" + name: "mos-updates" + uri: "http://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}" -+ suite: "mos9.0-updates" ++ suite: "mos10.0-updates" + section: "main restricted" + priority: 1050 + - type: "deb" + name: "mos-security" + uri: "http://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}" -+ suite: "mos9.0-security" ++ suite: "mos10.0-security" + section: "main restricted" + priority: 1050 + - type: "deb" + name: "mos-holdback" + uri: "http://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}" -+ suite: "mos9.0-holdback" ++ suite: "mos10.0-holdback" + section: "main restricted" + priority: 1100 + - type: "deb" diff --git a/patches/fuel-web/0002-Add-arch-to-nailgun-release-and-target-image.patch b/patches/fuel-web/0002-Add-arch-to-nailgun-release-and-target-image.patch index 0435891e..3a6d5e4b 100644 --- a/patches/fuel-web/0002-Add-arch-to-nailgun-release-and-target-image.patch +++ b/patches/fuel-web/0002-Add-arch-to-nailgun-release-and-target-image.patch @@ -1,10 +1,16 @@ From: Stanislaw Kardach <stanislaw.kardach@cavium.com> -Date: Wed, 24 Feb 2016 20:11:54 +0100 +Date: Sun, 18 Dec 2016 21:28:21 +0100 Subject: [PATCH] Add arch to nailgun release and target image This is required so that the TestVM image is created using a cirros image that is compatible with the architecture of the deployment setup. As a bonus, it is also used when building the target image. + +[ Alexandru Avadanii ] +Rebased onto Newton. + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Signed-off-by: Stanislaw Kardach <stanislaw.kardach@cavium.com> --- nailgun/nailgun/consts.py | 5 ++ .../alembic_migrations/versions/armband.py | 53 ++++++++++++++++++++++ @@ -20,7 +26,7 @@ As a bonus, it is also used when building the target image. create mode 100644 nailgun/nailgun/db/migration/alembic_migrations/versions/armband.py diff --git a/nailgun/nailgun/consts.py b/nailgun/nailgun/consts.py -index 4900ec0..59b0c91 100644 +index ccbe860..a3875ce 100644 --- a/nailgun/nailgun/consts.py +++ b/nailgun/nailgun/consts.py @@ -38,6 +38,11 @@ RELEASE_OS = Enum( @@ -58,14 +64,14 @@ index 0000000..6e42b3f +"""Armband patches + +Revision ID: f9b7fd91ac19 -+Revises: 675105097a69 ++Revises: c6edea552f1e +Create Date: 2016-03-01 23:18:58.712617 + +""" + +# revision identifiers, used by Alembic. +revision = 'f9b7fd91ac19' -+down_revision = '675105097a69' ++down_revision = 'c6edea552f1e' + +from alembic import op +from nailgun.utils.migration import drop_enum @@ -95,10 +101,10 @@ index 0000000..6e42b3f +def remove_release_arch(): + op.drop_column('releases', 'arch') diff --git a/nailgun/nailgun/db/sqlalchemy/models/release.py b/nailgun/nailgun/db/sqlalchemy/models/release.py -index 1b47de4..e209e72 100644 +index a069b61..882d32c 100644 --- a/nailgun/nailgun/db/sqlalchemy/models/release.py +++ b/nailgun/nailgun/db/sqlalchemy/models/release.py -@@ -39,6 +39,14 @@ class Release(Base): +@@ -40,6 +40,14 @@ class Release(Base): id = Column(Integer, primary_key=True) name = Column(Unicode(100), nullable=False) version = Column(String(30), nullable=False) @@ -114,54 +120,54 @@ index 1b47de4..e209e72 100644 operating_system = Column(String(50), nullable=False) state = Column( diff --git a/nailgun/nailgun/fixtures/openstack.yaml b/nailgun/nailgun/fixtures/openstack.yaml -index 65646d4..874d4f0 100644 +index ba967d7..c1d2a71 100644 --- a/nailgun/nailgun/fixtures/openstack.yaml +++ b/nailgun/nailgun/fixtures/openstack.yaml -@@ -1907,6 +1907,7 @@ - name: "Mitaka on CentOS 6.5" +@@ -2113,6 +2113,7 @@ + name: "Newton on CentOS 6.5" state: "unavailable" - version: "mitaka-9.0" + version: "newton-10.0" + arch: "amd64" operating_system: "CentOS" description: "This option will install the OpenStack Mitaka packages using a CentOS based operating system. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." attributes_metadata: -@@ -1980,6 +1981,7 @@ +@@ -2186,6 +2187,7 @@ fields: - name: "Mitaka on Ubuntu 14.04" - version: "mitaka-9.0" + name: "Newton on Ubuntu 16.04" + version: "newton-10.0" + arch: "amd64" operating_system: "Ubuntu" - description: "This option will install the OpenStack Mitaka packages using Ubuntu as a base operating system. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." + description: "This option will install the OpenStack Newton packages using Ubuntu as a base operating system. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." attributes_metadata: -@@ -2220,6 +2222,7 @@ +@@ -2417,6 +2419,7 @@ fields: - name: "Mitaka on Ubuntu+UCA 14.04" - version: "mitaka-9.0" + name: "Newton on Ubuntu+UCA 16.04" + version: "newton-10.0" + arch: "amd64" - description: "This option will install the OpenStack Mitaka packages using Ubuntu as a base operating system, including Ubuntu Cloud Archive OpenStack packages. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." + description: "This option will install the OpenStack Newton packages using Ubuntu as a base operating system, including Ubuntu Cloud Archive OpenStack packages. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." attributes_metadata: editable: -@@ -2320,6 +2323,7 @@ +@@ -2517,6 +2520,7 @@ fields: - name: "Mitaka on Ubuntu 14.04 (aarch64)" - version: "mitaka-9.0" + name: "Newton on Ubuntu 16.04 (aarch64)" + version: "newton-10.0" + arch: "arm64" operating_system: "Ubuntu" - description: "This option will install the OpenStack Mitaka packages using Ubuntu as a base operating system. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." + description: "This option will install the OpenStack Newton packages using Ubuntu as a base operating system. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." attributes_metadata: -@@ -2506,6 +2510,7 @@ +@@ -2754,6 +2758,7 @@ fields: - name: "Mitaka on Ubuntu+UCA 14.04 (aarch64)" - version: "mitaka-9.0" + name: "Newton on Ubuntu+UCA 16.04 (aarch64)" + version: "newton-10.0" + arch: "arm64" - description: "This option will install the OpenStack Mitaka packages using Ubuntu as a base operating system, including Ubuntu Cloud Archive OpenStack packages. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." + description: "This option will install the OpenStack Newton packages using Ubuntu as a base operating system, including Ubuntu Cloud Archive OpenStack packages. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." attributes_metadata: editable: diff --git a/nailgun/nailgun/orchestrator/deployment_serializers.py b/nailgun/nailgun/orchestrator/deployment_serializers.py -index 08eb9c3..e561324 100644 +index f983ae3..2f9cec2 100644 --- a/nailgun/nailgun/orchestrator/deployment_serializers.py +++ b/nailgun/nailgun/orchestrator/deployment_serializers.py -@@ -300,7 +300,13 @@ class DeploymentMultinodeSerializer(object): +@@ -261,7 +261,13 @@ class DeploymentMultinodeSerializer(object): img_dir = '/usr/share/cirros-testvm/' else: img_dir = '/opt/vm/' @@ -174,13 +180,13 @@ index 08eb9c3..e561324 100644 + image_data['img_path'] = '{0}cirros-{1}-disk.img'.format(img_dir, + arch) - glance_properties = [] + properties_data = {} diff --git a/nailgun/nailgun/orchestrator/provisioning_serializers.py b/nailgun/nailgun/orchestrator/provisioning_serializers.py -index 48a4753..aaac22a 100644 +index 7b2c6ac..f2c05e1 100644 --- a/nailgun/nailgun/orchestrator/provisioning_serializers.py +++ b/nailgun/nailgun/orchestrator/provisioning_serializers.py -@@ -307,7 +307,8 @@ class ProvisioningSerializer61(ProvisioningSerializer): +@@ -329,7 +329,8 @@ class ProvisioningSerializer61(ProvisioningSerializer): attrs['repo_setup']['repos'], attrs['provision'], cluster.id, @@ -188,13 +194,13 @@ index 48a4753..aaac22a 100644 + packages, + cluster.release.arch)) - # NOTE(kozhukalov): This pre-provision task is going to be - # removed by 7.0 because we need this only for classic way of + PriorityStrategy().one_by_one(tasks) + return tasks diff --git a/nailgun/nailgun/orchestrator/tasks_templates.py b/nailgun/nailgun/orchestrator/tasks_templates.py -index 51dfdf7..bc2bff9 100644 +index 8252891..84bebe6 100644 --- a/nailgun/nailgun/orchestrator/tasks_templates.py +++ b/nailgun/nailgun/orchestrator/tasks_templates.py -@@ -210,7 +210,7 @@ def make_reboot_task(uids, task): +@@ -208,7 +208,7 @@ def make_reboot_task(uids, task): def make_provisioning_images_task( @@ -203,7 +209,7 @@ index 51dfdf7..bc2bff9 100644 conf = { 'repos': repos, 'image_data': provision_data['image_data'], -@@ -232,7 +232,8 @@ def make_provisioning_images_task( +@@ -230,7 +230,8 @@ def make_provisioning_images_task( "--image_build_dir /var/lib/fuel/ibp " "--log-file /var/log/fuel-agent-env-{0}.log " "--data_driver nailgun_build_image " @@ -214,10 +220,10 @@ index 51dfdf7..bc2bff9 100644 'retries': 1}}) diff --git a/nailgun/nailgun/test/integration/test_cluster_changes_handler.py b/nailgun/nailgun/test/integration/test_cluster_changes_handler.py -index fb82be0..450155d 100644 +index d7e964e..bce8289 100644 --- a/nailgun/nailgun/test/integration/test_cluster_changes_handler.py +++ b/nailgun/nailgun/test/integration/test_cluster_changes_handler.py -@@ -154,7 +154,7 @@ class TestHandlers(BaseIntegrationTest): +@@ -148,7 +148,7 @@ class TestHandlers(BaseIntegrationTest): common_attrs['last_controller'] = controller_nodes[-1]['name'] common_attrs['storage']['pg_num'] = 128 @@ -226,16 +232,16 @@ index fb82be0..450155d 100644 'container_format': 'bare', 'public': 'true', 'disk_format': 'qcow2', -@@ -166,7 +166,7 @@ class TestHandlers(BaseIntegrationTest): - """--property murano_image_info=""" - """'{"title": "Murano Demo", "type": "cirros.demo"}'""" - ), +@@ -164,7 +164,7 @@ class TestHandlers(BaseIntegrationTest): + 'murano_image_info': """'{"title": "Murano Demo", "type":""" + """ "cirros.demo"}'""", + }, - } + }] critical_mapping = { 'primary-controller': True, -@@ -566,7 +566,7 @@ class TestHandlers(BaseIntegrationTest): +@@ -565,7 +565,7 @@ class TestHandlers(BaseIntegrationTest): common_attrs['last_controller'] = controller_nodes[-1]['name'] common_attrs['storage']['pg_num'] = 128 @@ -244,16 +250,16 @@ index fb82be0..450155d 100644 'container_format': 'bare', 'public': 'true', 'disk_format': 'qcow2', -@@ -578,7 +578,7 @@ class TestHandlers(BaseIntegrationTest): - """--property murano_image_info=""" - """'{"title": "Murano Demo", "type": "cirros.demo"}'""" - ), +@@ -581,7 +581,7 @@ class TestHandlers(BaseIntegrationTest): + 'murano_image_info': """'{"title": "Murano Demo", "type":""" + """ "cirros.demo"}'""", + }, - } + }] critical_mapping = { 'primary-controller': True, -@@ -1050,7 +1050,7 @@ class TestHandlers(BaseIntegrationTest): +@@ -1053,7 +1053,7 @@ class TestHandlers(BaseIntegrationTest): common_attrs['last_controller'] = controller_nodes[-1]['name'] common_attrs['storage']['pg_num'] = 128 @@ -262,20 +268,20 @@ index fb82be0..450155d 100644 'container_format': 'bare', 'public': 'true', 'disk_format': 'qcow2', -@@ -1062,7 +1062,7 @@ class TestHandlers(BaseIntegrationTest): - """--property murano_image_info=""" - """'{"title": "Murano Demo", "type": "cirros.demo"}'""" - ), +@@ -1069,7 +1069,7 @@ class TestHandlers(BaseIntegrationTest): + 'murano_image_info': """'{"title": "Murano Demo", "type":""" + """ "cirros.demo"}'""", + }, - } + }] critical_mapping = { 'primary-controller': True, diff --git a/nailgun/nailgun/test/integration/test_orchestrator_serializer.py b/nailgun/nailgun/test/integration/test_orchestrator_serializer.py -index 5db8f4e..ef208cb 100644 +index f399602..6034f30 100644 --- a/nailgun/nailgun/test/integration/test_orchestrator_serializer.py +++ b/nailgun/nailgun/test/integration/test_orchestrator_serializer.py -@@ -2584,12 +2584,12 @@ class BaseDeploymentSerializer(BaseSerializerTest): +@@ -2565,12 +2565,12 @@ class BaseDeploymentSerializer(BaseSerializerTest): def check_no_murano_data(self): glance_properties = self.serializer.generate_test_vm_image_data( @@ -289,9 +295,9 @@ index 5db8f4e..ef208cb 100644 + self.env.nodes[0])['test_vm_image'][0]['glance_properties'] self.assertIn('murano_image_info', glance_properties) - + @staticmethod diff --git a/nailgun/nailgun/test/unit/test_tasks_templates.py b/nailgun/nailgun/test/unit/test_tasks_templates.py -index 4e81a19..695e53d 100644 +index 4db183f..4f5d2cb 100644 --- a/nailgun/nailgun/test/unit/test_tasks_templates.py +++ b/nailgun/nailgun/test/unit/test_tasks_templates.py @@ -140,7 +140,8 @@ class TestMakeTask(base.BaseTestCase): diff --git a/patches/fuel-web/0003-AArch64-Cirros-TestVM-Use-AAVMF-guest-UEFI.patch b/patches/fuel-web/0003-AArch64-Cirros-TestVM-Use-AAVMF-guest-UEFI.patch new file mode 100644 index 00000000..40ceb751 --- /dev/null +++ b/patches/fuel-web/0003-AArch64-Cirros-TestVM-Use-AAVMF-guest-UEFI.patch @@ -0,0 +1,30 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Sat, 21 Jan 2017 00:01:16 +0100 +Subject: [PATCH] AArch64: Cirros TestVM: Use AAVMF (guest UEFI) + +Based on the direct kernel boot patch by Stan. + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Signed-off-by: Stanislaw Kardach <stanislaw.kardach@cavium.com> +--- + nailgun/nailgun/orchestrator/deployment_serializers.py | 7 +++++++ + 1 file changed, 7 insertions(+) + +diff --git a/nailgun/nailgun/orchestrator/deployment_serializers.py b/nailgun/nailgun/orchestrator/deployment_serializers.py +index 77fcba1..fd33b3f 100644 +--- a/nailgun/nailgun/orchestrator/deployment_serializers.py ++++ b/nailgun/nailgun/orchestrator/deployment_serializers.py +@@ -284,6 +284,13 @@ class DeploymentMultinodeSerializer(object): + 'hypervisor_type': 'vmware' + } + ++ # AArch64: Use AAVMF (guest UEFI) ++ if arch == 'aarch64': ++ properties_data = { ++ 'hw_firmware_type': 'uefi', ++ 'short_id': 'ubuntu16.04' ++ } ++ + # NOTE(aschultz): properties was added as part of N and should be + # used infavor of glance_properties + image_data['glance_properties'] = self.generate_properties_arguments( diff --git a/patches/fuel-web/direct-kernel-boot/0000-Add-arch-to-nailgun-release-and-target-image.patch b/patches/fuel-web/direct-kernel-boot/0000-Add-arch-to-nailgun-release-and-target-image.patch deleted file mode 120000 index 64ff45c3..00000000 --- a/patches/fuel-web/direct-kernel-boot/0000-Add-arch-to-nailgun-release-and-target-image.patch +++ /dev/null @@ -1 +0,0 @@ -../0002-Add-arch-to-nailgun-release-and-target-image.patch
\ No newline at end of file diff --git a/patches/fuel-web/direct-kernel-boot/0001-direct-kernel-boot-for-cirros.patch b/patches/fuel-web/direct-kernel-boot/0001-direct-kernel-boot-for-cirros.patch deleted file mode 100644 index 0f9395c3..00000000 --- a/patches/fuel-web/direct-kernel-boot/0001-direct-kernel-boot-for-cirros.patch +++ /dev/null @@ -1,137 +0,0 @@ -From: Stanislaw Kardach <stanislaw.kardach@cavium.com> -Date: Tue, 15 Mar 2016 15:01:51 +0100 -Subject: [PATCH] direct kernel boot for cirros - ---- - nailgun/nailgun/orchestrator/base_serializers.py | 16 +++-- - .../nailgun/orchestrator/deployment_serializers.py | 68 ++++++++++++++++------ - 2 files changed, 61 insertions(+), 23 deletions(-) - -diff --git a/nailgun/nailgun/orchestrator/base_serializers.py b/nailgun/nailgun/orchestrator/base_serializers.py -index 2fb0da5..ac787ed 100644 ---- a/nailgun/nailgun/orchestrator/base_serializers.py -+++ b/nailgun/nailgun/orchestrator/base_serializers.py -@@ -145,12 +145,16 @@ class MuranoMetadataSerializerMixin(object): - - def inject_murano_settings(self, image_data): - """Adds murano metadata to the test image""" -- test_vm_image = image_data['test_vm_image'] -- existing_properties = test_vm_image['glance_properties'] -- murano_data = ' '.join(["""--property murano_image_info='{"title":""" -- """ "Murano Demo", "type": "cirros.demo"}'"""]) -- test_vm_image['glance_properties'] = existing_properties + murano_data -- return {'test_vm_image': test_vm_image} -+ test_vm_images = [] -+ for test_vm_image in image_data['test_vm_image']: -+ existing_properties = test_vm_image['glance_properties'] -+ murano_data = ' '.join( -+ ["""--property murano_image_info='{"title":""" -+ """ "Murano Demo", "type": "cirros.demo"}'"""]) -+ test_vm_image['glance_properties'] = existing_properties + \ -+ murano_data -+ test_vm_images += [test_vm_image] -+ return {'test_vm_image': test_vm_images} - - - class VmwareDeploymentSerializerMixin(object): -diff --git a/nailgun/nailgun/orchestrator/deployment_serializers.py b/nailgun/nailgun/orchestrator/deployment_serializers.py -index e561324..601301b 100644 ---- a/nailgun/nailgun/orchestrator/deployment_serializers.py -+++ b/nailgun/nailgun/orchestrator/deployment_serializers.py -@@ -283,6 +283,9 @@ class DeploymentMultinodeSerializer(object): - return {'glance': {'image_cache_max_size': image_cache_max_size}} - - def generate_test_vm_image_data(self, node): -+ images = [] -+ # By default always prepare the disk image -+ - # Instantiate all default values in dict. - image_data = { - 'container_format': 'bare', -@@ -305,8 +308,8 @@ class DeploymentMultinodeSerializer(object): - arch = "x86_64" - elif arch == "arm64": - arch = "aarch64" -- image_data['img_path'] = '{0}cirros-{1}-disk.img'.format(img_dir, -- arch) -+ image_data['img_path'] = \ -+ '{0}cirros-{1}-disk.img'.format(img_dir, arch) - - glance_properties = [] - -@@ -321,8 +324,38 @@ class DeploymentMultinodeSerializer(object): - glance_properties.append('--property hypervisor_type=vmware') - - image_data['glance_properties'] = ' '.join(glance_properties) -+ images += [image_data] -+ -+ # Now for aarch64 add also kernel and initrd because we direct -+ # kernel boot is used then -+ if arch == 'aarch64': -+ image_data['glance_properties'] += \ -+ ' --property os_command_line=\'root=/dev/vda1 rw rootwait' + \ -+ ' console=tty0 console=ttyS0 console=ttyAMA0\'' + \ -+ ' --property hw_video_model=vga' -+ kernel_data = { -+ 'container_format': 'aki', -+ 'public': 'true', -+ 'disk_format': 'aki', -+ 'img_name': 'TestVM-kernel', -+ 'img_path': '{0}cirros-{1}-kernel'.format(img_dir, arch), -+ 'os_name': 'cirros', -+ 'min_ram': 128, -+ 'glance_properties': '', -+ } -+ initrd_data = { -+ 'container_format': 'ari', -+ 'public': 'true', -+ 'disk_format': 'ari', -+ 'img_name': 'TestVM-initramfs', -+ 'img_path': '{0}cirros-{1}-initramfs'.format(img_dir, arch), -+ 'os_name': 'cirros', -+ 'min_ram': 128, -+ 'glance_properties': '', -+ } -+ images += [kernel_data, initrd_data] - -- return {'test_vm_image': image_data} -+ return {'test_vm_image': images} - - @classmethod - def get_net_provider_serializer(cls, cluster): -@@ -521,20 +554,21 @@ class DeploymentHASerializer61(DeploymentHASerializer, - images_data['test_vm_image'] = [] - if attrs.get('editable', {}).get('common', {}). \ - get('use_vcenter', {}).get('value') is True: -- image_vmdk_data = deepcopy(image_data['test_vm_image']) -- img_path = image_vmdk_data['img_path']. \ -- replace('x86_64-disk.img', 'i386-disk.vmdk') -- image_vmdk_data.update({ -- 'img_name': 'TestVM-VMDK', -- 'disk_format': 'vmdk', -- 'img_path': img_path, -- }) -- image_vmdk_data['glance_properties'] = ' '.join([ -- '--property vmware_disktype=sparse', -- '--property vmware_adaptertype=lsiLogic', -- '--property hypervisor_type=vmware']) -- images_data['test_vm_image'].append(image_vmdk_data) -- images_data['test_vm_image'].append(image_data['test_vm_image']) -+ images_vmdk_data = deepcopy(image_data['test_vm_image']) -+ for image_vmdk_data in images_vmdk_data: -+ img_path = image_vmdk_data['img_path']. \ -+ replace('x86_64-disk.img', 'i386-disk.vmdk') -+ image_vmdk_data.update({ -+ 'img_name': 'TestVM-VMDK', -+ 'disk_format': 'vmdk', -+ 'img_path': img_path, -+ }) -+ image_vmdk_data['glance_properties'] = ' '.join([ -+ '--property vmware_disktype=sparse', -+ '--property vmware_adaptertype=lsiLogic', -+ '--property hypervisor_type=vmware']) -+ images_data['test_vm_image'] += [image_vmdk_data] -+ images_data['test_vm_image'] += image_data['test_vm_image'] - else: - images_data = image_data - diff --git a/patches/fuel-web/kernel-bump/0001-kernel-flavor-linux-image-generic-lts-xenial.patch b/patches/fuel-web/kernel-bump/0001-kernel-flavor-linux-image-generic-lts-xenial.patch deleted file mode 100644 index 80189744..00000000 --- a/patches/fuel-web/kernel-bump/0001-kernel-flavor-linux-image-generic-lts-xenial.patch +++ /dev/null @@ -1,39 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Thu, 4 Aug 2016 12:47:50 +0200 -Subject: [PATCH] kernel-flavor: linux-image-generic-lts-xenial - -Ubuntu arm64: switch from using kernel 4.2 for Trusty -(linux-image-generic-lts-trusty) to kernel 4.4 backported from Xenial -(linux-image-generic-lts-xenial). - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - nailgun/nailgun/fixtures/openstack.yaml | 8 ++++---- - 1 file changed, 4 insertions(+), 4 deletions(-) - -diff --git a/nailgun/nailgun/fixtures/openstack.yaml b/nailgun/nailgun/fixtures/openstack.yaml -index 874d4f0..c5087b6 100644 ---- a/nailgun/nailgun/fixtures/openstack.yaml -+++ b/nailgun/nailgun/fixtures/openstack.yaml -@@ -2081,8 +2081,8 @@ - i40e-dkms - linux-firmware - linux-firmware-nonfree -- linux-headers-generic-lts-trusty -- linux-image-generic-lts-trusty -+ linux-headers-generic-lts-xenial -+ linux-image-generic-lts-xenial - lvm2 - mcollective - mdadm -@@ -2424,8 +2424,8 @@ - i40e-dkms - linux-firmware - linux-firmware-nonfree -- linux-headers-generic-lts-trusty -- linux-image-generic-lts-trusty -+ linux-headers-generic-lts-xenial -+ linux-image-generic-lts-xenial - lvm2 - mcollective - mdadm diff --git a/patches/fuel-web/multiarch-fuel/0000-nailgun-Add-AArch64-Openstack-Mitaka-release-s.patch b/patches/fuel-web/multiarch-fuel/0000-nailgun-Add-AArch64-Openstack-Mitaka-release-s.patch deleted file mode 120000 index e0d56a81..00000000 --- a/patches/fuel-web/multiarch-fuel/0000-nailgun-Add-AArch64-Openstack-Mitaka-release-s.patch +++ /dev/null @@ -1 +0,0 @@ -../0001-nailgun-Add-AArch64-Openstack-Mitaka-release-s.patch
\ No newline at end of file diff --git a/patches/fuel-web/multiarch-fuel/0000-nailgun-Add-AArch64-Openstack-Newton-release-s.patch b/patches/fuel-web/multiarch-fuel/0000-nailgun-Add-AArch64-Openstack-Newton-release-s.patch new file mode 120000 index 00000000..2e9288ee --- /dev/null +++ b/patches/fuel-web/multiarch-fuel/0000-nailgun-Add-AArch64-Openstack-Newton-release-s.patch @@ -0,0 +1 @@ +../0001-nailgun-Add-AArch64-Openstack-Newton-release-s.patch
\ No newline at end of file diff --git a/patches/fuel-web/multiarch-fuel/0001-FIXME-Disable-amd64-Mitaka-on-Ubuntu-for-now.patch b/patches/fuel-web/multiarch-fuel/0001-FIXME-Disable-amd64-Newton-on-Ubuntu-for-now.patch index d76a1f6d..4c7d70e7 100644 --- a/patches/fuel-web/multiarch-fuel/0001-FIXME-Disable-amd64-Mitaka-on-Ubuntu-for-now.patch +++ b/patches/fuel-web/multiarch-fuel/0001-FIXME-Disable-amd64-Newton-on-Ubuntu-for-now.patch @@ -1,9 +1,9 @@ From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> Date: Sat, 19 Mar 2016 19:02:15 +0100 -Subject: [PATCH] FIXME: Disable amd64 Mitaka on Ubuntu for now. +Subject: [PATCH] FIXME: Disable amd64 Newton on Ubuntu for now. Until we fix all cohabitation of amd64 and arm64, disable Openstack -releases <Mitaka on Ubuntu 14.04> and <Mitaka on Ubuntu+UCA 14.04> +releases <Newton on Ubuntu 16.04> and <Newton on Ubuntu+UCA 16.04> (both x86_64) in openstack.yaml fixture. TODO: Propose renaming above releases to include arch in name. @@ -13,22 +13,20 @@ Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> 1 file changed, 2 insertions(+) diff --git a/nailgun/nailgun/fixtures/openstack.yaml b/nailgun/nailgun/fixtures/openstack.yaml -index c5087b6..ba25093 100644 --- a/nailgun/nailgun/fixtures/openstack.yaml +++ b/nailgun/nailgun/fixtures/openstack.yaml @@ -1980,6 +1980,7 @@ extend: *base_release fields: - name: "Mitaka on Ubuntu 14.04" + name: "Newton on Ubuntu 16.04" + state: "unavailable" - version: "mitaka-9.0" + version: "newton-10.0" arch: "amd64" operating_system: "Ubuntu" -@@ -2221,6 +2222,7 @@ - extend: *ubuntu_release +@@ -2221,5 +2222,6 @@ fields: - name: "Mitaka on Ubuntu+UCA 14.04" + name: "Newton on Ubuntu+UCA 16.04" + state: "unavailable" - version: "mitaka-9.0" + version: "newton-10.0" arch: "amd64" - description: "This option will install the OpenStack Mitaka packages using Ubuntu as a base operating system, including Ubuntu Cloud Archive OpenStack packages. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." + description: "This option will install the OpenStack Newton packages using Ubuntu as a base operating system, including Ubuntu Cloud Archive OpenStack packages. With high availability features built in, you are getting a robust, enterprise-grade OpenStack deployment." diff --git a/patches/opnfv-fuel/0002-Revert-Point-to-specific-snapshot-of-mos-repositorie.patch b/patches/opnfv-fuel/0002-Revert-Point-to-specific-snapshot-of-mos-repositorie.patch deleted file mode 100644 index 05f9e123..00000000 --- a/patches/opnfv-fuel/0002-Revert-Point-to-specific-snapshot-of-mos-repositorie.patch +++ /dev/null @@ -1,43 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Tue, 18 Oct 2016 12:26:10 +0200 -Subject: [PATCH] Revert "Point to specific snapshot of mos repositories" - -Armband does not publish snapshots yet. - -This reverts commit fe3fc91c23ace520f32830b6af5205c39088f922. ---- - build/Makefile | 4 +--- - build/f_isoroot/f_repobuild/Makefile | 4 ++-- - 2 files changed, 3 insertions(+), 5 deletions(-) - -diff --git a/build/Makefile b/build/Makefile -index 5069def..59fa918 100644 ---- a/build/Makefile -+++ b/build/Makefile -@@ -63,9 +63,7 @@ export MIRROR_UBUNTU_ROOT := $(shell echo -n '/' ; echo "$(MIRROR_UBUNTU_URL)" | - export LATEST_MIRROR_ID_URL := http://$(shell ./select_closest_fuel_mirror.py) - - export MIRROR_MOS_UBUNTU ?= $(shell echo "$(LATEST_MIRROR_ID_URL)" | cut -d'/' -f3) --#export LATEST_TARGET_UBUNTU := $(shell curl -sSf "$(MIRROR_MOS_UBUNTU)/mos-repos/ubuntu/$(MOSVERSION).target.txt" | head -1) --# Fuel 9.0 --export LATEST_TARGET_UBUNTU := snapshots/9.0-2016-06-23-164100-copy -+export LATEST_TARGET_UBUNTU := $(shell curl -sSf "$(MIRROR_MOS_UBUNTU)/mos-repos/ubuntu/$(MOSVERSION).target.txt" | head -1) - export MIRROR_MOS_UBUNTU_ROOT := /mos-repos/ubuntu/$(LATEST_TARGET_UBUNTU) - - export LATEST_TARGET_CENTOS := $(shell curl -sSf "$(LATEST_MIRROR_ID_URL)/mos-repos/centos/mos$(MOSVERSION)-centos7/os.target.txt" | head -1) -diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile -index 8beb882..ad1a934 100644 ---- a/build/f_isoroot/f_repobuild/Makefile -+++ b/build/f_isoroot/f_repobuild/Makefile -@@ -53,9 +53,9 @@ release:nailgun - @cp -Rp nailgun ../release/opnfv/nailgun - @cp fuel_bootstrap_cli.yaml ../release/opnfv/ - --############################################################################ -+############################################################################# - # Cache operations - only used when building through ci/build.sh --############################################################################ -+############################################################################# - - # Create a unique hash to be used for getting and putting cache, based on: - # - Year and week (causing the cache to be rebuilt weekly) diff --git a/patches/opnfv-fuel/0003-dea_base-replace-grub-pc-by-grub-efi-arm64.patch b/patches/opnfv-fuel/0003-dea_base-replace-grub-pc-by-grub-efi-arm64.patch index fae14001..7d2f7ce0 100644 --- a/patches/opnfv-fuel/0003-dea_base-replace-grub-pc-by-grub-efi-arm64.patch +++ b/patches/opnfv-fuel/0003-dea_base-replace-grub-pc-by-grub-efi-arm64.patch @@ -11,12 +11,10 @@ diff --git a/deploy/config/dea_base.yaml b/deploy/config/dea_base.yaml index 0b8485b..658b328 100644 --- a/deploy/config/dea_base.yaml +++ b/deploy/config/dea_base.yaml -@@ -462,7 +462,7 @@ settings: +@@ -462,5 +462,5 @@ settings: gdisk - grub-pc + grub-efi-arm64 - hpsa-dkms - diff --git a/patches/opnfv-fuel/0005-deploy.py-add-a-time-stamp-to-the-new-ISO-image.patch b/patches/opnfv-fuel/0005-deploy.py-add-a-time-stamp-to-the-new-ISO-image.patch index 61847ab8..b832848b 100644 --- a/patches/opnfv-fuel/0005-deploy.py-add-a-time-stamp-to-the-new-ISO-image.patch +++ b/patches/opnfv-fuel/0005-deploy.py-add-a-time-stamp-to-the-new-ISO-image.patch @@ -23,8 +23,8 @@ index 041ba2f..cebef97 100755 import sys +import time import yaml - import errno import signal + import netaddr @@ -100,8 +101,9 @@ class AutoDeploy(object): def install_fuel_master(self): diff --git a/patches/opnfv-fuel/arm64-bug-fixes/0002-net-check-add-support-for-faulty-operstate.patch b/patches/opnfv-fuel/arm64-bug-fixes/0002-net-check-add-support-for-faulty-operstate.patch deleted file mode 100644 index a7006fcc..00000000 --- a/patches/opnfv-fuel/arm64-bug-fixes/0002-net-check-add-support-for-faulty-operstate.patch +++ /dev/null @@ -1,63 +0,0 @@ -From: Stefan Sicleru <stefan.sicleru@enea.com> -Date: Tue, 30 Aug 2016 17:53:41 +0200 -Subject: [PATCH] net-check: add support for faulty operstate - -Some eth drivers, such as those for APM X-Gene Mustang Board's NICs, do -not advertise operstate properly in sysfs, ie. it is advertised as -"unknown" whereas ethtool shows the NIC as fully functional with link -detected. This further affects "ip link show" output which is parsed -within _check_iface_ready() method. - -Replace "ip link show" command with "ethtool" in order to obtain proper -results when operstate is unknown. - -Signed-off-by: Stefan Sicleru <stefan.sicleru@enea.com> ---- - ...et-check-add-support-for-faulty-operstate.patch | 38 ++++++++++++++++++++++ - 1 file changed, 38 insertions(+) - create mode 100644 build/f_repos/patch/network-checker/0010-net-check-add-support-for-faulty-operstate.patch - -diff --git a/build/f_repos/patch/network-checker/0010-net-check-add-support-for-faulty-operstate.patch b/build/f_repos/patch/network-checker/0010-net-check-add-support-for-faulty-operstate.patch -new file mode 100644 -index 0000000..71e7b73 ---- /dev/null -+++ b/build/f_repos/patch/network-checker/0010-net-check-add-support-for-faulty-operstate.patch -@@ -0,0 +1,38 @@ -+From: Stefan Sicleru <stefan.sicleru@enea.com> -+Date: Tue, 30 Aug 2016 17:30:24 +0200 -+Subject: [PATCH] net-check: add support for faulty operstate -+ -+Some eth drivers, such as those for APM X-Gene Mustang Board's NICs, do -+not advertise operstate properly in sysfs, ie. it is advertised as -+"unknown" whereas ethtool shows the NIC as fully functional with link -+detected. This further affects "ip link show" output which is parsed -+within _check_iface_ready() method. -+ -+Replace "ip link show" command with "ethtool" in order to obtain proper -+results when operstate is unknown. -+ -+Signed-off-by: Stefan Sicleru <stefan.sicleru@enea.com> -+--- -+ network_checker/net_check/api.py | 9 ++++++++- -+ 1 file changed, 8 insertions(+), 1 deletion(-) -+ -+diff --git a/network_checker/net_check/api.py b/network_checker/net_check/api.py -+index e3c3b4e..87aa257 100755 -+--- a/network_checker/net_check/api.py -++++ b/network_checker/net_check/api.py -+@@ -195,7 +195,14 @@ class Actor(object): -+ def _check_iface_ready(self, iface, vid=None): -+ check_iface = self._iface_name(iface, vid) -+ output = self._execute(['ip', '-o', 'link', 'show', check_iface]) -+- return 'state UP' in '\n'.join(output) -++ if 'state UP' in '\n'.join(output): -++ return True -++ -++ if 'state UNKNOWN' in '\n'.join(output) and vid == None: -++ output = self._execute(['ethtool', check_iface]) -++ return 'Link detected: yes' in '\n'.join(output).replace('\n', ' ') -++ -++ return False -+ -+ def _ensure_iface_up(self, iface, vid=None): -+ """Ensures interface is with vid up.""" diff --git a/patches/opnfv-fuel/arm64-bug-fixes/0002-patch-packages-cirros_testvm-Use-Armband-version.patch b/patches/opnfv-fuel/arm64-bug-fixes/0002-patch-packages-cirros_testvm-Use-Armband-version.patch new file mode 100644 index 00000000..3bf33b27 --- /dev/null +++ b/patches/opnfv-fuel/arm64-bug-fixes/0002-patch-packages-cirros_testvm-Use-Armband-version.patch @@ -0,0 +1,28 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Fri, 13 Jan 2017 21:16:30 +0100 +Subject: [PATCH] patch-packages: cirros_testvm: Use Armband version + +Armband repackages cirros_testvm to add AArch64 binaries, and hence +rebuilds x86 binaries too, to align package versions. + +Account for Armband's cirros_testvm version by using its specific +version string instead of the old MOS one. + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + build/patch-packages/cirros_testvm/Makefile | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/build/patch-packages/cirros_testvm/Makefile b/build/patch-packages/cirros_testvm/Makefile +index b6a56d4..62c27f6 100644 +--- a/build/patch-packages/cirros_testvm/Makefile ++++ b/build/patch-packages/cirros_testvm/Makefile +@@ -23,7 +23,7 @@ clean: + + .PHONY: release + release: +- ../tools/deb_unpack c/cirros-testvm/cirros-testvm_0.3.4-2~u16.04+mos5_amd64.deb $(ORIGISO) ++ ../tools/deb_unpack c/cirros-testvm/cirros-testvm_0.3.4-2+amos1~u16.04+mos5_amd64.deb $(ORIGISO) + @rm -rf package/usr/share/cirros-testvm/* + wget -O package/usr/share/cirros-testvm/cirros-x86_64-disk.img http://download.cirros-cloud.net/daily/20160722/cirros-d160722-x86_64-disk.img + ../tools/deb_pack $(REVSTATE) diff --git a/patches/opnfv-fuel/cross-bootstrap/0001-UX-Update-bootstrap-target-build-time-estimate.patch b/patches/opnfv-fuel/cross-bootstrap/0001-UX-Update-bootstrap-target-build-time-estimate.patch index 91ba18a0..1b11f9e7 100644 --- a/patches/opnfv-fuel/cross-bootstrap/0001-UX-Update-bootstrap-target-build-time-estimate.patch +++ b/patches/opnfv-fuel/cross-bootstrap/0001-UX-Update-bootstrap-target-build-time-estimate.patch @@ -8,15 +8,14 @@ introduced by using qemu-user-static. Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> --- - .../0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch | 15 +++++++++++++++ + .../0001-Patches-for-OPNFV.patch | 15 +++++++++++++++ 1 file changed, 15 insertions(+) -diff --git a/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch b/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch +diff --git a/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch b/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch index 446d0b6..c2e664c 100644 ---- a/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch -+++ b/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch -@@ -10,6 +10,21 @@ diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh - index 3197c91..db3123d 100755 +--- a/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch ++++ b/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch +@@ -10,5 +10,20 @@ diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh --- a/iso/bootstrap_admin_node.sh +++ b/iso/bootstrap_admin_node.sh +@@ -86,9 +86,11 @@ @@ -34,6 +33,6 @@ index 446d0b6..c2e664c 100644 + bs_done_message="Default bootstrap image building done. Now you can boot new \ + nodes over PXE, they will be discovered and become available for installing \ + OpenStack on them" - @@ -339,8 +339,22 @@ fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail + @@ -347,8 +347,23 @@ fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail set +x echo "Done!" diff --git a/patches/opnfv-fuel/cross-bootstrap/0002-Build-bootstrap-image-for-arm64.patch b/patches/opnfv-fuel/cross-bootstrap/0002-Build-bootstrap-image-for-arm64.patch index 836cb2cf..433c87fb 100644 --- a/patches/opnfv-fuel/cross-bootstrap/0002-Build-bootstrap-image-for-arm64.patch +++ b/patches/opnfv-fuel/cross-bootstrap/0002-Build-bootstrap-image-for-arm64.patch @@ -6,15 +6,13 @@ For now the arch is hardcoded for simplicity. We should detect this but how? If we're multi-arch then one bootstrap image should be built per supported arch. This requires Fuel refactoring. --- - .../0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch | 9 +++++++++ + .../0001-Patches-for-OPNFV.patch | 9 +++++++++ 1 file changed, 9 insertions(+) -diff --git a/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch b/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch -index c2e664c..12d0dfb 100644 ---- a/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch -+++ b/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch -@@ -73,6 +73,15 @@ index 3197c91..db3123d 100755 - +diff --git a/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch b/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch +--- a/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch ++++ b/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch +@@ -45,3 +45,12 @@ # Enable iptables systemctl enable iptables.service +@@ -484,7 +490,7 @@ @@ -26,6 +24,4 @@ index c2e664c..12d0dfb 100644 + ret=0 + fuel notify --topic "done" --send "${bs_done_message}" + else - @@ -532,6 +545,16 @@ systemctl start ntpd - - bash /etc/rc.local + @@ -546,6 +560,16 @@ systemctl start ntpd diff --git a/patches/opnfv-fuel/cross-bootstrap/0004-post-scripts-Enable-systemd-binfmt-for-first-boot.patch b/patches/opnfv-fuel/cross-bootstrap/0004-post-scripts-Enable-systemd-binfmt-for-first-boot.patch index 26b898fe..c3dc220c 100644 --- a/patches/opnfv-fuel/cross-bootstrap/0004-post-scripts-Enable-systemd-binfmt-for-first-boot.patch +++ b/patches/opnfv-fuel/cross-bootstrap/0004-post-scripts-Enable-systemd-binfmt-for-first-boot.patch @@ -68,15 +68,15 @@ index 0000000..01eb2e2 +fi + +echo "Done preparing cross-building" -diff --git a/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch b/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch +diff --git a/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch b/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch index 12d0dfb..81d196b 100644 ---- a/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch -+++ b/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch -@@ -25,6 +25,19 @@ index 3197c91..db3123d 100755 +--- a/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch ++++ b/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch +@@ -25,6 +25,18 @@ index 3197c91..db3123d 100755 bs_done_message="Default bootstrap image building done. Now you can boot new \ nodes over PXE, they will be discovered and become available for installing \ OpenStack on them" -+@@ -239,6 +239,12 @@ ++@@ -239,5 +239,11 @@ + # /etc/fuel_openstack_version is provided by 'fuel-openstack-metadata' package + OPENSTACK_VERSION=$(cat /etc/fuel_openstack_version) + @@ -86,9 +86,8 @@ index 12d0dfb..81d196b 100644 ++ ${wwwdir}/${OPENSTACK_VERSION}/ubuntu/x86_64/dists/mos${FUEL_RELEASE}/Release > \ ++ /etc/fuel_openstack_arch ++ -+ # We do not ship debian-installer kernel and initrd on ISO. -+ # But we still need to be able to create ubuntu cobbler distro -+ # which requires kernel and initrd to be available. So, we - @@ -339,8 +339,22 @@ fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail ++ ++ touch /var/lib/hiera/common.yaml /etc/puppet/hiera.yaml + @@ -347,8 +347,23 @@ fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail set +x echo "Done!" diff --git a/patches/opnfv-fuel/cross-bootstrap/0008-m1.micro-Increase-profile-RAM-size-to-128MB.patch b/patches/opnfv-fuel/cross-bootstrap/0008-m1.micro-Increase-profile-RAM-size-to-256MB.patch index a54222a4..5f7a4949 100644 --- a/patches/opnfv-fuel/cross-bootstrap/0008-m1.micro-Increase-profile-RAM-size-to-128MB.patch +++ b/patches/opnfv-fuel/cross-bootstrap/0008-m1.micro-Increase-profile-RAM-size-to-256MB.patch @@ -1,6 +1,6 @@ From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> Date: Sat, 11 Jun 2016 23:34:48 +0200 -Subject: [PATCH] m1.micro: Increase profile RAM size to 128MB. +Subject: [PATCH] m1.micro: Increase profile RAM size to 256MB. TestVM (cirros) on aarch64 requires more than 64MB RAM. Keep profiles uniform across all archs. @@ -23,15 +23,15 @@ index c106bc3..0b48ca2 100755 + # + # TestVM (cirros) on aarch64 requires more than 64MB RAM. + # Keep profiles uniform across all archs by bumping micro flavor specs. -+ # m1.micro: Increase profile RAM size to 128MB. ++ # m1.micro: Increase profile RAM size to 256MB. + OPENSTACK_VERSION=$(cat /etc/fuel_openstack_version) + CONTROLLER_PP="/etc/puppet/${OPENSTACK_VERSION}/modules/openstack_tasks" -+ CONTROLLER_PP+="/manifests/openstack_controller/openstack_controller.pp" ++ CONTROLLER_PP+="/manifests/roles/controller.pp" + if [ -f ${CONTROLLER_PP} ]; then + sed -i.bak -r \ -+ 's/^(.*flavor-create.*m1\.micro auto)\s+[[:digit:]]+(.*)$/\1 128\2/' \ ++ 's/^(.*ram.*=>)\s+[[:digit:]]+(.*)$/\1 256\2/' \ + ${CONTROLLER_PP} -+ echo "INFO: [64] => [128] updated m1.micro profile RAM size (MB)." ++ echo "INFO: [64] => [256] updated m1.micro profile RAM size (MB)." + fi fi fi diff --git a/patches/opnfv-fuel/kernel-bump/0001-kernel-flavor-linux-image-generic-lts-xenial.patch b/patches/opnfv-fuel/kernel-bump/0001-kernel-flavor-linux-image-generic-lts-xenial.patch deleted file mode 100644 index c3a8da4a..00000000 --- a/patches/opnfv-fuel/kernel-bump/0001-kernel-flavor-linux-image-generic-lts-xenial.patch +++ /dev/null @@ -1,52 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Thu, 4 Aug 2016 12:50:19 +0200 -Subject: [PATCH] kernel-flavor: linux-image-generic-lts-xenial - -Ubuntu arm64: switch from using kernel 4.2 for Trusty -(linux-image-generic-lts-trusty) to kernel 4.4 backported from Xenial -(linux-image-generic-lts-xenial). - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml | 4 ++-- - deploy/config/dea_base.yaml | 4 ++-- - 2 files changed, 4 insertions(+), 4 deletions(-) - -diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml -index 17e161f..fb0ad1a 100644 ---- a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml -+++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml -@@ -13,7 +13,7 @@ - extra_dirs: - - /usr/share/fuel_bootstrap_cli/files/trusty - output_dir: /tmp/ -- kernel_flavor: linux-image-generic-lts-trusty -+ kernel_flavor: linux-image-generic-lts-xenial - packages: - - fuel-agent - - hwloc -@@ -21,7 +21,7 @@ - - i40e-dkms - - linux-firmware - - linux-firmware-nonfree -- - linux-headers-generic -+ - linux-headers-generic-lts-xenial - - live-boot - - live-boot-initramfs-tools - - lshw -diff --git a/deploy/config/dea_base.yaml b/deploy/config/dea_base.yaml -index 5a25040..f5a03e2 100644 ---- a/deploy/config/dea_base.yaml -+++ b/deploy/config/dea_base.yaml -@@ -474,9 +474,9 @@ settings: - - linux-firmware-nonfree - -- linux-headers-generic-lts-trusty -+ linux-headers-generic-lts-xenial - -- linux-image-generic-lts-trusty -+ linux-image-generic-lts-xenial - - lvm2 - diff --git a/patches/opnfv-fuel/multiarch-mirrors/0000-f_repobuild-Use-packetary-to-build-partial-mirror.patch b/patches/opnfv-fuel/multiarch-mirrors/0000-f_repobuild-Use-packetary-to-build-partial-mirror.patch new file mode 100644 index 00000000..54045aa1 --- /dev/null +++ b/patches/opnfv-fuel/multiarch-mirrors/0000-f_repobuild-Use-packetary-to-build-partial-mirror.patch @@ -0,0 +1,1221 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Wed, 6 Jul 2016 17:03:49 +0200 +Subject: [PATCH] f_repobuild: Use packetary to build partial mirror + +Packetary [1] will replace fuel-mirror in upstream Fuel. +fuel-mirror is using packetary under the hood already, so the +transition should not be too hard. + +Adapt OPNFV Ubuntu partial mirror build: +- obsolete opnfv-config.yaml (packetary uses CLI args for this); +- new opnfv_config.yaml will hold old ubuntu.yaml data, split at + build runtime into: + * mos_<arch>_mirror.yaml - MOS mirror cfg, only for deps analysis; + * ubuntu_<arch>_mirror.yaml - Ubuntu mirror cfg, for pkgs download; + * ubuntu_<arch>_packages.yaml - Ubuntu packages to download; + +While the consumers of our mirror still use `debootstrap`, which only +supports the "main" repo component, make sure the "main" package set +is complete, in the scope of: + * ubuntu_<arch>_mirror_main.yaml - Ubuntu [main only] mirror; + * ubuntu_<arch>_packages_main.yaml - Ubuntu packages for [main]; + +New workflow (executed for EACH architecture): +1. Collect bootstrap package deps from <fuel_bootstrap_cli.yaml>; +2. Collect all fixture release pkgs from fuel-web's <openstack.yaml>; +3. Add OPNFV extra pkg deps from "opnfv_config.yaml" + (based on old fuel-mirror) + + Currently, this is based on old fuel-mirror's + <fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml>, holding a + hardcoded list of package deps; plus eventual extra OPNFV packages, + like additional kernels, bootloaders etc. + Since we now inspect MOS repos for deps with packetary, this old + list should be easy to throw away as soon as we confirm each item + is directly required by another, already existing package in the + mirror. For now, keep the old list as safety net for missing deps. + + NOTE: Dependencies up to and including this group should also + be satisfiable from Ubuntu "main" repo component only. + +4. Append plugin dependencies to the package list; + +5. Run `packetary unresolved` for MOS repos, gathering MOS deps that + should be fetched from Ubuntu. The new packages will be appended + to the predefined list from old "ubuntu.yaml"; + +6. Run `packetary clone` to download all required pkgs for the + partial Ubuntu mirror. + +7. IF repo component merging is disabled, run `packetary clone` only + on Ubuntu[main] repo components to download any (still) missing + dependencies for `debootstrap`, which only looks at this component, + and not the whole mirror. + +8. IF repo component merging is enabled, run first `dpkg-scanpackages` + to filter out old duplicate versions of packages, then run + `packetary create` to merge all repo comps into a single "main". + +v1 -> v2: +- multiarch support (activated via UBUNTU_ARCH envvar); +- fixed debootstrap missing deps in "main" component of Ubuntu mirror; +- factored out some hardcodes into variables/functions; +- add .gitignore file; +- move generated config files to "opnfv_config" subdir; +- add arm64 Ubuntu repo URL (ports.ubuntu.com is separated from the + main Ubuntu mirror, archive.ubuntu.com, which only holds x86 pkgs); + +v3 -> v4: +- introduce env var "MIRROR_UBUNTU_MERGE" to control local mirror + repo-component merging into a single "main"; +- enable mirror repo-component merging by default, edit + fuel_bootstrap_cli.yaml accordingly; + +v4 -> v6: +- edit dea_base to use only main mirror; +- fix duplicated logs; +- remove `-q` flag for `create`; + +v6 -> v7: +- rebase for Newton fuel_bootstrap_cli.yaml (s/trusty/xenial/); +- include OPNFV plugin deps in local Ubuntu mirror; +- add `vgabios` dependency, required for arm64 guest firmware; +- remove all fuel-mirror references; +- merge old fuel-mirror pkglist to OPNFV pkglist; +- update packetary commit to include upstream bugfixes [2, 3]; + +v7 -> v11: +- re-enable f_repobuild after Newton rebase; +- fix leftover "trusty" ref to "xenial"; +- force f_repobuild rebuild to bypass cache (by editing Makefile); +- extend f_repobuild .cacheid to include scripts and OPNFV config; + +v11 -> v12: +- fix install/install.sh old refs to "trusty"; +- fix out of date select_ubuntu_repo.s refs to "trusty"; + +v12 -> v13: +- fix leftover mos9.0 / fuel-mirror references; +- fix most issues reported by `pylint opnfv_mirror_ubuntu.py`; + +v13 -> v14: +- Sync fuel_bootstrap_cli.yaml with settings.yaml.sample from 10.0: + [4] Stub pass for "Get root password for build image from settings" + [5] Sync pkg list with "Switch to Ubuntu Xenial" + [6] "Add puppet and daemonize to default packages list" + [7] "Delete kernel debug related parameter for performance" + [8] "Add to bootstrap image dependencies for network checker" +- Remove obsolete Trusty kernel images from pkg lists; +- Don't break when plugins set is empty; +- Fix missing "vim" package, which was resolved using secondary + provider packages like GUI-enabled vim flavors; + The proposed fix is quite sensitive, also cutting the final + mirror size in half (!), and allowing us to remove a previous + hacky workaround (re-running packetary only for main). +- Remove "main"-only packetary run (hacky), since the new fix for "vim" + also takes care of packetary solving dependencies from "universe". +- Add OPNFV blacklisting to workaround "upstart" package being installed, + together with "systemd" (having them both installed confuses puppet); + +NOTE: Without filtering out old package verisons using +`dpkg-scanpackages`, bootstrap build cannot solve all deps. + +FIXME: Packetary solves missing dependecies by also accepting +different packages that provide the same package name (e.g. "ifupdown" +is satisfied by "netscript" package). + +FIXME: Repo component merging is sort of slow, since packetary insists +on copying the source files to the destination dir even if they are +pointing to the same repo. + +FIXME: Packetary `create` uses a different directory scheme for the +created mirror than the upstream Ubuntu, e.g.: +[p] mirrors/ubuntu/pool/main/l/lvm2_2.02.98-6ubuntu2_amd64.deb +[U] mirrors/ubuntu/pool/main/l/lvm2/lvm2_2.02.98-6ubuntu2_amd64.deb + +This disencourages creating the "merged" mirror in the same location +as the source partial mirror. + +[1] https://wiki.openstack.org/wiki/Packetary +[2] https://review.openstack.org/#/c/392936/ +[3] https://review.openstack.org/#/c/392937/ +[4] https://review.openstack.org/#/c/277353/ +[5] https://review.openstack.org/#/c/345516/ +[6] https://review.openstack.org/#/c/344909/ +[7] https://review.openstack.org/#/c/400165/ +[8] https://review.openstack.org/#/c/415912/ + +JIRA: FUEL-218 +JIRA: FUEL-223 +JIRA: ARMBAND-169 +JIRA: ARMBAND-185 + +Change-Id: If2ee86f348b7683c83bfaf686baba4f1b8f555f0 +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + build/config.mk | 4 + + build/f_isoroot/Makefile | 8 +- + build/f_isoroot/f_repobuild/.gitignore | 4 + + ...s_tree-exact-match-trumps-other-providers.patch | 75 ++++++ + build/f_isoroot/f_repobuild/Makefile | 45 ++-- + build/f_isoroot/f_repobuild/config.mk | 23 ++ + .../f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml | 41 ++- + build/f_isoroot/f_repobuild/opnfv_config.yaml | 166 ++++++++++++ + build/f_isoroot/f_repobuild/opnfv_mirror_conf.py | 57 ----- + build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py | 279 +++++++++++++++++++++ + build/f_isoroot/f_repobuild/select_ubuntu_repo.sh | 41 +-- + build/install/apt-ftparchive-udeb.conf | 4 +- + build/install/install.sh | 14 +- + build/install/uninstall.sh | 16 +- + deploy/config/dea_base.yaml | 16 +- + 15 files changed, 636 insertions(+), 157 deletions(-) + create mode 100644 build/f_isoroot/f_repobuild/.gitignore + create mode 100644 build/f_isoroot/f_repobuild/0001-packages_tree-exact-match-trumps-other-providers.patch + create mode 100644 build/f_isoroot/f_repobuild/config.mk + create mode 100644 build/f_isoroot/f_repobuild/opnfv_config.yaml + delete mode 100755 build/f_isoroot/f_repobuild/opnfv_mirror_conf.py + create mode 100755 build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py + +diff --git a/build/config.mk b/build/config.mk +index 56317cb..5448c5e 100644 +--- a/build/config.mk ++++ b/build/config.mk +@@ -17,6 +17,10 @@ FUEL_MAIN_TAG = master + MOS_VERSION = 10.0 + OPENSTACK_VERSION = newton-10.0 + ++# List of space-separated Ubuntu architectures supported with current build ++# Format: same as `dpkg-architecture -qDEB_HOST_ARCH` ++# NOTE: Currently only amd64 is supported by Fuel@OPNFV. Armband adds arm64. ++export UBUNTU_ARCH ?= amd64 + + ############################################################################## + # Fuel components pinning / remote tracking; use submodules from f_repos +diff --git a/build/f_isoroot/Makefile b/build/f_isoroot/Makefile +index e189e16..b4ef1d0 100644 +--- a/build/f_isoroot/Makefile ++++ b/build/f_isoroot/Makefile +@@ -17,16 +17,12 @@ NON_10-0_REBASED_PLUGINS = f_odlpluginbuild f_onosfwpluginbuild f_ovsnfv-dpdk-pl + + # If the BUILD_FUEL_PLUGINS environment variable is set, only build the plugins + # indicated therein. +-# Temporarilu disabling f_repobuild that currently have some issues on +-# Fuel Newton. + ifdef BUILD_FUEL_PLUGINS + export BUILD_FUEL_PLUGINS +-#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS) +-SUBDIRS = f_kscfg f_bootstrap f_isolinux $(BUILD_FUEL_PLUGINS) ++SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(BUILD_FUEL_PLUGINS) + $(warning Overriding plugin build selection to $(BUILD_FUEL_PLUGINS)) + else +-#SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS) +-SUBDIRS = f_kscfg f_bootstrap f_isolinux $(PLUGINS) ++SUBDIRS = f_kscfg f_bootstrap f_isolinux f_repobuild $(PLUGINS) + endif + SUBCLEAN = $(addsuffix .clean,$(SUBDIRS)) + +diff --git a/build/f_isoroot/f_repobuild/.gitignore b/build/f_isoroot/f_repobuild/.gitignore +new file mode 100644 +index 0000000..09baca8 +--- /dev/null ++++ b/build/f_isoroot/f_repobuild/.gitignore +@@ -0,0 +1,4 @@ ++packetary ++fuel-web ++nailgun ++opnfv_config +diff --git a/build/f_isoroot/f_repobuild/0001-packages_tree-exact-match-trumps-other-providers.patch b/build/f_isoroot/f_repobuild/0001-packages_tree-exact-match-trumps-other-providers.patch +new file mode 100644 +index 0000000..1df292c +--- /dev/null ++++ b/build/f_isoroot/f_repobuild/0001-packages_tree-exact-match-trumps-other-providers.patch +@@ -0,0 +1,75 @@ ++From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ++Date: Sun, 15 Jan 2017 00:40:15 +0100 ++Subject: [PATCH] packages_tree: exact match trumps other providers ++ ++Ubuntu frequently has multiple providers for the same package name, ++e.g. "vim" is provided by "vim", "vim-gnome" etc. ++ ++Below is a sample dump of packetary's list of matches for a generic ++"vim" package requirement, without a version enforced, based on ++current Xenial repos: ++ ++vim-gtk3-py2 (2:7.4.1689-3ubuntu1) ++vim-gnome (2:7.4.1689-3ubuntu1) ++vim-gtk (2:7.4.1689-3ubuntu1) ++vim-nox (2:7.4.1689-3ubuntu1) ++vim-gtk-py2 (2:7.4.1689-3ubuntu1) ++vim-gnome-py2 (2:7.4.1689-3ubuntu1) ++vim-nox-py2 (2:7.4.1689-3ubuntu1) ++vim (2:7.4.1689-3ubuntu1) ++vim-gtk3 (2:7.4.1689-3ubuntu1) ++vim-athena (2:7.4.1689-3ubuntu1) ++vim-athena-py2 (2:7.4.1689-3ubuntu1) ++vim-gtk (2:7.4.1689-3ubuntu1.2) ++vim-gnome-py2 (2:7.4.1689-3ubuntu1.2) ++vim-gtk-py2 (2:7.4.1689-3ubuntu1.2) ++vim-nox (2:7.4.1689-3ubuntu1.2) ++vim-nox-py2 (2:7.4.1689-3ubuntu1.2) ++vim-gtk3 (2:7.4.1689-3ubuntu1.2) ++vim (2:7.4.1689-3ubuntu1.2) ++vim-athena (2:7.4.1689-3ubuntu1.2) ++vim-athena-py2 (2:7.4.1689-3ubuntu1.2) ++vim-gnome (2:7.4.1689-3ubuntu1.2) ++vim-gtk3-py2 (2:7.4.1689-3ubuntu1.2) ++ ++Currently, packetary solves the "vim" relation by using the last item ++in the sorted list, in this case "vim-gtk3-py2"; instead of using the ++exact package name match "vim". ++ ++This leads to our final mirror clone missing the "vim" package, and ++inherintely failing to build bootstrap/target images using only the ++partial Ubuntu mirror in OPNFV ISO. ++ ++The proposed fix is to first check the list for an exact name match, ++which would trump any secondary providers, even if they have a ++higher package version. ++ ++Change-Id: I7279aa6526ff9133829be2e316932c9b052c7814 ++Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ++--- ++ packetary/objects/packages_tree.py | 5 +++++ ++ 1 file changed, 5 insertions(+) ++ ++diff --git a/packetary/objects/packages_tree.py b/packetary/objects/packages_tree.py ++index f02df59..6081fa9 100644 ++--- a/packetary/objects/packages_tree.py +++++ b/packetary/objects/packages_tree.py ++@@ -18,6 +18,7 @@ ++ ++ from collections import defaultdict ++ +++import re ++ import six ++ ++ from packetary.objects.index import Index ++@@ -57,6 +58,10 @@ class PackagesTree(object): ++ """ ++ candidates = self.find_all(name, version_range) ++ if len(candidates) > 0: +++ # exact pkg name match trumps other providers' versions +++ for candidate in reversed(candidates): +++ if re.match('{0}\s.*'.format(re.escape(name)), str(candidate)): +++ return candidate ++ # we return candidates in sorted order, so let's take the highest ++ return candidates[-1] ++ return None +diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile +index 8beb882..76aadf9 100644 +--- a/build/f_isoroot/f_repobuild/Makefile ++++ b/build/f_isoroot/f_repobuild/Makefile +@@ -1,7 +1,8 @@ + ############################################################################## +-# Copyright (c) 2015 Ericsson AB and others. ++# Copyright (c) 2015,2016 Ericsson AB, Enea AB and others. + # stefan.k.berg@ericsson.com + # jonas.bjurel@ericsson.com ++# Alexandru.Avadanii@enea.com + # All rights reserved. This program and the accompanying materials + # are made available under the terms of the Apache License, Version 2.0 + # which accompanies this distribution, and is available at +@@ -10,41 +11,39 @@ + + SHELL := /bin/bash + TOP := $(shell pwd) +-TMP_ROOT_DIR := $(shell echo "$(MIRROR_UBUNTU_ROOT)" | cut -d "/" -f2) + + include ../../config.mk ++include config.mk + + export MOS_VERSION +-export OPENSTACK_VERSION ++export MIRROR_UBUNTU_OPNFV_PATH:=$(TOP)/nailgun/mirrors/ubuntu + + .PHONY: all + all: nailgun + + nailgun: +- sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml +- rm -Rf nailgun +- sudo mkdir -p /var/www/nailgun +- ln -sf ${F_SUBMOD_DIR}/fuel-mirror fuel-mirror +- sudo pip install -U -r ./fuel-mirror/requirements.txt +- sudo pip install ./fuel-mirror +- sudo pip install ./fuel-mirror/contrib/fuel_mirror +- ./opnfv_mirror_conf.py +- sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group ubuntu --pattern=ubuntu +- sudo chmod -R 755 /var/www/nailgun +- cp -Rp /var/www/nailgun . +- # In the end we want to have ubuntu repository in mirrors/ubuntu directory +- -if [ "$(MIRROR_UBUNTU_ROOT)" != "/ubuntu/" ]; then \ +- mkdir -p nailgun/mirrors/ubuntu;\ +- mv nailgun/mirrors$(MIRROR_UBUNTU_ROOT)* nailgun/mirrors/ubuntu;\ +- [ "$(MIRROR_UBUNTU_ROOT)" != "/" ] && rm -rf nailgun/mirrors/$(TMP_ROOT_DIR);\ ++ sudo apt-get install -y createrepo git libxml2-dev libxslt1-dev \ ++ python-dev zlib1g-dev ++ rm -Rf nailgun packetary opnfv_config && mkdir opnfv_config ++ # We will analyze fuel-web's fixture files for package lists ++ ln -sf ${F_SUBMOD_DIR}/fuel-web fuel-web ++ git clone --quiet $(PACKETARY_REPO) ++ if [ -n $(PACKETARY_COMMIT) ]; then \ ++ git -C packetary checkout $(PACKETARY_COMMIT); \ + fi ++ # FIXME(alav): Temporary, until we decide this is upstreamable ++ git -C packetary am $(TOP)/*.patch ++ sudo pip install -U -r ./packetary/requirements.txt ++ sudo pip install -U ./packetary ++ # Handle config and mirror build in one place ++ ./opnfv_mirror_ubuntu.py + # Store artifact in cache straight away if caching is enabled + # (no .cacheid will be present unless this is a cached build) + test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 + + .PHONY: clean + clean: +- @rm -rf ../release/opnfv/nailgun nailgun fuel-mirror opnfv-config.yaml ubuntu.yaml ++ @rm -rf ../release/opnfv/nailgun nailgun packetary fuel-web opnfv_config + + .PHONY: release + release:nailgun +@@ -59,10 +58,12 @@ release:nailgun + + # Create a unique hash to be used for getting and putting cache, based on: + # - Year and week (causing the cache to be rebuilt weekly) +-# - The contents of this Makefile ++# - The contents of this Makefile + all sh,mk,py,yaml files in CWD ++# - repo packages fingerprint ++# - repo arch list + .cacheid: + date +"Repocache %G%V" > .cachedata +- sha1sum Makefile >> .cachedata ++ sha1sum Makefile *.{sh,mk,py,yaml} >> .cachedata + $(CACHETOOL) packages >> .cachedata + echo -n $(UBUNTU_ARCH) | sha1sum | awk {'print $$1'} >> .cachedata + cat .cachedata | $(CACHETOOL) getid > .cacheid +diff --git a/build/f_isoroot/f_repobuild/config.mk b/build/f_isoroot/f_repobuild/config.mk +new file mode 100644 +index 0000000..9852093 +--- /dev/null ++++ b/build/f_isoroot/f_repobuild/config.mk +@@ -0,0 +1,23 @@ ++############################################################################## ++# Copyright (c) 2016 Ericsson AB, Enea AB and others. ++# stefan.k.berg@ericsson.com ++# jonas.bjurel@ericsson.com ++# Alexandru.Avadanii@enea.com ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++ ++# Use a recent master commit, since tags/branches are not yet mature ++export PACKETARY_REPO?=https://github.com/openstack/packetary ++export PACKETARY_COMMIT?=3021c001561b4baef352bf0b881d064ac687cc20 ++ ++# arm64 Ubuntu mirror is separated from archive.ubuntu.com ++export MIRROR_UBUNTU_URL_arm64=http://ports.ubuntu.com/ubuntu-ports/ ++export MIRROR_UBUNTU_ROOT_arm64=ubuntu-ports ++ ++# Merge all local mirror repo components/section into single "main" ++# NOTE: When changing this, make sure to also update all consumer config, like: ++# - fuel_bootstrap_cli.yaml ++export MIRROR_UBUNTU_MERGE=true +diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml +index fcf4257..81ca6eb 100644 +--- a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml ++++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml +@@ -8,19 +8,18 @@ + ############################################################################## + --- + root_ssh_authorized_file: /root/.ssh/id_rsa.pub +- extend_kopts: "biosdevname=0 net.ifnames=1 debug ignore_loglevel log_buf_len=10M print_fatal_signals=1 LOGLEVEL=8" +- ubuntu_release: trusty ++ extend_kopts: "biosdevname=0 net.ifnames=1" ++ ubuntu_release: xenial + extra_dirs: +- - /usr/share/fuel_bootstrap_cli/files/trusty ++ - /usr/share/fuel_bootstrap_cli/files/xenial + output_dir: /tmp/ +- kernel_flavor: linux-image-generic-lts-trusty ++ kernel_flavor: linux-image-generic-lts-xenial + packages: ++ - daemonize + - fuel-agent + - hwloc +- - hpsa-dkms + - i40e-dkms + - linux-firmware +- - linux-firmware-nonfree + - linux-headers-generic + - live-boot + - live-boot-initramfs-tools +@@ -33,13 +32,23 @@ + - nailgun-mcagents + - network-checker + - ntp ++ - ntpdate + - openssh-client + - openssh-server ++ - puppet + - squashfs-tools + - ubuntu-minimal + - vim + - wget + - xz-utils ++ # NOTE(el): Packages required for new generation ++ # network checker to be run without an access ++ # to repositories. ++ - sysfsutils ++ - bridge-utils ++ - ifenslave ++ - irqbalance ++ - iputils-arping + bootstrap_images_dir: /var/www/nailgun/bootstraps + active_bootstrap_symlink: /var/www/nailgun/bootstraps/active_bootstrap + flavor: ubuntu +@@ -47,29 +56,19 @@ + https_proxy: "" + repos: + - name: ubuntu +- section: "main universe multiverse" ++ section: "main" + uri: "http://127.0.0.1:8080/mirrors/ubuntu" + priority: +- suite: trusty +- type: deb +- - name: ubuntu-updates +- section: "main universe multiverse" +- uri: "http://127.0.0.1:8080/mirrors/ubuntu" +- priority: +- suite: trusty-updates +- type: deb +- - name: ubuntu-security +- section: "main universe multiverse" +- uri: "http://127.0.0.1:8080/mirrors/ubuntu" +- priority: +- suite: trusty-security ++ suite: xenial + type: deb + - name: mos + section: "main restricted" + uri: "http://127.0.0.1:8080/ubuntu/x86_64" + priority: 1050 +- suite: mos9.0 ++ suite: mos10.0 + type: deb + skip_default_img_build: false + direct_repo_addresses: + - "127.0.0.1" ++ # User can provide default hashed root password for bootstrap image ++ # hashed_root_password: "$6$IInX3Cqo$5xytL1VZbZTusOewFnG6couuF0Ia61yS3rbC6P5YbZP2TYclwHqMq9e3Tg8rvQxhxSlBXP1DZhdUamxdOBXK0." +diff --git a/build/f_isoroot/f_repobuild/opnfv_config.yaml b/build/f_isoroot/f_repobuild/opnfv_config.yaml +new file mode 100644 +index 0000000..b8d3441 +--- /dev/null ++++ b/build/f_isoroot/f_repobuild/opnfv_config.yaml +@@ -0,0 +1,166 @@ ++############################################################################## ++# Copyright (c) 2016 Enea AB and others. ++# Alexandru.Avadanii@enea.com ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++ ++# REPO definitions ++ ++# Based on old fuel-mirror data [1], adapted for OPNFV and packetary. ++# NOTE: 'uri' field will be added by opnfv_mirror_ubuntu.py from ENV. ++# [1] https://github.com/openstack/fuel-mirror/blob/stable/mitaka/ ++# contrib/fuel_mirror/data/ubuntu.yaml ++ ++# Main is a required parameter which defines what repository will be used ++# for images creation and that mirror should contain all packages for minimal ++# system creation. ++groups: ++ ubuntu: ++ - name: "ubuntu" ++ main: true ++ suite: "xenial" ++ section: ++ - "main" ++ - "multiverse" ++ - "restricted" ++ - "universe" ++ type: "deb" ++ priority: null ++ ++ - name: "ubuntu-updates" ++ suite: "xenial-updates" ++ section: ++ - "main" ++ - "multiverse" ++ - "restricted" ++ - "universe" ++ type: "deb" ++ priority: null ++ ++ - name: "ubuntu-security" ++ suite: "xenial-security" ++ section: ++ - "main" ++ - "multiverse" ++ - "restricted" ++ - "universe" ++ type: "deb" ++ priority: null ++ ++ mos: ++ - name: "mos" ++ suite: "mos$mos_version" ++ section: ++ - "main" ++ - "restricted" ++ type: "deb" ++ priority: 1000 ++ ++ - name: "mos-updates" ++ suite: "mos$mos_version-updates" ++ section: ++ - "main" ++ - "restricted" ++ type: "deb" ++ priority: 1000 ++ ++ - name: "mos-security" ++ suite: "mos$mos_version-security" ++ section: ++ - "main" ++ - "restricted" ++ type: "deb" ++ priority: 1000 ++ ++ - name: "mos-holdback" ++ suite: "mos$mos_version-holdback" ++ section: ++ - "main" ++ - "restricted" ++ type: "deb" ++ priority: 1000 ++ ++# PACKAGES ++ ++# This section lists packages that should be present in the local Ubuntu mirror, ++# but are not direct dependencies of any other packages from MOS or Ubuntu. ++# e.g.: additional kernels, bootloaders etc. ++ ++packages: ++ - name: "vgabios" ++# Packages are required to build bootstrap images for a system. ++# The mirror should contiain such packages in addition to local mirror. ++ - name: "acpi-support" ++ - name: "anacron" ++ - name: "aptitude" ++ - name: "atop" ++ - name: "acct" ++ - name: "bash-completion" ++ - name: "bc" ++ - name: "build-essential" ++ - name: "cloud-init" ++ - name: "conntrackd" ++ - name: "cpu-checker" ++ - name: "cpufrequtils" ++ - name: "debconf-utils" ++ - name: "devscripts" ++ - name: "fping" ++ - name: "git" ++ - name: "grub-pc" ++ - name: "htop" ++ - name: "hwloc" ++ - name: "ifenslave" ++ - name: "iperf" ++ - name: "iptables-persistent" ++ - name: "irqbalance" ++ - name: "language-pack-en" ++ - name: "libapache2-mod-fastcgi" ++ - name: "libnss3-tools" ++ - name: "linux-headers-generic-lts-xenial" ++ - name: "linux-image-generic-lts-xenial" ++ - name: "live-boot" ++ - name: "livecd-rootfs" ++ - name: "mc" ++ - name: "memcached" ++ - name: "monit" ++ - name: "msmtp-mta" ++ - name: "multipath-tools" ++ - name: "multipath-tools-boot" ++ - name: "nginx" ++ - name: "ntp" ++ - name: "openssh-server" ++ - name: "percona-toolkit" ++ - name: "percona-xtrabackup" ++ - name: "pm-utils" ++ - name: "puppet" ++ - name: "python-lesscpy" ++ - name: "python-pip" ++ - name: "rsyslog-gnutls" ++ - name: "rsyslog-relp" ++ - name: "screen" ++ - name: "squashfs-tools" ++ - name: "swift-plugin-s3" ++ - name: "sysfsutils" ++ - name: "sysstat" ++ - name: "telnet" ++ - name: "tmux" ++ - name: "traceroute" ++ - name: "ubuntu-standard" ++ - name: "vim" ++ - name: "virt-what" ++ - name: "xinetd" ++ - name: "xmlstarlet" ++ - name: "tftpd-hpa" ++ - name: "syslinux" ++ ++# OPNFV BLACKLIST ++ ++# Packetary's dependency solving mechanism brings in more than one alternative ++# For example, "Depends: upstart | systemd-sysv" brings both packages, which ++# leads to debootstrap using "upstart" (1st option available in local repo). ++ ++opnfv_blacklist: ++ - name: "upstart" +diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py +deleted file mode 100755 +index 1c7eb59..0000000 +--- a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py ++++ /dev/null +@@ -1,57 +0,0 @@ +-#!/usr/bin/env python +-############################################################################## +-# Copyright (c) 2015 Ericsson AB and others. +-# mskalski@mirantis.com +-# All rights reserved. This program and the accompanying materials +-# are made available under the terms of the Apache License, Version 2.0 +-# which accompanies this distribution, and is available at +-# http://www.apache.org/licenses/LICENSE-2.0 +-############################################################################## +-import os +-import yaml +- +-current_snapshot = os.environ["LATEST_TARGET_UBUNTU"] +-mos_version = os.environ['MOS_VERSION'] +-openstack_version = os.environ['OPENSTACK_VERSION'] +-mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU'] +-mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT'] +-mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL'] +-if os.environ.get('BUILD_FUEL_PLUGINS'): +- plugins = os.environ['BUILD_FUEL_PLUGINS'] +-else: +- plugins = os.environ['PLUGINS'] +- +- +-configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read() +-conf = yaml.load(configuration_file) +-conf['pattern_dir'] = '.' +-conf['openstack_version'] = openstack_version +-conf['mos_version'] = mos_version +- +-with open('opnfv-config.yaml', 'w') as outfile: +- outfile.write( yaml.dump(conf, default_flow_style=False) ) +- +-pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read() +-pattern = yaml.load(pattern_file) +-pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) +-pattern['ubuntu_baseurl'] = mirror_ubuntu +-for group in pattern['groups']['mos']: +- group['uri'] = pattern['mos_baseurl'] +-for group in pattern['groups']['ubuntu']: +- group['uri'] = pattern['ubuntu_baseurl'] +- +-for plugin in plugins.split(): +- path = "../{}/packages.yaml".format(plugin) +- if os.path.isfile(path): +- f = open(path).read() +- plugin_yaml = yaml.load(f) +- plugin_set = set(plugin_yaml['packages']) +- main_set = set(pattern['packages']) +- new_packages = plugin_set - main_set +- print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages)) +- pattern['packages'] = pattern['packages'] + list(new_packages) +- +-pattern['requirements']['ubuntu'] = pattern['packages'] +- +-with open('ubuntu.yaml', 'w') as outfile: +- outfile.write( yaml.safe_dump(pattern, default_flow_style=False) ) +diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py +new file mode 100755 +index 0000000..1a603ee +--- /dev/null ++++ b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py +@@ -0,0 +1,279 @@ ++#!/usr/bin/env python ++############################################################################## ++# Copyright (c) 2015,2016 Ericsson AB, Mirantis Inc., Enea AB and others. ++# mskalski@mirantis.com ++# Alexandru.Avadanii@enea.com ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++ ++"""Build multiarch partial local Ubuntu mirror using packetary""" ++ ++############################################################################## ++# Design quirks / workarounds: ++# 1. Fuel-agent uses `debootstrap` to build bootstrap and target chroots from ++# the local mirror; which only uses the "main" component from the first ++# repository, i.e. does not include "updates"/"security". ++# In order to fullfill all debootstrap dependencies in "main" repo, we will ++# do an extra packetary run using a reduced scope: ++# - only "main" component of the first mirror; ++# - reduced package dependency list (without MOS/OPNFV plugin deps). ++# 2. If repo structure is not mandatory to be in sync with official mirrors, ++# we can mitigate the issue by "merging" all repo-components into a single ++# "main". ++############################################################################## ++# Mirror build steps (for EACH architecture in UBUNTU_ARCH): ++# 1. Collect bootstrap package deps from <fuel_bootstrap_cli.yaml>; ++# 2. Collect all fixture release packages from fuel-web's <openstack.yaml>; ++# 3. Parse new "opnfv_config.yaml" list of packages (from old fuel-mirror); ++# 4. Inherit enviroment variable(s) for mirror URLs, paths etc. ++# - Allow arch-specific overrides for each env var; ++# 5. Mirror config is defined based on common config + OPNFV overrides; ++# - Convert old configuration format to packetary style where needed; ++# 6. Package lists are defined based on common config + OPNFV deps; ++# - Keep track of "main" packages separately, required by debootstrap; ++# 7. Clone/update all mirror components; ++# 8. IF mirror merging is disabled: ++# - Clone/update "main" mirror component (fix missing debootstrap deps); ++# 9. IF mirror merging is enabled: ++# - Use `dpkg-scanpackages` to filter out old versions of duplicate pkgs; ++# - Run `packetary create` on the set of downloaded packages, merging ++# them on the fly into a single-component mirror; ++############################################################################## ++ ++from copy import deepcopy ++import os ++import shutil ++import sys ++import yaml ++from contextlib import contextmanager ++from cStringIO import StringIO ++from packetary.cli.app import main ++ ++@contextmanager ++def capture_stdout(output): ++ """Context manager for capturing stdout""" ++ stdout = sys.stdout ++ sys.stdout = output ++ yield ++ sys.stdout = stdout ++ ++# FIXME: Find a better approach for eliminating duplicate logs than this ++def force_logger_reload(): ++ """Force logger reload (ugly hack to prevent log duplication)""" ++ for mod in sys.modules.keys(): ++ if mod.startswith('logging'): ++ try: ++ reload(sys.modules[mod]) ++ except: ++ pass ++ ++def get_unres_pkgs(architecture, cfg_mirror): ++ """Determine missing package dependecies for a mirror defition""" ++ unresolved_pkgs = list() ++ packetary_output = StringIO() ++ with capture_stdout(packetary_output): ++ main('unresolved -a {0} -r {1} -c name version --sep ;' ++ .format(_ARCH[architecture], cfg_mirror).split(' ')) ++ for dep_pkg in packetary_output.getvalue().splitlines(): ++ if dep_pkg.startswith('#'): ++ continue ++ dep = dep_pkg.split(';') ++ unresolved_pkgs += [{'name': dep[0], 'version': dep[1]}] ++ force_logger_reload() ++ return unresolved_pkgs ++ ++def from_legacy_pkglist(legacy_pkglist): ++ """Package list conversion from `old fuel-mirror` to `packetary` style""" ++ pkglist = list() ++ for pkg in legacy_pkglist: ++ pkglist += [{'name': pkg}] ++ return pkglist ++ ++def to_legacy_pkglist(pkglist): ++ """Package list conversion from `packetary` style to `old fuel-mirror`""" ++ legacy_pkglist = list() ++ for pkg in pkglist: ++ legacy_pkglist.append(pkg['name']) ++ return legacy_pkglist ++ ++def legacy_diff(base_pkglist, new_pkglist, requester, architecture): ++ """Package list diff (old format)""" ++ diff_set = set(new_pkglist) ++ if base_pkglist: ++ diff_set -= set(base_pkglist) ++ if diff_set: ++ print(' * {0} requires new packages for architecture [{1}]: {2}' ++ .format(requester, architecture, ', '.join(diff_set))) ++ return list(diff_set) ++ ++def do_local_repo(architecture, cfg_repo, cfg_packages_paths): ++ """Create single-component local repo (one architecture per call)""" ++ # Packetary does not use a global config file, so pass old settings here. ++ main('create -t deb -a {0} --repository {1} --package-files {2}' ++ ' --ignore-errors-num 2 --retries-num 3 --threads-num 10' ++ .format(_ARCH[architecture], cfg_repo, cfg_packages_paths).split(' ')) ++ force_logger_reload() ++ ++def do_partial_mirror(architecture, cfg_mirror, cfg_packages): ++ """Clone partial local mirror (one architecture per call)""" ++ # Note: '-d .' is ignored, as each mirror defines its own path. ++ main('clone -t deb -a {0} -r {1} -R {2} -d .' ++ ' --ignore-errors-num 2 --retries-num 3 --threads-num 10' ++ .format(_ARCH[architecture], cfg_mirror, cfg_packages).split(' ')) ++ force_logger_reload() ++ ++def write_cfg_file(cfg_mirror, data): ++ """Write configuration (yaml) file (package list / mirror defition)""" ++ with open(cfg_mirror, 'w') as outfile: ++ outfile.write(yaml.safe_dump(data, default_flow_style=False)) ++ ++def get_env(env_var, architecture=None): ++ """Evaluate architecture-specific overrides of env vars""" ++ if architecture: ++ env_var_arch = '{0}_{1}'.format(env_var, architecture) ++ if os.environ.get(env_var_arch): ++ return os.environ[env_var_arch] ++ if os.environ.get(env_var): ++ return os.environ[env_var] ++ return None ++ ++# Architecture name mapping (dpkg:packetary) for packetary CLI invocation ++_ARCH = { ++ "i386": "i386", ++ "amd64": "x86_64", ++ "arm64": "aarch64", ++} ++ ++# Arch-indepedent configuration (old fuel-mirror + OPNFV extra packages) ++CFG_D = 'opnfv_config' ++CFG_OPNFV = 'opnfv_config.yaml' ++MOS_VERSION = get_env('MOS_VERSION') ++UBUNTU_ARCH = get_env('UBUNTU_ARCH') ++MIRROR_UBUNTU_PATH = get_env('MIRROR_UBUNTU_OPNFV_PATH') ++MIRROR_UBUNTU_TMP_PATH = '{0}.tmp'.format(MIRROR_UBUNTU_PATH) ++MIRROR_UBUNTU_MERGE = get_env('MIRROR_UBUNTU_MERGE') ++CFG_MM_UBUNTU = '{0}/ubuntu_mirror_local.yaml'.format(CFG_D) ++FUEL_BOOTSTRAP_CLI_FILE = open('fuel_bootstrap_cli.yaml').read() ++FUEL_BOOTSTRAP_CLI = yaml.load(FUEL_BOOTSTRAP_CLI_FILE) ++FIXTURE_FILE = open('fuel-web/nailgun/nailgun/fixtures/openstack.yaml').read() ++FIXTURE = yaml.load(FIXTURE_FILE) ++OPNFV_CFG_YAML = open(CFG_OPNFV).read() ++OPNFV_CFG = yaml.load(OPNFV_CFG_YAML) ++ ++# Create local partial mirror using packetary, one arch at a time ++for arch in UBUNTU_ARCH.split(' '): ++ # Mirror / Package env vars, arch-overrideable ++ mos_ubuntu = get_env('MIRROR_MOS_UBUNTU', arch) ++ mos_ubuntu_root = get_env('MIRROR_MOS_UBUNTU_ROOT', arch) ++ mirror_ubuntu = get_env('MIRROR_UBUNTU_URL', arch) ++ plugins = get_env('BUILD_FUEL_PLUGINS', arch) ++ if plugins is None: ++ plugins = get_env('PLUGINS', arch) ++ ++ # Mirror / Package list configuration files (arch-specific) ++ cfg_m_mos = '{0}/mos_{1}_mirror.yaml'.format(CFG_D, arch) ++ cfg_m_ubuntu = '{0}/ubuntu_{1}_mirror.yaml'.format(CFG_D, arch) ++ cfg_p_ubuntu = '{0}/ubuntu_{1}_packages.yaml'.format(CFG_D, arch) ++ cfg_m_ubuntu_main = '{0}/ubuntu_{1}_mirror_main.yaml'.format(CFG_D, arch) ++ cfg_p_ubuntu_main = '{0}/ubuntu_{1}_packages_main.yaml'.format(CFG_D, arch) ++ ++ # Mirror config fork before customizing (arch-specific) ++ arch_mos = 'mos_{0}'.format(arch) ++ arch_ubuntu = 'ubuntu_{0}'.format(arch) ++ arch_packages = 'packages_{0}'.format(arch) ++ OPNFV_CFG['groups'][arch_mos] = deepcopy(OPNFV_CFG['groups']['mos']) ++ OPNFV_CFG['groups'][arch_ubuntu] = deepcopy(OPNFV_CFG['groups']['ubuntu']) ++ OPNFV_CFG[arch_packages] = OPNFV_CFG['packages'] ++ ++ # Mirror config update & conversion to packetary input ++ group_main_ubuntu = dict() ++ for group in OPNFV_CFG['groups'][arch_mos]: ++ group['uri'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) ++ group['suite'] = group['suite'].replace('$mos_version', MOS_VERSION) ++ for group in OPNFV_CFG['groups'][arch_ubuntu]: ++ group['uri'] = mirror_ubuntu ++ # FIXME: At `create`, packetary insists on copying all pkgs to dest dir, ++ # so configure it for another dir, which will replace the orig. ++ group['path'] = MIRROR_UBUNTU_TMP_PATH ++ if not group_main_ubuntu and 'main' in group: ++ group_main_ubuntu = [deepcopy(group)] ++ group_main_ubuntu[0]['section'] = ['main'] ++ ++ # Mirror config dump: MOS (for dep resolution), Ubuntu, Ubuntu[main] ++ write_cfg_file(cfg_m_mos, OPNFV_CFG['groups'][arch_mos]) ++ write_cfg_file(cfg_m_ubuntu, OPNFV_CFG['groups'][arch_ubuntu]) ++ if MIRROR_UBUNTU_MERGE is None: ++ write_cfg_file(cfg_m_ubuntu_main, group_main_ubuntu) ++ else: ++ # FIXME: For multiarch, only one dump would be enough ++ group_main_ubuntu[0]['origin'] = 'Ubuntu' ++ group_main_ubuntu[0]['path'] = MIRROR_UBUNTU_PATH ++ group_main_ubuntu[0]['uri'] = MIRROR_UBUNTU_PATH ++ write_cfg_file(CFG_MM_UBUNTU, group_main_ubuntu[0]) ++ ++ # Collect package dependencies from: ++ ## 1. fuel_bootstrap_cli.yaml (bootstrap image additional packages) ++ legacy_unresolved = legacy_diff(None, FUEL_BOOTSTRAP_CLI['packages'] + [ ++ FUEL_BOOTSTRAP_CLI['kernel_flavor'], ++ FUEL_BOOTSTRAP_CLI['kernel_flavor'].replace('image', 'headers')], ++ 'Bootstrap', arch) ++ ## 2. openstack.yaml FIXTURE definition (default target image packages) ++ for release in FIXTURE: ++ editable = release['fields']['attributes_metadata']['editable'] ++ if 'provision' in editable and 'packages' in editable['provision']: ++ release_pkgs = editable['provision']['packages']['value'].split() ++ legacy_unresolved += legacy_diff(legacy_unresolved, release_pkgs, ++ 'Release {0}'.format(release['fields']['name']), arch) ++ ## 3. OPNFV additional packages (includes old fuel-mirror ubuntu.yaml pkgs) ++ unresolved = dict() ++ unresolved['mandatory'] = 'exact' ++ unresolved['packages'] = from_legacy_pkglist(legacy_unresolved) ++ if 'packages' in OPNFV_CFG: ++ legacy_diff(legacy_unresolved, to_legacy_pkglist(OPNFV_CFG['packages']), ++ 'OPNFV config', arch) ++ unresolved['packages'] += OPNFV_CFG['packages'] ++ ++ # OPNFV plugins dependency resolution ++ if plugins: ++ for plugin in plugins.split(): ++ path = "../{}/packages.yaml".format(plugin) ++ if os.path.isfile(path): ++ f = open(path).read() ++ plugin_yaml = yaml.load(f) ++ new_pkgs = legacy_diff( ++ to_legacy_pkglist(unresolved['packages']), ++ plugin_yaml['packages'], 'Plugin {0}'.format(plugin), arch) ++ unresolved['packages'] += from_legacy_pkglist(new_pkgs) ++ ++ # Package list (reduced, i.e. no MOS deps, but with OPNFV plugin deps) ++ if MIRROR_UBUNTU_MERGE is None: ++ write_cfg_file(cfg_p_ubuntu_main, unresolved) ++ ++ # Mirror package list (full, including MOS/OPNFV plugin deps) ++ unresolved['packages'] += get_unres_pkgs(arch, cfg_m_mos) ++ write_cfg_file(cfg_p_ubuntu, unresolved) ++ do_partial_mirror(arch, cfg_m_ubuntu, cfg_p_ubuntu) ++ if MIRROR_UBUNTU_MERGE is None: ++ # Ubuntu[main] must be evaluated after Ubuntu ++ do_partial_mirror(arch, cfg_m_ubuntu_main, cfg_p_ubuntu_main) ++ ++if MIRROR_UBUNTU_MERGE is None: ++ shutil.move(MIRROR_UBUNTU_TMP_PATH, MIRROR_UBUNTU_PATH) ++else: ++ # Construct single-component mirror from all components ++ for arch in UBUNTU_ARCH.split(' '): ++ cfg_pp_ubuntu = '{0}/ubuntu_{1}_packages_paths.yaml'.format(CFG_D, arch) ++ # OPNFV blacklist ++ opnfv_blacklist = to_legacy_pkglist(OPNFV_CFG['opnfv_blacklist']) ++ # FIXME: We need scanpackages to omit older DEBs ++ # Inspired from http://askubuntu.com/questions/198474/ ++ os.system('dpkg-scanpackages -a {0} {1} 2>/dev/null | ' ++ 'grep -e "^Filename:" | sed "s|Filename: |- file://|g" | ' ++ 'grep -v -E "\/({2})_" > {3}' ++ .format(arch, MIRROR_UBUNTU_TMP_PATH, ++ '|'.join(opnfv_blacklist), cfg_pp_ubuntu)) ++ do_local_repo(arch, CFG_MM_UBUNTU, cfg_pp_ubuntu) ++ shutil.rmtree(MIRROR_UBUNTU_TMP_PATH) +diff --git a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh +index 5618eb7..c8c86db 100755 +--- a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh ++++ b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh +@@ -1,5 +1,6 @@ + #!/bin/bash + ++UBUNTU_DISTRO="xenial" + BLACKLIST="http://mirrors.se.eu.kernel.org/ubuntu/" + #BLACKLIST+=" http://foo.bar" + +@@ -28,26 +29,26 @@ blacklisted () { + check_mirror () { + mirror=$1 + status=0 +- for packdir in dists/trusty-updates/main/binary-amd64 \ +- dists/trusty-updates/restricted/binary-amd64 \ +- dists/trusty-updates/universe/binary-amd64 \ +- dists/trusty-updates/multiverse/binary-amd64 \ +- dists/trusty-security/main/binary-amd64 \ +- dists/trusty-security/restricted/binary-amd64 \ +- dists/trusty-security/universe/binary-amd64 \ +- dists/trusty-security/multiverse/binary-amd64 \ +- dists/trusty-proposed/main/binary-amd64 \ +- dists/trusty-proposed/restricted/binary-amd64 \ +- dists/trusty-proposed/universe/binary-amd64 \ +- dists/trusty-proposed/multiverse/binary-amd64 \ +- dists/trusty/main/binary-amd64 \ +- dists/trusty/restricted/binary-amd64 \ +- dists/trusty/universe/binary-amd64 \ +- dists/trusty/multiverse/binary-amd64 \ +- dists/trusty-backports/main/binary-amd64 \ +- dists/trusty-backports/restricted/binary-amd64 \ +- dists/trusty-backports/universe/binary-amd64 \ +- dists/trusty-backports/multiverse/binary-amd64 ++ for packdir in dists/${UBUNTU_DISTRO}-updates/main/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-updates/restricted/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-updates/universe/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-updates/multiverse/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-security/main/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-security/restricted/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-security/universe/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-security/multiverse/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-proposed/main/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-proposed/restricted/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-proposed/universe/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-proposed/multiverse/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}/main/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}/restricted/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}/universe/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}/multiverse/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-backports/main/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-backports/restricted/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-backports/universe/binary-amd64 \ ++ dists/${UBUNTU_DISTRO}-backports/multiverse/binary-amd64 + do + for packfile in Release Packages.gz + do +diff --git a/build/install/apt-ftparchive-udeb.conf b/build/install/apt-ftparchive-udeb.conf +index 3b5b239..0cabe94 100644 +--- a/build/install/apt-ftparchive-udeb.conf ++++ b/build/install/apt-ftparchive-udeb.conf +@@ -17,8 +17,8 @@ TreeDefault { + }; + + BinDirectory "pool/debian-installer" { +- Packages "dists/trusty/main/debian-installer/binary-amd64/Packages"; +- BinOverride "./indices/override.trusty.main.debian-installer"; ++ Packages "dists/xenial/main/debian-installer/binary-amd64/Packages"; ++ BinOverride "./indices/override.xenial.main.debian-installer"; + }; + + Default { +diff --git a/build/install/install.sh b/build/install/install.sh +index 198f26c..b2f6ccc 100755 +--- a/build/install/install.sh ++++ b/build/install/install.sh +@@ -122,17 +122,17 @@ prep_make_live() { + ssh-copy-id root@$FUELHOST + sshfs root@1${FUELHOST}:/ $TMP_HOSTMOUNT + +- if [ -f $REPO/dists/trusty/main/binary-amd64/Packages.backup ]; then ++ if [ -f $REPO/dists/xenial/main/binary-amd64/Packages.backup ]; then + echo "Error - found backup file for Packages!" + exit 1 + fi + +- if [ -f $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup ]; then ++ if [ -f $REPO/dists/xenial/main/binary-amd64/Packages.gz.backup ]; then + echo "Error - found backup file for Packages.gz!" + exit 1 + fi + +- if [ -f $REPO/dists/trusty/Release.backup ]; then ++ if [ -f $REPO/dists/xenial/Release.backup ]; then + echo "Error - found backup file for Release!" + exit 1 + fi +@@ -142,9 +142,9 @@ prep_make_live() { + exit 1 + fi + +- cp $REPO/dists/trusty/main/binary-amd64/Packages $REPO/dists/trusty/main/binary-amd64/Packages.backup +- cp $REPO/dists/trusty/main/binary-amd64/Packages.gz $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup +- cp $REPO/dists/trusty/Release $REPO/dists/trusty/Release.backup ++ cp $REPO/dists/xenial/main/binary-amd64/Packages $REPO/dists/xenial/main/binary-amd64/Packages.backup ++ cp $REPO/dists/xenial/main/binary-amd64/Packages.gz $REPO/dists/xenial/main/binary-amd64/Packages.gz.backup ++ cp $REPO/dists/xenial/Release $REPO/dists/xenial/Release.backup + cp -Rvp $DEST/etc/puppet $DEST/etc/puppet.backup + } + +@@ -443,7 +443,7 @@ copy_packages() { + echo Not running apt-ftparchive generate "${APT_UDEB_CONF}" + + # Fuel also needs this index file +- # cat dists/trusty/main/binary-amd64/Packages | \ ++ # cat dists/xenial/main/binary-amd64/Packages | \ + # awk '/^Package:/{pkg=$2} + # /^Version:/{print pkg ": \"" $2 "\""}' > ubuntu-versions.yaml + # cp ubuntu-versions.yaml $DEST +diff --git a/build/install/uninstall.sh b/build/install/uninstall.sh +index a9e74bc..5252c30 100755 +--- a/build/install/uninstall.sh ++++ b/build/install/uninstall.sh +@@ -31,17 +31,17 @@ DEST=$MOUNT + REPO=$DEST/var/www/nailgun/ubuntu/fuelweb/x86_64 + + cd $REPO +-if [ ! -f $REPO/dists/trusty/main/binary-amd64/Packages.backup ]; then ++if [ ! -f $REPO/dists/xenial/main/binary-amd64/Packages.backup ]; then + echo "Error - didn't find backup file for Packages!" + exit 1 + fi + +-if [ ! -f $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup ]; then ++if [ ! -f $REPO/dists/xenial/main/binary-amd64/Packages.gz.backup ]; then + echo "Error - didn't find backup file for Packages.gz!" + exit 1 + fi + +-if [ ! -f $REPO/dists/trusty/Release.backup ]; then ++if [ ! -f $REPO/dists/xenial/Release.backup ]; then + echo "Error - didn't find backup file for Release!" + exit 1 + fi +@@ -71,9 +71,9 @@ cd $REPO + + echo "Restoring backups of datafiles" + +-rm -f $REPO/dists/trusty/main/binary-amd64/Packages $REPO/dists/trusty/main/binary-amd64/Packages.gz +-rm -f $REPO/dists/trusty/Release $DEST/etc/puppet/manifests/site.pp +-mv $REPO/dists/trusty/main/binary-amd64/Packages.backup $REPO/dists/trusty/main/binary-amd64/Packages +-mv $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup $REPO/dists/trusty/main/binary-amd64/Packages.gz +-mv $REPO/dists/trusty/Release.backup $REPO/dists/trusty/Release ++rm -f $REPO/dists/xenial/main/binary-amd64/Packages $REPO/dists/xenial/main/binary-amd64/Packages.gz ++rm -f $REPO/dists/xenial/Release $DEST/etc/puppet/manifests/site.pp ++mv $REPO/dists/xenial/main/binary-amd64/Packages.backup $REPO/dists/xenial/main/binary-amd64/Packages ++mv $REPO/dists/xenial/main/binary-amd64/Packages.gz.backup $REPO/dists/xenial/main/binary-amd64/Packages.gz ++mv $REPO/dists/xenial/Release.backup $REPO/dists/xenial/Release + mv $DEST/etc/puppet/manifests/site.pp.backup $DEST/etc/puppet/manifests/site.pp +diff --git a/deploy/config/dea_base.yaml b/deploy/config/dea_base.yaml +index ce81994..2520cf5 100644 +--- a/deploy/config/dea_base.yaml ++++ b/deploy/config/dea_base.yaml +@@ -722,22 +722,10 @@ settings: + value: + - name: ubuntu + priority: null +- section: main universe multiverse ++ section: main + suite: xenial + type: deb +- uri: http://archive.ubuntu.com/ubuntu/ +- - name: ubuntu-updates +- priority: null +- section: main universe multiverse +- suite: xenial-updates +- type: deb +- uri: http://archive.ubuntu.com/ubuntu/ +- - name: ubuntu-security +- priority: null +- section: main universe multiverse +- suite: xenial-security +- type: deb +- uri: http://archive.ubuntu.com/ubuntu/ ++ uri: http://10.20.0.2:8080/mirrors/ubuntu/ + - name: mos + priority: 1050 + section: main restricted diff --git a/patches/opnfv-fuel/multiarch-mirrors/0001-build-install-Add-arm64-deb-repositories-setup.patch b/patches/opnfv-fuel/multiarch-mirrors/0001-build-install-Add-arm64-deb-repositories-setup.patch index 35129e01..5513309e 100644 --- a/patches/opnfv-fuel/multiarch-mirrors/0001-build-install-Add-arm64-deb-repositories-setup.patch +++ b/patches/opnfv-fuel/multiarch-mirrors/0001-build-install-Add-arm64-deb-repositories-setup.patch @@ -11,7 +11,7 @@ approach. FIXME: Use UBUNTU_ARCH env var instead of hardcoding the arch list. While fixing that, also take care of missed s/trusty/mos9.0/g -in {un,}install.sh. +and/or s/xenial/mos10.0/g in {un,}install.sh. Closes-bug: ARMBAND-163 @@ -58,13 +58,13 @@ index edcb1f2..81ead2b 100644 }; -TreeDefault { -+Tree "dists/mos9.0" { ++Tree "dists/mos10.0" { + Architectures "amd64 arm64"; Directory "pool"; -}; - -BinDirectory "pool/main" { -- Packages "dists/mos9.0/main/binary-amd64/Packages"; +- Packages "dists/mos10.0/main/binary-amd64/Packages"; + Sections "main"; }; @@ -73,14 +73,14 @@ diff --git a/build/install/apt-ftparchive-release.conf b/build/install/apt-ftpar index e40ff68..ceaf71a 100644 --- a/build/install/apt-ftparchive-release.conf +++ b/build/install/apt-ftparchive-release.conf -@@ -13,6 +13,7 @@ APT::FTPArchive::Release::Label "mos9.0"; - APT::FTPArchive::Release::Suite "mos9.0"; - APT::FTPArchive::Release::Version "14.04"; - APT::FTPArchive::Release::Codename "mos9.0"; +@@ -13,6 +13,7 @@ APT::FTPArchive::Release::Label "mos10.0"; + APT::FTPArchive::Release::Suite "mos10.0"; + APT::FTPArchive::Release::Version "16.04"; + APT::FTPArchive::Release::Codename "mos10.0"; -APT::FTPArchive::Release::Architectures "amd64"; +APT::FTPArchive::Release::Architectures "amd64 arm64"; APT::FTPArchive::Release::Components "main"; - APT::FTPArchive::Release::Description "Ubuntu Trusty Tahr 14.04 LTS"; + APT::FTPArchive::Release::Description "Ubuntu 16.04.1 LTS"; +APT::FTPArchive::Contents "no"; diff --git a/build/install/apt-ftparchive-udeb.conf b/build/install/apt-ftparchive-udeb.conf index 3b5b239..8852db7 100644 @@ -94,14 +94,14 @@ index 3b5b239..8852db7 100644 }; -TreeDefault { -+Tree "dists/mos9.0" { ++Tree "dists/mos10.0" { + Architectures "amd64 arm64"; Directory "pool"; -}; - -BinDirectory "pool/debian-installer" { -- Packages "dists/trusty/main/debian-installer/binary-amd64/Packages"; -- BinOverride "./indices/override.trusty.main.debian-installer"; +- Packages "dists/xenial/main/debian-installer/binary-amd64/Packages"; +- BinOverride "./indices/override.xenial.main.debian-installer"; + Sections "main"; }; @@ -114,26 +114,26 @@ index d1a49f3..0561207 100755 ssh-copy-id root@$FUELHOST sshfs root@1${FUELHOST}:/ $TMP_HOSTMOUNT -- if [ -f $REPO/dists/trusty/main/binary-amd64/Packages.backup ]; then +- if [ -f $REPO/dists/xenial/main/binary-amd64/Packages.backup ]; then + for arch in arm64 amd64; do -+ if [ -f $REPO/dists/trusty/main/binary-${arch}/Packages.backup ]; then ++ if [ -f $REPO/dists/xenial/main/binary-${arch}/Packages.backup ]; then + echo "Error - found backup file for Packages for ${arch}!" + exit 1 + fi + -+ if [ -f $REPO/dists/trusty/main/binary-${arch}/Packages.gz.backup ]; then ++ if [ -f $REPO/dists/xenial/main/binary-${arch}/Packages.gz.backup ]; then + echo "Error - found backup file for Packages.gz for ${arch}!" + exit 1 + fi + done + -+ if [ -f $REPO/dists/trusty/main/binary-arm64/Packages.backup ]; then ++ if [ -f $REPO/dists/xenial/main/binary-arm64/Packages.backup ]; then echo "Error - found backup file for Packages!" exit 1 fi -- if [ -f $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup ]; then -+ if [ -f $REPO/dists/trusty/main/binary-arm64/Packages.gz.backup ]; then +- if [ -f $REPO/dists/xenial/main/binary-amd64/Packages.gz.backup ]; then ++ if [ -f $REPO/dists/xenial/main/binary-arm64/Packages.gz.backup ]; then echo "Error - found backup file for Packages.gz!" exit 1 fi @@ -141,13 +141,13 @@ index d1a49f3..0561207 100755 exit 1 fi -- cp $REPO/dists/trusty/main/binary-amd64/Packages $REPO/dists/trusty/main/binary-amd64/Packages.backup -- cp $REPO/dists/trusty/main/binary-amd64/Packages.gz $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup +- cp $REPO/dists/xenial/main/binary-amd64/Packages $REPO/dists/xenial/main/binary-amd64/Packages.backup +- cp $REPO/dists/xenial/main/binary-amd64/Packages.gz $REPO/dists/xenial/main/binary-amd64/Packages.gz.backup + for arch in arm64 amd64; do -+ cp $REPO/dists/trusty/main/binary-${arch}/Packages $REPO/dists/trusty/main/binary-${arch}/Packages.backup -+ cp $REPO/dists/trusty/main/binary-${arch}/Packages.gz $REPO/dists/trusty/main/binary-${arch}/Packages.gz.backup ++ cp $REPO/dists/xenial/main/binary-${arch}/Packages $REPO/dists/xenial/main/binary-${arch}/Packages.backup ++ cp $REPO/dists/xenial/main/binary-${arch}/Packages.gz $REPO/dists/xenial/main/binary-${arch}/Packages.gz.backup + done - cp $REPO/dists/trusty/Release $REPO/dists/trusty/Release.backup + cp $REPO/dists/xenial/Release $REPO/dists/xenial/Release.backup cp -Rvp $DEST/etc/puppet $DEST/etc/puppet.backup } diff --git a/build/install/uninstall.sh b/build/install/uninstall.sh @@ -158,25 +158,25 @@ index a9e74bc..e90c632 100755 REPO=$DEST/var/www/nailgun/ubuntu/fuelweb/x86_64 cd $REPO --if [ ! -f $REPO/dists/trusty/main/binary-amd64/Packages.backup ]; then +-if [ ! -f $REPO/dists/xenial/main/binary-amd64/Packages.backup ]; then - echo "Error - didn't find backup file for Packages!" - exit 1 -fi +for arch in arm64 amd64; do -+ if [ ! -f $REPO/dists/trusty/main/binary-${arch}/Packages.backup ]; then ++ if [ ! -f $REPO/dists/xenial/main/binary-${arch}/Packages.backup ]; then + echo "Error - didn't find backup file for Packages for ${arch}!" + exit 1 + fi --if [ ! -f $REPO/dists/trusty/main/binary-amd64/Packages.gz.backup ]; then +-if [ ! -f $REPO/dists/xenial/main/binary-amd64/Packages.gz.backup ]; then - echo "Error - didn't find backup file for Packages.gz!" - exit 1 -fi -+ if [ ! -f $REPO/dists/trusty/main/binary-${arch}/Packages.gz.backup ]; then ++ if [ ! -f $REPO/dists/xenial/main/binary-${arch}/Packages.gz.backup ]; then + echo "Error - didn't find backup file for Packages.gz for ${arch}!" + exit 1 + fi +done - if [ ! -f $REPO/dists/trusty/Release.backup ]; then + if [ ! -f $REPO/dists/xenial/Release.backup ]; then echo "Error - didn't find backup file for Release!" diff --git a/patches/opnfv-fuel/upstream-backports/0001-deploy.sh-Remove-check-for-root-rm-umask-0000.patch b/patches/opnfv-fuel/upstream-backports/0001-deploy.sh-Remove-check-for-root-rm-umask-0000.patch index cd30166a..66515f65 100644 --- a/patches/opnfv-fuel/upstream-backports/0001-deploy.sh-Remove-check-for-root-rm-umask-0000.patch +++ b/patches/opnfv-fuel/upstream-backports/0001-deploy.sh-Remove-check-for-root-rm-umask-0000.patch @@ -52,18 +52,6 @@ index 8411714..081806c 100755 clean pushd ${DEPLOY_DIR} > /dev/null -diff --git a/deploy/deploy-config.py b/deploy/deploy-config.py -index 2a09aa3..02debe2 100644 ---- a/deploy/deploy-config.py -+++ b/deploy/deploy-config.py -@@ -41,7 +41,6 @@ from common import ( - check_file_exists, - create_dir_if_not_exists, - delete, -- check_if_root, - ArgParser, - ) - diff --git a/deploy/deploy.py b/deploy/deploy.py index fe213e5..08702d2 100755 --- a/deploy/deploy.py diff --git a/patches/opnfv-fuel/upstream-backports/0002-Backport-dpkg-checkbuilddeps-to-mk-build-deps.patch b/patches/opnfv-fuel/upstream-backports/0002-Backport-dpkg-checkbuilddeps-to-mk-build-deps.patch deleted file mode 100644 index eb81942e..00000000 --- a/patches/opnfv-fuel/upstream-backports/0002-Backport-dpkg-checkbuilddeps-to-mk-build-deps.patch +++ /dev/null @@ -1,58 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Wed, 6 Jul 2016 17:03:49 +0200 -Subject: [PATCH] Backport: dpkg-checkbuilddeps to mk-build-deps. - -Backported from [1]. - -[1] https://review.openstack.org/#/c/325210/ - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - ...ce-dpkg-checkbuilddeps-with-mk-build-deps.patch | 38 ++++++++++++++++++++++ - 1 file changed, 38 insertions(+) - create mode 100644 build/f_repos/patch/fuel-main/0005-replace-dpkg-checkbuilddeps-with-mk-build-deps.patch - -diff --git a/build/f_repos/patch/fuel-main/0005-replace-dpkg-checkbuilddeps-with-mk-build-deps.patch b/build/f_repos/patch/fuel-main/0005-replace-dpkg-checkbuilddeps-with-mk-build-deps.patch -new file mode 100644 -index 0000000..896f3cf ---- /dev/null -+++ b/build/f_repos/patch/fuel-main/0005-replace-dpkg-checkbuilddeps-with-mk-build-deps.patch -@@ -0,0 +1,38 @@ -+From: Sergii Golovatiuk <sgolovatiuk@mirantis.com> -+Date: Fri, 3 Jun 2016 12:26:19 +0200 -+Subject: [PATCH] Replace dpkg-checkbuilddeps with mk-build-deps -+ -+dpkg-checkbuilddeps doesn't check dependencies inside chroot as in many -+cases host system might not have required packages. This change replaces -+dpkg-checkbuilddeps with mk-build-deps in chroot -+ -+Implements: mos-xenial -+Change-Id: I27a65893c1bd33e6d82c11bf3d08423562dbe4d2 -+Signed-off-by: Sergii Golovatiuk <sgolovatiuk@mirantis.com> -+--- -+ packages/deb/module.mk | 5 ++--- -+ 1 file changed, 2 insertions(+), 3 deletions(-) -+ -+diff --git a/packages/deb/module.mk b/packages/deb/module.mk -+index 6c93441..c0ba737 100644 -+--- a/packages/deb/module.mk -++++ b/packages/deb/module.mk -+@@ -10,7 +10,7 @@ clean-deb: -+ done -+ sudo rm -rf $(BUILD_DIR)/packages/deb -+ -+-$(BUILD_DIR)/packages/deb/buildd.tar.gz: SANDBOX_DEB_PKGS:=wget bzip2 apt-utils build-essential python-setuptools python-pbr devscripts debhelper fakeroot -++$(BUILD_DIR)/packages/deb/buildd.tar.gz: SANDBOX_DEB_PKGS:=wget bzip2 apt-utils build-essential fakeroot devscripts equivs debhelper python-setuptools python-pbr -+ $(BUILD_DIR)/packages/deb/buildd.tar.gz: SANDBOX_UBUNTU:=$(BUILD_DIR)/packages/deb/chroot -+ $(BUILD_DIR)/packages/deb/buildd.tar.gz: export SANDBOX_UBUNTU_UP:=$(SANDBOX_UBUNTU_UP) -+ $(BUILD_DIR)/packages/deb/buildd.tar.gz: export SANDBOX_UBUNTU_DOWN:=$(SANDBOX_UBUNTU_DOWN) -+@@ -47,8 +47,7 @@ $(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/repos/repos.done -+ sudo -E dch -c $$(SANDBOX_UBUNTU)/tmp/$1/debian/changelog -D $(UBUNTU_RELEASE) -b --force-distribution \ -+ -v $(PACKAGE_VERSION)-`awk -F'=' '/DEBRELEASE/ {print $$$$2}' $(BUILD_DIR)/packages/sources/$1/version` \ -+ "`awk -F'=' '/DEBMSG/ {print $$$$2}' $(BUILD_DIR)/packages/sources/$1/version`" -+- dpkg-checkbuilddeps $(BUILD_DIR)/repos/$1/debian/control 2>&1 | sed 's/^dpkg-checkbuilddeps: Unmet build dependencies: //g' | sed 's/([^()]*)//g;s/|//g' | sudo tee $$(SANDBOX_UBUNTU)/tmp/$1.installdeps -+- sudo chroot $$(SANDBOX_UBUNTU) /bin/sh -c "cat /tmp/$1.installdeps | xargs --no-run-if-empty env LC_ALL=C DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true apt-get -y install" -++ sudo chroot $$(SANDBOX_UBUNTU) /bin/sh -c "mk-build-deps --install --remove --tool 'apt-get --yes --no-remove --no-install-recommends' /tmp/$1/debian/control" -+ sudo chroot $$(SANDBOX_UBUNTU) /bin/sh -c "cd /tmp/$1 ; DEB_BUILD_OPTIONS=nocheck debuild -us -uc -b -d" -+ cp $$(SANDBOX_UBUNTU)/tmp/*.deb $(BUILD_DIR)/packages/deb/packages -+ sudo sh -c "$$$${SANDBOX_UBUNTU_DOWN}" diff --git a/patches/opnfv-fuel/upstream-backports/0002-build-docker-Use-older-ffi-1.9.14.patch b/patches/opnfv-fuel/upstream-backports/0002-build-docker-Use-older-ffi-1.9.14.patch new file mode 100644 index 00000000..ccc37cb3 --- /dev/null +++ b/patches/opnfv-fuel/upstream-backports/0002-build-docker-Use-older-ffi-1.9.14.patch @@ -0,0 +1,32 @@ +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Fri, 13 Jan 2017 18:40:08 +0100 +Subject: [PATCH] build/docker: Use older ffi (1.9.14) + +`gem install fbm` tries to install the ffi gem too, which has a +problem in newer versions (>= 1.9.15) - see [1]: +"AbstractMemory.c:349:48: error: expected ')' before 'PRIsVALUE'" + +To bypass the newly introduced issues, use the old version. + +[1] https://jira.opnfv.org/browse/ARMBAND-191 + +JIRA: ARMBAND-191 + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + build/docker/Dockerfile | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile +index 04c4d2b..54aca35 100644 +--- a/build/docker/Dockerfile ++++ b/build/docker/Dockerfile +@@ -22,6 +22,8 @@ RUN apt-get install -y software-properties-common python-software-properties \ + ca-certificates sudo apt-utils lsb-release dosfstools debmirror p7zip-full \ + build-essential ruby-dev rubygems-integration python-pip git rpm createrepo dpkg-dev + ++# FIXME(armband): Newer ffi is broken - https://jira.opnfv.org/browse/ARMBAND-191 ++RUN gem install ffi -v 1.9.14 + RUN gem install fpm + + # Temporary: fpb needs to be built from sources diff --git a/patches/opnfv-fuel/upstream-backports/0003-f_repobuild-Use-packetary-to-build-partial-mirror.patch b/patches/opnfv-fuel/upstream-backports/0003-f_repobuild-Use-packetary-to-build-partial-mirror.patch deleted file mode 100644 index e7036e16..00000000 --- a/patches/opnfv-fuel/upstream-backports/0003-f_repobuild-Use-packetary-to-build-partial-mirror.patch +++ /dev/null @@ -1,825 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Wed, 6 Jul 2016 17:03:49 +0200 -Subject: [PATCH] f_repobuild: Use packetary to build partial mirror - -Packetary [1] will replace fuel-mirror in upstream Fuel. -fuel-mirror is using packetary under the hood already, so the -transition should not be too hard. - -Adapt OPNFV Ubuntu partial mirror build: -- obsolete opnfv-config.yaml (packetary uses CLI args for this); -- new opnfv_config.yaml will hold old ubuntu.yaml data, split at - build runtime into: - * mos_<arch>_mirror.yaml - MOS mirror cfg, only for deps analysis; - * ubuntu_<arch>_mirror.yaml - Ubuntu mirror cfg, for pkgs download; - * ubuntu_<arch>_packages.yaml - Ubuntu packages to download; - -While the consumers of our mirror still use `debootstrap`, which only -supports the "main" repo component, make sure the "main" package set -is complete, in the scope of: - * ubuntu_<arch>_mirror_main.yaml - Ubuntu [main only] mirror; - * ubuntu_<arch>_packages_main.yaml - Ubuntu packages for [main]; - -New workflow (executed for EACH architecture): -1. Collect bootstrap package deps from <fuel_bootstrap_cli.yaml>; -2. Collect all fixture release pkgs from fuel-web's <openstack.yaml>; -3. Add OPNFV extra pkg deps from "opnfv_config.yaml" - (based on old fuel-mirror) - - Currently, this is based on old fuel-mirror's - <fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml>, holding a - hardcoded list of package deps; plus eventual extra OPNFV packages, - like additional kernels, bootloaders etc. - Since we now inspect MOS repos for deps with packetary, this old - list should be easy to throw away as soon as we confirm each item - is directly required by another, already existing package in the - mirror. For now, keep the old list as safety net for missing deps. - - NOTE: Dependencies up to and including this group should also - be satisfiable from Ubuntu "main" repo component only. - -4. Append plugin dependencies to the package list; - -5. Run `packetary unresolved` for MOS repos, gathering MOS deps that - should be fetched from Ubuntu. The new packages will be appended - to the predefined list from old "ubuntu.yaml"; - -6. Run `packetary clone` to download all required pkgs for the - partial Ubuntu mirror. - -7. IF repo component merging is disabled, run `packetary clone` only - on Ubuntu[main] repo components to download any (still) missing - dependencies for `debootstrap`, which only looks at this component, - and not the whole mirror. - -8. IF repo component merging is enabled, run first `dpkg-scanpackages` - to filter out old duplicate versions of packages, then run - `packetary create` to merge all repo comps into a single "main". - -v1 -> v2: -- multiarch support (activated via UBUNTU_ARCH envvar); -- fixed debootstrap missing deps in "main" component of Ubuntu mirror; -- factored out some hardcodes into variables/functions; -- add .gitignore file; -- move generated config files to "opnfv_config" subdir; -- added arm64 Ubuntu repo URL (ports.ubuntu.com is separated from the - main Ubuntu mirror, archive.ubuntu.com, which only holds x86 pkgs); - -v3 -> v4: -- introduce env var "MIRROR_UBUNTU_MERGE" to control local mirror - repo-component merging into a single "main"; -- enable mirror repo-component merging by default, edit - fuel_bootstrap_cli.yaml accordingly; - -v4 -> v6: -- edit dea_base to use only main mirror; -- fix duplicated logs; -- remove `-q` flag for `create`; - -v6 -> v7: -- rebase for Newton fuel_bootstrap_cli.yaml (s/trusty/xenial/); -- include OPNFV plugin deps in local Ubuntu mirror; -- add `vgabios` dependency, required for arm64 guest firmware; -- remove all fuel-mirror references; -- merge old fuel-mirror pkglist to OPNFV pkglist; -- update packetary commit to include upstream bugfixes [2, 3]; - -NOTE: Without filtering out old package verisons using -`dpkg-scanpackages`, bootstrap build cannot solve all deps. - -FIXME: Packetary solves missing dependecies by also accepting -different packages that provide the same package name (e.g. "ifupdown" -is satisfied by "netscript" package). - -FIXME: Repo component merging is sort of slow, since packetary insists -on copying the source files to the destination dir even if they are -pointing to the same repo. - -FIXME: Packetary `create` uses a different directory scheme for the -created mirror than the upstream Ubuntu, e.g.: -[p] mirrors/ubuntu/pool/main/l/lvm2_2.02.98-6ubuntu2_amd64.deb -[U] mirrors/ubuntu/pool/main/l/lvm2/lvm2_2.02.98-6ubuntu2_amd64.deb - -This disencourages creating the "merged" mirror in the same location -as the source partial mirror. - -[1] https://wiki.openstack.org/wiki/Packetary -[2] https://review.openstack.org/#/c/392936/ -[3] https://review.openstack.org/#/c/392937/ - -JIRA: FUEL-218 -JIRA: FUEL-223 -JIRA: ARMBAND-169 -JIRA: ARMBAND-185 - -Change-Id: If2ee86f348b7683c83bfaf686baba4f1b8f555f0 -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - build/config.mk | 4 + - build/f_isoroot/f_repobuild/.gitignore | 4 + - build/f_isoroot/f_repobuild/Makefile | 37 ++- - build/f_isoroot/f_repobuild/config.mk | 23 ++ - .../f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml | 22 +- - build/f_isoroot/f_repobuild/opnfv_config.yaml | 160 ++++++++++++ - build/f_isoroot/f_repobuild/opnfv_mirror_conf.py | 57 ----- - build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py | 277 +++++++++++++++++++++ - deploy/config/dea_base.yaml | 16 +- - 9 files changed, 492 insertions(+), 108 deletions(-) - create mode 100644 build/f_isoroot/f_repobuild/.gitignore - create mode 100644 build/f_isoroot/f_repobuild/config.mk - create mode 100644 build/f_isoroot/f_repobuild/opnfv_config.yaml - delete mode 100755 build/f_isoroot/f_repobuild/opnfv_mirror_conf.py - create mode 100755 build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py - -diff --git a/build/config.mk b/build/config.mk -index 56317cb..5448c5e 100644 ---- a/build/config.mk -+++ b/build/config.mk -@@ -17,4 +17,9 @@ FUEL_MAIN_TAG = master - FUEL_TRACK_REMOTES = -+ -+# List of space-separated Ubuntu architectures supported with current build -+# Format: same as `dpkg-architecture -qDEB_HOST_ARCH` -+# NOTE: Currently only amd64 is supported by Fuel@OPNFV. Armband adds arm64. -+export UBUNTU_ARCH ?= amd64 - - ############################################################################## - # Fuel components pinning / remote tracking; use submodules from f_repos -diff --git a/build/f_isoroot/f_repobuild/.gitignore b/build/f_isoroot/f_repobuild/.gitignore -new file mode 100644 -index 0000000..09baca8 ---- /dev/null -+++ b/build/f_isoroot/f_repobuild/.gitignore -@@ -0,0 +1,4 @@ -+packetary -+fuel-web -+nailgun -+opnfv_config -diff --git a/build/f_isoroot/f_repobuild/Makefile b/build/f_isoroot/f_repobuild/Makefile -index 8beb882..9784edf 100644 ---- a/build/f_isoroot/f_repobuild/Makefile -+++ b/build/f_isoroot/f_repobuild/Makefile -@@ -1,7 +1,8 @@ - ############################################################################## --# Copyright (c) 2015 Ericsson AB and others. -+# Copyright (c) 2015,2016 Ericsson AB, Enea AB and others. - # stefan.k.berg@ericsson.com - # jonas.bjurel@ericsson.com -+# Alexandru.Avadanii@enea.com - # All rights reserved. This program and the accompanying materials - # are made available under the terms of the Apache License, Version 2.0 - # which accompanies this distribution, and is available at -@@ -10,41 +11,37 @@ - - SHELL := /bin/bash - TOP := $(shell pwd) --TMP_ROOT_DIR := $(shell echo "$(MIRROR_UBUNTU_ROOT)" | cut -d "/" -f2) - - include ../../config.mk -+include config.mk - - export MOS_VERSION --export OPENSTACK_VERSION -+export MIRROR_UBUNTU_OPNFV_PATH:=$(TOP)/nailgun/mirrors/ubuntu - - .PHONY: all - all: nailgun - - nailgun: -- sudo apt-get install -y git libxml2-dev libxslt-dev python-dev python-pip libz-dev libyaml-dev createrepo python-yaml -- rm -Rf nailgun -- sudo mkdir -p /var/www/nailgun -- ln -sf ${F_SUBMOD_DIR}/fuel-mirror fuel-mirror -- sudo pip install -U -r ./fuel-mirror/requirements.txt -- sudo pip install ./fuel-mirror -- sudo pip install ./fuel-mirror/contrib/fuel_mirror -- ./opnfv_mirror_conf.py -- sudo fuel-mirror --debug --config ./opnfv-config.yaml create --group ubuntu --pattern=ubuntu -- sudo chmod -R 755 /var/www/nailgun -- cp -Rp /var/www/nailgun . -- # In the end we want to have ubuntu repository in mirrors/ubuntu directory -- -if [ "$(MIRROR_UBUNTU_ROOT)" != "/ubuntu/" ]; then \ -- mkdir -p nailgun/mirrors/ubuntu;\ -- mv nailgun/mirrors$(MIRROR_UBUNTU_ROOT)* nailgun/mirrors/ubuntu;\ -- [ "$(MIRROR_UBUNTU_ROOT)" != "/" ] && rm -rf nailgun/mirrors/$(TMP_ROOT_DIR);\ -+ sudo apt-get install -y createrepo git libxml2-dev libxslt1-dev \ -+ python-dev zlib1g-dev -+ rm -Rf nailgun packetary opnfv_config && mkdir opnfv_config -+ # We will analyze fuel-web's fixture files for package lists -+ ln -sf ${F_SUBMOD_DIR}/fuel-web fuel-web -+ git clone --quiet $(PACKETARY_REPO) -+ if [ -n $(PACKETARY_COMMIT) ]; then \ -+ git -C packetary checkout $(PACKETARY_COMMIT); \ - fi -+ sudo pip install -U -r ./packetary/requirements.txt -+ sudo pip install ./packetary -+ # Handle config and mirror build in one place -+ ./opnfv_mirror_ubuntu.py - # Store artifact in cache straight away if caching is enabled - # (no .cacheid will be present unless this is a cached build) - test -f .cacheid && $(MAKE) -f Makefile put-cache || exit 0 - - .PHONY: clean - clean: -- @rm -rf ../release/opnfv/nailgun nailgun fuel-mirror opnfv-config.yaml ubuntu.yaml -+ @rm -rf ../release/opnfv/nailgun nailgun packetary fuel-mirror opnfv_config - - .PHONY: release - release:nailgun -diff --git a/build/f_isoroot/f_repobuild/config.mk b/build/f_isoroot/f_repobuild/config.mk -new file mode 100644 -index 0000000..9852093 ---- /dev/null -+++ b/build/f_isoroot/f_repobuild/config.mk -@@ -0,0 +1,23 @@ -+############################################################################## -+# Copyright (c) 2016 Ericsson AB, Enea AB and others. -+# stefan.k.berg@ericsson.com -+# jonas.bjurel@ericsson.com -+# Alexandru.Avadanii@enea.com -+# All rights reserved. This program and the accompanying materials -+# are made available under the terms of the Apache License, Version 2.0 -+# which accompanies this distribution, and is available at -+# http://www.apache.org/licenses/LICENSE-2.0 -+############################################################################## -+ -+# Use a recent master commit, since tags/branches are not yet mature -+export PACKETARY_REPO?=https://github.com/openstack/packetary -+export PACKETARY_COMMIT?=3021c001561b4baef352bf0b881d064ac687cc20 -+ -+# arm64 Ubuntu mirror is separated from archive.ubuntu.com -+export MIRROR_UBUNTU_URL_arm64=http://ports.ubuntu.com/ubuntu-ports/ -+export MIRROR_UBUNTU_ROOT_arm64=ubuntu-ports -+ -+# Merge all local mirror repo components/section into single "main" -+# NOTE: When changing this, make sure to also update all consumer config, like: -+# - fuel_bootstrap_cli.yaml -+export MIRROR_UBUNTU_MERGE=true -diff --git a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml -index fcf4257..7cbea29 100644 ---- a/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml -+++ b/build/f_isoroot/f_repobuild/fuel_bootstrap_cli.yaml -@@ -47,22 +47,10 @@ - https_proxy: "" - repos: - - name: ubuntu -- section: "main universe multiverse" -+ section: "main" - uri: "http://127.0.0.1:8080/mirrors/ubuntu" - priority: - suite: trusty -- type: deb -- - name: ubuntu-updates -- section: "main universe multiverse" -- uri: "http://127.0.0.1:8080/mirrors/ubuntu" -- priority: -- suite: trusty-updates -- type: deb -- - name: ubuntu-security -- section: "main universe multiverse" -- uri: "http://127.0.0.1:8080/mirrors/ubuntu" -- priority: -- suite: trusty-security - type: deb - - name: mos - section: "main restricted" -diff --git a/build/f_isoroot/f_repobuild/opnfv_config.yaml b/build/f_isoroot/f_repobuild/opnfv_config.yaml -new file mode 100644 -index 0000000..bb78747 ---- /dev/null -+++ b/build/f_isoroot/f_repobuild/opnfv_config.yaml -@@ -0,0 +1,160 @@ -+############################################################################## -+# Copyright (c) 2016 Enea AB and others. -+# Alexandru.Avadanii@enea.com -+# All rights reserved. This program and the accompanying materials -+# are made available under the terms of the Apache License, Version 2.0 -+# which accompanies this distribution, and is available at -+# http://www.apache.org/licenses/LICENSE-2.0 -+############################################################################## -+ -+# REPO definitions -+ -+# Based on old fuel-mirror data [1], adapted for OPNFV and packetary. -+# NOTE: 'uri' field will be added by opnfv_mirror_ubuntu.py from ENV. -+# [1] https://github.com/openstack/fuel-mirror/blob/stable/mitaka/ -+# contrib/fuel_mirror/data/ubuntu.yaml -+ -+# Main is a required parameter which defines what repository will be used -+# for images creation and that mirror should contain all packages for minimal -+# system creation. -+groups: -+ ubuntu: -+ - name: "ubuntu" -+ main: true -+ suite: "trusty" -+ section: -+ - "main" -+ - "multiverse" -+ - "restricted" -+ - "universe" -+ type: "deb" -+ priority: null -+ -+ - name: "ubuntu-updates" -+ suite: "trusty-updates" -+ section: -+ - "main" -+ - "multiverse" -+ - "restricted" -+ - "universe" -+ type: "deb" -+ priority: null -+ -+ - name: "ubuntu-security" -+ suite: "trusty-security" -+ section: -+ - "main" -+ - "multiverse" -+ - "restricted" -+ - "universe" -+ type: "deb" -+ priority: null -+ -+ mos: -+ - name: "mos" -+ suite: "mos$mos_version" -+ section: -+ - "main" -+ - "restricted" -+ type: "deb" -+ priority: 1000 -+ -+ - name: "mos-updates" -+ suite: "mos$mos_version-updates" -+ section: -+ - "main" -+ - "restricted" -+ type: "deb" -+ priority: 1000 -+ -+ - name: "mos-security" -+ suite: "mos$mos_version-security" -+ section: -+ - "main" -+ - "restricted" -+ type: "deb" -+ priority: 1000 -+ -+ - name: "mos-holdback" -+ suite: "mos$mos_version-holdback" -+ section: -+ - "main" -+ - "restricted" -+ type: "deb" -+ priority: 1000 -+ -+# PACKAGES -+ -+# This section lists packages that should be present in the local Ubuntu mirror, -+# but are not direct dependencies of any other packages from MOS or Ubuntu. -+# e.g.: additional kernels, bootloaders etc. -+ -+packages: -+ - name: "vgabios" -+# Packages are required to build bootstrap images for a system. -+# The mirror should contiain such packages in addition to local mirror. -+ - name: "acpi-support" -+ - name: "anacron" -+ - name: "aptitude" -+ - name: "atop" -+ - name: "acct" -+ - name: "bash-completion" -+ - name: "bc" -+ - name: "build-essential" -+ - name: "cloud-init" -+ - name: "conntrackd" -+ - name: "cpu-checker" -+ - name: "cpufrequtils" -+ - name: "debconf-utils" -+ - name: "devscripts" -+ - name: "fping" -+ - name: "git" -+ - name: "grub-pc" -+ - name: "htop" -+ - name: "hwloc" -+ - name: "ifenslave" -+ - name: "iperf" -+ - name: "iptables-persistent" -+ - name: "irqbalance" -+ - name: "language-pack-en" -+ - name: "libapache2-mod-fastcgi" -+ - name: "libnss3-tools" -+ - name: "linux-firmware-nonfree" -+ - name: "linux-headers-generic-lts-trusty" -+ - name: "linux-headers-generic-lts-xenial" -+ - name: "linux-image-generic-lts-trusty" -+ - name: "linux-image-generic-lts-xenial" -+ - name: "live-boot" -+ - name: "livecd-rootfs" -+ - name: "mc" -+ - name: "memcached" -+ - name: "monit" -+ - name: "msmtp-mta" -+ - name: "multipath-tools" -+ - name: "multipath-tools-boot" -+ - name: "nginx" -+ - name: "ntp" -+ - name: "openssh-server" -+ - name: "percona-toolkit" -+ - name: "percona-xtrabackup" -+ - name: "pm-utils" -+ - name: "puppet" -+ - name: "python-lesscpy" -+ - name: "python-pip" -+ - name: "rsyslog-gnutls" -+ - name: "rsyslog-relp" -+ - name: "screen" -+ - name: "squashfs-tools" -+ - name: "swift-plugin-s3" -+ - name: "sysfsutils" -+ - name: "sysstat" -+ - name: "telnet" -+ - name: "tmux" -+ - name: "traceroute" -+ - name: "ubuntu-standard" -+ - name: "vim" -+ - name: "virt-what" -+ - name: "xinetd" -+ - name: "xmlstarlet" -+ - name: "tftpd-hpa" -+ - name: "syslinux" -diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py b/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py -deleted file mode 100755 -index 1c7eb59..0000000 ---- a/build/f_isoroot/f_repobuild/opnfv_mirror_conf.py -+++ /dev/null -@@ -1,57 +0,0 @@ --#!/usr/bin/env python --############################################################################## --# Copyright (c) 2015 Ericsson AB and others. --# mskalski@mirantis.com --# All rights reserved. This program and the accompanying materials --# are made available under the terms of the Apache License, Version 2.0 --# which accompanies this distribution, and is available at --# http://www.apache.org/licenses/LICENSE-2.0 --############################################################################## --import os --import yaml -- --current_snapshot = os.environ["LATEST_TARGET_UBUNTU"] --mos_version = os.environ['MOS_VERSION'] --openstack_version = os.environ['OPENSTACK_VERSION'] --mos_ubuntu = os.environ['MIRROR_MOS_UBUNTU'] --mos_ubuntu_root = os.environ['MIRROR_MOS_UBUNTU_ROOT'] --mirror_ubuntu = os.environ['MIRROR_UBUNTU_URL'] --if os.environ.get('BUILD_FUEL_PLUGINS'): -- plugins = os.environ['BUILD_FUEL_PLUGINS'] --else: -- plugins = os.environ['PLUGINS'] -- -- --configuration_file = open('fuel-mirror/contrib/fuel_mirror/etc/config.yaml').read() --conf = yaml.load(configuration_file) --conf['pattern_dir'] = '.' --conf['openstack_version'] = openstack_version --conf['mos_version'] = mos_version -- --with open('opnfv-config.yaml', 'w') as outfile: -- outfile.write( yaml.dump(conf, default_flow_style=False) ) -- --pattern_file = open('fuel-mirror/contrib/fuel_mirror/data/ubuntu.yaml').read() --pattern = yaml.load(pattern_file) --pattern['mos_baseurl'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) --pattern['ubuntu_baseurl'] = mirror_ubuntu --for group in pattern['groups']['mos']: -- group['uri'] = pattern['mos_baseurl'] --for group in pattern['groups']['ubuntu']: -- group['uri'] = pattern['ubuntu_baseurl'] -- --for plugin in plugins.split(): -- path = "../{}/packages.yaml".format(plugin) -- if os.path.isfile(path): -- f = open(path).read() -- plugin_yaml = yaml.load(f) -- plugin_set = set(plugin_yaml['packages']) -- main_set = set(pattern['packages']) -- new_packages = plugin_set - main_set -- print "Plugin {} require new packages: {}".format(plugin, ', '.join(new_packages)) -- pattern['packages'] = pattern['packages'] + list(new_packages) -- --pattern['requirements']['ubuntu'] = pattern['packages'] -- --with open('ubuntu.yaml', 'w') as outfile: -- outfile.write( yaml.safe_dump(pattern, default_flow_style=False) ) -diff --git a/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py -new file mode 100755 -index 0000000..5b59182 ---- /dev/null -+++ b/build/f_isoroot/f_repobuild/opnfv_mirror_ubuntu.py -@@ -0,0 +1,277 @@ -+#!/usr/bin/env python -+############################################################################## -+# Copyright (c) 2015,2016 Ericsson AB, Mirantis Inc., Enea AB and others. -+# mskalski@mirantis.com -+# Alexandru.Avadanii@enea.com -+# All rights reserved. This program and the accompanying materials -+# are made available under the terms of the Apache License, Version 2.0 -+# which accompanies this distribution, and is available at -+# http://www.apache.org/licenses/LICENSE-2.0 -+############################################################################## -+ -+############################################################################## -+# Build multiarch partial local Ubuntu mirror -+############################################################################## -+# Design quirks / workarounds: -+# 1. Fuel-agent uses `debootstrap` to build bootstrap and target chroots from -+# the local mirror; which only uses the "main" component from the first -+# repository, i.e. does not include "updates"/"security". -+# In order to fullfill all debootstrap dependencies in "main" repo, we will -+# do an extra packetary run using a reduced scope: -+# - only "main" component of the first mirror; -+# - reduced package dependency list (without MOS/OPNFV plugin deps). -+# 2. If repo structure is not mandatory to be in sync with official mirrors, -+# we can mitigate the issue by "merging" all repo-components into a single -+# "main". -+############################################################################## -+# Mirror build steps (for EACH architecture in UBUNTU_ARCH): -+# 1. Collect bootstrap package deps from <fuel_bootstrap_cli.yaml>; -+# 2. Collect all fixture release packages from fuel-web's <openstack.yaml>; -+# 3. Parse new "opnfv_config.yaml" list of packages (from old fuel-mirror); -+# 4. Inherit enviroment variable(s) for mirror URLs, paths etc. -+# - Allow arch-specific overrides for each env var; -+# 5. Mirror config is defined based on common config + OPNFV overrides; -+# - Convert old configuration format to packetary style where needed; -+# 6. Package lists are defined based on common config + OPNFV deps; -+# - Keep track of "main" packages separately, required by debootstrap; -+# 7. Clone/update all mirror components; -+# 8. IF mirror merging is disabled OR workaround for ifupdown (see below): -+# - Clone/update "main" mirror component (fix missing debootstrap deps); -+# 9. IF mirror merging is enabled: -+# - Use `dpkg-scanpackages` to filter out old versions of duplicate pkgs; -+# - Run `packetary create` on the set of downloaded packages, merging -+# them on the fly into a single-component mirror; -+############################################################################## -+ -+import copy -+import os -+import shutil -+import sys -+import yaml -+from contextlib import contextmanager -+from cStringIO import StringIO -+from packetary.cli.app import main -+ -+@contextmanager -+def captureStdOut(output): -+ stdout = sys.stdout -+ sys.stdout = output -+ yield -+ sys.stdout = stdout -+ -+# FIXME: Find a better approach for eliminating duplicate logs than this -+def force_logger_reload(): -+ for mod in sys.modules.keys(): -+ if mod.startswith('logging'): -+ try: -+ reload(sys.modules[mod]) -+ except: -+ pass -+ -+# Determine missing package dependecies for a mirror defition -+def get_unres_pkgs(arch, cfg_mirror): -+ unresolved_pkgs = list() -+ packetary_output = StringIO() -+ with captureStdOut(packetary_output): -+ main('unresolved -a {0} -r {1} -c name version --sep ;' -+ .format(_ARCHITECTURES[arch], cfg_mirror).split(' ')) -+ for dep_pkg in packetary_output.getvalue().splitlines(): -+ if dep_pkg.startswith('#'): -+ continue -+ dep = dep_pkg.split(';') -+ unresolved_pkgs += [ {'name': dep[0], 'version': dep[1]} ] -+ force_logger_reload() -+ return unresolved_pkgs -+ -+# Package list conversion from `old fuel-mirror` to `packetary` style -+def from_legacy_pkglist(legacy_pkglist): -+ pkglist = list() -+ for pkg in legacy_pkglist: -+ pkglist += [ {'name': pkg} ] -+ return pkglist -+ -+def to_legacy_pkglist(pkglist): -+ legacy_pkglist = list() -+ for pkg in pkglist: -+ legacy_pkglist.append(pkg['name']) -+ return legacy_pkglist -+ -+def legacy_diff(base_pkglist, new_pkglist, requester, arch): -+ diff_set = set(new_pkglist) -+ if base_pkglist: -+ diff_set -= set(base_pkglist) -+ if diff_set: -+ print(' * {0} requires new packages for arch [{1}]: {2}' -+ .format(requester, arch, ', '.join(diff_set))) -+ return list(diff_set) -+ -+# Create single-component local repo (one arch per call) -+def do_local_repo(arch, cfg_repo, cfg_packages_paths): -+ # Packetary does not use a global config file, so pass old settings here. -+ main('create -t deb -a {0} --repository {1} --package-files {2}' -+ ' --ignore-errors-num 2 --retries-num 3 --threads-num 10' -+ .format(_ARCHITECTURES[arch], cfg_repo, cfg_packages_paths).split(' ')) -+ force_logger_reload() -+ -+# Clone partial local mirror (one arch per call) -+def do_partial_mirror(arch, cfg_mirror, cfg_packages): -+ # Note: '-d .' is ignored, as each mirror defines its own path. -+ main('clone -t deb -a {0} -r {1} -R {2} -d .' -+ ' --ignore-errors-num 2 --retries-num 3 --threads-num 10' -+ .format(_ARCHITECTURES[arch], cfg_mirror, cfg_packages).split(' ')) -+ force_logger_reload() -+ -+# Write configuration (yaml) file (package list / mirror defition) -+def write_cfg_file(cfg_mirror, data): -+ with open(cfg_mirror, 'w') as outfile: -+ outfile.write( yaml.safe_dump(data, default_flow_style=False) ) -+ -+# Allow arch-specific overrides of env vars -+def get_env(env_var, arch=None): -+ if arch: -+ env_var_arch = '{0}_{1}'.format(env_var, arch) -+ if os.environ.get(env_var_arch): -+ return os.environ[env_var_arch] -+ if os.environ.get(env_var): -+ return os.environ[env_var] -+ return None -+ -+# Architecture name mapping (dpkg:packetary) for packetary CLI invocation -+_ARCHITECTURES = { -+ "i386": "i386", -+ "amd64": "x86_64", -+ "arm64": "aarch64", -+} -+ -+# Arch-indepedent configuration (old fuel-mirror + OPNFV extra packages) -+cfg_dir = 'opnfv_config' -+cfg_p_opnfv = 'opnfv_config.yaml' -+mos_version = get_env('MOS_VERSION') -+ubuntu_arch = get_env('UBUNTU_ARCH') -+mirror_ubuntu_path = get_env('MIRROR_UBUNTU_OPNFV_PATH') -+mirror_ubuntu_tmp_path = '{0}.tmp'.format(mirror_ubuntu_path) -+mirror_ubuntu_merge = get_env('MIRROR_UBUNTU_MERGE') -+cfg_mm_ubuntu = '{0}/ubuntu_mirror_local.yaml'.format(cfg_dir) -+fuel_bootstrap_cli_file = open('fuel_bootstrap_cli.yaml').read() -+fuel_bootstrap_cli = yaml.load(fuel_bootstrap_cli_file) -+fixture_file = open('fuel-web/nailgun/nailgun/fixtures/openstack.yaml').read() -+fixture = yaml.load(fixture_file) -+opnfv_cfg_yaml = open(cfg_p_opnfv).read() -+opnfv_cfg = yaml.load(opnfv_cfg_yaml) -+ -+# FIXME: Packetary solves missing dependecies by also accepting -+# different packages that provide the same name (e.g. "ifupdown" dependency -+# is satisfied by "netscript" package from "universe" repo-component). -+# Work around this by resolving all deps in "main" repo-component, -+# then scan and keep only latest debs for the whole <merged> repo. -+mirror_ubuntu_resolve_main_deps = True -+ -+# Create local partial mirror using packetary, one arch at a time -+for arch in ubuntu_arch.split(' '): -+ # Mirror / Package env vars, arch-overrideable -+ mos_ubuntu = get_env('MIRROR_MOS_UBUNTU', arch) -+ mos_ubuntu_root = get_env('MIRROR_MOS_UBUNTU_ROOT', arch) -+ mirror_ubuntu = get_env('MIRROR_UBUNTU_URL', arch) -+ plugins = get_env('BUILD_FUEL_PLUGINS', arch) -+ if plugins is None: -+ plugins = get_env('PLUGINS', arch) -+ -+ # Mirror / Package list configuration files (arch-specific) -+ cfg_m_mos = '{0}/mos_{1}_mirror.yaml'.format(cfg_dir, arch) -+ cfg_m_ubuntu = '{0}/ubuntu_{1}_mirror.yaml'.format(cfg_dir, arch) -+ cfg_p_ubuntu = '{0}/ubuntu_{1}_packages.yaml'.format(cfg_dir, arch) -+ cfg_m_ubuntu_main = '{0}/ubuntu_{1}_mirror_main.yaml'.format(cfg_dir, arch) -+ cfg_p_ubuntu_main = '{0}/ubuntu_{1}_packages_main.yaml'.format(cfg_dir, arch) -+ -+ # Mirror config fork before customizing (arch-specific) -+ arch_group_mos = 'mos_{0}'.format(arch) -+ arch_group_ubuntu = 'ubuntu_{0}'.format(arch) -+ arch_packages = 'packages_{0}'.format(arch) -+ opnfv_cfg['groups'][arch_group_mos] = copy.deepcopy(opnfv_cfg['groups']['mos']) -+ opnfv_cfg['groups'][arch_group_ubuntu] = copy.deepcopy(opnfv_cfg['groups']['ubuntu']) -+ opnfv_cfg[arch_packages] = opnfv_cfg['packages'] -+ -+ # Mirror config update & conversion to packetary input -+ group_main_ubuntu = dict() -+ for group in opnfv_cfg['groups'][arch_group_mos]: -+ group['uri'] = "http://{}{}".format(mos_ubuntu, mos_ubuntu_root) -+ group['suite'] = group['suite'].replace('$mos_version', mos_version) -+ for group in opnfv_cfg['groups'][arch_group_ubuntu]: -+ group['uri'] = mirror_ubuntu -+ # FIXME: At `create`, packetary insists on copying all pkgs to dest dir, -+ # so configure it for another dir, which will replace the orig at the end. -+ group['path'] = mirror_ubuntu_tmp_path -+ if not group_main_ubuntu and 'main' in group: -+ group_main_ubuntu = [ copy.deepcopy(group) ] -+ group_main_ubuntu[0]['section'] = [ 'main' ] -+ -+ # Mirror config dump: MOS (for dep resolution), Ubuntu, Ubuntu[main] -+ write_cfg_file(cfg_m_mos, opnfv_cfg['groups'][arch_group_mos]) -+ write_cfg_file(cfg_m_ubuntu, opnfv_cfg['groups'][arch_group_ubuntu]) -+ if mirror_ubuntu_resolve_main_deps or mirror_ubuntu_merge is None: -+ write_cfg_file(cfg_m_ubuntu_main, group_main_ubuntu) -+ if mirror_ubuntu_merge is not None: -+ # FIXME: For multiarch, only one dump would be enough -+ group_main_ubuntu[0]['origin'] = 'Ubuntu' -+ group_main_ubuntu[0]['path'] = mirror_ubuntu_path -+ group_main_ubuntu[0]['uri'] = mirror_ubuntu_path -+ write_cfg_file(cfg_mm_ubuntu, group_main_ubuntu[0]) -+ -+ # Collect package dependencies from: -+ ## 1. fuel_bootstrap_cli.yaml (bootstrap image additional packages) -+ legacy_unresolved = legacy_diff(None, fuel_bootstrap_cli['packages'] + [ -+ fuel_bootstrap_cli['kernel_flavor'], -+ fuel_bootstrap_cli['kernel_flavor'].replace('image', 'headers')], -+ 'Bootstrap', arch) -+ ## 2. openstack.yaml fixture definition (default target image packages) -+ for release in fixture: -+ editable = release['fields']['attributes_metadata']['editable'] -+ if 'provision' in editable and 'packages' in editable['provision']: -+ release_pkgs = editable['provision']['packages']['value'].split() -+ legacy_unresolved += legacy_diff(legacy_unresolved, -+ release_pkgs, 'Release {0}'.format(release['fields']['name']), arch) -+ ## 3. OPNFV additional packages (includes old fuel-mirror ubuntu.yaml pkgs) -+ unresolved = dict() -+ unresolved['mandatory'] = 'exact' -+ unresolved['packages'] = from_legacy_pkglist(legacy_unresolved) -+ if 'packages' in opnfv_cfg: -+ legacy_diff(legacy_unresolved, to_legacy_pkglist(opnfv_cfg['packages']), -+ 'OPNFV config', arch) -+ unresolved['packages'] += opnfv_cfg['packages'] -+ -+ # OPNFV plugins dependency resolution -+ for plugin in plugins.split(): -+ path = "../{}/packages.yaml".format(plugin) -+ if os.path.isfile(path): -+ f = open(path).read() -+ plugin_yaml = yaml.load(f) -+ new_packages = legacy_diff(to_legacy_pkglist(unresolved['packages']), -+ plugin_yaml['packages'], 'Plugin {0}'.format(plugin), arch) -+ unresolved['packages'] += from_legacy_pkglist(new_packages) -+ -+ # Package list (reduced, i.e. no MOS deps, but with OPNFV plugin deps) -+ # FIXME: This helps work around packetary solving main deps from universe -+ if mirror_ubuntu_resolve_main_deps or mirror_ubuntu_merge is None: -+ write_cfg_file(cfg_p_ubuntu_main, unresolved) -+ -+ # Mirror package list (full, including MOS/OPNFV plugin deps) -+ unresolved['packages'] += get_unres_pkgs(arch, cfg_m_mos) -+ write_cfg_file(cfg_p_ubuntu, unresolved) -+ do_partial_mirror(arch, cfg_m_ubuntu, cfg_p_ubuntu) -+ if mirror_ubuntu_resolve_main_deps or mirror_ubuntu_merge is None: -+ # Ubuntu[main] must be evaluated after Ubuntu -+ do_partial_mirror(arch, cfg_m_ubuntu_main, cfg_p_ubuntu_main) -+ -+if mirror_ubuntu_merge is None: -+ shutil.move(mirror_ubuntu_tmp_path, mirror_ubuntu_path) -+else: -+ # Construct single-component mirror from all components -+ for arch in ubuntu_arch.split(' '): -+ cfg_pp_ubuntu = '{0}/ubuntu_{1}_packages_paths.yaml'.format(cfg_dir, arch) -+ # FIXME: We need scanpackages to omit older DEBs -+ # Inspired from http://askubuntu.com/questions/198474/ -+ os.system('dpkg-scanpackages -a {0} {1} 2>/dev/null | ' -+ 'grep -e "^Filename:" | sed "s|Filename: |- file://|g" > {2}' -+ .format(arch, mirror_ubuntu_tmp_path, cfg_pp_ubuntu)) -+ do_local_repo(arch, cfg_mm_ubuntu, cfg_pp_ubuntu) -+ shutil.rmtree(mirror_ubuntu_tmp_path) -diff --git a/deploy/config/dea_base.yaml b/deploy/config/dea_base.yaml -index ce81994..2520cf5 100644 ---- a/deploy/config/dea_base.yaml -+++ b/deploy/config/dea_base.yaml -@@ -622,22 +622,10 @@ settings: - value: - - name: ubuntu - priority: null -- section: main universe multiverse -+ section: main - suite: trusty - type: deb -- uri: http://10.20.0.2:8080/mirrors/ubuntu/ -- - name: ubuntu-updates -- priority: null -- section: main universe multiverse -- suite: trusty-updates -- type: deb -- uri: http://10.20.0.2:8080/mirrors/ubuntu/ -- - name: ubuntu-security -- priority: null -- section: main universe multiverse -- suite: trusty-security -- type: deb - uri: http://10.20.0.2:8080/mirrors/ubuntu/ - - name: mos - priority: 1050 - section: main restricted diff --git a/patches/opnfv-fuel/upstream-backports/0004-Fuel-rework-for-multiple-libvirt.patch b/patches/opnfv-fuel/upstream-backports/0004-Fuel-rework-for-multiple-libvirt.patch index 271e0fa0..e08026e6 100644 --- a/patches/opnfv-fuel/upstream-backports/0004-Fuel-rework-for-multiple-libvirt.patch +++ b/patches/opnfv-fuel/upstream-backports/0004-Fuel-rework-for-multiple-libvirt.patch @@ -736,7 +736,7 @@ index 3dd5f84..d738154 100644 @@ -41,17 +41,42 @@ dea-override-config: - id: 1 interfaces: interfaces_1 - role: mongo,controller + role: mongo,controller,congress + transformations: transformations_1 - id: 2 interfaces: interfaces_1 diff --git a/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch b/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch index 2315c5ad..48fd31db 100644 --- a/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch +++ b/patches/opnfv-fuel/upstream-backports/0005-CI-deploy-cache-Store-and-reuse-deploy-artifacts.patch @@ -63,9 +63,9 @@ index 0000000..80cd0f4 +--- a/iso/bootstrap_admin_node.sh ++++ b/iso/bootstrap_admin_node.sh +@@ -61,6 +61,8 @@ wget \ -+ + ASTUTE_YAML='/etc/fuel/astute.yaml' + BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf" ++ CUSTOM_REPOS="/root/default_deb_repos.yaml" ++OPNFV_CACHE_PATH="/var/cache/opnfv/bootstraps" ++OPNFV_CACHE_TAR="opnfv-bootstraps-cache.tar" + bs_build_log='/var/log/fuel-bootstrap-image-build.log' @@ -179,8 +179,7 @@ diff --git a/deploy/cloud/deployment.py b/deploy/cloud/deployment.py index 5dd0263..3db4c0d 100644 --- a/deploy/cloud/deployment.py +++ b/deploy/cloud/deployment.py -@@ -24,6 +24,8 @@ from common import ( - delete, +@@ -24,5 +24,7 @@ from common import ( ) +from deploy_cache import DeployCache diff --git a/patches/opnfv-fuel/upstream-backports/0006-Restart-sshd-service-instead-of-reload.patch b/patches/opnfv-fuel/upstream-backports/0006-Restart-sshd-service-instead-of-reload.patch deleted file mode 100644 index 6e876e85..00000000 --- a/patches/opnfv-fuel/upstream-backports/0006-Restart-sshd-service-instead-of-reload.patch +++ /dev/null @@ -1,26 +0,0 @@ -From: Michael Polenchuk <mpolenchuk@mirantis.com> -Date: Tue, 13 Dec 2016 17:42:23 +0400 -Subject: [PATCH] Restart sshd service instead of reload - -In the middle of fuel master node bootstrap the reload of sshd service -accidentally is losing main process pid, so restart it. - -JIRA: FUEL-237 -Change-Id: I89023177e71cb6da4b9d6beff5083d345cf66db8 -Signed-off-by: Michael Polenchuk <mpolenchuk@mirantis.com> ---- - build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch b/build/f_repos/patch/fuel-main/0001-Patches-for-OPNFV.patch -index 820e387..d01ce8b 100644 ---- a/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch -+++ b/build/f_repos/patch/fuel-main/0001-OPNFV-Additions-to-bootstrap_admin_node.sh.patch -@@ -41,6 +41,6 @@ index dbcac5f..1826390 100755 - -systemctl enable sshd - -systemctl start sshd --+systemctl reload sshd -++systemctl restart sshd - - # Enable iptables - systemctl enable iptables.service diff --git a/patches/opnfv-fuel/upstream-backports/0007-build-select_ubuntu_repo-break-on-err.patch b/patches/opnfv-fuel/upstream-backports/0007-build-select_ubuntu_repo-break-on-err.patch deleted file mode 100644 index fdf7be87..00000000 --- a/patches/opnfv-fuel/upstream-backports/0007-build-select_ubuntu_repo-break-on-err.patch +++ /dev/null @@ -1,70 +0,0 @@ -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Tue, 20 Dec 2016 00:49:47 +0100 -Subject: [PATCH] build: select_ubuntu_repo: break on err - -Stop ISO build process if no sane Ubuntu mirrors are available, -instead of continuing with an empty URL, which is bound to fail -later. - -While at it, skip re-assesing the output of select_ubuntu_repo.sh if -the URL var already is set. - -Also, stderr and stdout were both captured in UBUNTU_MIRROR_URL, -so disable debug by default, and only print debug info in case no -valid mirror is found. - -v2 -> v3: -- allow "One hour behind" mirror state to be considered, since - we already accept "Six hours behind" mirrors; - -JIRA: FUEL-240 - -Change-Id: I791942da234304528ff5cae1891415602e1a62b2 -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - build/Makefile | 10 +++++++--- - build/f_isoroot/f_repobuild/select_ubuntu_repo.sh | 4 ++-- - 2 files changed, 9 insertions(+), 5 deletions(-) - -diff --git a/build/Makefile b/build/Makefile -index c8fc793..d4b981a 100644 ---- a/build/Makefile -+++ b/build/Makefile -@@ -51,9 +51,13 @@ export REPOINFO := $(BUILD_BASE)/repo_info.sh - - # Use snapshots - # Use nearby repositories --export MIRROR_UBUNTU_URL := $(shell ./f_isoroot/f_repobuild/select_ubuntu_repo.sh --url) --export MIRROR_UBUNTU := $(shell echo "$(MIRROR_UBUNTU_URL)" | cut -d'/' -f3 ) --export MIRROR_UBUNTU_ROOT := $(shell echo -n '/' ; echo "$(MIRROR_UBUNTU_URL)" | cut -d'/' -f4-) -+export MIRROR_UBUNTU_URL ?= $(shell ./f_isoroot/f_repobuild/select_ubuntu_repo.sh) -+export MIRROR_UBUNTU ?= $(shell echo "$(MIRROR_UBUNTU_URL)" | cut -d'/' -f3 ) -+export MIRROR_UBUNTU_ROOT ?= $(shell echo -n '/' ; echo "$(MIRROR_UBUNTU_URL)" | cut -d'/' -f4-) -+ifeq (,$(MIRROR_UBUNTU_URL)) -+$(warning $(shell ./f_isoroot/f_repobuild/select_ubuntu_repo.sh -d)) -+$(error No sane Ubuntu mirror available) -+endif - - export LATEST_MIRROR_ID_URL := http://$(shell ./select_closest_fuel_mirror.py) - -diff --git a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh -index 24bd422..5618eb7 100755 ---- a/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh -+++ b/build/f_isoroot/f_repobuild/select_ubuntu_repo.sh -@@ -69,14 +69,14 @@ if [ "$1" == "-d" ]; then - fi - - # Hardcode for testing purposes --DEBUG=1 -+# DEBUG=1 - - TMPFILE=$(mktemp /tmp/mirrorsXXXXX)A - trap cleanup exit - - # Generate a list of mirrors considered as "up" - curl -s https://launchpad.net/ubuntu/+archivemirrors | \ -- grep -P -B8 "statusUP|statusSIX" | \ -+ grep -P -B8 "statusUP|statusONE|statusSIX" | \ - grep -o -P "(f|ht)tp.*\"" | \ - sed 's/"$//' | sort | uniq > $TMPFILE - |