Clean up and fix Libvirt and Ceph

[ Stanislaw Kardach <stanislaw.kardach@caviumnetworks.com> ]

* Fix upload_cirros timeout
* Workaround for default video device in nova
* Fix syntax and increase TestVM's RAM requirements

[ Alexandru Avadanii <Alexandru.Avadanii@enea.com> ]

* Cleanup: Obsolete SEPARATE_IMAGES.
* Nova libvirt drv: arm64: Update console default.
* fuel{,-main}: Cleanup obsolete patches.
* nova: Fix inject for direct boot with part table.
    See https://bugs.launchpad.net/nova/+bug/1290455 for full bug
    description. For now, we detect direct kernel boot with a partition
    table inside disk image by passing target_partition when root kernel
    arg points to a partition (instead of the whole disk).
* Drop obsolete/done TODO entries.
* Nova: Fix os cmd line parsing for list case.
* ceph: Fix obsolete XFS mount param.
* Fix out-of-order embedded patch hunks.

Signed-off-by: Stanislaw Kardach <stanislaw.kardach@caviumnetworks.com>
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

4 files changed, 264 insertions, 0 deletions

diff --git a/patches/fuel-library/0008-Increase-upload_cirros-timeout-for-multi-image.patch b/patches/fuel-library/0008-Increase-upload_cirros-timeout-for-multi-image.patch
new file mode 100644
index 00000000..7f240c8e
--- /dev/null
+++ b/patches/fuel-library/0008-Increase-upload_cirros-timeout-for-multi-image.patch
@@ -0,0 +1,21 @@
+From: Stanislaw Kardach <stanislaw.kardach@caviumnetworks.com>
+Date: Tue, 22 Mar 2016 10:54:32 +0100
+Subject: [PATCH] Increase upload_cirros timeout for multi-image
+
+---
+ deployment/puppet/osnailyfacter/modular/astute/tasks.yaml | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/deployment/puppet/osnailyfacter/modular/astute/tasks.yaml b/deployment/puppet/osnailyfacter/modular/astute/tasks.yaml
+index a2cb1da..d9d2dcc 100644
+--- a/deployment/puppet/osnailyfacter/modular/astute/tasks.yaml
++++ b/deployment/puppet/osnailyfacter/modular/astute/tasks.yaml
+@@ -19,7 +19,7 @@
+     cmd: ruby /etc/puppet/modules/osnailyfacter/modular/astute/upload_cirros.rb
+     retries: 3
+     interval: 20
+-    timeout: 180
++    timeout: 360
+ 
+ - id: upload_nodes_info
+   type: upload_file
diff --git a/patches/fuel-library/0009-Update-vga-console-defaults-for-armv7-and-aarch64.patch b/patches/fuel-library/0009-Update-vga-console-defaults-for-armv7-and-aarch64.patch
new file mode 100644
index 00000000..a48e925f
--- /dev/null
+++ b/patches/fuel-library/0009-Update-vga-console-defaults-for-armv7-and-aarch64.patch
@@ -0,0 +1,103 @@
+From: Stanislaw Kardach <stanislaw.kardach@caviumnetworks.com>
+Date: Tue, 22 Mar 2016 12:05:09 +0100
+Subject: [PATCH] Update vga, console defaults for armv7 and aarch64
+
+Nova hardcodes default options for both video=cirrus and console.
+armv7 and aarch64 VMs require video=vga, and since most applications
+use PL011 serial driver for guests, adding console=ttyAMA0 is also
+a nice UX addition.
+---
+ .../openstack/files/nova-libvirt-vga-console.patch | 39 ++++++++++++++++++++++
+ deployment/puppet/openstack/manifests/compute.pp   | 19 +++++++++++
+ 2 files changed, 58 insertions(+)
+ create mode 100644 deployment/puppet/openstack/files/nova-libvirt-vga-console.patch
+
+diff --git a/deployment/puppet/openstack/files/nova-libvirt-vga-console.patch b/deployment/puppet/openstack/files/nova-libvirt-vga-console.patch
+new file mode 100644
+index 0000000..d33d4ee
+--- /dev/null
++++ b/deployment/puppet/openstack/files/nova-libvirt-vga-console.patch
+@@ -0,0 +1,39 @@
++From: Stanislaw Kardach <stanislaw.kardach@caviumnetworks.com>
++Date: Tue, 22 Mar 2016 12:05:09 +0100
++Subject: [PATCH] Update vga, console defaults for armv7 and aarch64
++
++Nova hardcodes default options for both video=cirrus and console.
++armv7 and aarch64 VMs require video=vga, and since most applications
++use PL011 serial driver for guests, adding console=ttyAMA0 is also
++a nice UX addition.
++
++Signed-off-by: Stanislaw Kardach <stanislaw.kardach@caviumnetworks.com>
++Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
++---
++
++diff --git a/virt/libvirt/driver.py b/virt/libvirt/driver.py
++index 920a283..751b140 100644
++--- a/virt/libvirt/driver.py
+++++ b/virt/libvirt/driver.py
++@@ -3834,7 +3834,11 @@
++         if virt_type == "xen":
++             guest.os_cmdline = "ro root=%s" % root_device_name
++         else:
+++            guestarch = libvirt_utils.get_arch(image_meta)
++             guest.os_cmdline = ("root=%s %s" % (root_device_name, CONSOLE))
+++            if guestarch in (arch.ARMV7, arch.AARCH64):
+++                # NOTE(armband): ARM v7/v8 use PL011 drv, add ttyAMA0 console
+++                guest.os_cmdline += " console=ttyAMA0"
++             if virt_type == "qemu":
++                 guest.os_cmdline += " no_timer_check"
++         if instance.ramdisk_id:
++@@ -3972,7 +3972,8 @@ class LibvirtDriver(driver.ComputeDriver):
++             video.type = 'xen'
++         elif CONF.libvirt.virt_type == 'parallels':
++             video.type = 'vga'
++-        elif guestarch in (arch.PPC, arch.PPC64):
+++        elif guestarch in (arch.ARMV7, arch.AARCH64, arch.PPC, arch.PPC64):
+++            # NOTE(armband): Added ARM v7/v8, same as on PPC.
++             # NOTE(ldbragst): PowerKVM doesn't support 'cirrus' be default
++             # so use 'vga' instead when running on Power hardware.
++             video.type = 'vga'
+diff --git a/deployment/puppet/openstack/manifests/compute.pp b/deployment/puppet/openstack/manifests/compute.pp
+index c0349dc..46b1801 100644
+--- a/deployment/puppet/openstack/manifests/compute.pp
++++ b/deployment/puppet/openstack/manifests/compute.pp
+@@ -157,6 +157,12 @@ class openstack::compute (
+     }
+   }
+ 
++  if ! defined(Package['patch']) {
++    package { 'patch':
++      ensure => 'present',
++    }
++  }
++
+   $glance_connection = $glance_api_servers
+ 
+   case $::osfamily {
+@@ -250,6 +256,7 @@ class openstack::compute (
+ 
+   $memcached_addresses =  suffix($cache_server_ip, inline_template(":<%= @cache_server_port %>"))
+   $notify_on_state_change = 'vm_and_task_state'
++  $nova_path = '/usr/lib/python2.7/dist-packages/nova'
+ 
+   class { 'nova':
+       install_utilities      => false,
+@@ -273,6 +280,18 @@ class openstack::compute (
+       service_down_time      => $nova_service_down_time,
+       notify_on_state_change => $notify_on_state_change,
+       memcached_servers      => $memcached_addresses,
++  } ->
++  # FIXME(armband): Workaround for missing arm defaults in nova libvirt driver
++  file { "${nova_path}/libvirt-vga-console.patch":
++    ensure => "file",
++    source => "puppet:///modules/openstack/nova-libvirt-vga-console.patch",
++  } ->
++  exec { 'nova libvirt driver patch arm defaults':
++    path    => ['/usr/bin'],
++    command => "patch -p1 < ${nova_path}/libvirt-vga-console.patch",
++    unless  => "patch -p1 -R -N --dry-run < ${nova_path}/libvirt-vga-console.patch",
++    cwd     => $nova_path,
++    require => [Package['patch']],
+   }
+ 
+   if str2bool($::is_virtual) {
diff --git a/patches/fuel-library/0010-nova-Fix-inject-for-direct-boot-with-part-table.patch b/patches/fuel-library/0010-nova-Fix-inject-for-direct-boot-with-part-table.patch
new file mode 100644
index 00000000..b98984a7
--- /dev/null
+++ b/patches/fuel-library/0010-nova-Fix-inject-for-direct-boot-with-part-table.patch
@@ -0,0 +1,92 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Fri, 25 Mar 2016 16:12:08 +0100
+Subject: [PATCH] nova: Fix inject for direct boot with part table.
+
+See [1] for full bug description.
+For now, we detect direct kernel boot with a partition table inside
+disk image by passing target_partition when root kernel arg points
+to a partition (instead of the whole disk).
+
+[1] https://bugs.launchpad.net/nova/+bug/1290455
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Signed-off-by: Stanislaw Kardach <stanislaw.kardach@caviumnetworks.com>
+---
+ .../openstack/files/nova-libvirt-inject.patch      | 44 ++++++++++++++++++++++
+ deployment/puppet/openstack/manifests/compute.pp   | 12 ++++++
+ 2 files changed, 56 insertions(+)
+ create mode 100644 deployment/puppet/openstack/files/nova-libvirt-inject.patch
+
+diff --git a/deployment/puppet/openstack/files/nova-libvirt-inject.patch b/deployment/puppet/openstack/files/nova-libvirt-inject.patch
+new file mode 100644
+index 0000000..071aff3
+--- /dev/null
++++ b/deployment/puppet/openstack/files/nova-libvirt-inject.patch
+@@ -0,0 +1,44 @@
++From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
++Date: Tue, 22 Mar 2016 12:05:09 +0100
++Subject: [PATCH] Fix inject for direct boot with partition table
++
++See [1] for full bug description.
++For now, we detect direct kernel boot with a partition table inside
++disk image by passing target_partition when root kernel arg points
++to a partition (instead of the whole disk).
++
++[1] https://bugs.launchpad.net/nova/+bug/1290455
++
++Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
++Signed-off-by: Stanislaw Kardach <stanislaw.kardach@caviumnetworks.com>
++---
++
++diff --git a/virt/libvirt/driver.py b/virt/libvirt/driver.py
++index 920a283..751b140 100644
++--- a/virt/libvirt/driver.py
+++++ b/virt/libvirt/driver.py
++@@ -34,6 +34,7 @@
++ import mmap
++ import operator
++ import os
+++import re
++ import shutil
++ import tempfile
++ import time
++@@ -2820,7 +2821,15 @@
++         """
++         # Handles the partition need to be used.
++         target_partition = None
++-        if not instance.kernel_id:
+++        image_meta = objects.ImageMeta.from_instance(instance)
+++        # FIXME(armband): https://bugs.launchpad.net/nova/+bug/1290455
+++        # Support direct kernel boot with a partition table inside disk image:
+++        # pass target_partition when root kernel arg points to a partition
+++        # (instead of the whole disk).
+++        has_partition_table = re.search("root=/dev/.d.([1-9]+)",
+++            str(image_meta.properties.get("os_command_line")))
+++
+++        if not instance.kernel_id or has_partition_table:
++             target_partition = CONF.libvirt.inject_partition
++             if target_partition == 0:
++                 target_partition = None
+diff --git a/deployment/puppet/openstack/manifests/compute.pp b/deployment/puppet/openstack/manifests/compute.pp
+index 46b1801..d42d6a1 100644
+--- a/deployment/puppet/openstack/manifests/compute.pp
++++ b/deployment/puppet/openstack/manifests/compute.pp
+@@ -281,6 +281,18 @@ class openstack::compute (
+       notify_on_state_change => $notify_on_state_change,
+       memcached_servers      => $memcached_addresses,
+   } ->
++  # FIXME(armband): Fix inject for part image direct boot (bug LP #1469308)
++  file { "${nova_path}/libvirt-inject.patch":
++    ensure => "file",
++    source => "puppet:///modules/openstack/nova-libvirt-inject.patch",
++  } ->
++  exec { 'nova libvirt driver patch partitioned injection':
++    path    => ['/usr/bin'],
++    command => "patch -p1 < ${nova_path}/libvirt-inject.patch",
++    unless  => "patch -p1 -R -N --dry-run < ${nova_path}/libvirt-inject.patch",
++    cwd     => $nova_path,
++    require => [Package['patch']],
++  }
+   # FIXME(armband): Workaround for missing arm defaults in nova libvirt driver
+   file { "${nova_path}/libvirt-vga-console.patch":
+     ensure => "file",
diff --git a/patches/fuel-library/0011-ceph-Fix-obsolete-XFS-mount-param-delaylog.patch b/patches/fuel-library/0011-ceph-Fix-obsolete-XFS-mount-param-delaylog.patch
new file mode 100644
index 00000000..0d13efeb
--- /dev/null
+++ b/patches/fuel-library/0011-ceph-Fix-obsolete-XFS-mount-param-delaylog.patch
@@ -0,0 +1,48 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sun, 27 Mar 2016 20:32:50 +0200
+Subject: [PATCH] ceph: Fix obsolete XFS mount param "delaylog".
+
+According to [1], "delaylog" mount arg is now enforced by default,
+and passing it will lead to mount failure.
+
+Trim "delaylog" from default list of xfs mount args and leave it up to
+ceph-osd.pp to add this obsoleted arg only for targets running a kernel
+older than 4.0.
+
+[1] https://www.kernel.org/doc/Documentation/filesystems/xfs.txt
+---
+ deployment/puppet/osnailyfacter/modular/ceph/ceph-osd.pp   | 7 +++++++
+ deployment/puppet/osnailyfacter/modular/globals/globals.pp | 2 +-
+ 2 files changed, 8 insertions(+), 1 deletion(-)
+
+diff --git a/deployment/puppet/osnailyfacter/modular/ceph/ceph-osd.pp b/deployment/puppet/osnailyfacter/modular/ceph/ceph-osd.pp
+index f7da80e..22aab33 100644
+--- a/deployment/puppet/osnailyfacter/modular/ceph/ceph-osd.pp
++++ b/deployment/puppet/osnailyfacter/modular/ceph/ceph-osd.pp
+@@ -50,6 +50,13 @@ class {'ceph':
+ }
+ 
+ if $ceph_tuning_settings != {} {
++  if versioncmp($::kernelmajversion, '4.0') < 0 {
++    # FIXME(armband): XFS mount opt delaylog is deprecated in kernels >=4.0.
++    $ceph_tuning_settings['osd_mount_options_xfs'] = join([
++      $ceph_tuning_settings['osd_mount_options_xfs'],
++      'delaylog'
++    ], ',')
++  }
+   ceph_conf {
+     'global/debug_default'                    : value => $debug;
+     'global/max_open_files'                   : value => $ceph_tuning_settings['max_open_files'];
+diff --git a/deployment/puppet/osnailyfacter/modular/globals/globals.pp b/deployment/puppet/osnailyfacter/modular/globals/globals.pp
+index 268a5b1..69aed7b 100644
+--- a/deployment/puppet/osnailyfacter/modular/globals/globals.pp
++++ b/deployment/puppet/osnailyfacter/modular/globals/globals.pp
+@@ -123,7 +123,7 @@ if ($storage_hash['volumes_ceph'] or $storage_hash['images_ceph'] or $storage_ha
+   $ceph_tuning_settings = {
+     'max_open_files'                       => pick($storage_tuning_settings['max_open_files'], '131072'),
+     'osd_mkfs_type'                        => pick($storage_tuning_settings['osd_mkfs_type'], 'xfs'),
+-    'osd_mount_options_xfs'                => pick($storage_tuning_settings['osd_mount_options_xfs'], 'rw,relatime,inode64,logbsize=256k,delaylog,allocsize=4M'),
++    'osd_mount_options_xfs'                => pick($storage_tuning_settings['osd_mount_options_xfs'], 'rw,relatime,inode64,logbsize=256k,allocsize=4M'),
+     'osd_op_threads'                       => pick($storage_tuning_settings['osd_op_threads'], '20'),
+     'filestore_queue_max_ops'              => pick($storage_tuning_settings['filestore_queue_max_ops'], '500'),
+     'filestore_queue_committing_max_ops'   => pick($storage_tuning_settings['filestore_queue_committing_max_ops'], '5000'),