Merge "Rebase: deploy related patch series."

1 files changed, 233 insertions, 0 deletions

diff --git a/patches/opnfv-fuel/0006-virtual_fuel-initial-support-for-remote-libvirt.patch b/patches/opnfv-fuel/0006-virtual_fuel-initial-support-for-remote-libvirt.patch
new file mode 100644
index 00000000..f6a0f859
--- /dev/null
+++ b/patches/opnfv-fuel/0006-virtual_fuel-initial-support-for-remote-libvirt.patch
@@ -0,0 +1,233 @@
+From: Josep Puigdemont <josep.puigdemont@enea.com>
+Date: Wed, 4 May 2016 14:27:23 +0200
+Subject: [PATCH] virtual_fuel: initial support for remote libvirt
+
+With this patch it should be possible to create a fuel VM on a remote
+libvirt server by properly defining the LIBVIRT_DEFAULT_URI [1]
+environment variable. If the variable is not defined, then there should
+be no percievable change in behaviour for the script.
+
+This patch introduces the ability to create volumes (images) on a
+remote libvirt host where the Fuel VM is to be deployed. For now
+the volumes are created by default in a pool named jenkins, but
+the idea is to allow this to be configured, probably in the POD's
+DHA file.
+
+Since all virsh commands honor LIBVIRT_DEFAULT_URI, we use this
+environment variable to detect wheter we should create a volume or not.
+The rationale being that the variable will only be set if the user wants
+to to do the VM deployment on a remote libvirt host.
+
+We need to create a volume because we can not rely on being able to
+access the remote server's file system directly.
+
+The images are then transferred to the libvirt host using virsh
+commands. All this could also be done using scp and a user directory
+on the host machine, but using pools allows us to take advantage of
+libvirt's policies and file permissions.
+
+CHANGE: when LIBVIRT_DEFAULT_URI is defined, the script will not check
+for the presence of the required PXE bridge. This will still be checked
+when the Fuel VM is started and the bridge not found, but this happens
+at a later point than it does today.
+
+CHANGE: before this patch, the file system image was named like the VM:
+vm_name.raw. This patch introduces a change and adds a timestamp suffix
+to the image: vm_name-timestamp.raw. This is so to avoid collisions with
+an image with the same name on the remote pool (two PODs may be using
+the same pool). It may also be useful to keep around old file system
+images.
+
+FIXME: This patch requires a pool named "jenkins" in the remote libvirt
+server, and it will fail if it is not present. This should be
+configurable.
+
+Notice though that we can still define LIBVIRT_DEFAULT_URI as
+"qemu:///system" to create the Fuel VM on the local host.
+
+[1] https://libvirt.org/remote.html#Remote_URI_reference
+
+Change-Id: I40925ed31337d3ad9cf505f284f5c3d14e9129a0
+Signed-off-by: Josep Puigdemont <josep.puigdemont@enea.com>
+---
+ deploy/deploy.py                       |  5 +++
+ deploy/dha_adapters/libvirt_adapter.py | 31 +++++++++++++++++++
+ deploy/environments/virtual_fuel.py    | 56 ++++++++++++++++++++++++++++++++--
+ deploy/install_fuel_master.py          |  8 +++--
+ 4 files changed, 95 insertions(+), 5 deletions(-)
+
+diff --git a/deploy/deploy.py b/deploy/deploy.py
+index 742e76b..179ee7b 100755
+--- a/deploy/deploy.py
++++ b/deploy/deploy.py
+@@ -245,6 +245,11 @@ class AutoDeploy(object):
+ 
+ 
+ def check_bridge(pxe_bridge, dha_path):
++    # Assume that bridges on remote nodes exists, we could ssh but
++    # the remote user might not have a login shell.
++    if os.environ.get('LIBVIRT_DEFAULT_URI'):
++        return
++
+     with io.open(dha_path) as yaml_file:
+         dha_struct = yaml.load(yaml_file)
+     if dha_struct['adapter'] != 'libvirt':
+diff --git a/deploy/dha_adapters/libvirt_adapter.py b/deploy/dha_adapters/libvirt_adapter.py
+index 85913ac..466f134 100644
+--- a/deploy/dha_adapters/libvirt_adapter.py
++++ b/deploy/dha_adapters/libvirt_adapter.py
+@@ -11,6 +11,7 @@
+ from lxml import etree
+ from hardware_adapter import HardwareAdapter
+ import tempfile
++import os
+ 
+ from common import (
+     log,
+@@ -23,6 +24,15 @@ DEV = {'pxe': 'network',
+        'disk': 'hd',
+        'iso': 'cdrom'}
+ 
++VOL_XML_TEMPLATE = '''<volume type='file'>
++  <name>{name}</name>
++  <capacity unit='{unit}'>{size!s}</capacity>
++  <target>
++    <format type='{format_type}'/>
++  </target>
++</volume>'''
++
++DEFAULT_POOL = 'jenkins'
+ 
+ class LibvirtAdapter(HardwareAdapter):
+ 
+@@ -140,3 +150,24 @@ class LibvirtAdapter(HardwareAdapter):
+ 
+     def get_virt_net_conf_dir(self):
+         return self.dha_struct['virtNetConfDir']
++
++    def upload_iso(self, iso_file):
++        size = os.path.getsize(iso_file)
++        vol_name = os.path.basename(iso_file)
++        vol_xml = VOL_XML_TEMPLATE.format(name=vol_name, unit='bytes',
++                                          size=size, format_type='raw')
++        fd, fname = tempfile.mkstemp(text=True, suffix='deploy')
++        os.write(fd, vol_xml)
++        os.close(fd)
++
++        log(vol_xml)
++        pool = DEFAULT_POOL # FIXME
++        exec_cmd('virsh vol-create --pool %s %s' % (pool, fname))
++        vol_path = exec_cmd('virsh vol-path --pool %s %s' % (pool, vol_name))
++
++        exec_cmd('virsh vol-upload %s %s' % (vol_path, iso_file),
++                 attempts=5, delay=10, verbose=True)
++
++        delete(fname)
++
++        return vol_path
+diff --git a/deploy/environments/virtual_fuel.py b/deploy/environments/virtual_fuel.py
+index ac5fc53..f9f9f7a 100644
+--- a/deploy/environments/virtual_fuel.py
++++ b/deploy/environments/virtual_fuel.py
+@@ -11,14 +11,36 @@
+ from lxml import etree
+ from execution_environment import ExecutionEnvironment
+ import tempfile
++import os
++import re
++import time
+ 
+ from common import (
+     exec_cmd,
+     check_file_exists,
+     check_if_root,
+     delete,
++    log,
+ )
+ 
++VOL_XML_TEMPLATE = '''<volume type='file'>
++  <name>{name}</name>
++  <capacity unit='{unit}'>{size!s}</capacity>
++  <target>
++    <format type='{format_type}'/>
++  </target>
++</volume>'''
++
++DEFAULT_POOL = 'jenkins'
++
++def get_size_and_unit(s):
++    p = re.compile('^(\d+)\s*(\D+)')
++    m = p.match(s)
++    if m == None:
++        return None, None
++    size = m.groups()[0]
++    unit = m.groups()[1]
++    return size, unit
+ 
+ class VirtualFuel(ExecutionEnvironment):
+ 
+@@ -55,14 +77,42 @@ class VirtualFuel(ExecutionEnvironment):
+         with open(temp_vm_file, 'w') as f:
+             vm_xml.write(f, pretty_print=True, xml_declaration=True)
+ 
++    def create_volume(self, pool, name, su, img_type='qcow2'):
++        log('Creating image using Libvirt volumes in pool %s, name: %s' %
++            (pool, name))
++        size, unit = get_size_and_unit(su)
++        if size == None:
++            err('Could not determine size and unit of %s' % s)
++
++        vol_xml = VOL_XML_TEMPLATE.format(name=name, unit=unit, size=size,
++                                          format_type=img_type)
++        fname = os.path.join(self.temp_dir, '%s_vol.xml' % name)
++        with file(fname, 'w') as f:
++            f.write(vol_xml)
++
++        exec_cmd('virsh vol-create --pool %s %s' % (pool, fname))
++        vol_path = exec_cmd('virsh vol-path --pool %s %s' % (pool, name))
++
++        delete(fname)
++
++        return vol_path
++
+     def create_image(self, disk_path, disk_size):
+-        exec_cmd('qemu-img create -f qcow2 %s %s' % (disk_path, disk_size))
++        if os.environ.get('LIBVIRT_DEFAULT_URI') == None:
++            exec_cmd('qemu-img create -f qcow2 %s %s' % (disk_path, disk_size))
++        else:
++            pool = DEFAULT_POOL # FIXME
++            name = os.path.basename(disk_path)
++            disk_path = self.create_volume(pool, name, disk_size)
++
++        return disk_path
+ 
+     def create_vm(self):
+-        disk_path = '%s/%s.raw' % (self.storage_dir, self.vm_name)
++        stamp = time.strftime("%Y%m%d%H%M%S")
++        disk_path = '%s/%s-%s.raw' % (self.storage_dir, self.vm_name, stamp)
+         disk_sizes = self.dha.get_disks()
+         disk_size = disk_sizes['fuel']
+-        self.create_image(disk_path, disk_size)
++        disk_path = self.create_image(disk_path, disk_size)
+ 
+         temp_vm_file = '%s/%s' % (self.temp_dir, self.vm_name)
+         exec_cmd('cp %s %s' % (self.vm_template, temp_vm_file))
+diff --git a/deploy/install_fuel_master.py b/deploy/install_fuel_master.py
+index 631bf99..5adccef 100644
+--- a/deploy/install_fuel_master.py
++++ b/deploy/install_fuel_master.py
+@@ -54,8 +54,12 @@ class InstallFuelMaster(object):
+ 
+         self.dha.node_power_off(self.fuel_node_id)
+ 
+-        log('Zero the MBR')
+-        self.dha.node_zero_mbr(self.fuel_node_id)
++        if os.environ.get('LIBVIRT_DEFAULT_URI'):
++            log('Upload ISO to pool')
++            self.iso_file = self.dha.upload_iso(self.iso_file)
++        else:
++            log('Zero the MBR')
++            self.dha.node_zero_mbr(self.fuel_node_id)
+ 
+         self.dha.node_set_boot_order(self.fuel_node_id, ['disk', 'iso'])
+