Add support for extra properties in an image

This patch adds a new setting for images, extra_properties.
This field is a dict and can be used to create a 3-part image
by setting the kernel_id and the ramdisk_id of the main image.

Unit tests have been added to set some generic property, for
creation of the 3-part image, and for creating an instance
with that 3-part image

JIRA: SNAPS-32

Change-Id: Ifb53d1da1085fcd6429ddc0607c905522db5e8bb
Signed-off-by: Cristina Pauna <cristina.pauna@enea.com>

5 files changed, 330 insertions, 16 deletions

diff --git a/snaps/openstack/create_image.py b/snaps/openstack/create_image.py
index e1b8d94..bffa7de 100644
--- a/snaps/openstack/create_image.py
+++ b/snaps/openstack/create_image.py
@@ -150,7 +150,7 @@ class OpenStackImage:
 
 class ImageSettings:
     def __init__(self, config=None, name=None, image_user=None, img_format=None, url=None, image_file=None,
-                 nic_config_pb_loc=None):
+                 extra_properties=None, nic_config_pb_loc=None):
         """
 
         :param config: dict() object containing the configuration settings using the attribute names below as each
@@ -160,6 +160,8 @@ class ImageSettings:
         :param img_format: the image type (required)
         :param url: the image download location (requires url or img_file)
         :param image_file: the image file location (requires url or img_file)
+        :param extra_properties: dict() object containing extra parameters to pass when loading the image;
+                                 can be ids of kernel and initramfs images for a 3-part image
         :param nic_config_pb_loc: the file location to the Ansible Playbook that can configure multiple NICs
         """
 
@@ -169,6 +171,7 @@ class ImageSettings:
             self.format = config.get('format')
             self.url = config.get('download_url')
             self.image_file = config.get('image_file')
+            self.extra_properties = config.get('extra_properties')
             self.nic_config_pb_loc = config.get('nic_config_pb_loc')
         else:
             self.name = name
@@ -176,6 +179,7 @@ class ImageSettings:
             self.format = img_format
             self.url = url
             self.image_file = image_file
+            self.extra_properties = extra_properties
             self.nic_config_pb_loc = nic_config_pb_loc
 
         if not self.name or not self.image_user or not self.format:
diff --git a/snaps/openstack/tests/create_image_tests.py b/snaps/openstack/tests/create_image_tests.py
index 24bf0f2..753e83f 100644
--- a/snaps/openstack/tests/create_image_tests.py
+++ b/snaps/openstack/tests/create_image_tests.py
@@ -85,6 +85,19 @@ class ImageSettingsUnitTests(unittest.TestCase):
         self.assertIsNone(settings.image_file)
         self.assertIsNone(settings.nic_config_pb_loc)
 
+    def test_name_user_format_url_only_properties(self):
+        properties = {}
+        properties['hw_video_model'] = 'vga'
+        settings = ImageSettings(name='foo', image_user='bar', img_format='qcow2', url='http://foo.com', extra_properties=properties)
+        self.assertEquals('foo', settings.name)
+        self.assertEquals('bar', settings.image_user)
+        self.assertEquals('qcow2', settings.format)
+        self.assertEquals('http://foo.com', settings.url)
+        self.assertEquals(properties, settings.extra_properties)
+        self.assertIsNone(settings.image_file)
+        self.assertIsNone(settings.nic_config_pb_loc)
+
+
     def test_config_with_name_user_format_url_only(self):
         settings = ImageSettings(config={'name': 'foo', 'image_user': 'bar', 'format': 'qcow2',
                                          'download_url': 'http://foo.com'})
@@ -115,43 +128,55 @@ class ImageSettingsUnitTests(unittest.TestCase):
         self.assertIsNone(settings.nic_config_pb_loc)
 
     def test_all_url(self):
+        properties = {}
+        properties['hw_video_model'] = 'vga'
         settings = ImageSettings(name='foo', image_user='bar', img_format='qcow2', url='http://foo.com',
-                                 nic_config_pb_loc='/foo/bar')
+                                 extra_properties=properties, nic_config_pb_loc='/foo/bar')
         self.assertEquals('foo', settings.name)
         self.assertEquals('bar', settings.image_user)
         self.assertEquals('qcow2', settings.format)
         self.assertEquals('http://foo.com', settings.url)
+        self.assertEquals(properties, settings.extra_properties)
         self.assertIsNone(settings.image_file)
         self.assertEquals('/foo/bar', settings.nic_config_pb_loc)
 
     def test_config_all_url(self):
         settings = ImageSettings(config={'name': 'foo', 'image_user': 'bar', 'format': 'qcow2',
-                                         'download_url': 'http://foo.com', 'nic_config_pb_loc': '/foo/bar'})
+                                         'download_url': 'http://foo.com',
+                                         'extra_properties' : '{\'hw_video_model\': \'vga\'}',
+                                         'nic_config_pb_loc': '/foo/bar'})
         self.assertEquals('foo', settings.name)
         self.assertEquals('bar', settings.image_user)
         self.assertEquals('qcow2', settings.format)
         self.assertEquals('http://foo.com', settings.url)
+        self.assertEquals('{\'hw_video_model\': \'vga\'}', settings.extra_properties)
         self.assertIsNone(settings.image_file)
         self.assertEquals('/foo/bar', settings.nic_config_pb_loc)
 
     def test_all_file(self):
+        properties = {}
+        properties['hw_video_model'] = 'vga'
         settings = ImageSettings(name='foo', image_user='bar', img_format='qcow2', image_file='/foo/bar.qcow',
-                                 nic_config_pb_loc='/foo/bar')
+                                 extra_properties=properties, nic_config_pb_loc='/foo/bar')
         self.assertEquals('foo', settings.name)
         self.assertEquals('bar', settings.image_user)
         self.assertEquals('qcow2', settings.format)
         self.assertIsNone(settings.url)
         self.assertEquals('/foo/bar.qcow', settings.image_file)
+        self.assertEquals(properties, settings.extra_properties)
         self.assertEquals('/foo/bar', settings.nic_config_pb_loc)
 
     def test_config_all_file(self):
         settings = ImageSettings(config={'name': 'foo', 'image_user': 'bar', 'format': 'qcow2',
-                                         'image_file': '/foo/bar.qcow', 'nic_config_pb_loc': '/foo/bar'})
+                                         'image_file': '/foo/bar.qcow',
+                                         'extra_properties' : '{\'hw_video_model\' : \'vga\'}',
+                                         'nic_config_pb_loc': '/foo/bar'})
         self.assertEquals('foo', settings.name)
         self.assertEquals('bar', settings.image_user)
         self.assertEquals('qcow2', settings.format)
         self.assertIsNone(settings.url)
         self.assertEquals('/foo/bar.qcow', settings.image_file)
+        self.assertEquals('{\'hw_video_model\' : \'vga\'}', settings.extra_properties)
         self.assertEquals('/foo/bar', settings.nic_config_pb_loc)
 
 
@@ -206,6 +231,29 @@ class CreateImageSuccessTests(OSIntegrationTestCase):
         self.assertEquals(created_image.name, retrieved_image.name)
         self.assertEquals(created_image.id, retrieved_image.id)
 
+    def test_create_image_clean_url_properties(self):
+        """
+        Tests the creation of an OpenStack image from a URL and set properties.
+        """
+        # Set properties
+        properties = {}
+        properties['hw_video_model'] = 'vga'
+
+        # Create Image
+        os_image_settings = openstack_tests.cirros_url_image(name=self.image_name)
+        os_image_settings.extra_properties = properties
+        self.image_creator = create_image.OpenStackImage(self.os_creds, os_image_settings)
+
+        created_image = self.image_creator.create()
+        self.assertIsNotNone(created_image)
+
+        retrieved_image = glance_utils.get_image(self.nova, self.glance, os_image_settings.name)
+        self.assertIsNotNone(retrieved_image)
+
+        self.assertEquals(created_image.name, retrieved_image.name)
+        self.assertEquals(created_image.id, retrieved_image.id)
+        self.assertEquals(created_image.properties, retrieved_image.properties)
+
     def test_create_image_clean_file(self):
         """
         Tests the creation of an OpenStack image from a file.
@@ -360,3 +408,126 @@ class CreateImageNegativeTests(OSIntegrationTestCase):
                                        self.os_creds.project_name,
                                        proxy_settings=self.os_creds.proxy_settings),
                 os_image_settings)
+
+
+class CreateMultiPartImageTests(OSIntegrationTestCase):
+    """
+    Test for creating a 3-part image
+    """
+    def setUp(self):
+        """
+        Instantiates the CreateImage object that is responsible for
+        downloading and creating an OS image file within OpenStack
+        """
+        super(self.__class__, self).__start__()
+
+        guid = uuid.uuid4()
+        self.image_creators = list()
+        self.image_name = self.__class__.__name__ + '-' + str(guid)
+
+        self.nova = nova_utils.nova_client(self.os_creds)
+        self.glance = glance_utils.glance_client(self.os_creds)
+
+        self.tmp_dir = 'tmp/' + str(guid)
+        if not os.path.exists(self.tmp_dir):
+            os.makedirs(self.tmp_dir)
+
+    def tearDown(self):
+        """
+        Cleans the images and downloaded image file
+        """
+        while self.image_creators:
+            self.image_creators[0].clean()
+            self.image_creators.pop(0)
+
+        if os.path.exists(self.tmp_dir) and os.path.isdir(self.tmp_dir):
+            shutil.rmtree(self.tmp_dir)
+
+        super(self.__class__, self).__clean__()
+
+    def test_create_three_part_image_from_url(self):
+        """
+        Tests the creation of a 3-part OpenStack image from a URL.
+        """
+        # Set properties
+        properties = {}
+
+        # Create the kernel image
+        kernel_image_settings = openstack_tests.cirros_url_image(name=self.image_name+'_kernel',
+            url='http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-kernel')
+        self.image_creators.append(create_image.OpenStackImage(self.os_creds, kernel_image_settings))
+        kernel_image = self.image_creators[-1].create()
+        self.assertIsNotNone(kernel_image)
+
+        # Create the ramdisk image
+        ramdisk_image_settings = openstack_tests.cirros_url_image(name=self.image_name+'_ramdisk',
+            url='http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-initramfs')
+        self.image_creators.append(create_image.OpenStackImage(self.os_creds, ramdisk_image_settings))
+        ramdisk_image = self.image_creators[-1].create()
+        self.assertIsNotNone(ramdisk_image)
+
+        # Create the main image
+        os_image_settings = openstack_tests.cirros_url_image(name=self.image_name,
+            url='http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img')
+        properties['kernel_id'] = kernel_image.id
+        properties['ramdisk_id'] = ramdisk_image.id
+        os_image_settings.extra_properties = properties
+        self.image_creators.append(create_image.OpenStackImage(self.os_creds, os_image_settings))
+        created_image = self.image_creators[-1].create()
+        self.assertIsNotNone(created_image)
+        self.assertEqual(self.image_name, created_image.name)
+
+        retrieved_image = glance_utils.get_image(self.nova, self.glance, os_image_settings.name)
+        self.assertIsNotNone(retrieved_image)
+
+        self.assertEquals(created_image.name, retrieved_image.name)
+        self.assertEquals(created_image.id, retrieved_image.id)
+        self.assertEquals(created_image.properties, retrieved_image.properties)
+
+    def test_create_three_part_image_from_file(self):
+        """
+        Tests the creation of a 3-part OpenStack image from files.
+        """
+        # Set properties
+        properties = {}
+
+        # Create the kernel image
+        url_image_settings = openstack_tests.cirros_url_image('foo_kernel',
+            url='http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-kernel')
+        kernel_image_file = file_utils.download(url_image_settings.url, self.tmp_dir)
+        kernel_file_image_settings = openstack_tests.file_image_test_settings(
+            name=self.image_name+'_kernel', file_path=kernel_image_file.name)
+        self.image_creators.append(create_image.OpenStackImage(self.os_creds, kernel_file_image_settings))
+        kernel_image = self.image_creators[-1].create()
+        self.assertIsNotNone(kernel_image)
+
+        # Create the ramdisk image
+        url_image_settings = openstack_tests.cirros_url_image('foo_ramdisk',
+            url='http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-initramfs')
+        ramdisk_image_file = file_utils.download(url_image_settings.url, self.tmp_dir)
+        ramdisk_file_image_settings = openstack_tests.file_image_test_settings(
+            name=self.image_name+'_ramdisk', file_path=ramdisk_image_file.name)
+        self.image_creators.append(create_image.OpenStackImage(self.os_creds, ramdisk_file_image_settings))
+        ramdisk_image = self.image_creators[-1].create()
+        self.assertIsNotNone(ramdisk_image)
+
+        # Create the main image
+        url_image_settings = openstack_tests.cirros_url_image('foo',
+            url='http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img')
+        image_file = file_utils.download(url_image_settings.url, self.tmp_dir)
+        file_image_settings = openstack_tests.file_image_test_settings(name=self.image_name, file_path=image_file.name)
+        properties['kernel_id'] = kernel_image.id
+        properties['ramdisk_id'] = ramdisk_image.id
+        file_image_settings.extra_properties = properties
+        self.image_creators.append(create_image.OpenStackImage(self.os_creds, file_image_settings))
+        created_image = self.image_creators[-1].create()
+
+        self.assertIsNotNone(created_image)
+        self.assertEqual(self.image_name, created_image.name)
+
+        retrieved_image = glance_utils.get_image(self.nova, self.glance, file_image_settings.name)
+        self.assertIsNotNone(retrieved_image)
+
+        self.assertEquals(created_image.name, retrieved_image.name)
+        self.assertEquals(created_image.id, retrieved_image.id)
+        self.assertEquals(created_image.properties, retrieved_image.properties)
diff --git a/snaps/openstack/tests/create_instance_tests.py b/snaps/openstack/tests/create_instance_tests.py
index 756b45f..d733547 100644
--- a/snaps/openstack/tests/create_instance_tests.py
+++ b/snaps/openstack/tests/create_instance_tests.py
@@ -1472,3 +1472,128 @@ def validate_ssh_client(instance_creator):
         return True
 
     return False
+
+
+class CreateInstanceFromThreePartImage(OSIntegrationTestCase):
+    """
+    Test for the CreateInstance class for creating an image from a 3-part image
+    """
+
+    def setUp(self):
+        """
+        Instantiates the CreateImage object that is responsible for downloading and creating an OS image file
+        within OpenStack
+        """
+        super(self.__class__, self).__start__()
+
+        guid = self.__class__.__name__ + '-' + str(uuid.uuid4())
+        self.image_name = guid
+        self.vm_inst_name = guid + '-inst'
+        self.nova = nova_utils.nova_client(self.os_creds)
+
+        net_config = openstack_tests.get_priv_net_config(
+            net_name=guid + '-pub-net', subnet_name=guid + '-pub-subnet',
+            router_name=guid + '-pub-router', external_net=self.ext_net_name)
+
+        # Initialize for tearDown()
+        self.image_creators = list()
+        self.network_creator = None
+        self.flavor_creator = None
+        self.inst_creator = None
+
+        try:
+            # Create Images
+            # Create the kernel image
+            kernel_image_settings = openstack_tests.cirros_url_image(name=self.image_name+'_kernel',
+                url='http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-kernel')
+            self.image_creators.append(OpenStackImage(self.os_creds, kernel_image_settings))
+            kernel_image = self.image_creators[-1].create()
+
+            # Create the ramdisk image
+            ramdisk_image_settings = openstack_tests.cirros_url_image(name=self.image_name+'_ramdisk',
+                url='http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-initramfs')
+            self.image_creators.append(OpenStackImage(self.os_creds, ramdisk_image_settings))
+            ramdisk_image = self.image_creators[-1].create()
+            self.assertIsNotNone(ramdisk_image)
+
+            # Create the main image
+            os_image_settings = openstack_tests.cirros_url_image(name=self.image_name,
+                url='http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img')
+            properties = {}
+            properties['kernel_id'] = kernel_image.id
+            properties['ramdisk_id'] = ramdisk_image.id
+            os_image_settings.extra_properties = properties
+            self.image_creators.append(OpenStackImage(self.os_creds, os_image_settings))
+            created_image = self.image_creators[-1].create()
+
+            # Create Flavor
+            self.flavor_creator = OpenStackFlavor(
+                self.admin_os_creds,
+                FlavorSettings(name=guid + '-flavor-name', ram=2048, disk=10, vcpus=2))
+            self.flavor_creator.create()
+
+            # Create Network
+            self.network_creator = OpenStackNetwork(self.os_creds, net_config.network_settings)
+            self.network_creator.create()
+
+            self.port_settings = PortSettings(name=guid + '-port',
+                                              network_name=net_config.network_settings.name)
+        except Exception as e:
+            self.tearDown()
+            raise e
+
+    def tearDown(self):
+        """
+        Cleans the created object
+        """
+        if self.inst_creator:
+            try:
+                self.inst_creator.clean()
+            except Exception as e:
+                logger.error('Unexpected exception cleaning VM instance with message - ' + e.message)
+
+        if self.flavor_creator:
+            try:
+                self.flavor_creator.clean()
+            except Exception as e:
+                logger.error('Unexpected exception cleaning flavor with message - ' + e.message)
+
+        if self.network_creator:
+            try:
+                self.network_creator.clean()
+            except Exception as e:
+                logger.error('Unexpected exception cleaning network with message - ' + e.message)
+
+        if self.image_creators:
+            try:
+                while self.image_creators:
+                    self.image_creators[0].clean()
+                    self.image_creators.pop(0)
+            except Exception as e:
+                logger.error('Unexpected exception cleaning image with message - ' + e.message)
+
+        super(self.__class__, self).__clean__()
+
+    def test_create_delete_instance_from_three_part_image(self):
+        """
+        Tests the creation of an OpenStack instance from a 3-part image.
+        """
+        instance_settings = VmInstanceSettings(name=self.vm_inst_name, flavor=self.flavor_creator.flavor_settings.name,
+                                               port_settings=[self.port_settings])
+
+        # The last created image is the main image from which we create the instance
+        self.inst_creator = OpenStackVmInstance(
+            self.os_creds, instance_settings, self.image_creators[-1].image_settings)
+
+        vm_inst = self.inst_creator.create()
+        self.assertEquals(1, len(nova_utils.get_servers_by_name(self.nova, instance_settings.name)))
+
+        # Delete instance
+        nova_utils.delete_vm_instance(self.nova, vm_inst)
+
+        self.assertTrue(self.inst_creator.vm_deleted(block=True))
+        self.assertEquals(0, len(nova_utils.get_servers_by_name(self.nova, instance_settings.name)))
+
+        # Exception should not be thrown
+        self.inst_creator.clean()
+
diff --git a/snaps/openstack/tests/openstack_tests.py b/snaps/openstack/tests/openstack_tests.py
index dab2ea2..a1b7881 100644
--- a/snaps/openstack/tests/openstack_tests.py
+++ b/snaps/openstack/tests/openstack_tests.py
@@ -92,9 +92,10 @@ def get_credentials(os_env_file=None, proxy_settings_str=None, ssh_proxy_cmd=Non
     return os_creds
 
 
-def cirros_url_image(name):
-    return ImageSettings(name=name, image_user='cirros', img_format='qcow2',
-                         url='http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img')
+def cirros_url_image(name, url=None):
+    if not url:
+        url='http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img'
+    return ImageSettings(name=name, image_user='cirros', img_format='qcow2', url=url)
 
 
 def file_image_test_settings(name, file_path):
@@ -102,16 +103,17 @@ def file_image_test_settings(name, file_path):
                          image_file=file_path)
 
 
-def centos_url_image(name):
-    return ImageSettings(name=name, image_user='centos', img_format='qcow2',
-                         url='http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2',
-                         nic_config_pb_loc='./provisioning/ansible/centos-network-setup/playbooks/configure_host.yml')
+def centos_url_image(name, url=None):
+    if not url:
+        url='http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2'
+    return ImageSettings(name=name, image_user='centos', img_format='qcow2', url=url,
+        nic_config_pb_loc='./provisioning/ansible/centos-network-setup/playbooks/configure_host.yml')
 
 
-def ubuntu_url_image(name):
-    return ImageSettings(
-        name=name, image_user='ubuntu', img_format='qcow2',
-        url='http://uec-images.ubuntu.com/releases/trusty/14.04/ubuntu-14.04-server-cloudimg-amd64-disk1.img',
+def ubuntu_url_image(name, url=None):
+    if not url:
+        url='http://uec-images.ubuntu.com/releases/trusty/14.04/ubuntu-14.04-server-cloudimg-amd64-disk1.img'
+    return ImageSettings(name=name, image_user='ubuntu', img_format='qcow2', url=url,
         nic_config_pb_loc='./provisioning/ansible/ubuntu-network-setup/playbooks/configure_host.yml')
 
 
diff --git a/snaps/openstack/utils/glance_utils.py b/snaps/openstack/utils/glance_utils.py
index 6d90d3e..3be2a47 100644
--- a/snaps/openstack/utils/glance_utils.py
+++ b/snaps/openstack/utils/glance_utils.py
@@ -61,10 +61,22 @@ def create_image(glance, image_settings):
     :raise Exception if using a file and it cannot be found
     """
     if image_settings.url:
+        if image_settings.extra_properties:
+            return glance.images.create(name=image_settings.name,
+                                        disk_format=image_settings.format,
+                                        container_format="bare",
+                                        location=image_settings.url,
+                                        properties=image_settings.extra_properties)
         return glance.images.create(name=image_settings.name, disk_format=image_settings.format,
                                     container_format="bare", location=image_settings.url)
     elif image_settings.image_file:
         image_file = file_utils.get_file(image_settings.image_file)
+        if image_settings.extra_properties:
+            return glance.images.create(name=image_settings.name,
+                                        disk_format=image_settings.format,
+                                        container_format="bare",
+                                        data=image_file,
+                                        properties=image_settings.extra_properties)
         return glance.images.create(name=image_settings.name, disk_format=image_settings.format,
                                     container_format="bare", data=image_file)