From 0ef5c3265e8002e8b41d5b6f9c944bcdb09d7444 Mon Sep 17 00:00:00 2001 From: Florin Dumitrascu Date: Tue, 8 Mar 2016 14:17:01 +0100 Subject: Add fixes and improvements for arm64 deployment [ Florin Dumitrascu ] * arm64 support for OpenDaylight Fuel Plugin [ Stanislaw Kardach ] * Limit ESP to first drive * Remove default+timeout+tr from cobbler profile * Fix puppet syntax errors * Disable usb tablet on aarch64 * Support direct kernel boot for CirrOS TestVM on aarch64 * Remove git version signature not to confuse patches-export * Fix VGA support for CirrOS TestVM * Makefile: Add clean-{docker,build}. [ Alexandru Avadanii ] * Performance: Use gzip instead of xz compression. * Switch mirror proto from https to http. * Update TODO with remaining tasks. * Disable amd64 Liberty fixture (no multi-arch support yet). * m1.micro: Increase RAM size to 128MB for aarch64 images. Signed-off-by: Alexandru Avadanii Signed-off-by: Florin Dumitrascu Signed-off-by: Stanislaw Kardach (cherry picked from commit 80ba8c45cda5308010ded08b3d070343a92d58b5) Change-Id: I59823bc27b26e0749f27b6d39ed032847ca23fcd --- patches/fuel-web/0001-Add-arm64-fixture.patch | 11 +- ...-arch-to-nailgun-release-and-target-image.patch | 110 +++++++++++++--- .../0003-Update-docs-for-qemu-debootstrap.patch | 3 - .../0004-direct-kernel-boot-for-cirros.patch | 138 +++++++++++++++++++++ ...E-Disable-amd64-Liberty-on-Ubuntu-for-now.patch | 24 ++++ 5 files changed, 262 insertions(+), 24 deletions(-) create mode 100644 patches/fuel-web/0004-direct-kernel-boot-for-cirros.patch create mode 100644 patches/fuel-web/0005-FIXME-Disable-amd64-Liberty-on-Ubuntu-for-now.patch (limited to 'patches/fuel-web') diff --git a/patches/fuel-web/0001-Add-arm64-fixture.patch b/patches/fuel-web/0001-Add-arm64-fixture.patch index 58a64e49..95281c8c 100644 --- a/patches/fuel-web/0001-Add-arm64-fixture.patch +++ b/patches/fuel-web/0001-Add-arm64-fixture.patch @@ -7,7 +7,7 @@ Subject: [PATCH] Add arm64 fixture 1 file changed, 99 insertions(+) diff --git a/nailgun/nailgun/fixtures/openstack.yaml b/nailgun/nailgun/fixtures/openstack.yaml -index f9c8ab6..4daa2a2 100644 +index f9c8ab6..bf6f7c4 100644 --- a/nailgun/nailgun/fixtures/openstack.yaml +++ b/nailgun/nailgun/fixtures/openstack.yaml @@ -2079,3 +2079,102 @@ @@ -69,19 +69,19 @@ index f9c8ab6..4daa2a2 100644 + priority: 1050 + - type: "deb" + name: "mos-updates" -+ uri: "https://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}/" ++ uri: "http://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}" + suite: "mos8.0-updates" + section: "main restricted" + priority: 1050 + - type: "deb" + name: "mos-security" -+ uri: "https://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}/" ++ uri: "http://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}" + suite: "mos8.0-security" + section: "main restricted" + priority: 1050 + - type: "deb" + name: "mos-holdback" -+ uri: "https://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}/" ++ uri: "http://linux.enea.com/mos-repos/ubuntu/{cluster.release.environment_version}" + suite: "mos8.0-holdback" + section: "main restricted" + priority: 1100 @@ -113,6 +113,3 @@ index f9c8ab6..4daa2a2 100644 + uri: "http://{settings.MASTER_IP}:8080/targetimages/env_{cluster.id}_ubuntu_1404_arm64-boot.img.gz" + format: "ext2" + container: "gzip" --- -1.9.1 - diff --git a/patches/fuel-web/0002-Add-arch-to-nailgun-release-and-target-image.patch b/patches/fuel-web/0002-Add-arch-to-nailgun-release-and-target-image.patch index c5742f01..7af57961 100644 --- a/patches/fuel-web/0002-Add-arch-to-nailgun-release-and-target-image.patch +++ b/patches/fuel-web/0002-Add-arch-to-nailgun-release-and-target-image.patch @@ -6,15 +6,17 @@ Thsi is required so that the TestVM image is created using a cirros image that is compatible with the architecture of the deployment setup. As a bonus, it is also used when building the target image. --- - nailgun/nailgun/consts.py | 5 +++ - .../alembic_migrations/versions/armband.py | 47 ++++++++++++++++++++++ + nailgun/nailgun/consts.py | 5 ++ + .../alembic_migrations/versions/armband.py | 53 ++++++++++++++++++++++ nailgun/nailgun/db/sqlalchemy/models/release.py | 8 ++++ nailgun/nailgun/fixtures/openstack.yaml | 3 ++ - .../nailgun/orchestrator/deployment_serializers.py | 9 ++++- + .../nailgun/orchestrator/deployment_serializers.py | 9 +++- .../orchestrator/provisioning_serializers.py | 3 +- - nailgun/nailgun/orchestrator/tasks_templates.py | 5 ++- + nailgun/nailgun/orchestrator/tasks_templates.py | 5 +- + .../integration/test_cluster_changes_handler.py | 12 ++--- + .../integration/test_orchestrator_serializer.py | 4 +- nailgun/nailgun/test/unit/test_tasks_templates.py | 6 ++- - 8 files changed, 80 insertions(+), 6 deletions(-) + 10 files changed, 94 insertions(+), 14 deletions(-) create mode 100644 nailgun/nailgun/db/migration/alembic_migrations/versions/armband.py diff --git a/nailgun/nailgun/consts.py b/nailgun/nailgun/consts.py @@ -35,10 +37,10 @@ index bb69168..2c19ec9 100644 'ha_full', diff --git a/nailgun/nailgun/db/migration/alembic_migrations/versions/armband.py b/nailgun/nailgun/db/migration/alembic_migrations/versions/armband.py new file mode 100644 -index 0000000..8b832df +index 0000000..ad297b1 --- /dev/null +++ b/nailgun/nailgun/db/migration/alembic_migrations/versions/armband.py -@@ -0,0 +1,47 @@ +@@ -0,0 +1,53 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -66,23 +68,29 @@ index 0000000..8b832df +down_revision = '43b2cb64dae6' + +from alembic import op ++from nailgun.utils.migration import drop_enum +import sqlalchemy as sa -+from sqlalchemy.dialects import postgresql + +ENUMS = ( -+ 'release_arch') ++ 'release_arch', ++) ++ + +def upgrade(): + add_release_arch() + ++ +def downgrade(): + remove_release_arch() + map(drop_enum, ENUMS) + ++ +def add_release_arch(): + arch_enum = sa.Enum('amd64', 'arm64', name='release_arch') + arch_enum.create(op.get_bind(), checkfirst=False) -+ op.add_column('releases', sa.Column('arch', arch_enum, nullable=False)) ++ op.add_column('releases', sa.Column('arch', arch_enum, nullable=False, ++ server_default='amd64')) ++ + +def remove_release_arch(): + op.drop_column('releases', 'arch') @@ -106,7 +114,7 @@ index 96cf2ed..62ac2c1 100644 nullable=False, server_default='[]') description = Column(Unicode) diff --git a/nailgun/nailgun/fixtures/openstack.yaml b/nailgun/nailgun/fixtures/openstack.yaml -index 4daa2a2..733fa0c 100644 +index bf6f7c4..4979fb5 100644 --- a/nailgun/nailgun/fixtures/openstack.yaml +++ b/nailgun/nailgun/fixtures/openstack.yaml @@ -1913,6 +1913,7 @@ @@ -190,6 +198,83 @@ index 39f2067..cd2278c 100644 'timeout': settings.PROVISIONING_IMAGES_BUILD_TIMEOUT, 'retries': 1}}) +diff --git a/nailgun/nailgun/test/integration/test_cluster_changes_handler.py b/nailgun/nailgun/test/integration/test_cluster_changes_handler.py +index 56bd307..34b4ce3 100644 +--- a/nailgun/nailgun/test/integration/test_cluster_changes_handler.py ++++ b/nailgun/nailgun/test/integration/test_cluster_changes_handler.py +@@ -159,7 +159,7 @@ class TestHandlers(BaseIntegrationTest): + common_attrs['last_controller'] = controller_nodes[-1]['name'] + common_attrs['storage']['pg_num'] = 128 + +- common_attrs['test_vm_image'] = { ++ common_attrs['test_vm_image'] = [{ + 'container_format': 'bare', + 'public': 'true', + 'disk_format': 'qcow2', +@@ -171,7 +171,7 @@ class TestHandlers(BaseIntegrationTest): + """--property murano_image_info=""" + """'{"title": "Murano Demo", "type": "cirros.demo"}'""" + ), +- } ++ }] + + critical_mapping = { + 'primary-controller': True, +@@ -575,7 +575,7 @@ class TestHandlers(BaseIntegrationTest): + common_attrs['last_controller'] = controller_nodes[-1]['name'] + common_attrs['storage']['pg_num'] = 128 + +- common_attrs['test_vm_image'] = { ++ common_attrs['test_vm_image'] = [{ + 'container_format': 'bare', + 'public': 'true', + 'disk_format': 'qcow2', +@@ -587,7 +587,7 @@ class TestHandlers(BaseIntegrationTest): + """--property murano_image_info=""" + """'{"title": "Murano Demo", "type": "cirros.demo"}'""" + ), +- } ++ }] + + critical_mapping = { + 'primary-controller': True, +@@ -1062,7 +1062,7 @@ class TestHandlers(BaseIntegrationTest): + common_attrs['last_controller'] = controller_nodes[-1]['name'] + common_attrs['storage']['pg_num'] = 128 + +- common_attrs['test_vm_image'] = { ++ common_attrs['test_vm_image'] = [{ + 'container_format': 'bare', + 'public': 'true', + 'disk_format': 'qcow2', +@@ -1074,7 +1074,7 @@ class TestHandlers(BaseIntegrationTest): + """--property murano_image_info=""" + """'{"title": "Murano Demo", "type": "cirros.demo"}'""" + ), +- } ++ }] + + critical_mapping = { + 'primary-controller': True, +diff --git a/nailgun/nailgun/test/integration/test_orchestrator_serializer.py b/nailgun/nailgun/test/integration/test_orchestrator_serializer.py +index 8ce987a..e75b3c2 100644 +--- a/nailgun/nailgun/test/integration/test_orchestrator_serializer.py ++++ b/nailgun/nailgun/test/integration/test_orchestrator_serializer.py +@@ -2656,12 +2656,12 @@ class BaseDeploymentSerializer(base.BaseIntegrationTest): + + def check_no_murano_data(self): + glance_properties = self.serializer.generate_test_vm_image_data( +- self.env.nodes[0])['test_vm_image']['glance_properties'] ++ self.env.nodes[0])['test_vm_image'][0]['glance_properties'] + self.assertNotIn('murano_image_info', glance_properties) + + def check_murano_data(self): + glance_properties = self.serializer.generate_test_vm_image_data( +- self.env.nodes[0])['test_vm_image']['glance_properties'] ++ self.env.nodes[0])['test_vm_image'][0]['glance_properties'] + self.assertIn('murano_image_info', glance_properties) + + diff --git a/nailgun/nailgun/test/unit/test_tasks_templates.py b/nailgun/nailgun/test/unit/test_tasks_templates.py index a38e975..9cb9171 100644 --- a/nailgun/nailgun/test/unit/test_tasks_templates.py @@ -214,6 +299,3 @@ index a38e975..9cb9171 100644 self.assertEqual(jsonutils.loads(cmd), fuel_image_conf) def test_generate_ironic_bootstrap_keys_task(self): --- -1.9.1 - diff --git a/patches/fuel-web/0003-Update-docs-for-qemu-debootstrap.patch b/patches/fuel-web/0003-Update-docs-for-qemu-debootstrap.patch index b6ab7fe7..5a975fb7 100644 --- a/patches/fuel-web/0003-Update-docs-for-qemu-debootstrap.patch +++ b/patches/fuel-web/0003-Update-docs-for-qemu-debootstrap.patch @@ -19,6 +19,3 @@ index 227549c..47b9823 100644 sudo gem install bundler -v 1.2.1 sudo gem install builder sudo pip install xmlbuilder jinja2 --- -1.9.1 - diff --git a/patches/fuel-web/0004-direct-kernel-boot-for-cirros.patch b/patches/fuel-web/0004-direct-kernel-boot-for-cirros.patch new file mode 100644 index 00000000..0d133274 --- /dev/null +++ b/patches/fuel-web/0004-direct-kernel-boot-for-cirros.patch @@ -0,0 +1,138 @@ +From: Stanislaw Kardach +Date: Tue, 15 Mar 2016 15:01:51 +0100 +Subject: [PATCH] direct kernel boot for cirros + +--- + nailgun/nailgun/orchestrator/base_serializers.py | 16 +++-- + .../nailgun/orchestrator/deployment_serializers.py | 70 ++++++++++++++++------ + 2 files changed, 62 insertions(+), 24 deletions(-) + +diff --git a/nailgun/nailgun/orchestrator/base_serializers.py b/nailgun/nailgun/orchestrator/base_serializers.py +index 16dc3fa..23d6589 100644 +--- a/nailgun/nailgun/orchestrator/base_serializers.py ++++ b/nailgun/nailgun/orchestrator/base_serializers.py +@@ -36,12 +36,16 @@ class MuranoMetadataSerializerMixin(object): + self).generate_test_vm_image_data(node) + + # Add default Glance property for Murano. +- test_vm_image = image_data['test_vm_image'] +- existing_properties = test_vm_image['glance_properties'] +- murano_data = ' '.join(["""--property murano_image_info='{"title":""" +- """ "Murano Demo", "type": "cirros.demo"}'"""]) +- test_vm_image['glance_properties'] = existing_properties + murano_data +- return {'test_vm_image': test_vm_image} ++ test_vm_images = [] ++ for test_vm_image in image_data['test_vm_image']: ++ existing_properties = test_vm_image['glance_properties'] ++ murano_data = ' '.join( ++ ["""--property murano_image_info='{"title":""" ++ """ "Murano Demo", "type": "cirros.demo"}'"""]) ++ test_vm_image['glance_properties'] = existing_properties + \ ++ murano_data ++ test_vm_images += [test_vm_image] ++ return {'test_vm_image': test_vm_images} + + + class VmwareDeploymentSerializerMixin(object): +diff --git a/nailgun/nailgun/orchestrator/deployment_serializers.py b/nailgun/nailgun/orchestrator/deployment_serializers.py +index b8b38e2..54b1614 100644 +--- a/nailgun/nailgun/orchestrator/deployment_serializers.py ++++ b/nailgun/nailgun/orchestrator/deployment_serializers.py +@@ -274,6 +274,9 @@ class DeploymentMultinodeSerializer(object): + return {'glance': {'image_cache_max_size': image_cache_max_size}} + + def generate_test_vm_image_data(self, node): ++ images = [] ++ # By default always prepare the disk image ++ + # Instantiate all default values in dict. + image_data = { + 'container_format': 'bare', +@@ -297,8 +300,8 @@ class DeploymentMultinodeSerializer(object): + arch = "x86_64" + elif arch == "arm64": + arch = "aarch64" +- image_data['img_path'] = '{0}cirros-{1}-disk.img'.format(img_dir, +- arch) ++ image_data['img_path'] = \ ++ '{0}cirros-{1}-disk.img'.format(img_dir, arch) + + glance_properties = [] + +@@ -313,8 +316,38 @@ class DeploymentMultinodeSerializer(object): + glance_properties.append('--property hypervisor_type=vmware') + + image_data['glance_properties'] = ' '.join(glance_properties) +- +- return {'test_vm_image': image_data} ++ images += [image_data] ++ ++ # Now for aarch64 add also kernel and initrd because we direct ++ # kernel boot is used then ++ if arch == 'aarch64': ++ image_data['glance_properties'] += \ ++ ' --property os_command_line=\'root=/dev/vda1 rw rootwait' + \ ++ 'console=tty0 console=ttyS0 console=ttyAMA0\'' + \ ++ ' --property hw_video_model=vga' ++ kernel_data = { ++ 'container_format': 'aki', ++ 'public': 'true', ++ 'disk_format': 'aki', ++ 'img_name': 'TestVM-kernel', ++ 'img_path': '{0}cirros-{1}-kernel'.format(img_dir, arch), ++ 'os_name': 'cirros', ++ 'min_ram': 64, ++ 'glance_properties': '', ++ } ++ initrd_data = { ++ 'container_format': 'ari', ++ 'public': 'true', ++ 'disk_format': 'ari', ++ 'img_name': 'TestVM-initramfs', ++ 'img_path': '{0}cirros-{1}-initramfs'.format(img_dir, arch), ++ 'os_name': 'cirros', ++ 'min_ram': 64, ++ 'glance_properties': '', ++ } ++ images += [kernel_data, initrd_data] ++ ++ return {'test_vm_image': images} + + @classmethod + def get_net_provider_serializer(cls, cluster): +@@ -495,20 +528,21 @@ class DeploymentHASerializer61(DeploymentHASerializer, + images_data['test_vm_image'] = [] + if attrs.get('editable', {}).get('common', {}). \ + get('use_vcenter', {}).get('value') is True: +- image_vmdk_data = deepcopy(image_data['test_vm_image']) +- img_path = image_vmdk_data['img_path']. \ +- replace('x86_64-disk.img', 'i386-disk.vmdk') +- image_vmdk_data.update({ +- 'img_name': 'TestVM-VMDK', +- 'disk_format': 'vmdk', +- 'img_path': img_path, +- }) +- image_vmdk_data['glance_properties'] = ' '.join([ +- '--property vmware_disktype=sparse', +- '--property vmware_adaptertype=lsiLogic', +- '--property hypervisor_type=vmware']) +- images_data['test_vm_image'].append(image_vmdk_data) +- images_data['test_vm_image'].append(image_data['test_vm_image']) ++ images_vmdk_data = deepcopy(image_data['test_vm_image']) ++ for image_vmdk_data in images_vmdk_data: ++ img_path = image_vmdk_data['img_path']. \ ++ replace('x86_64-disk.img', 'i386-disk.vmdk') ++ image_vmdk_data.update({ ++ 'img_name': 'TestVM-VMDK', ++ 'disk_format': 'vmdk', ++ 'img_path': img_path, ++ }) ++ image_vmdk_data['glance_properties'] = ' '.join([ ++ '--property vmware_disktype=sparse', ++ '--property vmware_adaptertype=lsiLogic', ++ '--property hypervisor_type=vmware']) ++ images_data['test_vm_image'] += [image_vmdk_data] ++ images_data['test_vm_image'] += image_data['test_vm_image'] + else: + images_data = image_data + diff --git a/patches/fuel-web/0005-FIXME-Disable-amd64-Liberty-on-Ubuntu-for-now.patch b/patches/fuel-web/0005-FIXME-Disable-amd64-Liberty-on-Ubuntu-for-now.patch new file mode 100644 index 00000000..243d9c85 --- /dev/null +++ b/patches/fuel-web/0005-FIXME-Disable-amd64-Liberty-on-Ubuntu-for-now.patch @@ -0,0 +1,24 @@ +From: Alexandru Avadanii +Date: Sat, 19 Mar 2016 19:02:15 +0100 +Subject: [PATCH] FIXME: Disable amd64 Liberty on Ubuntu for now. + +Until we fix all cohabitation of amd64 and arm64, disable Openstack +release Liberty on Ubuntu (x86_64) in openstack.yaml fixture. + +TODO: maybe rename above fixture to include arch in name. +--- + nailgun/nailgun/fixtures/openstack.yaml | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/nailgun/nailgun/fixtures/openstack.yaml b/nailgun/nailgun/fixtures/openstack.yaml +index 4979fb5..037ceba 100644 +--- a/nailgun/nailgun/fixtures/openstack.yaml ++++ b/nailgun/nailgun/fixtures/openstack.yaml +@@ -1986,6 +1986,7 @@ + extend: *base_release + fields: + name: "Liberty on Ubuntu 14.04" ++ state: "unavailable" + version: "liberty-8.0" + arch: "amd64" + can_update_from_versions: [] -- cgit 1.2.3-korg