From 23d35dc2c56b8c2b5496b6f0a5fc62066b22bbc7 Mon Sep 17 00:00:00 2001 From: Sawyer Bergeron Date: Fri, 29 Oct 2021 15:11:29 -0400 Subject: Add Cloud Init Support Squashed commit of the following: commit afcee3cad5c091e78e909b83f8df49accf1af5b6 Author: Sawyer Bergeron Date: Mon Oct 11 22:02:16 2021 +0000 Prod cobbler hotfixes Signed-off-by: Sawyer Bergeron Change-Id: I092bc6d85a3b2c77bfbe24f3af0d2b7a5f75a8c3 commit 5ce0a52b17e530436c298e1b581d37bac853f5a7 Author: Sawyer Bergeron Date: Thu Oct 7 17:14:01 2021 -0400 Manually merge CI files Signed-off-by: Sawyer Bergeron Change-Id: Ic63d5da699578007ef2f2cc373350ded06c66971 commit 5b70b8f1b8bbbe6aeec43b8d8dfdc6b7cc68bc9c Author: Sawyer Bergeron Date: Thu Sep 30 16:33:01 2021 -0400 Fixes for collaborator field Signed-off-by: Sawyer Bergeron Change-Id: I3dbdedf26fa84617ea7680a0f99e032d88f1ea98 Signed-off-by: Sawyer Bergeron commit 529b2521627b17142284c55c744812129edc71e8 Merge: d555513 e9d72ce Author: Sawyer Bergeron Date: Thu Sep 30 14:03:55 2021 +0000 Merge "Push cloud config content for generated files into userdata_raw" into cobbler commit d55551394df73645e49ae2ae3e730a9f1c6af81d Author: Sawyer Bergeron Date: Thu Sep 30 10:02:32 2021 -0400 Better error handling for quick deploy Change-Id: I03a725dfee9ce2f119d72ef940cd08df5aee3dcc Signed-off-by: Sawyer Bergeron commit e9d72ce78a85c6ff2f3f8591bcbf4115f97318d5 Author: Sawyer Bergeron Date: Tue Sep 28 19:11:49 2021 -0400 Push cloud config content for generated files into userdata_raw Signed-off-by: Sawyer Bergeron Change-Id: Ieb8bd9b8b172b6bf11062f67f41fc78154cc7c89 commit 95d39c60f7e8062cabc8c1665080a2d2c8904234 Author: Sawyer Bergeron Date: Sat Sep 25 16:18:12 2021 -0400 Allow for "pod specific" vlan allocation for LFEDGE allocation case Signed-off-by: Sawyer Bergeron Change-Id: I8b75410145027f43eaf6de7bd5f1813af38d3e7f Signed-off-by: Sawyer Bergeron commit 2ebb82b5f344de1e17abd70c51c4cce765761dd1 Author: Sawyer Bergeron Date: Thu Sep 23 16:37:43 2021 -0400 Fix collaborator field with recent changes Signed-off-by: Sawyer Bergeron Change-Id: Id305de9b1567adf103c47d5180b0b28ebfdf1b5e commit a819fc1df86721eda36eee89d0235c89b3159d6b Author: Sawyer Bergeron Date: Tue Sep 7 11:28:35 2021 -0400 Add user specified CI file entry Signed-off-by: Sawyer Bergeron Change-Id: Ia920130612da8fcde9d1a0d5dde7861904857162 Signed-off-by: Sawyer Bergeron commit d93346a716bde5237b7cfef5c10ea56e4922b59a Author: Adam Hassick Date: Tue Jul 27 13:05:16 2021 +0000 Make C-I serialization work with current netconf rules Signed-off-by: Sawyer Bergeron Change-Id: If967e5e1f268c5bee3ad4496847662cf4de1187c Signed-off-by: Sawyer Bergeron commit 6ffb1fdf6ce7825770148bada5a4c54899e4ed36 Author: Adam Hassick Date: Tue Jun 29 16:49:27 2021 -0400 Cobbler model changes, new endpoints Signed-off-by: Adam Hassick Change-Id: If0a94730e92747127cef121ec4930a4c8bae6c92 Signed-off-by: Sawyer Bergeron Signed-off-by: Adam Hassick commit 49e2b407003b69551ddafa851639e83ec42a5b09 Author: Jacob Hodgdon Date: Fri May 14 15:42:56 2021 -0400 Color fixes for rebrand Signed-off-by: Jacob Hodgdon Change-Id: I5cf4ede598afa377db7ecec17d8dfef085e130ac commit a908da441bf6efcdb289a46d0c2761840138b1a5 Author: Sawyer Bergeron Date: Tue Jun 8 11:15:56 2021 -0400 Draft for cloud-init file generation Signed-off-by: Sawyer Bergeron Signed-off-by: Sawyer Bergeron Change-Id: I07f3a4a1ab67531cba2cc7e3de22e9bb860706e1 Signed-off-by: Sawyer Bergeron Signed-off-by: Sawyer Bergeron Change-Id: I392505174cbc07214c31c42aab2474a748e47913 Signed-off-by: Sawyer Bergeron --- src/booking/forms.py | 1 + src/booking/lib.py | 4 +- src/booking/migrations/0009_booking_complete.py | 18 ++++++ src/booking/models.py | 2 + src/booking/quick_deployer.py | 75 ++++++++++++++++--------- src/booking/views.py | 6 +- 6 files changed, 77 insertions(+), 29 deletions(-) create mode 100644 src/booking/migrations/0009_booking_complete.py (limited to 'src/booking') diff --git a/src/booking/forms.py b/src/booking/forms.py index cbc3407..ff829b2 100644 --- a/src/booking/forms.py +++ b/src/booking/forms.py @@ -22,6 +22,7 @@ class QuickBookingForm(forms.Form): purpose = forms.CharField(max_length=1000) project = forms.CharField(max_length=400) hostname = forms.CharField(required=False, max_length=400) + global_cloud_config = forms.CharField(widget=forms.Textarea, required=False) installer = forms.ModelChoiceField(queryset=Installer.objects.all(), required=False) scenario = forms.ModelChoiceField(queryset=Scenario.objects.all(), required=False) diff --git a/src/booking/lib.py b/src/booking/lib.py index 7a4c261..8c87979 100644 --- a/src/booking/lib.py +++ b/src/booking/lib.py @@ -28,9 +28,9 @@ def get_user_items(exclude=None): for up in qs: item = { 'id': up.id, - 'expanded_name': up.full_name, + 'expanded_name': up.full_name if up.full_name else up.user.username, 'small_name': up.user.username, - 'string': up.email_addr + 'string': up.email_addr if up.email_addr else up.user.username, } items[up.id] = item return items diff --git a/src/booking/migrations/0009_booking_complete.py b/src/booking/migrations/0009_booking_complete.py new file mode 100644 index 0000000..e291a83 --- /dev/null +++ b/src/booking/migrations/0009_booking_complete.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2021-09-07 15:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('booking', '0008_auto_20201109_1947'), + ] + + operations = [ + migrations.AddField( + model_name='booking', + name='complete', + field=models.BooleanField(default=False), + ), + ] diff --git a/src/booking/models.py b/src/booking/models.py index cfdf7bc..966f1c2 100644 --- a/src/booking/models.py +++ b/src/booking/models.py @@ -39,6 +39,8 @@ class Booking(models.Model): pdf = models.TextField(blank=True, default="") idf = models.TextField(blank=True, default="") + complete = models.BooleanField(default=False) + class Meta: db_table = 'booking' diff --git a/src/booking/quick_deployer.py b/src/booking/quick_deployer.py index 5e5bc8b..31865be 100644 --- a/src/booking/quick_deployer.py +++ b/src/booking/quick_deployer.py @@ -9,6 +9,7 @@ import json +import yaml from django.db.models import Q from django.db import transaction from datetime import timedelta @@ -18,7 +19,6 @@ from account.models import Lab, UserProfile from resource_inventory.models import ( ResourceTemplate, - Installer, Image, OPNFVRole, OPNFVConfig, @@ -27,6 +27,7 @@ from resource_inventory.models import ( NetworkConnection, InterfaceConfiguration, Network, + CloudInitFile, ) from resource_inventory.resource_manager import ResourceManager from resource_inventory.pdf_templater import PDFTemplater @@ -61,7 +62,7 @@ def parse_resource_field(resource_json): return lab, template -def update_template(old_template, image, hostname, user): +def update_template(old_template, image, hostname, user, global_cloud_config=None): """ Duplicate a template to the users account and update configured fields. @@ -113,9 +114,17 @@ def update_template(old_template, image, hostname, user): image=image_to_set, template=template, is_head_node=old_config.is_head_node, - name=hostname if len(old_template.getConfigs()) == 1 else old_config.name + name=hostname if len(old_template.getConfigs()) == 1 else old_config.name, + # cloud_init_files=old_config.cloud_init_files.set() ) + for file in old_config.cloud_init_files.all(): + config.cloud_init_files.add(file) + + if global_cloud_config: + config.cloud_init_files.add(global_cloud_config) + config.save() + for old_iface_config in old_config.interface_configs.all(): iface_config = InterfaceConfiguration.objects.create( profile=old_iface_config.profile, @@ -170,20 +179,11 @@ def generate_resource_bundle(template): def check_invariants(**kwargs): # TODO: This should really happen in the BookingForm validation methods - installer = kwargs['installer'] image = kwargs['image'] - scenario = kwargs['scenario'] lab = kwargs['lab'] length = kwargs['length'] # check that image os is compatible with installer if image: - if installer or scenario: - if installer in image.os.sup_installers.all(): - # if installer not here, we can omit that and not check for scenario - if not scenario: - raise ValidationError("An OPNFV Installer needs a scenario to be chosen to work properly") - if scenario not in installer.sup_scenarios.all(): - raise ValidationError("The chosen installer does not support the chosen scenario") if image.from_lab != lab: raise ValidationError("The chosen image is not available at the chosen hosting lab") # TODO @@ -200,9 +200,21 @@ def create_from_form(form, request): Parse data from QuickBookingForm to create booking """ resource_field = form.cleaned_data['filter_field'] + # users_field = form.cleaned_data['users'] + hostname = 'opnfv_host' if not form.cleaned_data['hostname'] else form.cleaned_data['hostname'] + + global_cloud_config = None if not form.cleaned_data['global_cloud_config'] else form.cleaned_data['global_cloud_config'] + + if global_cloud_config: + form.cleaned_data['global_cloud_config'] = create_ci_file(global_cloud_config) + + # image = form.cleaned_data['image'] + # scenario = form.cleaned_data['scenario'] + # installer = form.cleaned_data['installer'] lab, resource_template = parse_resource_field(resource_field) data = form.cleaned_data + data['hostname'] = hostname data['lab'] = lab data['resource_template'] = resource_template data['owner'] = request.user @@ -232,9 +244,26 @@ def create_from_API(body, user): data['lab'] = data['resource_template'].lab data['owner'] = user + if 'global_cloud_config' in data.keys(): + data['global_cloud_config'] = CloudInitFile.objects.get(id=data['global_cloud_config']) + return _create_booking(data) +def create_ci_file(data: str) -> CloudInitFile: + try: + d = yaml.load(data) + if not (type(d) is dict): + raise Exception("CI file was valid yaml but was not a dict") + except Exception: + raise ValidationError("The provided Cloud Config is not valid yaml, please refer to the Cloud Init documentation for expected structure") + print("about to create global cloud config") + config = CloudInitFile.create(text=data, priority=CloudInitFile.objects.count()) + print("made global cloud config") + + return config + + @transaction.atomic def _create_booking(data): check_invariants(**data) @@ -246,8 +275,11 @@ def _create_booking(data): raise PermissionError("You do not have permission to have more than 3 bookings at a time.") ResourceManager.getInstance().templateIsReservable(data['resource_template']) - data['resource_template'] = update_template(data['resource_template'], data['image'], 'opnfv_host' if not data['hostname'] else data['hostname'], data['owner']) - resource_bundle = generate_resource_bundle(data['resource_template']) + + resource_template = update_template(data['resource_template'], data['image'], data['hostname'], data['owner'], global_cloud_config=data['global_cloud_config']) + + # generate resource bundle + resource_bundle = generate_resource_bundle(resource_template) # generate booking booking = Booking.objects.create( @@ -263,7 +295,7 @@ def _create_booking(data): booking.pdf = PDFTemplater.makePDF(booking) - for collaborator in data['users']: # list of UserProfiles + for collaborator in data['users']: # list of Users (not UserProfile) booking.collaborators.add(collaborator.user) booking.save() @@ -284,23 +316,14 @@ def drop_filter(user): that installer is supported on that image """ installer_filter = {} - for image in Image.objects.all(): - installer_filter[image.id] = {} - for installer in image.os.sup_installers.all(): - installer_filter[image.id][installer.id] = 1 - scenario_filter = {} - for installer in Installer.objects.all(): - scenario_filter[installer.id] = {} - for scenario in installer.sup_scenarios.all(): - scenario_filter[installer.id][scenario.id] = 1 images = Image.objects.filter(Q(public=True) | Q(owner=user)) image_filter = {} for image in images: image_filter[image.id] = { 'lab': 'lab_' + str(image.from_lab.lab_user.id), - 'host_profile': str(image.host_type.id), + 'architecture': str(image.architecture), 'name': image.name } @@ -308,7 +331,7 @@ def drop_filter(user): templates = ResourceTemplate.objects.filter(Q(public=True) | Q(owner=user)) for rt in templates: profiles = [conf.profile for conf in rt.getConfigs()] - resource_filter["resource_" + str(rt.id)] = [str(p.id) for p in profiles] + resource_filter["resource_" + str(rt.id)] = [str(p.architecture) for p in profiles] return { 'installer_filter': json.dumps(installer_filter), diff --git a/src/booking/views.py b/src/booking/views.py index a418c82..940428b 100644 --- a/src/booking/views.py +++ b/src/booking/views.py @@ -28,6 +28,7 @@ from api.models import JobFactory from workflow.views import login from booking.forms import QuickBookingForm from booking.quick_deployer import create_from_form, drop_filter +import traceback def quick_create_clear_fields(request): @@ -62,6 +63,9 @@ def quick_create(request): "Check Account->My Bookings for the status of your new booking") return redirect(reverse('booking:booking_detail', kwargs={'booking_id': booking.id})) except Exception as e: + print("Error occurred while handling quick deployment:") + traceback.print_exc() + print(str(e)) messages.error(request, "Whoops, an error occurred: " + str(e)) context.update(drop_filter(request.user)) return render(request, 'booking/quick_deploy.html', context) @@ -137,7 +141,7 @@ def build_image_mapping(lab, user): for profile in ResourceProfile.objects.filter(labs=lab): images = Image.objects.filter( from_lab=lab, - host_type=profile + architecture=profile.architecture ).filter( Q(public=True) | Q(owner=user) ) -- cgit 1.2.3-korg