From a819fc1df86721eda36eee89d0235c89b3159d6b Mon Sep 17 00:00:00 2001 From: Sawyer Bergeron Date: Tue, 7 Sep 2021 11:28:35 -0400 Subject: Add user specified CI file entry Signed-off-by: Sawyer Bergeron Change-Id: Ia920130612da8fcde9d1a0d5dde7861904857162 Signed-off-by: Sawyer Bergeron --- src/api/migrations/0019_auto_20210907_1448.py | 29 ++++++++++ src/api/models.py | 26 ++++++--- src/api/urls.py | 8 ++- src/api/views.py | 25 ++++++--- src/booking/forms.py | 1 + src/booking/migrations/0009_booking_complete.py | 18 ++++++ src/booking/models.py | 2 + src/booking/quick_deployer.py | 30 +++++++++- src/dashboard/tasks.py | 6 +- .../migrations/0020_cloudinitfile.py | 21 +++++++ .../0021_resourceconfiguration_cloud_init_files.py | 18 ++++++ src/resource_inventory/models.py | 32 +++++++++++ src/templates/base/booking/booking_detail.html | 65 ++++++++++++++++++++++ src/templates/base/booking/quick_deploy.html | 20 ++++++- 14 files changed, 275 insertions(+), 26 deletions(-) create mode 100644 src/api/migrations/0019_auto_20210907_1448.py create mode 100644 src/booking/migrations/0009_booking_complete.py create mode 100644 src/resource_inventory/migrations/0020_cloudinitfile.py create mode 100644 src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py diff --git a/src/api/migrations/0019_auto_20210907_1448.py b/src/api/migrations/0019_auto_20210907_1448.py new file mode 100644 index 0000000..92140fb --- /dev/null +++ b/src/api/migrations/0019_auto_20210907_1448.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2 on 2021-09-07 14:48 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('booking', '0008_auto_20201109_1947'), + ('resource_inventory', '0020_cloudinitfile'), + ('api', '0018_cloudinitfile'), + ] + + operations = [ + migrations.CreateModel( + name='GeneratedCloudConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('resource_id', models.CharField(max_length=200)), + ('text', models.TextField(blank=True, null=True)), + ('booking', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='booking.Booking')), + ('rconfig', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='resource_inventory.ResourceConfiguration')), + ], + ), + migrations.DeleteModel( + name='CloudInitFile', + ), + ] diff --git a/src/api/models.py b/src/api/models.py index 3098111..ec163a1 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -32,7 +32,8 @@ from resource_inventory.models import ( OPNFVConfig, ConfigState, ResourceQuery, - ResourceConfiguration + ResourceConfiguration, + CloudInitFile ) from resource_inventory.idf_templater import IDFTemplater from resource_inventory.pdf_templater import PDFTemplater @@ -351,10 +352,11 @@ class LabManager(object): profile_ser.append(p) return profile_ser -class CloudInitFile(models.Model): +class GeneratedCloudConfig(models.Model): resource_id = models.CharField(max_length=200) booking = models.ForeignKey(Booking, on_delete=models.CASCADE) rconfig = models.ForeignKey(ResourceConfiguration, on_delete=models.CASCADE) + text = models.TextField(null=True, blank=True) def _normalize_username(self, username: str) -> str: # TODO: make usernames posix compliant @@ -447,8 +449,8 @@ class CloudInitFile(models.Model): return full_dict @classmethod - def get(cls, booking_id: int, resource_lab_id: str): - return CloudInitFile.objects.get(resource_id=resource_lab_id, booking__id=booking_id) + def get(cls, booking_id: int, resource_lab_id: str, file_id: int): + return GeneratedCloudConfig.objects.get(resource_id=resource_lab_id, booking__id=booking_id, file_id=file_id) def _resource(self): return ResourceQuery.get(labid=self.resource_id, lab=self.booking.lab) @@ -469,7 +471,7 @@ class CloudInitFile(models.Model): return main_dict def serialize(self) -> str: - return str("#cloud-config\n") + yaml.dump(self._to_dict()) + return yaml.dump(self._to_dict()) class Job(models.Model): """ @@ -804,7 +806,7 @@ class HardwareConfig(TaskConfig): return self.get_delta() def get_delta(self): - # TODO: grab the CloudInitFile urls from self.hosthardwarerelation.get_resource() + # TODO: grab the GeneratedCloudConfig urls from self.hosthardwarerelation.get_resource() return self.format_delta( self.hosthardwarerelation.get_resource().get_configuration(self.state), self.hosthardwarerelation.lab_token) @@ -1148,7 +1150,7 @@ class JobFactory(object): booking=booking, job=job ) - cls.makeCloudInitFiles( + cls.makeGeneratedCloudConfigs( resources=resources, job=job ) @@ -1176,11 +1178,17 @@ class JobFactory(object): continue @classmethod - def makeCloudInitFiles(cls, resources=[], job=Job()): + def makeGeneratedCloudConfigs(cls, resources=[], job=Job()): for res in resources: - cif = CloudInitFile.objects.create(resource_id=res.labid, booking=job.booking, rconfig=res.config) + cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=job.booking, rconfig=res.config) cif.save() + cif = CloudInitFile.create(priority=0, text=cif.serialize()) + cif.save() + + res.config.cloud_init_files.add(cif) + res.config.save() + @classmethod def makeHardwareConfigs(cls, resources=[], job=Job()): """ diff --git a/src/api/urls.py b/src/api/urls.py index 970ecf2..8dcfafe 100644 --- a/src/api/urls.py +++ b/src/api/urls.py @@ -48,10 +48,11 @@ from api.views import ( analytics_job, resource_ci_metadata, resource_ci_userdata, + resource_ci_userdata_directory, all_images, all_opsyss, single_image, - single_opsys + single_opsys, ) urlpatterns = [ @@ -69,8 +70,9 @@ urlpatterns = [ path('labs//booking//idf', get_idf, name="get-idf"), path('labs//jobs/', specific_job), path('labs//jobs//', specific_task), - path('labs//jobs//cidata//user-data', resource_ci_userdata), - path('labs//jobs//cidata//meta-data', resource_ci_metadata), + path('labs//jobs//cidata//user-data', resource_ci_userdata_directory, name="specific-user-data"), + path('labs//jobs//cidata//meta-data', resource_ci_metadata, name="specific-meta-data"), + path('labs//jobs//cidata///user-data', resource_ci_userdata, name="user-data-dir"), path('labs//jobs/new', new_jobs), path('labs//jobs/current', current_jobs), path('labs//jobs/done', done_jobs), diff --git a/src/api/views.py b/src/api/views.py index 7add23e..79da84c 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -24,14 +24,16 @@ from django.core.exceptions import ObjectDoesNotExist from api.serializers.booking_serializer import BookingSerializer from api.serializers.old_serializers import UserSerializer from api.forms import DowntimeForm -from account.models import UserProfile +from account.models import UserProfile, Lab from booking.models import Booking -from api.models import LabManagerTracker, get_task, CloudInitFile, Job +from api.models import LabManagerTracker, get_task, Job from notifier.manager import NotificationHandler from analytics.models import ActiveVPNUser from resource_inventory.models import ( Image, - Opsys + Opsys, + CloudInitFile, + ResourceQuery, ) import json @@ -248,7 +250,7 @@ def specific_job(request, lab_name="", job_id=""): return JsonResponse(lab_manager.get_job(job_id), safe=False) @csrf_exempt -def resource_ci_userdata(request, lab_name="", job_id="", resource_id=""): +def resource_ci_userdata(request, lab_name="", job_id="", resource_id="", file_id=0): #lab_token = request.META.get('HTTP_AUTH_TOKEN') #lab_manager = LabManagerTracker.get(lab_name, lab_token) @@ -257,16 +259,25 @@ def resource_ci_userdata(request, lab_name="", job_id="", resource_id=""): cifile = None try: - cifile = CloudInitFile.get(job.booking.id, resource_id) + cifile = CloudInitFile.objects.get(id=file_id) except ObjectDoesNotExist: return HttpResponseNotFound("Could not find a matching resource by id " + str(resource_id)) - return HttpResponse(cifile.serialize(), status=200) + return HttpResponse(cifile.text, status=200) @csrf_exempt -def resource_ci_metadata(request, lab_name="", job_id="", resource_id=""): +def resource_ci_metadata(request, lab_name="", job_id="", resource_id="", file_id=0): return HttpResponse("#cloud-config", status=200) +@csrf_exempt +def resource_ci_userdata_directory(request, lab_name="", job_id="", resource_id=""): + #files = [{"id": file.file_id, "priority": file.priority} for file in CloudInitFile.objects.filter(job__id=job_id, resource_id=resource_id).order_by("priority").all()] + resource = ResourceQuery.get(labid=resource_id, lab=Lab.objects.get(name=lab_name)) + files = resource.config.cloud_init_files + files = [{"id": file.id, "priority": file.priority} for file in files.order_by("priority").all()] + + return HttpResponse(json.dumps(files), status=200) + def new_jobs(request, lab_name=""): lab_token = request.META.get('HTTP_AUTH_TOKEN') diff --git a/src/booking/forms.py b/src/booking/forms.py index cbc3407..ff829b2 100644 --- a/src/booking/forms.py +++ b/src/booking/forms.py @@ -22,6 +22,7 @@ class QuickBookingForm(forms.Form): purpose = forms.CharField(max_length=1000) project = forms.CharField(max_length=400) hostname = forms.CharField(required=False, max_length=400) + global_cloud_config = forms.CharField(widget=forms.Textarea, required=False) installer = forms.ModelChoiceField(queryset=Installer.objects.all(), required=False) scenario = forms.ModelChoiceField(queryset=Scenario.objects.all(), required=False) diff --git a/src/booking/migrations/0009_booking_complete.py b/src/booking/migrations/0009_booking_complete.py new file mode 100644 index 0000000..e291a83 --- /dev/null +++ b/src/booking/migrations/0009_booking_complete.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2021-09-07 15:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('booking', '0008_auto_20201109_1947'), + ] + + operations = [ + migrations.AddField( + model_name='booking', + name='complete', + field=models.BooleanField(default=False), + ), + ] diff --git a/src/booking/models.py b/src/booking/models.py index cfdf7bc..966f1c2 100644 --- a/src/booking/models.py +++ b/src/booking/models.py @@ -39,6 +39,8 @@ class Booking(models.Model): pdf = models.TextField(blank=True, default="") idf = models.TextField(blank=True, default="") + complete = models.BooleanField(default=False) + class Meta: db_table = 'booking' diff --git a/src/booking/quick_deployer.py b/src/booking/quick_deployer.py index 9e53da5..9bdebc2 100644 --- a/src/booking/quick_deployer.py +++ b/src/booking/quick_deployer.py @@ -26,6 +26,7 @@ from resource_inventory.models import ( NetworkConnection, InterfaceConfiguration, Network, + CloudInitFile, ) from resource_inventory.resource_manager import ResourceManager from resource_inventory.pdf_templater import PDFTemplater @@ -60,7 +61,7 @@ def parse_resource_field(resource_json): return lab, template -def update_template(old_template, image, hostname, user): +def update_template(old_template, image, hostname, user, global_cloud_config=None): """ Duplicate a template to the users account and update configured fields. @@ -112,9 +113,17 @@ def update_template(old_template, image, hostname, user): image=image_to_set, template=template, is_head_node=old_config.is_head_node, - name=hostname if len(old_template.getConfigs()) == 1 else old_config.name + name=hostname if len(old_template.getConfigs()) == 1 else old_config.name, + #cloud_init_files=old_config.cloud_init_files.set() ) + for file in old_config.cloud_init_files.all(): + config.cloud_init_files.add(file); + + if global_cloud_config: + config.cloud_init_files.add(global_cloud_config) + config.save() + for old_iface_config in old_config.interface_configs.all(): iface_config = InterfaceConfiguration.objects.create( profile=old_iface_config.profile, @@ -186,6 +195,13 @@ def check_invariants(request, **kwargs): if length < 1 or length > 21: raise BookingLengthException("Booking must be between 1 and 21 days long") +# global_cloud_config is Option forming a valid yaml file if Some +def generate_cloud_configs(resource_bundle, global_cloud_config): + c_file = CloudInitFile.objects.new(priority=1) # apply after the internal + for host in resource_bundle.get_resources(): + #cfile = CloudInitFile::from_text( + pass + # TODO def create_from_form(form, request): """ @@ -200,6 +216,12 @@ def create_from_form(form, request): users_field = form.cleaned_data['users'] hostname = 'opnfv_host' if not form.cleaned_data['hostname'] else form.cleaned_data['hostname'] length = form.cleaned_data['length'] + global_cloud_config = None if not form.cleaned_data['global_cloud_config'] else form.cleaned_data['global_cloud_config'] + + if global_cloud_config: + print("about to create global cloud config") + global_cloud_config = CloudInitFile.create(text=global_cloud_config, priority=CloudInitFile.objects.count()) + print("made global cloud config") image = form.cleaned_data['image'] scenario = form.cleaned_data['scenario'] @@ -219,7 +241,7 @@ def create_from_form(form, request): ResourceManager.getInstance().templateIsReservable(resource_template) - resource_template = update_template(resource_template, image, hostname, request.user) + resource_template = update_template(resource_template, image, hostname, request.user, global_cloud_config=global_cloud_config) # if no installer provided, just create blank host opnfv_config = None @@ -231,6 +253,8 @@ def create_from_form(form, request): # generate resource bundle resource_bundle = generate_resource_bundle(resource_template) + #generate_cloud_configs(resource_bundle) + # generate booking booking = Booking.objects.create( purpose=purpose_field, diff --git a/src/dashboard/tasks.py b/src/dashboard/tasks.py index 3f88449..93e6a22 100644 --- a/src/dashboard/tasks.py +++ b/src/dashboard/tasks.py @@ -81,11 +81,15 @@ def free_hosts(): ).filter( end__lt=timezone.now(), job__complete=True, - resource__isnull=False + complete=False, + resource__isnull=False, ) for booking in bookings: ResourceManager.getInstance().releaseResourceBundle(booking.resource) + booking.complete = True + print("Booking", booking.id, "is now completed") + booking.save() @shared_task diff --git a/src/resource_inventory/migrations/0020_cloudinitfile.py b/src/resource_inventory/migrations/0020_cloudinitfile.py new file mode 100644 index 0000000..198181c --- /dev/null +++ b/src/resource_inventory/migrations/0020_cloudinitfile.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2 on 2021-09-07 14:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0019_auto_20210701_1947'), + ] + + operations = [ + migrations.CreateModel( + name='CloudInitFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('text', models.TextField()), + ('priority', models.IntegerField()), + ], + ), + ] diff --git a/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py new file mode 100644 index 0000000..6b0befc --- /dev/null +++ b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2021-09-10 18:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0020_cloudinitfile'), + ] + + operations = [ + migrations.AddField( + model_name='resourceconfiguration', + name='cloud_init_files', + field=models.ManyToManyField(blank=True, to='resource_inventory.CloudInitFile'), + ), + ] diff --git a/src/resource_inventory/models.py b/src/resource_inventory/models.py index fb4dad5..2c631dc 100644 --- a/src/resource_inventory/models.py +++ b/src/resource_inventory/models.py @@ -15,6 +15,7 @@ from django.db import models from django.db.models import Q import traceback import json +import yaml import re from collections import Counter @@ -152,6 +153,26 @@ with varying degrees of abstraction. """ +class CloudInitFile(models.Model): + text = models.TextField() + + # higher priority is applied later, so "on top" of existing files + priority = models.IntegerField() + + @classmethod + def merge_strategy(cls): + return [ + { 'name': 'list', 'settings': ['append'] }, + { 'name': 'dict', 'settings': ['recurse_list', 'replace'] }, + ] + + @classmethod + def create(cls, text="", priority=0): + prepended_text = "#cloud-config\n" + prepended_text = prepended_text + yaml.dump(CloudInitFile.merge_strategy()) + "\n" + print("in cloudinitfile create") + return CloudInitFile.objects.create(priority=priority, text=(prepended_text + text)) + class ResourceTemplate(models.Model): """ Models a "template" of a complete, configured collection of resources that can be booked. @@ -240,9 +261,14 @@ class ResourceConfiguration(models.Model): is_head_node = models.BooleanField(default=False) name = models.CharField(max_length=3000, default="opnfv_host") + cloud_init_files = models.ManyToManyField(CloudInitFile, blank=True) + def __str__(self): return str(self.name) + def ci_file_list(self): + return list(self.cloud_init_files.order_by("priority").all()) + def get_default_remote_info(): return RemoteInfo.objects.get_or_create( @@ -578,6 +604,12 @@ class NetworkRole(models.Model): network = models.ForeignKey(Network, on_delete=models.CASCADE) +def create_resource_ref_string(for_hosts: [str]) -> str: + # need to sort the list, then do dump + for_hosts.sort() + + return json.dumps(for_hosts) + class OPNFVConfig(models.Model): id = models.AutoField(primary_key=True) installer = models.ForeignKey(Installer, on_delete=models.CASCADE) diff --git a/src/templates/base/booking/booking_detail.html b/src/templates/base/booking/booking_detail.html index a014fea..4a8f35a 100644 --- a/src/templates/base/booking/booking_detail.html +++ b/src/templates/base/booking/booking_detail.html @@ -7,6 +7,12 @@ {% endblock %} + + {% block content %}
@@ -154,6 +160,65 @@
+
+
+

Diagnostic Information

+ +
+
+
+ + + + + + + + + {% for host in booking.resource.get_resources %} + + + + {% endfor %} +
Job ID: {{booking.job.id}}
CI Files
+ + + + + + + + + {% for ci_file in host.config.cloud_init_files.all %} + + + + + {% endfor %} +
Host:{{host.name}}
Configs:
{{ci_file.id}} + + +
+
+
+
+
diff --git a/src/templates/base/booking/quick_deploy.html b/src/templates/base/booking/quick_deploy.html index 1193aab..c51e234 100644 --- a/src/templates/base/booking/quick_deploy.html +++ b/src/templates/base/booking/quick_deploy.html @@ -3,6 +3,13 @@ {% load bootstrap4 %} {% block content %} + + {% bootstrap_form_errors form type='non_fields' %}
{% csrf_token %} @@ -18,7 +25,7 @@
-
+
{% bootstrap_field form.purpose %} {% bootstrap_field form.project %} @@ -31,19 +38,26 @@
{% block collab %} -
+
{{ form.users }}
{% endblock collab %} -
+
+
+
{% bootstrap_field form.hostname %} {% bootstrap_field form.image %}
+
+
+ {% bootstrap_field form.global_cloud_config %} +
+
-- cgit 1.2.3-korg