Add user specified CI file entry

Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu>
Change-Id: Ia920130612da8fcde9d1a0d5dde7861904857162
Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu>

14 files changed, 275 insertions, 26 deletions

diff --git a/src/api/migrations/0019_auto_20210907_1448.py b/src/api/migrations/0019_auto_20210907_1448.py
new file mode 100644
index 0000000..92140fb
--- /dev/null
+++ b/src/api/migrations/0019_auto_20210907_1448.py
@@ -0,0 +1,29 @@
+# Generated by Django 2.2 on 2021-09-07 14:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('booking', '0008_auto_20201109_1947'),
+        ('resource_inventory', '0020_cloudinitfile'),
+        ('api', '0018_cloudinitfile'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='GeneratedCloudConfig',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('resource_id', models.CharField(max_length=200)),
+                ('text', models.TextField(blank=True, null=True)),
+                ('booking', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='booking.Booking')),
+                ('rconfig', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='resource_inventory.ResourceConfiguration')),
+            ],
+        ),
+        migrations.DeleteModel(
+            name='CloudInitFile',
+        ),
+    ]
diff --git a/src/api/models.py b/src/api/models.py
index 3098111..ec163a1 100644
--- a/src/api/models.py
+++ b/src/api/models.py
@@ -32,7 +32,8 @@ from resource_inventory.models import (
     OPNFVConfig,
     ConfigState,
     ResourceQuery,
-    ResourceConfiguration
+    ResourceConfiguration,
+    CloudInitFile
 )
 from resource_inventory.idf_templater import IDFTemplater
 from resource_inventory.pdf_templater import PDFTemplater
@@ -351,10 +352,11 @@ class LabManager(object):
             profile_ser.append(p)
         return profile_ser
 
-class CloudInitFile(models.Model):
+class GeneratedCloudConfig(models.Model):
     resource_id = models.CharField(max_length=200)
     booking = models.ForeignKey(Booking, on_delete=models.CASCADE)
     rconfig = models.ForeignKey(ResourceConfiguration, on_delete=models.CASCADE)
+    text = models.TextField(null=True, blank=True)
 
     def _normalize_username(self, username: str) -> str:
         # TODO: make usernames posix compliant
@@ -447,8 +449,8 @@ class CloudInitFile(models.Model):
         return full_dict
 
     @classmethod
-    def get(cls, booking_id: int, resource_lab_id: str):
-        return CloudInitFile.objects.get(resource_id=resource_lab_id, booking__id=booking_id)
+    def get(cls, booking_id: int, resource_lab_id: str, file_id: int):
+        return GeneratedCloudConfig.objects.get(resource_id=resource_lab_id, booking__id=booking_id, file_id=file_id)
 
     def _resource(self):
         return ResourceQuery.get(labid=self.resource_id, lab=self.booking.lab)
@@ -469,7 +471,7 @@ class CloudInitFile(models.Model):
         return main_dict
 
     def serialize(self) -> str:
-        return str("#cloud-config\n") + yaml.dump(self._to_dict())
+        return yaml.dump(self._to_dict())
 
 class Job(models.Model):
     """
@@ -804,7 +806,7 @@ class HardwareConfig(TaskConfig):
         return self.get_delta()
 
     def get_delta(self):
-        # TODO: grab the CloudInitFile urls from self.hosthardwarerelation.get_resource()
+        # TODO: grab the GeneratedCloudConfig urls from self.hosthardwarerelation.get_resource()
         return self.format_delta(
             self.hosthardwarerelation.get_resource().get_configuration(self.state),
             self.hosthardwarerelation.lab_token)
@@ -1148,7 +1150,7 @@ class JobFactory(object):
             booking=booking,
             job=job
         )
-        cls.makeCloudInitFiles(
+        cls.makeGeneratedCloudConfigs(
             resources=resources,
             job=job
         )
@@ -1176,11 +1178,17 @@ class JobFactory(object):
                 continue
 
     @classmethod
-    def makeCloudInitFiles(cls, resources=[], job=Job()):
+    def makeGeneratedCloudConfigs(cls, resources=[], job=Job()):
         for res in resources:
-            cif = CloudInitFile.objects.create(resource_id=res.labid, booking=job.booking, rconfig=res.config)
+            cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=job.booking, rconfig=res.config)
             cif.save()
 
+            cif = CloudInitFile.create(priority=0, text=cif.serialize())
+            cif.save()
+
+            res.config.cloud_init_files.add(cif)
+            res.config.save()
+
     @classmethod
     def makeHardwareConfigs(cls, resources=[], job=Job()):
         """
diff --git a/src/api/urls.py b/src/api/urls.py
index 970ecf2..8dcfafe 100644
--- a/src/api/urls.py
+++ b/src/api/urls.py
@@ -48,10 +48,11 @@ from api.views import (
     analytics_job,
     resource_ci_metadata,
     resource_ci_userdata,
+    resource_ci_userdata_directory,
     all_images,
     all_opsyss,
     single_image,
-    single_opsys
+    single_opsys,
 )
 
 urlpatterns = [
@@ -69,8 +70,9 @@ urlpatterns = [
     path('labs/<slug:lab_name>/booking/<int:booking_id>/idf', get_idf, name="get-idf"),
     path('labs/<slug:lab_name>/jobs/<int:job_id>', specific_job),
     path('labs/<slug:lab_name>/jobs/<int:job_id>/<slug:task_id>', specific_task),
-    path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>/user-data', resource_ci_userdata),
-    path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>/meta-data', resource_ci_metadata),
+    path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>/user-data', resource_ci_userdata_directory, name="specific-user-data"),
+    path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>/meta-data', resource_ci_metadata, name="specific-meta-data"),
+    path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>/<int:file_id>/user-data', resource_ci_userdata, name="user-data-dir"),
     path('labs/<slug:lab_name>/jobs/new', new_jobs),
     path('labs/<slug:lab_name>/jobs/current', current_jobs),
     path('labs/<slug:lab_name>/jobs/done', done_jobs),
diff --git a/src/api/views.py b/src/api/views.py
index 7add23e..79da84c 100644
--- a/src/api/views.py
+++ b/src/api/views.py
@@ -24,14 +24,16 @@ from django.core.exceptions import ObjectDoesNotExist
 from api.serializers.booking_serializer import BookingSerializer
 from api.serializers.old_serializers import UserSerializer
 from api.forms import DowntimeForm
-from account.models import UserProfile
+from account.models import UserProfile, Lab
 from booking.models import Booking
-from api.models import LabManagerTracker, get_task, CloudInitFile, Job
+from api.models import LabManagerTracker, get_task, Job
 from notifier.manager import NotificationHandler
 from analytics.models import ActiveVPNUser
 from resource_inventory.models import (
     Image,
-    Opsys
+    Opsys,
+    CloudInitFile,
+    ResourceQuery,
 )
 
 import json
@@ -248,7 +250,7 @@ def specific_job(request, lab_name="", job_id=""):
     return JsonResponse(lab_manager.get_job(job_id), safe=False)
 
 @csrf_exempt
-def resource_ci_userdata(request, lab_name="", job_id="", resource_id=""):
+def resource_ci_userdata(request, lab_name="", job_id="", resource_id="", file_id=0):
     #lab_token = request.META.get('HTTP_AUTH_TOKEN')
     #lab_manager = LabManagerTracker.get(lab_name, lab_token)
 
@@ -257,16 +259,25 @@ def resource_ci_userdata(request, lab_name="", job_id="", resource_id=""):
 
     cifile = None
     try:
-        cifile = CloudInitFile.get(job.booking.id, resource_id)
+        cifile = CloudInitFile.objects.get(id=file_id)
     except ObjectDoesNotExist:
         return HttpResponseNotFound("Could not find a matching resource by id " + str(resource_id))
 
-    return HttpResponse(cifile.serialize(), status=200)
+    return HttpResponse(cifile.text, status=200)
 
 @csrf_exempt
-def resource_ci_metadata(request, lab_name="", job_id="", resource_id=""):
+def resource_ci_metadata(request, lab_name="", job_id="", resource_id="", file_id=0):
     return HttpResponse("#cloud-config", status=200)
 
+@csrf_exempt
+def resource_ci_userdata_directory(request, lab_name="", job_id="", resource_id=""):
+    #files = [{"id": file.file_id, "priority": file.priority} for file in CloudInitFile.objects.filter(job__id=job_id, resource_id=resource_id).order_by("priority").all()]
+    resource = ResourceQuery.get(labid=resource_id, lab=Lab.objects.get(name=lab_name))
+    files = resource.config.cloud_init_files
+    files = [{"id": file.id, "priority": file.priority} for file in files.order_by("priority").all()]
+
+    return HttpResponse(json.dumps(files), status=200)
+
 
 def new_jobs(request, lab_name=""):
     lab_token = request.META.get('HTTP_AUTH_TOKEN')
diff --git a/src/booking/forms.py b/src/booking/forms.py
index cbc3407..ff829b2 100644
--- a/src/booking/forms.py
+++ b/src/booking/forms.py
@@ -22,6 +22,7 @@ class QuickBookingForm(forms.Form):
     purpose = forms.CharField(max_length=1000)
     project = forms.CharField(max_length=400)
     hostname = forms.CharField(required=False, max_length=400)
+    global_cloud_config = forms.CharField(widget=forms.Textarea, required=False)
 
     installer = forms.ModelChoiceField(queryset=Installer.objects.all(), required=False)
     scenario = forms.ModelChoiceField(queryset=Scenario.objects.all(), required=False)
diff --git a/src/booking/migrations/0009_booking_complete.py b/src/booking/migrations/0009_booking_complete.py
new file mode 100644
index 0000000..e291a83
--- /dev/null
+++ b/src/booking/migrations/0009_booking_complete.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2 on 2021-09-07 15:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('booking', '0008_auto_20201109_1947'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='booking',
+            name='complete',
+            field=models.BooleanField(default=False),
+        ),
+    ]
diff --git a/src/booking/models.py b/src/booking/models.py
index cfdf7bc..966f1c2 100644
--- a/src/booking/models.py
+++ b/src/booking/models.py
@@ -39,6 +39,8 @@ class Booking(models.Model):
     pdf = models.TextField(blank=True, default="")
     idf = models.TextField(blank=True, default="")
 
+    complete = models.BooleanField(default=False)
+
     class Meta:
         db_table = 'booking'
 
diff --git a/src/booking/quick_deployer.py b/src/booking/quick_deployer.py
index 9e53da5..9bdebc2 100644
--- a/src/booking/quick_deployer.py
+++ b/src/booking/quick_deployer.py
@@ -26,6 +26,7 @@ from resource_inventory.models import (
     NetworkConnection,
     InterfaceConfiguration,
     Network,
+    CloudInitFile,
 )
 from resource_inventory.resource_manager import ResourceManager
 from resource_inventory.pdf_templater import PDFTemplater
@@ -60,7 +61,7 @@ def parse_resource_field(resource_json):
     return lab, template
 
 
-def update_template(old_template, image, hostname, user):
+def update_template(old_template, image, hostname, user, global_cloud_config=None):
     """
     Duplicate a template to the users account and update configured fields.
 
@@ -112,9 +113,17 @@ def update_template(old_template, image, hostname, user):
             image=image_to_set,
             template=template,
             is_head_node=old_config.is_head_node,
-            name=hostname if len(old_template.getConfigs()) == 1 else old_config.name
+            name=hostname if len(old_template.getConfigs()) == 1 else old_config.name,
+            #cloud_init_files=old_config.cloud_init_files.set()
         )
 
+        for file in old_config.cloud_init_files.all():
+            config.cloud_init_files.add(file);
+
+        if global_cloud_config:
+            config.cloud_init_files.add(global_cloud_config)
+            config.save()
+
         for old_iface_config in old_config.interface_configs.all():
             iface_config = InterfaceConfiguration.objects.create(
                 profile=old_iface_config.profile,
@@ -186,6 +195,13 @@ def check_invariants(request, **kwargs):
     if length < 1 or length > 21:
         raise BookingLengthException("Booking must be between 1 and 21 days long")
 
+# global_cloud_config is Option<str> forming a valid yaml file if Some
+def generate_cloud_configs(resource_bundle, global_cloud_config):
+    c_file = CloudInitFile.objects.new(priority=1) # apply after the internal 
+    for host in resource_bundle.get_resources():
+        #cfile = CloudInitFile::from_text(
+        pass
+        # TODO
 
 def create_from_form(form, request):
     """
@@ -200,6 +216,12 @@ def create_from_form(form, request):
     users_field = form.cleaned_data['users']
     hostname = 'opnfv_host' if not form.cleaned_data['hostname'] else form.cleaned_data['hostname']
     length = form.cleaned_data['length']
+    global_cloud_config = None if not form.cleaned_data['global_cloud_config'] else form.cleaned_data['global_cloud_config']
+
+    if global_cloud_config:
+        print("about to create global cloud config")
+        global_cloud_config = CloudInitFile.create(text=global_cloud_config, priority=CloudInitFile.objects.count())
+        print("made global cloud config")
 
     image = form.cleaned_data['image']
     scenario = form.cleaned_data['scenario']
@@ -219,7 +241,7 @@ def create_from_form(form, request):
 
     ResourceManager.getInstance().templateIsReservable(resource_template)
 
-    resource_template = update_template(resource_template, image, hostname, request.user)
+    resource_template = update_template(resource_template, image, hostname, request.user, global_cloud_config=global_cloud_config)
 
     # if no installer provided, just create blank host
     opnfv_config = None
@@ -231,6 +253,8 @@ def create_from_form(form, request):
     # generate resource bundle
     resource_bundle = generate_resource_bundle(resource_template)
 
+    #generate_cloud_configs(resource_bundle)
+
     # generate booking
     booking = Booking.objects.create(
         purpose=purpose_field,
diff --git a/src/dashboard/tasks.py b/src/dashboard/tasks.py
index 3f88449..93e6a22 100644
--- a/src/dashboard/tasks.py
+++ b/src/dashboard/tasks.py
@@ -81,11 +81,15 @@ def free_hosts():
     ).filter(
         end__lt=timezone.now(),
         job__complete=True,
-        resource__isnull=False
+        complete=False,
+        resource__isnull=False,
     )
 
     for booking in bookings:
         ResourceManager.getInstance().releaseResourceBundle(booking.resource)
+        booking.complete = True
+        print("Booking", booking.id, "is now completed")
+        booking.save()
 
 
 @shared_task
diff --git a/src/resource_inventory/migrations/0020_cloudinitfile.py b/src/resource_inventory/migrations/0020_cloudinitfile.py
new file mode 100644
index 0000000..198181c
--- /dev/null
+++ b/src/resource_inventory/migrations/0020_cloudinitfile.py
@@ -0,0 +1,21 @@
+# Generated by Django 2.2 on 2021-09-07 14:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('resource_inventory', '0019_auto_20210701_1947'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CloudInitFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('text', models.TextField()),
+                ('priority', models.IntegerField()),
+            ],
+        ),
+    ]
diff --git a/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py
new file mode 100644
index 0000000..6b0befc
--- /dev/null
+++ b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2 on 2021-09-10 18:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('resource_inventory', '0020_cloudinitfile'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='resourceconfiguration',
+            name='cloud_init_files',
+            field=models.ManyToManyField(blank=True, to='resource_inventory.CloudInitFile'),
+        ),
+    ]
diff --git a/src/resource_inventory/models.py b/src/resource_inventory/models.py
index fb4dad5..2c631dc 100644
--- a/src/resource_inventory/models.py
+++ b/src/resource_inventory/models.py
@@ -15,6 +15,7 @@ from django.db import models
 from django.db.models import Q
 import traceback
 import json
+import yaml
 
 import re
 from collections import Counter
@@ -152,6 +153,26 @@ with varying degrees of abstraction.
 """
 
 
+class CloudInitFile(models.Model):
+    text = models.TextField()
+
+    # higher priority is applied later, so "on top" of existing files
+    priority = models.IntegerField()
+
+    @classmethod
+    def merge_strategy(cls):
+        return [
+                { 'name': 'list', 'settings': ['append'] },
+                { 'name': 'dict', 'settings': ['recurse_list', 'replace'] },
+                ]
+
+    @classmethod
+    def create(cls, text="", priority=0):
+        prepended_text = "#cloud-config\n"
+        prepended_text = prepended_text + yaml.dump(CloudInitFile.merge_strategy()) + "\n"
+        print("in cloudinitfile create")
+        return CloudInitFile.objects.create(priority=priority, text=(prepended_text + text))
+
 class ResourceTemplate(models.Model):
     """
     Models a "template" of a complete, configured collection of resources that can be booked.
@@ -240,9 +261,14 @@ class ResourceConfiguration(models.Model):
     is_head_node = models.BooleanField(default=False)
     name = models.CharField(max_length=3000, default="opnfv_host")
 
+    cloud_init_files = models.ManyToManyField(CloudInitFile, blank=True)
+
     def __str__(self):
         return str(self.name)
 
+    def ci_file_list(self):
+        return list(self.cloud_init_files.order_by("priority").all())
+
 
 def get_default_remote_info():
     return RemoteInfo.objects.get_or_create(
@@ -578,6 +604,12 @@ class NetworkRole(models.Model):
     network = models.ForeignKey(Network, on_delete=models.CASCADE)
 
 
+def create_resource_ref_string(for_hosts: [str]) -> str:
+    # need to sort the list, then do dump
+    for_hosts.sort()
+
+    return json.dumps(for_hosts)
+
 class OPNFVConfig(models.Model):
     id = models.AutoField(primary_key=True)
     installer = models.ForeignKey(Installer, on_delete=models.CASCADE)
diff --git a/src/templates/base/booking/booking_detail.html b/src/templates/base/booking/booking_detail.html
index a014fea..4a8f35a 100644
--- a/src/templates/base/booking/booking_detail.html
+++ b/src/templates/base/booking/booking_detail.html
@@ -7,6 +7,12 @@
     <script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js?lang=yaml"></script>
 {% endblock %}
 
+<style>
+code {
+    overflow: scroll;
+}
+</style>
+
 {% block content %}
 <div class="row">
     <div class="col-12 col-lg-5">
@@ -154,6 +160,65 @@
                 </div>
             </div>
         </div>
+        <div class="card my-3">
+            <div class="card-header d-flex">
+                <h4 class="d-inline">Diagnostic Information</h4>
+                <button data-toggle="collapse" data-target="#diagnostics_panel" class="btn btn-outline-secondary ml-auto">Expand</button>
+            </div>
+            <div class="collapse" id="diagnostics_panel">
+                <div class="card-body">
+                    <table class="table m-0">
+                        <tr>
+                            <th>Job ID: </th>
+                            <td>{{booking.job.id}}</td>
+                        </tr>
+                        <tr>
+                            <th>CI Files</th>
+                        </tr>
+                        {% for host in booking.resource.get_resources %}
+                        <tr>
+                            <td>
+                                <table class="table m-0">
+                                    <tr>
+                                        <th>Host:</th>
+                                        <td>{{host.name}}</td>
+                                    </tr>
+                                    <tr>
+                                        <th>Configs:</th>
+                                    </tr>
+                                    {% for ci_file in host.config.cloud_init_files.all %}
+                                    <tr>
+                                        <td>{{ci_file.id}}</td>
+                                        <td>
+                                            <div class="modal fade" id="ci_file_modal_{{ci_file.id}}" tabindex="-1" role="dialog" aria-hidden="true">
+                                                <div class="modal-dialog modal-xl" role="document">
+                                                    <div class="modal-content">
+                                                        <div class="modal-header">
+                                                            <h4 class="modal-title d-inline float-left">Cloud Config Content</h4>
+                                                            <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+                                                                <span aria-hidden="true">&times;</span>
+                                                            </button>
+                                                        </div>
+                                                        <div class="card-body">
+                                                            <pre class="prettyprint lang-yaml m-0 border-0 text-break pre-wrap">
+{{ci_file.text}}
+                                                            </pre>
+                                                        </div>
+                                                    </div>
+                                                </div>
+                                            </div>
+                                            <button class="btn btn-primary" data-toggle="modal" data-target="#ci_file_modal_{{ci_file.id}}">Show File Content</button>
+                                        </td>
+                                    </tr>
+                                    {% endfor %}
+                                </table>
+                            </td>
+                        </tr>
+                        {% endfor %}
+                    </table>
+                </div>
+            </div>
+        </div>
     </div>
     <div class="col">
         <div class="card mb-3">
diff --git a/src/templates/base/booking/quick_deploy.html b/src/templates/base/booking/quick_deploy.html
index 1193aab..c51e234 100644
--- a/src/templates/base/booking/quick_deploy.html
+++ b/src/templates/base/booking/quick_deploy.html
@@ -3,6 +3,13 @@
 {% load bootstrap4 %}
 {% block content %}
 
+<style>
+/* hides images not in use. Not applied globally since doesn't make sense in all cases */
+select option:disabled {
+    display:none;
+}
+</style>
+
 {% bootstrap_form_errors form type='non_fields' %}
 <form id="quick_booking_form" action="/booking/quick/" method="POST" class="form  class="Anuket-Text"">
     {% csrf_token %}
@@ -18,7 +25,7 @@
             </div>
         </div>
         <div class="row justify-content-center">
-            <div class="col-12 col-lg-4 my-2">
+            <div class="col-12 col-lg-6 my-2">
                 <div class="col border rounded py-2 h-100">
                     {% bootstrap_field form.purpose %}
                     {% bootstrap_field form.project %}
@@ -31,19 +38,26 @@
                 </div>
             </div>
             {% block collab %}
-            <div class="col-12 col-lg-4 my-2">
+            <div class="col-12 col-lg-6 my-2">
                 <div class="col border rounded py-2 h-100">
                     <label>Collaborators</label>
                     {{ form.users }}
                 </div>
             </div>
             {% endblock collab %}
-            <div class="col-12 col-lg-4 my-2">
+        </div>
+        <div class="row justify-content-center">
+            <div class="col-12 col-lg-6 my-2">
                 <div class="col border rounded py-2 h-100">
                     {% bootstrap_field form.hostname %}
                     {% bootstrap_field form.image %}
                 </div>
             </div>
+            <div class="col-12 col-lg-6 my-2">
+                <div class="col border rounded py-2 h-100">
+                    {% bootstrap_field form.global_cloud_config %}
+                </div>
+            </div>
             <div class="col-12 d-flex mt-2 justify-content-end">
                 <button id="quick_booking_confirm" onclick="submit_form();" type="button" class="btn btn-success">Confirm</button>
             </div>