aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSawyer Bergeron <sbergeron@iol.unh.edu>2021-10-29 15:11:29 -0400
committerSawyer Bergeron <sbergeron@iol.unh.edu>2021-11-01 18:07:49 -0400
commit23d35dc2c56b8c2b5496b6f0a5fc62066b22bbc7 (patch)
treec8eca16091ce1646d088bff54345c728f3726041
parent35b9f39178cc502a5283a1b37a65f7dd0838ae05 (diff)
Add Cloud Init Support
Squashed commit of the following: commit afcee3cad5c091e78e909b83f8df49accf1af5b6 Author: Sawyer Bergeron <sbergeron@iol.unh.edu> Date: Mon Oct 11 22:02:16 2021 +0000 Prod cobbler hotfixes Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Change-Id: I092bc6d85a3b2c77bfbe24f3af0d2b7a5f75a8c3 commit 5ce0a52b17e530436c298e1b581d37bac853f5a7 Author: Sawyer Bergeron <sbergeron@iol.unh.edu> Date: Thu Oct 7 17:14:01 2021 -0400 Manually merge CI files Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Change-Id: Ic63d5da699578007ef2f2cc373350ded06c66971 commit 5b70b8f1b8bbbe6aeec43b8d8dfdc6b7cc68bc9c Author: Sawyer Bergeron <sbergeron@iol.unh.edu> Date: Thu Sep 30 16:33:01 2021 -0400 Fixes for collaborator field Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Change-Id: I3dbdedf26fa84617ea7680a0f99e032d88f1ea98 Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> commit 529b2521627b17142284c55c744812129edc71e8 Merge: d555513 e9d72ce Author: Sawyer Bergeron <sbergeron@iol.unh.edu> Date: Thu Sep 30 14:03:55 2021 +0000 Merge "Push cloud config content for generated files into userdata_raw" into cobbler commit d55551394df73645e49ae2ae3e730a9f1c6af81d Author: Sawyer Bergeron <sbergeron@iol.unh.edu> Date: Thu Sep 30 10:02:32 2021 -0400 Better error handling for quick deploy Change-Id: I03a725dfee9ce2f119d72ef940cd08df5aee3dcc Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> commit e9d72ce78a85c6ff2f3f8591bcbf4115f97318d5 Author: Sawyer Bergeron <sbergeron@iol.unh.edu> Date: Tue Sep 28 19:11:49 2021 -0400 Push cloud config content for generated files into userdata_raw Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Change-Id: Ieb8bd9b8b172b6bf11062f67f41fc78154cc7c89 commit 95d39c60f7e8062cabc8c1665080a2d2c8904234 Author: Sawyer Bergeron <sbergeron@iol.unh.edu> Date: Sat Sep 25 16:18:12 2021 -0400 Allow for "pod specific" vlan allocation for LFEDGE allocation case Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Change-Id: I8b75410145027f43eaf6de7bd5f1813af38d3e7f Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> commit 2ebb82b5f344de1e17abd70c51c4cce765761dd1 Author: Sawyer Bergeron <sbergeron@iol.unh.edu> Date: Thu Sep 23 16:37:43 2021 -0400 Fix collaborator field with recent changes Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Change-Id: Id305de9b1567adf103c47d5180b0b28ebfdf1b5e commit a819fc1df86721eda36eee89d0235c89b3159d6b Author: Sawyer Bergeron <sbergeron@iol.unh.edu> Date: Tue Sep 7 11:28:35 2021 -0400 Add user specified CI file entry Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Change-Id: Ia920130612da8fcde9d1a0d5dde7861904857162 Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> commit d93346a716bde5237b7cfef5c10ea56e4922b59a Author: Adam Hassick <ahassick@iol.unh.edu> Date: Tue Jul 27 13:05:16 2021 +0000 Make C-I serialization work with current netconf rules Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Change-Id: If967e5e1f268c5bee3ad4496847662cf4de1187c Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> commit 6ffb1fdf6ce7825770148bada5a4c54899e4ed36 Author: Adam Hassick <ahassick@iol.unh.edu> Date: Tue Jun 29 16:49:27 2021 -0400 Cobbler model changes, new endpoints Signed-off-by: Adam Hassick <ahassick@iol.unh.edu> Change-Id: If0a94730e92747127cef121ec4930a4c8bae6c92 Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Signed-off-by: Adam Hassick <ahassick@iol.unh.edu> commit 49e2b407003b69551ddafa851639e83ec42a5b09 Author: Jacob Hodgdon <jhodgdon@iol.unh.edu> Date: Fri May 14 15:42:56 2021 -0400 Color fixes for rebrand Signed-off-by: Jacob Hodgdon <jhodgdon@iol.unh.edu> Change-Id: I5cf4ede598afa377db7ecec17d8dfef085e130ac commit a908da441bf6efcdb289a46d0c2761840138b1a5 Author: Sawyer Bergeron <sbergeron@iol.unh.edu> Date: Tue Jun 8 11:15:56 2021 -0400 Draft for cloud-init file generation Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Change-Id: I07f3a4a1ab67531cba2cc7e3de22e9bb860706e1 Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Change-Id: I392505174cbc07214c31c42aab2474a748e47913 Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu>
-rw-r--r--docker-compose.override-dev.yml1
-rw-r--r--requirements.txt7
-rw-r--r--src/account/models.py23
-rw-r--r--src/api/migrations/0017_auto_20210630_1629.py18
-rw-r--r--src/api/migrations/0018_cloudinitfile.py25
-rw-r--r--src/api/migrations/0019_auto_20210907_1448.py29
-rw-r--r--src/api/models.py188
-rw-r--r--src/api/urls.py22
-rw-r--r--src/api/views.py177
-rw-r--r--src/booking/forms.py1
-rw-r--r--src/booking/lib.py4
-rw-r--r--src/booking/migrations/0009_booking_complete.py18
-rw-r--r--src/booking/models.py2
-rw-r--r--src/booking/quick_deployer.py75
-rw-r--r--src/booking/views.py6
-rw-r--r--src/dashboard/tasks.py6
-rw-r--r--src/dashboard/utils.py10
-rw-r--r--src/resource_inventory/migrations/0018_auto_20210630_1629.py101
-rw-r--r--src/resource_inventory/migrations/0019_auto_20210701_1947.py43
-rw-r--r--src/resource_inventory/migrations/0020_cloudinitfile.py21
-rw-r--r--src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py18
-rw-r--r--src/resource_inventory/migrations/0022_auto_20210925_2028.py23
-rw-r--r--src/resource_inventory/models.py131
-rw-r--r--src/resource_inventory/resource_manager.py8
-rw-r--r--src/resource_inventory/tests/test_models.py2
-rw-r--r--src/templates/base/base.html6
-rw-r--r--src/templates/base/booking/booking_detail.html65
-rw-r--r--src/templates/base/booking/quick_deploy.html33
-rw-r--r--src/templates/laas/base.html77
-rw-r--r--src/templates/lfedge/booking/quick_deploy.html8
-rw-r--r--src/workflow/resource_bundle_workflow.py2
31 files changed, 1076 insertions, 74 deletions
diff --git a/docker-compose.override-dev.yml b/docker-compose.override-dev.yml
index ee0b7a1..4d42569 100644
--- a/docker-compose.override-dev.yml
+++ b/docker-compose.override-dev.yml
@@ -20,7 +20,6 @@ services:
dockerfile: web/Dockerfile
command: >
sh -c "cd static && npm install && cd .. &&
- ./manage.py migrate &&
./manage.py runserver 0:8000"
volumes:
- ./src:/laas_dashboard
diff --git a/requirements.txt b/requirements.txt
index 72afbfa..e4650ea 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,6 +12,7 @@ psycopg2==2.8.6
PyJWT==2.1.0
requests==2.26.0
django-fernet-fields==0.6
-pyyaml==5.4.1
-pytz==2021.1
-mozilla-django-oidc==2.0.0 \ No newline at end of file
+pyyaml==3.13
+pytz==2018.5
+mozilla-django-oidc==1.2.3
+deepmerge==0.3
diff --git a/src/account/models.py b/src/account/models.py
index 210025e..32229b1 100644
--- a/src/account/models.py
+++ b/src/account/models.py
@@ -83,12 +83,14 @@ class VlanManager(models.Model):
# if they use QinQ or a vxlan overlay, for example
allow_overlapping = models.BooleanField()
- def get_vlans(self, count=1):
+ def get_vlans(self, count=1, within=None):
"""
Return the IDs of available vlans as a list[int], but does not reserve them.
Will throw index exception if not enough vlans are available.
Always returns a list of ints
+
+ If `within` is not none, will filter against that as a set, requiring that any vlans returned are within that set
"""
allocated = []
vlans = json.loads(self.vlans)
@@ -105,17 +107,28 @@ class VlanManager(models.Model):
continue
# vlan is available and not reserved, so safe to add
- allocated.append(i)
+ if within is not None:
+ if i in within:
+ allocated.append(i)
+ else:
+ allocated.append(i)
continue
if len(allocated) != count:
- raise ResourceAvailabilityException("can't allocate the vlans requested")
+ raise ResourceAvailabilityException("There were not enough available private vlans for the allocation. Please contact the administrators.")
return allocated
- def get_public_vlan(self):
+ def get_public_vlan(self, within=None):
"""Return reference to an available public network without reserving it."""
- return PublicNetwork.objects.filter(lab=self.lab_set.first(), in_use=False).first()
+ r = PublicNetwork.objects.filter(lab=self.lab_set.first(), in_use=False)
+ if within is not None:
+ r = r.filter(vlan__in=within)
+
+ if r.count() < 1:
+ raise ResourceAvailabilityException("There were not enough available public vlans for the allocation. Please contact the administrators.")
+
+ return r.first()
def reserve_public_vlan(self, vlan):
"""Reserves the Public Network that has the given vlan."""
diff --git a/src/api/migrations/0017_auto_20210630_1629.py b/src/api/migrations/0017_auto_20210630_1629.py
new file mode 100644
index 0000000..643ff5f
--- /dev/null
+++ b/src/api/migrations/0017_auto_20210630_1629.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2 on 2021-06-30 16:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('api', '0016_auto_20201109_2149'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='snapshotconfig',
+ name='image',
+ field=models.CharField(max_length=200, null=True),
+ ),
+ ]
diff --git a/src/api/migrations/0018_cloudinitfile.py b/src/api/migrations/0018_cloudinitfile.py
new file mode 100644
index 0000000..4e41b39
--- /dev/null
+++ b/src/api/migrations/0018_cloudinitfile.py
@@ -0,0 +1,25 @@
+# Generated by Django 2.2 on 2021-07-01 20:45
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('resource_inventory', '0019_auto_20210701_1947'),
+ ('booking', '0008_auto_20201109_1947'),
+ ('api', '0017_auto_20210630_1629'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='CloudInitFile',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('resource_id', models.CharField(max_length=200)),
+ ('booking', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='booking.Booking')),
+ ('rconfig', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='resource_inventory.ResourceConfiguration')),
+ ],
+ ),
+ ]
diff --git a/src/api/migrations/0019_auto_20210907_1448.py b/src/api/migrations/0019_auto_20210907_1448.py
new file mode 100644
index 0000000..92140fb
--- /dev/null
+++ b/src/api/migrations/0019_auto_20210907_1448.py
@@ -0,0 +1,29 @@
+# Generated by Django 2.2 on 2021-09-07 14:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('booking', '0008_auto_20201109_1947'),
+ ('resource_inventory', '0020_cloudinitfile'),
+ ('api', '0018_cloudinitfile'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='GeneratedCloudConfig',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('resource_id', models.CharField(max_length=200)),
+ ('text', models.TextField(blank=True, null=True)),
+ ('booking', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='booking.Booking')),
+ ('rconfig', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='resource_inventory.ResourceConfiguration')),
+ ],
+ ),
+ migrations.DeleteModel(
+ name='CloudInitFile',
+ ),
+ ]
diff --git a/src/api/models.py b/src/api/models.py
index d85f3e9..5928ea9 100644
--- a/src/api/models.py
+++ b/src/api/models.py
@@ -19,18 +19,22 @@ from django.utils import timezone
import json
import uuid
+import yaml
from booking.models import Booking
from resource_inventory.models import (
Lab,
ResourceProfile,
Image,
+ Opsys,
Interface,
ResourceOPNFVConfig,
RemoteInfo,
OPNFVConfig,
ConfigState,
- ResourceQuery
+ ResourceQuery,
+ ResourceConfiguration,
+ CloudInitFile
)
from resource_inventory.idf_templater import IDFTemplater
from resource_inventory.pdf_templater import PDFTemplater
@@ -84,6 +88,18 @@ class LabManager:
def __init__(self, lab):
self.lab = lab
+ def get_opsyss(self):
+ return Opsys.objects.filter(from_lab=self.lab)
+
+ def get_images(self):
+ return Image.objects.filter(from_lab=self.lab)
+
+ def get_image(self, image_id):
+ return Image.objects.filter(from_lab=self.lab, lab_id=image_id)
+
+ def get_opsys(self, opsys_id):
+ return Opsys.objects.filter(from_lab=self.lab, lab_id=opsys_id)
+
def get_downtime(self):
return Downtime.objects.filter(start__lt=timezone.now(), end__gt=timezone.now(), lab=self.lab)
@@ -338,6 +354,157 @@ class LabManager:
return profile_ser
+class GeneratedCloudConfig(models.Model):
+ resource_id = models.CharField(max_length=200)
+ booking = models.ForeignKey(Booking, on_delete=models.CASCADE)
+ rconfig = models.ForeignKey(ResourceConfiguration, on_delete=models.CASCADE)
+ text = models.TextField(null=True, blank=True)
+
+ def _normalize_username(self, username: str) -> str:
+ # TODO: make usernames posix compliant
+ return username
+
+ def _get_ssh_string(self, username: str) -> str:
+ user = User.objects.get(username=username)
+ uprofile = user.userprofile
+
+ ssh_file = uprofile.ssh_public_key
+
+ escaped_file = ssh_file.open().read().decode(encoding="UTF-8").replace("\n", " ")
+
+ return escaped_file
+
+ def _serialize_users(self):
+ """
+ returns the dictionary to be placed behind the `users` field of the toplevel c-i dict
+ """
+ # conserves distro default user
+ user_array = ["default"]
+
+ users = list(self.booking.collaborators.all())
+ users.append(self.booking.owner)
+ for collaborator in users:
+ userdict = {}
+
+ # TODO: validate if usernames are valid as linux usernames (and provide an override potentially)
+ userdict['name'] = self._normalize_username(collaborator.username)
+
+ userdict['groups'] = "sudo"
+ userdict['sudo'] = "ALL=(ALL) NOPASSWD:ALL"
+
+ userdict['ssh_authorized_keys'] = [self._get_ssh_string(collaborator.username)]
+
+ user_array.append(userdict)
+
+ # user_array.append({
+ # "name": "opnfv",
+ # "passwd": "$6$k54L.vim1cLaEc4$5AyUIrufGlbtVBzuCWOlA1yV6QdD7Gr2MzwIs/WhuYR9ebSfh3Qlb7djkqzjwjxpnSAonK1YOabPP6NxUDccu.",
+ # "ssh_redirect_user": True,
+ # "sudo": "ALL=(ALL) NOPASSWD:ALL",
+ # "groups": "sudo",
+ # })
+
+ return user_array
+
+ # TODO: make this configurable
+ def _serialize_sysinfo(self):
+ defuser = {}
+ defuser['name'] = 'opnfv'
+ defuser['plain_text_passwd'] = 'OPNFV_HOST'
+ defuser['home'] = '/home/opnfv'
+ defuser['shell'] = '/bin/bash'
+ defuser['lock_passwd'] = True
+ defuser['gecos'] = 'Lab Manager User'
+ defuser['groups'] = 'sudo'
+
+ return {'default_user': defuser}
+
+ # TODO: make this configurable
+ def _serialize_runcmds(self):
+ cmdlist = []
+
+ # have hosts run dhcp on boot
+ cmdlist.append(['sudo', 'dhclient', '-r'])
+ cmdlist.append(['sudo', 'dhclient'])
+
+ return cmdlist
+
+ def _serialize_netconf_v1(self):
+ # interfaces = {} # map from iface_name => dhcp_config
+ # vlans = {} # map from vlan_id => dhcp_config
+
+ config_arr = []
+
+ for interface in self._resource().interfaces.all():
+ interface_name = interface.profile.name
+ interface_mac = interface.mac_address
+
+ iface_dict_entry = {
+ "type": "physical",
+ "name": interface_name,
+ "mac_address": interface_mac,
+ }
+
+ for vlan in interface.config.all():
+ if vlan.tagged:
+ vlan_dict_entry = {'type': 'vlan'}
+ vlan_dict_entry['name'] = str(interface_name) + "." + str(vlan.vlan_id)
+ vlan_dict_entry['vlan_link'] = str(interface_name)
+ vlan_dict_entry['vlan_id'] = int(vlan.vlan_id)
+ vlan_dict_entry['mac_address'] = str(interface_mac)
+ if vlan.public:
+ vlan_dict_entry["subnets"] = [{"type": "dhcp"}]
+ config_arr.append(vlan_dict_entry)
+ if (not vlan.tagged) and vlan.public:
+ iface_dict_entry["subnets"] = [{"type": "dhcp"}]
+
+ # vlan_dict_entry['mtu'] = # TODO, determine override MTU if needed
+
+ config_arr.append(iface_dict_entry)
+
+ ns_dict = {
+ 'type': 'nameserver',
+ 'address': ['10.64.0.1', '8.8.8.8']
+ }
+
+ config_arr.append(ns_dict)
+
+ full_dict = {'version': 1, 'config': config_arr}
+
+ return full_dict
+
+ @classmethod
+ def get(cls, booking_id: int, resource_lab_id: str, file_id: int):
+ return GeneratedCloudConfig.objects.get(resource_id=resource_lab_id, booking__id=booking_id, file_id=file_id)
+
+ def _resource(self):
+ return ResourceQuery.get(labid=self.resource_id, lab=self.booking.lab)
+
+ # def _get_facts(self):
+ # resource = self._resource()
+
+ # hostname = self.rconfig.name
+ # iface_configs = for_config.interface_configs.all()
+
+ def _to_dict(self):
+ main_dict = {}
+
+ main_dict['users'] = self._serialize_users()
+ main_dict['network'] = self._serialize_netconf_v1()
+ main_dict['hostname'] = self.rconfig.name
+
+ # add first startup commands
+ main_dict['runcmd'] = self._serialize_runcmds()
+
+ # configure distro default user
+ main_dict['system_info'] = self._serialize_sysinfo()
+
+ return main_dict
+
+ def serialize(self) -> str:
+ return yaml.dump(self._to_dict())
+
+
class APILog(models.Model):
user = models.ForeignKey(User, on_delete=models.PROTECT)
call_time = models.DateTimeField(auto_now=True)
@@ -761,6 +928,7 @@ class HardwareConfig(TaskConfig):
return self.get_delta()
def get_delta(self):
+ # TODO: grab the GeneratedCloudConfig urls from self.hosthardwarerelation.get_resource()
return self.format_delta(
self.hosthardwarerelation.get_resource().get_configuration(self.state),
self.hosthardwarerelation.lab_token)
@@ -813,7 +981,7 @@ class NetworkConfig(TaskConfig):
class SnapshotConfig(TaskConfig):
resource_id = models.CharField(max_length=200, default="default_id")
- image = models.IntegerField(null=True)
+ image = models.CharField(max_length=200, null=True) # cobbler ID
dashboard_id = models.IntegerField()
delta = models.TextField(default="{}")
@@ -1104,6 +1272,10 @@ class JobFactory(object):
booking=booking,
job=job
)
+ cls.makeGeneratedCloudConfigs(
+ resources=resources,
+ job=job
+ )
all_users = list(booking.collaborators.all())
all_users.append(booking.owner)
cls.makeAccessConfig(
@@ -1128,6 +1300,18 @@ class JobFactory(object):
continue
@classmethod
+ def makeGeneratedCloudConfigs(cls, resources=[], job=Job()):
+ for res in resources:
+ cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=job.booking, rconfig=res.config)
+ cif.save()
+
+ cif = CloudInitFile.create(priority=0, text=cif.serialize())
+ cif.save()
+
+ res.config.cloud_init_files.add(cif)
+ res.config.save()
+
+ @classmethod
def makeHardwareConfigs(cls, resources=[], job=Job()):
"""
Create and save HardwareConfig.
diff --git a/src/api/urls.py b/src/api/urls.py
index 52a6fc7..3693979 100644
--- a/src/api/urls.py
+++ b/src/api/urls.py
@@ -46,17 +46,28 @@ from api.views import (
lab_user,
GenerateTokenView,
analytics_job,
+ resource_ci_metadata,
+ resource_ci_userdata,
+ resource_ci_userdata_directory,
+ all_images,
+ all_opsyss,
+ single_image,
+ single_opsys,
user_bookings,
- make_booking,
- available_templates,
- images_for_template,
specific_booking,
extend_booking,
+ make_booking,
+ list_labs,
all_users,
- list_labs
+ images_for_template,
+ available_templates,
)
urlpatterns = [
+ path('labs/<slug:lab_name>/opsys/<slug:opsys_id>', single_opsys),
+ path('labs/<slug:lab_name>/image/<slug:image_id>', single_image),
+ path('labs/<slug:lab_name>/opsys', all_opsyss),
+ path('labs/<slug:lab_name>/image', all_images),
path('labs/<slug:lab_name>/profile', lab_profile),
path('labs/<slug:lab_name>/status', lab_status),
path('labs/<slug:lab_name>/inventory', lab_inventory),
@@ -67,6 +78,9 @@ urlpatterns = [
path('labs/<slug:lab_name>/booking/<int:booking_id>/idf', get_idf, name="get-idf"),
path('labs/<slug:lab_name>/jobs/<int:job_id>', specific_job),
path('labs/<slug:lab_name>/jobs/<int:job_id>/<slug:task_id>', specific_task),
+ path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>/user-data', resource_ci_userdata_directory, name="specific-user-data"),
+ path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>/meta-data', resource_ci_metadata, name="specific-meta-data"),
+ path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>/<int:file_id>/user-data', resource_ci_userdata, name="user-data-dir"),
path('labs/<slug:lab_name>/jobs/new', new_jobs),
path('labs/<slug:lab_name>/jobs/current', current_jobs),
path('labs/<slug:lab_name>/jobs/done', done_jobs),
diff --git a/src/api/views.py b/src/api/views.py
index c0da1bc..84d19cc 100644
--- a/src/api/views.py
+++ b/src/api/views.py
@@ -19,24 +19,34 @@ from django.shortcuts import redirect, get_object_or_404
from django.utils.decorators import method_decorator
from django.utils import timezone
from django.views import View
+from django.http import HttpResponseNotFound
from django.http.response import JsonResponse, HttpResponse
from rest_framework import viewsets
from rest_framework.authtoken.models import Token
from django.views.decorators.csrf import csrf_exempt
from django.core.exceptions import ObjectDoesNotExist
+from django.db.models import Q
from api.serializers.booking_serializer import BookingSerializer
from api.serializers.old_serializers import UserSerializer
from api.forms import DowntimeForm
from account.models import UserProfile, Lab
from booking.models import Booking
-from api.models import LabManagerTracker, AutomationAPIManager, get_task, APILog
+from booking.quick_deployer import create_from_API
+from api.models import LabManagerTracker, get_task, Job, AutomationAPIManager, APILog
from notifier.manager import NotificationHandler
from analytics.models import ActiveVPNUser
-from booking.quick_deployer import create_from_API
-from resource_inventory.models import ResourceTemplate
-from django.db.models import Q
-
+from resource_inventory.models import (
+ Image,
+ Opsys,
+ CloudInitFile,
+ ResourceQuery,
+ ResourceTemplate,
+)
+
+import yaml
+import uuid
+from deepmerge import Merger
"""
API views.
@@ -88,6 +98,83 @@ def lab_host(request, lab_name="", host_id=""):
if request.method == "POST":
return JsonResponse(lab_manager.update_host(host_id, request.POST), safe=False)
+# API extension for Cobbler integration
+
+
+def all_images(request, lab_name=""):
+ a = []
+ for i in Image.objects.all():
+ a.append(i.serialize())
+ return JsonResponse(a, safe=False)
+
+
+def all_opsyss(request, lab_name=""):
+ a = []
+ for opsys in Opsys.objects.all():
+ a.append(opsys.serialize())
+
+ return JsonResponse(a, safe=False)
+
+
+@csrf_exempt
+def single_image(request, lab_name="", image_id=""):
+ lab_token = request.META.get('HTTP_AUTH_TOKEN')
+ lab_manager = LabManagerTracker.get(lab_name, lab_token)
+ img = lab_manager.get_image(image_id).first()
+
+ if request.method == "GET":
+ if not img:
+ return HttpResponse(status=404)
+ return JsonResponse(img.serialize(), safe=False)
+
+ if request.method == "POST":
+ # get POST data
+ data = json.loads(request.body.decode('utf-8'))
+ if img:
+ img.update(data)
+ else:
+ # append lab name and the ID from the URL
+ data['from_lab_id'] = lab_name
+ data['lab_id'] = image_id
+
+ # create and save a new Image object
+ img = Image.new_from_data(data)
+
+ img.save()
+
+ # indicate success in response
+ return HttpResponse(status=200)
+ return HttpResponse(status=405)
+
+
+@csrf_exempt
+def single_opsys(request, lab_name="", opsys_id=""):
+ lab_token = request.META.get('HTTP_AUTH_TOKEN')
+ lab_manager = LabManagerTracker.get(lab_name, lab_token)
+ opsys = lab_manager.get_opsys(opsys_id).first()
+
+ if request.method == "GET":
+ if not opsys:
+ return HttpResponse(status=404)
+ return JsonResponse(opsys.serialize(), safe=False)
+
+ if request.method == "POST":
+ data = json.loads(request.body.decode('utf-8'))
+ if opsys:
+ opsys.update(data)
+ else:
+ # only name, available, and obsolete are needed to create an Opsys
+ # other fields are derived from the URL parameters
+ data['from_lab_id'] = lab_name
+ data['lab_id'] = opsys_id
+ opsys = Opsys.new_from_data(data)
+
+ opsys.save()
+ return HttpResponse(status=200)
+ return HttpResponse(status=405)
+
+# end API extension
+
def get_pdf(request, lab_name="", booking_id=""):
lab_token = request.META.get('HTTP_AUTH_TOKEN')
@@ -175,6 +262,86 @@ def specific_job(request, lab_name="", job_id=""):
return JsonResponse(lab_manager.get_job(job_id), safe=False)
+@csrf_exempt
+def resource_ci_userdata(request, lab_name="", job_id="", resource_id="", file_id=0):
+ # lab_token = request.META.get('HTTP_AUTH_TOKEN')
+ # lab_manager = LabManagerTracker.get(lab_name, lab_token)
+
+ # job = lab_manager.get_job(job_id)
+ Job.objects.get(id=job_id) # verify a valid job was given, even if we don't use it
+
+ cifile = None
+ try:
+ cifile = CloudInitFile.objects.get(id=file_id)
+ except ObjectDoesNotExist:
+ return HttpResponseNotFound("Could not find a matching resource by id " + str(resource_id))
+
+ text = cifile.text
+
+ prepended_text = "#cloud-config\n"
+ # mstrat = CloudInitFile.merge_strategy()
+ # prepended_text = prepended_text + yaml.dump({"merge_strategy": mstrat}) + "\n"
+ # print("in cloudinitfile create")
+ text = prepended_text + text
+ cloud_dict = {
+ "datasource": {
+ "None": {
+ "metadata": {
+ "instance-id": str(uuid.uuid4())
+ },
+ "userdata_raw": text,
+ },
+ },
+ "datasource_list": ["None"],
+ }
+
+ return HttpResponse(yaml.dump(cloud_dict), status=200)
+
+
+@csrf_exempt
+def resource_ci_metadata(request, lab_name="", job_id="", resource_id="", file_id=0):
+ return HttpResponse("#cloud-config", status=200)
+
+
+@csrf_exempt
+def resource_ci_userdata_directory(request, lab_name="", job_id="", resource_id=""):
+ # files = [{"id": file.file_id, "priority": file.priority} for file in CloudInitFile.objects.filter(job__id=job_id, resource_id=resource_id).order_by("priority").all()]
+ resource = ResourceQuery.get(labid=resource_id, lab=Lab.objects.get(name=lab_name))
+ files = resource.config.cloud_init_files
+ files = [{"id": file.id, "priority": file.priority} for file in files.order_by("priority").all()]
+
+ d = {
+ 'merge_failures': []
+ }
+
+ merger = Merger(
+ [
+ (list, ["append"]),
+ (dict, ["merge"]),
+ ],
+ ["override"], # fallback
+ ["override"], # if types conflict (shouldn't happen in CI, but handle case)
+ )
+
+ for f in resource.config.cloud_init_files.order_by("priority").all():
+ try:
+ other_dict = yaml.load(f.text)
+ if not (type(d) is dict):
+ raise Exception("CI file was valid yaml but was not a dict")
+
+ merger.merge(d, other_dict)
+ except Exception as e:
+ # if fail to merge, then just skip
+ print("Failed to merge file in, as it had invalid content:", f.id)
+ print("File text was:")
+ print(f.text)
+ d['merge_failures'].append({f.id: str(e)})
+
+ file = CloudInitFile.create(text=yaml.dump(d), priority=0)
+
+ return HttpResponse(json.dumps([{"id": file.id, "priority": file.priority}]), status=200)
+
+
def new_jobs(request, lab_name=""):
lab_token = request.META.get('HTTP_AUTH_TOKEN')
lab_manager = LabManagerTracker.get(lab_name, lab_token)
diff --git a/src/booking/forms.py b/src/booking/forms.py
index cbc3407..ff829b2 100644
--- a/src/booking/forms.py
+++ b/src/booking/forms.py
@@ -22,6 +22,7 @@ class QuickBookingForm(forms.Form):
purpose = forms.CharField(max_length=1000)
project = forms.CharField(max_length=400)
hostname = forms.CharField(required=False, max_length=400)
+ global_cloud_config = forms.CharField(widget=forms.Textarea, required=False)
installer = forms.ModelChoiceField(queryset=Installer.objects.all(), required=False)
scenario = forms.ModelChoiceField(queryset=Scenario.objects.all(), required=False)
diff --git a/src/booking/lib.py b/src/booking/lib.py
index 7a4c261..8c87979 100644
--- a/src/booking/lib.py
+++ b/src/booking/lib.py
@@ -28,9 +28,9 @@ def get_user_items(exclude=None):
for up in qs:
item = {
'id': up.id,
- 'expanded_name': up.full_name,
+ 'expanded_name': up.full_name if up.full_name else up.user.username,
'small_name': up.user.username,
- 'string': up.email_addr
+ 'string': up.email_addr if up.email_addr else up.user.username,
}
items[up.id] = item
return items
diff --git a/src/booking/migrations/0009_booking_complete.py b/src/booking/migrations/0009_booking_complete.py
new file mode 100644
index 0000000..e291a83
--- /dev/null
+++ b/src/booking/migrations/0009_booking_complete.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2 on 2021-09-07 15:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('booking', '0008_auto_20201109_1947'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='booking',
+ name='complete',
+ field=models.BooleanField(default=False),
+ ),
+ ]
diff --git a/src/booking/models.py b/src/booking/models.py
index cfdf7bc..966f1c2 100644
--- a/src/booking/models.py
+++ b/src/booking/models.py
@@ -39,6 +39,8 @@ class Booking(models.Model):
pdf = models.TextField(blank=True, default="")
idf = models.TextField(blank=True, default="")
+ complete = models.BooleanField(default=False)
+
class Meta:
db_table = 'booking'
diff --git a/src/booking/quick_deployer.py b/src/booking/quick_deployer.py
index 5e5bc8b..31865be 100644
--- a/src/booking/quick_deployer.py
+++ b/src/booking/quick_deployer.py
@@ -9,6 +9,7 @@
import json
+import yaml
from django.db.models import Q
from django.db import transaction
from datetime import timedelta
@@ -18,7 +19,6 @@ from account.models import Lab, UserProfile
from resource_inventory.models import (
ResourceTemplate,
- Installer,
Image,
OPNFVRole,
OPNFVConfig,
@@ -27,6 +27,7 @@ from resource_inventory.models import (
NetworkConnection,
InterfaceConfiguration,
Network,
+ CloudInitFile,
)
from resource_inventory.resource_manager import ResourceManager
from resource_inventory.pdf_templater import PDFTemplater
@@ -61,7 +62,7 @@ def parse_resource_field(resource_json):
return lab, template
-def update_template(old_template, image, hostname, user):
+def update_template(old_template, image, hostname, user, global_cloud_config=None):
"""
Duplicate a template to the users account and update configured fields.
@@ -113,9 +114,17 @@ def update_template(old_template, image, hostname, user):
image=image_to_set,
template=template,
is_head_node=old_config.is_head_node,
- name=hostname if len(old_template.getConfigs()) == 1 else old_config.name
+ name=hostname if len(old_template.getConfigs()) == 1 else old_config.name,
+ # cloud_init_files=old_config.cloud_init_files.set()
)
+ for file in old_config.cloud_init_files.all():
+ config.cloud_init_files.add(file)
+
+ if global_cloud_config:
+ config.cloud_init_files.add(global_cloud_config)
+ config.save()
+
for old_iface_config in old_config.interface_configs.all():
iface_config = InterfaceConfiguration.objects.create(
profile=old_iface_config.profile,
@@ -170,20 +179,11 @@ def generate_resource_bundle(template):
def check_invariants(**kwargs):
# TODO: This should really happen in the BookingForm validation methods
- installer = kwargs['installer']
image = kwargs['image']
- scenario = kwargs['scenario']
lab = kwargs['lab']
length = kwargs['length']
# check that image os is compatible with installer
if image:
- if installer or scenario:
- if installer in image.os.sup_installers.all():
- # if installer not here, we can omit that and not check for scenario
- if not scenario:
- raise ValidationError("An OPNFV Installer needs a scenario to be chosen to work properly")
- if scenario not in installer.sup_scenarios.all():
- raise ValidationError("The chosen installer does not support the chosen scenario")
if image.from_lab != lab:
raise ValidationError("The chosen image is not available at the chosen hosting lab")
# TODO
@@ -200,9 +200,21 @@ def create_from_form(form, request):
Parse data from QuickBookingForm to create booking
"""
resource_field = form.cleaned_data['filter_field']
+ # users_field = form.cleaned_data['users']
+ hostname = 'opnfv_host' if not form.cleaned_data['hostname'] else form.cleaned_data['hostname']
+
+ global_cloud_config = None if not form.cleaned_data['global_cloud_config'] else form.cleaned_data['global_cloud_config']
+
+ if global_cloud_config:
+ form.cleaned_data['global_cloud_config'] = create_ci_file(global_cloud_config)
+
+ # image = form.cleaned_data['image']
+ # scenario = form.cleaned_data['scenario']
+ # installer = form.cleaned_data['installer']
lab, resource_template = parse_resource_field(resource_field)
data = form.cleaned_data
+ data['hostname'] = hostname
data['lab'] = lab
data['resource_template'] = resource_template
data['owner'] = request.user
@@ -232,9 +244,26 @@ def create_from_API(body, user):
data['lab'] = data['resource_template'].lab
data['owner'] = user
+ if 'global_cloud_config' in data.keys():
+ data['global_cloud_config'] = CloudInitFile.objects.get(id=data['global_cloud_config'])
+
return _create_booking(data)
+def create_ci_file(data: str) -> CloudInitFile:
+ try:
+ d = yaml.load(data)
+ if not (type(d) is dict):
+ raise Exception("CI file was valid yaml but was not a dict")
+ except Exception:
+ raise ValidationError("The provided Cloud Config is not valid yaml, please refer to the Cloud Init documentation for expected structure")
+ print("about to create global cloud config")
+ config = CloudInitFile.create(text=data, priority=CloudInitFile.objects.count())
+ print("made global cloud config")
+
+ return config
+
+
@transaction.atomic
def _create_booking(data):
check_invariants(**data)
@@ -246,8 +275,11 @@ def _create_booking(data):
raise PermissionError("You do not have permission to have more than 3 bookings at a time.")
ResourceManager.getInstance().templateIsReservable(data['resource_template'])
- data['resource_template'] = update_template(data['resource_template'], data['image'], 'opnfv_host' if not data['hostname'] else data['hostname'], data['owner'])
- resource_bundle = generate_resource_bundle(data['resource_template'])
+
+ resource_template = update_template(data['resource_template'], data['image'], data['hostname'], data['owner'], global_cloud_config=data['global_cloud_config'])
+
+ # generate resource bundle
+ resource_bundle = generate_resource_bundle(resource_template)
# generate booking
booking = Booking.objects.create(
@@ -263,7 +295,7 @@ def _create_booking(data):
booking.pdf = PDFTemplater.makePDF(booking)
- for collaborator in data['users']: # list of UserProfiles
+ for collaborator in data['users']: # list of Users (not UserProfile)
booking.collaborators.add(collaborator.user)
booking.save()
@@ -284,23 +316,14 @@ def drop_filter(user):
that installer is supported on that image
"""
installer_filter = {}
- for image in Image.objects.all():
- installer_filter[image.id] = {}
- for installer in image.os.sup_installers.all():
- installer_filter[image.id][installer.id] = 1
-
scenario_filter = {}
- for installer in Installer.objects.all():
- scenario_filter[installer.id] = {}
- for scenario in installer.sup_scenarios.all():
- scenario_filter[installer.id][scenario.id] = 1
images = Image.objects.filter(Q(public=True) | Q(owner=user))
image_filter = {}
for image in images:
image_filter[image.id] = {
'lab': 'lab_' + str(image.from_lab.lab_user.id),
- 'host_profile': str(image.host_type.id),
+ 'architecture': str(image.architecture),
'name': image.name
}
@@ -308,7 +331,7 @@ def drop_filter(user):
templates = ResourceTemplate.objects.filter(Q(public=True) | Q(owner=user))
for rt in templates:
profiles = [conf.profile for conf in rt.getConfigs()]
- resource_filter["resource_" + str(rt.id)] = [str(p.id) for p in profiles]
+ resource_filter["resource_" + str(rt.id)] = [str(p.architecture) for p in profiles]
return {
'installer_filter': json.dumps(installer_filter),
diff --git a/src/booking/views.py b/src/booking/views.py
index a418c82..940428b 100644
--- a/src/booking/views.py
+++ b/src/booking/views.py
@@ -28,6 +28,7 @@ from api.models import JobFactory
from workflow.views import login
from booking.forms import QuickBookingForm
from booking.quick_deployer import create_from_form, drop_filter
+import traceback
def quick_create_clear_fields(request):
@@ -62,6 +63,9 @@ def quick_create(request):
"Check Account->My Bookings for the status of your new booking")
return redirect(reverse('booking:booking_detail', kwargs={'booking_id': booking.id}))
except Exception as e:
+ print("Error occurred while handling quick deployment:")
+ traceback.print_exc()
+ print(str(e))
messages.error(request, "Whoops, an error occurred: " + str(e))
context.update(drop_filter(request.user))
return render(request, 'booking/quick_deploy.html', context)
@@ -137,7 +141,7 @@ def build_image_mapping(lab, user):
for profile in ResourceProfile.objects.filter(labs=lab):
images = Image.objects.filter(
from_lab=lab,
- host_type=profile
+ architecture=profile.architecture
).filter(
Q(public=True) | Q(owner=user)
)
diff --git a/src/dashboard/tasks.py b/src/dashboard/tasks.py
index 3f88449..93e6a22 100644
--- a/src/dashboard/tasks.py
+++ b/src/dashboard/tasks.py
@@ -81,11 +81,15 @@ def free_hosts():
).filter(
end__lt=timezone.now(),
job__complete=True,
- resource__isnull=False
+ complete=False,
+ resource__isnull=False,
)
for booking in bookings:
ResourceManager.getInstance().releaseResourceBundle(booking.resource)
+ booking.complete = True
+ print("Booking", booking.id, "is now completed")
+ booking.save()
@shared_task
diff --git a/src/dashboard/utils.py b/src/dashboard/utils.py
index d6b697a..97c9ac7 100644
--- a/src/dashboard/utils.py
+++ b/src/dashboard/utils.py
@@ -7,7 +7,7 @@
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from django.core.exceptions import ObjectDoesNotExist
+from django.core.exceptions import (ObjectDoesNotExist, MultipleObjectsReturned)
class AbstractModelQuery():
@@ -38,7 +38,15 @@ class AbstractModelQuery():
@classmethod
def get(cls, *args, **kwargs):
+ """
+ Gets a single matching resource
+ Throws ObjectDoesNotExist if none found matching, or MultipleObjectsReturned if
+ the query does not narrow to a single object
+ """
try:
+ ls = cls.filter(*args, **kwargs)
+ if len(ls) > 1:
+ raise MultipleObjectsReturned()
return cls.filter(*args, **kwargs)[0]
except IndexError:
raise ObjectDoesNotExist()
diff --git a/src/resource_inventory/migrations/0018_auto_20210630_1629.py b/src/resource_inventory/migrations/0018_auto_20210630_1629.py
new file mode 100644
index 0000000..19e53e4
--- /dev/null
+++ b/src/resource_inventory/migrations/0018_auto_20210630_1629.py
@@ -0,0 +1,101 @@
+# Generated by Django 2.2 on 2021-06-30 16:29
+
+from django.db import migrations, models
+import django.db.models.deletion
+from account.models import Lab
+
+
+def set_availability(apps, schema_editor):
+ models = [apps.get_model('resource_inventory', 'Image'), apps.get_model('resource_inventory', 'Opsys')]
+
+ for model in models:
+ for obj in model.objects.all():
+ obj.available = False
+ obj.obsolete = True
+ obj.save()
+
+
+def set_rconfig_arch(apps, schema_editor):
+ rprofs = apps.get_model('resource_inventory', 'ResourceProfile')
+
+ for rprof in rprofs.objects.all():
+ rprof.architecture = rprof.cpuprofile.first().architecture
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('account', '0009_auto_20210324_2107'),
+ ('resource_inventory', '0017_auto_20201218_1516'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='image',
+ name='host_type',
+ ),
+ migrations.AlterField(
+ model_name='image',
+ name='lab_id',
+ field=models.CharField(default='none (retired)', max_length=100),
+ preserve_default=True,
+ ),
+ migrations.RemoveField(
+ model_name='opsys',
+ name='sup_installers',
+ ),
+
+ migrations.AddField(
+ model_name='image',
+ name='architecture',
+ field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50),
+ preserve_default=False,
+ ),
+
+ migrations.AddField(
+ model_name='image',
+ name='available',
+ field=models.BooleanField(default=True),
+ ),
+ migrations.AddField(
+ model_name='image',
+ name='obsolete',
+ field=models.BooleanField(default=False),
+ ),
+
+ migrations.AddField(
+ model_name='opsys',
+ name='available',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AddField(
+ model_name='opsys',
+ name='obsolete',
+ field=models.BooleanField(default=True),
+ ),
+
+ migrations.RunPython(set_availability),
+
+ migrations.AddField(
+ model_name='opsys',
+ name='lab_id',
+ field=models.CharField(default="none (retired)", max_length=100),
+ preserve_default=False,
+ ),
+
+ migrations.AddField(
+ model_name='opsys',
+ name='from_lab',
+ field=models.ForeignKey(default=Lab.objects.first, on_delete=django.db.models.deletion.CASCADE, to='account.Lab'),
+ preserve_default=False,
+ ),
+
+ migrations.AddField(
+ model_name='resourceprofile',
+ name='architecture',
+ field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50),
+ preserve_default=False,
+ ),
+
+ migrations.RunPython(set_rconfig_arch),
+ ]
diff --git a/src/resource_inventory/migrations/0019_auto_20210701_1947.py b/src/resource_inventory/migrations/0019_auto_20210701_1947.py
new file mode 100644
index 0000000..e64d174
--- /dev/null
+++ b/src/resource_inventory/migrations/0019_auto_20210701_1947.py
@@ -0,0 +1,43 @@
+# Generated by Django 2.2 on 2021-07-01 19:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('resource_inventory', '0018_auto_20210630_1629'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='image',
+ name='lab_id',
+ field=models.CharField(max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='image',
+ name='name',
+ field=models.CharField(max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='network',
+ name='name',
+ field=models.CharField(max_length=200),
+ ),
+ migrations.AlterField(
+ model_name='opsys',
+ name='available',
+ field=models.BooleanField(default=True),
+ ),
+ migrations.AlterField(
+ model_name='opsys',
+ name='obsolete',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AlterField(
+ model_name='resourceprofile',
+ name='architecture',
+ field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64')], max_length=50),
+ ),
+ ]
diff --git a/src/resource_inventory/migrations/0020_cloudinitfile.py b/src/resource_inventory/migrations/0020_cloudinitfile.py
new file mode 100644
index 0000000..198181c
--- /dev/null
+++ b/src/resource_inventory/migrations/0020_cloudinitfile.py
@@ -0,0 +1,21 @@
+# Generated by Django 2.2 on 2021-09-07 14:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('resource_inventory', '0019_auto_20210701_1947'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='CloudInitFile',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('text', models.TextField()),
+ ('priority', models.IntegerField()),
+ ],
+ ),
+ ]
diff --git a/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py
new file mode 100644
index 0000000..6b0befc
--- /dev/null
+++ b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2 on 2021-09-10 18:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('resource_inventory', '0020_cloudinitfile'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='resourceconfiguration',
+ name='cloud_init_files',
+ field=models.ManyToManyField(blank=True, to='resource_inventory.CloudInitFile'),
+ ),
+ ]
diff --git a/src/resource_inventory/migrations/0022_auto_20210925_2028.py b/src/resource_inventory/migrations/0022_auto_20210925_2028.py
new file mode 100644
index 0000000..2b0b902
--- /dev/null
+++ b/src/resource_inventory/migrations/0022_auto_20210925_2028.py
@@ -0,0 +1,23 @@
+# Generated by Django 2.2 on 2021-09-25 20:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('resource_inventory', '0021_resourceconfiguration_cloud_init_files'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='resourcetemplate',
+ name='private_vlan_pool',
+ field=models.TextField(default=''),
+ ),
+ migrations.AddField(
+ model_name='resourcetemplate',
+ name='public_vlan_pool',
+ field=models.TextField(default=''),
+ ),
+ ]
diff --git a/src/resource_inventory/models.py b/src/resource_inventory/models.py
index 7fe479a..aefd5ce 100644
--- a/src/resource_inventory/models.py
+++ b/src/resource_inventory/models.py
@@ -9,10 +9,12 @@
##############################################################################
from django.contrib.auth.models import User
+
from django.core.exceptions import ValidationError
from django.db import models
from django.db.models import Q
import traceback
+import json
import re
from collections import Counter
@@ -20,7 +22,6 @@ from collections import Counter
from account.models import Lab
from dashboard.utils import AbstractModelQuery
-
"""
Profiles of resources hosted by labs.
@@ -33,6 +34,10 @@ Profile models (e.g. an x86 server profile and armv8 server profile.
class ResourceProfile(models.Model):
id = models.AutoField(primary_key=True)
name = models.CharField(max_length=200, unique=True)
+ architecture = models.CharField(max_length=50, choices=[
+ ("x86_64", "x86_64"),
+ ("aarch64", "aarch64")
+ ])
description = models.TextField()
labs = models.ManyToManyField(Lab, related_name="resourceprofiles")
@@ -147,6 +152,24 @@ with varying degrees of abstraction.
"""
+class CloudInitFile(models.Model):
+ text = models.TextField()
+
+ # higher priority is applied later, so "on top" of existing files
+ priority = models.IntegerField()
+
+ @classmethod
+ def merge_strategy(cls):
+ return [
+ {'name': 'list', 'settings': ['append']},
+ {'name': 'dict', 'settings': ['recurse_list', 'replace']},
+ ]
+
+ @classmethod
+ def create(cls, text="", priority=0):
+ return CloudInitFile.objects.create(priority=priority, text=text)
+
+
class ResourceTemplate(models.Model):
"""
Models a "template" of a complete, configured collection of resources that can be booked.
@@ -167,6 +190,24 @@ class ResourceTemplate(models.Model):
temporary = models.BooleanField(default=False)
copy_of = models.ForeignKey("ResourceTemplate", blank=True, null=True, on_delete=models.SET_NULL)
+ # if these fields are empty ("") then they are implicitly "every vlan",
+ # otherwise we filter any allocations we try to instantiate against this list
+ # they should be represented as a json list of integers
+ private_vlan_pool = models.TextField(default="")
+ public_vlan_pool = models.TextField(default="")
+
+ def private_vlan_pool_set(self):
+ if self.private_vlan_pool != "":
+ return set(json.loads(self.private_vlan_pool))
+ else:
+ return None
+
+ def public_vlan_pool_set(self):
+ if self.private_vlan_pool != "":
+ return set(json.loads(self.public_vlan_pool))
+ else:
+ return None
+
def getConfigs(self):
configs = self.resourceConfigurations.all()
return list(configs)
@@ -235,9 +276,14 @@ class ResourceConfiguration(models.Model):
is_head_node = models.BooleanField(default=False)
name = models.CharField(max_length=3000, default="opnfv_host")
+ cloud_init_files = models.ManyToManyField(CloudInitFile, blank=True)
+
def __str__(self):
return str(self.name)
+ def ci_file_list(self):
+ return list(self.cloud_init_files.order_by("priority").all())
+
def get_default_remote_info():
return RemoteInfo.objects.get_or_create(
@@ -369,10 +415,43 @@ class Server(Resource):
return isinstance(other, Server) and other.name == self.name
+def is_serializable(data):
+ try:
+ json.dumps(data)
+ return True
+ except Exception:
+ return False
+
+
class Opsys(models.Model):
id = models.AutoField(primary_key=True)
name = models.CharField(max_length=100)
- sup_installers = models.ManyToManyField("Installer", blank=True)
+ lab_id = models.CharField(max_length=100)
+ obsolete = models.BooleanField(default=False)
+ available = models.BooleanField(default=True) # marked true by Cobbler if it exists there
+ from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE)
+
+ indexes = [
+ models.Index(fields=['cobbler_id'])
+ ]
+
+ def new_from_data(data):
+ opsys = Opsys()
+ opsys.update(data)
+ return opsys
+
+ def serialize(self):
+ d = {}
+ for field in vars(self):
+ attr = getattr(self, field)
+ if is_serializable(attr):
+ d[field] = attr
+ return d
+
+ def update(self, data):
+ for field in vars(self):
+ if field in data:
+ setattr(self, field, data[field] if data[field] else getattr(self, field))
def __str__(self):
return self.name
@@ -382,18 +461,51 @@ class Image(models.Model):
"""Model for representing OS images / snapshots of hosts."""
id = models.AutoField(primary_key=True)
- lab_id = models.IntegerField() # ID the lab who holds this image knows
from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE)
- name = models.CharField(max_length=200)
+ architecture = models.CharField(max_length=50, choices=[
+ ("x86_64", "x86_64"),
+ ("aarch64", "aarch64"),
+ ("unknown", "unknown"),
+ ])
+ lab_id = models.CharField(max_length=100)
+ name = models.CharField(max_length=100)
owner = models.ForeignKey(User, null=True, on_delete=models.SET_NULL)
public = models.BooleanField(default=True)
- host_type = models.ForeignKey(ResourceProfile, on_delete=models.CASCADE)
description = models.TextField()
os = models.ForeignKey(Opsys, null=True, on_delete=models.CASCADE)
+ available = models.BooleanField(default=True) # marked True by cobbler if it exists there
+ obsolete = models.BooleanField(default=False)
+
+ indexes = [
+ models.Index(fields=['architecture']),
+ models.Index(fields=['cobbler_id'])
+ ]
+
def __str__(self):
return self.name
+ def is_obsolete(self):
+ return self.obsolete or self.os.obsolete
+
+ def serialize(self):
+ d = {}
+ for field in vars(self):
+ attr = getattr(self, field)
+ if is_serializable(attr):
+ d[field] = attr
+ return d
+
+ def update(self, data):
+ for field in vars(self):
+ if field in data:
+ setattr(self, field, data[field] if data[field] else getattr(self, field))
+
+ def new_from_data(data):
+ img = Image()
+ img.update(data)
+ return img
+
def in_use(self):
for resource in ResourceQuery.filter(config__image=self):
if resource.is_reserved():
@@ -409,7 +521,7 @@ Networking configuration models
class Network(models.Model):
id = models.AutoField(primary_key=True)
- name = models.CharField(max_length=100)
+ name = models.CharField(max_length=200)
bundle = models.ForeignKey(ResourceTemplate, on_delete=models.CASCADE, related_name="networks")
is_public = models.BooleanField()
@@ -507,6 +619,13 @@ class NetworkRole(models.Model):
network = models.ForeignKey(Network, on_delete=models.CASCADE)
+def create_resource_ref_string(for_hosts: [str]) -> str:
+ # need to sort the list, then do dump
+ for_hosts.sort()
+
+ return json.dumps(for_hosts)
+
+
class OPNFVConfig(models.Model):
id = models.AutoField(primary_key=True)
installer = models.ForeignKey(Installer, on_delete=models.CASCADE)
diff --git a/src/resource_inventory/resource_manager.py b/src/resource_inventory/resource_manager.py
index 37bf33c..52af824 100644
--- a/src/resource_inventory/resource_manager.py
+++ b/src/resource_inventory/resource_manager.py
@@ -6,7 +6,8 @@
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from __future__ import annotations
+
+from __future__ import annotations # noqa: F407
import re
from typing import Optional
@@ -85,12 +86,13 @@ class ResourceManager:
vlan_manager = resourceTemplate.lab.vlan_manager
for network in resourceTemplate.networks.all():
if network.is_public:
- public_net = vlan_manager.get_public_vlan()
+ # already throws if can't get requested count, so can always expect public_net to be Some
+ public_net = vlan_manager.get_public_vlan(within=resourceTemplate.public_vlan_pool_set())
vlan_manager.reserve_public_vlan(public_net.vlan)
networks[network.name] = public_net.vlan
else:
# already throws if can't get requested count, so can always index in @ 0
- vlans = vlan_manager.get_vlans(count=1)
+ vlans = vlan_manager.get_vlans(count=1, within=resourceTemplate.private_vlan_pool_set())
vlan_manager.reserve_vlans(vlans[0])
networks[network.name] = vlans[0]
return networks
diff --git a/src/resource_inventory/tests/test_models.py b/src/resource_inventory/tests/test_models.py
index e1b2106..3f2d1d8 100644
--- a/src/resource_inventory/tests/test_models.py
+++ b/src/resource_inventory/tests/test_models.py
@@ -80,7 +80,7 @@ class ConfigUtil():
)
return Image.objects.create(
- lab_id=0,
+ cobbler_id="profile1",
from_lab=lab,
name="an image for testing",
owner=owner
diff --git a/src/templates/base/base.html b/src/templates/base/base.html
index 704bc3b..a5b79af 100644
--- a/src/templates/base/base.html
+++ b/src/templates/base/base.html
@@ -156,6 +156,12 @@
<a href="{% url 'account:my-bookings' %}" class="list-group-item list-group-item-action list-group-item-secondary dropDown-bg">
My Bookings
</a>
+ <a href="{% url 'account:my-configurations' %}" class="list-group-item list-group-item-action list-group-item-secondary dropDown-bg">
+ My Configurations
+ </a>
+ <a href="{% url 'account:my-images' %}" class="list-group-item list-group-item-action list-group-item-secondary dropDown-bg">
+ My Snapshots
+ </a>
</div>
<a href="{% url 'dashboard:all_labs' %}" class="list-group-item list-group-item-action nav-bg">
Lab Info
diff --git a/src/templates/base/booking/booking_detail.html b/src/templates/base/booking/booking_detail.html
index a014fea..4a8f35a 100644
--- a/src/templates/base/booking/booking_detail.html
+++ b/src/templates/base/booking/booking_detail.html
@@ -7,6 +7,12 @@
<script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js?lang=yaml"></script>
{% endblock %}
+<style>
+code {
+ overflow: scroll;
+}
+</style>
+
{% block content %}
<div class="row">
<div class="col-12 col-lg-5">
@@ -154,6 +160,65 @@
</div>
</div>
</div>
+ <div class="card my-3">
+ <div class="card-header d-flex">
+ <h4 class="d-inline">Diagnostic Information</h4>
+ <button data-toggle="collapse" data-target="#diagnostics_panel" class="btn btn-outline-secondary ml-auto">Expand</button>
+ </div>
+ <div class="collapse" id="diagnostics_panel">
+ <div class="card-body">
+ <table class="table m-0">
+ <tr>
+ <th>Job ID: </th>
+ <td>{{booking.job.id}}</td>
+ </tr>
+ <tr>
+ <th>CI Files</th>
+ </tr>
+ {% for host in booking.resource.get_resources %}
+ <tr>
+ <td>
+ <table class="table m-0">
+ <tr>
+ <th>Host:</th>
+ <td>{{host.name}}</td>
+ </tr>
+ <tr>
+ <th>Configs:</th>
+ </tr>
+ {% for ci_file in host.config.cloud_init_files.all %}
+ <tr>
+ <td>{{ci_file.id}}</td>
+ <td>
+ <div class="modal fade" id="ci_file_modal_{{ci_file.id}}" tabindex="-1" role="dialog" aria-hidden="true">
+ <div class="modal-dialog modal-xl" role="document">
+ <div class="modal-content">
+ <div class="modal-header">
+ <h4 class="modal-title d-inline float-left">Cloud Config Content</h4>
+ <button type="button" class="close" data-dismiss="modal" aria-label="Close">
+ <span aria-hidden="true">&times;</span>
+ </button>
+ </div>
+ <div class="card-body">
+ <pre class="prettyprint lang-yaml m-0 border-0 text-break pre-wrap">
+{{ci_file.text}}
+ </pre>
+ </div>
+ </div>
+ </div>
+ </div>
+ <button class="btn btn-primary" data-toggle="modal" data-target="#ci_file_modal_{{ci_file.id}}">Show File Content</button>
+ </td>
+ </tr>
+ {% endfor %}
+ </table>
+ </td>
+ </tr>
+ {% endfor %}
+ </table>
+ </div>
+ </div>
+ </div>
</div>
<div class="col">
<div class="card mb-3">
diff --git a/src/templates/base/booking/quick_deploy.html b/src/templates/base/booking/quick_deploy.html
index 5dc41e2..c51e234 100644
--- a/src/templates/base/booking/quick_deploy.html
+++ b/src/templates/base/booking/quick_deploy.html
@@ -3,6 +3,13 @@
{% load bootstrap4 %}
{% block content %}
+<style>
+/* hides images not in use. Not applied globally since doesn't make sense in all cases */
+select option:disabled {
+ display:none;
+}
+</style>
+
{% bootstrap_form_errors form type='non_fields' %}
<form id="quick_booking_form" action="/booking/quick/" method="POST" class="form class="Anuket-Text"">
{% csrf_token %}
@@ -18,7 +25,7 @@
</div>
</div>
<div class="row justify-content-center">
- <div class="col-12 col-lg-4 my-2">
+ <div class="col-12 col-lg-6 my-2">
<div class="col border rounded py-2 h-100">
{% bootstrap_field form.purpose %}
{% bootstrap_field form.project %}
@@ -31,19 +38,26 @@
</div>
</div>
{% block collab %}
- <div class="col-12 col-lg-4 my-2">
+ <div class="col-12 col-lg-6 my-2">
<div class="col border rounded py-2 h-100">
<label>Collaborators</label>
{{ form.users }}
</div>
</div>
{% endblock collab %}
- <div class="col-12 col-lg-4 my-2">
+ </div>
+ <div class="row justify-content-center">
+ <div class="col-12 col-lg-6 my-2">
<div class="col border rounded py-2 h-100">
{% bootstrap_field form.hostname %}
{% bootstrap_field form.image %}
</div>
</div>
+ <div class="col-12 col-lg-6 my-2">
+ <div class="col border rounded py-2 h-100">
+ {% bootstrap_field form.global_cloud_config %}
+ </div>
+ </div>
<div class="col-12 d-flex mt-2 justify-content-end">
<button id="quick_booking_confirm" onclick="submit_form();" type="button" class="btn btn-success">Confirm</button>
</div>
@@ -88,15 +102,22 @@
function imageFilter() {
var drop = document.getElementById("id_image");
var lab_pk = get_selected_value("lab");
- var host_pk = get_selected_value("resource");
+ var profile_pk = get_selected_value("resource");
for (const childNode of drop.childNodes) {
var image_object = sup_image_dict[childNode.value];
if (image_object) //weed out empty option
{
+ console.log("image object:");
+ console.log(image_object);
const img_at_lab = image_object.lab == lab_pk;
- const profiles = resource_profile_map[host_pk];
- const img_in_template = profiles && profiles.indexOf(image_object.host_profile) > -1
+ const profiles = resource_profile_map[profile_pk];
+ console.log("profiles are:");
+ console.log(profiles);
+ console.log("profile map is:");
+ console.log(resource_profile_map);
+ console.log("host profile is" + image_object.architecture);
+ const img_in_template = profiles && profiles.indexOf(image_object.architecture) > -1
childNode.disabled = !img_at_lab || !img_in_template;
}
}
diff --git a/src/templates/laas/base.html b/src/templates/laas/base.html
index 69e4976..f980268 100644
--- a/src/templates/laas/base.html
+++ b/src/templates/laas/base.html
@@ -2,11 +2,84 @@
{% load staticfiles %}
{% block logo %}
-<link rel="stylesheet" href="{% static "css/anuket.css" %}">
+<style>
+ nav ,body{
+ background-color:#fff !important;
+ color:#343a40 !important;
+ }
+
+ header{
+ background-color:#f8f9fa !important;
+ color:#343a40 !important;
+ }
+
+ .nav-bg{
+ background-color:#fff !important;
+ color:#343a40 !important;
+ }
+
+ .nav-bg:hover{
+ background-color:#f8f9fa !important;
+ transition-duration:0.2s;
+ }
+
+ .dropDown-bg{
+ background-color:#d6d8db !important;
+ color:#343a40 !important;
+ }
+
+ .btnAnuket {
+ color: #343a40;
+ background-color: #6BDAD5;
+ transition-duration:0.2s;
+ border:0px
+ }
+ .btnAnuket:hover{
+ color: #f8f9fa;
+ background-color: #007473;
+ border:0px
+ }
+
+ .btnAnuket:focus{
+ color: #f8f9fa !important;
+ background-color: #007473 !important;
+ border:0px
+ }
+
+ .alertAnuket{
+ background-color: #e6b3c1;
+ color:#820c2c;
+ border:0px;
+ }
+ .inTextLink{
+ text-decoration: underline;
+ }
+
+ .Anuket-Text{
+ color:#343a40 !important;
+ }
+
+ h1, h2{
+ color:#343a40 !important;
+ }
+
+ p, h3, h4, h5{
+ color:#343a40 !important;
+ }
+
+ ::selection {
+ background: #BCE194;
+ color:#343a40;
+ }
+ ::-moz-selection {
+ background: #BCE194;
+ color:#343a40;
+ }
+ </style>
<div class="col-12 col-sm order-1 order-sm-2 text-center text-lg-left">
<a href="https://anuket.io/" class="navbar-brand">
- <img src="{% static "img/Anuket-logo.svg" %}" alt="Anuket logo" width="134.2" height="50" style="vertical-align:middle; margin:12px 12px 12px -20px;">
+ <img src="{% static "img/Anuket-logo.svg" %}" width="134.2" height="50" style="vertical-align:middle; margin:12px 12px 12px -20px;">
</a>
<a class="navbar-brand d-none d-lg-inline Anuket-Text" href={% url 'dashboard:index' %} style="margin-left:10px; font-size:26px; vertical-align:middle;">
LaaS Dashboard
diff --git a/src/templates/lfedge/booking/quick_deploy.html b/src/templates/lfedge/booking/quick_deploy.html
index dac3815..ccafd90 100644
--- a/src/templates/lfedge/booking/quick_deploy.html
+++ b/src/templates/lfedge/booking/quick_deploy.html
@@ -11,7 +11,7 @@
</p>
{% endblock form-text %}
{% block collab %}
-<div class="col-12 col-lg-4 my-2">
+<div class="col-12 col-lg-6 my-2">
<div class="col border rounded py-2 h-100">
<label>Collaborators</label>
{{ form.users }}
@@ -21,8 +21,8 @@
{% block image_script %}
<script type="text/javascript">
- document.getElementById("id_image").disabled = true;
- document.getElementById("id_image").style.display = 'none';
- document.getElementById("id_image").previousElementSibling.style.display = 'none';
+// document.getElementById("id_image").disabled = true;
+// document.getElementById("id_image").style.display = 'none';
+// document.getElementById("id_image").previousElementSibling.style.display = 'none';
</script>
{% endblock image_script %}
diff --git a/src/workflow/resource_bundle_workflow.py b/src/workflow/resource_bundle_workflow.py
index 63a9519..a461e9a 100644
--- a/src/workflow/resource_bundle_workflow.py
+++ b/src/workflow/resource_bundle_workflow.py
@@ -196,7 +196,7 @@ class Define_Software(WorkflowStep):
for i, host_data in enumerate(hosts_data):
host = ResourceConfiguration.objects.get(pk=host_data['host_id'])
wrong_owner = Image.objects.exclude(owner=user).exclude(public=True)
- wrong_host = Image.objects.exclude(host_type=host.profile)
+ wrong_host = Image.objects.exclude(architecture=host.profile.architecture)
wrong_lab = Image.objects.exclude(from_lab=lab)
excluded_images = wrong_owner | wrong_host | wrong_lab
filter_data.append([])