diff options
author | Adam Hassick <ahassick@iol.unh.edu> | 2021-06-29 16:49:27 -0400 |
---|---|---|
committer | Adam Hassick <ahassick@iol.unh.edu> | 2021-07-23 16:22:54 +0000 |
commit | 6ffb1fdf6ce7825770148bada5a4c54899e4ed36 (patch) | |
tree | da5d8390a4d46a898840083a761809af47bd7f52 | |
parent | 49e2b407003b69551ddafa851639e83ec42a5b09 (diff) |
Cobbler model changes, new endpoints
Signed-off-by: Adam Hassick <ahassick@iol.unh.edu>
Change-Id: If0a94730e92747127cef121ec4930a4c8bae6c92
Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu>
Signed-off-by: Adam Hassick <ahassick@iol.unh.edu>
-rw-r--r-- | src/api/migrations/0017_auto_20210630_1629.py | 18 | ||||
-rw-r--r-- | src/api/migrations/0018_cloudinitfile.py (renamed from src/api/migrations/0017_cloudinitfile.py) | 6 | ||||
-rw-r--r-- | src/api/models.py | 18 | ||||
-rw-r--r-- | src/api/urls.py | 10 | ||||
-rw-r--r-- | src/api/views.py | 90 | ||||
-rw-r--r-- | src/booking/quick_deployer.py | 20 | ||||
-rw-r--r-- | src/booking/views.py | 2 | ||||
-rw-r--r-- | src/resource_inventory/migrations/0018_auto_20210630_1629.py | 105 | ||||
-rw-r--r-- | src/resource_inventory/migrations/0019_auto_20210701_1947.py | 43 | ||||
-rw-r--r-- | src/resource_inventory/models.py | 83 | ||||
-rw-r--r-- | src/resource_inventory/tests/test_models.py | 2 | ||||
-rw-r--r-- | src/templates/base/booking/quick_deploy.html | 13 | ||||
-rw-r--r-- | src/workflow/resource_bundle_workflow.py | 2 |
13 files changed, 371 insertions, 41 deletions
diff --git a/src/api/migrations/0017_auto_20210630_1629.py b/src/api/migrations/0017_auto_20210630_1629.py new file mode 100644 index 0000000..643ff5f --- /dev/null +++ b/src/api/migrations/0017_auto_20210630_1629.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2021-06-30 16:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0016_auto_20201109_2149'), + ] + + operations = [ + migrations.AlterField( + model_name='snapshotconfig', + name='image', + field=models.CharField(max_length=200, null=True), + ), + ] diff --git a/src/api/migrations/0017_cloudinitfile.py b/src/api/migrations/0018_cloudinitfile.py index f14aea1..4e41b39 100644 --- a/src/api/migrations/0017_cloudinitfile.py +++ b/src/api/migrations/0018_cloudinitfile.py @@ -1,4 +1,4 @@ -# Generated by Django 2.2 on 2021-06-11 20:42 +# Generated by Django 2.2 on 2021-07-01 20:45 from django.db import migrations, models import django.db.models.deletion @@ -7,9 +7,9 @@ import django.db.models.deletion class Migration(migrations.Migration): dependencies = [ - ('resource_inventory', '0017_auto_20201218_1516'), + ('resource_inventory', '0019_auto_20210701_1947'), ('booking', '0008_auto_20201109_1947'), - ('api', '0016_auto_20201109_2149'), + ('api', '0017_auto_20210630_1629'), ] operations = [ diff --git a/src/api/models.py b/src/api/models.py index 36d1b8c..a207044 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -25,6 +25,7 @@ from resource_inventory.models import ( Lab, ResourceProfile, Image, + Opsys, Interface, ResourceOPNFVConfig, RemoteInfo, @@ -85,6 +86,18 @@ class LabManager(object): def __init__(self, lab): self.lab = lab + def get_opsyss(self): + return Opsys.objects.filter(from_lab=self.lab) + + def get_images(self): + return Image.objects.filter(from_lab=self.lab) + + def get_image(self, image_id): + return Image.objects.filter(from_lab=self.lab, lab_id=image_id) + + def get_opsys(self, opsys_id): + return Opsys.objects.filter(from_lab=self.lab, lab_id=opsys_id) + def get_downtime(self): return Downtime.objects.filter(start__lt=timezone.now(), end__gt=timezone.now(), lab=self.lab) @@ -408,7 +421,7 @@ class CloudInitFile(models.Model): return full_dict @classmethod - def get(booking_id: int, resource_lab_id: str): + def get(cls, booking_id: int, resource_lab_id: str): return CloudInitFile.objects.get(resource_id=resource_lab_id, booking__id=booking_id) def _resource(self): @@ -768,7 +781,6 @@ class HardwareConfig(TaskConfig): # TODO: grab the CloudInitFile urls from self.hosthardwarerelation.get_resource() return self.format_delta( self.hosthardwarerelation.get_resource().get_configuration(self.state), - self.cloudinit_file.get_delta_url(), self.hosthardwarerelation.lab_token) @@ -819,7 +831,7 @@ class NetworkConfig(TaskConfig): class SnapshotConfig(TaskConfig): resource_id = models.CharField(max_length=200, default="default_id") - image = models.IntegerField(null=True) + image = models.CharField(max_length=200,null=True) # cobbler ID dashboard_id = models.IntegerField() delta = models.TextField(default="{}") diff --git a/src/api/urls.py b/src/api/urls.py index 7adeef6..e5ddd97 100644 --- a/src/api/urls.py +++ b/src/api/urls.py @@ -47,9 +47,17 @@ from api.views import ( GenerateTokenView, analytics_job, resource_cidata, + all_images, + all_opsyss, + single_image, + single_opsys ) urlpatterns = [ + path('labs/<slug:lab_name>/opsys/<slug:opsys_id>', single_opsys), + path('labs/<slug:lab_name>/image/<slug:image_id>', single_image), + path('labs/<slug:lab_name>/opsys', all_opsyss), + path('labs/<slug:lab_name>/image', all_images), path('labs/<slug:lab_name>/profile', lab_profile), path('labs/<slug:lab_name>/status', lab_status), path('labs/<slug:lab_name>/inventory', lab_inventory), @@ -60,7 +68,7 @@ urlpatterns = [ path('labs/<slug:lab_name>/booking/<int:booking_id>/idf', get_idf, name="get-idf"), path('labs/<slug:lab_name>/jobs/<int:job_id>', specific_job), path('labs/<slug:lab_name>/jobs/<int:job_id>/<slug:task_id>', specific_task), - path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id', resource_cidata), + path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>', resource_cidata), path('labs/<slug:lab_name>/jobs/new', new_jobs), path('labs/<slug:lab_name>/jobs/current', current_jobs), path('labs/<slug:lab_name>/jobs/done', done_jobs), diff --git a/src/api/views.py b/src/api/views.py index 3a3effa..4b887e6 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -14,6 +14,7 @@ from django.shortcuts import redirect from django.utils.decorators import method_decorator from django.utils import timezone from django.views import View +from django.http import QueryDict from django.http.response import JsonResponse, HttpResponse from rest_framework import viewsets from rest_framework.authtoken.models import Token @@ -25,9 +26,14 @@ from api.serializers.old_serializers import UserSerializer from api.forms import DowntimeForm from account.models import UserProfile from booking.models import Booking -from api.models import LabManagerTracker, get_task, CloudInitFile +from api.models import LabManagerTracker, get_task, CloudInitFile, Job from notifier.manager import NotificationHandler from analytics.models import ActiveVPNUser +from resource_inventory.models import ( + Image, + Opsys +) + import json """ @@ -80,6 +86,81 @@ def lab_host(request, lab_name="", host_id=""): if request.method == "POST": return JsonResponse(lab_manager.update_host(host_id, request.POST), safe=False) +# API extension for Cobbler integration + +def all_images(request, lab_name=""): + a = [] + for i in Image.objects.all(): + a.append(i.serialize()) + return JsonResponse(a, safe=False) + + +def all_opsyss(request, lab_name=""): + a = [] + for opsys in Opsys.objects.all(): + a.append(opsys.serialize()) + + return JsonResponse(a, safe=False) + +@csrf_exempt +def single_image(request, lab_name="", image_id=""): + lab_token = request.META.get('HTTP_AUTH_TOKEN') + lab_manager = LabManagerTracker.get(lab_name, lab_token) + img = lab_manager.get_image(image_id).first() + + if request.method == "GET": + if not img: + return HttpResponse(status=404) + return JsonResponse(img.serialize(), safe=False) + + if request.method == "POST": + # get POST data + data = json.loads(request.body.decode('utf-8')) + if img: + img.update(data) + else: + # append lab name and the ID from the URL + data['from_lab_id'] = lab_name + data['lab_id'] = image_id + + # create and save a new Image object + img = Image.new_from_data(data) + + img.save() + + # indicate success in response + return HttpResponse(status=200) + return HttpResponse(status=405) + + +@csrf_exempt +def single_opsys(request, lab_name="", opsys_id=""): + lab_token = request.META.get('HTTP_AUTH_TOKEN') + lab_manager = LabManagerTracker.get(lab_name, lab_token) + opsys = lab_manager.get_opsys(opsys_id).first() + + if request.method == "GET": + if not opsys: + return HttpResponse(status=404) + return JsonResponse(opsys.serialize(), safe=False) + + if request.method == "POST": + data = json.loads(request.body.decode('utf-8')) + if opsys: + opsys.update(data) + else: + # only name, available, and obsolete are needed to create an Opsys + # other fields are derived from the URL parameters + + data['from_lab_id'] = lab_name + data['lab_id'] = opsys_id + opsys = Opsys.new_from_data(data) + + opsys.save() + return HttpResponse(status=200) + return HttpResponse(status=405) + +# end API extension def get_pdf(request, lab_name="", booking_id=""): lab_token = request.META.get('HTTP_AUTH_TOKEN') @@ -168,10 +249,11 @@ def specific_job(request, lab_name="", job_id=""): @csrf_exempt def resource_cidata(request, lab_name="", job_id="", resource_id=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) + #lab_token = request.META.get('HTTP_AUTH_TOKEN') + #lab_manager = LabManagerTracker.get(lab_name, lab_token) - job = lab_manager.get_job(job_id) + #job = lab_manager.get_job(job_id) + job = Job.objects.get(id=job_id) cifile = None try: diff --git a/src/booking/quick_deployer.py b/src/booking/quick_deployer.py index 0a3bfc6..9e53da5 100644 --- a/src/booking/quick_deployer.py +++ b/src/booking/quick_deployer.py @@ -176,13 +176,6 @@ def check_invariants(request, **kwargs): length = kwargs['length'] # check that image os is compatible with installer if image: - if installer or scenario: - if installer in image.os.sup_installers.all(): - # if installer not here, we can omit that and not check for scenario - if not scenario: - raise ValidationError("An OPNFV Installer needs a scenario to be chosen to work properly") - if scenario not in installer.sup_scenarios.all(): - raise ValidationError("The chosen installer does not support the chosen scenario") if image.from_lab != lab: raise ValidationError("The chosen image is not available at the chosen hosting lab") # TODO @@ -272,23 +265,14 @@ def drop_filter(user): that installer is supported on that image """ installer_filter = {} - for image in Image.objects.all(): - installer_filter[image.id] = {} - for installer in image.os.sup_installers.all(): - installer_filter[image.id][installer.id] = 1 - scenario_filter = {} - for installer in Installer.objects.all(): - scenario_filter[installer.id] = {} - for scenario in installer.sup_scenarios.all(): - scenario_filter[installer.id][scenario.id] = 1 images = Image.objects.filter(Q(public=True) | Q(owner=user)) image_filter = {} for image in images: image_filter[image.id] = { 'lab': 'lab_' + str(image.from_lab.lab_user.id), - 'host_profile': str(image.host_type.id), + 'architecture': str(image.architecture), 'name': image.name } @@ -296,7 +280,7 @@ def drop_filter(user): templates = ResourceTemplate.objects.filter(Q(public=True) | Q(owner=user)) for rt in templates: profiles = [conf.profile for conf in rt.getConfigs()] - resource_filter["resource_" + str(rt.id)] = [str(p.id) for p in profiles] + resource_filter["resource_" + str(rt.id)] = [str(p.architecture) for p in profiles] return { 'installer_filter': json.dumps(installer_filter), diff --git a/src/booking/views.py b/src/booking/views.py index 2b910e7..ea038dd 100644 --- a/src/booking/views.py +++ b/src/booking/views.py @@ -138,7 +138,7 @@ def build_image_mapping(lab, user): for profile in ResourceProfile.objects.filter(labs=lab): images = Image.objects.filter( from_lab=lab, - host_type=profile + architecture=profile.architecture ).filter( Q(public=True) | Q(owner=user) ) diff --git a/src/resource_inventory/migrations/0018_auto_20210630_1629.py b/src/resource_inventory/migrations/0018_auto_20210630_1629.py new file mode 100644 index 0000000..8062205 --- /dev/null +++ b/src/resource_inventory/migrations/0018_auto_20210630_1629.py @@ -0,0 +1,105 @@ +# Generated by Django 2.2 on 2021-06-30 16:29 + +from django.db import migrations, models +import django.db.models.deletion +from account.models import * + +#def set_architectures(apps, schema_editor): +# model = apps.get_model('resource_inventory', 'Image') +# +# #while model.objects.filter(architecture=' +# for obj in model.objects.all(): +# obj.architecture = + +def set_availability(apps, schema_editor): + models = [apps.get_model('resource_inventory', 'Image'), apps.get_model('resource_inventory', 'Opsys')] + + for model in models: + for obj in model.objects.all(): + obj.available = False + obj.obsolete = True + obj.save() + +def set_rconfig_arch(apps, schema_editor): + rprofs = apps.get_model('resource_inventory', 'ResourceProfile') + + for rprof in rprofs.objects.all(): + rprof.architecture = rprof.cpuprofile.first().architecture + +class Migration(migrations.Migration): + + dependencies = [ + ('account', '0009_auto_20210324_2107'), + ('resource_inventory', '0017_auto_20201218_1516'), + ] + + operations = [ + migrations.RemoveField( + model_name='image', + name='host_type', + ), + migrations.AlterField( + model_name='image', + name='lab_id', + field=models.CharField(default='none (retired)', max_length=100), + preserve_default=True, + ), + migrations.RemoveField( + model_name='opsys', + name='sup_installers', + ), + + migrations.AddField( + model_name='image', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50), + preserve_default=False, + ), + + migrations.AddField( + model_name='image', + name='available', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='image', + name='obsolete', + field=models.BooleanField(default=False), + ), + + migrations.AddField( + model_name='opsys', + name='available', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='opsys', + name='obsolete', + field=models.BooleanField(default=True), + ), + + migrations.RunPython(set_availability), + + migrations.AddField( + model_name='opsys', + name='lab_id', + field=models.CharField(default="none (retired)", max_length=100), + preserve_default=False, + ), + + migrations.AddField( + model_name='opsys', + name='from_lab', + field=models.ForeignKey(default=Lab.objects.first, on_delete=django.db.models.deletion.CASCADE, to='account.Lab'), + preserve_default=False, + ), + + migrations.AddField( + model_name='resourceprofile', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50), + preserve_default=False, + ), + + migrations.RunPython(set_rconfig_arch), + ] diff --git a/src/resource_inventory/migrations/0019_auto_20210701_1947.py b/src/resource_inventory/migrations/0019_auto_20210701_1947.py new file mode 100644 index 0000000..e64d174 --- /dev/null +++ b/src/resource_inventory/migrations/0019_auto_20210701_1947.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2 on 2021-07-01 19:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0018_auto_20210630_1629'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='lab_id', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='image', + name='name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='network', + name='name', + field=models.CharField(max_length=200), + ), + migrations.AlterField( + model_name='opsys', + name='available', + field=models.BooleanField(default=True), + ), + migrations.AlterField( + model_name='opsys', + name='obsolete', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='resourceprofile', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64')], max_length=50), + ), + ] diff --git a/src/resource_inventory/models.py b/src/resource_inventory/models.py index 7fe479a..fb4dad5 100644 --- a/src/resource_inventory/models.py +++ b/src/resource_inventory/models.py @@ -9,10 +9,12 @@ ############################################################################## from django.contrib.auth.models import User + from django.core.exceptions import ValidationError from django.db import models from django.db.models import Q import traceback +import json import re from collections import Counter @@ -20,7 +22,6 @@ from collections import Counter from account.models import Lab from dashboard.utils import AbstractModelQuery - """ Profiles of resources hosted by labs. @@ -33,6 +34,10 @@ Profile models (e.g. an x86 server profile and armv8 server profile. class ResourceProfile(models.Model): id = models.AutoField(primary_key=True) name = models.CharField(max_length=200, unique=True) + architecture = models.CharField(max_length=50, choices=[ + ("x86_64", "x86_64"), + ("aarch64", "aarch64") + ]) description = models.TextField() labs = models.ManyToManyField(Lab, related_name="resourceprofiles") @@ -369,10 +374,43 @@ class Server(Resource): return isinstance(other, Server) and other.name == self.name +def is_serializable(data): + try: + json.dumps(data) + return True + except: + return False + + class Opsys(models.Model): id = models.AutoField(primary_key=True) name = models.CharField(max_length=100) - sup_installers = models.ManyToManyField("Installer", blank=True) + lab_id = models.CharField(max_length=100) + obsolete = models.BooleanField(default=False) + available = models.BooleanField(default=True) # marked true by Cobbler if it exists there + from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE) + + indexes = [ + models.Index(fields=['cobbler_id']) + ] + + def new_from_data(data): + opsys = Opsys() + opsys.update(data) + return opsys + + def serialize(self): + d = {} + for field in vars(self): + attr = getattr(self, field) + if is_serializable(attr): + d[field] = attr + return d + + def update(self, data): + for field in vars(self): + if field in data: + setattr(self, field, data[field] if data[field] else getattr(self, field)) def __str__(self): return self.name @@ -382,18 +420,51 @@ class Image(models.Model): """Model for representing OS images / snapshots of hosts.""" id = models.AutoField(primary_key=True) - lab_id = models.IntegerField() # ID the lab who holds this image knows from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE) - name = models.CharField(max_length=200) + architecture = models.CharField(max_length=50, choices=[ + ("x86_64", "x86_64"), + ("aarch64", "aarch64"), + ("unknown", "unknown"), + ]) + lab_id = models.CharField(max_length=100) + name = models.CharField(max_length=100) owner = models.ForeignKey(User, null=True, on_delete=models.SET_NULL) public = models.BooleanField(default=True) - host_type = models.ForeignKey(ResourceProfile, on_delete=models.CASCADE) description = models.TextField() os = models.ForeignKey(Opsys, null=True, on_delete=models.CASCADE) + + available = models.BooleanField(default=True) # marked True by cobbler if it exists there + obsolete = models.BooleanField(default=False) + + indexes = [ + models.Index(fields=['architecture']), + models.Index(fields=['cobbler_id']) + ] def __str__(self): return self.name + def is_obsolete(self): + return self.obsolete or self.os.obsolete + + def serialize(self): + d = {} + for field in vars(self): + attr = getattr(self, field) + if is_serializable(attr): + d[field] = attr + return d + + def update(self, data): + for field in vars(self): + if field in data: + setattr(self, field, data[field] if data[field] else getattr(self, field)) + + def new_from_data(data): + img = Image() + img.update(data) + return img + def in_use(self): for resource in ResourceQuery.filter(config__image=self): if resource.is_reserved(): @@ -409,7 +480,7 @@ Networking configuration models class Network(models.Model): id = models.AutoField(primary_key=True) - name = models.CharField(max_length=100) + name = models.CharField(max_length=200) bundle = models.ForeignKey(ResourceTemplate, on_delete=models.CASCADE, related_name="networks") is_public = models.BooleanField() diff --git a/src/resource_inventory/tests/test_models.py b/src/resource_inventory/tests/test_models.py index e1b2106..3f2d1d8 100644 --- a/src/resource_inventory/tests/test_models.py +++ b/src/resource_inventory/tests/test_models.py @@ -80,7 +80,7 @@ class ConfigUtil(): ) return Image.objects.create( - lab_id=0, + cobbler_id="profile1", from_lab=lab, name="an image for testing", owner=owner diff --git a/src/templates/base/booking/quick_deploy.html b/src/templates/base/booking/quick_deploy.html index 5dc41e2..1193aab 100644 --- a/src/templates/base/booking/quick_deploy.html +++ b/src/templates/base/booking/quick_deploy.html @@ -88,15 +88,22 @@ function imageFilter() { var drop = document.getElementById("id_image"); var lab_pk = get_selected_value("lab"); - var host_pk = get_selected_value("resource"); + var profile_pk = get_selected_value("resource"); for (const childNode of drop.childNodes) { var image_object = sup_image_dict[childNode.value]; if (image_object) //weed out empty option { + console.log("image object:"); + console.log(image_object); const img_at_lab = image_object.lab == lab_pk; - const profiles = resource_profile_map[host_pk]; - const img_in_template = profiles && profiles.indexOf(image_object.host_profile) > -1 + const profiles = resource_profile_map[profile_pk]; + console.log("profiles are:"); + console.log(profiles); + console.log("profile map is:"); + console.log(resource_profile_map); + console.log("host profile is" + image_object.architecture); + const img_in_template = profiles && profiles.indexOf(image_object.architecture) > -1 childNode.disabled = !img_at_lab || !img_in_template; } } diff --git a/src/workflow/resource_bundle_workflow.py b/src/workflow/resource_bundle_workflow.py index 63a9519..a461e9a 100644 --- a/src/workflow/resource_bundle_workflow.py +++ b/src/workflow/resource_bundle_workflow.py @@ -196,7 +196,7 @@ class Define_Software(WorkflowStep): for i, host_data in enumerate(hosts_data): host = ResourceConfiguration.objects.get(pk=host_data['host_id']) wrong_owner = Image.objects.exclude(owner=user).exclude(public=True) - wrong_host = Image.objects.exclude(host_type=host.profile) + wrong_host = Image.objects.exclude(architecture=host.profile.architecture) wrong_lab = Image.objects.exclude(from_lab=lab) excluded_images = wrong_owner | wrong_host | wrong_lab filter_data.append([]) |