From a09db9f287a02873c0226759f8ea444bb304cd59 Mon Sep 17 00:00:00 2001 From: Justin Choquette Date: Thu, 8 Jun 2023 12:46:53 -0400 Subject: LaaS 3.0 Almost MVP Change-Id: Ided9a43cf3088bb58a233dc459711c03f43e11b8 Signed-off-by: Justin Choquette --- src/account/admin.py | 4 +- src/account/migrations/0010_auto_20230608_1913.py | 23 + src/account/models.py | 178 --- src/account/tests/__init__.py | 8 - src/account/tests/test_general.py | 60 - src/account/urls.py | 4 - src/account/views.py | 82 +- src/analytics/admin.py | 5 - .../migrations/0003_delete_activevpnuser.py | 16 + src/analytics/models.py | 22 - src/api/admin.py | 21 - .../migrations/0022_add_cifile_generated_field.py | 15 - src/api/migrations/0022_merge_20211102_2136.py | 14 - src/api/migrations/0022_merge_20230607_1948.py | 14 + src/api/migrations/0023_auto_20230608_1913.py | 172 ++ src/api/models.py | 1275 +-------------- src/api/serializers/__init__.py | 8 - src/api/serializers/booking_serializer.py | 173 -- src/api/serializers/old_serializers.py | 21 - src/api/tests/__init__.py | 8 - src/api/tests/test_models_unittest.py | 271 ---- src/api/urls.py | 44 - src/api/views.py | 756 ++++----- src/booking/forms.py | 114 -- src/booking/migrations/0010_auto_20230608_1913.py | 29 + src/booking/migrations/0011_booking_aggregateid.py | 19 + src/booking/models.py | 33 +- src/booking/quick_deployer.py | 343 ---- src/booking/stats.py | 109 -- src/booking/tests/__init__.py | 8 - src/booking/tests/test_models.py | 210 --- src/booking/tests/test_quick_booking.py | 180 --- src/booking/tests/test_stats.py | 59 - src/booking/urls.py | 8 - src/booking/views.py | 123 +- src/dashboard/admin_utils.py | 811 ---------- src/dashboard/populate_db_iol.py | 352 ----- src/dashboard/tasks.py | 93 +- src/dashboard/testing_utils.py | 315 ---- src/dashboard/tests/__init__.py | 8 - src/dashboard/tests/test_views.py | 30 - src/dashboard/views.py | 34 +- src/laas_dashboard/model_test.py | 110 -- src/laas_dashboard/settings.py | 26 +- src/laas_dashboard/urls.py | 2 - src/notifier/admin.py | 5 - src/notifier/manager.py | 162 -- src/notifier/migrations/0008_auto_20230608_1913.py | 30 + src/notifier/models.py | 48 - src/notifier/tasks.py | 51 - src/notifier/tests/test_dispatcher.py | 15 - src/notifier/tests/test_models.py | 30 - src/notifier/urls.py | 13 +- src/notifier/views.py | 50 - src/resource_inventory/admin.py | 59 - src/resource_inventory/forms.py | 31 - src/resource_inventory/idf_templater.py | 148 -- .../migrations/0023_cloudinitfile_generated.py | 2 +- .../migrations/0024_auto_20230608_1913.py | 262 +++ src/resource_inventory/models.py | 689 +------- src/resource_inventory/pdf_templater.py | 176 --- src/resource_inventory/resource_manager.py | 197 --- src/resource_inventory/tests/test_managers.py | 301 ---- src/resource_inventory/tests/test_models.py | 173 -- src/resource_inventory/urls.py | 27 - src/resource_inventory/views.py | 30 - src/static/css/base.css | 124 +- src/static/js/dashboard.js | 1664 -------------------- src/static/js/workflows/book-a-pod.js | 708 +++++++++ src/static/js/workflows/common-models.js | 189 +++ src/static/js/workflows/design-a-pod.js | 1186 ++++++++++++++ src/static/js/workflows/workflow.js | 246 +++ src/templates/base/account/booking_list.html | 45 +- src/templates/base/account/resource_list.html | 77 +- src/templates/base/base.html | 71 +- src/templates/base/booking/booking_detail.html | 320 +--- src/templates/base/dashboard/genericselect.html | 30 - src/templates/base/dashboard/landing.html | 35 +- .../base/snapshot_workflow/steps/meta.html | 17 - .../base/snapshot_workflow/steps/select_host.html | 83 - src/templates/base/workflow/book_a_pod.html | 121 ++ src/templates/base/workflow/confirm.html | 56 - src/templates/base/workflow/design_a_pod.html | 214 +++ src/templates/base/workflow/no_workflow.html | 3 - src/templates/base/workflow/viewport-base.html | 82 - src/templates/base/workflow/viewport-element.html | 17 - src/workflow/README | 32 +- src/workflow/booking_workflow.py | 182 --- src/workflow/forms.py | 263 +--- src/workflow/models.py | 687 +------- src/workflow/opnfv_workflow.py | 292 ---- src/workflow/resource_bundle_workflow.py | 614 -------- src/workflow/snapshot_workflow.py | 116 -- src/workflow/tests/__init__.py | 8 - src/workflow/tests/constants.py | 198 --- src/workflow/tests/test_fixtures.py | 2 - src/workflow/tests/test_steps.py | 269 ---- src/workflow/tests/test_workflows.py | 99 -- src/workflow/urls.py | 11 +- src/workflow/views.py | 137 +- src/workflow/workflow_factory.py | 126 -- src/workflow/workflow_manager.py | 270 ---- 102 files changed, 3860 insertions(+), 13443 deletions(-) create mode 100644 src/account/migrations/0010_auto_20230608_1913.py delete mode 100644 src/account/tests/__init__.py delete mode 100644 src/account/tests/test_general.py create mode 100644 src/analytics/migrations/0003_delete_activevpnuser.py delete mode 100644 src/api/migrations/0022_add_cifile_generated_field.py delete mode 100644 src/api/migrations/0022_merge_20211102_2136.py create mode 100644 src/api/migrations/0022_merge_20230607_1948.py create mode 100644 src/api/migrations/0023_auto_20230608_1913.py delete mode 100644 src/api/serializers/__init__.py delete mode 100644 src/api/serializers/booking_serializer.py delete mode 100644 src/api/serializers/old_serializers.py delete mode 100644 src/api/tests/__init__.py delete mode 100644 src/api/tests/test_models_unittest.py create mode 100644 src/booking/migrations/0010_auto_20230608_1913.py create mode 100644 src/booking/migrations/0011_booking_aggregateid.py delete mode 100644 src/booking/quick_deployer.py delete mode 100644 src/booking/stats.py delete mode 100644 src/booking/tests/__init__.py delete mode 100644 src/booking/tests/test_models.py delete mode 100644 src/booking/tests/test_quick_booking.py delete mode 100644 src/booking/tests/test_stats.py delete mode 100644 src/dashboard/admin_utils.py delete mode 100644 src/dashboard/populate_db_iol.py delete mode 100644 src/dashboard/testing_utils.py delete mode 100644 src/dashboard/tests/__init__.py delete mode 100644 src/dashboard/tests/test_views.py delete mode 100644 src/laas_dashboard/model_test.py delete mode 100644 src/notifier/manager.py create mode 100644 src/notifier/migrations/0008_auto_20230608_1913.py delete mode 100644 src/notifier/tasks.py delete mode 100644 src/notifier/tests/test_dispatcher.py delete mode 100644 src/notifier/tests/test_models.py delete mode 100644 src/resource_inventory/forms.py delete mode 100644 src/resource_inventory/idf_templater.py create mode 100644 src/resource_inventory/migrations/0024_auto_20230608_1913.py delete mode 100644 src/resource_inventory/pdf_templater.py delete mode 100644 src/resource_inventory/resource_manager.py delete mode 100644 src/resource_inventory/tests/test_managers.py delete mode 100644 src/resource_inventory/tests/test_models.py delete mode 100644 src/static/js/dashboard.js create mode 100644 src/static/js/workflows/book-a-pod.js create mode 100644 src/static/js/workflows/common-models.js create mode 100644 src/static/js/workflows/design-a-pod.js create mode 100644 src/static/js/workflows/workflow.js delete mode 100644 src/templates/base/dashboard/genericselect.html delete mode 100644 src/templates/base/snapshot_workflow/steps/meta.html delete mode 100644 src/templates/base/snapshot_workflow/steps/select_host.html create mode 100644 src/templates/base/workflow/book_a_pod.html delete mode 100644 src/templates/base/workflow/confirm.html create mode 100644 src/templates/base/workflow/design_a_pod.html delete mode 100644 src/templates/base/workflow/no_workflow.html delete mode 100644 src/templates/base/workflow/viewport-base.html delete mode 100644 src/templates/base/workflow/viewport-element.html delete mode 100644 src/workflow/booking_workflow.py delete mode 100644 src/workflow/opnfv_workflow.py delete mode 100644 src/workflow/resource_bundle_workflow.py delete mode 100644 src/workflow/snapshot_workflow.py delete mode 100644 src/workflow/tests/__init__.py delete mode 100644 src/workflow/tests/constants.py delete mode 100644 src/workflow/tests/test_fixtures.py delete mode 100644 src/workflow/tests/test_steps.py delete mode 100644 src/workflow/tests/test_workflows.py delete mode 100644 src/workflow/workflow_factory.py delete mode 100644 src/workflow/workflow_manager.py (limited to 'src') diff --git a/src/account/admin.py b/src/account/admin.py index b4c142c..fd03e60 100644 --- a/src/account/admin.py +++ b/src/account/admin.py @@ -11,9 +11,7 @@ from django.contrib import admin -from account.models import UserProfile, Lab, VlanManager, PublicNetwork +from account.models import UserProfile, Lab admin.site.register(UserProfile) admin.site.register(Lab) -admin.site.register(VlanManager) -admin.site.register(PublicNetwork) diff --git a/src/account/migrations/0010_auto_20230608_1913.py b/src/account/migrations/0010_auto_20230608_1913.py new file mode 100644 index 0000000..3e597b9 --- /dev/null +++ b/src/account/migrations/0010_auto_20230608_1913.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2 on 2023-06-08 19:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('account', '0009_auto_20210324_2107'), + ] + + operations = [ + migrations.RemoveField( + model_name='lab', + name='vlan_manager', + ), + migrations.DeleteModel( + name='PublicNetwork', + ), + migrations.DeleteModel( + name='VlanManager', + ), + ] diff --git a/src/account/models.py b/src/account/models.py index 32229b1..f1deca7 100644 --- a/src/account/models.py +++ b/src/account/models.py @@ -16,9 +16,6 @@ import random from collections import Counter -from dashboard.exceptions import ResourceAvailabilityException - - class LabStatus(object): """ A Poor man's enum for the status of a lab. @@ -63,163 +60,6 @@ class UserProfile(models.Model): def __str__(self): return self.user.username - -class VlanManager(models.Model): - """ - Keeps track of the vlans for a lab. - - Vlans are represented as indexes into a 4096 element list. - This list is serialized to JSON for storing in the DB. - """ - - # list of length 4096 containing either 0 (not available) or 1 (available) - vlans = models.TextField() - # list of length 4096 containing either 0 (not reserved) or 1 (reserved) - reserved_vlans = models.TextField() - - block_size = models.IntegerField() - - # True if the lab allows two different users to have the same private vlans - # if they use QinQ or a vxlan overlay, for example - allow_overlapping = models.BooleanField() - - def get_vlans(self, count=1, within=None): - """ - Return the IDs of available vlans as a list[int], but does not reserve them. - - Will throw index exception if not enough vlans are available. - Always returns a list of ints - - If `within` is not none, will filter against that as a set, requiring that any vlans returned are within that set - """ - allocated = [] - vlans = json.loads(self.vlans) - reserved = json.loads(self.reserved_vlans) - - for i in range(0, len(vlans) - 1): - if len(allocated) >= count: - break - - if vlans[i] == 0 and self.allow_overlapping is False: - continue - - if reserved[i] == 1: - continue - - # vlan is available and not reserved, so safe to add - if within is not None: - if i in within: - allocated.append(i) - else: - allocated.append(i) - continue - - if len(allocated) != count: - raise ResourceAvailabilityException("There were not enough available private vlans for the allocation. Please contact the administrators.") - - return allocated - - def get_public_vlan(self, within=None): - """Return reference to an available public network without reserving it.""" - r = PublicNetwork.objects.filter(lab=self.lab_set.first(), in_use=False) - if within is not None: - r = r.filter(vlan__in=within) - - if r.count() < 1: - raise ResourceAvailabilityException("There were not enough available public vlans for the allocation. Please contact the administrators.") - - return r.first() - - def reserve_public_vlan(self, vlan): - """Reserves the Public Network that has the given vlan.""" - net = PublicNetwork.objects.get(lab=self.lab_set.first(), vlan=vlan, in_use=False) - net.in_use = True - net.save() - - def release_public_vlan(self, vlan): - """Un-reserves a public network with the given vlan.""" - net = PublicNetwork.objects.get(lab=self.lab_set.first(), vlan=vlan, in_use=True) - net.in_use = False - net.save() - - def public_vlan_is_available(self, vlan): - """ - Whether the public vlan is available. - - returns true if the network with the given vlan is free to use, - False otherwise - """ - net = PublicNetwork.objects.get(lab=self.lab_set.first(), vlan=vlan) - return not net.in_use - - def is_available(self, vlans): - """ - If the vlans are available. - - 'vlans' is either a single vlan id integer or a list of integers - will return true (available) or false - """ - if self.allow_overlapping: - return True - - reserved = json.loads(self.reserved_vlans) - vlan_master_list = json.loads(self.vlans) - try: - iter(vlans) - except Exception: - vlans = [vlans] - - for vlan in vlans: - if not vlan_master_list[vlan] or reserved[vlan]: - return False - return True - - def release_vlans(self, vlans): - """ - Make the vlans available for another booking. - - 'vlans' is either a single vlan id integer or a list of integers - will make the vlans available - doesnt return a value - """ - my_vlans = json.loads(self.vlans) - - try: - iter(vlans) - except Exception: - vlans = [vlans] - - for vlan in vlans: - my_vlans[vlan] = 1 - self.vlans = json.dumps(my_vlans) - self.save() - - def reserve_vlans(self, vlans): - """ - Reserves all given vlans or throws a ValueError. - - vlans can be an integer or a list of integers. - """ - my_vlans = json.loads(self.vlans) - - reserved = json.loads(self.reserved_vlans) - - try: - iter(vlans) - except Exception: - vlans = [vlans] - - vlans = set(vlans) - - for vlan in vlans: - if my_vlans[vlan] == 0 or reserved[vlan] == 1: - raise ValueError("vlan " + str(vlan) + " is not available") - - my_vlans[vlan] = 0 - self.vlans = json.dumps(my_vlans) - self.save() - - class Lab(models.Model): """ Model representing a Hosting Lab. @@ -233,7 +73,6 @@ class Lab(models.Model): contact_email = models.EmailField(max_length=200, null=True, blank=True) contact_phone = models.CharField(max_length=20, null=True, blank=True) status = models.IntegerField(default=LabStatus.UP) - vlan_manager = models.ForeignKey(VlanManager, on_delete=models.CASCADE, null=True) location = models.TextField(default="unknown") # This token must apear in API requests from this lab api_token = models.CharField(max_length=50) @@ -250,26 +89,9 @@ class Lab(models.Model): key += random.choice(alphabet) return key - def get_available_resources(self): - # Cannot import model normally due to ciruclar import - Server = apps.get_model('resource_inventory', 'Server') # TODO: Find way to import ResourceQuery - resources = [str(resource.profile) for resource in Server.objects.filter(lab=self, working=True, booked=False)] - return dict(Counter(resources)) - def __str__(self): return self.name - -class PublicNetwork(models.Model): - """L2/L3 network that can reach the internet.""" - - vlan = models.IntegerField() - lab = models.ForeignKey(Lab, on_delete=models.CASCADE) - in_use = models.BooleanField(default=False) - cidr = models.CharField(max_length=50, default="0.0.0.0/0") - gateway = models.CharField(max_length=50, default="0.0.0.0") - - class Downtime(models.Model): """ A Downtime event. diff --git a/src/account/tests/__init__.py b/src/account/tests/__init__.py deleted file mode 100644 index b6fef6c..0000000 --- a/src/account/tests/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## diff --git a/src/account/tests/test_general.py b/src/account/tests/test_general.py deleted file mode 100644 index 4020d89..0000000 --- a/src/account/tests/test_general.py +++ /dev/null @@ -1,60 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from django.contrib.auth.models import User -from django.test import Client -from django.test import TestCase -from django.urls import reverse -from django.utils import timezone - -from account.models import UserProfile - - -class AccountMiddlewareTestCase(TestCase): - def setUp(self): - self.client = Client() - self.user1 = User.objects.create(username='user1') - self.user1.set_password('user1') - self.user1profile = UserProfile.objects.create(user=self.user1) - self.user1.save() - - def test_timezone_middleware(self): - """ - Verify timezone is being set by Middleware. - - The timezone should be UTC for anonymous users, - for authenticated users it should be set to user.userprofile.timezone - """ - # default - self.assertEqual(timezone.get_current_timezone_name(), 'UTC') - - url = reverse('account:settings') - # anonymous request - self.client.get(url) - self.assertEqual(timezone.get_current_timezone_name(), 'UTC') - - # authenticated user with UTC timezone (userprofile default) - self.client.login(username='user1', password='user1') - self.client.get(url) - self.assertEqual(timezone.get_current_timezone_name(), 'UTC') - - # authenticated user with custom timezone (userprofile default) - self.user1profile.timezone = 'Etc/Greenwich' - self.user1profile.save() - self.client.get(url) - self.assertEqual(timezone.get_current_timezone_name(), 'GMT') - - # if there is no profile for a user, it should be created - user2 = User.objects.create(username='user2') - user2.set_password('user2') - user2.save() - self.client.login(username='user2', password='user2') - self.client.get(url) - self.assertTrue(user2.userprofile) diff --git a/src/account/urls.py b/src/account/urls.py index 6d4ef2f..23ce122 100644 --- a/src/account/urls.py +++ b/src/account/urls.py @@ -35,11 +35,9 @@ from account.views import ( UserListView, account_resource_view, account_booking_view, - account_images_view, account_detail_view, template_delete_view, booking_cancel_view, - image_delete_view, ) app_name = 'account' @@ -53,7 +51,5 @@ urlpatterns = [ path('my/resources/delete/', template_delete_view), url(r'^my/bookings/$', account_booking_view, name='my-bookings'), path('my/bookings/cancel/', booking_cancel_view), - url(r'^my/images/$', account_images_view, name='my-images'), - path('my/images/delete/', image_delete_view), url(r'^my/$', account_detail_view, name='my-account'), ] diff --git a/src/account/views.py b/src/account/views.py index 8976ff9..2d280cd 100644 --- a/src/account/views.py +++ b/src/account/views.py @@ -9,6 +9,7 @@ ############################################################################## +import json import os from django.utils import timezone @@ -30,9 +31,7 @@ from mozilla_django_oidc.auth import OIDCAuthenticationBackend from account.forms import AccountSettingsForm from account.models import UserProfile from booking.models import Booking -from resource_inventory.models import ResourceTemplate, Image - - +from api.views import delete_template, liblaas_templates @method_decorator(login_required, name='dispatch') class AccountSettingsView(UpdateView): model = UserProfile @@ -134,17 +133,21 @@ def account_resource_view(request): return render(request, "dashboard/login.html", {'title': 'Authentication Required'}) template = "account/resource_list.html" - active_bundles = [book.resource for book in Booking.objects.filter( - owner=request.user, end__gte=timezone.now(), resource__template__temporary=False)] - active_resources = [bundle.template.id for bundle in active_bundles] - resource_list = list(ResourceTemplate.objects.filter(owner=request.user, temporary=False)) + if request.method == "GET": - context = { - "resources": resource_list, - "active_resources": active_resources, - "title": "My Resources" - } - return render(request, template, context=context) + r = liblaas_templates(request) + usable_templates = r.json() + user_templates = [ t for t in usable_templates if t["owner"] == str(request.user)] + context = { + "templates": user_templates, + "title": "My Resources" + } + return render(request, template, context=context) + + if request.method == "POST": + return delete_template(request) + + return HttpResponse(status_code=405) def account_booking_view(request): @@ -165,37 +168,20 @@ def account_booking_view(request): return render(request, template, context=context) -def account_images_view(request): - if not request.user.is_authenticated: - return render(request, "dashboard/login.html", {'title': 'Authentication Required'}) - template = "account/image_list.html" - my_images = Image.objects.filter(owner=request.user) - public_images = Image.objects.filter(public=True) - used_images = {} - for image in my_images: - if image.in_use(): - used_images[image.id] = "true" - context = { - "title": "Images", - "images": my_images, - "public_images": public_images, - "used_images": used_images - } - return render(request, template, context=context) - def template_delete_view(request, resource_id=None): - if not request.user.is_authenticated: - return HttpResponse(status=403) - template = get_object_or_404(ResourceTemplate, pk=resource_id) - if not request.user.id == template.owner.id: - return HttpResponse(status=403) - if Booking.objects.filter(resource__template=template, end__gt=timezone.now()).exists(): - return HttpResponse(status=403) - template.public = False - template.temporary = True - template.save() - return HttpResponse(status=200) + # if not request.user.is_authenticated: + # return HttpResponse(status=403) + # template = get_object_or_404(ResourceTemplate, pk=resource_id) + # if not request.user.id == template.owner.id: + # return HttpResponse(status=403) + # if Booking.objects.filter(resource__template=template, end__gt=timezone.now()).exists(): + # return HttpResponse(status=403) + # template.public = False + # template.temporary = True + # template.save() + # return HttpResponse(status=200) + return HttpResponse(status=404) # todo - LL Integration def booking_cancel_view(request, booking_id=None): @@ -212,15 +198,3 @@ def booking_cancel_view(request, booking_id=None): booking.save() return HttpResponse('') - -def image_delete_view(request, image_id=None): - if not request.user.is_authenticated: - return HttpResponse('no') # 403? - image = get_object_or_404(Image, pk=image_id) - if image.public or image.owner.id != request.user.id: - return HttpResponse('no') # 403? - # check if used in booking - if image.in_use(): - return HttpResponse('no') # 403? - image.delete() - return HttpResponse('') diff --git a/src/analytics/admin.py b/src/analytics/admin.py index 63f139f..043bad6 100644 --- a/src/analytics/admin.py +++ b/src/analytics/admin.py @@ -6,8 +6,3 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - -from django.contrib import admin -from analytics.models import ActiveVPNUser - -admin.site.register(ActiveVPNUser) diff --git a/src/analytics/migrations/0003_delete_activevpnuser.py b/src/analytics/migrations/0003_delete_activevpnuser.py new file mode 100644 index 0000000..4d21250 --- /dev/null +++ b/src/analytics/migrations/0003_delete_activevpnuser.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2 on 2023-06-08 19:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('analytics', '0002_auto_20201109_2149'), + ] + + operations = [ + migrations.DeleteModel( + name='ActiveVPNUser', + ), + ] diff --git a/src/analytics/models.py b/src/analytics/models.py index 10baa0c..043bad6 100644 --- a/src/analytics/models.py +++ b/src/analytics/models.py @@ -6,25 +6,3 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - -from django.db import models -from account.models import Lab - - -class ActiveVPNUser(models.Model): - """ Keeps track of how many VPN Users are connected to Lab """ - time_stamp = models.DateTimeField(auto_now_add=True) - lab = models.ForeignKey(Lab, on_delete=models.CASCADE, null=False) - active_users = models.IntegerField() - - @classmethod - def create(cls, lab_name, active_users): - """ - This creates an Active VPN Users entry from - from lab_name as a string - """ - - lab = Lab.objects.get(name=lab_name) - avu = cls(lab=lab, active_users=active_users) - avu.save() - return avu diff --git a/src/api/admin.py b/src/api/admin.py index 1e243a0..74b023e 100644 --- a/src/api/admin.py +++ b/src/api/admin.py @@ -12,16 +12,6 @@ from django.apps import AppConfig from django.contrib import admin from api.models import ( - Job, - OpnfvApiConfig, - HardwareConfig, - NetworkConfig, - SoftwareConfig, - AccessConfig, - AccessRelation, - SoftwareRelation, - HostHardwareRelation, - HostNetworkRelation, APILog ) @@ -29,15 +19,4 @@ from api.models import ( class ApiConfig(AppConfig): name = 'apiJobs' - -admin.site.register(Job) -admin.site.register(OpnfvApiConfig) -admin.site.register(HardwareConfig) -admin.site.register(NetworkConfig) -admin.site.register(SoftwareConfig) -admin.site.register(AccessConfig) -admin.site.register(AccessRelation) -admin.site.register(SoftwareRelation) -admin.site.register(HostHardwareRelation) -admin.site.register(HostNetworkRelation) admin.site.register(APILog) diff --git a/src/api/migrations/0022_add_cifile_generated_field.py b/src/api/migrations/0022_add_cifile_generated_field.py deleted file mode 100644 index f83a102..0000000 --- a/src/api/migrations/0022_add_cifile_generated_field.py +++ /dev/null @@ -1,15 +0,0 @@ -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ('api', '0018_cloudinitfile'), - ] - - operations = [ - migrations.AddField( - model_name="CloudInitFile", - name="generated", - field=models.BooleanField(default=False) - ), - ] diff --git a/src/api/migrations/0022_merge_20211102_2136.py b/src/api/migrations/0022_merge_20211102_2136.py deleted file mode 100644 index bb27ae4..0000000 --- a/src/api/migrations/0022_merge_20211102_2136.py +++ /dev/null @@ -1,14 +0,0 @@ -# Generated by Django 2.2 on 2021-11-02 21:36 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('api', '0019_auto_20210907_1448'), - ('api', '0021_auto_20210405_1943'), - ] - - operations = [ - ] diff --git a/src/api/migrations/0022_merge_20230607_1948.py b/src/api/migrations/0022_merge_20230607_1948.py new file mode 100644 index 0000000..2c6fae5 --- /dev/null +++ b/src/api/migrations/0022_merge_20230607_1948.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2 on 2023-06-07 19:48 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0021_auto_20210405_1943'), + ('api', '0019_auto_20210907_1448'), + ] + + operations = [ + ] diff --git a/src/api/migrations/0023_auto_20230608_1913.py b/src/api/migrations/0023_auto_20230608_1913.py new file mode 100644 index 0000000..2bc986c --- /dev/null +++ b/src/api/migrations/0023_auto_20230608_1913.py @@ -0,0 +1,172 @@ +# Generated by Django 2.2 on 2023-06-08 19:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0022_merge_20230607_1948'), + ] + + operations = [ + migrations.RemoveField( + model_name='accessrelation', + name='config', + ), + migrations.RemoveField( + model_name='accessrelation', + name='job', + ), + migrations.RemoveField( + model_name='activeusersrelation', + name='config', + ), + migrations.RemoveField( + model_name='activeusersrelation', + name='job', + ), + migrations.RemoveField( + model_name='bridgeconfig', + name='interfaces', + ), + migrations.RemoveField( + model_name='bridgeconfig', + name='opnfv_config', + ), + migrations.RemoveField( + model_name='generatedcloudconfig', + name='booking', + ), + migrations.RemoveField( + model_name='generatedcloudconfig', + name='rconfig', + ), + migrations.RemoveField( + model_name='hardwareconfig', + name='taskconfig_ptr', + ), + migrations.RemoveField( + model_name='hosthardwarerelation', + name='config', + ), + migrations.RemoveField( + model_name='hosthardwarerelation', + name='job', + ), + migrations.RemoveField( + model_name='hostnetworkrelation', + name='config', + ), + migrations.RemoveField( + model_name='hostnetworkrelation', + name='job', + ), + migrations.RemoveField( + model_name='job', + name='booking', + ), + migrations.RemoveField( + model_name='networkconfig', + name='interfaces', + ), + migrations.RemoveField( + model_name='networkconfig', + name='taskconfig_ptr', + ), + migrations.RemoveField( + model_name='opnfvapiconfig', + name='bridge_config', + ), + migrations.RemoveField( + model_name='opnfvapiconfig', + name='opnfv_config', + ), + migrations.RemoveField( + model_name='opnfvapiconfig', + name='roles', + ), + migrations.RemoveField( + model_name='snapshotconfig', + name='taskconfig_ptr', + ), + migrations.RemoveField( + model_name='snapshotrelation', + name='config', + ), + migrations.RemoveField( + model_name='snapshotrelation', + name='job', + ), + migrations.RemoveField( + model_name='snapshotrelation', + name='snapshot', + ), + migrations.RemoveField( + model_name='softwareconfig', + name='opnfv', + ), + migrations.RemoveField( + model_name='softwareconfig', + name='taskconfig_ptr', + ), + migrations.RemoveField( + model_name='softwarerelation', + name='config', + ), + migrations.RemoveField( + model_name='softwarerelation', + name='job', + ), + migrations.DeleteModel( + name='AccessConfig', + ), + migrations.DeleteModel( + name='AccessRelation', + ), + migrations.DeleteModel( + name='ActiveUsersConfig', + ), + migrations.DeleteModel( + name='ActiveUsersRelation', + ), + migrations.DeleteModel( + name='BridgeConfig', + ), + migrations.DeleteModel( + name='GeneratedCloudConfig', + ), + migrations.DeleteModel( + name='HardwareConfig', + ), + migrations.DeleteModel( + name='HostHardwareRelation', + ), + migrations.DeleteModel( + name='HostNetworkRelation', + ), + migrations.DeleteModel( + name='Job', + ), + migrations.DeleteModel( + name='NetworkConfig', + ), + migrations.DeleteModel( + name='OpnfvApiConfig', + ), + migrations.DeleteModel( + name='SnapshotConfig', + ), + migrations.DeleteModel( + name='SnapshotRelation', + ), + migrations.DeleteModel( + name='SoftwareConfig', + ), + migrations.DeleteModel( + name='SoftwareRelation', + ), + migrations.DeleteModel( + name='TaskConfig', + ), + ] diff --git a/src/api/models.py b/src/api/models.py index 93168f5..ca33ed8 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -23,40 +23,10 @@ import yaml import re from booking.models import Booking -from resource_inventory.models import ( - Lab, - ResourceProfile, - Image, - Opsys, - Interface, - ResourceOPNFVConfig, - RemoteInfo, - OPNFVConfig, - ConfigState, - ResourceQuery, - ResourceConfiguration, - CloudInitFile -) -from resource_inventory.idf_templater import IDFTemplater -from resource_inventory.pdf_templater import PDFTemplater from account.models import Downtime, UserProfile from dashboard.utils import AbstractModelQuery -class JobStatus: - """ - A poor man's enum for a job's status. - - A job is NEW if it has not been started or recognized by the Lab - A job is CURRENT if it has been started by the lab but it is not yet completed - a job is DONE if all the tasks are complete and the booking is ready to use - """ - - NEW = 0 - CURRENT = 100 - DONE = 200 - ERROR = 300 - class LabManagerTracker: @@ -89,18 +59,6 @@ class LabManager: def __init__(self, lab): self.lab = lab - def get_opsyss(self): - return Opsys.objects.filter(from_lab=self.lab) - - def get_images(self): - return Image.objects.filter(from_lab=self.lab) - - def get_image(self, image_id): - return Image.objects.filter(from_lab=self.lab, lab_id=image_id) - - def get_opsys(self, opsys_id): - return Opsys.objects.filter(from_lab=self.lab, lab_id=opsys_id) - def get_downtime(self): return Downtime.objects.filter(start__lt=timezone.now(), end__gt=timezone.now(), lab=self.lab) @@ -135,50 +93,6 @@ class LabManager: ) return self.get_downtime_json() - def update_host_remote_info(self, data, res_id): - resource = ResourceQuery.filter(labid=res_id, lab=self.lab) - if len(resource) != 1: - return HttpResponseNotFound("Could not find single host with id " + str(res_id)) - resource = resource[0] - info = {} - try: - info['address'] = data['address'] - info['mac_address'] = data['mac_address'] - info['password'] = data['password'] - info['user'] = data['user'] - info['type'] = data['type'] - info['versions'] = json.dumps(data['versions']) - except Exception as e: - return {"error": "invalid arguement: " + str(e)} - remote_info = resource.remote_management - if "default" in remote_info.mac_address: - remote_info = RemoteInfo() - remote_info.address = info['address'] - remote_info.mac_address = info['mac_address'] - remote_info.password = info['password'] - remote_info.user = info['user'] - remote_info.type = info['type'] - remote_info.versions = info['versions'] - remote_info.save() - resource.remote_management = remote_info - resource.save() - booking = Booking.objects.get(resource=resource.bundle) - self.update_xdf(booking) - return {"status": "success"} - - def update_xdf(self, booking): - booking.pdf = PDFTemplater.makePDF(booking) - booking.idf = IDFTemplater().makeIDF(booking) - booking.save() - - def get_pdf(self, booking_id): - booking = get_object_or_404(Booking, pk=booking_id, lab=self.lab) - return booking.pdf - - def get_idf(self, booking_id): - booking = get_object_or_404(Booking, pk=booking_id, lab=self.lab) - return booking.idf - def get_profile(self): prof = {} prof['name'] = self.lab.name @@ -214,299 +128,12 @@ class LabManager: return json.dumps(self.format_user(profile)) - def get_inventory(self): - inventory = {} - resources = ResourceQuery.filter(lab=self.lab) - images = Image.objects.filter(from_lab=self.lab) - profiles = ResourceProfile.objects.filter(labs=self.lab) - inventory['resources'] = self.serialize_resources(resources) - inventory['images'] = self.serialize_images(images) - inventory['host_types'] = self.serialize_host_profiles(profiles) - return inventory - - def get_host(self, hostname): - resource = ResourceQuery.filter(labid=hostname, lab=self.lab) - if len(resource) != 1: - return HttpResponseNotFound("Could not find single host with id " + str(hostname)) - resource = resource[0] - return { - "booked": resource.booked, - "working": resource.working, - "type": resource.profile.name - } - - def update_host(self, hostname, data): - resource = ResourceQuery.filter(labid=hostname, lab=self.lab) - if len(resource) != 1: - return HttpResponseNotFound("Could not find single host with id " + str(hostname)) - resource = resource[0] - if "working" in data: - working = data['working'] == "true" - resource.working = working - resource.save() - return self.get_host(hostname) - def get_status(self): return {"status": self.lab.status} def set_status(self, payload): {} - def get_current_jobs(self): - jobs = Job.objects.filter(booking__lab=self.lab) - - return self.serialize_jobs(jobs, status=JobStatus.CURRENT) - - def get_new_jobs(self): - jobs = Job.objects.filter(booking__lab=self.lab) - - return self.serialize_jobs(jobs, status=JobStatus.NEW) - - def get_done_jobs(self): - jobs = Job.objects.filter(booking__lab=self.lab) - - return self.serialize_jobs(jobs, status=JobStatus.DONE) - - def get_analytics_job(self): - """ Get analytics job with status new """ - jobs = Job.objects.filter( - booking__lab=self.lab, - job_type='DATA' - ) - - return self.serialize_jobs(jobs, status=JobStatus.NEW) - - def get_job(self, jobid): - return Job.objects.get(pk=jobid).to_dict() - - def update_job(self, jobid, data): - {} - - def serialize_jobs(self, jobs, status=JobStatus.NEW): - job_ser = [] - for job in jobs: - jsonized_job = job.get_delta(status) - if len(jsonized_job['payload']) < 1: - continue - job_ser.append(jsonized_job) - - return job_ser - - def serialize_resources(self, resources): - # TODO: rewrite for Resource model - host_ser = [] - for res in resources: - r = { - 'interfaces': [], - 'hostname': res.name, - 'host_type': res.profile.name - } - for iface in res.get_interfaces(): - r['interfaces'].append({ - 'mac': iface.mac_address, - 'busaddr': iface.bus_address, - 'name': iface.name, - 'switchport': {"switch_name": iface.switch_name, "port_name": iface.port_name} - }) - return host_ser - - def serialize_images(self, images): - images_ser = [] - for image in images: - images_ser.append( - { - "name": image.name, - "lab_id": image.lab_id, - "dashboard_id": image.id - } - ) - return images_ser - - def serialize_resource_profiles(self, profiles): - profile_ser = [] - for profile in profiles: - p = {} - p['cpu'] = { - "cores": profile.cpuprofile.first().cores, - "arch": profile.cpuprofile.first().architecture, - "cpus": profile.cpuprofile.first().cpus, - } - p['disks'] = [] - for disk in profile.storageprofile.all(): - d = { - "size": disk.size, - "type": disk.media_type, - "name": disk.name - } - p['disks'].append(d) - p['description'] = profile.description - p['interfaces'] = [] - for iface in profile.interfaceprofile.all(): - p['interfaces'].append( - { - "speed": iface.speed, - "name": iface.name - } - ) - - p['ram'] = {"amount": profile.ramprofile.first().amount} - p['name'] = profile.name - profile_ser.append(p) - return profile_ser - - -class GeneratedCloudConfig(models.Model): - resource_id = models.CharField(max_length=200) - booking = models.ForeignKey(Booking, on_delete=models.CASCADE) - rconfig = models.ForeignKey(ResourceConfiguration, on_delete=models.CASCADE) - text = models.TextField(null=True, blank=True) - - def _normalize_username(self, username: str) -> str: - # TODO: make usernames posix compliant - s = re.sub(r'\W+', '', username) - return s - - def _get_ssh_string(self, username: str) -> str: - user = User.objects.get(username=username) - uprofile = user.userprofile - - ssh_file = uprofile.ssh_public_key - - escaped_file = ssh_file.open().read().decode(encoding="UTF-8").replace("\n", " ") - - return escaped_file - - def _serialize_users(self): - """ - returns the dictionary to be placed behind the `users` field of the toplevel c-i dict - """ - # conserves distro default user - user_array = ["default"] - - users = list(self.booking.collaborators.all()) - users.append(self.booking.owner) - for collaborator in users: - userdict = {} - - # TODO: validate if usernames are valid as linux usernames (and provide an override potentially) - userdict['name'] = self._normalize_username(collaborator.username) - - userdict['groups'] = "sudo" - userdict['sudo'] = "ALL=(ALL) NOPASSWD:ALL" - - userdict['ssh_authorized_keys'] = [self._get_ssh_string(collaborator.username)] - - user_array.append(userdict) - - # user_array.append({ - # "name": "opnfv", - # "passwd": "$6$k54L.vim1cLaEc4$5AyUIrufGlbtVBzuCWOlA1yV6QdD7Gr2MzwIs/WhuYR9ebSfh3Qlb7djkqzjwjxpnSAonK1YOabPP6NxUDccu.", - # "ssh_redirect_user": True, - # "sudo": "ALL=(ALL) NOPASSWD:ALL", - # "groups": "sudo", - # }) - - return user_array - - # TODO: make this configurable - def _serialize_sysinfo(self): - defuser = {} - defuser['name'] = 'opnfv' - defuser['plain_text_passwd'] = 'OPNFV_HOST' - defuser['home'] = '/home/opnfv' - defuser['shell'] = '/bin/bash' - defuser['lock_passwd'] = True - defuser['gecos'] = 'Lab Manager User' - defuser['groups'] = 'sudo' - - return {'default_user': defuser} - - # TODO: make this configurable - def _serialize_runcmds(self): - cmdlist = [] - - # have hosts run dhcp on boot - cmdlist.append(['sudo', 'dhclient', '-r']) - cmdlist.append(['sudo', 'dhclient']) - - return cmdlist - - def _serialize_netconf_v1(self): - # interfaces = {} # map from iface_name => dhcp_config - # vlans = {} # map from vlan_id => dhcp_config - - config_arr = [] - - for interface in self._resource().interfaces.all(): - interface_name = interface.profile.name - interface_mac = interface.mac_address - - iface_dict_entry = { - "type": "physical", - "name": interface_name, - "mac_address": interface_mac, - } - - for vlan in interface.config.all(): - if vlan.tagged: - vlan_dict_entry = {'type': 'vlan'} - vlan_dict_entry['name'] = str(interface_name) + "." + str(vlan.vlan_id) - vlan_dict_entry['vlan_link'] = str(interface_name) - vlan_dict_entry['vlan_id'] = int(vlan.vlan_id) - vlan_dict_entry['mac_address'] = str(interface_mac) - if vlan.public: - vlan_dict_entry["subnets"] = [{"type": "dhcp"}] - config_arr.append(vlan_dict_entry) - if (not vlan.tagged) and vlan.public: - iface_dict_entry["subnets"] = [{"type": "dhcp"}] - - # vlan_dict_entry['mtu'] = # TODO, determine override MTU if needed - - config_arr.append(iface_dict_entry) - - ns_dict = { - 'type': 'nameserver', - 'address': ['10.64.0.1', '8.8.8.8'] - } - - config_arr.append(ns_dict) - - full_dict = {'version': 1, 'config': config_arr} - - return full_dict - - @classmethod - def get(cls, booking_id: int, resource_lab_id: str, file_id: int): - return GeneratedCloudConfig.objects.get(resource_id=resource_lab_id, booking__id=booking_id, file_id=file_id) - - def _resource(self): - return ResourceQuery.get(labid=self.resource_id, lab=self.booking.lab) - - # def _get_facts(self): - # resource = self._resource() - - # hostname = self.rconfig.name - # iface_configs = for_config.interface_configs.all() - - def _to_dict(self): - main_dict = {} - - main_dict['users'] = self._serialize_users() - main_dict['network'] = self._serialize_netconf_v1() - main_dict['hostname'] = self.rconfig.name - - # add first startup commands - main_dict['runcmd'] = self._serialize_runcmds() - - # configure distro default user - main_dict['system_info'] = self._serialize_sysinfo() - - return main_dict - - def serialize(self) -> str: - return yaml.dump(self._to_dict(), width=float("inf")) - - class APILog(models.Model): user = models.ForeignKey(User, on_delete=models.PROTECT) call_time = models.DateTimeField(auto_now=True) @@ -534,7 +161,6 @@ class AutomationAPIManager: sbook['end'] = booking.end sbook['lab'] = AutomationAPIManager.serialize_lab(booking.lab) sbook['purpose'] = booking.purpose - sbook['resourceBundle'] = AutomationAPIManager.serialize_bundle(booking.resource) return sbook @staticmethod @@ -544,51 +170,6 @@ class AutomationAPIManager: slab['name'] = lab.name return slab - @staticmethod - def serialize_bundle(bundle): - sbundle = {} - sbundle['id'] = bundle.pk - sbundle['resources'] = [ - AutomationAPIManager.serialize_server(server) - for server in bundle.get_resources()] - return sbundle - - @staticmethod - def serialize_server(server): - sserver = {} - sserver['id'] = server.pk - sserver['name'] = server.name - return sserver - - @staticmethod - def serialize_resource_profile(profile): - sprofile = {} - sprofile['id'] = profile.pk - sprofile['name'] = profile.name - return sprofile - - @staticmethod - def serialize_template(rec_temp_and_count): - template = rec_temp_and_count[0] - count = rec_temp_and_count[1] - - stemplate = {} - stemplate['id'] = template.pk - stemplate['name'] = template.name - stemplate['count_available'] = count - stemplate['resourceProfiles'] = [ - AutomationAPIManager.serialize_resource_profile(config.profile) - for config in template.getConfigs() - ] - return stemplate - - @staticmethod - def serialize_image(image): - simage = {} - simage['id'] = image.pk - simage['name'] = image.name - return simage - @staticmethod def serialize_userprofile(up): sup = {} @@ -596,858 +177,6 @@ class AutomationAPIManager: sup['username'] = up.user.username return sup - -class Job(models.Model): - """ - A Job to be performed by the Lab. - - The API uses Jobs and Tasks to communicate actions that need to be taken to the Lab - that is hosting a booking. A booking from a user has an associated Job which tells - the lab how to configure the hardware, networking, etc to fulfill the booking - for the user. - This is the class that is serialized and put into the api - """ - - JOB_TYPES = ( - ('BOOK', 'Booking'), - ('DATA', 'Analytics') - ) - - booking = models.OneToOneField(Booking, on_delete=models.CASCADE, null=True) - status = models.IntegerField(default=JobStatus.NEW) - complete = models.BooleanField(default=False) - job_type = models.CharField( - max_length=4, - choices=JOB_TYPES, - default='BOOK' - ) - - def to_dict(self): - d = {} - for relation in self.get_tasklist(): - if relation.job_key not in d: - d[relation.job_key] = {} - d[relation.job_key][relation.task_id] = relation.config.to_dict() - - return {"id": self.id, "payload": d} - - def get_tasklist(self, status="all"): - if status != "all": - return JobTaskQuery.filter(job=self, status=status) - return JobTaskQuery.filter(job=self) - - def is_fulfilled(self): - """ - If a job has been completed by the lab. - - This method should return true if all of the job's tasks are done, - and false otherwise - """ - my_tasks = self.get_tasklist() - for task in my_tasks: - if task.status != JobStatus.DONE: - return False - return True - - def get_delta(self, status): - d = {} - for relation in self.get_tasklist(status=status): - if relation.job_key not in d: - d[relation.job_key] = {} - d[relation.job_key][relation.task_id] = relation.config.get_delta() - - return {"id": self.id, "payload": d} - - def to_json(self): - return json.dumps(self.to_dict()) - - -class TaskConfig(models.Model): - state = models.IntegerField(default=ConfigState.NEW) - - keys = set() # TODO: This needs to be an instance variable, not a class variable - delta_keys_list = models.CharField(max_length=200, default="[]") - - @property - def delta_keys(self): - return list(set(json.loads(self.delta_keys_list))) - - @delta_keys.setter - def delta_keys(self, keylist): - self.delta_keys_list = json.dumps(keylist) - - def to_dict(self): - raise NotImplementedError - - def get_delta(self): - raise NotImplementedError - - def format_delta(self, config, token): - delta = {k: config[k] for k in self.delta_keys} - delta['lab_token'] = token - return delta - - def to_json(self): - return json.dumps(self.to_dict()) - - def clear_delta(self): - self.delta_keys = [] - - def set(self, *args): - dkeys = self.delta_keys - for arg in args: - if arg in self.keys: - dkeys.append(arg) - self.delta_keys = dkeys - - -class BridgeConfig(models.Model): - """Displays mapping between jumphost interfaces and bridges.""" - - interfaces = models.ManyToManyField(Interface) - opnfv_config = models.ForeignKey(OPNFVConfig, on_delete=models.CASCADE) - - def to_dict(self): - d = {} - hid = ResourceQuery.get(interface__pk=self.interfaces.first().pk).labid - d[hid] = {} - for interface in self.interfaces.all(): - d[hid][interface.mac_address] = [] - for vlan in interface.config.all(): - network_role = self.opnfv_model.networks().filter(network=vlan.network) - bridge = IDFTemplater.bridge_names[network_role.name] - br_config = { - "vlan_id": vlan.vlan_id, - "tagged": vlan.tagged, - "bridge": bridge - } - d[hid][interface.mac_address].append(br_config) - return d - - def to_json(self): - return json.dumps(self.to_dict()) - - -class ActiveUsersConfig(models.Model): - """ - Task for getting active VPN users - - StackStorm needs no information to run this job - so this task is very bare, but neccessary to fit - job creation convention. - """ - - def clear_delta(self): - self.delta = '{}' - - def get_delta(self): - return json.loads(self.to_json()) - - def to_json(self): - return json.dumps(self.to_dict()) - - def to_dict(self): - return {} - - -class OpnfvApiConfig(models.Model): - - installer = models.CharField(max_length=200) - scenario = models.CharField(max_length=300) - roles = models.ManyToManyField(ResourceOPNFVConfig) - # pdf and idf are url endpoints, not the actual file - pdf = models.CharField(max_length=100) - idf = models.CharField(max_length=100) - bridge_config = models.OneToOneField(BridgeConfig, on_delete=models.CASCADE, null=True) - delta = models.TextField() - opnfv_config = models.ForeignKey(OPNFVConfig, null=True, on_delete=models.SET_NULL) - - def to_dict(self): - d = {} - if not self.opnfv_config: - return d - if self.installer: - d['installer'] = self.installer - if self.scenario: - d['scenario'] = self.scenario - if self.pdf: - d['pdf'] = self.pdf - if self.idf: - d['idf'] = self.idf - if self.bridge_config: - d['bridged_interfaces'] = self.bridge_config.to_dict() - - hosts = self.roles.all() - if hosts.exists(): - d['roles'] = [] - for host in hosts: - d['roles'].append({ - host.labid: self.opnfv_config.host_opnfv_config.get( - host_config__pk=host.config.pk - ).role.name - }) - - return d - - def to_json(self): - return json.dumps(self.to_dict()) - - def set_installer(self, installer): - self.installer = installer - d = json.loads(self.delta) - d['installer'] = installer - self.delta = json.dumps(d) - - def set_scenario(self, scenario): - self.scenario = scenario - d = json.loads(self.delta) - d['scenario'] = scenario - self.delta = json.dumps(d) - - def set_xdf(self, booking, update_delta=True): - kwargs = {'lab_name': booking.lab.name, 'booking_id': booking.id} - self.pdf = reverse('get-pdf', kwargs=kwargs) - self.idf = reverse('get-idf', kwargs=kwargs) - if update_delta: - d = json.loads(self.delta) - d['pdf'] = self.pdf - d['idf'] = self.idf - self.delta = json.dumps(d) - - def add_role(self, host): - self.roles.add(host) - d = json.loads(self.delta) - if 'role' not in d: - d['role'] = [] - d['roles'].append({host.labid: host.config.opnfvRole.name}) - self.delta = json.dumps(d) - - def clear_delta(self): - self.delta = '{}' - - def get_delta(self): - return json.loads(self.to_json()) - - -class AccessConfig(TaskConfig): - access_type = models.CharField(max_length=50) - user = models.ForeignKey(User, on_delete=models.CASCADE) - revoke = models.BooleanField(default=False) - context = models.TextField(default="") - delta = models.TextField(default="{}") - - def to_dict(self): - d = {} - d['access_type'] = self.access_type - d['user'] = self.user.id - d['revoke'] = self.revoke - try: - d['context'] = json.loads(self.context) - except Exception: - pass - return d - - def get_delta(self): - d = json.loads(self.to_json()) - d["lab_token"] = self.accessrelation.lab_token - - return d - - def to_json(self): - return json.dumps(self.to_dict()) - - def clear_delta(self): - d = {} - d["lab_token"] = self.accessrelation.lab_token - self.delta = json.dumps(d) - - def set_access_type(self, access_type): - self.access_type = access_type - d = json.loads(self.delta) - d['access_type'] = access_type - self.delta = json.dumps(d) - - def set_user(self, user): - self.user = user - d = json.loads(self.delta) - d['user'] = self.user.id - self.delta = json.dumps(d) - - def set_revoke(self, revoke): - self.revoke = revoke - d = json.loads(self.delta) - d['revoke'] = revoke - self.delta = json.dumps(d) - - def set_context(self, context): - self.context = json.dumps(context) - d = json.loads(self.delta) - d['context'] = context - self.delta = json.dumps(d) - - -class SoftwareConfig(TaskConfig): - """Handles software installations, such as OPNFV or ONAP.""" - - opnfv = models.ForeignKey(OpnfvApiConfig, on_delete=models.CASCADE) - - def to_dict(self): - d = {} - if self.opnfv: - d['opnfv'] = self.opnfv.to_dict() - - d["lab_token"] = self.softwarerelation.lab_token - self.delta = json.dumps(d) - - return d - - def get_delta(self): - d = {} - d['opnfv'] = self.opnfv.get_delta() - d['lab_token'] = self.softwarerelation.lab_token - - return d - - def clear_delta(self): - self.opnfv.clear_delta() - - def to_json(self): - return json.dumps(self.to_dict()) - - -class HardwareConfig(TaskConfig): - """Describes the desired configuration of the hardware.""" - - image = models.CharField(max_length=100, default="defimage") - power = models.CharField(max_length=100, default="off") - hostname = models.CharField(max_length=100, default="hostname") - ipmi_create = models.BooleanField(default=False) - delta = models.TextField() - - keys = set(["id", "image", "power", "hostname", "ipmi_create"]) - - def to_dict(self): - return self.get_delta() - - def get_delta(self): - # TODO: grab the GeneratedCloudConfig urls from self.hosthardwarerelation.get_resource() - return self.format_delta( - self.hosthardwarerelation.get_resource().get_configuration(self.state), - self.hosthardwarerelation.lab_token) - - -class NetworkConfig(TaskConfig): - """Handles network configuration.""" - - interfaces = models.ManyToManyField(Interface) - delta = models.TextField() - - def to_dict(self): - d = {} - hid = self.hostnetworkrelation.resource_id - d[hid] = {} - for interface in self.interfaces.all(): - d[hid][interface.mac_address] = [] - if self.state != ConfigState.CLEAN: - for vlan in interface.config.all(): - # TODO: should this come from the interface? - # e.g. will different interfaces for different resources need different configs? - d[hid][interface.mac_address].append({"vlan_id": vlan.vlan_id, "tagged": vlan.tagged}) - - return d - - def to_json(self): - return json.dumps(self.to_dict()) - - def get_delta(self): - d = json.loads(self.to_json()) - d['lab_token'] = self.hostnetworkrelation.lab_token - return d - - def clear_delta(self): - self.delta = json.dumps(self.to_dict()) - self.save() - - def add_interface(self, interface): - self.interfaces.add(interface) - d = json.loads(self.delta) - hid = self.hostnetworkrelation.resource_id - if hid not in d: - d[hid] = {} - d[hid][interface.mac_address] = [] - for vlan in interface.config.all(): - d[hid][interface.mac_address].append({"vlan_id": vlan.vlan_id, "tagged": vlan.tagged}) - self.delta = json.dumps(d) - - -class SnapshotConfig(TaskConfig): - - resource_id = models.CharField(max_length=200, default="default_id") - image = models.CharField(max_length=200, null=True) # cobbler ID - dashboard_id = models.IntegerField() - delta = models.TextField(default="{}") - - def to_dict(self): - d = {} - if self.host: - d['host'] = self.host.labid - if self.image: - d['image'] = self.image - d['dashboard_id'] = self.dashboard_id - return d - - def to_json(self): - return json.dumps(self.to_dict()) - - def get_delta(self): - d = json.loads(self.to_json()) - return d - - def clear_delta(self): - self.delta = json.dumps(self.to_dict()) - self.save() - - def set_host(self, host): - self.host = host - d = json.loads(self.delta) - d['host'] = host.labid - self.delta = json.dumps(d) - - def set_image(self, image): - self.image = image - d = json.loads(self.delta) - d['image'] = self.image - self.delta = json.dumps(d) - - def clear_image(self): - self.image = None - d = json.loads(self.delta) - d.pop("image", None) - self.delta = json.dumps(d) - - def set_dashboard_id(self, dash): - self.dashboard_id = dash - d = json.loads(self.delta) - d['dashboard_id'] = self.dashboard_id - self.delta = json.dumps(d) - - def save(self, *args, **kwargs): - if len(ResourceQuery.filter(labid=self.resource_id)) != 1: - raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource") - super().save(*args, **kwargs) - - -def get_task(task_id): - for taskclass in [AccessRelation, SoftwareRelation, HostHardwareRelation, HostNetworkRelation, SnapshotRelation]: - try: - ret = taskclass.objects.get(task_id=task_id) - return ret - except taskclass.DoesNotExist: - pass - from django.core.exceptions import ObjectDoesNotExist - raise ObjectDoesNotExist("Could not find matching TaskRelation instance") - - +# Needs to exist for migrations def get_task_uuid(): - return str(uuid.uuid4()) - - -class TaskRelation(models.Model): - """ - Relates a Job to a TaskConfig. - - superclass that relates a Job to tasks anc maintains information - like status and messages from the lab - """ - - status = models.IntegerField(default=JobStatus.NEW) - job = models.ForeignKey(Job, on_delete=models.CASCADE) - config = models.OneToOneField(TaskConfig, on_delete=models.CASCADE) - task_id = models.CharField(default=get_task_uuid, max_length=37) - lab_token = models.CharField(default="null", max_length=50) - message = models.TextField(default="") - - job_key = None - - def delete(self, *args, **kwargs): - self.config.delete() - return super(self.__class__, self).delete(*args, **kwargs) - - def type_str(self): - return "Generic Task" - - class Meta: - abstract = True - - -class AccessRelation(TaskRelation): - config = models.OneToOneField(AccessConfig, on_delete=models.CASCADE) - job_key = "access" - - def type_str(self): - return "Access Task" - - def delete(self, *args, **kwargs): - self.config.delete() - return super(self.__class__, self).delete(*args, **kwargs) - - -class SoftwareRelation(TaskRelation): - config = models.OneToOneField(SoftwareConfig, on_delete=models.CASCADE) - job_key = "software" - - def type_str(self): - return "Software Configuration Task" - - def delete(self, *args, **kwargs): - self.config.delete() - return super(self.__class__, self).delete(*args, **kwargs) - - -class HostHardwareRelation(TaskRelation): - resource_id = models.CharField(max_length=200, default="default_id") - config = models.OneToOneField(HardwareConfig, on_delete=models.CASCADE) - job_key = "hardware" - - def type_str(self): - return "Hardware Configuration Task" - - def get_delta(self): - return self.config.to_dict() - - def delete(self, *args, **kwargs): - self.config.delete() - return super(self.__class__, self).delete(*args, **kwargs) - - def save(self, *args, **kwargs): - if len(ResourceQuery.filter(labid=self.resource_id)) != 1: - raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource") - super().save(*args, **kwargs) - - def get_resource(self): - return ResourceQuery.get(labid=self.resource_id) - - -class HostNetworkRelation(TaskRelation): - resource_id = models.CharField(max_length=200, default="default_id") - config = models.OneToOneField(NetworkConfig, on_delete=models.CASCADE) - job_key = "network" - - def type_str(self): - return "Network Configuration Task" - - def delete(self, *args, **kwargs): - self.config.delete() - return super(self.__class__, self).delete(*args, **kwargs) - - def save(self, *args, **kwargs): - if len(ResourceQuery.filter(labid=self.resource_id)) != 1: - raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource") - super().save(*args, **kwargs) - - def get_resource(self): - return ResourceQuery.get(labid=self.resource_id) - - -class SnapshotRelation(TaskRelation): - snapshot = models.ForeignKey(Image, on_delete=models.CASCADE) - config = models.OneToOneField(SnapshotConfig, on_delete=models.CASCADE) - job_key = "snapshot" - - def type_str(self): - return "Snapshot Task" - - def get_delta(self): - return self.config.to_dict() - - def delete(self, *args, **kwargs): - self.config.delete() - return super(self.__class__, self).delete(*args, **kwargs) - - -class ActiveUsersRelation(TaskRelation): - config = models.OneToOneField(ActiveUsersConfig, on_delete=models.CASCADE) - job_key = "active users task" - - def type_str(self): - return "Active Users Task" - - -class JobFactory(object): - """This class creates all the API models (jobs, tasks, etc) needed to fulfill a booking.""" - - @classmethod - def reimageHost(cls, new_image, booking, host): - """Modify an existing job to reimage the given host.""" - job = Job.objects.get(booking=booking) - # make hardware task new - hardware_relation = HostHardwareRelation.objects.get(resource_id=host, job=job) - hardware_relation.config.image = new_image.lab_id - hardware_relation.config.save() - hardware_relation.status = JobStatus.NEW - - # re-apply networking after host is reset - net_relation = HostNetworkRelation.objects.get(resource_id=host, job=job) - net_relation.status = JobStatus.NEW - - # re-apply ssh access after host is reset - for relation in AccessRelation.objects.filter(job=job, config__access_type="ssh"): - relation.status = JobStatus.NEW - relation.save() - - hardware_relation.save() - net_relation.save() - - @classmethod - def makeSnapshotTask(cls, image, booking, host): - relation = SnapshotRelation() - job = Job.objects.get(booking=booking) - config = SnapshotConfig.objects.create(dashboard_id=image.id) - - relation.job = job - relation.config = config - relation.config.save() - relation.config = relation.config - relation.snapshot = image - relation.save() - - config.clear_delta() - config.set_host(host) - config.save() - - @classmethod - def makeActiveUsersTask(cls): - """ Append active users task to analytics job """ - config = ActiveUsersConfig() - relation = ActiveUsersRelation() - job = Job.objects.get(job_type='DATA') - - job.status = JobStatus.NEW - - relation.job = job - relation.config = config - relation.config.save() - relation.config = relation.config - relation.save() - config.save() - - @classmethod - def makeAnalyticsJob(cls, booking): - """ - Create the analytics job - - This will only run once since there will only be one analytics job. - All analytics tasks get appended to analytics job. - """ - - if len(Job.objects.filter(job_type='DATA')) > 0: - raise Exception("Cannot have more than one analytics job") - - if booking.resource: - raise Exception("Booking is not marker for analytics job, has resoure") - - job = Job() - job.booking = booking - job.job_type = 'DATA' - job.save() - - cls.makeActiveUsersTask() - - @classmethod - def makeCompleteJob(cls, booking): - """Create everything that is needed to fulfill the given booking.""" - resources = booking.resource.get_resources() - job = None - try: - job = Job.objects.get(booking=booking) - except Exception: - job = Job.objects.create(status=JobStatus.NEW, booking=booking) - cls.makeHardwareConfigs( - resources=resources, - job=job - ) - cls.makeNetworkConfigs( - resources=resources, - job=job - ) - cls.makeSoftware( - booking=booking, - job=job - ) - cls.makeGeneratedCloudConfigs( - resources=resources, - job=job - ) - all_users = list(booking.collaborators.all()) - all_users.append(booking.owner) - cls.makeAccessConfig( - users=all_users, - access_type="vpn", - revoke=False, - job=job - ) - for user in all_users: - try: - cls.makeAccessConfig( - users=[user], - access_type="ssh", - revoke=False, - job=job, - context={ - "key": user.userprofile.ssh_public_key.open().read().decode(encoding="UTF-8"), - "hosts": [r.labid for r in resources] - } - ) - except Exception: - continue - - @classmethod - def makeGeneratedCloudConfigs(cls, resources=[], job=Job()): - for res in resources: - cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=job.booking, rconfig=res.config) - cif.save() - - cif = CloudInitFile.create(priority=0, text=cif.serialize()) - cif.save() - - res.config.cloud_init_files.add(cif) - res.config.save() - - @classmethod - def makeHardwareConfigs(cls, resources=[], job=Job()): - """ - Create and save HardwareConfig. - - Helper function to create the tasks related to - configuring the hardware - """ - for res in resources: - hardware_config = None - try: - hardware_config = HardwareConfig.objects.get(relation__resource_id=res.labid) - except Exception: - hardware_config = HardwareConfig() - - relation = HostHardwareRelation() - relation.resource_id = res.labid - relation.job = job - relation.config = hardware_config - relation.config.save() - relation.config = relation.config - relation.save() - - hardware_config.set("id", "image", "hostname", "power", "ipmi_create") - hardware_config.save() - - @classmethod - def makeAccessConfig(cls, users, access_type, revoke=False, job=Job(), context=False): - """ - Create and save AccessConfig. - - Helper function to create the tasks related to - configuring the VPN, SSH, etc access for users - """ - for user in users: - relation = AccessRelation() - relation.job = job - config = AccessConfig() - config.access_type = access_type - config.user = user - config.save() - relation.config = config - relation.save() - config.clear_delta() - if context: - config.set_context(context) - config.set_access_type(access_type) - config.set_revoke(revoke) - config.set_user(user) - config.save() - - @classmethod - def makeNetworkConfigs(cls, resources=[], job=Job()): - """ - Create and save NetworkConfig. - - Helper function to create the tasks related to - configuring the networking - """ - for res in resources: - network_config = None - try: - network_config = NetworkConfig.objects.get(relation__host=res) - except Exception: - network_config = NetworkConfig.objects.create() - - relation = HostNetworkRelation() - relation.resource_id = res.labid - relation.job = job - network_config.save() - relation.config = network_config - relation.save() - network_config.clear_delta() - - # TODO: use get_interfaces() on resource - for interface in res.interfaces.all(): - network_config.add_interface(interface) - network_config.save() - - @classmethod - def make_bridge_config(cls, booking): - if len(booking.resource.get_resources()) < 2: - return None - try: - jumphost_config = ResourceOPNFVConfig.objects.filter( - role__name__iexact="jumphost" - ) - jumphost = ResourceQuery.filter( - bundle=booking.resource, - config=jumphost_config.resource_config - )[0] - except Exception: - return None - br_config = BridgeConfig.objects.create(opnfv_config=booking.opnfv_config) - for iface in jumphost.interfaces.all(): - br_config.interfaces.add(iface) - return br_config - - @classmethod - def makeSoftware(cls, booking=None, job=Job()): - """ - Create and save SoftwareConfig. - - Helper function to create the tasks related to - configuring the desired software, e.g. an OPNFV deployment - """ - if not booking.opnfv_config: - return None - - opnfv_api_config = OpnfvApiConfig.objects.create( - opnfv_config=booking.opnfv_config, - installer=booking.opnfv_config.installer.name, - scenario=booking.opnfv_config.scenario.name, - bridge_config=cls.make_bridge_config(booking) - ) - - opnfv_api_config.set_xdf(booking, False) - opnfv_api_config.save() - - for host in booking.resource.get_resources(): - opnfv_api_config.roles.add(host) - software_config = SoftwareConfig.objects.create(opnfv=opnfv_api_config) - software_relation = SoftwareRelation.objects.create(job=job, config=software_config) - return software_relation - - -JOB_TASK_CLASSLIST = [ - HostHardwareRelation, - AccessRelation, - HostNetworkRelation, - SoftwareRelation, - SnapshotRelation, - ActiveUsersRelation -] - - -class JobTaskQuery(AbstractModelQuery): - model_list = JOB_TASK_CLASSLIST + pass \ No newline at end of file diff --git a/src/api/serializers/__init__.py b/src/api/serializers/__init__.py deleted file mode 100644 index e0408fa..0000000 --- a/src/api/serializers/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## diff --git a/src/api/serializers/booking_serializer.py b/src/api/serializers/booking_serializer.py deleted file mode 100644 index 993eb22..0000000 --- a/src/api/serializers/booking_serializer.py +++ /dev/null @@ -1,173 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from rest_framework import serializers - -from resource_inventory.models import ( - ResourceConfiguration, - CpuProfile, - DiskProfile, - InterfaceProfile, - RamProfile, - Image, - Interface -) - - -class BookingField(serializers.Field): - - def to_representation(self, booking): - """ - Take in a booking object. - - Returns a dictionary of primitives representing that booking - """ - ser = {} - ser['id'] = booking.id - # main loop to grab relevant info out of booking - host_configs = {} # mapping hostname -> config - networks = {} # mapping vlan id -> network_hosts - for host in booking.resource.hosts.all(): - host_configs[host.name] = ResourceConfiguration.objects.get(host=host.template) - if "jumphost" not in ser and host_configs[host.name].opnfvRole.name.lower() == "jumphost": - ser['jumphost'] = host.name - # host is a Host model - for i in range(len(host.interfaces.all())): - interface = host.interfaces.all()[i] - # interface is an Interface model - for vlan in interface.config.all(): - # vlan is Vlan model - if vlan.id not in networks: - networks[vlan.id] = [] - net_host = {"hostname": host.name, "tagged": vlan.tagged, "interface": i} - networks[vlan.id].append(net_host) - # creates networking object of proper form - networking = [] - for vlanid in networks: - network = {} - network['vlan_id'] = vlanid - network['hosts'] = networks[vlanid] - - ser['networking'] = networking - - # creates hosts object of correct form - hosts = [] - for hostname in host_configs: - host = {"hostname": hostname} - host['deploy_image'] = True # TODO? - image = host_configs[hostname].image - host['image'] = { - "name": image.name, - "lab_id": image.lab_id, - "dashboard_id": image.id - } - hosts.append(host) - - ser['hosts'] = hosts - - return ser - - def to_internal_value(self, data): - """ - Take in a dictionary of primitives, and return a booking object. - - This is not going to be implemented or allowed. - If someone needs to create a booking through the api, - they will send a different booking object - """ - return None - - -class BookingSerializer(serializers.Serializer): - - booking = BookingField() - - -# Host Type stuff, for inventory -class CPUSerializer(serializers.ModelSerializer): - class Meta: - model = CpuProfile - fields = ('cores', 'architecture', 'cpus') - - -class DiskSerializer(serializers.ModelSerializer): - class Meta: - model = DiskProfile - fields = ('size', 'media_type', 'name') - - -class InterfaceProfileSerializer(serializers.ModelSerializer): - class Meta: - model = InterfaceProfile - fields = ('speed', 'name') - - -class RamSerializer(serializers.ModelSerializer): - class Meta: - model = RamProfile - fields = ('amount', 'channels') - - -class HostTypeSerializer(serializers.Serializer): - name = serializers.CharField(max_length=200) - ram = RamSerializer() - interface = InterfaceProfileSerializer() - description = serializers.CharField(max_length=1000) - disks = DiskSerializer() - cpu = CPUSerializer() - - -# the rest of the inventory stuff -class NetworkSerializer(serializers.Serializer): - cidr = serializers.CharField(max_length=200) - gateway = serializers.IPAddressField(max_length=200) - vlan = serializers.IntegerField() - - -class ImageSerializer(serializers.ModelSerializer): - lab_id = serializers.IntegerField() - id = serializers.IntegerField(source="dashboard_id") - name = serializers.CharField(max_length=50) - description = serializers.CharField(max_length=200) - - class Meta: - model = Image - - -class InterfaceField(serializers.Field): - def to_representation(self, interface): - pass - - def to_internal_value(self, data): - """Take in a serialized interface and creates an Interface model.""" - mac = data['mac'] - bus_address = data['busaddr'] - switch_name = data['switchport']['switch_name'] - port_name = data['switchport']['port_name'] - # TODO config?? - return Interface.objects.create( - mac_address=mac, - bus_address=bus_address, - switch_name=switch_name, - port_name=port_name - ) - - -class InventoryHostSerializer(serializers.Serializer): - hostname = serializers.CharField(max_length=100) - host_type = serializers.CharField(max_length=100) - interfaces = InterfaceField() - - -class InventorySerializer(serializers.Serializer): - hosts = InventoryHostSerializer() - networks = NetworkSerializer() - images = ImageSerializer() - host_types = HostTypeSerializer() diff --git a/src/api/serializers/old_serializers.py b/src/api/serializers/old_serializers.py deleted file mode 100644 index 0944881..0000000 --- a/src/api/serializers/old_serializers.py +++ /dev/null @@ -1,21 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from rest_framework import serializers - -from account.models import UserProfile - - -class UserSerializer(serializers.ModelSerializer): - username = serializers.CharField(source='user.username') - - class Meta: - model = UserProfile - fields = ('user', 'username', 'ssh_public_key', 'pgp_public_key', 'email_addr') diff --git a/src/api/tests/__init__.py b/src/api/tests/__init__.py deleted file mode 100644 index 2435a9f..0000000 --- a/src/api/tests/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Parker Berberian and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## diff --git a/src/api/tests/test_models_unittest.py b/src/api/tests/test_models_unittest.py deleted file mode 100644 index 2dee29b..0000000 --- a/src/api/tests/test_models_unittest.py +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright (c) 2019 Sawyer Bergeron, Parker Berberian, and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from api.models import ( - Job, - JobStatus, - JobFactory, - HostNetworkRelation, - HostHardwareRelation, - SoftwareRelation, - AccessConfig, - SnapshotRelation -) - -from resource_inventory.models import ( - OPNFVRole, - HostProfile, - ConfigState, -) - -from django.test import TestCase, Client - -from dashboard.testing_utils import ( - make_host, - make_user, - make_user_profile, - make_lab, - make_installer, - make_image, - make_scenario, - make_os, - make_complete_host_profile, - make_booking, -) - - -class ValidBookingCreatesValidJob(TestCase): - @classmethod - def setUpTestData(cls): - cls.user = make_user(False, username="newtestuser", password="testpassword") - cls.userprofile = make_user_profile(cls.user) - cls.lab = make_lab() - - cls.host_profile = make_complete_host_profile(cls.lab) - cls.scenario = make_scenario() - cls.installer = make_installer([cls.scenario]) - os = make_os([cls.installer]) - cls.image = make_image(cls.lab, 1, cls.user, os, cls.host_profile) - for i in range(30): - make_host(cls.host_profile, cls.lab, name="host" + str(i), labid="host" + str(i)) - cls.client = Client() - - def setUp(self): - self.booking, self.compute_hostnames, self.jump_hostname = self.create_multinode_generic_booking() - - def create_multinode_generic_booking(self): - topology = {} - - compute_hostnames = ["cmp01", "cmp02", "cmp03"] - - host_type = HostProfile.objects.first() - - universal_networks = [ - {"name": "public", "tagged": False, "public": True}, - {"name": "admin", "tagged": True, "public": False}] - compute_networks = [{"name": "private", "tagged": True, "public": False}] - jumphost_networks = [{"name": "external", "tagged": True, "public": True}] - - # generate a bunch of extra networks - for i in range(10): - net = {"tagged": False, "public": False} - net["name"] = "net" + str(i) - universal_networks.append(net) - - jumphost_info = { - "type": host_type, - "role": OPNFVRole.objects.get_or_create(name="Jumphost")[0], - "nets": self.make_networks(host_type, jumphost_networks + universal_networks), - "image": self.image - } - topology["jump"] = jumphost_info - - for hostname in compute_hostnames: - host_info = { - "type": host_type, - "role": OPNFVRole.objects.get_or_create(name="Compute")[0], - "nets": self.make_networks(host_type, compute_networks + universal_networks), - "image": self.image - } - topology[hostname] = host_info - - booking = make_booking( - owner=self.user, - lab=self.lab, - topology=topology, - installer=self.installer, - scenario=self.scenario - ) - - if not booking.resource: - raise Exception("Booking does not have a resource when trying to pass to makeCompleteJob") - return booking, compute_hostnames, "jump" - - def make_networks(self, hostprofile, nets): - """ - Distribute nets accross hostprofile's interfaces. - - returns a 2D array - """ - network_struct = [] - count = hostprofile.interfaceprofile.all().count() - for i in range(count): - network_struct.append([]) - while (nets): - index = len(nets) % count - network_struct[index].append(nets.pop()) - - return network_struct - - ################################################################# - # Complete Job Tests - ################################################################# - - def test_complete_job_makes_access_configs(self): - JobFactory.makeCompleteJob(self.booking) - job = Job.objects.get(booking=self.booking) - self.assertIsNotNone(job) - - access_configs = AccessConfig.objects.filter(accessrelation__job=job) - - vpn_configs = access_configs.filter(access_type="vpn") - ssh_configs = access_configs.filter(access_type="ssh") - - self.assertFalse(AccessConfig.objects.exclude(access_type__in=["vpn", "ssh"]).exists()) - - all_users = list(self.booking.collaborators.all()) - all_users.append(self.booking.owner) - - for user in all_users: - self.assertTrue(vpn_configs.filter(user=user).exists()) - self.assertTrue(ssh_configs.filter(user=user).exists()) - - def test_complete_job_makes_network_configs(self): - JobFactory.makeCompleteJob(self.booking) - job = Job.objects.get(booking=self.booking) - self.assertIsNotNone(job) - - booking_hosts = self.booking.resource.hosts.all() - - netrelations = HostNetworkRelation.objects.filter(job=job) - netconfigs = [r.config for r in netrelations] - - netrelation_hosts = [r.host for r in netrelations] - - for config in netconfigs: - for interface in config.interfaces.all(): - self.assertTrue(interface.host in booking_hosts) - - # if no interfaces are referenced that shouldn't have vlans, - # and no vlans exist outside those accounted for in netconfigs, - # then the api is faithfully representing networks - # as netconfigs reference resource_inventory models directly - - # this test relies on the assumption that - # every interface is configured, whether it does or does not have vlans - # if this is not true, the test fails - - for host in booking_hosts: - self.assertTrue(host in netrelation_hosts) - relation = HostNetworkRelation.objects.filter(job=job).get(host=host) - - # do 2 direction matching that interfaces are one to one - config = relation.config - for interface in config.interfaces.all(): - self.assertTrue(interface in host.interfaces) - for interface in host.interfaces.all(): - self.assertTrue(interface in config.interfaces) - - for host in netrelation_hosts: - self.assertTrue(host in booking_hosts) - - def test_complete_job_makes_hardware_configs(self): - JobFactory.makeCompleteJob(self.booking) - job = Job.objects.get(booking=self.booking) - self.assertIsNotNone(job) - - hardware_relations = HostHardwareRelation.objects.filter(job=job) - - job_hosts = [r.host for r in hardware_relations] - - booking_hosts = self.booking.resource.hosts.all() - - self.assertEqual(len(booking_hosts), len(job_hosts)) - - for relation in hardware_relations: - self.assertTrue(relation.host in booking_hosts) - self.assertEqual(relation.status, JobStatus.NEW) - config = relation.config - host = relation.host - self.assertEqual(config.get_delta()["hostname"], host.template.resource.name) - - def test_complete_job_makes_software_configs(self): - JobFactory.makeCompleteJob(self.booking) - job = Job.objects.get(booking=self.booking) - self.assertIsNotNone(job) - - srelation = SoftwareRelation.objects.filter(job=job).first() - self.assertIsNotNone(srelation) - - sconfig = srelation.config - self.assertIsNotNone(sconfig) - - oconfig = sconfig.opnfv - self.assertIsNotNone(oconfig) - - # not onetoone in models, but first() is safe here based on how ConfigBundle and a matching OPNFVConfig are created - # this should, however, be made explicit - self.assertEqual(oconfig.installer, self.booking.config_bundle.opnfv_config.first().installer.name) - self.assertEqual(oconfig.scenario, self.booking.config_bundle.opnfv_config.first().scenario.name) - - for host in oconfig.roles.all(): - role_name = host.config.host_opnfv_config.first().role.name - if str(role_name).lower() == "jumphost": - self.assertEqual(host.template.resource.name, self.jump_hostname) - elif str(role_name).lower() == "compute": - self.assertTrue(host.template.resource.name in self.compute_hostnames) - else: - self.fail(msg="Host with non-configured role name related to job: " + str(role_name)) - - def test_make_snapshot_task(self): - host = self.booking.resource.hosts.first() - image = make_image(self.lab, -1, None, None, host.profile) - - Job.objects.create(booking=self.booking) - - JobFactory.makeSnapshotTask(image, self.booking, host) - - snap_relation = SnapshotRelation.objects.get(job=self.booking.job) - config = snap_relation.config - self.assertEqual(host.id, config.host.id) - self.assertEqual(config.dashboard_id, image.id) - self.assertEqual(snap_relation.snapshot.id, image.id) - - def test_make_hardware_configs(self): - hosts = self.booking.resource.hosts.all() - job = Job.objects.create(booking=self.booking) - JobFactory.makeHardwareConfigs(hosts=hosts, job=job) - - hardware_relations = HostHardwareRelation.objects.filter(job=job) - - self.assertEqual(hardware_relations.count(), hosts.count()) - - host_set = set([h.id for h in hosts]) - - for relation in hardware_relations: - try: - host_set.remove(relation.host.id) - except KeyError: - self.fail("Hardware Relation/Config not created for host " + str(relation.host)) - # TODO: ConfigState needs to be fixed in factory methods - relation.config.state = ConfigState.NEW - self.assertEqual(relation.config.get_delta()["power"], "on") - self.assertTrue(relation.config.get_delta()["ipmi_create"]) - # TODO: the rest of hwconf attrs - - self.assertEqual(len(host_set), 0) diff --git a/src/api/urls.py b/src/api/urls.py index cbb453c..b009aeb 100644 --- a/src/api/urls.py +++ b/src/api/urls.py @@ -31,62 +31,23 @@ from django.urls import path from api.views import ( lab_profile, lab_status, - lab_inventory, lab_downtime, - specific_job, - specific_task, - new_jobs, - current_jobs, - done_jobs, - update_host_bmc, - lab_host, - get_pdf, - get_idf, lab_users, lab_user, GenerateTokenView, - analytics_job, user_bookings, specific_booking, extend_booking, make_booking, list_labs, all_users, - images_for_template, - available_templates, - resource_ci_metadata, - resource_ci_userdata, - resource_ci_userdata_directory, - all_images, - all_opsyss, - single_image, - single_opsys, - create_ci_file, booking_details, ) urlpatterns = [ - path('labs//opsys/', single_opsys), - path('labs//image/', single_image), - path('labs//opsys', all_opsyss), - path('labs//image', all_images), path('labs//profile', lab_profile), path('labs//status', lab_status), - path('labs//inventory', lab_inventory), path('labs//downtime', lab_downtime), - path('labs//hosts/', lab_host), - path('labs//hosts//bmc', update_host_bmc), - path('labs//booking//pdf', get_pdf, name="get-pdf"), - path('labs//booking//idf', get_idf, name="get-idf"), - path('labs//jobs/', specific_job), - path('labs//jobs//', specific_task), - path('labs//jobs//cidata//user-data', resource_ci_userdata_directory, name="specific-user-data"), - path('labs//jobs//cidata//meta-data', resource_ci_metadata, name="specific-meta-data"), - path('labs//jobs//cidata///user-data', resource_ci_userdata, name="user-data-dir"), - path('labs//jobs/new', new_jobs), - path('labs//jobs/current', current_jobs), - path('labs//jobs/done', done_jobs), - path('labs//jobs/getByType/DATA', analytics_job), path('labs//users', lab_users), path('labs//users/', lab_user), @@ -96,11 +57,6 @@ urlpatterns = [ path('booking/makeBooking', make_booking), path('booking//details', booking_details), - path('resource_inventory/availableTemplates', available_templates), - path('resource_inventory//images', images_for_template), - - path('resource_inventory/cloud/create', create_ci_file), - path('users', all_users), path('labs', list_labs), diff --git a/src/api/views.py b/src/api/views.py index d5966ed..ea36a6d 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -10,6 +10,7 @@ import json import math +import os import traceback import sys from datetime import timedelta @@ -21,28 +22,18 @@ from django.utils import timezone from django.views import View from django.http import HttpResponseNotFound from django.http.response import JsonResponse, HttpResponse +import requests from rest_framework import viewsets from rest_framework.authtoken.models import Token from django.views.decorators.csrf import csrf_exempt from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q +from django.contrib.auth.models import User -from api.serializers.booking_serializer import BookingSerializer -from api.serializers.old_serializers import UserSerializer from api.forms import DowntimeForm from account.models import UserProfile, Lab from booking.models import Booking -from booking.quick_deployer import create_from_API -from api.models import LabManagerTracker, get_task, Job, AutomationAPIManager, APILog, GeneratedCloudConfig -from notifier.manager import NotificationHandler -from analytics.models import ActiveVPNUser -from resource_inventory.models import ( - Image, - Opsys, - CloudInitFile, - ResourceQuery, - ResourceTemplate, -) +from api.models import LabManagerTracker,AutomationAPIManager, APILog import yaml import uuid @@ -61,17 +52,6 @@ the correct thing will happen """ -class BookingViewSet(viewsets.ModelViewSet): - queryset = Booking.objects.all() - serializer_class = BookingSerializer - filter_fields = ('resource', 'id') - - -class UserViewSet(viewsets.ModelViewSet): - queryset = UserProfile.objects.all() - serializer_class = UserSerializer - - @method_decorator(login_required, name='dispatch') class GenerateTokenView(View): def get(self, request, *args, **kwargs): @@ -83,111 +63,6 @@ class GenerateTokenView(View): return redirect('account:settings') -def lab_inventory(request, lab_name=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - return JsonResponse(lab_manager.get_inventory(), safe=False) - - -@csrf_exempt -def lab_host(request, lab_name="", host_id=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - if request.method == "GET": - return JsonResponse(lab_manager.get_host(host_id), safe=False) - if request.method == "POST": - return JsonResponse(lab_manager.update_host(host_id, request.POST), safe=False) - -# API extension for Cobbler integration - - -def all_images(request, lab_name=""): - a = [] - for i in Image.objects.all(): - a.append(i.serialize()) - return JsonResponse(a, safe=False) - - -def all_opsyss(request, lab_name=""): - a = [] - for opsys in Opsys.objects.all(): - a.append(opsys.serialize()) - - return JsonResponse(a, safe=False) - - -@csrf_exempt -def single_image(request, lab_name="", image_id=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - img = lab_manager.get_image(image_id).first() - - if request.method == "GET": - if not img: - return HttpResponse(status=404) - return JsonResponse(img.serialize(), safe=False) - - if request.method == "POST": - # get POST data - data = json.loads(request.body.decode('utf-8')) - if img: - img.update(data) - else: - # append lab name and the ID from the URL - data['from_lab_id'] = lab_name - data['lab_id'] = image_id - - # create and save a new Image object - img = Image.new_from_data(data) - - img.save() - - # indicate success in response - return HttpResponse(status=200) - return HttpResponse(status=405) - - -@csrf_exempt -def single_opsys(request, lab_name="", opsys_id=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - opsys = lab_manager.get_opsys(opsys_id).first() - - if request.method == "GET": - if not opsys: - return HttpResponse(status=404) - return JsonResponse(opsys.serialize(), safe=False) - - if request.method == "POST": - data = json.loads(request.body.decode('utf-8')) - if opsys: - opsys.update(data) - else: - # only name, available, and obsolete are needed to create an Opsys - # other fields are derived from the URL parameters - data['from_lab_id'] = lab_name - data['lab_id'] = opsys_id - opsys = Opsys.new_from_data(data) - - opsys.save() - return HttpResponse(status=200) - return HttpResponse(status=405) - -# end API extension - - -def get_pdf(request, lab_name="", booking_id=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - return HttpResponse(lab_manager.get_pdf(booking_id), content_type="text/plain") - - -def get_idf(request, lab_name="", booking_id=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - return HttpResponse(lab_manager.get_idf(booking_id), content_type="text/plain") - - def lab_status(request, lab_name=""): lab_token = request.META.get('HTTP_AUTH_TOKEN') lab_manager = LabManagerTracker.get(lab_name, lab_token) @@ -208,171 +83,12 @@ def lab_user(request, lab_name="", user_id=-1): return HttpResponse(lab_manager.get_user(user_id), content_type="text/plain") -@csrf_exempt -def update_host_bmc(request, lab_name="", host_id=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - if request.method == "POST": - # update / create RemoteInfo for host - return JsonResponse( - lab_manager.update_host_remote_info(request.POST, host_id), - safe=False - ) - - def lab_profile(request, lab_name=""): lab_token = request.META.get('HTTP_AUTH_TOKEN') lab_manager = LabManagerTracker.get(lab_name, lab_token) return JsonResponse(lab_manager.get_profile(), safe=False) -@csrf_exempt -def specific_task(request, lab_name="", job_id="", task_id=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - LabManagerTracker.get(lab_name, lab_token) # Authorize caller, but we dont need the result - - if request.method == "POST": - task = get_task(task_id) - if 'status' in request.POST: - task.status = request.POST.get('status') - if 'message' in request.POST: - task.message = request.POST.get('message') - if 'lab_token' in request.POST: - task.lab_token = request.POST.get('lab_token') - task.save() - NotificationHandler.task_updated(task) - d = {} - d['task'] = task.config.get_delta() - m = {} - m['status'] = task.status - m['job'] = str(task.job) - m['message'] = task.message - d['meta'] = m - return JsonResponse(d, safe=False) - elif request.method == "GET": - return JsonResponse(get_task(task_id).config.get_delta()) - - -@csrf_exempt -def specific_job(request, lab_name="", job_id=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - if request.method == "POST": - return JsonResponse(lab_manager.update_job(job_id, request.POST), safe=False) - return JsonResponse(lab_manager.get_job(job_id), safe=False) - - -@csrf_exempt -def resource_ci_userdata(request, lab_name="", job_id="", resource_id="", file_id=0): - # lab_token = request.META.get('HTTP_AUTH_TOKEN') - # lab_manager = LabManagerTracker.get(lab_name, lab_token) - - # job = lab_manager.get_job(job_id) - Job.objects.get(id=job_id) # verify a valid job was given, even if we don't use it - - cifile = None - try: - cifile = CloudInitFile.objects.get(id=file_id) - except ObjectDoesNotExist: - return HttpResponseNotFound("Could not find a matching resource by id " + str(resource_id)) - - text = cifile.text - - prepended_text = "#cloud-config\n" - # mstrat = CloudInitFile.merge_strategy() - # prepended_text = prepended_text + yaml.dump({"merge_strategy": mstrat}) + "\n" - # print("in cloudinitfile create") - text = prepended_text + text - cloud_dict = { - "datasource": { - "None": { - "metadata": { - "instance-id": str(uuid.uuid4()) - }, - "userdata_raw": text, - }, - }, - "datasource_list": ["None"], - } - - return HttpResponse(yaml.dump(cloud_dict, width=float("inf")), status=200) - - -@csrf_exempt -def resource_ci_metadata(request, lab_name="", job_id="", resource_id="", file_id=0): - return HttpResponse("#cloud-config", status=200) - - -@csrf_exempt -def resource_ci_userdata_directory(request, lab_name="", job_id="", resource_id=""): - # files = [{"id": file.file_id, "priority": file.priority} for file in CloudInitFile.objects.filter(job__id=job_id, resource_id=resource_id).order_by("priority").all()] - resource = ResourceQuery.get(labid=resource_id, lab=Lab.objects.get(name=lab_name)) - files = resource.config.cloud_init_files - files = [{"id": file.id, "priority": file.priority} for file in files.order_by("priority").all()] - - d = {} - - merge_failures = [] - - merger = Merger( - [ - (list, ["append"]), - (dict, ["merge"]), - ], - ["override"], # fallback - ["override"], # if types conflict (shouldn't happen in CI, but handle case) - ) - - for f in resource.config.cloud_init_files.order_by("priority").all(): - try: - other_dict = yaml.safe_load(f.text) - if not (type(d) is dict): - raise Exception("CI file was valid yaml but was not a dict") - - merger.merge(d, other_dict) - except Exception as e: - # if fail to merge, then just skip - print("Failed to merge file in, as it had invalid content:", f.id) - print("File text was:") - print(f.text) - merge_failures.append({f.id: str(e)}) - - if len(merge_failures) > 0: - d['merge_failures'] = merge_failures - - file = CloudInitFile.create(text=yaml.dump(d, width=float("inf")), priority=0) - - return HttpResponse(json.dumps([{"id": file.id, "priority": file.priority}]), status=200) - - -def new_jobs(request, lab_name=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - return JsonResponse(lab_manager.get_new_jobs(), safe=False) - - -def current_jobs(request, lab_name=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - return JsonResponse(lab_manager.get_current_jobs(), safe=False) - - -@csrf_exempt -def analytics_job(request, lab_name=""): - """ returns all jobs with type booking""" - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - if request.method == "GET": - return JsonResponse(lab_manager.get_analytics_job(), safe=False) - if request.method == "POST": - users = json.loads(request.body.decode('utf-8'))['active_users'] - try: - ActiveVPNUser.create(lab_name, users) - except ObjectDoesNotExist: - return JsonResponse('Lab does not exist!', safe=False) - return HttpResponse(status=200) - return HttpResponse(status=405) - def lab_downtime(request, lab_name=""): lab_token = request.META.get('HTTP_AUTH_TOKEN') @@ -408,12 +124,6 @@ def delete_lab_downtime(lab_manager): return JsonResponse({"error": "Lab is not in downtime"}, status=422) -def done_jobs(request, lab_name=""): - lab_token = request.META.get('HTTP_AUTH_TOKEN') - lab_manager = LabManagerTracker.get(lab_name, lab_token) - return JsonResponse(lab_manager.get_done_jobs(), safe=False) - - def auth_and_log(request, endpoint): """ Function to authenticate an API user and log info @@ -471,37 +181,41 @@ Booking API Views def user_bookings(request): - token = auth_and_log(request, 'booking') + # token = auth_and_log(request, 'booking') - if isinstance(token, HttpResponse): - return token + # if isinstance(token, HttpResponse): + # return token - bookings = Booking.objects.filter(owner=token.user, end__gte=timezone.now()) - output = [AutomationAPIManager.serialize_booking(booking) - for booking in bookings] - return JsonResponse(output, safe=False) + # bookings = Booking.objects.filter(owner=token.user, end__gte=timezone.now()) + # output = [AutomationAPIManager.serialize_booking(booking) + # for booking in bookings] + # return JsonResponse(output, safe=False) + # todo - LL Integration + return HttpResponse(status=404) @csrf_exempt def specific_booking(request, booking_id=""): - token = auth_and_log(request, 'booking/{}'.format(booking_id)) + # token = auth_and_log(request, 'booking/{}'.format(booking_id)) - if isinstance(token, HttpResponse): - return token + # if isinstance(token, HttpResponse): + # return token - booking = get_object_or_404(Booking, pk=booking_id, owner=token.user) - if request.method == "GET": - sbooking = AutomationAPIManager.serialize_booking(booking) - return JsonResponse(sbooking, safe=False) + # booking = get_object_or_404(Booking, pk=booking_id, owner=token.user) + # if request.method == "GET": + # sbooking = AutomationAPIManager.serialize_booking(booking) + # return JsonResponse(sbooking, safe=False) - if request.method == "DELETE": + # if request.method == "DELETE": - if booking.end < timezone.now(): - return HttpResponse("Booking already over", status=400) + # if booking.end < timezone.now(): + # return HttpResponse("Booking already over", status=400) - booking.end = timezone.now() - booking.save() - return HttpResponse("Booking successfully cancelled") + # booking.end = timezone.now() + # booking.save() + # return HttpResponse("Booking successfully cancelled") + # todo - LL Integration + return HttpResponse(status=404) @csrf_exempt @@ -531,70 +245,82 @@ def extend_booking(request, booking_id="", days=""): @csrf_exempt def make_booking(request): - token = auth_and_log(request, 'booking/makeBooking') - - if isinstance(token, HttpResponse): - return token - + print("received call to make_booking") + data = json.loads(request.body) + print("incoming data is ", data) + + allowed_users = list(data["allowed_users"]) + allowed_users.append(str(request.user)) + + bookingBlob = { + "template_id": data["template_id"], + "allowed_users": allowed_users, + "global_cifile": data["global_cifile"], + "metadata": { + "booking_id": None, # fill in after creating django object + "owner": str(request.user), + "lab": "UNH_IOL", + "purpose": data["metadata"]["purpose"], + "project": data["metadata"]["project"], + "length": data["metadata"]["length"] + } + } + + print("allowed users are ", bookingBlob["allowed_users"]) try: - booking = create_from_API(request.body, token.user) + booking = Booking.objects.create( + purpose=bookingBlob["metadata"]["purpose"], + project=bookingBlob["metadata"]['project'], + lab=Lab.objects.get(name='UNH_IOL'), + owner=request.user, + start=timezone.now(), + end=timezone.now() + timedelta(days=int(bookingBlob["metadata"]['length'])), + ) + print("successfully created booking object with id ", booking.id) + + # Now add collabs + for c in bookingBlob["allowed_users"]: + if c != bookingBlob["metadata"]["owner"]: # Don't add self as a collab + booking.collaborators.add(User.objects.get(username=c)) + print("successfully added collabs") + + # Now create it in liblaas + bookingBlob["metadata"]["booking_id"] = str(booking.id) + liblaas_endpoint = os.environ.get("LIBLAAS_BASE_URL") + 'booking/create' + liblaas_response = requests.post(liblaas_endpoint, data=json.dumps(bookingBlob), headers={'Content-Type': 'application/json'}) + if liblaas_response.status_code != 200: + print("received non success from liblaas") + return JsonResponse( + data={}, + status=500, + safe=False + ) + aggregateId = json.loads(liblaas_response.content) + print("successfully created aggregate in liblaas") - except Exception: - finalTrace = '' - exc_type, exc_value, exc_traceback = sys.exc_info() - for i in traceback.format_exception(exc_type, exc_value, exc_traceback): - finalTrace += '
' + i.strip() - return HttpResponse(finalTrace, status=400) + # Now update the agg_id + booking.aggregateId = aggregateId + booking.save() + print("sucessfully updated aggreagateId in booking object") - sbooking = AutomationAPIManager.serialize_booking(booking) - return JsonResponse(sbooking, safe=False) + return JsonResponse( + data = {"bookingId": booking.id}, + status=200, + safe=False + ) + except Exception as error: + print(error) + return JsonResponse( + data={}, + status=500, + safe=False + ) """ Resource Inventory API Views """ - - -def available_templates(request): - token = auth_and_log(request, 'resource_inventory/availableTemplates') - - if isinstance(token, HttpResponse): - return token - - # get available templates - # mirrors MultipleSelectFilter Widget - avt = [] - for lab in Lab.objects.all(): - for template in ResourceTemplate.objects.filter(Q(owner=token.user) | Q(public=True), lab=lab, temporary=False): - available_resources = lab.get_available_resources() - required_resources = template.get_required_resources() - least_available = 100 - - for resource, count_required in required_resources.items(): - try: - curr_count = math.floor(available_resources[str(resource)] / count_required) - if curr_count < least_available: - least_available = curr_count - except KeyError: - least_available = 0 - - if least_available > 0: - avt.append((template, least_available)) - - savt = [AutomationAPIManager.serialize_template(temp) - for temp in avt] - - return JsonResponse(savt, safe=False) - - -def images_for_template(request, template_id=""): - _ = auth_and_log(request, 'resource_inventory/{}/images'.format(template_id)) - - template = get_object_or_404(ResourceTemplate, pk=template_id) - images = [AutomationAPIManager.serialize_image(config.image) - for config in template.getConfigs()] - return JsonResponse(images, safe=False) - +# todo - LL Integration """ User API Views @@ -613,25 +339,6 @@ def all_users(request): return JsonResponse(users, safe=False) -def create_ci_file(request): - token = auth_and_log(request, 'booking/makeCloudConfig') - - if isinstance(token, HttpResponse): - return token - - try: - cconf = request.body - d = yaml.load(cconf) - if not (type(d) is dict): - raise Exception() - - cconf = CloudInitFile.create(text=cconf, priority=CloudInitFile.objects.count()) - - return JsonResponse({"id": cconf.id}) - except Exception: - return JsonResponse({"error": "Provided config file was not valid yaml or was not a dict at the top level"}) - - """ Lab API Views """ @@ -662,95 +369,188 @@ Booking Details API Views def booking_details(request, booking_id=""): - token = auth_and_log(request, 'booking/{}/details'.format(booking_id)) - - if isinstance(token, HttpResponse): - return token - - booking = get_object_or_404(Booking, pk=booking_id, owner=token.user) + # token = auth_and_log(request, 'booking/{}/details'.format(booking_id)) + + # if isinstance(token, HttpResponse): + # return token + + # booking = get_object_or_404(Booking, pk=booking_id, owner=token.user) + + # # overview + # overview = { + # 'username': GeneratedCloudConfig._normalize_username(None, str(token.user)), + # 'purpose': booking.purpose, + # 'project': booking.project, + # 'start_time': booking.start, + # 'end_time': booking.end, + # 'pod_definitions': booking.resource.template, + # 'lab': booking.lab + # } + + # # deployment progress + # task_list = [] + # for task in booking.job.get_tasklist(): + # task_info = { + # 'name': str(task), + # 'status': 'DONE', + # 'lab_response': 'No response provided (yet)' + # } + # if task.status < 100: + # task_info['status'] = 'PENDING' + # elif task.status < 200: + # task_info['status'] = 'IN PROGRESS' + + # if task.message: + # if task.type_str == "Access Task" and request.user.id != task.config.user.id: + # task_info['lab_response'] = '--secret--' + # else: + # task_info['lab_response'] = str(task.message) + # task_list.append(task_info) + + # # pods + # pod_list = [] + # for host in booking.resource.get_resources(): + # pod_info = { + # 'hostname': host.config.name, + # 'machine': host.name, + # 'role': '', + # 'is_headnode': host.config.is_head_node, + # 'image': host.config.image, + # 'ram': {'amount': str(host.profile.ramprofile.first().amount) + 'G', 'channels': host.profile.ramprofile.first().channels}, + # 'cpu': {'arch': host.profile.cpuprofile.first().architecture, 'cores': host.profile.cpuprofile.first().cores, 'sockets': host.profile.cpuprofile.first().cpus}, + # 'disk': {'size': str(host.profile.storageprofile.first().size) + 'GiB', 'type': host.profile.storageprofile.first().media_type, 'mount_point': host.profile.storageprofile.first().name}, + # 'interfaces': [], + # } + # try: + # pod_info['role'] = host.template.opnfvRole + # except Exception: + # pass + # for intprof in host.profile.interfaceprofile.all(): + # int_info = { + # 'name': intprof.name, + # 'speed': intprof.speed + # } + # pod_info['interfaces'].append(int_info) + # pod_list.append(pod_info) + + # # diagnostic info + # diagnostic_info = { + # 'job_id': booking.job.id, + # 'ci_files': '', + # 'pods': [] + # } + # for host in booking.resource.get_resources(): + # pod = { + # 'host': host.name, + # 'configs': [], + + # } + # for ci_file in host.config.cloud_init_files.all(): + # ci_info = { + # 'id': ci_file.id, + # 'text': ci_file.text + # } + # pod['configs'].append(ci_info) + # diagnostic_info['pods'].append(pod) + + # details = { + # 'overview': overview, + # 'deployment_progress': task_list, + # 'pods': pod_list, + # 'diagnostic_info': diagnostic_info, + # 'pdf': booking.pdf + # } + # return JsonResponse(str(details), safe=False) + # todo - LL Integration + return HttpResponse(status=404) + + +""" Forwards a request to the LibLaaS API from a workflow """ +def liblaas_request(request) -> JsonResponse: + print("handing liblaas request... ", request.method) + print(request.body) + if request.method != 'POST': + return JsonResponse({"error" : "405 Method not allowed"}) + + liblaas_base_url = os.environ.get("LIBLAAS_BASE_URL") + post_data = json.loads(request.body) + print("post data is " + str(post_data)) + http_method = post_data["method"] + liblaas_endpoint = post_data["endpoint"] + payload = post_data["workflow_data"] + # Fill in actual username + liblaas_endpoint = liblaas_endpoint.replace("[username]", str(request.user)) + liblaas_endpoint = liblaas_base_url + liblaas_endpoint + print("processed endpoint is ", liblaas_endpoint) + + if (http_method == "GET"): + response = requests.get(liblaas_endpoint, data=json.dumps(payload)) + elif (http_method == "POST"): + response = requests.post(liblaas_endpoint, data=json.dumps(payload), headers={'Content-Type': 'application/json'}) + elif (http_method == "DELETE"): + response = requests.delete(liblaas_endpoint, data=json.dumps(payload)) + elif (http_method == "PUT"): + response = requests.put(liblaas_endpoint, data=json.dumps(payload)) + else: + return JsonResponse( + data={}, + status=405, + safe=False + ) + try: + return JsonResponse( + data=json.loads(response.content.decode('utf8')), + status=200, + safe=False + ) + except Exception as e: + print("fail") + print(e) + return JsonResponse( + data = {}, + status=500, + safe=False + ) - # overview - overview = { - 'username': GeneratedCloudConfig._normalize_username(None, str(token.user)), - 'purpose': booking.purpose, - 'project': booking.project, - 'start_time': booking.start, - 'end_time': booking.end, - 'pod_definitions': booking.resource.template, - 'lab': booking.lab - } +def liblaas_templates(request): + liblaas_url = os.environ.get("LIBLAAS_BASE_URL") + "template/list/" + str(request.user) + print("api call to " + liblaas_url) + return requests.get(liblaas_url) - # deployment progress - task_list = [] - for task in booking.job.get_tasklist(): - task_info = { - 'name': str(task), - 'status': 'DONE', - 'lab_response': 'No response provided (yet)' - } - if task.status < 100: - task_info['status'] = 'PENDING' - elif task.status < 200: - task_info['status'] = 'IN PROGRESS' - - if task.message: - if task.type_str == "Access Task" and request.user.id != task.config.user.id: - task_info['lab_response'] = '--secret--' - else: - task_info['lab_response'] = str(task.message) - task_list.append(task_info) - - # pods - pod_list = [] - for host in booking.resource.get_resources(): - pod_info = { - 'hostname': host.config.name, - 'machine': host.name, - 'role': '', - 'is_headnode': host.config.is_head_node, - 'image': host.config.image, - 'ram': {'amount': str(host.profile.ramprofile.first().amount) + 'G', 'channels': host.profile.ramprofile.first().channels}, - 'cpu': {'arch': host.profile.cpuprofile.first().architecture, 'cores': host.profile.cpuprofile.first().cores, 'sockets': host.profile.cpuprofile.first().cpus}, - 'disk': {'size': str(host.profile.storageprofile.first().size) + 'GiB', 'type': host.profile.storageprofile.first().media_type, 'mount_point': host.profile.storageprofile.first().name}, - 'interfaces': [], - } - try: - pod_info['role'] = host.template.opnfvRole - except Exception: - pass - for intprof in host.profile.interfaceprofile.all(): - int_info = { - 'name': intprof.name, - 'speed': intprof.speed - } - pod_info['interfaces'].append(int_info) - pod_list.append(pod_info) - - # diagnostic info - diagnostic_info = { - 'job_id': booking.job.id, - 'ci_files': '', - 'pods': [] - } - for host in booking.resource.get_resources(): - pod = { - 'host': host.name, - 'configs': [], +def delete_template(request): + endpoint = json.loads(request.body)["endpoint"] + liblaas_url = os.environ.get("LIBLAAS_BASE_URL") + endpoint + print("api call to ", liblaas_url) + try: + response = requests.delete(liblaas_url) + return JsonResponse( + data={}, + status=response.status_code, + safe=False + ) + except: + return JsonResponse( + data={}, + status=500, + safe=False + ) - } - for ci_file in host.config.cloud_init_files.all(): - ci_info = { - 'id': ci_file.id, - 'text': ci_file.text - } - pod['configs'].append(ci_info) - diagnostic_info['pods'].append(pod) - - details = { - 'overview': overview, - 'deployment_progress': task_list, - 'pods': pod_list, - 'diagnostic_info': diagnostic_info, - 'pdf': booking.pdf - } - return JsonResponse(str(details), safe=False) +def get_booking_status(bookingObject): + liblaas_url = os.environ.get("LIBLAAS_BASE_URL") + "booking/" + bookingObject.aggregateId + "/status" + print("Getting booking status at: ", liblaas_url) + response = requests.get(liblaas_url) + try: + return json.loads(response.content) + except: + print("failed to get status") + return [] + +def liblaas_end_booking(aggregateId): + liblaas_url = os.environ.get('LIBLAAS_BASE_URL') + "booking/" + str(aggregateId) + "/end" + print("Ending booking at ", liblaas_url) + response = requests.delete(liblaas_url) + try: + return response + except: + print("failed to end booking") + return HttpResponse(status=500) \ No newline at end of file diff --git a/src/booking/forms.py b/src/booking/forms.py index 9c9b053..c7169bb 100644 --- a/src/booking/forms.py +++ b/src/booking/forms.py @@ -7,117 +7,3 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import django.forms as forms -from django.forms.widgets import NumberInput - -from workflow.forms import ( - MultipleSelectFilterField, - MultipleSelectFilterWidget) -from account.models import UserProfile -from resource_inventory.models import Image, Installer, Scenario -from workflow.forms import SearchableSelectMultipleField -from booking.lib import get_user_items, get_user_field_opts - - -class QuickBookingForm(forms.Form): - # Django Form class for Express Booking - purpose = forms.CharField(max_length=1000) - project = forms.CharField(max_length=400) - hostname = forms.CharField(required=False, max_length=400) - global_cloud_config = forms.CharField(widget=forms.Textarea, required=False) - - installer = forms.ModelChoiceField(queryset=Installer.objects.all(), required=False) - scenario = forms.ModelChoiceField(queryset=Scenario.objects.all(), required=False) - - def __init__(self, data=None, user=None, lab_data=None, *args, **kwargs): - if "default_user" in kwargs: - default_user = kwargs.pop("default_user") - else: - default_user = "you" - self.default_user = default_user - - super(QuickBookingForm, self).__init__(data=data, **kwargs) - - image_help_text = 'Image can be set only for single-node bookings. For multi-node bookings set image through Design a POD.' - self.fields["image"] = forms.ModelChoiceField( - Image.objects.filter(public=True) | Image.objects.filter(owner=user), required=False - ) - - self.fields['image'].widget.attrs.update({ - 'class': 'has-popover', - 'data-content': image_help_text, - 'data-placement': 'bottom', - 'data-container': 'body' - }) - - self.fields['users'] = SearchableSelectMultipleField( - queryset=UserProfile.objects.filter(public_user=True).select_related('user').exclude(user=user), - items=get_user_items(exclude=user), - required=False, - **get_user_field_opts() - ) - - self.fields['length'] = forms.IntegerField( - widget=NumberInput( - attrs={ - "type": "range", - 'min': "1", - "max": "21", - "value": "1" - } - ) - ) - - self.fields['filter_field'] = MultipleSelectFilterField(widget=MultipleSelectFilterWidget(**lab_data)) - - hostname_help_text = 'Hostname can be set only for single-node bookings. For multi-node bookings set hostname through Design a POD.' - self.fields['hostname'].widget.attrs.update({ - 'class': 'has-popover', - 'data-content': hostname_help_text, - 'data-placement': 'top', - 'data-container': 'body' - }) - - def build_user_list(self): - """ - Build list of UserProfiles. - - returns a mapping of UserProfile ids to displayable objects expected by - searchable multiple select widget - """ - try: - users = {} - d_qset = UserProfile.objects.select_related('user').all().exclude(user__username=self.default_user) - for userprofile in d_qset: - user = { - 'id': userprofile.user.id, - 'expanded_name': userprofile.full_name, - 'small_name': userprofile.user.username, - 'string': userprofile.email_addr - } - - users[userprofile.user.id] = user - - return users - except Exception: - pass - - def build_search_widget_attrs(self, chosen_users, default_user="you"): - - attrs = { - 'set': self.build_user_list(), - 'show_from_noentry': "false", - 'show_x_results': 10, - 'scrollable': "false", - 'selectable_limit': -1, - 'name': "users", - 'placeholder': "username", - 'initial': chosen_users, - 'edit': False - } - return attrs - - -class HostReImageForm(forms.Form): - - image_id = forms.IntegerField() - host_id = forms.IntegerField() diff --git a/src/booking/migrations/0010_auto_20230608_1913.py b/src/booking/migrations/0010_auto_20230608_1913.py new file mode 100644 index 0000000..66bf63b --- /dev/null +++ b/src/booking/migrations/0010_auto_20230608_1913.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2 on 2023-06-08 19:13 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('booking', '0009_booking_complete'), + ] + + operations = [ + migrations.RemoveField( + model_name='booking', + name='jira_issue_id', + ), + migrations.RemoveField( + model_name='booking', + name='jira_issue_status', + ), + migrations.RemoveField( + model_name='booking', + name='opnfv_config', + ), + migrations.RemoveField( + model_name='booking', + name='resource', + ), + ] diff --git a/src/booking/migrations/0011_booking_aggregateid.py b/src/booking/migrations/0011_booking_aggregateid.py new file mode 100644 index 0000000..111b36b --- /dev/null +++ b/src/booking/migrations/0011_booking_aggregateid.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2 on 2023-07-17 15:25 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('booking', '0010_auto_20230608_1913'), + ] + + operations = [ + migrations.AddField( + model_name='booking', + name='aggregateId', + field=models.CharField(blank=True, max_length=36, validators=[django.core.validators.RegexValidator(code='nomatch', message='aggregate_id must be a valid UUID', regex='^[0-9a-fA-F]{8}\x08-[0-9a-fA-F]{4}\x08-[0-9a-fA-F]{4}\x08-[0-9a-fA-F]{4}\x08-[0-9a-fA-F]{12}$')]), + ), + ] diff --git a/src/booking/models.py b/src/booking/models.py index 966f1c2..09244d3 100644 --- a/src/booking/models.py +++ b/src/booking/models.py @@ -9,11 +9,10 @@ ############################################################################## -from resource_inventory.models import ResourceBundle, OPNFVConfig from account.models import Lab from django.contrib.auth.models import User from django.db import models -import resource_inventory.resource_manager +from django.core.validators import RegexValidator class Booking(models.Model): @@ -26,47 +25,21 @@ class Booking(models.Model): start = models.DateTimeField() end = models.DateTimeField() reset = models.BooleanField(default=False) - jira_issue_id = models.IntegerField(null=True, blank=True) - jira_issue_status = models.CharField(max_length=50, blank=True) purpose = models.CharField(max_length=300, blank=False) # bookings can be extended a limited number of times ext_count = models.IntegerField(default=2) # the hardware that the user has booked - resource = models.ForeignKey(ResourceBundle, on_delete=models.SET_NULL, null=True, blank=True) - opnfv_config = models.ForeignKey(OPNFVConfig, on_delete=models.SET_NULL, null=True, blank=True) project = models.CharField(max_length=100, default="", blank=True, null=True) lab = models.ForeignKey(Lab, null=True, on_delete=models.SET_NULL) pdf = models.TextField(blank=True, default="") idf = models.TextField(blank=True, default="") + # Associated LibLaaS aggregate + aggregateId = models.CharField(blank=True, max_length=36, validators=[RegexValidator(regex='^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$', message='aggregate_id must be a valid UUID', code='nomatch')]) complete = models.BooleanField(default=False) class Meta: db_table = 'booking' - def save(self, *args, **kwargs): - """ - Save the booking if self.user is authorized and there is no overlapping booking. - - Raise PermissionError if the user is not authorized - Raise ValueError if there is an overlapping booking - """ - if self.start >= self.end: - raise ValueError('Start date is after end date') - # conflicts end after booking starts, and start before booking ends - conflicting_dates = Booking.objects.filter(resource=self.resource).exclude(id=self.id) - conflicting_dates = conflicting_dates.filter(end__gt=self.start) - conflicting_dates = conflicting_dates.filter(start__lt=self.end) - if conflicting_dates.count() > 0: - raise ValueError('This booking overlaps with another booking') - return super(Booking, self).save(*args, **kwargs) - - def delete(self, *args, **kwargs): - res = self.resource - self.resource = None - self.save() - resource_inventory.resource_manager.ResourceManager.getInstance().deleteResourceBundle(res) - return super(self.__class__, self).delete(*args, **kwargs) - def __str__(self): return str(self.purpose) + ' from ' + str(self.start) + ' until ' + str(self.end) diff --git a/src/booking/quick_deployer.py b/src/booking/quick_deployer.py deleted file mode 100644 index 4b85d76..0000000 --- a/src/booking/quick_deployer.py +++ /dev/null @@ -1,343 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -import json -import yaml -from django.db.models import Q -from django.db import transaction -from datetime import timedelta -from django.utils import timezone -from django.core.exceptions import ValidationError -from account.models import Lab, UserProfile - -from resource_inventory.models import ( - ResourceTemplate, - Image, - OPNFVRole, - OPNFVConfig, - ResourceOPNFVConfig, - ResourceConfiguration, - NetworkConnection, - InterfaceConfiguration, - Network, - CloudInitFile, -) -from resource_inventory.resource_manager import ResourceManager -from resource_inventory.pdf_templater import PDFTemplater -from notifier.manager import NotificationHandler -from booking.models import Booking -from dashboard.exceptions import BookingLengthException -from api.models import JobFactory - - -def parse_resource_field(resource_json): - """ - Parse the json from the frontend. - - returns a reference to the selected Lab and ResourceTemplate objects - """ - lab, template = (None, None) - lab_dict = resource_json['lab'] - for lab_info in lab_dict.values(): - if lab_info['selected']: - lab = Lab.objects.get(lab_user__id=lab_info['id']) - - resource_dict = resource_json['resource'] - for resource_info in resource_dict.values(): - if resource_info['selected']: - template = ResourceTemplate.objects.get(pk=resource_info['id']) - - if lab is None: - raise ValidationError("No lab was selected") - if template is None: - raise ValidationError("No Host was selected") - - return lab, template - - -def update_template(old_template, image, hostname, user, global_cloud_config=None): - """ - Duplicate a template to the users account and update configured fields. - - The dashboard presents users with preconfigured resource templates, - but the user might want to make small modifications, e.g hostname and - linux distro. So we copy the public template and create a private version - to the user's profile, and mark it temporary. When the booking ends the - new template is deleted - """ - name = user.username + "'s Copy of '" + old_template.name + "'" - num_copies = ResourceTemplate.objects.filter(name__startswith=name).count() - template = ResourceTemplate.objects.create( - name=name if num_copies == 0 else name + " (" + str(num_copies) + ")", - xml=old_template.xml, - owner=user, - lab=old_template.lab, - description=old_template.description, - public=False, - temporary=True, - private_vlan_pool=old_template.private_vlan_pool, - public_vlan_pool=old_template.public_vlan_pool, - copy_of=old_template - ) - - for old_network in old_template.networks.all(): - Network.objects.create( - name=old_network.name, - bundle=template, - is_public=old_network.is_public - ) - # We are assuming there is only one opnfv config per public resource template - old_opnfv = template.opnfv_config.first() - if old_opnfv: - opnfv_config = OPNFVConfig.objects.create( - installer=old_opnfv.installer, - scenario=old_opnfv.installer, - template=template, - name=old_opnfv.installer, - ) - # I am explicitly leaving opnfv_config.networks empty to avoid - # problems with duplicated / shared networks. In the quick deploy, - # there is never multiple networks anyway. This may have to change in the future - - for old_config in old_template.getConfigs(): - image_to_set = image - if not image: - image_to_set = old_config.image - - config = ResourceConfiguration.objects.create( - profile=old_config.profile, - image=image_to_set, - template=template, - is_head_node=old_config.is_head_node, - name=hostname if len(old_template.getConfigs()) == 1 else old_config.name, - # cloud_init_files=old_config.cloud_init_files.set() - ) - - for file in old_config.cloud_init_files.all(): - config.cloud_init_files.add(file) - - if global_cloud_config: - config.cloud_init_files.add(global_cloud_config) - config.save() - - for old_iface_config in old_config.interface_configs.all(): - iface_config = InterfaceConfiguration.objects.create( - profile=old_iface_config.profile, - resource_config=config - ) - - for old_connection in old_iface_config.connections.all(): - iface_config.connections.add(NetworkConnection.objects.create( - network=template.networks.get(name=old_connection.network.name), - vlan_is_tagged=old_connection.vlan_is_tagged - )) - - for old_res_opnfv in old_config.resource_opnfv_config.all(): - if old_opnfv: - ResourceOPNFVConfig.objects.create( - role=old_opnfv.role, - resource_config=config, - opnfv_config=opnfv_config - ) - return template - - -def generate_opnfvconfig(scenario, installer, template): - return OPNFVConfig.objects.create( - scenario=scenario, - installer=installer, - template=template - ) - - -def generate_hostopnfv(hostconfig, opnfvconfig): - role = None - try: - role = OPNFVRole.objects.get(name="Jumphost") - except Exception: - role = OPNFVRole.objects.create( - name="Jumphost", - description="Single server jumphost role" - ) - return ResourceOPNFVConfig.objects.create( - role=role, - host_config=hostconfig, - opnfv_config=opnfvconfig - ) - - -def generate_resource_bundle(template): - resource_manager = ResourceManager.getInstance() - resource_bundle = resource_manager.instantiateTemplate(template) - return resource_bundle - - -def check_invariants(**kwargs): - # TODO: This should really happen in the BookingForm validation methods - image = kwargs['image'] - lab = kwargs['lab'] - length = kwargs['length'] - # check that image os is compatible with installer - if image: - if image.from_lab != lab: - raise ValidationError("The chosen image is not available at the chosen hosting lab") - # TODO - # if image.host_type != host_profile: - # raise ValidationError("The chosen image is not available for the chosen host type") - if not image.public and image.owner != kwargs['owner']: - raise ValidationError("You are not the owner of the chosen private image") - if length < 1 or length > 21: - raise BookingLengthException("Booking must be between 1 and 21 days long") - - -def create_from_form(form, request): - """ - Parse data from QuickBookingForm to create booking - """ - resource_field = form.cleaned_data['filter_field'] - # users_field = form.cleaned_data['users'] - hostname = 'opnfv_host' if not form.cleaned_data['hostname'] else form.cleaned_data['hostname'] - - global_cloud_config = None if not form.cleaned_data['global_cloud_config'] else form.cleaned_data['global_cloud_config'] - - if global_cloud_config: - form.cleaned_data['global_cloud_config'] = create_ci_file(global_cloud_config) - - # image = form.cleaned_data['image'] - # scenario = form.cleaned_data['scenario'] - # installer = form.cleaned_data['installer'] - - lab, resource_template = parse_resource_field(resource_field) - data = form.cleaned_data - data['hostname'] = hostname - data['lab'] = lab - data['resource_template'] = resource_template - data['owner'] = request.user - - return _create_booking(data) - - -def create_from_API(body, user): - """ - Parse data from Automation API to create booking - """ - booking_info = json.loads(body.decode('utf-8')) - - data = {} - data['purpose'] = booking_info['purpose'] - data['project'] = booking_info['project'] - data['users'] = [UserProfile.objects.get(user__username=username) - for username in booking_info['collaborators']] - data['hostname'] = booking_info['hostname'] - data['length'] = booking_info['length'] - data['installer'] = None - data['scenario'] = None - - data['image'] = Image.objects.get(pk=booking_info['imageLabID']) - - data['resource_template'] = ResourceTemplate.objects.get(pk=booking_info['templateID']) - data['lab'] = data['resource_template'].lab - data['owner'] = user - - if 'global_cloud_config' in data.keys(): - data['global_cloud_config'] = CloudInitFile.objects.get(id=data['global_cloud_config']) - - return _create_booking(data) - - -def create_ci_file(data: str) -> CloudInitFile: - try: - d = yaml.load(data) - if not (type(d) is dict): - raise Exception("CI file was valid yaml but was not a dict") - except Exception: - raise ValidationError("The provided Cloud Config is not valid yaml, please refer to the Cloud Init documentation for expected structure") - print("about to create global cloud config") - config = CloudInitFile.create(text=data, priority=CloudInitFile.objects.count()) - print("made global cloud config") - - return config - - -@transaction.atomic -def _create_booking(data): - check_invariants(**data) - - # check booking privileges - # TODO: use the canonical booking_allowed method because now template might have multiple - # machines - if Booking.objects.filter(owner=data['owner'], end__gt=timezone.now()).count() >= 3 and not data['owner'].userprofile.booking_privledge: - raise PermissionError("You do not have permission to have more than 3 bookings at a time.") - - ResourceManager.getInstance().templateIsReservable(data['resource_template']) - - resource_template = update_template(data['resource_template'], data['image'], data['hostname'], data['owner'], global_cloud_config=data['global_cloud_config']) - - # generate resource bundle - resource_bundle = generate_resource_bundle(resource_template) - - # generate booking - booking = Booking.objects.create( - purpose=data['purpose'], - project=data['project'], - lab=data['lab'], - owner=data['owner'], - start=timezone.now(), - end=timezone.now() + timedelta(days=int(data['length'])), - resource=resource_bundle, - opnfv_config=None - ) - - booking.pdf = PDFTemplater.makePDF(booking) - - for collaborator in data['users']: # list of Users (not UserProfile) - booking.collaborators.add(collaborator.user) - - booking.save() - - # generate job - JobFactory.makeCompleteJob(booking) - NotificationHandler.notify_new_booking(booking) - - return booking - - -def drop_filter(user): - """ - Return a dictionary that contains filters. - - Only certain installlers are supported on certain images, etc - so the image filter indexed at [imageid][installerid] is truthy if - that installer is supported on that image - """ - installer_filter = {} - scenario_filter = {} - - images = Image.objects.filter(Q(public=True) | Q(owner=user)) - image_filter = {} - for image in images: - image_filter[image.id] = { - 'lab': 'lab_' + str(image.from_lab.lab_user.id), - 'architecture': str(image.architecture), - 'name': image.name - } - - resource_filter = {} - templates = ResourceTemplate.objects.filter(Q(public=True) | Q(owner=user)) - for rt in templates: - profiles = [conf.profile for conf in rt.getConfigs()] - resource_filter["resource_" + str(rt.id)] = [str(p.architecture) for p in profiles] - - return { - 'installer_filter': json.dumps(installer_filter), - 'scenario_filter': json.dumps(scenario_filter), - 'image_filter': json.dumps(image_filter), - 'resource_profile_map': json.dumps(resource_filter), - } diff --git a/src/booking/stats.py b/src/booking/stats.py deleted file mode 100644 index 5a59d32..0000000 --- a/src/booking/stats.py +++ /dev/null @@ -1,109 +0,0 @@ -############################################################################## -# Copyright (c) 2020 Parker Berberian, Sawyer Bergeron, Sean Smith and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import os -from booking.models import Booking -from resource_inventory.models import ResourceQuery, ResourceProfile -from datetime import datetime, timedelta -from collections import Counter -import pytz - - -class StatisticsManager(object): - - @staticmethod - def getContinuousBookingTimeSeries(span=28): - """ - Calculate Booking usage data points. - - Gathers all active bookings that fall in interval [(now - span), (now + 1 week)]. - x data points are every 12 hours - y values are the integer number of bookings/users active at time - """ - - anuket_colors = [ - '#6BDAD5', # Turquoise - '#E36386', # Pale Violet Red - '#F5B335', # Sandy Brown - '#007473', # Teal - '#BCE194', # Gainsboro - '#00CE7C', # Sea Green - ] - - lfedge_colors = [ - '#0049B0', - '#B481A5', - '#6CAFE4', - '#D33668', - '#28245A' - ] - - x = [] - y = [] - users = [] - projects = [] - profiles = {str(profile): [] for profile in ResourceProfile.objects.all()} - - now = datetime.now(pytz.utc) - delta = timedelta(days=span) - start = now - delta - end = now + timedelta(weeks=1) - - bookings = Booking.objects.filter( - start__lte=end, - end__gte=start - ).prefetch_related("collaborators") - - # get data - while start <= end: - active_users = 0 - - books = bookings.filter( - start__lte=start, - end__gte=start - ).prefetch_related("collaborators") - - for booking in books: - active_users += booking.collaborators.all().count() + 1 - - x.append(str(start.month) + '-' + str(start.day)) - y.append(books.count()) - - step_profiles = Counter([ - str(config.profile) - for book in books - for config in book.resource.template.getConfigs() - ]) - - for profile in ResourceProfile.objects.all(): - profiles[str(profile)].append(step_profiles[str(profile)]) - users.append(active_users) - - start += timedelta(hours=12) - - in_use = len(ResourceQuery.filter(working=True, booked=True)) - not_in_use = len(ResourceQuery.filter(working=True, booked=False)) - maintenance = len(ResourceQuery.filter(working=False)) - - projects = [x.project for x in bookings] - proj_count = sorted(Counter(projects).items(), key=lambda x: x[1]) - - project_keys = [proj[0] for proj in proj_count[-5:]] - project_keys = ['None' if x is None else x for x in project_keys] - project_counts = [proj[1] for proj in proj_count[-5:]] - - resources = {key: [x, value] for key, value in profiles.items()} - - return { - "resources": resources, - "booking": [x, y], - "user": [x, users], - "utils": [in_use, not_in_use, maintenance], - "projects": [project_keys, project_counts], - "colors": anuket_colors if os.environ.get('TEMPLATE_OVERRIDE_DIR') == 'laas' else lfedge_colors - } diff --git a/src/booking/tests/__init__.py b/src/booking/tests/__init__.py deleted file mode 100644 index b6fef6c..0000000 --- a/src/booking/tests/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## diff --git a/src/booking/tests/test_models.py b/src/booking/tests/test_models.py deleted file mode 100644 index 37eb655..0000000 --- a/src/booking/tests/test_models.py +++ /dev/null @@ -1,210 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from datetime import timedelta - -from django.contrib.auth.models import User -from django.test import TestCase -from django.utils import timezone - -from booking.models import Booking -from dashboard.testing_utils import make_resource_template, make_user - - -class BookingModelTestCase(TestCase): - """ - Test the Booking model. - - Creates all the scafolding needed and tests the Booking model - """ - - def setUp(self): - """ - Prepare for Booking model tests. - - Creates all the needed models, such as users, resources, and configurations - """ - self.owner = User.objects.create(username='owner') - self.res1 = make_resource_template(name="Test template 1") - self.res2 = make_resource_template(name="Test template 2") - self.user1 = make_user(username='user1') - - def test_start_end(self): - """ - Verify the start and end fields. - - if the start of a booking is greater or equal then the end, - saving should raise a ValueException - """ - start = timezone.now() - end = start - timedelta(weeks=1) - self.assertRaises( - ValueError, - Booking.objects.create, - start=start, - end=end, - resource=self.res1, - owner=self.user1, - ) - end = start - self.assertRaises( - ValueError, - Booking.objects.create, - start=start, - end=end, - resource=self.res1, - owner=self.user1, - ) - - def test_conflicts(self): - """ - Verify conflicting dates are dealt with. - - saving an overlapping booking on the same resource - should raise a ValueException - saving for different resources should succeed - """ - start = timezone.now() - end = start + timedelta(weeks=1) - self.assertTrue( - Booking.objects.create( - start=start, - end=end, - owner=self.user1, - resource=self.res1, - ) - ) - - self.assertRaises( - ValueError, - Booking.objects.create, - start=start, - end=end, - resource=self.res1, - owner=self.user1, - ) - - self.assertRaises( - ValueError, - Booking.objects.create, - start=start + timedelta(days=1), - end=end - timedelta(days=1), - resource=self.res1, - owner=self.user1, - ) - - self.assertRaises( - ValueError, - Booking.objects.create, - start=start - timedelta(days=1), - end=end, - resource=self.res1, - owner=self.user1, - ) - - self.assertRaises( - ValueError, - Booking.objects.create, - start=start - timedelta(days=1), - end=end - timedelta(days=1), - resource=self.res1, - owner=self.user1, - ) - - self.assertRaises( - ValueError, - Booking.objects.create, - start=start, - end=end + timedelta(days=1), - resource=self.res1, - owner=self.user1, - ) - - self.assertRaises( - ValueError, - Booking.objects.create, - start=start + timedelta(days=1), - end=end + timedelta(days=1), - resource=self.res1, - owner=self.user1, - ) - - self.assertTrue( - Booking.objects.create( - start=start - timedelta(days=1), - end=start, - owner=self.user1, - resource=self.res1, - ) - ) - - self.assertTrue( - Booking.objects.create( - start=end, - end=end + timedelta(days=1), - owner=self.user1, - resource=self.res1, - ) - ) - - self.assertTrue( - Booking.objects.create( - start=start - timedelta(days=2), - end=start - timedelta(days=1), - owner=self.user1, - resource=self.res1, - ) - ) - - self.assertTrue( - Booking.objects.create( - start=end + timedelta(days=1), - end=end + timedelta(days=2), - owner=self.user1, - resource=self.res1, - ) - ) - - self.assertTrue( - Booking.objects.create( - start=start, - end=end, - owner=self.user1, - resource=self.res2, - ) - ) - - def test_extensions(self): - """ - Test booking extensions. - - saving a booking with an extended end time is allows to happen twice, - and each extension must be a maximum of one week long - """ - start = timezone.now() - end = start + timedelta(weeks=1) - self.assertTrue( - Booking.objects.create( - start=start, - end=end, - owner=self.user1, - resource=self.res1, - ) - ) - - booking = Booking.objects.all().first() # should be only thing in db - - self.assertEquals(booking.ext_count, 2) - booking.end = booking.end + timedelta(days=3) - try: - booking.save() - except Exception: - self.fail("save() threw an exception") diff --git a/src/booking/tests/test_quick_booking.py b/src/booking/tests/test_quick_booking.py deleted file mode 100644 index f405047..0000000 --- a/src/booking/tests/test_quick_booking.py +++ /dev/null @@ -1,180 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import datetime -import json - -from django.test import TestCase, Client - -from booking.models import Booking -from dashboard.testing_utils import ( - make_user, - make_user_profile, - make_lab, - make_image, - make_os, - make_opnfv_role, - make_public_net, - make_resource_template, - make_server -) - - -class QuickBookingValidFormTestCase(TestCase): - @classmethod - def setUpTestData(cls): - cls.user = make_user(False, username="newtestuser") - cls.user.set_password("testpassword") - cls.user.save() - make_user_profile(cls.user, True) - - cls.lab = make_lab() - - cls.res_template = make_resource_template(owner=cls.user, lab=cls.lab) - cls.res_profile = cls.res_template.getConfigs()[0].profile - os = make_os() - cls.image = make_image(cls.res_profile, lab=cls.lab, owner=cls.user, os=os) - cls.server = make_server(cls.res_profile, cls.lab) - cls.role = make_opnfv_role() - cls.pubnet = make_public_net(10, cls.lab) - - cls.post_data = cls.build_post_data() - cls.client = Client() - - @classmethod - def build_post_data(cls): - return { - 'filter_field': json.dumps({ - "resource": { - "resource_" + str(cls.res_profile.id): { - "selected": True, - "id": cls.res_template.id - } - }, - "lab": { - "lab_" + str(cls.lab.lab_user.id): { - "selected": True, - "id": cls.lab.lab_user.id - } - } - }), - 'purpose': 'my_purpose', - 'project': 'my_project', - 'length': '3', - 'ignore_this': 1, - 'users': '', - 'hostname': 'my_host', - 'image': str(cls.image.id), - } - - def post(self, changed_fields={}): - payload = self.post_data.copy() - payload.update(changed_fields) - response = self.client.post('/booking/quick/', payload) - return response - - def setUp(self): - self.client.login(username="newtestuser", password="testpassword") - - def assertValidBooking(self, booking): - self.assertEqual(booking.owner, self.user) - self.assertEqual(booking.purpose, 'my_purpose') - self.assertEqual(booking.project, 'my_project') - delta = booking.end - booking.start - delta -= datetime.timedelta(days=3) - self.assertLess(delta, datetime.timedelta(minutes=1)) - - resource_bundle = booking.resource - - host = resource_bundle.get_resources()[0] - self.assertEqual(host.profile, self.res_profile) - self.assertEqual(host.name, 'my_host') - - def test_with_too_long_length(self): - response = self.post({'length': '22'}) - - self.assertEqual(response.status_code, 200) - self.assertIsNone(Booking.objects.first()) - - def test_with_negative_length(self): - response = self.post({'length': '-1'}) - - self.assertEqual(response.status_code, 200) - self.assertIsNone(Booking.objects.first()) - - def test_with_invalid_installer(self): - response = self.post({'installer': str(self.installer.id + 100)}) - - self.assertEqual(response.status_code, 200) - self.assertIsNone(Booking.objects.first()) - - def test_with_invalid_scenario(self): - response = self.post({'scenario': str(self.scenario.id + 100)}) - - self.assertEqual(response.status_code, 200) - self.assertIsNone(Booking.objects.first()) - - def test_with_invalid_host_id(self): - response = self.post({'filter_field': json.dumps({ - "resource": { - "resource_" + str(self.res_profile.id + 100): { - "selected": True, - "id": self.res_profile.id + 100 - } - }, - "lab": { - "lab_" + str(self.lab.lab_user.id): { - "selected": True, - "id": self.lab.lab_user.id - } - } - })}) - - self.assertEqual(response.status_code, 200) - self.assertIsNone(Booking.objects.first()) - - def test_with_invalid_lab_id(self): - response = self.post({'filter_field': json.dumps({ - "resource": { - "resource_" + str(self.res_profile.id): { - "selected": True, - "id": self.res_profile.id - } - }, - "lab": { - "lab_" + str(self.lab.lab_user.id + 100): { - "selected": True, - "id": self.lab.lab_user.id + 100 - } - } - })}) - - self.assertEqual(response.status_code, 200) - self.assertIsNone(Booking.objects.first()) - - def test_with_invalid_empty_filter_field(self): - response = self.post({'filter_field': ''}) - - self.assertEqual(response.status_code, 200) - self.assertIsNone(Booking.objects.first()) - - def test_with_garbage_users_field(self): # expected behavior: treat as though field is empty if it has garbage data - response = self.post({'users': ['X�]QP�槰DP�+m���h�U�_�yJA:.rDi��QN|.��C��n�P��F!��D�����5ȅj�9�LV��']}) # output from /dev/urandom - - self.assertEqual(response.status_code, 200) - booking = Booking.objects.first() - self.assertIsNone(booking) - - def test_with_valid_form(self): - response = self.post() - - self.assertEqual(response.status_code, 302) # success should redirect - booking = Booking.objects.first() - self.assertIsNotNone(booking) - self.assertValidBooking(booking) diff --git a/src/booking/tests/test_stats.py b/src/booking/tests/test_stats.py deleted file mode 100644 index 5501355..0000000 --- a/src/booking/tests/test_stats.py +++ /dev/null @@ -1,59 +0,0 @@ -############################################################################# -# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, Sean Smith, and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -import pytz -from datetime import timedelta, datetime - -from django.test import TestCase - -from booking.models import Booking -from booking.stats import StatisticsManager as sm -from dashboard.testing_utils import make_user - - -class StatsTestCases(TestCase): - - def test_no_booking_outside_span(self): - now = datetime.now(pytz.utc) - - bad_date = now + timedelta(days=1200) - Booking.objects.create(start=now, end=bad_date, owner=make_user(username='jj')) - - actual = sm.getContinuousBookingTimeSeries() - dates = actual['booking'][0] - - for date in dates: - self.assertNotEqual(date, bad_date) - - def check_booking_and_user_counts(self): - now = datetime.now(pytz.utc) - - for i in range(20): - Booking.objects.create( - start=now, - end=now + timedelta(weeks=3), - owner=make_user(username='a')) - - for i in range(30): - Booking.objects.create( - start=now + timedelta(days=5), - end=now + timedelta(weeks=3, days=5), - owner=make_user(username='a')) - - for i in range(120): - Booking.objects.create( - start=now + timedelta(weeks=1), - end=now + timedelta(weeks=4), - owner=make_user(username='a')) - - dates = [[now, 20], [now + timedelta(days=5), 30], [now + timedelta(weeks=1), 120]] - actual = sm.getContinuousBookingTimeSeries() - - for date in dates: - self.assertEqual(date[1], actual['booking'][date[0]]) - self.assertEqual(date[1], actual['booking'][date[1]]) diff --git a/src/booking/urls.py b/src/booking/urls.py index 0b60351..9784fc5 100644 --- a/src/booking/urls.py +++ b/src/booking/urls.py @@ -32,10 +32,6 @@ from booking.views import ( BookingDeleteView, bookingDelete, BookingListView, - booking_stats_view, - booking_stats_json, - quick_create, - booking_modify_image ) app_name = 'booking' @@ -45,9 +41,5 @@ urlpatterns = [ url(r'^delete/$', BookingDeleteView.as_view(), name='delete_prefix'), url(r'^delete/(?P[0-9]+)/$', BookingDeleteView.as_view(), name='delete'), url(r'^delete/(?P[0-9]+)/confirm/$', bookingDelete, name='delete_booking'), - url(r'^modify/(?P[0-9]+)/image/$', booking_modify_image, name='modify_booking_image'), url(r'^list/$', BookingListView.as_view(), name='list'), - url(r'^stats/$', booking_stats_view, name='stats'), - url(r'^stats/json$', booking_stats_json, name='stats_json'), - url(r'^quick/$', quick_create, name='quick_create'), ] diff --git a/src/booking/views.py b/src/booking/views.py index 367a18d..25cac43 100644 --- a/src/booking/views.py +++ b/src/booking/views.py @@ -18,63 +18,9 @@ from django.shortcuts import redirect, render from django.db.models import Q from django.urls import reverse -from resource_inventory.models import ResourceBundle, ResourceProfile, Image, ResourceQuery from account.models import Downtime, Lab +from api.views import get_booking_status from booking.models import Booking -from booking.stats import StatisticsManager -from booking.forms import HostReImageForm -from workflow.forms import FormUtils -from api.models import JobFactory, GeneratedCloudConfig -from workflow.views import login -from booking.forms import QuickBookingForm -from booking.quick_deployer import create_from_form, drop_filter -import traceback - - -def quick_create_clear_fields(request): - request.session['quick_create_forminfo'] = None - - -def quick_create(request): - if not request.user.is_authenticated: - return login(request) - - if request.method == 'GET': - context = {} - attrs = FormUtils.getLabData(user=request.user) - context['form'] = QuickBookingForm(lab_data=attrs, default_user=request.user.username, user=request.user) - context['lab_profile_map'] = {} - context.update(drop_filter(request.user)) - context['contact_email'] = Lab.objects.filter(name="UNH_IOL").first().contact_email - return render(request, 'booking/quick_deploy.html', context) - - if request.method == 'POST': - attrs = FormUtils.getLabData(user=request.user) - form = QuickBookingForm(request.POST, lab_data=attrs, user=request.user) - - context = {} - context['lab_profile_map'] = {} - context['form'] = form - - if form.is_valid(): - try: - booking = create_from_form(form, request) - messages.success(request, "We've processed your request. " - "Check Account->My Bookings for the status of your new booking") - return redirect(reverse('booking:booking_detail', kwargs={'booking_id': booking.id})) - except Exception as e: - print("Error occurred while handling quick deployment:") - traceback.print_exc() - print(str(e)) - messages.error(request, "Whoops, an error occurred: " + str(e)) - context.update(drop_filter(request.user)) - return render(request, 'booking/quick_deploy.html', context) - else: - messages.error(request, "Looks like the form didn't validate. Check that you entered everything correctly") - context['status'] = 'false' - context.update(drop_filter(request.user)) - return render(request, 'booking/quick_deploy.html', context) - class BookingView(TemplateView): template_name = "booking/booking_detail.html" @@ -123,31 +69,6 @@ class BookingListView(TemplateView): return context -class ResourceBookingsJSON(View): - def get(self, request, *args, **kwargs): - resource = get_object_or_404(ResourceBundle, id=self.kwargs['resource_id']) - bookings = resource.booking_set.get_queryset().values( - 'id', - 'start', - 'end', - 'purpose', - 'config_bundle__name' - ) - return JsonResponse({'bookings': list(bookings)}) - - -def build_image_mapping(lab, user): - mapping = {} - for profile in ResourceProfile.objects.filter(labs=lab): - images = Image.objects.filter( - from_lab=lab, - architecture=profile.architecture - ).filter( - Q(public=True) | Q(owner=user) - ) - mapping[profile.name] = [{"name": image.name, "value": image.id} for image in images] - return mapping - def booking_detail_view(request, booking_id): user = None @@ -157,18 +78,17 @@ def booking_detail_view(request, booking_id): return render(request, "dashboard/login.html", {'title': 'Authentication Required'}) booking = get_object_or_404(Booking, id=booking_id) + statuses = get_booking_status(booking) allowed_users = set(list(booking.collaborators.all())) allowed_users.add(booking.owner) if user not in allowed_users: return render(request, "dashboard/login.html", {'title': 'This page is private'}) - + context = { 'title': 'Booking Details', 'booking': booking, - 'pdf': booking.pdf, - 'user_id': user.id, - 'image_mapping': build_image_mapping(booking.lab, user), - 'posix_username': GeneratedCloudConfig._normalize_username(None, user.username) + 'statuses': statuses, + 'collab_string': ', '.join(map(str, booking.collaborators.all())) } return render( @@ -176,36 +96,3 @@ def booking_detail_view(request, booking_id): "booking/booking_detail.html", context ) - - -def booking_modify_image(request, booking_id): - form = HostReImageForm(request.POST) - if form.is_valid(): - booking = Booking.objects.get(id=booking_id) - if request.user != booking.owner: - return HttpResponse("unauthorized") - if timezone.now() > booking.end: - return HttpResponse("unauthorized") - new_image = Image.objects.get(id=form.cleaned_data['image_id']) - host = ResourceQuery.get(id=form.cleaned_data['host_id']) - host.config.image = new_image - host.config.save() - JobFactory.reimageHost(new_image, booking, host) - return HttpResponse(new_image.name) - return HttpResponse("error") - - -def booking_stats_view(request): - return render( - request, - "booking/stats.html", - context={"data": StatisticsManager.getContinuousBookingTimeSeries(), "title": ""} - ) - - -def booking_stats_json(request): - try: - span = int(request.GET.get("days", 14)) - except Exception: - span = 14 - return JsonResponse(StatisticsManager.getContinuousBookingTimeSeries(span), safe=False) diff --git a/src/dashboard/admin_utils.py b/src/dashboard/admin_utils.py deleted file mode 100644 index 75e4f3e..0000000 --- a/src/dashboard/admin_utils.py +++ /dev/null @@ -1,811 +0,0 @@ -############################################################################## -# Copyright (c) 2021 Sawyer Bergeron and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from resource_inventory.models import ( - ResourceTemplate, - Image, - Server, - ResourceBundle, - ResourceProfile, - InterfaceProfile, - PhysicalNetwork, - ResourceConfiguration, - NetworkConnection, - InterfaceConfiguration, - Network, - DiskProfile, - CpuProfile, - RamProfile, - Interface, - CloudInitFile, -) - -import json -import yaml -import sys -import inspect -import pydoc -import csv - -from django.contrib.auth.models import User - -from account.models import ( - Lab, - PublicNetwork -) - -from resource_inventory.resource_manager import ResourceManager -from resource_inventory.pdf_templater import PDFTemplater - -from booking.quick_deployer import update_template - -from datetime import timedelta, date, datetime, timezone - -from booking.models import Booking -from notifier.manager import NotificationHandler -from api.models import JobFactory - -from api.models import JobStatus, Job, GeneratedCloudConfig - - -def print_div(): - """ - Utility function for printing dividers, does nothing directly useful as a utility - """ - print("=" * 68) - - -def book_host(owner_username, host_labid, lab_username, hostname, image_id, template_name, length_days=21, collaborator_usernames=[], purpose="internal", project="LaaS"): - """ - creates a quick booking using the given host - - @owner_username is the simple username for the user who will own the resulting booking. - Do not set this to a lab username! - - @image_id is the django id of the image in question, NOT the labid of the image. - Query Image objects by their public status and compatible host types - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username for iol is `unh_iol`, other labs will be documented here - - @hostname the hostname that the resulting host should have set - - @template_name the name of the (public, or user accessible) template to use for this booking - - @length_days how long the booking should be, no hard limit currently - - @collaborator_usernames a list of usernames for collaborators to the booking - - @purpose what this booking will be used for - - @project what project/group this booking is on behalf of or the owner represents - """ - lab = Lab.objects.get(lab_user__username=lab_username) - host = Server.objects.filter(lab=lab).get(labid=host_labid) - if host.booked: - print("Can't book host, already marked as booked") - return - else: - host.booked = True - host.save() - - template = ResourceTemplate.objects.filter(public=True).get(name=template_name) - image = Image.objects.get(id=image_id) - - owner = User.objects.get(username=owner_username) - - new_template = update_template(template, image, hostname, owner) - - rmanager = ResourceManager.getInstance() - - vlan_map = rmanager.get_vlans(new_template) - - # only a single host so can reuse var for iter here - resource_bundle = ResourceBundle.objects.create(template=new_template) - res_configs = new_template.getConfigs() - - for config in res_configs: - try: - host.bundle = resource_bundle - host.config = config - rmanager.configureNetworking(resource_bundle, host, vlan_map) - host.save() - except Exception: - host.booked = False - host.save() - print("Failed to book host due to error configuring it") - return - - new_template.save() - - booking = Booking.objects.create( - purpose=purpose, - project=project, - lab=lab, - owner=owner, - start=timezone.now(), - end=timezone.now() + timedelta(days=int(length_days)), - resource=resource_bundle, - opnfv_config=None - ) - - booking.pdf = PDFTemplater.makePDF(booking) - - booking.save() - - for collaborator_username in collaborator_usernames: - try: - user = User.objects.get(username=collaborator_username) - booking.collaborators.add(user) - except Exception: - print("couldn't add user with username ", collaborator_username) - - booking.save() - - JobFactory.makeCompleteJob(booking) - NotificationHandler.notify_new_booking(booking) - - -def mark_working(host_labid, lab_username, working=True): - """ - Mark a host working/not working so that it is either bookable or hidden in the dashboard. - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username: param of the form `unh_iol` or similar - - @working: bool, whether by the end of execution the host should be considered working or not working - """ - - lab = Lab.objects.get(lab_user__username=lab_username) - server = Server.objects.filter(lab=lab).get(labid=host_labid) - print("changing server working status from ", server.working, "to", working) - server.working = working - server.save() - - -def mark_booked(host_labid, lab_username, booked=True): - """ - Mark a host as booked/unbooked - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username: param of the form `unh_iol` or similar - - @working: bool, whether by the end of execution the host should be considered booked or not booked - """ - - lab = Lab.objects.get(lab_user__username=lab_username) - server = Server.objects.filter(lab=lab).get(labid=host_labid) - print("changing server booked status from ", server.booked, "to", booked) - server.booked = booked - server.save() - - -def get_host(host_labid, lab_username): - """ - Returns host filtered by lab and then unique id within lab - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username: param of the form `unh_iol` or similar - """ - lab = Lab.objects.get(lab_user__username=lab_username) - return Server.objects.filter(lab=lab).get(labid=host_labid) - - -def get_info(host_labid, lab_username): - """ - Returns various information on the host queried by the given parameters - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username: param of the form `unh_iol` or similar - """ - info = {} - host = get_host(host_labid, lab_username) - info['host_labid'] = host_labid - info['booked'] = host.booked - info['working'] = host.working - info['profile'] = str(host.profile) - if host.bundle: - binfo = {} - info['bundle'] = binfo - if host.config: - cinfo = {} - info['config'] = cinfo - - return info - - -class CumulativeData: - use_days = 0 - count_bookings = 0 - count_extensions = 0 - - def __init__(self, file_writer): - self.file_writer = file_writer - - def account(self, booking, usage_days): - self.count_bookings += 1 - self.count_extensions += booking.ext_count - self.use_days += usage_days - - def write_cumulative(self): - self.file_writer.writerow([]) - self.file_writer.writerow([]) - self.file_writer.writerow(['Lab Use Days', 'Count of Bookings', 'Total Extensions Used']) - self.file_writer.writerow([self.use_days, self.count_bookings, (self.count_bookings * 2) - self.count_extensions]) - - -def get_years_booking_data(start_year=None, end_year=None): - """ - Outputs yearly booking information from the past 'start_year' years (default: current year) - until the last day of the end year (default current year) as a csv file. - """ - if start_year is None and end_year is None: - start = datetime.combine(date(datetime.now().year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - end = datetime.combine(date(start.year + 1, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - elif end_year is None: - start = datetime.combine(date(start_year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - end = datetime.combine(date(datetime.now().year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - else: - start = datetime.combine(date(start_year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - end = datetime.combine(date(end_year + 1, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - - if (start.year == end.year - 1): - file_name = "yearly_booking_data_" + str(start.year) + ".csv" - else: - file_name = "yearly_booking_data_" + str(start.year) + "-" + str(end.year - 1) + ".csv" - - with open(file_name, "w", newline="") as file: - file_writer = csv.writer(file) - cumulative_data = CumulativeData(file_writer) - file_writer.writerow( - [ - 'ID', - 'Project', - 'Purpose', - 'User', - 'Collaborators', - 'Extensions Left', - 'Usage Days', - 'Start', - 'End' - ] - ) - - for booking in Booking.objects.filter(start__gte=start, start__lte=end): - filtered = False - booking_filter = [279] - user_filter = ["ParkerBerberian", "ssmith", "ahassick", "sbergeron", "jhodgdon", "rhodgdon", "aburch", "jspewock"] - user = booking.owner.username if booking.owner.username is not None else "None" - - for b in booking_filter: - if b == booking.id: - filtered = True - - for u in user_filter: - if u == user: - filtered = True - # trims time delta to the the specified year(s) if between years - usage_days = ((end if booking.end > end else booking.end) - (start if booking.start < start else booking.start)).days - collaborators = [] - - for c in booking.collaborators.all(): - collaborators.append(c.username) - - if (not filtered): - cumulative_data.account(booking, usage_days) - file_writer.writerow([ - str(booking.id), - str(booking.project), - str(booking.purpose), - str(booking.owner.username), - ','.join(collaborators), - str(booking.ext_count), - str(usage_days), - str(booking.start), - str(booking.end) - ]) - cumulative_data.write_cumulative() - - -def map_cntt_interfaces(labid: str): - """ - Use this during cntt migrations, call it with a host labid and it will change profiles for this host - as well as mapping its interfaces across. interface ens1f2 should have the mac address of interface eno50 - as an invariant before calling this function - """ - host = get_host(labid, "unh_iol") - host.profile = ResourceProfile.objects.get(name="HPE x86 CNTT") - host.save() - host = get_host(labid, "unh_iol") - - for iface in host.interfaces.all(): - new_ifprofile = None - if iface.profile.name == "ens1f2": - new_ifprofile = InterfaceProfile.objects.get(host=host.profile, name="eno50") - else: - new_ifprofile = InterfaceProfile.objects.get(host=host.profile, name=iface.profile.name) - - iface.profile = new_ifprofile - - iface.save() - - -def detect_leaked_hosts(labid="unh_iol"): - """ - Use this to try to detect leaked hosts. - These hosts may still be in the process of unprovisioning, - but if they are not (or unprovisioning is frozen) then - these hosts are instead leaked - """ - working_servers = Server.objects.filter(working=True, lab__lab_user__username=labid) - booked = working_servers.filter(booked=True) - filtered = booked - print_div() - print("In use now:") - for booking in Booking.objects.filter(end__gte=timezone.now()): - res_for_booking = booking.resource.get_resources() - print(res_for_booking) - for resource in res_for_booking: - filtered = filtered.exclude(id=resource.id) - print_div() - print("Possibly leaked:") - for host in filtered: - print(host) - print_div() - return filtered - - -def booking_for_host(host_labid: str, lab_username="unh_iol"): - """ - Returns the booking that this server is a part of, if any. - Fails with an exception if no such booking exists - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username: param of the form `unh_iol` or similar - """ - server = Server.objects.get(lab__lab_user__username=lab_username, labid=host_labid) - booking = server.bundle.booking_set.first() - print_div() - print(booking) - print("id:", booking.id) - print("owner:", booking.owner) - print("job (id):", booking.job, "(" + str(booking.job.id) + ")") - print_div() - return booking - - -def force_release_booking(booking_id: int): - """ - Takes a booking id and forces the booking to end whether or not the tasks have - completed normally. - - Use with caution! Hosts may or may not be released depending on other underlying issues - - @booking_id: the id of the Booking object to be released - """ - booking = Booking.objects.get(id=booking_id) - job = booking.job - tasks = job.get_tasklist() - for task in tasks: - task.status = JobStatus.DONE - task.save() - - -def free_leaked_public_vlans(safety_buffer_days=2): - for lab in Lab.objects.all(): - current_booking_set = Booking.objects.filter(end__gte=timezone.now() + timedelta(days=safety_buffer_days)) - - marked_nets = set() - - for booking in current_booking_set: - for network in get_network_metadata(booking.id): - marked_nets.add(network["vlan_id"]) - - for net in PublicNetwork.objects.filter(lab=lab).filter(in_use=True): - if net.vlan not in marked_nets: - lab.vlan_manager.release_public_vlan(net.vlan) - - -def get_network_metadata(booking_id: int): - """ - Takes a booking id and prints all (known) networks that are owned by it. - Returns an object of the form {: {"vlan_id": int, "netname": str , "public": bool BEFORE THIS - @filenames: array of host import file names to import - """ - - for filename in filenames: - - # open import file - file = open("dashboard/" + filename + "-import.yaml", "r") - data = yaml.safe_load(file) - - # if a new profile is needed create one and a matching template - if (data["new_profile"]): - add_profile(data) - print("Profile: " + data["name"] + " created!") - make_default_template( - ResourceProfile.objects.get(name=data["name"]), - Image.objects.get(lab_id=data["image"]).id, - None, - None, - False, - False, - data["owner"], - "unh_iol", - True, - False, - data["temp_desc"] - ) - - print(" Template: " + data["temp_name"] + " created!") - - # add the server - add_server( - ResourceProfile.objects.get(name=data["name"]), - data["hostname"], - data["interfaces"], - data["lab"], - data["vendor"], - data["model"] - ) - - print(data["hostname"] + " imported!") - - -def convert_inspect_results(files): - """ - Converts an array of inspection result files into templates (filename-import.yaml) to be filled out for importing the servers into the dashboard - @files an array of file names (not including the file type. i.e hpe44). Default: [] - """ - for filename in files: - # open host inspect file - file = open("dashboard/" + filename + ".yaml") - output = open("dashboard/" + filename + "-import.yaml", "w") - data = json.load(file) - - # gather data about disks - disk_data = {} - for i in data["disk"]: - - # don't include loops in disks - if "loop" not in i: - disk_data[i["name"]] = { - "capacity": i["size"][:-3], - "media_type": "<\"SSD\" or \"HDD\">", - "interface": "<\"sata\", \"sas\", \"ssd\", \"nvme\", \"scsi\", or \"iscsi\">", - } - - # gather interface data - interface_data = {} - for i in data["interfaces"]: - interface_data[data["interfaces"][i]["name"]] = { - "speed": data["interfaces"][i]["speed"], - "nic_type": "<\"onboard\" or \"pcie\">", - "order": "", - "mac_address": data["interfaces"][i]["mac"], - "bus_addr": data["interfaces"][i]["busaddr"], - } - - # gather cpu data - cpu_data = { - "cores": data["cpu"]["cores"], - "architecture": data["cpu"]["arch"], - "cpus": data["cpu"]["cpus"], - "cflags": "", - } - - # gather ram data - ram_data = { - "amount": data["memory"][:-1], - "channels": "", - } - - # assemble data for host import file - import_data = { - "new_profile": " (Set to True to create a new profile for the host's type)", - "name": " (Used to set the profile of a host and for creating a new profile)", - "description": "", - "labs": "", - "temp_name": "