diff options
Diffstat (limited to 'src')
58 files changed, 1790 insertions, 544 deletions
diff --git a/src/account/jira_util.py b/src/account/jira_util.py deleted file mode 100644 index a522594..0000000 --- a/src/account/jira_util.py +++ /dev/null @@ -1,65 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -import base64 -import os - -import oauth2 as oauth -from django.conf import settings -from jira import JIRA -from tlslite.utils import keyfactory - - -class SignatureMethod_RSA_SHA1(oauth.SignatureMethod): - name = 'RSA-SHA1' - - def signing_base(self, request, consumer, token): - if not hasattr(request, 'normalized_url') or request.normalized_url is None: - raise ValueError("Base URL for request is not set.") - - sig = ( - oauth.escape(request.method), - oauth.escape(request.normalized_url), - oauth.escape(request.get_normalized_parameters()), - ) - - key = '%s&' % oauth.escape(consumer.secret) - if token: - key += oauth.escape(token.secret) - raw = '&'.join(sig) - return key, raw - - def sign(self, request, consumer, token): - """Build the base signature string.""" - key, raw = self.signing_base(request, consumer, token) - - module_dir = os.path.dirname(__file__) # get current directory - with open(module_dir + '/rsa.pem', 'r') as f: - data = f.read() - privateKeyString = data.strip() - privatekey = keyfactory.parsePrivateKey(privateKeyString) - raw = str.encode(raw) - signature = privatekey.hashAndSign(raw) - return base64.b64encode(signature) - - -def get_jira(user): - module_dir = os.path.dirname(__file__) # get current directory - with open(module_dir + '/rsa.pem', 'r') as f: - key_cert = f.read() - - oauth_dict = { - 'access_token': user.userprofile.oauth_token, - 'access_token_secret': user.userprofile.oauth_secret, - 'consumer_key': settings.OAUTH_CONSUMER_KEY, - 'key_cert': key_cert - } - - return JIRA(server=settings.JIRA_URL, oauth=oauth_dict) diff --git a/src/account/models.py b/src/account/models.py index b71f0ac..32229b1 100644 --- a/src/account/models.py +++ b/src/account/models.py @@ -51,6 +51,7 @@ class UserProfile(models.Model): oauth_secret = models.CharField(max_length=1024, blank=False) jira_url = models.CharField(max_length=100, null=True, blank=True, default='') + full_name = models.CharField(max_length=100, null=True, blank=True, default='') booking_privledge = models.BooleanField(default=False) @@ -82,12 +83,14 @@ class VlanManager(models.Model): # if they use QinQ or a vxlan overlay, for example allow_overlapping = models.BooleanField() - def get_vlans(self, count=1): + def get_vlans(self, count=1, within=None): """ Return the IDs of available vlans as a list[int], but does not reserve them. Will throw index exception if not enough vlans are available. Always returns a list of ints + + If `within` is not none, will filter against that as a set, requiring that any vlans returned are within that set """ allocated = [] vlans = json.loads(self.vlans) @@ -104,17 +107,28 @@ class VlanManager(models.Model): continue # vlan is available and not reserved, so safe to add - allocated.append(i) + if within is not None: + if i in within: + allocated.append(i) + else: + allocated.append(i) continue if len(allocated) != count: - raise ResourceAvailabilityException("can't allocate the vlans requested") + raise ResourceAvailabilityException("There were not enough available private vlans for the allocation. Please contact the administrators.") return allocated - def get_public_vlan(self): + def get_public_vlan(self, within=None): """Return reference to an available public network without reserving it.""" - return PublicNetwork.objects.filter(lab=self.lab_set.first(), in_use=False).first() + r = PublicNetwork.objects.filter(lab=self.lab_set.first(), in_use=False) + if within is not None: + r = r.filter(vlan__in=within) + + if r.count() < 1: + raise ResourceAvailabilityException("There were not enough available public vlans for the allocation. Please contact the administrators.") + + return r.first() def reserve_public_vlan(self, vlan): """Reserves the Public Network that has the given vlan.""" diff --git a/src/account/tasks.py b/src/account/tasks.py deleted file mode 100644 index df98c73..0000000 --- a/src/account/tasks.py +++ /dev/null @@ -1,37 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from celery import shared_task -from django.contrib.auth.models import User -from jira import JIRAError - -from account.jira_util import get_jira - - -@shared_task -def sync_jira_accounts(): - users = User.objects.all() - for user in users: - jira = get_jira(user) - try: - user_dict = jira.myself() - except JIRAError: - # User can be anonymous (local django admin account) - continue - try: - user.email = user_dict['emailAddress'] - except KeyError: - pass - user.userprofile.url = user_dict['self'] - user.userprofile.full_name = user_dict['displayName'] - - user.userprofile.save() - user.save() diff --git a/src/account/urls.py b/src/account/urls.py index 97d8c77..6d4ef2f 100644 --- a/src/account/urls.py +++ b/src/account/urls.py @@ -30,46 +30,30 @@ from django.urls import path from account.views import ( AccountSettingsView, - JiraAuthenticatedView, - JiraLoginView, OIDCLoginView, - JiraLogoutView, + LogoutView, UserListView, account_resource_view, account_booking_view, account_images_view, - account_configuration_view, account_detail_view, - resource_delete_view, + template_delete_view, booking_cancel_view, image_delete_view, - configuration_delete_view ) -from laas_dashboard import settings +app_name = 'account' - -def get_login_view(): - if (settings.AUTH_SETTING == 'LFID'): - return OIDCLoginView.as_view() - else: - return JiraLoginView.as_view() - - -app_name = "account" urlpatterns = [ url(r'^settings/', AccountSettingsView.as_view(), name='settings'), - url(r'^authenticated/$', JiraAuthenticatedView.as_view(), name='authenticated'), - url(r'^login/$', get_login_view(), name='login'), - url(r'^logout/$', JiraLogoutView.as_view(), name='logout'), + url(r'^login/$', OIDCLoginView.as_view(), name='login'), + url(r'^logout/$', LogoutView.as_view(), name='logout'), url(r'^users/$', UserListView.as_view(), name='users'), - url(r'^my/resources/$', account_resource_view, name="my-resources"), - path('my/resources/delete/<int:resource_id>', resource_delete_view), - url(r'^my/bookings/$', account_booking_view, name="my-bookings"), + url(r'^my/resources/$', account_resource_view, name='my-resources'), + path('my/resources/delete/<int:resource_id>', template_delete_view), + url(r'^my/bookings/$', account_booking_view, name='my-bookings'), path('my/bookings/cancel/<int:booking_id>', booking_cancel_view), - url(r'^my/images/$', account_images_view, name="my-images"), + url(r'^my/images/$', account_images_view, name='my-images'), path('my/images/delete/<int:image_id>', image_delete_view), - url(r'^my/configurations/$', account_configuration_view, name="my-configurations"), - path('my/configurations/delete/<int:config_id>', configuration_delete_view), - url(r'^my/$', account_detail_view, name="my-account"), + url(r'^my/$', account_detail_view, name='my-account'), ] diff --git a/src/account/views.py b/src/account/views.py index b74126e..8976ff9 100644 --- a/src/account/views.py +++ b/src/account/views.py @@ -10,13 +10,10 @@ import os -import urllib -import oauth2 as oauth -from django.conf import settings from django.utils import timezone from django.contrib import messages -from django.contrib.auth import logout, authenticate, login +from django.contrib.auth import logout from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin from django.contrib.auth.models import User @@ -26,13 +23,11 @@ from django.shortcuts import get_object_or_404 from django.utils.decorators import method_decorator from django.views.generic import RedirectView, TemplateView, UpdateView from django.shortcuts import render -from jira import JIRA from rest_framework.authtoken.models import Token from mozilla_django_oidc.auth import OIDCAuthenticationBackend from account.forms import AccountSettingsForm -from account.jira_util import SignatureMethod_RSA_SHA1 from account.models import UserProfile from booking.models import Booking from resource_inventory.models import ResourceTemplate, Image @@ -69,7 +64,7 @@ class MyOIDCAB(OIDCAuthenticationBackend): If this changes we will need to match users based on some other criterea. """ - username = claims.get(os.environ['CLAIMS_ENDPOINT'] + 'username') + username = claims.get(os.environ.get('CLAIMS_ENDPOINT') + 'username') if not username: return HttpResponse('No username provided, contact support.') @@ -101,109 +96,17 @@ class MyOIDCAB(OIDCAuthenticationBackend): return user -class JiraLoginView(RedirectView): - def get_redirect_url(self, *args, **kwargs): - consumer = oauth.Consumer(settings.OAUTH_CONSUMER_KEY, settings.OAUTH_CONSUMER_SECRET) - client = oauth.Client(consumer) - client.set_signature_method(SignatureMethod_RSA_SHA1()) - - # Step 1. Get a request token from Jira. - try: - resp, content = client.request(settings.OAUTH_REQUEST_TOKEN_URL, "POST") - except Exception: - messages.add_message(self.request, messages.ERROR, - 'Error: Connection to Jira failed. Please contact an Administrator') - return '/' - if resp['status'] != '200': - messages.add_message(self.request, messages.ERROR, - 'Error: Connection to Jira failed. Please contact an Administrator') - return '/' - - # Step 2. Store the request token in a session for later use. - self.request.session['request_token'] = dict(urllib.parse.parse_qsl(content.decode())) - # Step 3. Redirect the user to the authentication URL. - url = settings.OAUTH_AUTHORIZE_URL + '?oauth_token=' + \ - self.request.session['request_token']['oauth_token'] + \ - '&oauth_callback=' + settings.OAUTH_CALLBACK_URL - return url - - class OIDCLoginView(RedirectView): def get_redirect_url(self, *args, **kwargs): return reverse('oidc_authentication_init') -class JiraLogoutView(LoginRequiredMixin, RedirectView): +class LogoutView(LoginRequiredMixin, RedirectView): def get_redirect_url(self, *args, **kwargs): logout(self.request) return '/' -class JiraAuthenticatedView(RedirectView): - def get_redirect_url(self, *args, **kwargs): - # Step 1. Use the request token in the session to build a new client. - consumer = oauth.Consumer(settings.OAUTH_CONSUMER_KEY, settings.OAUTH_CONSUMER_SECRET) - token = oauth.Token(self.request.session['request_token']['oauth_token'], - self.request.session['request_token']['oauth_token_secret']) - client = oauth.Client(consumer, token) - client.set_signature_method(SignatureMethod_RSA_SHA1()) - - # Step 2. Request the authorized access token from Jira. - try: - resp, content = client.request(settings.OAUTH_ACCESS_TOKEN_URL, "POST") - except Exception: - messages.add_message(self.request, messages.ERROR, - 'Error: Connection to Jira failed. Please contact an Administrator') - return '/' - if resp['status'] != '200': - messages.add_message(self.request, messages.ERROR, - 'Error: Connection to Jira failed. Please contact an Administrator') - return '/' - - access_token = dict(urllib.parse.parse_qsl(content.decode())) - - module_dir = os.path.dirname(__file__) # get current directory - with open(module_dir + '/rsa.pem', 'r') as f: - key_cert = f.read() - - oauth_dict = { - 'access_token': access_token['oauth_token'], - 'access_token_secret': access_token['oauth_token_secret'], - 'consumer_key': settings.OAUTH_CONSUMER_KEY, - 'key_cert': key_cert - } - - jira = JIRA(server=settings.JIRA_URL, oauth=oauth_dict) - username = jira.current_user() - email = "" - try: - email = jira.user(username).emailAddress - except AttributeError: - email = "" - url = '/' - # Step 3. Lookup the user or create them if they don't exist. - try: - user = User.objects.get(username=username) - except User.DoesNotExist: - # Save our permanent token and secret for later. - user = User.objects.create_user(username=username, - password=access_token['oauth_token_secret']) - profile = UserProfile() - profile.user = user - profile.save() - user.userprofile.email_addr = email - url = reverse('account:settings') - user.userprofile.oauth_token = access_token['oauth_token'] - user.userprofile.oauth_secret = access_token['oauth_token_secret'] - user.userprofile.save() - user.set_password(access_token['oauth_token_secret']) - user.save() - user = authenticate(username=username, password=access_token['oauth_token_secret']) - login(self.request, user) - # redirect user to settings page to complete profile - return url - - @method_decorator(login_required, name='dispatch') class UserListView(TemplateView): template_name = "account/user_list.html" @@ -232,9 +135,9 @@ def account_resource_view(request): template = "account/resource_list.html" active_bundles = [book.resource for book in Booking.objects.filter( - owner=request.user, end__gte=timezone.now())] + owner=request.user, end__gte=timezone.now(), resource__template__temporary=False)] active_resources = [bundle.template.id for bundle in active_bundles] - resource_list = list(ResourceTemplate.objects.filter(owner=request.user)) + resource_list = list(ResourceTemplate.objects.filter(owner=request.user, temporary=False)) context = { "resources": resource_list, @@ -262,15 +165,6 @@ def account_booking_view(request): return render(request, template, context=context) -def account_configuration_view(request): - if not request.user.is_authenticated: - return render(request, "dashboard/login.html", {'title': 'Authentication Required'}) - template = "account/configuration_list.html" - configs = list(ResourceTemplate.objects.filter(owner=request.user)) - context = {"title": "Configuration List", "configurations": configs} - return render(request, template, context=context) - - def account_images_view(request): if not request.user.is_authenticated: return render(request, "dashboard/login.html", {'title': 'Authentication Required'}) @@ -290,28 +184,18 @@ def account_images_view(request): return render(request, template, context=context) -def resource_delete_view(request, resource_id=None): - if not request.user.is_authenticated: - return HttpResponse('no') # 403? - grb = get_object_or_404(ResourceTemplate, pk=resource_id) - if not request.user.id == grb.owner.id: - return HttpResponse('no') # 403? - if Booking.objects.filter(resource__template=grb, end__gt=timezone.now()).exists(): - return HttpResponse('no') # 403? - grb.delete() - return HttpResponse('') - - -def configuration_delete_view(request, config_id=None): +def template_delete_view(request, resource_id=None): if not request.user.is_authenticated: - return HttpResponse('no') # 403? - config = get_object_or_404(ResourceTemplate, pk=config_id) - if not request.user.id == config.owner.id: - return HttpResponse('no') # 403? - if Booking.objects.filter(resource__template=config, end__gt=timezone.now()).exists(): - return HttpResponse('no') - config.delete() - return HttpResponse('') + return HttpResponse(status=403) + template = get_object_or_404(ResourceTemplate, pk=resource_id) + if not request.user.id == template.owner.id: + return HttpResponse(status=403) + if Booking.objects.filter(resource__template=template, end__gt=timezone.now()).exists(): + return HttpResponse(status=403) + template.public = False + template.temporary = True + template.save() + return HttpResponse(status=200) def booking_cancel_view(request, booking_id=None): diff --git a/src/api/admin.py b/src/api/admin.py index 8b2fcb3..1e243a0 100644 --- a/src/api/admin.py +++ b/src/api/admin.py @@ -22,6 +22,7 @@ from api.models import ( SoftwareRelation, HostHardwareRelation, HostNetworkRelation, + APILog ) @@ -39,3 +40,4 @@ admin.site.register(AccessRelation) admin.site.register(SoftwareRelation) admin.site.register(HostHardwareRelation) admin.site.register(HostNetworkRelation) +admin.site.register(APILog) diff --git a/src/api/migrations/0017_apilog.py b/src/api/migrations/0017_apilog.py new file mode 100644 index 0000000..d209aef --- /dev/null +++ b/src/api/migrations/0017_apilog.py @@ -0,0 +1,27 @@ +# Generated by Django 2.2 on 2021-03-19 20:45 + +from django.conf import settings +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('api', '0016_auto_20201109_2149'), + ] + + operations = [ + migrations.CreateModel( + name='APILog', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('call_time', models.DateTimeField(auto_now=True)), + ('endpoint', models.CharField(max_length=300)), + ('body', django.contrib.postgres.fields.jsonb.JSONField()), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/src/api/migrations/0017_auto_20210630_1629.py b/src/api/migrations/0017_auto_20210630_1629.py new file mode 100644 index 0000000..643ff5f --- /dev/null +++ b/src/api/migrations/0017_auto_20210630_1629.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2021-06-30 16:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0016_auto_20201109_2149'), + ] + + operations = [ + migrations.AlterField( + model_name='snapshotconfig', + name='image', + field=models.CharField(max_length=200, null=True), + ), + ] diff --git a/src/api/migrations/0018_apilog_ip_addr.py b/src/api/migrations/0018_apilog_ip_addr.py new file mode 100644 index 0000000..4b7ce39 --- /dev/null +++ b/src/api/migrations/0018_apilog_ip_addr.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2021-03-22 18:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0017_apilog'), + ] + + operations = [ + migrations.AddField( + model_name='apilog', + name='ip_addr', + field=models.GenericIPAddressField(null=True), + ), + ] diff --git a/src/api/migrations/0018_cloudinitfile.py b/src/api/migrations/0018_cloudinitfile.py new file mode 100644 index 0000000..4e41b39 --- /dev/null +++ b/src/api/migrations/0018_cloudinitfile.py @@ -0,0 +1,25 @@ +# Generated by Django 2.2 on 2021-07-01 20:45 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0019_auto_20210701_1947'), + ('booking', '0008_auto_20201109_1947'), + ('api', '0017_auto_20210630_1629'), + ] + + operations = [ + migrations.CreateModel( + name='CloudInitFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('resource_id', models.CharField(max_length=200)), + ('booking', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='booking.Booking')), + ('rconfig', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='resource_inventory.ResourceConfiguration')), + ], + ), + ] diff --git a/src/api/migrations/0019_auto_20210322_1823.py b/src/api/migrations/0019_auto_20210322_1823.py new file mode 100644 index 0000000..b3c4cdf --- /dev/null +++ b/src/api/migrations/0019_auto_20210322_1823.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2 on 2021-03-22 18:23 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0018_apilog_ip_addr'), + ] + + operations = [ + migrations.AlterField( + model_name='apilog', + name='body', + field=django.contrib.postgres.fields.jsonb.JSONField(null=True), + ), + ] diff --git a/src/api/migrations/0019_auto_20210907_1448.py b/src/api/migrations/0019_auto_20210907_1448.py new file mode 100644 index 0000000..92140fb --- /dev/null +++ b/src/api/migrations/0019_auto_20210907_1448.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2 on 2021-09-07 14:48 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('booking', '0008_auto_20201109_1947'), + ('resource_inventory', '0020_cloudinitfile'), + ('api', '0018_cloudinitfile'), + ] + + operations = [ + migrations.CreateModel( + name='GeneratedCloudConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('resource_id', models.CharField(max_length=200)), + ('text', models.TextField(blank=True, null=True)), + ('booking', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='booking.Booking')), + ('rconfig', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='resource_inventory.ResourceConfiguration')), + ], + ), + migrations.DeleteModel( + name='CloudInitFile', + ), + ] diff --git a/src/api/migrations/0020_auto_20210322_2218.py b/src/api/migrations/0020_auto_20210322_2218.py new file mode 100644 index 0000000..0252c79 --- /dev/null +++ b/src/api/migrations/0020_auto_20210322_2218.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2 on 2021-03-22 22:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0019_auto_20210322_1823'), + ] + + operations = [ + migrations.AddField( + model_name='apilog', + name='method', + field=models.CharField(max_length=4, null=True), + ), + migrations.AlterField( + model_name='apilog', + name='endpoint', + field=models.CharField(max_length=300, null=True), + ), + ] diff --git a/src/api/migrations/0021_auto_20210405_1943.py b/src/api/migrations/0021_auto_20210405_1943.py new file mode 100644 index 0000000..ca6e741 --- /dev/null +++ b/src/api/migrations/0021_auto_20210405_1943.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2021-04-05 19:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0020_auto_20210322_2218'), + ] + + operations = [ + migrations.AlterField( + model_name='apilog', + name='method', + field=models.CharField(max_length=6, null=True), + ), + ] diff --git a/src/api/migrations/0022_merge_20211102_2136.py b/src/api/migrations/0022_merge_20211102_2136.py new file mode 100644 index 0000000..bb27ae4 --- /dev/null +++ b/src/api/migrations/0022_merge_20211102_2136.py @@ -0,0 +1,14 @@ +# Generated by Django 2.2 on 2021-11-02 21:36 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0019_auto_20210907_1448'), + ('api', '0021_auto_20210405_1943'), + ] + + operations = [ + ] diff --git a/src/api/migrations/0023_add_cifile_generated_field.py b/src/api/migrations/0023_add_cifile_generated_field.py new file mode 100644 index 0000000..df2b6d7 --- /dev/null +++ b/src/api/migrations/0023_add_cifile_generated_field.py @@ -0,0 +1,14 @@ +from django.db import migrations, models + +class Migration(migrations.Migration): + dependencies = [ + ('api', '0022_merge_20211102_2136'), + ] + + operations = [ + migrations.AddField( + model_name="CloudInitFile", + name="generated", + field=models.BooleanField(default=False) + ), + ] diff --git a/src/api/models.py b/src/api/models.py index d1bb692..93168f5 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -12,24 +12,30 @@ from django.contrib.auth.models import User from django.db import models from django.core.exceptions import PermissionDenied, ValidationError from django.shortcuts import get_object_or_404 +from django.contrib.postgres.fields import JSONField from django.http import HttpResponseNotFound from django.urls import reverse from django.utils import timezone import json import uuid +import yaml +import re from booking.models import Booking from resource_inventory.models import ( Lab, ResourceProfile, Image, + Opsys, Interface, ResourceOPNFVConfig, RemoteInfo, OPNFVConfig, ConfigState, - ResourceQuery + ResourceQuery, + ResourceConfiguration, + CloudInitFile ) from resource_inventory.idf_templater import IDFTemplater from resource_inventory.pdf_templater import PDFTemplater @@ -37,7 +43,7 @@ from account.models import Downtime, UserProfile from dashboard.utils import AbstractModelQuery -class JobStatus(object): +class JobStatus: """ A poor man's enum for a job's status. @@ -52,7 +58,7 @@ class JobStatus(object): ERROR = 300 -class LabManagerTracker(object): +class LabManagerTracker: @classmethod def get(cls, lab_name, token): @@ -72,7 +78,7 @@ class LabManagerTracker(object): raise PermissionDenied("Lab not authorized") -class LabManager(object): +class LabManager: """ Handles all lab REST calls. @@ -83,6 +89,18 @@ class LabManager(object): def __init__(self, lab): self.lab = lab + def get_opsyss(self): + return Opsys.objects.filter(from_lab=self.lab) + + def get_images(self): + return Image.objects.filter(from_lab=self.lab) + + def get_image(self, image_id): + return Image.objects.filter(from_lab=self.lab, lab_id=image_id) + + def get_opsys(self, opsys_id): + return Opsys.objects.filter(from_lab=self.lab, lab_id=opsys_id) + def get_downtime(self): return Downtime.objects.filter(start__lt=timezone.now(), end__gt=timezone.now(), lab=self.lab) @@ -337,6 +355,248 @@ class LabManager(object): return profile_ser +class GeneratedCloudConfig(models.Model): + resource_id = models.CharField(max_length=200) + booking = models.ForeignKey(Booking, on_delete=models.CASCADE) + rconfig = models.ForeignKey(ResourceConfiguration, on_delete=models.CASCADE) + text = models.TextField(null=True, blank=True) + + def _normalize_username(self, username: str) -> str: + # TODO: make usernames posix compliant + s = re.sub(r'\W+', '', username) + return s + + def _get_ssh_string(self, username: str) -> str: + user = User.objects.get(username=username) + uprofile = user.userprofile + + ssh_file = uprofile.ssh_public_key + + escaped_file = ssh_file.open().read().decode(encoding="UTF-8").replace("\n", " ") + + return escaped_file + + def _serialize_users(self): + """ + returns the dictionary to be placed behind the `users` field of the toplevel c-i dict + """ + # conserves distro default user + user_array = ["default"] + + users = list(self.booking.collaborators.all()) + users.append(self.booking.owner) + for collaborator in users: + userdict = {} + + # TODO: validate if usernames are valid as linux usernames (and provide an override potentially) + userdict['name'] = self._normalize_username(collaborator.username) + + userdict['groups'] = "sudo" + userdict['sudo'] = "ALL=(ALL) NOPASSWD:ALL" + + userdict['ssh_authorized_keys'] = [self._get_ssh_string(collaborator.username)] + + user_array.append(userdict) + + # user_array.append({ + # "name": "opnfv", + # "passwd": "$6$k54L.vim1cLaEc4$5AyUIrufGlbtVBzuCWOlA1yV6QdD7Gr2MzwIs/WhuYR9ebSfh3Qlb7djkqzjwjxpnSAonK1YOabPP6NxUDccu.", + # "ssh_redirect_user": True, + # "sudo": "ALL=(ALL) NOPASSWD:ALL", + # "groups": "sudo", + # }) + + return user_array + + # TODO: make this configurable + def _serialize_sysinfo(self): + defuser = {} + defuser['name'] = 'opnfv' + defuser['plain_text_passwd'] = 'OPNFV_HOST' + defuser['home'] = '/home/opnfv' + defuser['shell'] = '/bin/bash' + defuser['lock_passwd'] = True + defuser['gecos'] = 'Lab Manager User' + defuser['groups'] = 'sudo' + + return {'default_user': defuser} + + # TODO: make this configurable + def _serialize_runcmds(self): + cmdlist = [] + + # have hosts run dhcp on boot + cmdlist.append(['sudo', 'dhclient', '-r']) + cmdlist.append(['sudo', 'dhclient']) + + return cmdlist + + def _serialize_netconf_v1(self): + # interfaces = {} # map from iface_name => dhcp_config + # vlans = {} # map from vlan_id => dhcp_config + + config_arr = [] + + for interface in self._resource().interfaces.all(): + interface_name = interface.profile.name + interface_mac = interface.mac_address + + iface_dict_entry = { + "type": "physical", + "name": interface_name, + "mac_address": interface_mac, + } + + for vlan in interface.config.all(): + if vlan.tagged: + vlan_dict_entry = {'type': 'vlan'} + vlan_dict_entry['name'] = str(interface_name) + "." + str(vlan.vlan_id) + vlan_dict_entry['vlan_link'] = str(interface_name) + vlan_dict_entry['vlan_id'] = int(vlan.vlan_id) + vlan_dict_entry['mac_address'] = str(interface_mac) + if vlan.public: + vlan_dict_entry["subnets"] = [{"type": "dhcp"}] + config_arr.append(vlan_dict_entry) + if (not vlan.tagged) and vlan.public: + iface_dict_entry["subnets"] = [{"type": "dhcp"}] + + # vlan_dict_entry['mtu'] = # TODO, determine override MTU if needed + + config_arr.append(iface_dict_entry) + + ns_dict = { + 'type': 'nameserver', + 'address': ['10.64.0.1', '8.8.8.8'] + } + + config_arr.append(ns_dict) + + full_dict = {'version': 1, 'config': config_arr} + + return full_dict + + @classmethod + def get(cls, booking_id: int, resource_lab_id: str, file_id: int): + return GeneratedCloudConfig.objects.get(resource_id=resource_lab_id, booking__id=booking_id, file_id=file_id) + + def _resource(self): + return ResourceQuery.get(labid=self.resource_id, lab=self.booking.lab) + + # def _get_facts(self): + # resource = self._resource() + + # hostname = self.rconfig.name + # iface_configs = for_config.interface_configs.all() + + def _to_dict(self): + main_dict = {} + + main_dict['users'] = self._serialize_users() + main_dict['network'] = self._serialize_netconf_v1() + main_dict['hostname'] = self.rconfig.name + + # add first startup commands + main_dict['runcmd'] = self._serialize_runcmds() + + # configure distro default user + main_dict['system_info'] = self._serialize_sysinfo() + + return main_dict + + def serialize(self) -> str: + return yaml.dump(self._to_dict(), width=float("inf")) + + +class APILog(models.Model): + user = models.ForeignKey(User, on_delete=models.PROTECT) + call_time = models.DateTimeField(auto_now=True) + method = models.CharField(null=True, max_length=6) + endpoint = models.CharField(null=True, max_length=300) + ip_addr = models.GenericIPAddressField(protocol="both", null=True, unpack_ipv4=False) + body = JSONField(null=True) + + def __str__(self): + return "Call to {} at {} by {}".format( + self.endpoint, + self.call_time, + self.user.username + ) + + +class AutomationAPIManager: + @staticmethod + def serialize_booking(booking): + sbook = {} + sbook['id'] = booking.pk + sbook['owner'] = booking.owner.username + sbook['collaborators'] = [user.username for user in booking.collaborators.all()] + sbook['start'] = booking.start + sbook['end'] = booking.end + sbook['lab'] = AutomationAPIManager.serialize_lab(booking.lab) + sbook['purpose'] = booking.purpose + sbook['resourceBundle'] = AutomationAPIManager.serialize_bundle(booking.resource) + return sbook + + @staticmethod + def serialize_lab(lab): + slab = {} + slab['id'] = lab.pk + slab['name'] = lab.name + return slab + + @staticmethod + def serialize_bundle(bundle): + sbundle = {} + sbundle['id'] = bundle.pk + sbundle['resources'] = [ + AutomationAPIManager.serialize_server(server) + for server in bundle.get_resources()] + return sbundle + + @staticmethod + def serialize_server(server): + sserver = {} + sserver['id'] = server.pk + sserver['name'] = server.name + return sserver + + @staticmethod + def serialize_resource_profile(profile): + sprofile = {} + sprofile['id'] = profile.pk + sprofile['name'] = profile.name + return sprofile + + @staticmethod + def serialize_template(rec_temp_and_count): + template = rec_temp_and_count[0] + count = rec_temp_and_count[1] + + stemplate = {} + stemplate['id'] = template.pk + stemplate['name'] = template.name + stemplate['count_available'] = count + stemplate['resourceProfiles'] = [ + AutomationAPIManager.serialize_resource_profile(config.profile) + for config in template.getConfigs() + ] + return stemplate + + @staticmethod + def serialize_image(image): + simage = {} + simage['id'] = image.pk + simage['name'] = image.name + return simage + + @staticmethod + def serialize_userprofile(up): + sup = {} + sup['id'] = up.pk + sup['username'] = up.user.username + return sup + + class Job(models.Model): """ A Job to be performed by the Lab. @@ -670,6 +930,7 @@ class HardwareConfig(TaskConfig): return self.get_delta() def get_delta(self): + # TODO: grab the GeneratedCloudConfig urls from self.hosthardwarerelation.get_resource() return self.format_delta( self.hosthardwarerelation.get_resource().get_configuration(self.state), self.hosthardwarerelation.lab_token) @@ -722,7 +983,7 @@ class NetworkConfig(TaskConfig): class SnapshotConfig(TaskConfig): resource_id = models.CharField(max_length=200, default="default_id") - image = models.IntegerField(null=True) + image = models.CharField(max_length=200, null=True) # cobbler ID dashboard_id = models.IntegerField() delta = models.TextField(default="{}") @@ -1013,6 +1274,10 @@ class JobFactory(object): booking=booking, job=job ) + cls.makeGeneratedCloudConfigs( + resources=resources, + job=job + ) all_users = list(booking.collaborators.all()) all_users.append(booking.owner) cls.makeAccessConfig( @@ -1037,6 +1302,18 @@ class JobFactory(object): continue @classmethod + def makeGeneratedCloudConfigs(cls, resources=[], job=Job()): + for res in resources: + cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=job.booking, rconfig=res.config) + cif.save() + + cif = CloudInitFile.create(priority=0, text=cif.serialize()) + cif.save() + + res.config.cloud_init_files.add(cif) + res.config.save() + + @classmethod def makeHardwareConfigs(cls, resources=[], job=Job()): """ Create and save HardwareConfig. diff --git a/src/api/urls.py b/src/api/urls.py index bae86ea..acef947 100644 --- a/src/api/urls.py +++ b/src/api/urls.py @@ -45,10 +45,30 @@ from api.views import ( lab_users, lab_user, GenerateTokenView, - analytics_job + analytics_job, + user_bookings, + specific_booking, + extend_booking, + make_booking, + list_labs, + all_users, + images_for_template, + available_templates, + resource_ci_metadata, + resource_ci_userdata, + resource_ci_userdata_directory, + all_images, + all_opsyss, + single_image, + single_opsys, + create_ci_file, ) urlpatterns = [ + path('labs/<slug:lab_name>/opsys/<slug:opsys_id>', single_opsys), + path('labs/<slug:lab_name>/image/<slug:image_id>', single_image), + path('labs/<slug:lab_name>/opsys', all_opsyss), + path('labs/<slug:lab_name>/image', all_images), path('labs/<slug:lab_name>/profile', lab_profile), path('labs/<slug:lab_name>/status', lab_status), path('labs/<slug:lab_name>/inventory', lab_inventory), @@ -59,11 +79,28 @@ urlpatterns = [ path('labs/<slug:lab_name>/booking/<int:booking_id>/idf', get_idf, name="get-idf"), path('labs/<slug:lab_name>/jobs/<int:job_id>', specific_job), path('labs/<slug:lab_name>/jobs/<int:job_id>/<slug:task_id>', specific_task), + path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>/user-data', resource_ci_userdata_directory, name="specific-user-data"), + path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>/meta-data', resource_ci_metadata, name="specific-meta-data"), + path('labs/<slug:lab_name>/jobs/<int:job_id>/cidata/<slug:resource_id>/<int:file_id>/user-data', resource_ci_userdata, name="user-data-dir"), path('labs/<slug:lab_name>/jobs/new', new_jobs), path('labs/<slug:lab_name>/jobs/current', current_jobs), path('labs/<slug:lab_name>/jobs/done', done_jobs), path('labs/<slug:lab_name>/jobs/getByType/DATA', analytics_job), path('labs/<slug:lab_name>/users', lab_users), path('labs/<slug:lab_name>/users/<int:user_id>', lab_user), + + path('booking', user_bookings), + path('booking/<int:booking_id>', specific_booking), + path('booking/<int:booking_id>/extendBooking/<int:days>', extend_booking), + path('booking/makeBooking', make_booking), + + path('resource_inventory/availableTemplates', available_templates), + path('resource_inventory/<int:template_id>/images', images_for_template), + + path('resource_inventory/cloud/create', create_ci_file), + + path('users', all_users), + path('labs', list_labs), + url(r'^token$', GenerateTokenView.as_view(), name='generate_token'), ] diff --git a/src/api/views.py b/src/api/views.py index 2e5f33f..1516374 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -8,27 +8,45 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## +import json +import math +import traceback +import sys +from datetime import timedelta from django.contrib.auth.decorators import login_required -from django.shortcuts import redirect +from django.shortcuts import redirect, get_object_or_404 from django.utils.decorators import method_decorator from django.utils import timezone from django.views import View +from django.http import HttpResponseNotFound from django.http.response import JsonResponse, HttpResponse from rest_framework import viewsets from rest_framework.authtoken.models import Token from django.views.decorators.csrf import csrf_exempt from django.core.exceptions import ObjectDoesNotExist +from django.db.models import Q from api.serializers.booking_serializer import BookingSerializer from api.serializers.old_serializers import UserSerializer from api.forms import DowntimeForm -from account.models import UserProfile +from account.models import UserProfile, Lab from booking.models import Booking -from api.models import LabManagerTracker, get_task +from booking.quick_deployer import create_from_API +from api.models import LabManagerTracker, get_task, Job, AutomationAPIManager, APILog from notifier.manager import NotificationHandler from analytics.models import ActiveVPNUser -import json +from resource_inventory.models import ( + Image, + Opsys, + CloudInitFile, + ResourceQuery, + ResourceTemplate, +) + +import yaml +import uuid +from deepmerge import Merger """ API views. @@ -80,6 +98,83 @@ def lab_host(request, lab_name="", host_id=""): if request.method == "POST": return JsonResponse(lab_manager.update_host(host_id, request.POST), safe=False) +# API extension for Cobbler integration + + +def all_images(request, lab_name=""): + a = [] + for i in Image.objects.all(): + a.append(i.serialize()) + return JsonResponse(a, safe=False) + + +def all_opsyss(request, lab_name=""): + a = [] + for opsys in Opsys.objects.all(): + a.append(opsys.serialize()) + + return JsonResponse(a, safe=False) + + +@csrf_exempt +def single_image(request, lab_name="", image_id=""): + lab_token = request.META.get('HTTP_AUTH_TOKEN') + lab_manager = LabManagerTracker.get(lab_name, lab_token) + img = lab_manager.get_image(image_id).first() + + if request.method == "GET": + if not img: + return HttpResponse(status=404) + return JsonResponse(img.serialize(), safe=False) + + if request.method == "POST": + # get POST data + data = json.loads(request.body.decode('utf-8')) + if img: + img.update(data) + else: + # append lab name and the ID from the URL + data['from_lab_id'] = lab_name + data['lab_id'] = image_id + + # create and save a new Image object + img = Image.new_from_data(data) + + img.save() + + # indicate success in response + return HttpResponse(status=200) + return HttpResponse(status=405) + + +@csrf_exempt +def single_opsys(request, lab_name="", opsys_id=""): + lab_token = request.META.get('HTTP_AUTH_TOKEN') + lab_manager = LabManagerTracker.get(lab_name, lab_token) + opsys = lab_manager.get_opsys(opsys_id).first() + + if request.method == "GET": + if not opsys: + return HttpResponse(status=404) + return JsonResponse(opsys.serialize(), safe=False) + + if request.method == "POST": + data = json.loads(request.body.decode('utf-8')) + if opsys: + opsys.update(data) + else: + # only name, available, and obsolete are needed to create an Opsys + # other fields are derived from the URL parameters + data['from_lab_id'] = lab_name + data['lab_id'] = opsys_id + opsys = Opsys.new_from_data(data) + + opsys.save() + return HttpResponse(status=200) + return HttpResponse(status=405) + +# end API extension + def get_pdf(request, lab_name="", booking_id=""): lab_token = request.META.get('HTTP_AUTH_TOKEN') @@ -167,6 +262,89 @@ def specific_job(request, lab_name="", job_id=""): return JsonResponse(lab_manager.get_job(job_id), safe=False) +@csrf_exempt +def resource_ci_userdata(request, lab_name="", job_id="", resource_id="", file_id=0): + # lab_token = request.META.get('HTTP_AUTH_TOKEN') + # lab_manager = LabManagerTracker.get(lab_name, lab_token) + + # job = lab_manager.get_job(job_id) + Job.objects.get(id=job_id) # verify a valid job was given, even if we don't use it + + cifile = None + try: + cifile = CloudInitFile.objects.get(id=file_id) + except ObjectDoesNotExist: + return HttpResponseNotFound("Could not find a matching resource by id " + str(resource_id)) + + text = cifile.text + + prepended_text = "#cloud-config\n" + # mstrat = CloudInitFile.merge_strategy() + # prepended_text = prepended_text + yaml.dump({"merge_strategy": mstrat}) + "\n" + # print("in cloudinitfile create") + text = prepended_text + text + cloud_dict = { + "datasource": { + "None": { + "metadata": { + "instance-id": str(uuid.uuid4()) + }, + "userdata_raw": text, + }, + }, + "datasource_list": ["None"], + } + + return HttpResponse(yaml.dump(cloud_dict, width=float("inf")), status=200) + + +@csrf_exempt +def resource_ci_metadata(request, lab_name="", job_id="", resource_id="", file_id=0): + return HttpResponse("#cloud-config", status=200) + + +@csrf_exempt +def resource_ci_userdata_directory(request, lab_name="", job_id="", resource_id=""): + # files = [{"id": file.file_id, "priority": file.priority} for file in CloudInitFile.objects.filter(job__id=job_id, resource_id=resource_id).order_by("priority").all()] + resource = ResourceQuery.get(labid=resource_id, lab=Lab.objects.get(name=lab_name)) + files = resource.config.cloud_init_files + files = [{"id": file.id, "priority": file.priority} for file in files.order_by("priority").all()] + + d = {} + + merge_failures = [] + + merger = Merger( + [ + (list, ["append"]), + (dict, ["merge"]), + ], + ["override"], # fallback + ["override"], # if types conflict (shouldn't happen in CI, but handle case) + ) + + for f in resource.config.cloud_init_files.order_by("priority").all(): + try: + other_dict = yaml.safe_load(f.text) + if not (type(d) is dict): + raise Exception("CI file was valid yaml but was not a dict") + + merger.merge(d, other_dict) + except Exception as e: + # if fail to merge, then just skip + print("Failed to merge file in, as it had invalid content:", f.id) + print("File text was:") + print(f.text) + merge_failures.append({f.id: str(e)}) + + if len(merge_failures) > 0: + d['merge_failures'] = merge_failures + + file = CloudInitFile.create(text=yaml.dump(d, width=float("inf")), priority=0) + + return HttpResponse(json.dumps([{"id": file.id, "priority": file.priority}]), status=200) + + def new_jobs(request, lab_name=""): lab_token = request.META.get('HTTP_AUTH_TOKEN') lab_manager = LabManagerTracker.get(lab_name, lab_token) @@ -234,3 +412,241 @@ def done_jobs(request, lab_name=""): lab_token = request.META.get('HTTP_AUTH_TOKEN') lab_manager = LabManagerTracker.get(lab_name, lab_token) return JsonResponse(lab_manager.get_done_jobs(), safe=False) + + +def auth_and_log(request, endpoint): + """ + Function to authenticate an API user and log info + in the API log model. This is to keep record of + all calls to the dashboard + """ + user_token = request.META.get('HTTP_AUTH_TOKEN') + response = None + + if user_token is None: + return HttpResponse('Unauthorized', status=401) + + try: + token = Token.objects.get(key=user_token) + except Token.DoesNotExist: + token = None + response = HttpResponse('Unauthorized', status=401) + + x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') + if x_forwarded_for: + ip = x_forwarded_for.split(',')[0] + else: + ip = request.META.get('REMOTE_ADDR') + + body = None + + if request.method in ['POST', 'PUT']: + try: + body = json.loads(request.body.decode('utf-8')), + except Exception: + response = HttpResponse('Invalid Request Body', status=400) + + APILog.objects.create( + user=token.user, + call_time=timezone.now(), + method=request.method, + endpoint=endpoint, + body=body, + ip_addr=ip + ) + + if response: + return response + else: + return token + + +""" +Booking API Views +""" + + +def user_bookings(request): + token = auth_and_log(request, 'booking') + + if isinstance(token, HttpResponse): + return token + + bookings = Booking.objects.filter(owner=token.user, end__gte=timezone.now()) + output = [AutomationAPIManager.serialize_booking(booking) + for booking in bookings] + return JsonResponse(output, safe=False) + + +@csrf_exempt +def specific_booking(request, booking_id=""): + token = auth_and_log(request, 'booking/{}'.format(booking_id)) + + if isinstance(token, HttpResponse): + return token + + booking = get_object_or_404(Booking, pk=booking_id, owner=token.user) + if request.method == "GET": + sbooking = AutomationAPIManager.serialize_booking(booking) + return JsonResponse(sbooking, safe=False) + + if request.method == "DELETE": + + if booking.end < timezone.now(): + return HttpResponse("Booking already over", status=400) + + booking.end = timezone.now() + booking.save() + return HttpResponse("Booking successfully cancelled") + + +@csrf_exempt +def extend_booking(request, booking_id="", days=""): + token = auth_and_log(request, 'booking/{}/extendBooking/{}'.format(booking_id, days)) + + if isinstance(token, HttpResponse): + return token + + booking = get_object_or_404(Booking, pk=booking_id, owner=token.user) + + if booking.end < timezone.now(): + return HttpResponse("This booking is already over, cannot extend") + + if days > 30: + return HttpResponse("Cannot extend a booking longer than 30 days") + + if booking.ext_count == 0: + return HttpResponse("Booking has already been extended 2 times, cannot extend again") + + booking.end += timedelta(days=days) + booking.ext_count -= 1 + booking.save() + + return HttpResponse("Booking successfully extended") + + +@csrf_exempt +def make_booking(request): + token = auth_and_log(request, 'booking/makeBooking') + + if isinstance(token, HttpResponse): + return token + + try: + booking = create_from_API(request.body, token.user) + + except Exception: + finalTrace = '' + exc_type, exc_value, exc_traceback = sys.exc_info() + for i in traceback.format_exception(exc_type, exc_value, exc_traceback): + finalTrace += '<br>' + i.strip() + return HttpResponse(finalTrace, status=400) + + sbooking = AutomationAPIManager.serialize_booking(booking) + return JsonResponse(sbooking, safe=False) + + +""" +Resource Inventory API Views +""" + + +def available_templates(request): + token = auth_and_log(request, 'resource_inventory/availableTemplates') + + if isinstance(token, HttpResponse): + return token + + # get available templates + # mirrors MultipleSelectFilter Widget + avt = [] + for lab in Lab.objects.all(): + for template in ResourceTemplate.objects.filter(Q(owner=token.user) | Q(public=True), lab=lab, temporary=False): + available_resources = lab.get_available_resources() + required_resources = template.get_required_resources() + least_available = 100 + + for resource, count_required in required_resources.items(): + try: + curr_count = math.floor(available_resources[str(resource)] / count_required) + if curr_count < least_available: + least_available = curr_count + except KeyError: + least_available = 0 + + if least_available > 0: + avt.append((template, least_available)) + + savt = [AutomationAPIManager.serialize_template(temp) + for temp in avt] + + return JsonResponse(savt, safe=False) + + +def images_for_template(request, template_id=""): + _ = auth_and_log(request, 'resource_inventory/{}/images'.format(template_id)) + + template = get_object_or_404(ResourceTemplate, pk=template_id) + images = [AutomationAPIManager.serialize_image(config.image) + for config in template.getConfigs()] + return JsonResponse(images, safe=False) + + +""" +User API Views +""" + + +def all_users(request): + token = auth_and_log(request, 'users') + + if token is None: + return HttpResponse('Unauthorized', status=401) + + users = [AutomationAPIManager.serialize_userprofile(up) + for up in UserProfile.objects.filter(public_user=True)] + + return JsonResponse(users, safe=False) + + +def create_ci_file(request): + token = auth_and_log(request, 'booking/makeCloudConfig') + + if isinstance(token, HttpResponse): + return token + + try: + cconf = request.body + d = yaml.load(cconf) + if not (type(d) is dict): + raise Exception() + + cconf = CloudInitFile.create(text=cconf, priority=CloudInitFile.objects.count()) + + return JsonResponse({"id": cconf.id}) + except Exception: + return JsonResponse({"error": "Provided config file was not valid yaml or was not a dict at the top level"}) + + +""" +Lab API Views +""" + + +def list_labs(request): + lab_list = [] + for lab in Lab.objects.all(): + lab_info = { + 'name': lab.name, + 'username': lab.lab_user.username, + 'status': lab.status, + 'project': lab.project, + 'description': lab.description, + 'location': lab.location, + 'info': lab.lab_info_link, + 'email': lab.contact_email, + 'phone': lab.contact_phone + } + lab_list.append(lab_info) + + return JsonResponse(lab_list, safe=False) diff --git a/src/booking/forms.py b/src/booking/forms.py index cbc3407..ff829b2 100644 --- a/src/booking/forms.py +++ b/src/booking/forms.py @@ -22,6 +22,7 @@ class QuickBookingForm(forms.Form): purpose = forms.CharField(max_length=1000) project = forms.CharField(max_length=400) hostname = forms.CharField(required=False, max_length=400) + global_cloud_config = forms.CharField(widget=forms.Textarea, required=False) installer = forms.ModelChoiceField(queryset=Installer.objects.all(), required=False) scenario = forms.ModelChoiceField(queryset=Scenario.objects.all(), required=False) diff --git a/src/booking/lib.py b/src/booking/lib.py index 7a4c261..8c87979 100644 --- a/src/booking/lib.py +++ b/src/booking/lib.py @@ -28,9 +28,9 @@ def get_user_items(exclude=None): for up in qs: item = { 'id': up.id, - 'expanded_name': up.full_name, + 'expanded_name': up.full_name if up.full_name else up.user.username, 'small_name': up.user.username, - 'string': up.email_addr + 'string': up.email_addr if up.email_addr else up.user.username, } items[up.id] = item return items diff --git a/src/booking/migrations/0009_booking_complete.py b/src/booking/migrations/0009_booking_complete.py new file mode 100644 index 0000000..e291a83 --- /dev/null +++ b/src/booking/migrations/0009_booking_complete.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2021-09-07 15:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('booking', '0008_auto_20201109_1947'), + ] + + operations = [ + migrations.AddField( + model_name='booking', + name='complete', + field=models.BooleanField(default=False), + ), + ] diff --git a/src/booking/models.py b/src/booking/models.py index cfdf7bc..966f1c2 100644 --- a/src/booking/models.py +++ b/src/booking/models.py @@ -39,6 +39,8 @@ class Booking(models.Model): pdf = models.TextField(blank=True, default="") idf = models.TextField(blank=True, default="") + complete = models.BooleanField(default=False) + class Meta: db_table = 'booking' diff --git a/src/booking/quick_deployer.py b/src/booking/quick_deployer.py index 0a3bfc6..4b85d76 100644 --- a/src/booking/quick_deployer.py +++ b/src/booking/quick_deployer.py @@ -9,15 +9,16 @@ import json +import yaml from django.db.models import Q +from django.db import transaction from datetime import timedelta from django.utils import timezone from django.core.exceptions import ValidationError -from account.models import Lab +from account.models import Lab, UserProfile from resource_inventory.models import ( ResourceTemplate, - Installer, Image, OPNFVRole, OPNFVConfig, @@ -26,6 +27,7 @@ from resource_inventory.models import ( NetworkConnection, InterfaceConfiguration, Network, + CloudInitFile, ) from resource_inventory.resource_manager import ResourceManager from resource_inventory.pdf_templater import PDFTemplater @@ -60,7 +62,7 @@ def parse_resource_field(resource_json): return lab, template -def update_template(old_template, image, hostname, user): +def update_template(old_template, image, hostname, user, global_cloud_config=None): """ Duplicate a template to the users account and update configured fields. @@ -80,6 +82,8 @@ def update_template(old_template, image, hostname, user): description=old_template.description, public=False, temporary=True, + private_vlan_pool=old_template.private_vlan_pool, + public_vlan_pool=old_template.public_vlan_pool, copy_of=old_template ) @@ -112,9 +116,17 @@ def update_template(old_template, image, hostname, user): image=image_to_set, template=template, is_head_node=old_config.is_head_node, - name=hostname if len(old_template.getConfigs()) == 1 else old_config.name + name=hostname if len(old_template.getConfigs()) == 1 else old_config.name, + # cloud_init_files=old_config.cloud_init_files.set() ) + for file in old_config.cloud_init_files.all(): + config.cloud_init_files.add(file) + + if global_cloud_config: + config.cloud_init_files.add(global_cloud_config) + config.save() + for old_iface_config in old_config.interface_configs.all(): iface_config = InterfaceConfiguration.objects.create( profile=old_iface_config.profile, @@ -167,28 +179,19 @@ def generate_resource_bundle(template): return resource_bundle -def check_invariants(request, **kwargs): +def check_invariants(**kwargs): # TODO: This should really happen in the BookingForm validation methods - installer = kwargs['installer'] image = kwargs['image'] - scenario = kwargs['scenario'] lab = kwargs['lab'] length = kwargs['length'] # check that image os is compatible with installer if image: - if installer or scenario: - if installer in image.os.sup_installers.all(): - # if installer not here, we can omit that and not check for scenario - if not scenario: - raise ValidationError("An OPNFV Installer needs a scenario to be chosen to work properly") - if scenario not in installer.sup_scenarios.all(): - raise ValidationError("The chosen installer does not support the chosen scenario") if image.from_lab != lab: raise ValidationError("The chosen image is not available at the chosen hosting lab") # TODO # if image.host_type != host_profile: # raise ValidationError("The chosen image is not available for the chosen host type") - if not image.public and image.owner != request.user: + if not image.public and image.owner != kwargs['owner']: raise ValidationError("You are not the owner of the chosen private image") if length < 1 or length > 21: raise BookingLengthException("Booking must be between 1 and 21 days long") @@ -196,62 +199,105 @@ def check_invariants(request, **kwargs): def create_from_form(form, request): """ - Create a Booking from the user's form. - - Large, nasty method to create a booking or return a useful error - based on the form from the frontend + Parse data from QuickBookingForm to create booking """ resource_field = form.cleaned_data['filter_field'] - purpose_field = form.cleaned_data['purpose'] - project_field = form.cleaned_data['project'] - users_field = form.cleaned_data['users'] + # users_field = form.cleaned_data['users'] hostname = 'opnfv_host' if not form.cleaned_data['hostname'] else form.cleaned_data['hostname'] - length = form.cleaned_data['length'] - image = form.cleaned_data['image'] - scenario = form.cleaned_data['scenario'] - installer = form.cleaned_data['installer'] + global_cloud_config = None if not form.cleaned_data['global_cloud_config'] else form.cleaned_data['global_cloud_config'] + + if global_cloud_config: + form.cleaned_data['global_cloud_config'] = create_ci_file(global_cloud_config) + + # image = form.cleaned_data['image'] + # scenario = form.cleaned_data['scenario'] + # installer = form.cleaned_data['installer'] lab, resource_template = parse_resource_field(resource_field) data = form.cleaned_data + data['hostname'] = hostname data['lab'] = lab data['resource_template'] = resource_template - check_invariants(request, **data) + data['owner'] = request.user + + return _create_booking(data) + + +def create_from_API(body, user): + """ + Parse data from Automation API to create booking + """ + booking_info = json.loads(body.decode('utf-8')) + + data = {} + data['purpose'] = booking_info['purpose'] + data['project'] = booking_info['project'] + data['users'] = [UserProfile.objects.get(user__username=username) + for username in booking_info['collaborators']] + data['hostname'] = booking_info['hostname'] + data['length'] = booking_info['length'] + data['installer'] = None + data['scenario'] = None + + data['image'] = Image.objects.get(pk=booking_info['imageLabID']) + + data['resource_template'] = ResourceTemplate.objects.get(pk=booking_info['templateID']) + data['lab'] = data['resource_template'].lab + data['owner'] = user + + if 'global_cloud_config' in data.keys(): + data['global_cloud_config'] = CloudInitFile.objects.get(id=data['global_cloud_config']) + + return _create_booking(data) + + +def create_ci_file(data: str) -> CloudInitFile: + try: + d = yaml.load(data) + if not (type(d) is dict): + raise Exception("CI file was valid yaml but was not a dict") + except Exception: + raise ValidationError("The provided Cloud Config is not valid yaml, please refer to the Cloud Init documentation for expected structure") + print("about to create global cloud config") + config = CloudInitFile.create(text=data, priority=CloudInitFile.objects.count()) + print("made global cloud config") + + return config + + +@transaction.atomic +def _create_booking(data): + check_invariants(**data) # check booking privileges # TODO: use the canonical booking_allowed method because now template might have multiple # machines - if Booking.objects.filter(owner=request.user, end__gt=timezone.now()).count() >= 3 and not request.user.userprofile.booking_privledge: + if Booking.objects.filter(owner=data['owner'], end__gt=timezone.now()).count() >= 3 and not data['owner'].userprofile.booking_privledge: raise PermissionError("You do not have permission to have more than 3 bookings at a time.") - ResourceManager.getInstance().templateIsReservable(resource_template) + ResourceManager.getInstance().templateIsReservable(data['resource_template']) - resource_template = update_template(resource_template, image, hostname, request.user) - - # if no installer provided, just create blank host - opnfv_config = None - if installer: - hconf = resource_template.getConfigs()[0] - opnfv_config = generate_opnfvconfig(scenario, installer, resource_template) - generate_hostopnfv(hconf, opnfv_config) + resource_template = update_template(data['resource_template'], data['image'], data['hostname'], data['owner'], global_cloud_config=data['global_cloud_config']) # generate resource bundle resource_bundle = generate_resource_bundle(resource_template) # generate booking booking = Booking.objects.create( - purpose=purpose_field, - project=project_field, - lab=lab, - owner=request.user, + purpose=data['purpose'], + project=data['project'], + lab=data['lab'], + owner=data['owner'], start=timezone.now(), - end=timezone.now() + timedelta(days=int(length)), + end=timezone.now() + timedelta(days=int(data['length'])), resource=resource_bundle, - opnfv_config=opnfv_config + opnfv_config=None ) + booking.pdf = PDFTemplater.makePDF(booking) - for collaborator in users_field: # list of UserProfiles + for collaborator in data['users']: # list of Users (not UserProfile) booking.collaborators.add(collaborator.user) booking.save() @@ -272,23 +318,14 @@ def drop_filter(user): that installer is supported on that image """ installer_filter = {} - for image in Image.objects.all(): - installer_filter[image.id] = {} - for installer in image.os.sup_installers.all(): - installer_filter[image.id][installer.id] = 1 - scenario_filter = {} - for installer in Installer.objects.all(): - scenario_filter[installer.id] = {} - for scenario in installer.sup_scenarios.all(): - scenario_filter[installer.id][scenario.id] = 1 images = Image.objects.filter(Q(public=True) | Q(owner=user)) image_filter = {} for image in images: image_filter[image.id] = { 'lab': 'lab_' + str(image.from_lab.lab_user.id), - 'host_profile': str(image.host_type.id), + 'architecture': str(image.architecture), 'name': image.name } @@ -296,7 +333,7 @@ def drop_filter(user): templates = ResourceTemplate.objects.filter(Q(public=True) | Q(owner=user)) for rt in templates: profiles = [conf.profile for conf in rt.getConfigs()] - resource_filter["resource_" + str(rt.id)] = [str(p.id) for p in profiles] + resource_filter["resource_" + str(rt.id)] = [str(p.architecture) for p in profiles] return { 'installer_filter': json.dumps(installer_filter), diff --git a/src/booking/stats.py b/src/booking/stats.py index 626ed79..70f91fa 100644 --- a/src/booking/stats.py +++ b/src/booking/stats.py @@ -104,5 +104,5 @@ class StatisticsManager(object): "user": [x, users], "utils": [in_use, not_in_use, maintenance], "projects": [project_keys, project_counts], - "colors": anuket_colors if os.environ['TEMPLATE_OVERRIDE_DIR'] == 'laas' else lfedge_colors + "colors": anuket_colors if os.environ.get('TEMPLATE_OVERRIDE_DIR') == 'laas' else lfedge_colors } diff --git a/src/booking/urls.py b/src/booking/urls.py index cdf18ae..0b60351 100644 --- a/src/booking/urls.py +++ b/src/booking/urls.py @@ -38,7 +38,7 @@ from booking.views import ( booking_modify_image ) -app_name = "booking" +app_name = 'booking' urlpatterns = [ url(r'^detail/(?P<booking_id>[0-9]+)/$', booking_detail_view, name='detail'), url(r'^(?P<booking_id>[0-9]+)/$', booking_detail_view, name='booking_detail'), diff --git a/src/booking/views.py b/src/booking/views.py index 2b910e7..940428b 100644 --- a/src/booking/views.py +++ b/src/booking/views.py @@ -28,6 +28,7 @@ from api.models import JobFactory from workflow.views import login from booking.forms import QuickBookingForm from booking.quick_deployer import create_from_form, drop_filter +import traceback def quick_create_clear_fields(request): @@ -62,6 +63,9 @@ def quick_create(request): "Check Account->My Bookings for the status of your new booking") return redirect(reverse('booking:booking_detail', kwargs={'booking_id': booking.id})) except Exception as e: + print("Error occurred while handling quick deployment:") + traceback.print_exc() + print(str(e)) messages.error(request, "Whoops, an error occurred: " + str(e)) context.update(drop_filter(request.user)) return render(request, 'booking/quick_deploy.html', context) @@ -127,7 +131,6 @@ class ResourceBookingsJSON(View): 'start', 'end', 'purpose', - 'jira_issue_status', 'config_bundle__name' ) return JsonResponse({'bookings': list(bookings)}) @@ -138,7 +141,7 @@ def build_image_mapping(lab, user): for profile in ResourceProfile.objects.filter(labs=lab): images = Image.objects.filter( from_lab=lab, - host_type=profile + architecture=profile.architecture ).filter( Q(public=True) | Q(owner=user) ) diff --git a/src/dashboard/admin_utils.py b/src/dashboard/admin_utils.py index b105e96..045caeb 100644 --- a/src/dashboard/admin_utils.py +++ b/src/dashboard/admin_utils.py @@ -22,7 +22,8 @@ from resource_inventory.models import ( DiskProfile, CpuProfile, RamProfile, - Interface + Interface, + CloudInitFile, ) import json @@ -50,7 +51,7 @@ from booking.models import Booking from notifier.manager import NotificationHandler from api.models import JobFactory -from api.models import JobStatus +from api.models import JobStatus, Job, GeneratedCloudConfig def print_div(): @@ -528,6 +529,30 @@ def extend_booking(booking_id, days=0, hours=0, minutes=0, weeks=0): booking.save() +def regenerate_cloud_configs(booking_id): + b = Booking.objects.get(id=booking_id) + for res in b.resource.get_resources(): + res.config.cloud_init_files.set(res.config.cloud_init_files.filter(generated=False)) # careful! + res.config.save() + cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=b, rconfig=res.config) + cif.save() + cif = CloudInitFile.create(priority=0, text=cif.serialize()) + cif.save() + res.config.cloud_init_files.add(cif) + res.config.save() + + +def set_job_new(job_id): + j = Job.objects.get(id=job_id) + b = j.booking + regenerate_cloud_configs(b.id) + for task in j.get_tasklist(): + task.status = JobStatus.NEW + task.save() + j.status = JobStatus.NEW + j.save() + + def docs(function=None, fulltext=False): """ Print documentation for a given function in admin_utils. diff --git a/src/dashboard/tasks.py b/src/dashboard/tasks.py index 3f88449..93e6a22 100644 --- a/src/dashboard/tasks.py +++ b/src/dashboard/tasks.py @@ -81,11 +81,15 @@ def free_hosts(): ).filter( end__lt=timezone.now(), job__complete=True, - resource__isnull=False + complete=False, + resource__isnull=False, ) for booking in bookings: ResourceManager.getInstance().releaseResourceBundle(booking.resource) + booking.complete = True + print("Booking", booking.id, "is now completed") + booking.save() @shared_task diff --git a/src/dashboard/templatetags/jira_filters.py b/src/dashboard/templatetags/jira_filters.py deleted file mode 100644 index 9a97c1d..0000000 --- a/src/dashboard/templatetags/jira_filters.py +++ /dev/null @@ -1,17 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from django.conf import settings -from django.template.defaultfilters import register - - -@register.filter -def jira_issue_url(issue): - return settings.JIRA_URL + '/browse/' + str(issue) diff --git a/src/dashboard/urls.py b/src/dashboard/urls.py index d5dad57..c87dacc 100644 --- a/src/dashboard/urls.py +++ b/src/dashboard/urls.py @@ -33,7 +33,7 @@ from dashboard.views import ( host_profile_detail_view ) -app_name = "dashboard" +app_name = 'dashboard' urlpatterns = [ url(r'^$', landing_view, name='index'), url(r'^lab/$', lab_list_view, name='all_labs'), diff --git a/src/dashboard/utils.py b/src/dashboard/utils.py index d6b697a..97c9ac7 100644 --- a/src/dashboard/utils.py +++ b/src/dashboard/utils.py @@ -7,7 +7,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import (ObjectDoesNotExist, MultipleObjectsReturned) class AbstractModelQuery(): @@ -38,7 +38,15 @@ class AbstractModelQuery(): @classmethod def get(cls, *args, **kwargs): + """ + Gets a single matching resource + Throws ObjectDoesNotExist if none found matching, or MultipleObjectsReturned if + the query does not narrow to a single object + """ try: + ls = cls.filter(*args, **kwargs) + if len(ls) > 1: + raise MultipleObjectsReturned() return cls.filter(*args, **kwargs)[0] except IndexError: raise ObjectDoesNotExist() diff --git a/src/laas_dashboard/celery.py b/src/laas_dashboard/celery.py index 65b9211..362bbdb 100644 --- a/src/laas_dashboard/celery.py +++ b/src/laas_dashboard/celery.py @@ -23,6 +23,7 @@ app = Celery('laas_dashboard') # pickle the object when using Windows. app.config_from_object('django.conf:settings') app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) +app.config_from_object('django.conf:settings', namespace='CELERY') @app.task(bind=True) diff --git a/src/laas_dashboard/settings.py b/src/laas_dashboard/settings.py index 6b3ed09..7e27c8d 100644 --- a/src/laas_dashboard/settings.py +++ b/src/laas_dashboard/settings.py @@ -15,8 +15,8 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # SECURITY WARNING: don't run with debug turned on in production! # NOTE: os.environ only returns strings, so making a comparison to # 'True' here will convert it to the correct Boolean value. -DEBUG = os.environ['DEBUG'] == 'True' -TESTING = os.environ['TEST'] == 'True' +DEBUG = os.environ.get('DEBUG') == 'True' +TESTING = os.environ.get('TEST') == 'True' # Application definition @@ -53,29 +53,34 @@ MIDDLEWARE = [ 'account.middleware.TimezoneMiddleware', ] -AUTH_SETTING = os.environ.get('AUTH_SETTING', 'JIRA') -if AUTH_SETTING == 'LFID': - AUTHENTICATION_BACKENDS = ['account.views.MyOIDCAB'] +# AUTHENTICATION_BACKENDS = ['django.contrib.auth.backends.ModelBackend', 'account.views.MyOIDCAB'] +AUTHENTICATION_BACKENDS = ['django.contrib.auth.backends.ModelBackend'] + +AUTH_SETTING = os.environ.get('AUTH_SETTING') +if AUTH_SETTING == 'LFID': # OpenID Authentications - OIDC_RP_CLIENT_ID = os.environ['OIDC_CLIENT_ID'] - OIDC_RP_CLIENT_SECRET = os.environ['OIDC_CLIENT_SECRET'] + AUTHENTICATION_BACKENDS.append('account.views.MyOIDCAB') + OIDC_RP_CLIENT_ID = os.environ.get('OIDC_CLIENT_ID') + OIDC_RP_CLIENT_SECRET = os.environ.get('OIDC_CLIENT_SECRET') - OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ['OIDC_AUTHORIZATION_ENDPOINT'] - OIDC_OP_TOKEN_ENDPOINT = os.environ['OIDC_TOKEN_ENDPOINT'] - OIDC_OP_USER_ENDPOINT = os.environ['OIDC_USER_ENDPOINT'] + OIDC_OP_AUTHORIZATION_ENDPOINT = os.environ.get('OIDC_AUTHORIZATION_ENDPOINT') + OIDC_OP_TOKEN_ENDPOINT = os.environ.get('OIDC_TOKEN_ENDPOINT') + OIDC_OP_USER_ENDPOINT = os.environ.get('OIDC_USER_ENDPOINT') - LOGIN_REDIRECT_URL = os.environ['DASHBOARD_URL'] - LOGOUT_REDIRECT_URL = os.environ['DASHBOARD_URL'] + LOGIN_REDIRECT_URL = os.environ.get('DASHBOARD_URL') + LOGOUT_REDIRECT_URL = os.environ.get('DASHBOARD_URL') - OIDC_RP_SIGN_ALGO = os.environ["OIDC_RP_SIGN_ALGO"] + OIDC_RP_SIGN_ALGO = os.environ.get("OIDC_RP_SIGN_ALGO") if OIDC_RP_SIGN_ALGO == "RS256": - OIDC_OP_JWKS_ENDPOINT = os.environ["OIDC_OP_JWKS_ENDPOINT"] + OIDC_OP_JWKS_ENDPOINT = os.environ.get("OIDC_OP_JWKS_ENDPOINT") +else: + raise Exception('AUTH_SETTING set to invalid value') # This is for LFID auth setups w/ an HTTPS proxy -if os.environ['EXPECT_HOST_FORWARDING'] == 'True': +if os.environ.get('EXPECT_HOST_FORWARDING') == 'True': SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', "https") USE_X_FORWARDED_HOST = True @@ -162,7 +167,7 @@ STATICFILES_DIRS = [ LOGIN_REDIRECT_URL = '/' # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.environ['SECRET_KEY'] +SECRET_KEY = os.environ.get('SECRET_KEY') BOOTSTRAP3 = { 'set_placeholder': False, @@ -175,11 +180,11 @@ ALLOWED_HOSTS = ['*'] DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql', - 'NAME': os.environ['DB_NAME'], - 'USER': os.environ['DB_USER'], - 'PASSWORD': os.environ['DB_PASS'], - 'HOST': os.environ['DB_SERVICE'], - 'PORT': os.environ['DB_PORT'] + 'NAME': os.environ.get('DB_NAME'), + 'USER': os.environ.get('DB_USER'), + 'PASSWORD': os.environ.get('DB_PASS'), + 'HOST': os.environ.get('DB_SERVICE'), + 'PORT': os.environ.get('DB_PORT') } } @@ -198,33 +203,23 @@ REST_FRAMEWORK = { MEDIA_ROOT = '/media' STATIC_ROOT = '/static' -# Jira Settings -CREATE_JIRA_TICKET = False - -JIRA_URL = os.environ['JIRA_URL'] - -JIRA_USER_NAME = os.environ['JIRA_USER_NAME'] -JIRA_USER_PASSWORD = os.environ['JIRA_USER_PASSWORD'] - -OAUTH_CONSUMER_KEY = os.environ['OAUTH_CONSUMER_KEY'] -OAUTH_CONSUMER_SECRET = os.environ['OAUTH_CONSUMER_SECRET'] - -OAUTH_REQUEST_TOKEN_URL = JIRA_URL + '/plugins/servlet/oauth/request-token' -OAUTH_ACCESS_TOKEN_URL = JIRA_URL + '/plugins/servlet/oauth/access-token' -OAUTH_AUTHORIZE_URL = JIRA_URL + '/plugins/servlet/oauth/authorize' +OAUTH_CONSUMER_KEY = os.environ.get('OAUTH_CONSUMER_KEY') +OAUTH_CONSUMER_SECRET = os.environ.get('OAUTH_CONSUMER_SECRET') -OAUTH_CALLBACK_URL = os.environ['DASHBOARD_URL'] + '/accounts/authenticated' +OAUTH_CALLBACK_URL = os.environ.get('DASHBOARD_URL') + '/accounts/authenticated' # Celery Settings CELERY_TIMEZONE = 'UTC' RABBITMQ_URL = 'rabbitmq' +# RABBITMQ_DEFAULT_USER = os.environ['DEFAULT_USER'] +# RABBITMQ_DEFAULT_PASS = os.environ['DEFAULT_PASS'] RABBITMQ_DEFAULT_USER = os.environ['RABBITMQ_DEFAULT_USER'] RABBITMQ_DEFAULT_PASS = os.environ['RABBITMQ_DEFAULT_PASS'] -BROKER_URL = 'amqp://' + RABBITMQ_DEFAULT_USER + ':' + RABBITMQ_DEFAULT_PASS + '@rabbitmq:5672//' +CELERY_BROKER_URL = 'amqp://' + RABBITMQ_DEFAULT_USER + ':' + RABBITMQ_DEFAULT_PASS + '@rabbitmq:5672//' -CELERYBEAT_SCHEDULE = { +CELERY_BEAT_SCHEDULE = { 'booking_poll': { 'task': 'dashboard.tasks.booking_poll', 'schedule': timedelta(minutes=1) @@ -248,10 +243,10 @@ CELERYBEAT_SCHEDULE = { } # Notifier Settings -EMAIL_HOST = os.environ['EMAIL_HOST'] -EMAIL_PORT = os.environ['EMAIL_PORT'] -EMAIL_HOST_USER = os.environ['EMAIL_HOST_USER'] -EMAIL_HOST_PASSWORD = os.environ['EMAIL_HOST_PASSWORD'] +EMAIL_HOST = os.environ.get('EMAIL_HOST') +EMAIL_PORT = os.environ.get('EMAIL_PORT') +EMAIL_HOST_USER = os.environ.get('EMAIL_HOST_USER') +EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD') EMAIL_USE_TLS = True DEFAULT_EMAIL_FROM = os.environ.get('DEFAULT_EMAIL_FROM', 'webmaster@localhost') SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies" diff --git a/src/notifier/urls.py b/src/notifier/urls.py index fedb9e8..923cc33 100644 --- a/src/notifier/urls.py +++ b/src/notifier/urls.py @@ -12,7 +12,7 @@ from django.conf.urls import url from notifier.views import InboxView, NotificationView -app_name = "notifier" +app_name = 'notifier' urlpatterns = [ url(r'^$', InboxView, name='messages'), url(r'^notification/(?P<notification_id>[0-9]+)/$', NotificationView, name='notifier_single') diff --git a/src/resource_inventory/migrations/0018_auto_20210630_1629.py b/src/resource_inventory/migrations/0018_auto_20210630_1629.py new file mode 100644 index 0000000..19e53e4 --- /dev/null +++ b/src/resource_inventory/migrations/0018_auto_20210630_1629.py @@ -0,0 +1,101 @@ +# Generated by Django 2.2 on 2021-06-30 16:29 + +from django.db import migrations, models +import django.db.models.deletion +from account.models import Lab + + +def set_availability(apps, schema_editor): + models = [apps.get_model('resource_inventory', 'Image'), apps.get_model('resource_inventory', 'Opsys')] + + for model in models: + for obj in model.objects.all(): + obj.available = False + obj.obsolete = True + obj.save() + + +def set_rconfig_arch(apps, schema_editor): + rprofs = apps.get_model('resource_inventory', 'ResourceProfile') + + for rprof in rprofs.objects.all(): + rprof.architecture = rprof.cpuprofile.first().architecture + + +class Migration(migrations.Migration): + + dependencies = [ + ('account', '0009_auto_20210324_2107'), + ('resource_inventory', '0017_auto_20201218_1516'), + ] + + operations = [ + migrations.RemoveField( + model_name='image', + name='host_type', + ), + migrations.AlterField( + model_name='image', + name='lab_id', + field=models.CharField(default='none (retired)', max_length=100), + preserve_default=True, + ), + migrations.RemoveField( + model_name='opsys', + name='sup_installers', + ), + + migrations.AddField( + model_name='image', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50), + preserve_default=False, + ), + + migrations.AddField( + model_name='image', + name='available', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='image', + name='obsolete', + field=models.BooleanField(default=False), + ), + + migrations.AddField( + model_name='opsys', + name='available', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='opsys', + name='obsolete', + field=models.BooleanField(default=True), + ), + + migrations.RunPython(set_availability), + + migrations.AddField( + model_name='opsys', + name='lab_id', + field=models.CharField(default="none (retired)", max_length=100), + preserve_default=False, + ), + + migrations.AddField( + model_name='opsys', + name='from_lab', + field=models.ForeignKey(default=Lab.objects.first, on_delete=django.db.models.deletion.CASCADE, to='account.Lab'), + preserve_default=False, + ), + + migrations.AddField( + model_name='resourceprofile', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50), + preserve_default=False, + ), + + migrations.RunPython(set_rconfig_arch), + ] diff --git a/src/resource_inventory/migrations/0019_auto_20210701_1947.py b/src/resource_inventory/migrations/0019_auto_20210701_1947.py new file mode 100644 index 0000000..e64d174 --- /dev/null +++ b/src/resource_inventory/migrations/0019_auto_20210701_1947.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2 on 2021-07-01 19:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0018_auto_20210630_1629'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='lab_id', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='image', + name='name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='network', + name='name', + field=models.CharField(max_length=200), + ), + migrations.AlterField( + model_name='opsys', + name='available', + field=models.BooleanField(default=True), + ), + migrations.AlterField( + model_name='opsys', + name='obsolete', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='resourceprofile', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64')], max_length=50), + ), + ] diff --git a/src/resource_inventory/migrations/0020_cloudinitfile.py b/src/resource_inventory/migrations/0020_cloudinitfile.py new file mode 100644 index 0000000..198181c --- /dev/null +++ b/src/resource_inventory/migrations/0020_cloudinitfile.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2 on 2021-09-07 14:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0019_auto_20210701_1947'), + ] + + operations = [ + migrations.CreateModel( + name='CloudInitFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('text', models.TextField()), + ('priority', models.IntegerField()), + ], + ), + ] diff --git a/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py new file mode 100644 index 0000000..6b0befc --- /dev/null +++ b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2021-09-10 18:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0020_cloudinitfile'), + ] + + operations = [ + migrations.AddField( + model_name='resourceconfiguration', + name='cloud_init_files', + field=models.ManyToManyField(blank=True, to='resource_inventory.CloudInitFile'), + ), + ] diff --git a/src/resource_inventory/migrations/0022_auto_20210925_2028.py b/src/resource_inventory/migrations/0022_auto_20210925_2028.py new file mode 100644 index 0000000..2b0b902 --- /dev/null +++ b/src/resource_inventory/migrations/0022_auto_20210925_2028.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2 on 2021-09-25 20:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0021_resourceconfiguration_cloud_init_files'), + ] + + operations = [ + migrations.AddField( + model_name='resourcetemplate', + name='private_vlan_pool', + field=models.TextField(default=''), + ), + migrations.AddField( + model_name='resourcetemplate', + name='public_vlan_pool', + field=models.TextField(default=''), + ), + ] diff --git a/src/resource_inventory/models.py b/src/resource_inventory/models.py index 7fe479a..5d87430 100644 --- a/src/resource_inventory/models.py +++ b/src/resource_inventory/models.py @@ -9,10 +9,12 @@ ############################################################################## from django.contrib.auth.models import User + from django.core.exceptions import ValidationError from django.db import models from django.db.models import Q import traceback +import json import re from collections import Counter @@ -20,7 +22,6 @@ from collections import Counter from account.models import Lab from dashboard.utils import AbstractModelQuery - """ Profiles of resources hosted by labs. @@ -33,6 +34,10 @@ Profile models (e.g. an x86 server profile and armv8 server profile. class ResourceProfile(models.Model): id = models.AutoField(primary_key=True) name = models.CharField(max_length=200, unique=True) + architecture = models.CharField(max_length=50, choices=[ + ("x86_64", "x86_64"), + ("aarch64", "aarch64") + ]) description = models.TextField() labs = models.ManyToManyField(Lab, related_name="resourceprofiles") @@ -147,6 +152,25 @@ with varying degrees of abstraction. """ +class CloudInitFile(models.Model): + text = models.TextField() + + # higher priority is applied later, so "on top" of existing files + priority = models.IntegerField() + generated = models.BooleanField(default=False) + + @classmethod + def merge_strategy(cls): + return [ + {'name': 'list', 'settings': ['append']}, + {'name': 'dict', 'settings': ['recurse_list', 'replace']}, + ] + + @classmethod + def create(cls, text="", priority=0): + return CloudInitFile.objects.create(priority=priority, text=text) + + class ResourceTemplate(models.Model): """ Models a "template" of a complete, configured collection of resources that can be booked. @@ -167,6 +191,24 @@ class ResourceTemplate(models.Model): temporary = models.BooleanField(default=False) copy_of = models.ForeignKey("ResourceTemplate", blank=True, null=True, on_delete=models.SET_NULL) + # if these fields are empty ("") then they are implicitly "every vlan", + # otherwise we filter any allocations we try to instantiate against this list + # they should be represented as a json list of integers + private_vlan_pool = models.TextField(default="") + public_vlan_pool = models.TextField(default="") + + def private_vlan_pool_set(self): + if self.private_vlan_pool != "": + return set(json.loads(self.private_vlan_pool)) + else: + return None + + def public_vlan_pool_set(self): + if self.private_vlan_pool != "": + return set(json.loads(self.public_vlan_pool)) + else: + return None + def getConfigs(self): configs = self.resourceConfigurations.all() return list(configs) @@ -235,9 +277,14 @@ class ResourceConfiguration(models.Model): is_head_node = models.BooleanField(default=False) name = models.CharField(max_length=3000, default="opnfv_host") + cloud_init_files = models.ManyToManyField(CloudInitFile, blank=True) + def __str__(self): return str(self.name) + def ci_file_list(self): + return list(self.cloud_init_files.order_by("priority").all()) + def get_default_remote_info(): return RemoteInfo.objects.get_or_create( @@ -369,10 +416,43 @@ class Server(Resource): return isinstance(other, Server) and other.name == self.name +def is_serializable(data): + try: + json.dumps(data) + return True + except Exception: + return False + + class Opsys(models.Model): id = models.AutoField(primary_key=True) name = models.CharField(max_length=100) - sup_installers = models.ManyToManyField("Installer", blank=True) + lab_id = models.CharField(max_length=100) + obsolete = models.BooleanField(default=False) + available = models.BooleanField(default=True) # marked true by Cobbler if it exists there + from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE) + + indexes = [ + models.Index(fields=['cobbler_id']) + ] + + def new_from_data(data): + opsys = Opsys() + opsys.update(data) + return opsys + + def serialize(self): + d = {} + for field in vars(self): + attr = getattr(self, field) + if is_serializable(attr): + d[field] = attr + return d + + def update(self, data): + for field in vars(self): + if field in data: + setattr(self, field, data[field] if data[field] else getattr(self, field)) def __str__(self): return self.name @@ -382,18 +462,51 @@ class Image(models.Model): """Model for representing OS images / snapshots of hosts.""" id = models.AutoField(primary_key=True) - lab_id = models.IntegerField() # ID the lab who holds this image knows from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE) - name = models.CharField(max_length=200) + architecture = models.CharField(max_length=50, choices=[ + ("x86_64", "x86_64"), + ("aarch64", "aarch64"), + ("unknown", "unknown"), + ]) + lab_id = models.CharField(max_length=100) + name = models.CharField(max_length=100) owner = models.ForeignKey(User, null=True, on_delete=models.SET_NULL) public = models.BooleanField(default=True) - host_type = models.ForeignKey(ResourceProfile, on_delete=models.CASCADE) description = models.TextField() os = models.ForeignKey(Opsys, null=True, on_delete=models.CASCADE) + available = models.BooleanField(default=True) # marked True by cobbler if it exists there + obsolete = models.BooleanField(default=False) + + indexes = [ + models.Index(fields=['architecture']), + models.Index(fields=['cobbler_id']) + ] + def __str__(self): return self.name + def is_obsolete(self): + return self.obsolete or self.os.obsolete + + def serialize(self): + d = {} + for field in vars(self): + attr = getattr(self, field) + if is_serializable(attr): + d[field] = attr + return d + + def update(self, data): + for field in vars(self): + if field in data: + setattr(self, field, data[field] if data[field] else getattr(self, field)) + + def new_from_data(data): + img = Image() + img.update(data) + return img + def in_use(self): for resource in ResourceQuery.filter(config__image=self): if resource.is_reserved(): @@ -409,7 +522,7 @@ Networking configuration models class Network(models.Model): id = models.AutoField(primary_key=True) - name = models.CharField(max_length=100) + name = models.CharField(max_length=200) bundle = models.ForeignKey(ResourceTemplate, on_delete=models.CASCADE, related_name="networks") is_public = models.BooleanField() @@ -507,6 +620,13 @@ class NetworkRole(models.Model): network = models.ForeignKey(Network, on_delete=models.CASCADE) +def create_resource_ref_string(for_hosts: [str]) -> str: + # need to sort the list, then do dump + for_hosts.sort() + + return json.dumps(for_hosts) + + class OPNFVConfig(models.Model): id = models.AutoField(primary_key=True) installer = models.ForeignKey(Installer, on_delete=models.CASCADE) diff --git a/src/resource_inventory/resource_manager.py b/src/resource_inventory/resource_manager.py index 9406977..52af824 100644 --- a/src/resource_inventory/resource_manager.py +++ b/src/resource_inventory/resource_manager.py @@ -6,20 +6,29 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## + +from __future__ import annotations # noqa: F407 + import re +from typing import Optional from django.db.models import Q from dashboard.exceptions import ResourceAvailabilityException from resource_inventory.models import ( + Resource, ResourceBundle, ResourceTemplate, + ResourceConfiguration, Network, Vlan, PhysicalNetwork, InterfaceConfiguration, ) +from account.models import Lab +from django.contrib.auth.models import User + class ResourceManager: @@ -29,19 +38,19 @@ class ResourceManager: pass @staticmethod - def getInstance(): + def getInstance() -> ResourceManager: if ResourceManager.instance is None: ResourceManager.instance = ResourceManager() return ResourceManager.instance - def getAvailableResourceTemplates(self, lab, user=None): + def getAvailableResourceTemplates(self, lab: Lab, user: Optional[User] = None) -> list[ResourceTemplate]: filter = Q(public=True) if user: filter = filter | Q(owner=user) filter = filter & Q(temporary=False) & Q(lab=lab) return ResourceTemplate.objects.filter(filter) - def templateIsReservable(self, resource_template): + def templateIsReservable(self, resource_template: ResourceTemplate): """ Check if the required resources to reserve this template is available. @@ -63,28 +72,32 @@ class ResourceManager: return True # public interface - def deleteResourceBundle(self, resourceBundle): + def deleteResourceBundle(self, resourceBundle: ResourceBundle): raise NotImplementedError("Resource Bundle Deletion Not Implemented") - def releaseResourceBundle(self, resourceBundle): + def releaseResourceBundle(self, resourceBundle: ResourceBundle): resourceBundle.release() - def get_vlans(self, resourceTemplate): + def get_vlans(self, resourceTemplate: ResourceTemplate) -> dict[str, int]: + """ + returns: dict from network name to the associated vlan number (backend vlan id) + """ networks = {} vlan_manager = resourceTemplate.lab.vlan_manager for network in resourceTemplate.networks.all(): if network.is_public: - public_net = vlan_manager.get_public_vlan() + # already throws if can't get requested count, so can always expect public_net to be Some + public_net = vlan_manager.get_public_vlan(within=resourceTemplate.public_vlan_pool_set()) vlan_manager.reserve_public_vlan(public_net.vlan) networks[network.name] = public_net.vlan else: # already throws if can't get requested count, so can always index in @ 0 - vlans = vlan_manager.get_vlans(count=1) + vlans = vlan_manager.get_vlans(count=1, within=resourceTemplate.private_vlan_pool_set()) vlan_manager.reserve_vlans(vlans[0]) networks[network.name] = vlans[0] return networks - def instantiateTemplate(self, resource_template): + def instantiateTemplate(self, resource_template: ResourceTemplate): """ Convert a ResourceTemplate into a ResourceBundle. @@ -113,16 +126,18 @@ class ResourceManager: return resource_bundle - def configureNetworking(self, resource_bundle, resource, vlan_map): + def configureNetworking(self, resource_bundle: ResourceBundle, resource: Resource, vlan_map: dict[str, int]): + """ + @vlan_map: dict from network name to the associated vlan number (backend vlan id) + """ for physical_interface in resource.interfaces.all(): - # assign interface configs - iface_configs = InterfaceConfiguration.objects.filter( + # assign interface configs + iface_config = InterfaceConfiguration.objects.get( profile=physical_interface.profile, resource_config=resource.config ) - iface_config = iface_configs.first() physical_interface.acts_as = iface_config physical_interface.acts_as.save() @@ -143,7 +158,7 @@ class ResourceManager: ) # private interface - def acquireHost(self, resource_config): + def acquireHost(self, resource_config: ResourceConfiguration) -> Resource: resources = resource_config.profile.get_resources( lab=resource_config.template.lab, unreserved=True diff --git a/src/resource_inventory/tests/test_models.py b/src/resource_inventory/tests/test_models.py index e1b2106..3f2d1d8 100644 --- a/src/resource_inventory/tests/test_models.py +++ b/src/resource_inventory/tests/test_models.py @@ -80,7 +80,7 @@ class ConfigUtil(): ) return Image.objects.create( - lab_id=0, + cobbler_id="profile1", from_lab=lab, name="an image for testing", owner=owner diff --git a/src/resource_inventory/urls.py b/src/resource_inventory/urls.py index a008176..a9a4d43 100644 --- a/src/resource_inventory/urls.py +++ b/src/resource_inventory/urls.py @@ -29,7 +29,7 @@ from django.conf.urls import url from resource_inventory.views import HostView, hostprofile_detail_view -app_name = "resource" +app_name = 'resource' urlpatterns = [ url(r'^hosts$', HostView.as_view(), name='hosts'), url(r'^profiles/(?P<hostprofile_id>.+)/$', hostprofile_detail_view, name='host_detail'), diff --git a/src/static/css/anuket.css b/src/static/css/anuket.css new file mode 100644 index 0000000..6bbdb3f --- /dev/null +++ b/src/static/css/anuket.css @@ -0,0 +1,115 @@ +nav ,body{ + background-color:#fff !important; + color:#343a40 !important; + } + + header{ + background-color:#f8f9fa !important; + color:#343a40 !important; + } + + p, h1, h2, h3, h4, h5{ + color:#343a40 !important; + } + + a, .page-link { + color: #007473 !important; + } + + .page-item.active .page-link{ + color: #f8f9fa !important; + background-color: #007473 !important; + } + + .topcrumb.active > span { + background: #007473 !important; + } + + .nav-bg{ + background-color:#fff !important; + color:#343a40 !important; + } + + .nav-bg:hover{ + background-color:#f8f9fa !important; + transition-duration:0.2s; + } + + .dropDown-bg{ + background-color:#d6d8db !important; + color:#343a40 !important; + } + + .btn-primary{ + color: #f8f9fa !important; + background-color: #007473 !important; + border:0px !important; + transition-duration:0.2s !important; + } + + .btn-primary:hover{ + color: #343a40 !important; + background-color: #6BDAD5 !important; + border:0px !important; + } + + .btn-primary:focus{ + color: #343a40 !important; + background-color: #6BDAD5 !important; + border:0px !important; + } + + .btn-success{ + color: #f8f9fa; + background-color: #008852; + border:0px !important; + } + + .btn-success:hover{ + color: #343a40; + background-color: #00CE7C; + border:0px !important; + } + + .btn-success:focus{ + color: #343a40; + background-color: #00CE7C; + border:0px !important; + } + + .btn-danger { + color: #f8f9fa; + background-color: #af2b38; + border:0px !important; + } + + .btn-danger:hover { + color: #f8f9fa; + background-color: #dc3545; + border:0px !important; + } + + .alert-danger{ + background-color: #e6b3c1 !important; + color:#820c2c !important; + border:0px !important; + } + + .Anuket-Text{ + color:#343a40 !important; + } + + .selected_node { + border-color: #008852; + box-shadow: 0 1px 1px rgb(0 0 0 / 8%), 0 0 8px rgb(109 243 76 / 60%); + transition: border-color ease-in-out .1s,box-shadow ease-in-out .1s; + } + + ::selection { + background: #BCE194; + color:#343a40; + } + ::-moz-selection { + background: #BCE194; + color:#343a40; + }
\ No newline at end of file diff --git a/src/static/css/lfedge.css b/src/static/css/lfedge.css new file mode 100644 index 0000000..328d71a --- /dev/null +++ b/src/static/css/lfedge.css @@ -0,0 +1,14 @@ +.LFEdge { + background: #0049b0; + margin-left: -25px; +} + +.wtext { + font-size: 18px; + color: #FFFFFF; +} + +.wtext:hover { + color: #FFFFFF; + text-decoration: none; +}
\ No newline at end of file diff --git a/src/templates/base/account/configuration_list.html b/src/templates/base/account/configuration_list.html deleted file mode 100644 index fee6e83..0000000 --- a/src/templates/base/account/configuration_list.html +++ /dev/null @@ -1,85 +0,0 @@ -{% extends "base.html" %} -{% block content %} -<div class="row"> -{% for config in configurations %} - <div class="col-12 col-md-6 col-lg-4 col-xl-3 mb-3"> - <div class="card h-100"> - <div class="card-header"> - <h3>Configuration {{config.id}}</h3> - </div> - <ul class="list-group list-group-flush h-100"> - <li class="list-group-item">id: {{config.id}}</li> - <li class="list-group-item">name: {{config.name}}</li> - <li class="list-group-item">description: {{config.description}}</li> - <li class="list-group-item">resource: {{config.bundle}}</li> - </ul> - <div class="card-footer"> - <button - class="btn btn-danger w-100" - onclick='delete_config({{config.id}});' - data-toggle="modal" - data-target="#configModal" - >Delete</button> - </div> - </div> - </div> -{% empty %} - <div class="col"> - <p>You don't have any configurations. You can create a configuration by configuring a pod.</p> - </div> -{% endfor %} -</div> - -<script> - var current_config_id = -1; - function delete_config(config_id) { - current_config_id = config_id; - } - - function submit_delete_form() { - var ajaxForm = $("#config_delete_form"); - var formData = ajaxForm.serialize(); - req = new XMLHttpRequest(); - var url = "delete/" + current_config_id; - req.onreadystatechange = function() { - if (this.readyState == 4 && this.status == 200) { - location.reload(); - } - }; - req.open("POST", url, true); - req.setRequestHeader("Content-Type", "application/x-www-form-urlencoded"); - req.onerror = function() { alert("problem submitting form"); } - req.send(formData); - } -</script> - -<div class="modal fade" id="configModal" tabindex="-1" role="dialog" aria-hidden="true"> - <div class="modal-dialog" role="document"> - <div class="modal-content"> - <div class="modal-header"> - <h4 class="modal-title d-inline float-left">Delete Configuration?</h4> - <button type="button" class="close" data-dismiss="modal" aria-label="Close"> - <span aria-hidden="true">×</span> - </button> - </div> - <form id="config_delete_form"> - {% csrf_token %} - </form> - <div class="modal-footer d-flex flex-column"> - <div class="mb-2"> - <button type="button" class="btn btn-outline-secondary" data-dismiss="modal">Close</button> - <button type="button" class="btn btn-danger" data-toggle="collapse" data-target="#warning">Delete</button> - </div> - <div class="collapse w-100 text-center border-top" id="warning"> - <div class="p-4"> - <h3>Are You Sure?</h3> - <p>This cannot be undone</p> - <button class="btn btn-outline-secondary" data-dismiss="modal">Nevermind</button> - <button class="btn btn-danger" data-dismiss="modal" onclick="submit_delete_form();">I'm Sure</button> - </div> - </div> - </div> - </div> - </div> -</div> -{% endblock %} diff --git a/src/templates/base/account/details.html b/src/templates/base/account/details.html index 3092ad0..ad59c9a 100644 --- a/src/templates/base/account/details.html +++ b/src/templates/base/account/details.html @@ -4,6 +4,5 @@ <h1>Account Details</h1> <a class="btn btn-primary" href="{% url 'account:my-resources' %}">My Resources</a> <a class="btn btn-primary" href="{% url 'account:my-bookings' %}">My Bookings</a> -<a class="btn btn-primary" href="{% url 'account:my-configurations' %}">My Configurations</a> <a class="btn btn-primary" href="{% url 'account:my-images' %}">My Snapshots</a> {% endblock content %} diff --git a/src/templates/base/base.html b/src/templates/base/base.html index a628ab4..351bd9a 100644 --- a/src/templates/base/base.html +++ b/src/templates/base/base.html @@ -87,7 +87,7 @@ {% else %} <a href="{% url 'account:login' %}" class="dropdown-item Anuket-Text"> <i class="fas fa-sign-in-alt"></i> - Login with Jira + Login </a> {% endif %} {% endif %} @@ -156,9 +156,6 @@ <a href="{% url 'account:my-bookings' %}" class="list-group-item list-group-item-action list-group-item-secondary dropDown-bg"> My Bookings </a> - <a href="{% url 'account:my-configurations' %}" class="list-group-item list-group-item-action list-group-item-secondary dropDown-bg"> - My Configurations - </a> <a href="{% url 'account:my-images' %}" class="list-group-item list-group-item-action list-group-item-secondary dropDown-bg"> My Snapshots </a> diff --git a/src/templates/base/booking/booking_delete.html b/src/templates/base/booking/booking_delete.html index b89eb15..4afa370 100644 --- a/src/templates/base/booking/booking_delete.html +++ b/src/templates/base/booking/booking_delete.html @@ -1,4 +1,3 @@ -{% load jira_filters %} {% load bootstrap4 %} <p> diff --git a/src/templates/base/booking/booking_detail.html b/src/templates/base/booking/booking_detail.html index a014fea..4a8f35a 100644 --- a/src/templates/base/booking/booking_detail.html +++ b/src/templates/base/booking/booking_detail.html @@ -7,6 +7,12 @@ <script src="https://cdn.rawgit.com/google/code-prettify/master/loader/run_prettify.js?lang=yaml"></script> {% endblock %} +<style> +code { + overflow: scroll; +} +</style> + {% block content %} <div class="row"> <div class="col-12 col-lg-5"> @@ -154,6 +160,65 @@ </div> </div> </div> + <div class="card my-3"> + <div class="card-header d-flex"> + <h4 class="d-inline">Diagnostic Information</h4> + <button data-toggle="collapse" data-target="#diagnostics_panel" class="btn btn-outline-secondary ml-auto">Expand</button> + </div> + <div class="collapse" id="diagnostics_panel"> + <div class="card-body"> + <table class="table m-0"> + <tr> + <th>Job ID: </th> + <td>{{booking.job.id}}</td> + </tr> + <tr> + <th>CI Files</th> + </tr> + {% for host in booking.resource.get_resources %} + <tr> + <td> + <table class="table m-0"> + <tr> + <th>Host:</th> + <td>{{host.name}}</td> + </tr> + <tr> + <th>Configs:</th> + </tr> + {% for ci_file in host.config.cloud_init_files.all %} + <tr> + <td>{{ci_file.id}}</td> + <td> + <div class="modal fade" id="ci_file_modal_{{ci_file.id}}" tabindex="-1" role="dialog" aria-hidden="true"> + <div class="modal-dialog modal-xl" role="document"> + <div class="modal-content"> + <div class="modal-header"> + <h4 class="modal-title d-inline float-left">Cloud Config Content</h4> + <button type="button" class="close" data-dismiss="modal" aria-label="Close"> + <span aria-hidden="true">×</span> + </button> + </div> + <div class="card-body"> + <pre class="prettyprint lang-yaml m-0 border-0 text-break pre-wrap"> +{{ci_file.text}} + </pre> + </div> + </div> + </div> + </div> + <button class="btn btn-primary" data-toggle="modal" data-target="#ci_file_modal_{{ci_file.id}}">Show File Content</button> + </td> + </tr> + {% endfor %} + </table> + </td> + </tr> + {% endfor %} + </table> + </div> + </div> + </div> </div> <div class="col"> <div class="card mb-3"> diff --git a/src/templates/base/booking/booking_table.html b/src/templates/base/booking/booking_table.html index 1b95433..b4a713a 100644 --- a/src/templates/base/booking/booking_table.html +++ b/src/templates/base/booking/booking_table.html @@ -1,4 +1,4 @@ -{% load jira_filters %} + <thead> diff --git a/src/templates/base/booking/quick_deploy.html b/src/templates/base/booking/quick_deploy.html index 5dc41e2..c51e234 100644 --- a/src/templates/base/booking/quick_deploy.html +++ b/src/templates/base/booking/quick_deploy.html @@ -3,6 +3,13 @@ {% load bootstrap4 %} {% block content %} +<style> +/* hides images not in use. Not applied globally since doesn't make sense in all cases */ +select option:disabled { + display:none; +} +</style> + {% bootstrap_form_errors form type='non_fields' %} <form id="quick_booking_form" action="/booking/quick/" method="POST" class="form class="Anuket-Text""> {% csrf_token %} @@ -18,7 +25,7 @@ </div> </div> <div class="row justify-content-center"> - <div class="col-12 col-lg-4 my-2"> + <div class="col-12 col-lg-6 my-2"> <div class="col border rounded py-2 h-100"> {% bootstrap_field form.purpose %} {% bootstrap_field form.project %} @@ -31,19 +38,26 @@ </div> </div> {% block collab %} - <div class="col-12 col-lg-4 my-2"> + <div class="col-12 col-lg-6 my-2"> <div class="col border rounded py-2 h-100"> <label>Collaborators</label> {{ form.users }} </div> </div> {% endblock collab %} - <div class="col-12 col-lg-4 my-2"> + </div> + <div class="row justify-content-center"> + <div class="col-12 col-lg-6 my-2"> <div class="col border rounded py-2 h-100"> {% bootstrap_field form.hostname %} {% bootstrap_field form.image %} </div> </div> + <div class="col-12 col-lg-6 my-2"> + <div class="col border rounded py-2 h-100"> + {% bootstrap_field form.global_cloud_config %} + </div> + </div> <div class="col-12 d-flex mt-2 justify-content-end"> <button id="quick_booking_confirm" onclick="submit_form();" type="button" class="btn btn-success">Confirm</button> </div> @@ -88,15 +102,22 @@ function imageFilter() { var drop = document.getElementById("id_image"); var lab_pk = get_selected_value("lab"); - var host_pk = get_selected_value("resource"); + var profile_pk = get_selected_value("resource"); for (const childNode of drop.childNodes) { var image_object = sup_image_dict[childNode.value]; if (image_object) //weed out empty option { + console.log("image object:"); + console.log(image_object); const img_at_lab = image_object.lab == lab_pk; - const profiles = resource_profile_map[host_pk]; - const img_in_template = profiles && profiles.indexOf(image_object.host_profile) > -1 + const profiles = resource_profile_map[profile_pk]; + console.log("profiles are:"); + console.log(profiles); + console.log("profile map is:"); + console.log(resource_profile_map); + console.log("host profile is" + image_object.architecture); + const img_in_template = profiles && profiles.indexOf(image_object.architecture) > -1 childNode.disabled = !img_at_lab || !img_in_template; } } diff --git a/src/templates/lfedge/base.html b/src/templates/lfedge/base.html index 64c05a4..4413340 100644 --- a/src/templates/lfedge/base.html +++ b/src/templates/lfedge/base.html @@ -1,29 +1,15 @@ {% extends "base/base.html" %} {% load staticfiles %} {% block bgColor %} -<style> -.LFEdge { - background: #0049b0; - margin-left: -25px; -} +<link rel="stylesheet" href="{% static "css/lfedge.css" %}"> -.wtext { - font-size: 18px; - color: #FFFFFF; -} - -.wtext:hover { - color: #FFFFFF; - text-decoration: none; -} -</style> <nav class="navbar navbar-light LFEdge navbar-fixed-top border-bottom py-0 mb-0" role="navigation"> {% endblock bgColor %} {% block logo %} <div class="barClamp col-12 col-sm order-1 order-sm-2 text-center text-lg-left"> <a href="https://www.lfedge.org/" class="navbar-brand"> - <img src="{% static "img/lfedge-logo.png" %}"> + <img src="{% static "img/lfedge-logo.png" %}" alt="lfedge logo"> </a> <a class="wtext d-none d-lg-inline" href={% url 'dashboard:index' %}> diff --git a/src/templates/lfedge/booking/booking_table.html b/src/templates/lfedge/booking/booking_table.html index 4afb4d2..4020b5e 100644 --- a/src/templates/lfedge/booking/booking_table.html +++ b/src/templates/lfedge/booking/booking_table.html @@ -1,4 +1,4 @@ -{% load jira_filters %} + <thead> diff --git a/src/templates/lfedge/booking/quick_deploy.html b/src/templates/lfedge/booking/quick_deploy.html index dac3815..ccafd90 100644 --- a/src/templates/lfedge/booking/quick_deploy.html +++ b/src/templates/lfedge/booking/quick_deploy.html @@ -11,7 +11,7 @@ </p> {% endblock form-text %} {% block collab %} -<div class="col-12 col-lg-4 my-2"> +<div class="col-12 col-lg-6 my-2"> <div class="col border rounded py-2 h-100"> <label>Collaborators</label> {{ form.users }} @@ -21,8 +21,8 @@ {% block image_script %} <script type="text/javascript"> - document.getElementById("id_image").disabled = true; - document.getElementById("id_image").style.display = 'none'; - document.getElementById("id_image").previousElementSibling.style.display = 'none'; +// document.getElementById("id_image").disabled = true; +// document.getElementById("id_image").style.display = 'none'; +// document.getElementById("id_image").previousElementSibling.style.display = 'none'; </script> {% endblock image_script %} diff --git a/src/workflow/models.py b/src/workflow/models.py index f550a38..91a216c 100644 --- a/src/workflow/models.py +++ b/src/workflow/models.py @@ -160,7 +160,7 @@ class BookingAuthManager(): return True # admin override for this user if Booking.objects.filter(owner=booking.owner, end__gt=timezone.now()).count() >= 3: return False - if len(booking.resource.template.getResources()) < 2: + if len(booking.resource.template.get_required_resources()) < 2: return True # if they only have one server, we dont care if repo.BOOKING_INFO_FILE not in repo.el: return False # INFO file not provided diff --git a/src/workflow/resource_bundle_workflow.py b/src/workflow/resource_bundle_workflow.py index 63a9519..a461e9a 100644 --- a/src/workflow/resource_bundle_workflow.py +++ b/src/workflow/resource_bundle_workflow.py @@ -196,7 +196,7 @@ class Define_Software(WorkflowStep): for i, host_data in enumerate(hosts_data): host = ResourceConfiguration.objects.get(pk=host_data['host_id']) wrong_owner = Image.objects.exclude(owner=user).exclude(public=True) - wrong_host = Image.objects.exclude(host_type=host.profile) + wrong_host = Image.objects.exclude(architecture=host.profile.architecture) wrong_lab = Image.objects.exclude(from_lab=lab) excluded_images = wrong_owner | wrong_host | wrong_lab filter_data.append([]) |