From 25275685e9a735e51fae8b1a936ba5733f6fb770 Mon Sep 17 00:00:00 2001 From: Parker Berberian Date: Wed, 10 Oct 2018 16:06:47 -0400 Subject: Lab as a Service 2.0 See changes here: https://wiki.opnfv.org/display/INF/Pharos+Laas Change-Id: I59ada5f98e70a28d7f8c14eab3239597e236ca26 Signed-off-by: Sawyer Bergeron Signed-off-by: Parker Berberian --- dashboard/src/dashboard/__init__.py | 2 - dashboard/src/dashboard/admin.py | 6 +- dashboard/src/dashboard/context_processors.py | 12 + dashboard/src/dashboard/exceptions.py | 46 +++ dashboard/src/dashboard/migrations/0001_initial.py | 64 ---- .../migrations/0002_auto_20170505_0815.py | 42 --- .../migrations/0003_resource_resource_lab.py | 22 -- dashboard/src/dashboard/migrations/__init__.py | 10 - dashboard/src/dashboard/models.py | 90 +----- dashboard/src/dashboard/populate_db_iol.py | 346 +++++++++++++++++++++ dashboard/src/dashboard/tasks.py | 113 +++++-- .../src/dashboard/templatetags/jenkins_filters.py | 38 --- dashboard/src/dashboard/tests/test_models.py | 69 ---- dashboard/src/dashboard/tests/test_views.py | 75 ----- dashboard/src/dashboard/urls.py | 17 +- dashboard/src/dashboard/views.py | 196 +++++------- 16 files changed, 585 insertions(+), 563 deletions(-) create mode 100644 dashboard/src/dashboard/context_processors.py create mode 100644 dashboard/src/dashboard/exceptions.py delete mode 100644 dashboard/src/dashboard/migrations/0001_initial.py delete mode 100644 dashboard/src/dashboard/migrations/0002_auto_20170505_0815.py delete mode 100644 dashboard/src/dashboard/migrations/0003_resource_resource_lab.py delete mode 100644 dashboard/src/dashboard/migrations/__init__.py create mode 100644 dashboard/src/dashboard/populate_db_iol.py delete mode 100644 dashboard/src/dashboard/templatetags/jenkins_filters.py delete mode 100644 dashboard/src/dashboard/tests/test_models.py delete mode 100644 dashboard/src/dashboard/tests/test_views.py (limited to 'dashboard/src/dashboard') diff --git a/dashboard/src/dashboard/__init__.py b/dashboard/src/dashboard/__init__.py index b5914ce..b6fef6c 100644 --- a/dashboard/src/dashboard/__init__.py +++ b/dashboard/src/dashboard/__init__.py @@ -6,5 +6,3 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - - diff --git a/dashboard/src/dashboard/admin.py b/dashboard/src/dashboard/admin.py index 0bfdef8..43b5386 100644 --- a/dashboard/src/dashboard/admin.py +++ b/dashboard/src/dashboard/admin.py @@ -1,5 +1,6 @@ ############################################################################## # Copyright (c) 2016 Max Breitenfeldt and others. +# Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -10,11 +11,6 @@ from django.contrib import admin -from dashboard.models import * admin.site.site_header = "Pharos Dashboard Administration" admin.site.site_title = "Pharos Dashboard" - -admin.site.register(Resource) -admin.site.register(Server) -admin.site.register(ResourceStatus) diff --git a/dashboard/src/dashboard/context_processors.py b/dashboard/src/dashboard/context_processors.py new file mode 100644 index 0000000..32c70b8 --- /dev/null +++ b/dashboard/src/dashboard/context_processors.py @@ -0,0 +1,12 @@ +############################################################################## +# Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from django.conf import settings + +def debug(context): + return {'DEBUG': settings.DEBUG} diff --git a/dashboard/src/dashboard/exceptions.py b/dashboard/src/dashboard/exceptions.py new file mode 100644 index 0000000..bc3fcac --- /dev/null +++ b/dashboard/src/dashboard/exceptions.py @@ -0,0 +1,46 @@ +############################################################################## +# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class ResourceProvisioningException(Exception): + """ + Resources could not be provisioned + """ + pass + +class ModelValidationException(Exception): + """ + Validation before saving model returned issues + """ + pass + +class ResourceAvailabilityException(ResourceProvisioningException): + """ + Requested resources are not *currently* available + """ + pass + +class ResourceExistenceException(ResourceAvailabilityException): + """ + Requested resources do not exist or do not match any known resources + """ + pass + + +class NonUniqueHostnameException(Exception): + pass + +class InvalidHostnameException(Exception): + pass + +class InvalidVlanConfigurationException(Exception): + pass + +class NetworkExistsException(Exception): + pass diff --git a/dashboard/src/dashboard/migrations/0001_initial.py b/dashboard/src/dashboard/migrations/0001_initial.py deleted file mode 100644 index aaf3945..0000000 --- a/dashboard/src/dashboard/migrations/0001_initial.py +++ /dev/null @@ -1,64 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.10 on 2016-11-03 13:33 -from __future__ import unicode_literals - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('jenkins', '0001_initial'), - ] - - operations = [ - migrations.CreateModel( - name='Resource', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('name', models.CharField(max_length=100, unique=True)), - ('description', models.CharField(blank=True, max_length=300, null=True)), - ('url', models.CharField(blank=True, max_length=100, null=True)), - ('owner', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='user_lab_owner', to=settings.AUTH_USER_MODEL)), - ('slave', models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='jenkins.JenkinsSlave')), - ('vpn_users', models.ManyToManyField(blank=True, related_name='user_vpn_users', to=settings.AUTH_USER_MODEL)), - ], - options={ - 'db_table': 'resource', - }, - ), - migrations.CreateModel( - name='ResourceStatus', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('timestamp', models.DateTimeField(auto_now_add=True)), - ('type', models.CharField(max_length=20)), - ('title', models.CharField(max_length=50)), - ('content', models.CharField(max_length=5000)), - ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dashboard.Resource')), - ], - options={ - 'db_table': 'resource_status', - }, - ), - migrations.CreateModel( - name='Server', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('name', models.CharField(blank=True, max_length=100)), - ('model', models.CharField(blank=True, max_length=100)), - ('cpu', models.CharField(blank=True, max_length=100)), - ('ram', models.CharField(blank=True, max_length=100)), - ('storage', models.CharField(blank=True, max_length=100)), - ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dashboard.Resource')), - ], - options={ - 'db_table': 'server', - }, - ), - ] diff --git a/dashboard/src/dashboard/migrations/0002_auto_20170505_0815.py b/dashboard/src/dashboard/migrations/0002_auto_20170505_0815.py deleted file mode 100644 index 4285b88..0000000 --- a/dashboard/src/dashboard/migrations/0002_auto_20170505_0815.py +++ /dev/null @@ -1,42 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -# -*- coding: utf-8 -*- -# Generated by Django 1.10 on 2017-05-05 08:15 -from __future__ import unicode_literals - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('dashboard', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='resource', - name='dev_pod', - field=models.BooleanField(default=False), - ), - migrations.AlterField( - model_name='resource', - name='owner', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='user_lab_owner', to=settings.AUTH_USER_MODEL), - ), - migrations.AlterField( - model_name='resource', - name='slave', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='jenkins.JenkinsSlave'), - ), - ] diff --git a/dashboard/src/dashboard/migrations/0003_resource_resource_lab.py b/dashboard/src/dashboard/migrations/0003_resource_resource_lab.py deleted file mode 100644 index fff93fd..0000000 --- a/dashboard/src/dashboard/migrations/0003_resource_resource_lab.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.10 on 2018-01-10 16:36 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('account', '0002_auto_20180110_1636'), - ('dashboard', '0002_auto_20170505_0815'), - ] - - operations = [ - migrations.AddField( - model_name='resource', - name='resource_lab', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='lab_resource_set', to='account.Lab'), - ), - ] diff --git a/dashboard/src/dashboard/migrations/__init__.py b/dashboard/src/dashboard/migrations/__init__.py deleted file mode 100644 index b5914ce..0000000 --- a/dashboard/src/dashboard/migrations/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - diff --git a/dashboard/src/dashboard/models.py b/dashboard/src/dashboard/models.py index ff55232..f9bd07e 100644 --- a/dashboard/src/dashboard/models.py +++ b/dashboard/src/dashboard/models.py @@ -1,97 +1,9 @@ ############################################################################## # Copyright (c) 2016 Max Breitenfeldt and others. +# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## - - -from datetime import timedelta - -from django.contrib.auth.models import User -from django.db import models -from django.utils import timezone - -from jenkins.models import JenkinsSlave -from account.models import Lab - - -class Resource(models.Model): - id = models.AutoField(primary_key=True) - name = models.CharField(max_length=100, unique=True) - description = models.CharField(max_length=300, blank=True, null=True) - url = models.CharField(max_length=100, blank=True, null=True) - resource_lab = models.ForeignKey(Lab, related_name='lab_resource_set', null=True, blank=True) - owner = models.ForeignKey(User, related_name='user_lab_owner', null=True, blank=True) - vpn_users = models.ManyToManyField(User, related_name='user_vpn_users', blank=True) - slave = models.ForeignKey(JenkinsSlave, on_delete=models.DO_NOTHING, null=True, blank=True) - dev_pod = models.BooleanField(default=False) - - def get_booking_utilization(self, weeks): - """ - Return a dictionary containing the count of booked and free seconds for a resource in the - range [now,now + weeks] if weeks is positive, - or [now-weeks, now] if weeks is negative - """ - - length = timedelta(weeks=abs(weeks)) - now = timezone.now() - - start = now - end = now + length - if weeks < 0: - start = now - length - end = now - - bookings = self.booking_set.filter(start__lt=start + length, end__gt=start) - - booked_seconds = 0 - for booking in bookings: - booking_start = booking.start - booking_end = booking.end - if booking_start < start: - booking_start = start - if booking_end > end: - booking_end = start + length - total = booking_end - booking_start - booked_seconds += total.total_seconds() - - return {'booked_seconds': booked_seconds, - 'available_seconds': length.total_seconds() - booked_seconds} - - class Meta: - db_table = 'resource' - - def __str__(self): - return self.name - -class Server(models.Model): - id = models.AutoField(primary_key=True) - resource = models.ForeignKey(Resource, on_delete=models.CASCADE) - name = models.CharField(max_length=100, blank=True) - model = models.CharField(max_length=100, blank=True) - cpu = models.CharField(max_length=100, blank=True) - ram = models.CharField(max_length=100, blank=True) - storage = models.CharField(max_length=100, blank=True) - - class Meta: - db_table = 'server' - - def __str__(self): - return self.name - -class ResourceStatus(models.Model): - id = models.AutoField(primary_key=True) - resource = models.ForeignKey(Resource, on_delete=models.CASCADE) - timestamp = models.DateTimeField(auto_now_add=True) - type = models.CharField(max_length=20) - title = models.CharField(max_length=50) - content = models.CharField(max_length=5000) - - class Meta: - db_table = 'resource_status' - - def __str__(self): - return self.resource.name + ': ' + self.title + ' ' + str(self.timestamp) diff --git a/dashboard/src/dashboard/populate_db_iol.py b/dashboard/src/dashboard/populate_db_iol.py new file mode 100644 index 0000000..8c8b271 --- /dev/null +++ b/dashboard/src/dashboard/populate_db_iol.py @@ -0,0 +1,346 @@ +############################################################################## +# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from django.test import TestCase +from booking.models import Booking +from resource_inventory.models import * +from account.models import * +from api.serializers.booking_serializer import * +from datetime import timedelta +from django.utils import timezone +from django.contrib.auth.models import Permission, User +import json +import yaml + + +class Populator: + + def __init__(self): + self.host_profile_count = 0 + self.generic_host_count = 0 + self.host_profiles = [] + self.generic_bundle_count = 0 + self.booking_count = 0 + + + def make_host_profile(self, lab, data): + hostProfile = HostProfile.objects.create( + host_type=data['host']['type'], + name=data['host']['name'], + description=data['host']['description'] + ) + hostProfile.save() + + for iface_data in data['interfaces']: + + interfaceProfile = InterfaceProfile.objects.create( + speed=iface_data['speed'], + name=iface_data['name'], + host=hostProfile + ) + interfaceProfile.save() + + + for disk_data in data['disks']: + + diskProfile = DiskProfile.objects.create( + size=disk_data['size'], + media_type=disk_data['type'], + name=disk_data['name'], + host=hostProfile + ) + diskProfile.save() + + cpuProfile = CpuProfile.objects.create( + cores=data['cpu']['cores'], + architecture=data['cpu']['arch'], + cpus=data['cpu']['cpus'], + host=hostProfile + ) + cpuProfile.save() + ramProfile = RamProfile.objects.create( + amount=data['ram']['amount'], + channels=data['ram']['channels'], + host=hostProfile + ) + ramProfile.save() + hostProfile.labs.add(lab) + return hostProfile + + def make_users(self): + user_pberberian = User.objects.create(username="pberberian") + user_pberberian.save() + user_pberberian_prof = UserProfile.objects.create(user=user_pberberian) + user_pberberian_prof.save() + + user_sbergeron = User.objects.create(username="sbergeron") + user_sbergeron.save() + user_sbergeron_prof = UserProfile.objects.create(user=user_sbergeron) + user_sbergeron_prof.save() + return [user_sbergeron, user_pberberian,] + + + def make_labs(self): + unh_iol = User.objects.create(username="unh_iol") + unh_iol.save() + vlans = [] + reserved = [] + for i in range(1,4096): + vlans.append(1) + reserved.append(0) + # TODO: put reserved vlans here + iol = Lab.objects.create( + lab_user=unh_iol, + name="UNH_IOL", + vlan_manager=VlanManager.objects.create( + vlans = json.dumps(vlans), + reserved_vlans = json.dumps(reserved), + allow_overlapping = False, + block_size = 20, + ), + api_token = Lab.make_api_token(), + contact_email = "nfv-lab@iol.unh.edu", + location = "University of New Hampshire, Durham NH, 03824 USA" + ) + return [iol] + + + def make_configurations(self): + #scenarios + scen1 = Scenario.objects.create(name="os-nosdn-nofeature-noha") + scen2 = Scenario.objects.create(name="os-odl-kvm-ha") + scen3 = Scenario.objects.create(name="os-nosdn-nofeature-ha") + + #installers + fuel = Installer.objects.create(name="Fuel") + fuel.sup_scenarios.add(scen1) + fuel.sup_scenarios.add(scen3) + fuel.save() + joid = Installer.objects.create(name="Joid") + joid.sup_scenarios.add(scen1) + joid.sup_scenarios.add(scen2) + joid.save() + apex = Installer.objects.create(name="Apex") + apex.sup_scenarios.add(scen2) + apex.sup_scenarios.add(scen3) + apex.save() + daisy = Installer.objects.create(name="Daisy") + daisy.sup_scenarios.add(scen1) + daisy.sup_scenarios.add(scen2) + daisy.sup_scenarios.add(scen3) + daisy.save() + compass = Installer.objects.create(name="Compass") + compass.sup_scenarios.add(scen1) + compass.sup_scenarios.add(scen3) + compass.save() + + #operating systems + ubuntu = Opsys.objects.create(name="Ubuntu") + ubuntu.sup_installers.add(compass) + ubuntu.sup_installers.add(joid) + ubuntu.save() + centos = Opsys.objects.create(name="CentOs") + centos.sup_installers.add(apex) + centos.sup_installers.add(fuel) + centos.save() + suse = Opsys.objects.create(name="Suse") + suse.sup_installers.add(fuel) + suse.save() + + + #opnfv roles + compute = OPNFVRole.objects.create(name="Compute", description="Does the heavy lifting") + controller = OPNFVRole.objects.create(name="Controller", description="Controls everything") + jumphost = OPNFVRole.objects.create(name="Jumphost", description="Entry Point") + + lab = Lab.objects.first() + user = UserProfile.objects.first().user + image = Image.objects.create( + lab_id=23, + name="hpe centos", + from_lab=lab, + owner=user, + host_type=HostProfile.objects.get(name="hpe") + ) + image = Image.objects.create( + lab_id=25, + name="hpe ubuntu", + from_lab=lab, + owner=user, + host_type=HostProfile.objects.get(name="hpe") + ) + + image = Image.objects.create( + lab_id=26, + name="hpe suse", + from_lab=lab, + owner=user, + host_type=HostProfile.objects.get(name="hpe") + ) + image = Image.objects.create( + lab_id=27, + name="arm ubuntu", + from_lab=lab, + owner=user, + host_type=HostProfile.objects.get(name="arm") + ) + + def make_lab_hosts(self, hostcount, profile, lab, data, offset=1): + for i in range(hostcount): + name="Host_" + lab.name + "_" + profile.name + "_" + str(i + offset) + host = Host.objects.create( + name=name, + lab=lab, + profile=profile, + labid=data[i]['labid'] + ) + for iface_profile in profile.interfaceprofile.all(): + iface_data = data[i]['interfaces'][iface_profile.name] + Interface.objects.create( + mac_address=iface_data['mac'], + bus_address=iface_data['bus'], + name=iface_profile.name, + host=host + ) + + def make_profile_data(self): + """ + returns a dictionary of data from the yaml files + created by inspection scripts + """ + data = [] + for prof in ["hpe", "arm"]: # TODO + profile_dict = {} + host = { + "name": prof, + "type": 0, + "description": "some LaaS servers" + } + profile_dict['host'] = host + profile_dict['interfaces'] = [] + for interface in [{"name": "eno1", "speed": 1000}, {"name": "eno2", "speed": 10000}]: # TODO + iface_dict = {} + iface_dict["name"] = interface['name'] + iface_dict['speed'] = interface['speed'] + profile_dict['interfaces'].append(iface_dict) + + profile_dict['disks'] = [] + for disk in [{"size": 1000, "type": "ssd", "name": "sda"}]: # TODO + disk_dict = {} + disk_dict['size'] = disk['size'] + disk_dict['type'] = disk['type'] + disk_dict['name'] = disk['name'] + profile_dict['disks'].append(disk_dict) + + # cpu + cpu = {} + cpu['cores'] = 4 + cpu['arch'] = "x86" + cpu['cpus'] = 2 + profile_dict['cpu'] = cpu + + # ram + ram = {} + ram['amount'] = 256 + ram['channels'] = 4 + profile_dict['ram'] = ram + + data.append(profile_dict) + + return data + + def get_lab_data(self, lab): + data = {} + path = "/pharos_dashboard/data/" + lab.name + "/" + host_file = open(path + "hostlist.json") + host_structure = json.loads(host_file.read()) + host_file.close() + for profile in host_structure['profiles'].keys(): + data[profile] = {} + prof_path = path + profile + for host in host_structure['profiles'][profile]: + host_file = open(prof_path + "/" + host + ".yaml") + host_data = yaml.load(host_file.read()) + host_file.close() + data[profile][host] = host_data + return data + + def make_profiles_and_hosts(self, lab, lab_data): + for host_profile_name, host_data_dict in lab_data.items(): + if len(host_data_dict) < 1: + continue + host_profile = HostProfile.objects.create( + name=host_profile_name, + description="" + ) + host_profile.labs.add(lab) + example_host_data = list(host_data_dict.values())[0] + + cpu_data = example_host_data['cpu'] + CpuProfile.objects.create( + cores=cpu_data['cores'], + architecture=cpu_data['arch'], + cpus=cpu_data['cpus'], + host=host_profile + ) + + ram_data = example_host_data['memory'] + RamProfile.objects.create( + amount=int(ram_data[:-1]), + channels=1, + host=host_profile + ) + + disks_data = example_host_data['disk'] + for disk_data in disks_data: + size = 0 + try: + size=int(disk_data['size'].split('.')[0]) + except: + size=int(disk_data['size'].split('.')[0][:-1]) + DiskProfile.objects.create( + size=size, + media_type="SSD", + name=disk_data['name'], + host=host_profile + ) + + ifaces_data = example_host_data['interface'] + for iface_data in ifaces_data: + InterfaceProfile.objects.create( + speed=iface_data['speed'], + name=iface_data['name'], + host=host_profile + ) + + # all profiles created + for hostname, host_data in host_data_dict.items(): + host = Host.objects.create( + name=hostname, + labid=hostname, + profile=host_profile, + lab=lab + ) + for iface_data in host_data['interface']: + Interface.objects.create( + mac_address=iface_data['mac'], + bus_address=iface_data['busaddr'], + name=iface_data['name'], + host=host + ) + + def populate(self): + self.labs = self.make_labs() + # We should use the existing users, not creating our own + for lab in self.labs: + lab_data = self.get_lab_data(lab) + self.make_profiles_and_hosts(lab, lab_data) + + # We will add opnfv info and images as they are created and supported diff --git a/dashboard/src/dashboard/tasks.py b/dashboard/src/dashboard/tasks.py index fa2ee9d..827c7c5 100644 --- a/dashboard/src/dashboard/tasks.py +++ b/dashboard/src/dashboard/tasks.py @@ -1,5 +1,6 @@ ############################################################################## # Copyright (c) 2016 Max Breitenfeldt and others. +# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -7,32 +8,100 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from datetime import timedelta from celery import shared_task from django.utils import timezone -from django.conf import settings +from django.db.models import Q from booking.models import Booking +from notifier.manager import * +from notifier.models import * +from api.models import * +from resource_inventory.resource_manager import ResourceManager -from jenkins.models import JenkinsStatistic @shared_task -def database_cleanup(): - now = timezone.now() - JenkinsStatistic.objects.filter(timestamp__lt=now - timedelta(weeks=4)).delete() - -def booking_cleanup(): - expire_time = timedelta(days=int(settings.BOOKING_EXP_TIME)) - expire_number = int(settings.BOOKING_MAX_NUM) - expired_set = Booking.objects.filter(end__lte=timezone.now()) - expired_count = len(expired_set) - - for booking in expired_set: - if timezone.now() - booking.end > expire_time: - booking.delete() - expired_count = expired_count - 1 - - if expired_count > expire_number: - oldest = expired_set.order_by("end")[:expired_count-expire_number] - for booking in oldest: - booking.delete() +def conjure_aggregate_notifiers(): + NotifyPeriodic.task() + + +@shared_task +def booking_poll(): + def cleanup_hardware(qs): + for hostrelation in qs: + config = hostrelation.config + config.clear_delta() + config.set_power("off") + config.save() + hostrelation.status=JobStatus.NEW + hostrelation.save() + + def cleanup_network(qs): + for hostrelation in qs: + network = hostrelation.config + network.interfaces.clear() + host = hostrelation.host + network.clear_delta() + vlans = [] + for interface in host.interfaces.all(): + for vlan in interface.config: + if vlan.public: + try: + host.lab.vlan_manager.release_public_vlan(vlan.vlan_id) + except: # will fail if we already released in this loop + pass + else: + vlans.append(vlan.vlan_id) + + # release all vlans + if len(vlans) > 0: + host.lab.vlan_manager.release_vlans(vlans) + + interface.config.clear() + network.add_interface(interface) + network.save() + hostrelation.status=JobStatus.NEW + hostrelation.save() + + def cleanup_software(qs): + if qs.exists(): + relation = qs.first() + software = relation.config.opnfv + software.clear_delta() + software.save() + relation.status=JobStatus.NEW + relation.save() + + def cleanup_access(qs): + for relation in qs: + pass # TODO + + cleanup_set = Booking.objects.filter(end__lte=timezone.now()).filter(job__complete=False) + + for booking in cleanup_set: + if not booking.job.complete: + job = booking.job + cleanup_software(SoftwareRelation.objects.filter(job=job)) + cleanup_hardware(HostHardwareRelation.objects.filter(job=job)) + cleanup_network(HostNetworkRelation.objects.filter(job=job)) + cleanup_access(AccessRelation.objects.filter(job=job)) + job.complete = True + job.save() + + +@shared_task +def free_hosts(): + """ + gets all hosts from the database that need to be freed and frees them + """ + networks = ~Q(~Q(job__hostnetworkrelation__status=200)) + hardware = ~Q(~Q(job__hosthardwarerelation__status=200)) + + bookings = Booking.objects.filter( + networks, + hardware, + end__lt=timezone.now(), + job__complete=True, + resource__isnull=False + ) + for booking in bookings: + ResourceManager.getInstance().deleteResourceBundle(booking.resource) diff --git a/dashboard/src/dashboard/templatetags/jenkins_filters.py b/dashboard/src/dashboard/templatetags/jenkins_filters.py deleted file mode 100644 index e7e1425..0000000 --- a/dashboard/src/dashboard/templatetags/jenkins_filters.py +++ /dev/null @@ -1,38 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from django.template.defaultfilters import register - - -@register.filter -def jenkins_job_color(job_result): - if job_result == 'SUCCESS': - return '#5cb85c' - if job_result == 'FAILURE': - return '#d9534f' - if job_result == 'UNSTABLE': - return '#EDD62B' - return '#646F73' # job is still building - - -@register.filter -def jenkins_status_color(slave_status): - if slave_status == 'offline': - return '#d9534f' - if slave_status == 'online': - return '#5cb85c' - if slave_status == 'online / idle': - return '#5bc0de' - - -@register.filter -def jenkins_job_blink(job_result): - if job_result == '': # job is still building - return 'class=blink_me' diff --git a/dashboard/src/dashboard/tests/test_models.py b/dashboard/src/dashboard/tests/test_models.py deleted file mode 100644 index 3a3aeab..0000000 --- a/dashboard/src/dashboard/tests/test_models.py +++ /dev/null @@ -1,69 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from datetime import timedelta -from math import ceil, floor - -from django.test import TestCase -from django.utils import timezone - -from booking.models import * -from dashboard.models import Resource -from jenkins.models import JenkinsSlave - - -class ResourceModelTestCase(TestCase): - def setUp(self): - self.slave = JenkinsSlave.objects.create(name='test', url='test') - self.owner = User.objects.create(username='owner') - - self.res1 = Resource.objects.create(name='res1', slave=self.slave, description='x', - url='x', owner=self.owner) - - def test_booking_utilization(self): - utilization = self.res1.get_booking_utilization(1) - self.assertTrue(utilization['booked_seconds'] == 0) - self.assertTrue(utilization['available_seconds'] == timedelta(weeks=1).total_seconds()) - - start = timezone.now() + timedelta(days=1) - end = start + timedelta(days=1) - booking = Booking.objects.create(start=start, end=end, purpose='test', resource=self.res1, - user=self.owner) - - utilization = self.res1.get_booking_utilization(1) - booked_seconds = timedelta(days=1).total_seconds() - self.assertEqual(utilization['booked_seconds'], booked_seconds) - - utilization = self.res1.get_booking_utilization(-1) - self.assertEqual(utilization['booked_seconds'], 0) - - booking.delete() - start = timezone.now() - timedelta(days=1) - end = start + timedelta(days=2) - booking = Booking.objects.create(start=start, end=end, purpose='test', resource=self.res1, - user=self.owner) - booked_seconds = self.res1.get_booking_utilization(1)['booked_seconds'] - # use ceil because a fraction of the booked time has already passed now - booked_seconds = ceil(booked_seconds) - self.assertEqual(booked_seconds, timedelta(days=1).total_seconds()) - - booking.delete() - start = timezone.now() + timedelta(days=6) - end = start + timedelta(days=2) - booking = Booking.objects.create(start=start, end=end, purpose='test', resource=self.res1, - user=self.owner) - booked_seconds = self.res1.get_booking_utilization(1)['booked_seconds'] - booked_seconds = floor(booked_seconds) - self.assertEqual(booked_seconds, timedelta(days=1).total_seconds()) - - - - - diff --git a/dashboard/src/dashboard/tests/test_views.py b/dashboard/src/dashboard/tests/test_views.py deleted file mode 100644 index f5e17c2..0000000 --- a/dashboard/src/dashboard/tests/test_views.py +++ /dev/null @@ -1,75 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from django.test import TestCase -from django.urls import reverse - -from dashboard.models import Resource -from jenkins.models import JenkinsSlave - - -class DashboardViewTestCase(TestCase): - def setUp(self): - self.slave_active = JenkinsSlave.objects.create(name='slave_active', url='x', active=True) - self.slave_inactive = JenkinsSlave.objects.create(name='slave_inactive', url='x', - active=False) - self.res_active = Resource.objects.create(name='res_active', slave=self.slave_active, - description='x', url='x') - self.res_inactive = Resource.objects.create(name='res_inactive', slave=self.slave_inactive, - description='x', url='x') - - def test_booking_utilization_json(self): - url = reverse('dashboard:booking_utilization', kwargs={'resource_id': 0, 'weeks': 0}) - self.assertEqual(self.client.get(url).status_code, 404) - - url = reverse('dashboard:booking_utilization', kwargs={'resource_id': self.res_active.id, - 'weeks': 0}) - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertContains(response, 'data') - - def test_jenkins_utilization_json(self): - url = reverse('dashboard:jenkins_utilization', kwargs={'resource_id': 0, 'weeks': 0}) - self.assertEqual(self.client.get(url).status_code, 404) - - url = reverse('dashboard:jenkins_utilization', kwargs={'resource_id': self.res_active.id, - 'weeks': 0}) - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertContains(response, 'data') - - def test_jenkins_slaves_view(self): - url = reverse('dashboard:jenkins_slaves') - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertIn(self.slave_active, response.context['slaves']) - self.assertNotIn(self.slave_inactive, response.context['slaves']) - - def test_ci_pods_view(self): - url = reverse('dashboard:ci_pods') - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.context['ci_pods']), 0) - - self.slave_active.ci_slave = True - self.slave_inactive.ci_slave = True - self.slave_active.save() - self.slave_inactive.save() - - response = self.client.get(url) - self.assertIn(self.res_active, response.context['ci_pods']) - self.assertNotIn(self.res_inactive, response.context['ci_pods']) - - def test_dev_pods_view(self): - url = reverse('dashboard:dev_pods') - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertEqual(len(response.context['dev_pods']), 0) - diff --git a/dashboard/src/dashboard/urls.py b/dashboard/src/dashboard/urls.py index 609e5d6..0d7ee87 100644 --- a/dashboard/src/dashboard/urls.py +++ b/dashboard/src/dashboard/urls.py @@ -1,5 +1,6 @@ ############################################################################## # Copyright (c) 2016 Max Breitenfeldt and others. +# Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -24,18 +25,12 @@ Including another URLconf 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) """ from django.conf.urls import url - from dashboard.views import * +app_name="dashboard" urlpatterns = [ - url(r'^ci_pods/$', CIPodsView.as_view(), name='ci_pods'), - url(r'^dev_pods/$', DevelopmentPodsView.as_view(), name='dev_pods'), - url(r'^jenkins_slaves/$', JenkinsSlavesView.as_view(), name='jenkins_slaves'), - url(r'^resource/all/$', LabOwnerView.as_view(), name='resources'), - url(r'^resource/(?P[0-9]+)/$', ResourceView.as_view(), name='resource'), - url(r'^resource/(?P[0-9]+)/booking_utilization/(?P-?\d+)/$', - BookingUtilizationJSON.as_view(), name='booking_utilization'), - url(r'^resource/(?P[0-9]+)/jenkins_utilization/(?P-?\d+)/$', - JenkinsUtilizationJSON.as_view(), name='jenkins_utilization'), - url(r'^$', DevelopmentPodsView.as_view(), name="index"), + url(r'^$', landing_view, name='index'), + url(r'^lab/$', lab_list_view, name='all_labs'), + url(r'^lab/(?P.+)/$', lab_detail_view, name='lab_detail'), + url(r'^hosts/$', host_profile_detail_view, name="hostprofile_detail") ] diff --git a/dashboard/src/dashboard/views.py b/dashboard/src/dashboard/views.py index 4bab036..2d1f8b2 100644 --- a/dashboard/src/dashboard/views.py +++ b/dashboard/src/dashboard/views.py @@ -1,5 +1,6 @@ ############################################################################## # Copyright (c) 2016 Max Breitenfeldt and others. +# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 @@ -8,139 +9,106 @@ ############################################################################## -from datetime import timedelta - -from django.http import JsonResponse from django.shortcuts import get_object_or_404 -from django.utils import timezone -from django.views import View from django.views.generic import TemplateView +from django.shortcuts import render +from django.http import HttpResponseRedirect from booking.models import Booking -from dashboard.models import Resource -from jenkins.models import JenkinsSlave +from account.models import Lab +from resource_inventory.models import * +from workflow.views import * +from workflow.workflow_manager import * -class JenkinsSlavesView(TemplateView): - template_name = "dashboard/jenkins_slaves.html" - def get_context_data(self, **kwargs): - slaves = JenkinsSlave.objects.filter(active=True) - context = super(JenkinsSlavesView, self).get_context_data(**kwargs) - context.update({'title': "Jenkins Slaves", 'slaves': slaves}) - return context +def lab_list_view(request): + labs = Lab.objects.all() + context = {"labs": labs} + return render(request, "dashboard/lab_list.html", context) -class CIPodsView(TemplateView): - template_name = "dashboard/ci_pods.html" - def get_context_data(self, **kwargs): - ci_pods = Resource.objects.filter(slave__ci_slave=True, slave__active=True) - context = super(CIPodsView, self).get_context_data(**kwargs) - context.update({'title': "CI Pods", 'ci_pods': ci_pods}) - return context +def lab_detail_view(request, lab_name): + user = None + if request.user.is_authenticated: + user = request.user + lab = get_object_or_404(Lab, name=lab_name) -class DevelopmentPodsView(TemplateView): - template_name = "dashboard/dev_pods.html" + images = Image.objects.filter(from_lab=lab).filter(public=True) + if user: + images = images | Image.objects.filter(from_lab=lab).filter(owner=user) - def get_context_data(self, **kwargs): - resources = Resource.objects.filter(dev_pod=True) - - bookings = Booking.objects.filter(start__lte=timezone.now()) - bookings = bookings.filter(end__gt=timezone.now()) - - dev_pods = [] - for resource in resources: - booking_utilization = resource.get_booking_utilization(weeks=4) - total = booking_utilization['booked_seconds'] + booking_utilization['available_seconds'] - try: - utilization_percentage = "%d%%" % (float(booking_utilization['booked_seconds']) / - total * 100) - except (ValueError, ZeroDivisionError): - return "" - - dev_pod = (resource, None, utilization_percentage) - for booking in bookings: - if booking.resource == resource: - dev_pod = (resource, booking, utilization_percentage) - dev_pods.append(dev_pod) - - context = super(DevelopmentPodsView, self).get_context_data(**kwargs) - context.update({'title': "Development Pods", 'dev_pods': dev_pods}) - return context + return render(request, "dashboard/lab_detail.html", + {'title': "Lab Overview", + 'lab': lab, + 'hostprofiles': lab.hostprofiles.all(), + 'images': images}) -class ResourceView(TemplateView): - template_name = "dashboard/resource.html" +def host_profile_detail_view(request): - def get_context_data(self, **kwargs): - resource = get_object_or_404(Resource, id=self.kwargs['resource_id']) - bookings = Booking.objects.filter(resource=resource, end__gt=timezone.now()) - context = super(ResourceView, self).get_context_data(**kwargs) - context.update({'title': str(resource), 'resource': resource, 'bookings': bookings}) - return context + return render(request, "dashboard/host_profile_detail.html", + {'title': "Host Types", + }) -class LabOwnerView(TemplateView): - template_name = "dashboard/resource_all.html" +def landing_view(request): + manager = None + manager_detected = False + if 'manager_session' in request.session: - def get_context_data(self, **kwargs): - resources = Resource.objects.filter(slave__dev_pod=True, slave__active=True) - pods = [] - for resource in resources: - utilization = resource.slave.get_utilization(timedelta(days=7)) - bookings = Booking.objects.filter(resource=resource, end__gt=timezone.now()) - pods.append((resource, utilization, bookings)) - context = super(LabOwnerView, self).get_context_data(**kwargs) - context.update({'title': "Overview", 'pods': pods}) - return context + try: + manager = ManagerTracker.managers[request.session['manager_session']] + + + except KeyError as e: + pass + + if manager is not None: + #no manager detected, don't display continue button + manager_detected = True + if request.method == 'GET': + return render(request, 'dashboard/landing.html', {'manager': manager_detected, 'title': "Welcome!"}) -class BookingUtilizationJSON(View): - def get(self, request, *args, **kwargs): - resource = get_object_or_404(Resource, id=kwargs['resource_id']) - utilization = resource.get_booking_utilization(int(kwargs['weeks'])) - utilization = [ - { - 'label': 'Booked', - 'data': utilization['booked_seconds'], - 'color': '#d9534f' - }, - { - 'label': 'Available', - 'data': utilization['available_seconds'], - 'color': '#5cb85c' - }, - ] - return JsonResponse({'data': utilization}) - - -class JenkinsUtilizationJSON(View): - def get(self, request, *args, **kwargs): - resource = get_object_or_404(Resource, id=kwargs['resource_id']) - weeks = int(kwargs['weeks']) + if request.method == 'POST': try: - utilization = resource.slave.get_utilization(timedelta(weeks=weeks)) - utilization = [ - { - 'label': 'Offline', - 'data': utilization['offline'], - 'color': '#d9534f' - }, - { - 'label': 'Online', - 'data': utilization['online'], - 'color': '#5cb85c' - }, - { - 'label': 'Idle', - 'data': utilization['idle'], - 'color': '#5bc0de' - }, - ] - jutilization = JsonResponse({'data': utilization}) - except AttributeError: - return JsonResponse({'data': ''}) - if jutilization: - return jutilization + create = request.POST['create'] + + if manager is not None: + del manager + + mgr_uuid = create_session(create, request=request,) + request.session['manager_session'] = mgr_uuid + return HttpResponseRedirect('/wf/') + + except KeyError as e: + pass + + +class LandingView(TemplateView): + template_name = "dashboard/landing.html" + + def get_context_data(self, **kwargs): + context = super(LandingView, self).get_context_data(**kwargs) + + hosts = [] + + for host_profile in HostProfile.objects.all(): + name = host_profile.name + description = host_profile.description + in_labs = host_profile.labs + + interfaces = host_profile.interfaceprofile + storage = host_profile.storageprofile + cpu = host_profile.cpuprofile + ram = host_profile.ramprofile + + host = (name, description, in_labs, interfaces, storage, cpu, ram) + hosts.append(host) + + context.update({'hosts': hosts}) + + return context -- cgit 1.2.3-korg