diff options
Diffstat (limited to 'src/dashboard')
-rw-r--r-- | src/dashboard/__init__.py | 8 | ||||
-rw-r--r-- | src/dashboard/actions.py | 47 | ||||
-rw-r--r-- | src/dashboard/admin.py | 16 | ||||
-rw-r--r-- | src/dashboard/apps.py | 15 | ||||
-rw-r--r-- | src/dashboard/context_processors.py | 13 | ||||
-rw-r--r-- | src/dashboard/exceptions.py | 56 | ||||
-rw-r--r-- | src/dashboard/fixtures/dashboard.json | 164 | ||||
-rw-r--r-- | src/dashboard/models.py | 9 | ||||
-rw-r--r-- | src/dashboard/populate_db_iol.py | 350 | ||||
-rw-r--r-- | src/dashboard/tasks.py | 108 | ||||
-rw-r--r-- | src/dashboard/templatetags/__init__.py | 8 | ||||
-rw-r--r-- | src/dashboard/templatetags/jira_filters.py | 17 | ||||
-rw-r--r-- | src/dashboard/testing_utils.py | 396 | ||||
-rw-r--r-- | src/dashboard/tests/__init__.py | 8 | ||||
-rw-r--r-- | src/dashboard/urls.py | 41 | ||||
-rw-r--r-- | src/dashboard/views.py | 121 |
16 files changed, 1377 insertions, 0 deletions
diff --git a/src/dashboard/__init__.py b/src/dashboard/__init__.py new file mode 100644 index 0000000..b6fef6c --- /dev/null +++ b/src/dashboard/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/src/dashboard/actions.py b/src/dashboard/actions.py new file mode 100644 index 0000000..44b1fdd --- /dev/null +++ b/src/dashboard/actions.py @@ -0,0 +1,47 @@ +############################################################################## +# Copyright (c) 2019 Parker Berberian, Sawyer Bergeron, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from resource_inventory.models import Host, Vlan +from account.models import Lab +from booking.models import Booking +from datetime import timedelta +from django.utils import timezone + + +def free_leaked_hosts(free_old_bookings=False, old_booking_age=timedelta(days=1)): + bundles = [booking.resource for booking in Booking.objects.filter(end__gt=timezone.now())] + active_hosts = set() + for bundle in bundles: + active_hosts.update([host for host in bundle.hosts.all()]) + + marked_hosts = set(Host.objects.filter(booked=True)) + + for host in (marked_hosts - active_hosts): + host.booked = False + host.save() + + +def free_leaked_public_vlans(): + booked_host_interfaces = [] + + for lab in Lab.objects.all(): + + for host in Host.objects.filter(booked=True).filter(lab=lab): + for interface in host.interfaces.all(): + booked_host_interfaces.append(interface) + + in_use_vlans = Vlan.objects.filter(public=True).distinct('vlan_id').filter(interface__in=booked_host_interfaces) + + manager = lab.vlan_manager + + for vlan in Vlan.objects.all(): + if vlan not in in_use_vlans: + if vlan.public: + manager.release_public_vlan(vlan.vlan_id) + manager.release_vlans(vlan) diff --git a/src/dashboard/admin.py b/src/dashboard/admin.py new file mode 100644 index 0000000..43b5386 --- /dev/null +++ b/src/dashboard/admin.py @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from django.contrib import admin + + +admin.site.site_header = "Pharos Dashboard Administration" +admin.site.site_title = "Pharos Dashboard" diff --git a/src/dashboard/apps.py b/src/dashboard/apps.py new file mode 100644 index 0000000..e0c4f44 --- /dev/null +++ b/src/dashboard/apps.py @@ -0,0 +1,15 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from django.apps import AppConfig + + +class DashboardConfig(AppConfig): + name = 'dashboard' diff --git a/src/dashboard/context_processors.py b/src/dashboard/context_processors.py new file mode 100644 index 0000000..338f609 --- /dev/null +++ b/src/dashboard/context_processors.py @@ -0,0 +1,13 @@ +############################################################################## +# Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from django.conf import settings + + +def debug(context): + return {'DEBUG': settings.DEBUG} diff --git a/src/dashboard/exceptions.py b/src/dashboard/exceptions.py new file mode 100644 index 0000000..7111bf8 --- /dev/null +++ b/src/dashboard/exceptions.py @@ -0,0 +1,56 @@ +############################################################################## +# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +class ResourceProvisioningException(Exception): + """ + Resources could not be provisioned + """ + pass + + +class ModelValidationException(Exception): + """ + Validation before saving model returned issues + """ + pass + + +class ResourceAvailabilityException(ResourceProvisioningException): + """ + Requested resources are not *currently* available + """ + pass + + +class ResourceExistenceException(ResourceAvailabilityException): + """ + Requested resources do not exist or do not match any known resources + """ + pass + + +class NonUniqueHostnameException(Exception): + pass + + +class InvalidHostnameException(Exception): + pass + + +class InvalidVlanConfigurationException(Exception): + pass + + +class NetworkExistsException(Exception): + pass + + +class BookingLengthException(Exception): + pass diff --git a/src/dashboard/fixtures/dashboard.json b/src/dashboard/fixtures/dashboard.json new file mode 100644 index 0000000..f0ac3b2 --- /dev/null +++ b/src/dashboard/fixtures/dashboard.json @@ -0,0 +1,164 @@ +[ +{ + "model": "dashboard.resource", + "pk": 1, + "fields": { + "name": "Linux Foundation POD 1", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab" + } +}, +{ + "model": "dashboard.resource", + "pk": 2, + "fields": { + "name": "Linux Foundation POD 2", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab" + } +}, +{ + "model": "dashboard.resource", + "pk": 3, + "fields": { + "name": "Ericsson POD 2", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process" + } +}, +{ + "model": "dashboard.resource", + "pk": 4, + "fields": { + "name": "Intel POD 2", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod2" + } +}, +{ + "model": "dashboard.resource", + "pk": 5, + "fields": { + "name": "Intel POD 5", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod5" + } +}, +{ + "model": "dashboard.resource", + "pk": 6, + "fields": { + "name": "Intel POD 6", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod6" + } +}, +{ + "model": "dashboard.resource", + "pk": 7, + "fields": { + "name": "Intel POD 8", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod8" + } +}, +{ + "model": "dashboard.resource", + "pk": 8, + "fields": { + "name": "Huawei POD 1", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" + } +}, +{ + "model": "dashboard.resource", + "pk": 9, + "fields": { + "name": "Intel POD 3", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod3" + } +}, +{ + "model": "dashboard.resource", + "pk": 10, + "fields": { + "name": "Dell POD 1", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting" + } +}, +{ + "model": "dashboard.resource", + "pk": 11, + "fields": { + "name": "Dell POD 2", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting" + } +}, +{ + "model": "dashboard.resource", + "pk": 12, + "fields": { + "name": "Orange POD 2", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Opnfv-orange-pod2" + } +}, +{ + "model": "dashboard.resource", + "pk": 13, + "fields": { + "name": "Arm POD 1", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Enea-pharos-lab" + } +}, +{ + "model": "dashboard.resource", + "pk": 14, + "fields": { + "name": "Ericsson POD 1", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process" + } +}, +{ + "model": "dashboard.resource", + "pk": 15, + "fields": { + "name": "Huawei POD 2", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" + } +}, +{ + "model": "dashboard.resource", + "pk": 16, + "fields": { + "name": "Huawei POD 3", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" + } +}, +{ + "model": "dashboard.resource", + "pk": 17, + "fields": { + "name": "Huawei POD 4", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" + } +}, +{ + "model": "dashboard.resource", + "pk": 18, + "fields": { + "name": "Intel POD 9", + "description": "Some description", + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod9" + } +} +] diff --git a/src/dashboard/models.py b/src/dashboard/models.py new file mode 100644 index 0000000..f9bd07e --- /dev/null +++ b/src/dashboard/models.py @@ -0,0 +1,9 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/src/dashboard/populate_db_iol.py b/src/dashboard/populate_db_iol.py new file mode 100644 index 0000000..916dd97 --- /dev/null +++ b/src/dashboard/populate_db_iol.py @@ -0,0 +1,350 @@ +############################################################################## +# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import json +import yaml + +from account.models import Lab, UserProfile +from django.contrib.auth.models import User +from resource_inventory.models import ( + HostProfile, + InterfaceProfile, + DiskProfile, + CpuProfile, + RamProfile, + VlanManager, + Scenario, + Installer, + Opsys, + OPNFVRole, + Image, + Interface, + Host +) + + +class Populator: + + def __init__(self): + self.host_profile_count = 0 + self.generic_host_count = 0 + self.host_profiles = [] + self.generic_bundle_count = 0 + self.booking_count = 0 + + def make_host_profile(self, lab, data): + hostProfile = HostProfile.objects.create( + host_type=data['host']['type'], + name=data['host']['name'], + description=data['host']['description'] + ) + hostProfile.save() + + for iface_data in data['interfaces']: + + interfaceProfile = InterfaceProfile.objects.create( + speed=iface_data['speed'], + name=iface_data['name'], + host=hostProfile + ) + interfaceProfile.save() + + for disk_data in data['disks']: + + diskProfile = DiskProfile.objects.create( + size=disk_data['size'], + media_type=disk_data['type'], + name=disk_data['name'], + host=hostProfile + ) + diskProfile.save() + + cpuProfile = CpuProfile.objects.create( + cores=data['cpu']['cores'], + architecture=data['cpu']['arch'], + cpus=data['cpu']['cpus'], + host=hostProfile + ) + cpuProfile.save() + ramProfile = RamProfile.objects.create( + amount=data['ram']['amount'], + channels=data['ram']['channels'], + host=hostProfile + ) + ramProfile.save() + hostProfile.labs.add(lab) + return hostProfile + + def make_users(self): + user_pberberian = User.objects.create(username="pberberian") + user_pberberian.save() + user_pberberian_prof = UserProfile.objects.create(user=user_pberberian) + user_pberberian_prof.save() + + user_sbergeron = User.objects.create(username="sbergeron") + user_sbergeron.save() + user_sbergeron_prof = UserProfile.objects.create(user=user_sbergeron) + user_sbergeron_prof.save() + return [user_sbergeron, user_pberberian] + + def make_labs(self): + unh_iol = User.objects.create(username="unh_iol") + unh_iol.save() + vlans = [] + reserved = [] + for i in range(1, 4096): + vlans.append(1) + reserved.append(0) + iol = Lab.objects.create( + lab_user=unh_iol, + name="UNH_IOL", + vlan_manager=VlanManager.objects.create( + vlans=json.dumps(vlans), + reserved_vlans=json.dumps(reserved), + allow_overlapping=False, + block_size=20, + ), + api_token=Lab.make_api_token(), + contact_email="nfv-lab@iol.unh.edu", + location="University of New Hampshire, Durham NH, 03824 USA" + ) + return [iol] + + def make_configurations(self): + # scenarios + scen1 = Scenario.objects.create(name="os-nosdn-nofeature-noha") + scen2 = Scenario.objects.create(name="os-odl-kvm-ha") + scen3 = Scenario.objects.create(name="os-nosdn-nofeature-ha") + + # installers + fuel = Installer.objects.create(name="Fuel") + fuel.sup_scenarios.add(scen1) + fuel.sup_scenarios.add(scen3) + fuel.save() + joid = Installer.objects.create(name="Joid") + joid.sup_scenarios.add(scen1) + joid.sup_scenarios.add(scen2) + joid.save() + apex = Installer.objects.create(name="Apex") + apex.sup_scenarios.add(scen2) + apex.sup_scenarios.add(scen3) + apex.save() + daisy = Installer.objects.create(name="Daisy") + daisy.sup_scenarios.add(scen1) + daisy.sup_scenarios.add(scen2) + daisy.sup_scenarios.add(scen3) + daisy.save() + compass = Installer.objects.create(name="Compass") + compass.sup_scenarios.add(scen1) + compass.sup_scenarios.add(scen3) + compass.save() + + # operating systems + ubuntu = Opsys.objects.create(name="Ubuntu") + ubuntu.sup_installers.add(compass) + ubuntu.sup_installers.add(joid) + ubuntu.save() + centos = Opsys.objects.create(name="CentOs") + centos.sup_installers.add(apex) + centos.sup_installers.add(fuel) + centos.save() + suse = Opsys.objects.create(name="Suse") + suse.sup_installers.add(fuel) + suse.save() + + # opnfv roles + OPNFVRole.objects.create(name="Compute", description="Does the heavy lifting") + OPNFVRole.objects.create(name="Controller", description="Controls everything") + OPNFVRole.objects.create(name="Jumphost", description="Entry Point") + + lab = Lab.objects.first() + user = UserProfile.objects.first().user + Image.objects.create( + lab_id=23, + name="hpe centos", + from_lab=lab, + owner=user, + host_type=HostProfile.objects.get(name="hpe") + ) + Image.objects.create( + lab_id=25, + name="hpe ubuntu", + from_lab=lab, + owner=user, + host_type=HostProfile.objects.get(name="hpe") + ) + + Image.objects.create( + lab_id=26, + name="hpe suse", + from_lab=lab, + owner=user, + host_type=HostProfile.objects.get(name="hpe") + ) + + Image.objects.create( + lab_id=27, + name="arm ubuntu", + from_lab=lab, + owner=user, + host_type=HostProfile.objects.get(name="arm") + ) + + def make_lab_hosts(self, hostcount, profile, lab, data, offset=1): + for i in range(hostcount): + name = "Host_" + lab.name + "_" + profile.name + "_" + str(i + offset) + host = Host.objects.create( + name=name, + lab=lab, + profile=profile, + labid=data[i]['labid'] + ) + for iface_profile in profile.interfaceprofile.all(): + iface_data = data[i]['interfaces'][iface_profile.name] + Interface.objects.create( + mac_address=iface_data['mac'], + bus_address=iface_data['bus'], + name=iface_profile.name, + host=host + ) + + def make_profile_data(self): + """ + returns a dictionary of data from the yaml files + created by inspection scripts + """ + data = [] + for prof in ["hpe", "arm"]: # TODO + profile_dict = {} + host = { + "name": prof, + "type": 0, + "description": "some LaaS servers" + } + profile_dict['host'] = host + profile_dict['interfaces'] = [] + for interface in [{"name": "eno1", "speed": 1000}, {"name": "eno2", "speed": 10000}]: # TODO + iface_dict = {} + iface_dict["name"] = interface['name'] + iface_dict['speed'] = interface['speed'] + profile_dict['interfaces'].append(iface_dict) + + profile_dict['disks'] = [] + for disk in [{"size": 1000, "type": "ssd", "name": "sda"}]: # TODO + disk_dict = {} + disk_dict['size'] = disk['size'] + disk_dict['type'] = disk['type'] + disk_dict['name'] = disk['name'] + profile_dict['disks'].append(disk_dict) + + # cpu + cpu = {} + cpu['cores'] = 4 + cpu['arch'] = "x86" + cpu['cpus'] = 2 + profile_dict['cpu'] = cpu + + # ram + ram = {} + ram['amount'] = 256 + ram['channels'] = 4 + profile_dict['ram'] = ram + + data.append(profile_dict) + + return data + + def get_lab_data(self, lab): + data = {} + path = "/pharos_dashboard/data/" + lab.name + "/" + host_file = open(path + "hostlist.json") + host_structure = json.loads(host_file.read()) + host_file.close() + for profile in host_structure['profiles'].keys(): + data[profile] = {} + prof_path = path + profile + for host in host_structure['profiles'][profile]: + host_file = open(prof_path + "/" + host + ".yaml") + host_data = yaml.load(host_file.read()) + host_file.close() + data[profile][host] = host_data + return data + + def make_profiles_and_hosts(self, lab, lab_data): + for host_profile_name, host_data_dict in lab_data.items(): + if len(host_data_dict) < 1: + continue + host_profile = HostProfile.objects.create( + name=host_profile_name, + description="" + ) + host_profile.labs.add(lab) + example_host_data = list(host_data_dict.values())[0] + + cpu_data = example_host_data['cpu'] + CpuProfile.objects.create( + cores=cpu_data['cores'], + architecture=cpu_data['arch'], + cpus=cpu_data['cpus'], + host=host_profile + ) + + ram_data = example_host_data['memory'] + RamProfile.objects.create( + amount=int(ram_data[:-1]), + channels=1, + host=host_profile + ) + + disks_data = example_host_data['disk'] + for disk_data in disks_data: + size = 0 + try: + size = int(disk_data['size'].split('.')[0]) + except Exception: + size = int(disk_data['size'].split('.')[0][:-1]) + DiskProfile.objects.create( + size=size, + media_type="SSD", + name=disk_data['name'], + host=host_profile + ) + + ifaces_data = example_host_data['interface'] + for iface_data in ifaces_data: + InterfaceProfile.objects.create( + speed=iface_data['speed'], + name=iface_data['name'], + host=host_profile + ) + + # all profiles created + for hostname, host_data in host_data_dict.items(): + host = Host.objects.create( + name=hostname, + labid=hostname, + profile=host_profile, + lab=lab + ) + for iface_data in host_data['interface']: + Interface.objects.create( + mac_address=iface_data['mac'], + bus_address=iface_data['busaddr'], + name=iface_data['name'], + host=host + ) + + def populate(self): + self.labs = self.make_labs() + # We should use the existing users, not creating our own + for lab in self.labs: + lab_data = self.get_lab_data(lab) + self.make_profiles_and_hosts(lab, lab_data) + + # We will add opnfv info and images as they are created and supported diff --git a/src/dashboard/tasks.py b/src/dashboard/tasks.py new file mode 100644 index 0000000..597629f --- /dev/null +++ b/src/dashboard/tasks.py @@ -0,0 +1,108 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from celery import shared_task +from django.utils import timezone +from booking.models import Booking +from notifier.manager import NotificationHandler +from api.models import Job, JobStatus, SoftwareRelation, HostHardwareRelation, HostNetworkRelation, AccessRelation +from resource_inventory.resource_manager import ResourceManager + + +@shared_task +def booking_poll(): + def cleanup_hardware(qs): + for hostrelation in qs: + config = hostrelation.config + config.clear_delta() + config.set_power("off") + config.save() + hostrelation.status = JobStatus.NEW + hostrelation.save() + + def cleanup_network(qs): + for hostrelation in qs: + network = hostrelation.config + network.interfaces.clear() + host = hostrelation.host + network.clear_delta() + vlans = [] + for interface in host.interfaces.all(): + for vlan in interface.config.all(): + if vlan.public: + try: + host.lab.vlan_manager.release_public_vlan(vlan.vlan_id) + except Exception: # will fail if we already released in this loop + pass + else: + vlans.append(vlan.vlan_id) + + # release all vlans + if len(vlans) > 0: + host.lab.vlan_manager.release_vlans(vlans) + + interface.config.clear() + network.add_interface(interface) + network.save() + hostrelation.status = JobStatus.NEW + hostrelation.save() + + def cleanup_software(qs): + if qs.exists(): + relation = qs.first() + software = relation.config.opnfv + software.clear_delta() + software.save() + relation.status = JobStatus.NEW + relation.save() + + def cleanup_access(qs): + for relation in qs: + if "vpn" in relation.config.access_type.lower(): + relation.config.set_revoke(True) + relation.config.save() + relation.status = JobStatus.NEW + relation.save() + + cleanup_set = Booking.objects.filter(end__lte=timezone.now()).filter(job__complete=False) + + for booking in cleanup_set: + if not booking.job.complete: + job = booking.job + cleanup_software(SoftwareRelation.objects.filter(job=job)) + cleanup_hardware(HostHardwareRelation.objects.filter(job=job)) + cleanup_network(HostNetworkRelation.objects.filter(job=job)) + cleanup_access(AccessRelation.objects.filter(job=job)) + job.complete = True + job.save() + NotificationHandler.notify_booking_end(booking) + + +@shared_task +def free_hosts(): + """ + gets all hosts from the database that need to be freed and frees them + """ + undone_statuses = [JobStatus.NEW, JobStatus.CURRENT, JobStatus.ERROR] + undone_jobs = Job.objects.filter( + hostnetworkrelation__status__in=undone_statuses, + hosthardwarerelation__status__in=undone_statuses + ) + + bookings = Booking.objects.exclude( + job__in=undone_jobs + ).filter( + end__lt=timezone.now(), + job__complete=True, + resource__isnull=False + ) + for booking in bookings: + ResourceManager.getInstance().deleteResourceBundle(booking.resource) diff --git a/src/dashboard/templatetags/__init__.py b/src/dashboard/templatetags/__init__.py new file mode 100644 index 0000000..b6fef6c --- /dev/null +++ b/src/dashboard/templatetags/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/src/dashboard/templatetags/jira_filters.py b/src/dashboard/templatetags/jira_filters.py new file mode 100644 index 0000000..9a97c1d --- /dev/null +++ b/src/dashboard/templatetags/jira_filters.py @@ -0,0 +1,17 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from django.conf import settings +from django.template.defaultfilters import register + + +@register.filter +def jira_issue_url(issue): + return settings.JIRA_URL + '/browse/' + str(issue) diff --git a/src/dashboard/testing_utils.py b/src/dashboard/testing_utils.py new file mode 100644 index 0000000..a96b6d0 --- /dev/null +++ b/src/dashboard/testing_utils.py @@ -0,0 +1,396 @@ +############################################################################## +# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from django.contrib.auth.models import User +from django.core.files.base import ContentFile +from django.utils import timezone + +import json +import re +from datetime import timedelta + +from dashboard.exceptions import InvalidHostnameException +from booking.models import Booking +from account.models import UserProfile, Lab, LabStatus, VlanManager, PublicNetwork +from resource_inventory.models import ( + Host, + HostProfile, + InterfaceProfile, + DiskProfile, + CpuProfile, + Opsys, + Image, + Scenario, + Installer, + OPNFVRole, + RamProfile, + Network, + GenericResourceBundle, + GenericResource, + GenericHost, + ConfigBundle, + GenericInterface, + HostConfiguration, + OPNFVConfig, + NetworkConnection, + HostOPNFVConfig +) +from resource_inventory.resource_manager import ResourceManager + +""" +Info for make_booking() function: +[topology] argument structure: + the [topology] argument should describe the structure of the pod + the top level should be a dictionary, with each key being a hostname + each value in the top level should be a dictionary with two keys: + "type" should map to a host profile instance + "nets" should map to a list of interfaces each with a list of + dictionaries each defining a network in the format + { "name": "netname", "tagged": True|False, "public": True|False } + each network is defined if a matching name is not found + + sample argument structure: + topology={ + "host1": { + "type": instanceOf HostProfile, + "role": instanceOf OPNFVRole + "image": instanceOf Image + "nets": [ + 0: [ + 0: { "name": "public", "tagged": True, "public": True }, + 1: { "name": "private", "tagged": False, "public": False }, + ] + 1: [] + ] + } + } +""" + + +def make_booking(owner=None, start=timezone.now(), + end=timezone.now() + timedelta(days=1), + lab=None, purpose="my_purpose", + project="my_project", collaborators=[], + topology={}, installer=None, scenario=None): + + grb, host_set = make_grb(topology, owner, lab) + config_bundle, opnfv_bundle = make_config_bundle(grb, owner, topology, host_set, installer, scenario) + resource = ResourceManager.getInstance().convertResourceBundle(grb, config=config_bundle) + if not resource: + raise Exception("Resource not created") + + return Booking.objects.create( + resource=resource, + config_bundle=config_bundle, + start=start, + end=end, + owner=owner, + purpose=purpose, + project=project, + lab=lab, + opnfv_config=opnfv_bundle + ) + + +def make_config_bundle(grb, owner, topology={}, host_set={}, + installer=None, scenario=None): + cb = ConfigBundle.objects.create( + owner=owner, + name="config bundle " + str(ConfigBundle.objects.count()), + description="cb generated by make_config_bundle() method" + ) + + opnfv_config = OPNFVConfig.objects.create( + installer=installer, + scenario=scenario, + bundle=cb + ) + + # generate host configurations based on topology and host set + for hostname, host_info in topology.items(): + host_config = HostConfiguration.objects.create( + host=host_set[hostname], + image=host_info["image"], + bundle=cb, + is_head_node=host_info['role'].name.lower() == "jumphost" + ) + HostOPNFVConfig.objects.create( + role=host_info["role"], + host_config=host_config, + opnfv_config=opnfv_config + ) + return cb, opnfv_config + + +def make_network(name, lab, grb, public): + network = Network(name=name, bundle=grb, is_public=public) + if public: + public_net = lab.vlan_manager.get_public_vlan() + if not public_net: + raise Exception("No more public networks available") + lab.vlan_manager.reserve_public_vlan(public_net.vlan) + network.vlan_id = public_net.vlan + else: + private_net = lab.vlan_manager.get_vlan() + if not private_net: + raise Exception("No more generic vlans are available") + lab.vlan_manager.reserve_vlans([private_net]) + network.vlan_id = private_net + + network.save() + return network + + +def make_grb(topology, owner, lab): + + grb = GenericResourceBundle.objects.create( + owner=owner, + lab=lab, + name="Generic ResourceBundle " + str(GenericResourceBundle.objects.count()), + description="grb generated by make_grb() method" + ) + + networks = {} + host_set = {} + + for hostname, info in topology.items(): + host_profile = info["type"] + + # need to construct host from hostname and type + generic_host = make_generic_host(grb, host_profile, hostname) + host_set[hostname] = generic_host + + # set up networks + nets = info["nets"] + for interface_index, interface_profile in enumerate(host_profile.interfaceprofile.all()): + generic_interface = GenericInterface.objects.create(host=generic_host, profile=interface_profile) + netconfig = nets[interface_index] + for network_info in netconfig: + network_name = network_info["name"] + if network_name not in networks: + networks[network_name] = make_network(network_name, lab, grb, network_info['public']) + + generic_interface.connections.add(NetworkConnection.objects.create( + network=networks[network_name], + vlan_is_tagged=network_info["tagged"] + )) + + return grb, host_set + + +def make_generic_host(grb, host_profile, hostname): + if not re.match(r"(?=^.{1,253}$)(^([A-Za-z0-9-_]{1,62}\.)*[A-Za-z0-9-_]{1,63})$", hostname): + raise InvalidHostnameException("Hostname must comply to RFC 952 and all extensions") + gresource = GenericResource.objects.create(bundle=grb, name=hostname) + + return GenericHost.objects.create(resource=gresource, profile=host_profile) + + +def make_user(is_superuser=False, username="testuser", + password="testpassword", email="default_email@user.com"): + user = User.objects.create_user(username=username, email=email, password=password) + user.is_superuser = is_superuser + user.save() + + return user + + +def make_user_profile(user=None, email_addr="email@email.com", + company="company", full_name="John Doe", + booking_privledge=True, ssh_file=None): + user = user or User.objects.first() or make_user() + profile = UserProfile.objects.create( + email_addr=email_addr, + company=company, + full_name=full_name, + booking_privledge=booking_privledge, + user=user + ) + profile.ssh_public_key.save("user_ssh_key", ssh_file if ssh_file else ContentFile("public key content string")) + + return profile + + +def make_vlan_manager(vlans=None, block_size=20, allow_overlapping=False, reserved_vlans=None): + if not vlans: + vlans = [vlan % 2 for vlan in range(4095)] + if not reserved_vlans: + reserved_vlans = [0 for i in range(4095)] + + return VlanManager.objects.create( + vlans=json.dumps(vlans), + reserved_vlans=json.dumps(vlans), + block_size=block_size, + allow_overlapping=allow_overlapping + ) + + +def make_lab(user=None, name="Test_Lab_Instance", + status=LabStatus.UP, vlan_manager=None, + pub_net_count=5): + if not vlan_manager: + vlan_manager = make_vlan_manager() + + if not user: + user = make_user() + + lab = Lab.objects.create( + lab_user=user, + name=name, + contact_email='test_lab@test_site.org', + contact_phone='603 123 4567', + status=status, + vlan_manager=vlan_manager, + description='test lab instantiation', + api_token='12345678' + ) + + for i in range(pub_net_count): + make_public_net(vlan=i * 2 + 1, lab=lab) + + return lab + + +""" +resource_inventory instantiation section for permanent resources +""" + + +def make_complete_host_profile(lab, name="test_hostprofile"): + host_profile = make_host_profile(lab, name=name) + make_disk_profile(host_profile, 500, name=name) + make_cpu_profile(host_profile) + make_interface_profile(host_profile, name=name) + make_ram_profile(host_profile) + + return host_profile + + +def make_host_profile(lab, host_type=0, name="test hostprofile"): + host_profile = HostProfile.objects.create( + host_type=host_type, + name=name, + description='test hostprofile instance' + ) + host_profile.labs.add(lab) + + return host_profile + + +def make_ram_profile(host, channels=4, amount=256): + return RamProfile.objects.create( + host=host, + amount=amount, + channels=channels + ) + + +def make_disk_profile(hostprofile, size=0, media_type="SSD", + name="test diskprofile", rotation=0, + interface="sata"): + return DiskProfile.objects.create( + name=name, + size=size, + media_type=media_type, + host=hostprofile, + rotation=rotation, + interface=interface + ) + + +def make_cpu_profile(hostprofile, + cores=4, + architecture="x86_64", + cpus=4,): + return CpuProfile.objects.create( + cores=cores, + architecture=architecture, + cpus=cpus, + host=hostprofile, + cflags='' + ) + + +def make_interface_profile(hostprofile, + speed=1000, + name="test interface profile", + nic_type="pcie"): + return InterfaceProfile.objects.create( + host=hostprofile, + name=name, + speed=speed, + nic_type=nic_type + ) + + +def make_image(lab, lab_id, owner, os, host_profile, + public=True, name="default image", description="default image"): + return Image.objects.create( + from_lab=lab, + lab_id=lab_id, + os=os, + host_type=host_profile, + public=public, + name=name, + description=description + ) + + +def make_scenario(name="test scenario"): + return Scenario.objects.create(name=name) + + +def make_installer(scenarios, name="test installer"): + installer = Installer.objects.create(name=name) + for scenario in scenarios: + installer.sup_scenarios.add(scenario) + + return installer + + +def make_os(installers, name="test OS"): + os = Opsys.objects.create(name=name) + for installer in installers: + os.sup_installers.add(installer) + + return os + + +def make_host(host_profile, lab, labid="test_host", name="test_host", + booked=False, working=True, config=None, template=None, + bundle=None, model="Model 1", vendor="ACME"): + return Host.objects.create( + lab=lab, + profile=host_profile, + name=name, + booked=booked, + working=working, + config=config, + template=template, + bundle=bundle, + model=model, + vendor=vendor + ) + + +def make_opnfv_role(name="Jumphost", description="test opnfvrole"): + return OPNFVRole.objects.create( + name=name, + description=description + ) + + +def make_public_net(vlan, lab, in_use=False, + cidr="0.0.0.0/0", gateway="0.0.0.0"): + return PublicNetwork.objects.create( + lab=lab, + vlan=vlan, + cidr=cidr, + gateway=gateway + ) diff --git a/src/dashboard/tests/__init__.py b/src/dashboard/tests/__init__.py new file mode 100644 index 0000000..b6fef6c --- /dev/null +++ b/src/dashboard/tests/__init__.py @@ -0,0 +1,8 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## diff --git a/src/dashboard/urls.py b/src/dashboard/urls.py new file mode 100644 index 0000000..571a987 --- /dev/null +++ b/src/dashboard/urls.py @@ -0,0 +1,41 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# Copyright (c) 2018 Sawyer Bergeron, Parker Berberian, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +"""pharos_dashboard URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/1.10/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.conf.urls import url, include + 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) +""" +from django.conf.urls import url +from dashboard.views import ( + landing_view, + lab_list_view, + lab_detail_view, + host_profile_detail_view +) + +app_name = "dashboard" +urlpatterns = [ + url(r'^$', landing_view, name='index'), + url(r'^lab/$', lab_list_view, name='all_labs'), + url(r'^lab/(?P<lab_name>.+)/$', lab_detail_view, name='lab_detail'), + url(r'^hosts/$', host_profile_detail_view, name="hostprofile_detail") +] diff --git a/src/dashboard/views.py b/src/dashboard/views.py new file mode 100644 index 0000000..aaad7ab --- /dev/null +++ b/src/dashboard/views.py @@ -0,0 +1,121 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# Copyright (c) 2018 Parker Berberian, Sawyer Bergeron, and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from django.shortcuts import get_object_or_404 +from django.views.generic import TemplateView +from django.shortcuts import render +from django.http import HttpResponseRedirect + +from account.models import Lab + +from resource_inventory.models import Image, HostProfile +from workflow.views import create_session +from workflow.workflow_manager import ManagerTracker + + +def lab_list_view(request): + labs = Lab.objects.all() + context = {"labs": labs} + + return render(request, "dashboard/lab_list.html", context) + + +def lab_detail_view(request, lab_name): + user = None + if request.user.is_authenticated: + user = request.user + + lab = get_object_or_404(Lab, name=lab_name) + + images = Image.objects.filter(from_lab=lab).filter(public=True) + if user: + images = images | Image.objects.filter(from_lab=lab).filter(owner=user) + + return render( + request, + "dashboard/lab_detail.html", + { + 'title': "Lab Overview", + 'lab': lab, + 'hostprofiles': lab.hostprofiles.all(), + 'images': images, + } + ) + + +def host_profile_detail_view(request): + + return render( + request, + "dashboard/host_profile_detail.html", + { + 'title': "Host Types", + } + ) + + +def landing_view(request): + manager = None + manager_detected = False + if 'manager_session' in request.session: + + try: + manager = ManagerTracker.managers[request.session['manager_session']] + + except KeyError: + pass + + if manager is not None: + # no manager detected, don't display continue button + manager_detected = True + + if request.method == 'GET': + return render(request, 'dashboard/landing.html', {'manager': manager_detected, 'title': "Welcome to the Lab as a Service Dashboard"}) + + if request.method == 'POST': + try: + create = request.POST['create'] + + if manager is not None: + del manager + + mgr_uuid = create_session(create, request=request,) + request.session['manager_session'] = mgr_uuid + return HttpResponseRedirect('/wf/') + + except KeyError: + pass + + +class LandingView(TemplateView): + template_name = "dashboard/landing.html" + + def get_context_data(self, **kwargs): + context = super(LandingView, self).get_context_data(**kwargs) + + hosts = [] + + for host_profile in HostProfile.objects.all(): + name = host_profile.name + description = host_profile.description + in_labs = host_profile.labs + + interfaces = host_profile.interfaceprofile + storage = host_profile.storageprofile + cpu = host_profile.cpuprofile + ram = host_profile.ramprofile + + host = (name, description, in_labs, interfaces, storage, cpu, ram) + hosts.append(host) + + context.update({'hosts': hosts}) + + return context |