From 2ec0d7b9f5c1354977b821c6b06c24a3ffa13142 Mon Sep 17 00:00:00 2001 From: Gergely Csatari Date: Thu, 26 Oct 2023 10:33:28 +0300 Subject: Removing project content and adding a note that the development continues in GitHub Change-Id: I25c58a679dbf92b2367d826429b7cda936bf9f0e Signed-off-by: Gergely Csatari --- src/dashboard/admin_utils.py | 811 ------------------------------------------- 1 file changed, 811 deletions(-) delete mode 100644 src/dashboard/admin_utils.py (limited to 'src/dashboard/admin_utils.py') diff --git a/src/dashboard/admin_utils.py b/src/dashboard/admin_utils.py deleted file mode 100644 index 75e4f3e..0000000 --- a/src/dashboard/admin_utils.py +++ /dev/null @@ -1,811 +0,0 @@ -############################################################################## -# Copyright (c) 2021 Sawyer Bergeron and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -from resource_inventory.models import ( - ResourceTemplate, - Image, - Server, - ResourceBundle, - ResourceProfile, - InterfaceProfile, - PhysicalNetwork, - ResourceConfiguration, - NetworkConnection, - InterfaceConfiguration, - Network, - DiskProfile, - CpuProfile, - RamProfile, - Interface, - CloudInitFile, -) - -import json -import yaml -import sys -import inspect -import pydoc -import csv - -from django.contrib.auth.models import User - -from account.models import ( - Lab, - PublicNetwork -) - -from resource_inventory.resource_manager import ResourceManager -from resource_inventory.pdf_templater import PDFTemplater - -from booking.quick_deployer import update_template - -from datetime import timedelta, date, datetime, timezone - -from booking.models import Booking -from notifier.manager import NotificationHandler -from api.models import JobFactory - -from api.models import JobStatus, Job, GeneratedCloudConfig - - -def print_div(): - """ - Utility function for printing dividers, does nothing directly useful as a utility - """ - print("=" * 68) - - -def book_host(owner_username, host_labid, lab_username, hostname, image_id, template_name, length_days=21, collaborator_usernames=[], purpose="internal", project="LaaS"): - """ - creates a quick booking using the given host - - @owner_username is the simple username for the user who will own the resulting booking. - Do not set this to a lab username! - - @image_id is the django id of the image in question, NOT the labid of the image. - Query Image objects by their public status and compatible host types - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username for iol is `unh_iol`, other labs will be documented here - - @hostname the hostname that the resulting host should have set - - @template_name the name of the (public, or user accessible) template to use for this booking - - @length_days how long the booking should be, no hard limit currently - - @collaborator_usernames a list of usernames for collaborators to the booking - - @purpose what this booking will be used for - - @project what project/group this booking is on behalf of or the owner represents - """ - lab = Lab.objects.get(lab_user__username=lab_username) - host = Server.objects.filter(lab=lab).get(labid=host_labid) - if host.booked: - print("Can't book host, already marked as booked") - return - else: - host.booked = True - host.save() - - template = ResourceTemplate.objects.filter(public=True).get(name=template_name) - image = Image.objects.get(id=image_id) - - owner = User.objects.get(username=owner_username) - - new_template = update_template(template, image, hostname, owner) - - rmanager = ResourceManager.getInstance() - - vlan_map = rmanager.get_vlans(new_template) - - # only a single host so can reuse var for iter here - resource_bundle = ResourceBundle.objects.create(template=new_template) - res_configs = new_template.getConfigs() - - for config in res_configs: - try: - host.bundle = resource_bundle - host.config = config - rmanager.configureNetworking(resource_bundle, host, vlan_map) - host.save() - except Exception: - host.booked = False - host.save() - print("Failed to book host due to error configuring it") - return - - new_template.save() - - booking = Booking.objects.create( - purpose=purpose, - project=project, - lab=lab, - owner=owner, - start=timezone.now(), - end=timezone.now() + timedelta(days=int(length_days)), - resource=resource_bundle, - opnfv_config=None - ) - - booking.pdf = PDFTemplater.makePDF(booking) - - booking.save() - - for collaborator_username in collaborator_usernames: - try: - user = User.objects.get(username=collaborator_username) - booking.collaborators.add(user) - except Exception: - print("couldn't add user with username ", collaborator_username) - - booking.save() - - JobFactory.makeCompleteJob(booking) - NotificationHandler.notify_new_booking(booking) - - -def mark_working(host_labid, lab_username, working=True): - """ - Mark a host working/not working so that it is either bookable or hidden in the dashboard. - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username: param of the form `unh_iol` or similar - - @working: bool, whether by the end of execution the host should be considered working or not working - """ - - lab = Lab.objects.get(lab_user__username=lab_username) - server = Server.objects.filter(lab=lab).get(labid=host_labid) - print("changing server working status from ", server.working, "to", working) - server.working = working - server.save() - - -def mark_booked(host_labid, lab_username, booked=True): - """ - Mark a host as booked/unbooked - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username: param of the form `unh_iol` or similar - - @working: bool, whether by the end of execution the host should be considered booked or not booked - """ - - lab = Lab.objects.get(lab_user__username=lab_username) - server = Server.objects.filter(lab=lab).get(labid=host_labid) - print("changing server booked status from ", server.booked, "to", booked) - server.booked = booked - server.save() - - -def get_host(host_labid, lab_username): - """ - Returns host filtered by lab and then unique id within lab - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username: param of the form `unh_iol` or similar - """ - lab = Lab.objects.get(lab_user__username=lab_username) - return Server.objects.filter(lab=lab).get(labid=host_labid) - - -def get_info(host_labid, lab_username): - """ - Returns various information on the host queried by the given parameters - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username: param of the form `unh_iol` or similar - """ - info = {} - host = get_host(host_labid, lab_username) - info['host_labid'] = host_labid - info['booked'] = host.booked - info['working'] = host.working - info['profile'] = str(host.profile) - if host.bundle: - binfo = {} - info['bundle'] = binfo - if host.config: - cinfo = {} - info['config'] = cinfo - - return info - - -class CumulativeData: - use_days = 0 - count_bookings = 0 - count_extensions = 0 - - def __init__(self, file_writer): - self.file_writer = file_writer - - def account(self, booking, usage_days): - self.count_bookings += 1 - self.count_extensions += booking.ext_count - self.use_days += usage_days - - def write_cumulative(self): - self.file_writer.writerow([]) - self.file_writer.writerow([]) - self.file_writer.writerow(['Lab Use Days', 'Count of Bookings', 'Total Extensions Used']) - self.file_writer.writerow([self.use_days, self.count_bookings, (self.count_bookings * 2) - self.count_extensions]) - - -def get_years_booking_data(start_year=None, end_year=None): - """ - Outputs yearly booking information from the past 'start_year' years (default: current year) - until the last day of the end year (default current year) as a csv file. - """ - if start_year is None and end_year is None: - start = datetime.combine(date(datetime.now().year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - end = datetime.combine(date(start.year + 1, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - elif end_year is None: - start = datetime.combine(date(start_year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - end = datetime.combine(date(datetime.now().year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - else: - start = datetime.combine(date(start_year, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - end = datetime.combine(date(end_year + 1, 1, 1), datetime.min.time()).replace(tzinfo=timezone.utc) - - if (start.year == end.year - 1): - file_name = "yearly_booking_data_" + str(start.year) + ".csv" - else: - file_name = "yearly_booking_data_" + str(start.year) + "-" + str(end.year - 1) + ".csv" - - with open(file_name, "w", newline="") as file: - file_writer = csv.writer(file) - cumulative_data = CumulativeData(file_writer) - file_writer.writerow( - [ - 'ID', - 'Project', - 'Purpose', - 'User', - 'Collaborators', - 'Extensions Left', - 'Usage Days', - 'Start', - 'End' - ] - ) - - for booking in Booking.objects.filter(start__gte=start, start__lte=end): - filtered = False - booking_filter = [279] - user_filter = ["ParkerBerberian", "ssmith", "ahassick", "sbergeron", "jhodgdon", "rhodgdon", "aburch", "jspewock"] - user = booking.owner.username if booking.owner.username is not None else "None" - - for b in booking_filter: - if b == booking.id: - filtered = True - - for u in user_filter: - if u == user: - filtered = True - # trims time delta to the the specified year(s) if between years - usage_days = ((end if booking.end > end else booking.end) - (start if booking.start < start else booking.start)).days - collaborators = [] - - for c in booking.collaborators.all(): - collaborators.append(c.username) - - if (not filtered): - cumulative_data.account(booking, usage_days) - file_writer.writerow([ - str(booking.id), - str(booking.project), - str(booking.purpose), - str(booking.owner.username), - ','.join(collaborators), - str(booking.ext_count), - str(usage_days), - str(booking.start), - str(booking.end) - ]) - cumulative_data.write_cumulative() - - -def map_cntt_interfaces(labid: str): - """ - Use this during cntt migrations, call it with a host labid and it will change profiles for this host - as well as mapping its interfaces across. interface ens1f2 should have the mac address of interface eno50 - as an invariant before calling this function - """ - host = get_host(labid, "unh_iol") - host.profile = ResourceProfile.objects.get(name="HPE x86 CNTT") - host.save() - host = get_host(labid, "unh_iol") - - for iface in host.interfaces.all(): - new_ifprofile = None - if iface.profile.name == "ens1f2": - new_ifprofile = InterfaceProfile.objects.get(host=host.profile, name="eno50") - else: - new_ifprofile = InterfaceProfile.objects.get(host=host.profile, name=iface.profile.name) - - iface.profile = new_ifprofile - - iface.save() - - -def detect_leaked_hosts(labid="unh_iol"): - """ - Use this to try to detect leaked hosts. - These hosts may still be in the process of unprovisioning, - but if they are not (or unprovisioning is frozen) then - these hosts are instead leaked - """ - working_servers = Server.objects.filter(working=True, lab__lab_user__username=labid) - booked = working_servers.filter(booked=True) - filtered = booked - print_div() - print("In use now:") - for booking in Booking.objects.filter(end__gte=timezone.now()): - res_for_booking = booking.resource.get_resources() - print(res_for_booking) - for resource in res_for_booking: - filtered = filtered.exclude(id=resource.id) - print_div() - print("Possibly leaked:") - for host in filtered: - print(host) - print_div() - return filtered - - -def booking_for_host(host_labid: str, lab_username="unh_iol"): - """ - Returns the booking that this server is a part of, if any. - Fails with an exception if no such booking exists - - @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object - - @lab_username: param of the form `unh_iol` or similar - """ - server = Server.objects.get(lab__lab_user__username=lab_username, labid=host_labid) - booking = server.bundle.booking_set.first() - print_div() - print(booking) - print("id:", booking.id) - print("owner:", booking.owner) - print("job (id):", booking.job, "(" + str(booking.job.id) + ")") - print_div() - return booking - - -def force_release_booking(booking_id: int): - """ - Takes a booking id and forces the booking to end whether or not the tasks have - completed normally. - - Use with caution! Hosts may or may not be released depending on other underlying issues - - @booking_id: the id of the Booking object to be released - """ - booking = Booking.objects.get(id=booking_id) - job = booking.job - tasks = job.get_tasklist() - for task in tasks: - task.status = JobStatus.DONE - task.save() - - -def free_leaked_public_vlans(safety_buffer_days=2): - for lab in Lab.objects.all(): - current_booking_set = Booking.objects.filter(end__gte=timezone.now() + timedelta(days=safety_buffer_days)) - - marked_nets = set() - - for booking in current_booking_set: - for network in get_network_metadata(booking.id): - marked_nets.add(network["vlan_id"]) - - for net in PublicNetwork.objects.filter(lab=lab).filter(in_use=True): - if net.vlan not in marked_nets: - lab.vlan_manager.release_public_vlan(net.vlan) - - -def get_network_metadata(booking_id: int): - """ - Takes a booking id and prints all (known) networks that are owned by it. - Returns an object of the form {: {"vlan_id": int, "netname": str , "public": bool BEFORE THIS - @filenames: array of host import file names to import - """ - - for filename in filenames: - - # open import file - file = open("dashboard/" + filename + "-import.yaml", "r") - data = yaml.safe_load(file) - - # if a new profile is needed create one and a matching template - if (data["new_profile"]): - add_profile(data) - print("Profile: " + data["name"] + " created!") - make_default_template( - ResourceProfile.objects.get(name=data["name"]), - Image.objects.get(lab_id=data["image"]).id, - None, - None, - False, - False, - data["owner"], - "unh_iol", - True, - False, - data["temp_desc"] - ) - - print(" Template: " + data["temp_name"] + " created!") - - # add the server - add_server( - ResourceProfile.objects.get(name=data["name"]), - data["hostname"], - data["interfaces"], - data["lab"], - data["vendor"], - data["model"] - ) - - print(data["hostname"] + " imported!") - - -def convert_inspect_results(files): - """ - Converts an array of inspection result files into templates (filename-import.yaml) to be filled out for importing the servers into the dashboard - @files an array of file names (not including the file type. i.e hpe44). Default: [] - """ - for filename in files: - # open host inspect file - file = open("dashboard/" + filename + ".yaml") - output = open("dashboard/" + filename + "-import.yaml", "w") - data = json.load(file) - - # gather data about disks - disk_data = {} - for i in data["disk"]: - - # don't include loops in disks - if "loop" not in i: - disk_data[i["name"]] = { - "capacity": i["size"][:-3], - "media_type": "<\"SSD\" or \"HDD\">", - "interface": "<\"sata\", \"sas\", \"ssd\", \"nvme\", \"scsi\", or \"iscsi\">", - } - - # gather interface data - interface_data = {} - for i in data["interfaces"]: - interface_data[data["interfaces"][i]["name"]] = { - "speed": data["interfaces"][i]["speed"], - "nic_type": "<\"onboard\" or \"pcie\">", - "order": "", - "mac_address": data["interfaces"][i]["mac"], - "bus_addr": data["interfaces"][i]["busaddr"], - } - - # gather cpu data - cpu_data = { - "cores": data["cpu"]["cores"], - "architecture": data["cpu"]["arch"], - "cpus": data["cpu"]["cpus"], - "cflags": "", - } - - # gather ram data - ram_data = { - "amount": data["memory"][:-1], - "channels": "", - } - - # assemble data for host import file - import_data = { - "new_profile": " (Set to True to create a new profile for the host's type)", - "name": " (Used to set the profile of a host and for creating a new profile)", - "description": "", - "labs": "", - "temp_name": "