From e4488315862998d198d69cd8e4649428276e4f82 Mon Sep 17 00:00:00 2001 From: Sawyer Bergeron Date: Thu, 8 Apr 2021 14:19:54 -0400 Subject: Merge master for RC Squashed commit of the following: commit 4f12ae24d0de22f526d6dd59dbc437bdaa1874f1 Author: Sean Smith Date: Wed Apr 7 15:16:29 2021 -0400 Check if hostname null in resource filter Signed-off-by: Sean Smith Change-Id: Ie28c83109faf529f0a3bd4c61b1f96e064c1577c commit e9ed42e7ace16f36532a23d360300bb7f6b0387e Author: Sean Smith Date: Wed Apr 7 13:15:08 2021 -0400 Fix typo for image disable Change-Id: I74314aba4d1ee6f89b450666fd76deabdd99c11d Signed-off-by: Sean Smith commit 593ea6dc98ab12c1e70d65b16103f92f8b17abbc Merge: 241ff30 2271036 Author: Sawyer Bergeron Date: Wed Apr 7 15:20:07 2021 +0000 Merge "Restrict image on multi-node booking" commit 2271036232bce7ce4f2136ee6e031e9e77d9c1c2 Author: Sean Smith Date: Tue Apr 6 16:18:36 2021 -0400 Restrict image on multi-node booking Change-Id: I280dba83bceed74195a77b28f3016421c462cc5a Signed-off-by: Sean Smith commit 241ff304980c8bf542fdd1a2a47f73accf912ee1 Merge: f145622 d007fa6 Author: Sawyer Bergeron Date: Tue Apr 6 17:38:49 2021 +0000 Merge "Remove exposure of users on dashboard" commit f1456220fcc098cb0f5e9fc60124680ff8aba6af Merge: 6366b17 3abc7dc Author: Sawyer Bergeron Date: Tue Mar 30 18:01:05 2021 +0000 Merge "Hostname not required" commit 6366b1776bc51c29d4ba4c256f51d16acc52d871 Merge: a35c01b 0573f51 Author: Sawyer Bergeron Date: Tue Mar 30 17:54:21 2021 +0000 Merge "Update actions to free hosts and vlans" commit d007fa661e2faf77e70cb75c8d520941e907ebf7 Author: Sean Smith Date: Wed Mar 24 17:22:54 2021 -0400 Remove exposure of users on dashboard Signed-off-by: Sean Smith Change-Id: I83b93d9247a7eafb54e4a5761d1423a504d86400 commit 3abc7dcc9603cf978e5b7c2762e80998a4a6eef5 Author: Sean Smith Date: Mon Mar 29 15:00:52 2021 -0400 Hostname not required Change-Id: I7d639a17452e426e92671c558b111d6bcec34e8c Signed-off-by: Sean Smith commit a35c01bccec1137ccd8a0f07f832daecb4ee2da3 Author: Sean Smith Date: Mon Mar 29 14:14:48 2021 -0400 Resource display remains on error Signed-off-by: Sean Smith Change-Id: I0a59ef9da908599cf75e379b68b33e280a4deb16 commit 7addb00710a5865a9e6bc38f1d826aa0323ee67e Merge: f6472a3 f6753ae Author: Sawyer Bergeron Date: Mon Mar 29 18:05:56 2021 +0000 Merge "Fixes error on installer fail" commit f6472a3cca58ca428200265acf720174ed36ea7e Merge: 2e9a8c8 e3dec82 Author: Sawyer Bergeron Date: Fri Mar 26 23:26:33 2021 +0000 Merge "Remove duplicate migration" commit 2e9a8c854a93705d080dc54b8c6e3ba36eaba366 Author: Sean Smith Date: Thu Mar 25 15:16:50 2021 -0400 Additional fix for email Signed-off-by: Sean Smith Change-Id: I0599e29c7930386651f77bceccf05d069ebd4101 commit f6753ae2c5f54fb79375214590e565218b1bf0d7 Author: Sean Smith Date: Tue Sep 8 15:13:23 2020 -0400 Fixes error on installer fail Signed-off-by: Sean Smith Change-Id: I823f3c5dd2b7677b428b88e1af4bba78d7dc698c commit e3dec8238442865939bed379aa25219103f4d287 Author: Sean Smith Date: Wed Mar 24 16:09:15 2021 -0400 Remove duplicate migration Signed-off-by: Sean Smith Change-Id: I456b61f3ad25823dbcf2468b303c1e68ff41df7b commit e0a5e206a3a68bbae373bf8f9257b5aba14932c5 Merge: e231274 8559820 Author: Sawyer Bergeron Date: Wed Mar 24 15:56:51 2021 +0000 Merge "Filters non working servers in quick booking" commit e23127473b81bdbc1f4630036ddec3c46ee17dda Merge: b14e617 e817c32 Author: Sawyer Bergeron Date: Wed Mar 24 15:48:39 2021 +0000 Merge "Front end fixes for enforcing hostname" commit e817c3203fde0ca9bfb66a00919050cbdd1179e8 Author: Sean Smith Date: Wed Feb 24 12:18:45 2021 -0500 Front end fixes for enforcing hostname Signed-off-by: Sean Smith Change-Id: I0a077307a2e1bcfef5b31acdaf4ecb73be64fff9 commit 0573f51f18c1e55fe162dbd5a3f9acbd5394e065 Author: Sean Smith Date: Tue Feb 23 15:41:59 2021 -0500 Update actions to free hosts and vlans Signed-off-by: Sean Smith Change-Id: I94425ca5a48cccdf0b5382ef2bb16a989ee6b32d Signed-off-by: Sawyer Bergeron commit b14e617edb86628f31648847c0b96206764ef44c Merge: 13cdfee babedf3 Author: Sawyer Bergeron Date: Tue Mar 23 18:34:16 2021 +0000 Merge "Added LF Edge template overrides." commit babedf30038b453ea7d123c9bb744f4acb14e625 Author: Adam Hassick Date: Tue Mar 16 14:18:31 2021 -0400 Added LF Edge template overrides. Signed-off-by: Adam Hassick Change-Id: If119646eee67fb7e145c5e92e64fa4f28a947dbe commit 13cdfee5d9656c2195bb6238d33d66cb637c22e4 Author: Sawyer Bergeron Date: Wed Mar 3 14:09:35 2021 -0500 Add copyright notice to admin_utils file Signed-off-by: Sawyer Bergeron Change-Id: I4b8a53371c0da274347dcfb7ea969ab4ce403f66 commit 72585921008fc89b5a32f4cb116d5ea9fe1a8ead Merge: 8a902d3 de4cbcb Author: Sawyer Bergeron Date: Wed Mar 17 17:07:40 2021 +0000 Merge "Add documentation to admin_utils functions" commit de4cbcb7b770bd6ae14916715db361e987d1d3ec Author: Sawyer Bergeron Date: Wed Mar 3 14:06:07 2021 -0500 Add documentation to admin_utils functions Signed-off-by: Sawyer Bergeron Change-Id: I27204f49513b3d646776261ed869d9f4d5c6fd64 Signed-off-by: Sawyer Bergeron commit 85598208ad7b65de9e4c60b509331a9b6ce46e52 Author: Sean Smith Date: Mon Mar 8 17:15:42 2021 -0500 Filters non working servers in quick booking Signed-off-by: Sean Smith Change-Id: I47776a3e239a333d544d9e6e86f702b1d299baeb commit 8a902d357fa6eca07384d25fac32a6e0521a63e5 Author: Sean Smith Date: Mon Mar 8 16:21:56 2021 -0500 Release VLAN on booking end Signed-off-by: Sean Smith Change-Id: Ib17fd871501a2af46bb78bbdfc68112a66e763c0 commit a320df5bf04978fc1e73a2e49d1a4bed5b82b849 Merge: 402e6e1 4caae41 Author: Sawyer Bergeron Date: Wed Mar 3 19:07:19 2021 +0000 Merge "Add documentation request functions and use hints to admin utils" commit 4caae41287310cb1309c9b30e2871297d3ed21ef Author: Sawyer Bergeron Date: Wed Mar 3 12:10:02 2021 -0500 Add documentation request functions and use hints to admin utils Signed-off-by: Sawyer Bergeron Change-Id: I3211194fea5dc7d0c3569db6c1d42fe2f4aa53e1 Signed-off-by: Sawyer Bergeron commit 402e6e1111964f28235f6f7ec53ba51c76a8298f Author: Sean Smith Date: Mon Feb 22 13:05:46 2021 -0500 Update celery schedule to send emails Change-Id: I0fbf7080d440cea98d358a30ac5df88ce888a8a0 Signed-off-by: Sean Smith Update celery Change-Id: Icd20d67b561bacbccf1f9d75335f76ebdbef4f1d Signed-off-by: Sean Smith commit db4c2dc4c4f0d86e0a00e8409eed74a0bcffb20b Author: Sawyer Bergeron Date: Mon Feb 8 11:42:41 2021 -0500 Add migration to set rconfig hostname to a valid linux hostname Signed-off-by: Sawyer Bergeron Change-Id: Ib737dc25de37dd9d1c29a4e15f95da0fb68ddfc8 Signed-off-by: Sawyer Bergeron commit 9c200b5a15a336e3a3fc8d7ddac53053bc3b6ffe Author: Sawyer Bergeron Date: Tue Jan 19 12:38:32 2021 -0500 Add booking extension function to admin utils Signed-off-by: Sawyer Bergeron Change-Id: I4efab99e7cb82f69d01de8f74f30047263b4b0c2 Signed-off-by: Sawyer Bergeron commit c77b9169b09b7b53572b45a4f3679e602b8ca6a2 Author: Sawyer Bergeron Date: Mon Jan 11 15:02:03 2021 -0500 Add server create function to admin utils Signed-off-by: Sawyer Bergeron Change-Id: I5776213bfc2eda310049c47dcb1fa1a601bd1896 commit 76b884730a00fecff5ed39d8ff4360941f313fe0 Author: Sawyer Bergeron Date: Mon Jan 11 11:45:23 2021 -0500 Add scripts for creating default templates Signed-off-by: Sawyer Bergeron Change-Id: I522c617ff038531914f3a525b083d47c079489c1 Signed-off-by: Sawyer Bergeron commit d41310f9c3b62e8ab462fa94f2bf1b6b7d2b5e23 Merge: 107d8bb bf82270 Author: Sawyer Bergeron Date: Mon Jan 4 21:15:58 2021 +0000 Merge "Fix issue where users are redirected to Jira login in LFID mode in certain cases" commit bf822708949ad2e57b3966ad2bf485588fe4b29d Author: Adam Hassick Date: Mon Jan 4 20:47:35 2021 +0000 Fix issue where users are redirected to Jira login in LFID mode in certain cases Change-Id: If8a82d3a2628a7f55f902d321388be2419524c8b Signed-off-by: Adam Hassick Change-Id: Ieb9de968514b88a8c450967466d06c9860427d83 commit 107d8bb663083cd2e5fc09417ccd41560bc272c9 Author: Sawyer Bergeron Date: Fri Dec 18 12:16:57 2020 -0500 Reduce log spam: deletion on resource bundles should be deprecated Signed-off-by: Sawyer Bergeron Change-Id: I6bffc393f7a6f93c4b8cd8d4c8bacba0c23fb256 Signed-off-by: Sawyer Bergeron commit 17388941fc3cff40cf2c7e7290e6bad9a27ca578 Author: Sawyer Bergeron Date: Fri Dec 18 10:17:02 2020 -0500 Repair migration chain Signed-off-by: Sawyer Bergeron Change-Id: Iaedcaf76111694bc7a3d8c44bd98f29d508610c5 commit c089784cc3b3708cf89f79962d62de66b009fc05 Author: Sawyer Bergeron Date: Fri Dec 18 09:53:49 2020 -0500 Fix hostname default to be valid linux hostname Signed-off-by: Sawyer Bergeron Change-Id: I898c4e4a03261a1bf0f3f56dbc32104c9b5f3ee9 Signed-off-by: Sawyer Bergeron Change-Id: Ieb010345d5c44356db37f0d4df625686d143a81d Signed-off-by: Sawyer Bergeron --- src/dashboard/admin_utils.py | 290 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 240 insertions(+), 50 deletions(-) (limited to 'src/dashboard/admin_utils.py') diff --git a/src/dashboard/admin_utils.py b/src/dashboard/admin_utils.py index 6d990c9..186a64f 100644 --- a/src/dashboard/admin_utils.py +++ b/src/dashboard/admin_utils.py @@ -1,3 +1,12 @@ +############################################################################## +# Copyright (c) 2021 Sawyer Bergeron and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + from resource_inventory.models import ( ResourceTemplate, Image, @@ -17,10 +26,16 @@ from resource_inventory.models import ( ) import json +import sys +import inspect +import pydoc from django.contrib.auth.models import User -from account.models import Lab +from account.models import ( + Lab, + PublicNetwork +) from resource_inventory.resource_manager import ResourceManager from resource_inventory.pdf_templater import PDFTemplater @@ -39,12 +54,37 @@ from api.models import JobStatus def print_div(): - print("====================================================================") + """ + Utility function for printing dividers, does nothing directly useful as a utility + """ + print("=" * 68) def book_host(owner_username, host_labid, lab_username, hostname, image_id, template_name, length_days=21, collaborator_usernames=[], purpose="internal", project="LaaS"): """ creates a quick booking using the given host + + @owner_username is the simple username for the user who will own the resulting booking. + Do not set this to a lab username! + + @image_id is the django id of the image in question, NOT the labid of the image. + Query Image objects by their public status and compatible host types + + @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object + + @lab_username for iol is `unh_iol`, other labs will be documented here + + @hostname the hostname that the resulting host should have set + + @template_name the name of the (public, or user accessible) template to use for this booking + + @length_days how long the booking should be, no hard limit currently + + @collaborator_usernames a list of usernames for collaborators to the booking + + @purpose what this booking will be used for + + @project what project/group this booking is on behalf of or the owner represents """ lab = Lab.objects.get(lab_user__username=lab_username) host = Server.objects.filter(lab=lab).get(labid=host_labid) @@ -113,6 +153,16 @@ def book_host(owner_username, host_labid, lab_username, hostname, image_id, temp def mark_working(host_labid, lab_username, working=True): + """ + Mark a host working/not working so that it is either bookable or hidden in the dashboard. + + @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object + + @lab_username: param of the form `unh_iol` or similar + + @working: bool, whether by the end of execution the host should be considered working or not working + """ + lab = Lab.objects.get(lab_user__username=lab_username) server = Server.objects.filter(lab=lab).get(labid=host_labid) print("changing server working status from ", server.working, "to", working) @@ -121,6 +171,16 @@ def mark_working(host_labid, lab_username, working=True): def mark_booked(host_labid, lab_username, booked=True): + """ + Mark a host as booked/unbooked + + @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object + + @lab_username: param of the form `unh_iol` or similar + + @working: bool, whether by the end of execution the host should be considered booked or not booked + """ + lab = Lab.objects.get(lab_user__username=lab_username) server = Server.objects.filter(lab=lab).get(labid=host_labid) print("changing server booked status from ", server.booked, "to", booked) @@ -128,13 +188,26 @@ def mark_booked(host_labid, lab_username, booked=True): server.save() -# returns host filtered by lab and then unique id within lab def get_host(host_labid, lab_username): + """ + Returns host filtered by lab and then unique id within lab + + @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object + + @lab_username: param of the form `unh_iol` or similar + """ lab = Lab.objects.get(lab_user__username=lab_username) return Server.objects.filter(lab=lab).get(labid=host_labid) def get_info(host_labid, lab_username): + """ + Returns various information on the host queried by the given parameters + + @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object + + @lab_username: param of the form `unh_iol` or similar + """ info = {} host = get_host(host_labid, lab_username) info['host_labid'] = host_labid @@ -199,8 +272,16 @@ def detect_leaked_hosts(labid="unh_iol"): return filtered -def booking_for_host(host_labid: str, labid="unh_iol"): - server = Server.objects.get(lab__lab_user__username=labid, labid=host_labid) +def booking_for_host(host_labid: str, lab_username="unh_iol"): + """ + Returns the booking that this server is a part of, if any. + Fails with an exception if no such booking exists + + @host_labid is usually of the form `hpe3` or similar, is the labid of the Server (subtype of Resource) object + + @lab_username: param of the form `unh_iol` or similar + """ + server = Server.objects.get(lab__lab_user__username=lab_username, lab_username=host_labid) booking = server.bundle.booking_set.first() print_div() print(booking) @@ -211,7 +292,15 @@ def booking_for_host(host_labid: str, labid="unh_iol"): return booking -def force_release_booking(booking_id): +def force_release_booking(booking_id: int): + """ + Takes a booking id and forces the booking to end whether or not the tasks have + completed normally. + + Use with caution! Hosts may or may not be released depending on other underlying issues + + @booking_id: the id of the Booking object to be released + """ booking = Booking.objects.get(id=booking_id) job = booking.job tasks = job.get_tasklist() @@ -220,7 +309,28 @@ def force_release_booking(booking_id): task.save() +def free_leaked_public_vlans(safety_buffer_days=2): + for lab in Lab.objects.all(): + current_booking_set = Booking.objects.filter(end__gte=timezone.now() + timedelta(days=safety_buffer_days)) + + marked_nets = set() + + for booking in current_booking_set: + for network in get_network_metadata(booking.id): + marked_nets.add(network["vlan_id"]) + + for net in PublicNetwork.objects.filter(lab=lab).filter(in_use=True): + if net.vlan not in marked_nets: + lab.vlan_manager.release_public_vlan(net.vlan) + + def get_network_metadata(booking_id: int): + """ + Takes a booking id and prints all (known) networks that are owned by it. + Returns an object of the form {: {"vlan_id": int, "netname": str , "public": bool : { - capacity: int (GiB) - media_type: str ("SSD" or "HDD") - interface: str ("sata", "sas", "ssd", "nvme", "scsi", or "iscsi") +def add_profile(data): + """ + Used for adding a host profile to the dashboard + + schema (of dict passed as "data" param): + { + "name": str + "description": str + "labs": [ + str (lab username) + ] + "disks": { + : { + capacity: int (GiB) + media_type: str ("SSD" or "HDD") + interface: str ("sata", "sas", "ssd", "nvme", "scsi", or "iscsi") + } } - } - interfaces: { - : { - "speed": int (mbit) - "nic_type": str ("onboard" or "pcie") - "order": int (compared to the other interfaces, indicates the "order" that the ports are laid out) + interfaces: { + : { + "speed": int (mbit) + "nic_type": str ("onboard" or "pcie") + "order": int (compared to the other interfaces, indicates the "order" that the ports are laid out) + } + } + cpus: { + cores: int (hardware threads count) + architecture: str (x86_64" or "aarch64") + cpus: int (number of sockets) + cflags: str + } + ram: { + amount: int (GiB) + channels: int } } - cpus: { - cores: int (hardware threads count) - architecture: str (x86_64" or "aarch64") - cpus: int (number of sockets) - cflags: str - } - ram: { - amount: int (GiB) - channels: int - } -} -""" - - -def add_profile(data): + """ base_profile = ResourceProfile.objects.create(name=data['name'], description=data['description']) base_profile.save() @@ -303,6 +417,11 @@ def add_profile(data): def make_default_template(resource_profile, image_id=None, template_name=None, connected_interface_names=None, interfaces_tagged=False, connected_interface_tagged=False, owner_username="root", lab_username="unh_iol", public=True, temporary=False, description=""): + """ + Do not call this function without reading the related source code, it may have unintended effects. + + Used for creating a default template from some host profile + """ if not resource_profile: raise Exception("No viable continuation from none resource_profile") @@ -349,16 +468,27 @@ def make_default_template(resource_profile, image_id=None, template_name=None, c connection.save() -""" -Note: interfaces should be dict from interface name (eg ens1f0) to dict of schema: - { - mac_address: , - bus_addr: , //this field is optional, "" is default - } -""" +def add_server(profile, name, interfaces, lab_username="unh_iol", vendor="unknown", model="unknown"): + """ + Used to enroll a new host of some profile + @profile: the ResourceProfile in question (by reference to a model object) + + @name: the unique name of the server, currently indistinct from labid + + @interfaces: interfaces should be dict from interface name (eg ens1f0) to dict of schema: + { + mac_address: , + bus_addr: , //this field is optional, "" is default + } -def add_server(profile, uname, interfaces, lab_username="unh_iol", vendor="unknown", model="unknown"): + @lab_username: username of the lab to be added to + + @vendor: vendor name of the host, such as "HPE" or "Gigabyte" + + @model: specific model of the host, such as "DL380 Gen 9" + + """ server = Server.objects.create( bundle=None, profile=profile, @@ -366,9 +496,9 @@ def add_server(profile, uname, interfaces, lab_username="unh_iol", vendor="unkno working=True, vendor=vendor, model=model, - labid=uname, + labid=name, lab=Lab.objects.get(lab_user__username=lab_username), - name=uname, + name=name, booked=False) for iface_prof in InterfaceProfile.objects.filter(host=profile).all(): @@ -382,3 +512,63 @@ def add_server(profile, uname, interfaces, lab_username="unh_iol", vendor="unkno server.interfaces.add(iface) server.save() + + +def extend_booking(booking_id, days=0, hours=0, minutes=0, weeks=0): + """ + Extend a booking by n + + @booking_id: id of the booking + + @days/@hours/@minutes/@weeks: the cumulative amount of delta to add to the length of the booking + """ + + booking = Booking.objects.get(id=booking_id) + booking.end = booking.end + timedelta(days=days, hours=hours, minutes=minutes, weeks=weeks) + booking.save() + + +def docs(function=None, fulltext=False): + """ + Print documentation for a given function in admin_utils. + Call without arguments for more information + """ + + fn = None + + if isinstance(function, str): + try: + fn = globals()[function] + except KeyError: + print("Couldn't find a function by the given name") + return + elif callable(function): + fn = function + else: + print("docs(function: callable | str, fulltext: bool) was called with a 'function' that was neither callable nor a string name of a function") + print("usage: docs('some_function_in_admin_utils', fulltext=True)") + print("The 'fulltext' argument is used to choose if you want the complete source of the function printed. If this argument is false then you will only see the pydoc rendered documentation for the function") + return + + if not fn: + print("couldn't find a function by that name") + + if not fulltext: + print("Pydoc documents the function as such:") + print(pydoc.render_doc(fn)) + else: + print("The full source of the function is this:") + print(inspect.getsource(fn)) + + +def admin_functions(): + """ + List functions available to call within admin_utils + """ + + return [name for name, func in inspect.getmembers(sys.modules[__name__]) if (inspect.isfunction(func) and func.__module__ == __name__)] + + +print("Hint: call `docs()` or `admin_functions()` for help on using the admin utils") +print("docs() displays documentation on a given function") +print("admin_functions() lists all functions available to call within this module") -- cgit 1.2.3-korg