diff options
| author |   Parker Berberian <pberberian@iol.unh.edu> | 2020-02-06 18:34:17 +0000 |
|---|---|---|
| committer |   Gerrit Code Review <gerrit@opnfv.org> | 2020-02-06 18:34:17 +0000 |
| commit | 868dc419abbec2988dfe48cbd9d6f7cf56a48079 (patch) | |
| tree | 7ce55022654d3fee68ed4833888a4f15590e61ae | |
| parent | 078273eb7db5a481a4131d44a943f3c9e34b6b88 (diff) | |
| parent | 77377d5e9362bd35a3b300df231e82ee974675e1 (diff) | |
Merge "Comments and Documentation"
42 files changed, 389 insertions, 144 deletions
@@ -0,0 +1,2 @@ +[flake8] +ignore = E501,D10 diff --git a/config.env.sample b/config.env.sample index 01194b9..a21d060 100644 --- a/config.env.sample +++ b/config.env.sample @@ -2,8 +2,17 @@ DASHBOARD_URL=http://labs.opnfv.org # SECURITY WARNING: don't run with debug turned on in production! DEBUG=False +# TEST should be True if you want to run some tests in your local dev environment TEST=False +# These configure the postgres container and +# tell django how to access the database +# You shouldn't really need to change these, unless +# You want a specific user / pass on the DB +# The POSTGRES_ vars and DB_ vars should be kept in sync, eg +# POSTGRES_DB == DB_NAME +# POSTGRES_USER == DB_USER +# POSTGRES_PASSWORD == DB_PASS POSTGRES_DB=sample_name POSTGRES_USER=sample_user POSTGRES_PASSWORD=sample_pass @@ -19,6 +28,9 @@ SECRET_KEY=http://www.miniwebtool.com/django-secret-key-generator/ OAUTH_CONSUMER_KEY=sample_key OAUTH_CONSUMER_SECRET=sample_secret +# access information for Jira +# In addition to this, the rsa keys from your jira admin +# need to go into src/account JIRA_URL=sample_url JIRA_USER_NAME=sample_jira_user JIRA_USER_PASSWORD=sample_jira_pass @@ -27,7 +39,7 @@ JIRA_USER_PASSWORD=sample_jira_pass RABBITMQ_DEFAULT_USER=opnfv RABBITMQ_DEFAULT_PASS=opnfvopnfv -#Jenkins Build Server +# Jenkins Build Server JENKINS_URL=https://build.opnfv.org/ci # Email Settings @@ -20,6 +20,7 @@ Deployment: - install docker, docker-compose - run 'make data' - run 'make up' to run the dashboard (or 'make dev-up' for development) +- get the rsa.pem and rsa.pub keys from your jira admin and place them in src/account Production will be running on port 80 by default. Development will be running on port 8000 by default. diff --git a/src/account/jira_util.py b/src/account/jira_util.py index 18b0e26..a522594 100644 --- a/src/account/jira_util.py +++ b/src/account/jira_util.py @@ -37,7 +37,7 @@ class SignatureMethod_RSA_SHA1(oauth.SignatureMethod): return key, raw def sign(self, request, consumer, token): - """Builds the base signature string.""" + """Build the base signature string.""" key, raw = self.signing_base(request, consumer, token) module_dir = os.path.dirname(__file__) # get current directory diff --git a/src/account/middleware.py b/src/account/middleware.py index 0f1dbd8..6a46dfe 100644 --- a/src/account/middleware.py +++ b/src/account/middleware.py @@ -16,9 +16,12 @@ from account.models import UserProfile class TimezoneMiddleware(MiddlewareMixin): """ + Manage user's Timezone preference. + Activate the timezone from request.user.userprofile if user is authenticated, deactivate the timezone otherwise and use default (UTC) """ + def process_request(self, request): if request.user.is_authenticated: try: diff --git a/src/account/models.py b/src/account/models.py index 4862231..294e109 100644 --- a/src/account/models.py +++ b/src/account/models.py @@ -15,6 +15,14 @@ import random class LabStatus(object): + """ + A Poor man's enum for the status of a lab. + + If everything is working fine at a lab, it is UP. + If it is down temporarily e.g. for maintenance, it is TEMP_DOWN + If its broken, its DOWN + """ + UP = 0 TEMP_DOWN = 100 DOWN = 200 @@ -25,6 +33,8 @@ def upload_to(object, filename): class UserProfile(models.Model): + """Extend the Django User model.""" + user = models.OneToOneField(User, on_delete=models.CASCADE) timezone = models.CharField(max_length=100, blank=False, default='UTC') ssh_public_key = models.FileField(upload_to=upload_to, null=True, blank=True) @@ -47,14 +57,31 @@ class UserProfile(models.Model): class VlanManager(models.Model): + """ + Keeps track of the vlans for a lab. + + Vlans are represented as indexes into a 4096 element list. + This list is serialized to JSON for storing in the DB. + """ + # list of length 4096 containing either 0 (not available) or 1 (available) vlans = models.TextField() + # list of length 4096 containing either 0 (not reserved) or 1 (reserved) + reserved_vlans = models.TextField() + block_size = models.IntegerField() + + # True if the lab allows two different users to have the same private vlans + # if they use QinQ or a vxlan overlay, for example allow_overlapping = models.BooleanField() - # list of length 4096 containing either 0 (not rexerved) or 1 (reserved) - reserved_vlans = models.TextField() def get_vlan(self, count=1): + """ + Return the ID of available vlans, but does not reserve them. + + Will throw index exception if not enough vlans are available. + If count == 1, the return value is an int. Otherwise, it is a list of ints. + """ allocated = [] vlans = json.loads(self.vlans) for i in range(count): @@ -66,24 +93,35 @@ class VlanManager(models.Model): return allocated def get_public_vlan(self): + """Return reference to an available public network without reserving it.""" return PublicNetwork.objects.filter(lab=self.lab_set.first(), in_use=False).first() def reserve_public_vlan(self, vlan): + """Reserves the Public Network that has the given vlan.""" net = PublicNetwork.objects.get(lab=self.lab_set.first(), vlan=vlan, in_use=False) net.in_use = True net.save() def release_public_vlan(self, vlan): + """Un-reserves a public network with the given vlan.""" net = PublicNetwork.objects.get(lab=self.lab_set.first(), vlan=vlan, in_use=True) net.in_use = False net.save() def public_vlan_is_available(self, vlan): + """ + Whether the public vlan is available. + + returns true if the network with the given vlan is free to use, + False otherwise + """ net = PublicNetwork.objects.get(lab=self.lab_set.first(), vlan=vlan) return not net.in_use def is_available(self, vlans): """ + If the vlans are available. + 'vlans' is either a single vlan id integer or a list of integers will return true (available) or false """ @@ -104,6 +142,8 @@ class VlanManager(models.Model): def release_vlans(self, vlans): """ + Make the vlans available for another booking. + 'vlans' is either a single vlan id integer or a list of integers will make the vlans available doesnt return a value @@ -121,6 +161,11 @@ class VlanManager(models.Model): self.save() def reserve_vlans(self, vlans): + """ + Reserves all given vlans or throws a ValueError. + + vlans can be an integer or a list of integers. + """ my_vlans = json.loads(self.vlans) try: @@ -140,6 +185,13 @@ class VlanManager(models.Model): class Lab(models.Model): + """ + Model representing a Hosting Lab. + + Anybody that wants to host resources for LaaS needs to have a Lab model + We associate hardware with Labs so we know what is available and where. + """ + lab_user = models.OneToOneField(User, on_delete=models.CASCADE) name = models.CharField(max_length=200, primary_key=True, unique=True, null=False, blank=False) contact_email = models.EmailField(max_length=200, null=True, blank=True) @@ -147,11 +199,13 @@ class Lab(models.Model): status = models.IntegerField(default=LabStatus.UP) vlan_manager = models.ForeignKey(VlanManager, on_delete=models.CASCADE, null=True) location = models.TextField(default="unknown") + # This token must apear in API requests from this lab api_token = models.CharField(max_length=50) description = models.CharField(max_length=240) @staticmethod def make_api_token(): + """Generate random 45 character string for API token.""" alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" key = "" for i in range(45): @@ -163,6 +217,8 @@ class Lab(models.Model): class PublicNetwork(models.Model): + """L2/L3 network that can reach the internet.""" + vlan = models.IntegerField() lab = models.ForeignKey(Lab, on_delete=models.CASCADE) in_use = models.BooleanField(default=False) @@ -171,6 +227,13 @@ class PublicNetwork(models.Model): class Downtime(models.Model): + """ + A Downtime event. + + Labs can create Downtime objects so the dashboard can + alert users that the lab is down, etc + """ + start = models.DateTimeField() end = models.DateTimeField() lab = models.ForeignKey(Lab, on_delete=models.CASCADE) diff --git a/src/account/tests/test_general.py b/src/account/tests/test_general.py index 3fb52b0..4020d89 100644 --- a/src/account/tests/test_general.py +++ b/src/account/tests/test_general.py @@ -27,8 +27,10 @@ class AccountMiddlewareTestCase(TestCase): def test_timezone_middleware(self): """ - The timezone should be UTC for anonymous users, for authenticated users it should be set - to user.userprofile.timezone + Verify timezone is being set by Middleware. + + The timezone should be UTC for anonymous users, + for authenticated users it should be set to user.userprofile.timezone """ # default self.assertEqual(timezone.get_current_timezone_name(), 'UTC') diff --git a/src/account/urls.py b/src/account/urls.py index 47400e5..0c01ee0 100644 --- a/src/account/urls.py +++ b/src/account/urls.py @@ -9,7 +9,8 @@ ############################################################################## -"""laas_dashboard URL Configuration +""" +laas_dashboard URL Configuration. The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/1.10/topics/http/urls/ diff --git a/src/account/views.py b/src/account/views.py index 5b91550..ccc4c8d 100644 --- a/src/account/views.py +++ b/src/account/views.py @@ -169,6 +169,8 @@ def account_detail_view(request): def account_resource_view(request): """ + Display a user's resources. + gathers a users genericResoureBundles and turns them into displayable objects """ diff --git a/src/api/models.py b/src/api/models.py index 520e747..1e5a2da 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -36,6 +36,14 @@ from account.models import Downtime class JobStatus(object): + """ + A poor man's enum for a job's status. + + A job is NEW if it has not been started or recognized by the Lab + A job is CURRENT if it has been started by the lab but it is not yet completed + a job is DONE if all the tasks are complete and the booking is ready to use + """ + NEW = 0 CURRENT = 100 DONE = 200 @@ -47,8 +55,11 @@ class LabManagerTracker(object): @classmethod def get(cls, lab_name, token): """ + Get a LabManager. + Takes in a lab name (from a url path) returns a lab manager instance for that lab, if it exists + Also checks that the given API token is correct """ try: lab = Lab.objects.get(name=lab_name) @@ -61,8 +72,8 @@ class LabManagerTracker(object): class LabManager(object): """ - This is the class that will ultimately handle all REST calls to - lab endpoints. + Handles all lab REST calls. + handles jobs, inventory, status, etc may need to create helper classes """ @@ -86,7 +97,9 @@ class LabManager(object): def create_downtime(self, form): """ - takes in a dictionary that describes the model. + Create a downtime event. + + Takes in a dictionary that describes the model. { "start": utc timestamp "end": utc timestamp @@ -287,8 +300,15 @@ class LabManager(object): class Job(models.Model): """ + A Job to be performed by the Lab. + + The API uses Jobs and Tasks to communicate actions that need to be taken to the Lab + that is hosting a booking. A booking from a user has an associated Job which tells + the lab how to configure the hardware, networking, etc to fulfill the booking + for the user. This is the class that is serialized and put into the api """ + booking = models.OneToOneField(Booking, on_delete=models.CASCADE, null=True) status = models.IntegerField(default=JobStatus.NEW) complete = models.BooleanField(default=False) @@ -321,6 +341,8 @@ class Job(models.Model): def is_fulfilled(self): """ + If a job has been completed by the lab. + This method should return true if all of the job's tasks are done, and false otherwise """ @@ -383,10 +405,8 @@ class TaskConfig(models.Model): class BridgeConfig(models.Model): - """ - Displays mapping between jumphost interfaces and - bridges - """ + """Displays mapping between jumphost interfaces and bridges.""" + interfaces = models.ManyToManyField(Interface) opnfv_config = models.ForeignKey(OPNFVConfig, on_delete=models.CASCADE) @@ -554,9 +574,8 @@ class AccessConfig(TaskConfig): class SoftwareConfig(TaskConfig): - """ - handled opnfv installations, etc - """ + """Handles software installations, such as OPNFV or ONAP.""" + opnfv = models.ForeignKey(OpnfvApiConfig, on_delete=models.CASCADE) def to_dict(self): @@ -584,9 +603,8 @@ class SoftwareConfig(TaskConfig): class HardwareConfig(TaskConfig): - """ - handles imaging, user accounts, etc - """ + """Describes the desired configuration of the hardware.""" + image = models.CharField(max_length=100, default="defimage") power = models.CharField(max_length=100, default="off") hostname = models.CharField(max_length=100, default="hostname") @@ -602,9 +620,8 @@ class HardwareConfig(TaskConfig): class NetworkConfig(TaskConfig): - """ - handles network configuration - """ + """Handles network configuration.""" + interfaces = models.ManyToManyField(Interface) delta = models.TextField() @@ -718,6 +735,13 @@ def get_task_uuid(): class TaskRelation(models.Model): + """ + Relates a Job to a TaskConfig. + + superclass that relates a Job to tasks anc maintains information + like status and messages from the lab + """ + status = models.IntegerField(default=JobStatus.NEW) job = models.ForeignKey(Job, on_delete=models.CASCADE) config = models.OneToOneField(TaskConfig, on_delete=models.CASCADE) @@ -808,13 +832,11 @@ class SnapshotRelation(TaskRelation): class JobFactory(object): + """This class creates all the API models (jobs, tasks, etc) needed to fulfill a booking.""" @classmethod def reimageHost(cls, new_image, booking, host): - """ - This method will make all necessary changes to make a lab - reimage a host. - """ + """Modify an existing job to reimage the given host.""" job = Job.objects.get(booking=booking) # make hardware task new hardware_relation = HostHardwareRelation.objects.get(host=host, job=job) @@ -853,6 +875,7 @@ class JobFactory(object): @classmethod def makeCompleteJob(cls, booking): + """Create everything that is needed to fulfill the given booking.""" hosts = Host.objects.filter(bundle=booking.resource) job = None try: @@ -896,6 +919,12 @@ class JobFactory(object): @classmethod def makeHardwareConfigs(cls, hosts=[], job=Job()): + """ + Create and save HardwareConfig. + + Helper function to create the tasks related to + configuring the hardware + """ for host in hosts: hardware_config = None try: @@ -916,6 +945,12 @@ class JobFactory(object): @classmethod def makeAccessConfig(cls, users, access_type, revoke=False, job=Job(), context=False): + """ + Create and save AccessConfig. + + Helper function to create the tasks related to + configuring the VPN, SSH, etc access for users + """ for user in users: relation = AccessRelation() relation.job = job @@ -935,6 +970,12 @@ class JobFactory(object): @classmethod def makeNetworkConfigs(cls, hosts=[], job=Job()): + """ + Create and save NetworkConfig. + + Helper function to create the tasks related to + configuring the networking + """ for host in hosts: network_config = None try: @@ -975,7 +1016,12 @@ class JobFactory(object): @classmethod def makeSoftware(cls, booking=None, job=Job()): + """ + Create and save SoftwareConfig. + Helper function to create the tasks related to + configuring the desired software, e.g. an OPNFV deployment + """ if not booking.opnfv_config: return None diff --git a/src/api/serializers/booking_serializer.py b/src/api/serializers/booking_serializer.py index 9b5c059..46a2348 100644 --- a/src/api/serializers/booking_serializer.py +++ b/src/api/serializers/booking_serializer.py @@ -25,7 +25,8 @@ class BookingField(serializers.Field): def to_representation(self, booking): """ - Takes in a booking object. + Take in a booking object. + Returns a dictionary of primitives representing that booking """ ser = {} @@ -75,8 +76,7 @@ class BookingField(serializers.Field): def to_internal_value(self, data): """ - Takes in a dictionary of primitives - Returns a booking object + Take in a dictionary of primitives, and return a booking object. This is not going to be implemented or allowed. If someone needs to create a booking through the api, @@ -146,9 +146,7 @@ class InterfaceField(serializers.Field): pass def to_internal_value(self, data): - """ - takes in a serialized interface and creates an Interface model - """ + """Take in a serialized interface and creates an Interface model.""" mac = data['mac'] bus_address = data['busaddr'] switch_name = data['switchport']['switch_name'] diff --git a/src/api/tests/test_models_unittest.py b/src/api/tests/test_models_unittest.py index 2ecbe42..2a6fa0b 100644 --- a/src/api/tests/test_models_unittest.py +++ b/src/api/tests/test_models_unittest.py @@ -108,7 +108,8 @@ class ValidBookingCreatesValidJob(TestCase): def make_networks(self, hostprofile, nets): """ - distributes nets accross hostprofile's interfaces + Distribute nets accross hostprofile's interfaces. + returns a 2D array """ network_struct = [] diff --git a/src/api/urls.py b/src/api/urls.py index 0e84a6a..39f07df 100644 --- a/src/api/urls.py +++ b/src/api/urls.py @@ -9,7 +9,8 @@ ############################################################################## -"""laas_dashboard URL Configuration +""" +laas_dashboard URL Configuration. The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/1.10/topics/http/urls/ diff --git a/src/api/views.py b/src/api/views.py index a5153d7..bc01562 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -27,6 +27,18 @@ from booking.models import Booking from api.models import LabManagerTracker, get_task from notifier.manager import NotificationHandler +""" +API views. + +All functions return a Json blob +Most functions that deal with info from a specific lab (tasks, host info) +requires the Lab auth token. + for example, curl -H auth-token:mylabsauthtoken url + +Most functions let you GET or POST to the same endpoint, and +the correct thing will happen +""" + class BookingViewSet(viewsets.ModelViewSet): queryset = Booking.objects.all() diff --git a/src/booking/forms.py b/src/booking/forms.py index df88cc6..9b4db86 100644 --- a/src/booking/forms.py +++ b/src/booking/forms.py @@ -62,6 +62,8 @@ class QuickBookingForm(forms.Form): def build_user_list(self): """ + Build list of UserProfiles. + returns a mapping of UserProfile ids to displayable objects expected by searchable multiple select widget """ diff --git a/src/booking/models.py b/src/booking/models.py index 9836730..8f2446f 100644 --- a/src/booking/models.py +++ b/src/booking/models.py @@ -18,16 +18,22 @@ import resource_inventory.resource_manager class Booking(models.Model): id = models.AutoField(primary_key=True) + # All bookings are owned by the user who requested it owner = models.ForeignKey(User, on_delete=models.PROTECT, related_name='owner') + # an owner can add other users to the booking collaborators = models.ManyToManyField(User, related_name='collaborators') + # start and end time start = models.DateTimeField() end = models.DateTimeField() reset = models.BooleanField(default=False) jira_issue_id = models.IntegerField(null=True, blank=True) jira_issue_status = models.CharField(max_length=50, blank=True) purpose = models.CharField(max_length=300, blank=False) + # bookings can be extended a limited number of times ext_count = models.IntegerField(default=2) + # the hardware that the user has booked resource = models.ForeignKey(ResourceBundle, on_delete=models.SET_NULL, null=True) + # configuration for the above hardware config_bundle = models.ForeignKey(ConfigBundle, on_delete=models.SET_NULL, null=True) opnfv_config = models.ForeignKey(OPNFVConfig, on_delete=models.SET_NULL, null=True) project = models.CharField(max_length=100, default="", blank=True, null=True) @@ -41,6 +47,7 @@ class Booking(models.Model): def save(self, *args, **kwargs): """ Save the booking if self.user is authorized and there is no overlapping booking. + Raise PermissionError if the user is not authorized Raise ValueError if there is an overlapping booking """ diff --git a/src/booking/quick_deployer.py b/src/booking/quick_deployer.py index 4ec488e..743cdcf 100644 --- a/src/booking/quick_deployer.py +++ b/src/booking/quick_deployer.py @@ -97,6 +97,11 @@ class BookingPermissionException(Exception): def parse_host_field(host_json): + """ + Parse the json from the frontend. + + returns a reference to the selected Lab and HostProfile objects + """ lab, profile = (None, None) lab_dict = host_json['lab'] for lab_info in lab_dict.values(): @@ -117,6 +122,12 @@ def parse_host_field(host_json): def check_available_matching_host(lab, hostprofile): + """ + Check the resources are available. + + Returns true if the requested host type is availble, + Or throws an exception + """ available_host_types = ResourceManager.getInstance().getAvailableHostTypes(lab) if hostprofile not in available_host_types: # TODO: handle deleting generic resource in this instance along with grb @@ -129,7 +140,10 @@ def check_available_matching_host(lab, hostprofile): return True +# Functions to create models + def generate_grb(owner, lab, common_id): + """Create a Generic Resource Bundle.""" grbundle = GenericResourceBundle(owner=owner) grbundle.lab = lab grbundle.name = "grbundle for quick booking with uid " + common_id @@ -140,6 +154,7 @@ def generate_grb(owner, lab, common_id): def generate_gresource(bundle, hostname): + """Create a Generic Resource.""" if not re.match(r"(?=^.{1,253}$)(^([A-Za-z0-9-_]{1,62}\.)*[A-Za-z0-9-_]{1,63})$", hostname): raise InvalidHostnameException("Hostname must comply to RFC 952 and all extensions to it until this point") gresource = GenericResource(bundle=bundle, name=hostname) @@ -149,6 +164,7 @@ def generate_gresource(bundle, hostname): def generate_ghost(generic_resource, host_profile): + """Create a Generic Host.""" ghost = GenericHost() ghost.resource = generic_resource ghost.profile = host_profile @@ -158,6 +174,7 @@ def generate_ghost(generic_resource, host_profile): def generate_config_bundle(owner, common_id, grbundle): + """Create a Configuration Bundle.""" cbundle = ConfigBundle() cbundle.owner = owner cbundle.name = "configbundle for quick booking with uid " + common_id @@ -169,6 +186,7 @@ def generate_config_bundle(owner, common_id, grbundle): def generate_opnfvconfig(scenario, installer, config_bundle): + """Create an OPNFV Configuration.""" opnfvconfig = OPNFVConfig() opnfvconfig.scenario = scenario opnfvconfig.installer = installer @@ -179,6 +197,7 @@ def generate_opnfvconfig(scenario, installer, config_bundle): def generate_hostconfig(generic_host, image, config_bundle): + """Create a Host Configuration.""" hconf = HostConfiguration() hconf.host = generic_host hconf.image = image @@ -190,6 +209,7 @@ def generate_hostconfig(generic_host, image, config_bundle): def generate_hostopnfv(hostconfig, opnfvconfig): + """Relate the Host and OPNFV Configs.""" config = HostOPNFVConfig() role = None try: @@ -207,6 +227,7 @@ def generate_hostopnfv(hostconfig, opnfvconfig): def generate_resource_bundle(generic_resource_bundle, config_bundle): # warning: requires cleanup + """Create a Resource Bundle.""" try: resource_manager = ResourceManager.getInstance() resource_bundle = resource_manager.convertResourceBundle(generic_resource_bundle, config=config_bundle) @@ -218,6 +239,11 @@ def generate_resource_bundle(generic_resource_bundle, config_bundle): # warning def check_invariants(request, **kwargs): + """ + Verify all the contraints on the requested booking. + + verifies software compatibility, booking length, etc + """ installer = kwargs['installer'] image = kwargs['image'] scenario = kwargs['scenario'] @@ -256,6 +282,12 @@ def configure_networking(grb, config): def create_from_form(form, request): + """ + Create a Booking from the user's form. + + Large, nasty method to create a booking or return a useful error + based on the form from the frontend + """ quick_booking_id = str(uuid.uuid4()) host_field = form.cleaned_data['filter_field'] @@ -330,6 +362,13 @@ def create_from_form(form, request): def drop_filter(user): + """ + Return a dictionary that contains filters. + + Only certain installlers are supported on certain images, etc + so the image filter indexed at [imageid][installerid] is truthy if + that installer is supported on that image + """ installer_filter = {} for image in Image.objects.all(): installer_filter[image.id] = {} diff --git a/src/booking/stats.py b/src/booking/stats.py index 383723a..47de80b 100644 --- a/src/booking/stats.py +++ b/src/booking/stats.py @@ -16,6 +16,8 @@ class StatisticsManager(object): @staticmethod def getContinuousBookingTimeSeries(span=28): """ + Calculate Booking usage data points. + Will return a dictionary of names and 2-D array of x and y data points. e.g. {"plot1": [["x1", "x2", "x3"],["y1", "y2", "y3]]} x values will be dates in string diff --git a/src/booking/tests/test_models.py b/src/booking/tests/test_models.py index 6170295..c8c8ea8 100644 --- a/src/booking/tests/test_models.py +++ b/src/booking/tests/test_models.py @@ -21,10 +21,20 @@ from resource_inventory.models import ResourceBundle, GenericResourceBundle, Con class BookingModelTestCase(TestCase): + """ + Test the Booking model. + + Creates all the scafolding needed and tests the Booking model + """ count = 0 def setUp(self): + """ + Prepare for Booking model tests. + + Creates all the needed models, such as users, resources, and configurations + """ self.owner = User.objects.create(username='owner') self.res1 = ResourceBundle.objects.create( @@ -52,6 +62,8 @@ class BookingModelTestCase(TestCase): def test_start_end(self): """ + Verify the start and end fields. + if the start of a booking is greater or equal then the end, saving should raise a ValueException """ @@ -79,6 +91,8 @@ class BookingModelTestCase(TestCase): def test_conflicts(self): """ + Verify conflicting dates are dealt with. + saving an overlapping booking on the same resource should raise a ValueException saving for different resources should succeed @@ -207,6 +221,8 @@ class BookingModelTestCase(TestCase): def test_extensions(self): """ + Test booking extensions. + saving a booking with an extended end time is allows to happen twice, and each extension must be a maximum of one week long """ diff --git a/src/booking/urls.py b/src/booking/urls.py index 54e29c9..d5287e9 100644 --- a/src/booking/urls.py +++ b/src/booking/urls.py @@ -9,7 +9,8 @@ ############################################################################## -"""laas_dashboard URL Configuration +""" +laas_dashboard URL Configuration. The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/1.10/topics/http/urls/ diff --git a/src/dashboard/exceptions.py b/src/dashboard/exceptions.py index 7111bf8..61df145 100644 --- a/src/dashboard/exceptions.py +++ b/src/dashboard/exceptions.py @@ -9,30 +9,26 @@ class ResourceProvisioningException(Exception): - """ - Resources could not be provisioned - """ + """Resources could not be provisioned.""" + pass class ModelValidationException(Exception): - """ - Validation before saving model returned issues - """ + """Validation before saving model returned issues.""" + pass class ResourceAvailabilityException(ResourceProvisioningException): - """ - Requested resources are not *currently* available - """ + """Requested resources are not *currently* available.""" + pass class ResourceExistenceException(ResourceAvailabilityException): - """ - Requested resources do not exist or do not match any known resources - """ + """Requested resources do not exist or do not match any known resources.""" + pass diff --git a/src/dashboard/populate_db_iol.py b/src/dashboard/populate_db_iol.py index 57ebd40..d8df03f 100644 --- a/src/dashboard/populate_db_iol.py +++ b/src/dashboard/populate_db_iol.py @@ -216,6 +216,8 @@ class Populator: def make_profile_data(self): """ + Create Profile Data. + returns a dictionary of data from the yaml files created by inspection scripts """ diff --git a/src/dashboard/tasks.py b/src/dashboard/tasks.py index 71afed2..ac4d36f 100644 --- a/src/dashboard/tasks.py +++ b/src/dashboard/tasks.py @@ -88,9 +88,7 @@ def booking_poll(): @shared_task def free_hosts(): - """ - gets all hosts from the database that need to be freed and frees them - """ + """Free all hosts that should be freed.""" undone_statuses = [JobStatus.NEW, JobStatus.CURRENT, JobStatus.ERROR] undone_jobs = Job.objects.filter( hostnetworkrelation__status__in=undone_statuses, diff --git a/src/dashboard/urls.py b/src/dashboard/urls.py index 60334f6..d5dad57 100644 --- a/src/dashboard/urls.py +++ b/src/dashboard/urls.py @@ -9,7 +9,8 @@ ############################################################################## -"""laas_dashboard URL Configuration +""" +laas_dashboard URL Configuration. The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/1.10/topics/http/urls/ diff --git a/src/laas_dashboard/settings.py b/src/laas_dashboard/settings.py index d41ad0c..62fc9ec 100644 --- a/src/laas_dashboard/settings.py +++ b/src/laas_dashboard/settings.py @@ -219,5 +219,5 @@ EMAIL_HOST_PASSWORD = os.environ['EMAIL_HOST_PASSWORD'] EMAIL_USE_TLS = True DEFAULT_EMAIL_FROM = os.environ.get('DEFAULT_EMAIL_FROM', 'webmaster@localhost') SESSION_ENGINE = "django.contrib.sessions.backends.signed_cookies" -EXPIRE_LIFETIME = 12 # Minimum lifetime of booking to send notification -EXPIRE_HOURS = 48 # Notify when booking is expiring within this many hours +EXPIRE_LIFETIME = 12 # Minimum lifetime of booking to send notification +EXPIRE_HOURS = 48 # Notify when booking is expiring within this many hours diff --git a/src/laas_dashboard/urls.py b/src/laas_dashboard/urls.py index f90f18b..17cbe84 100644 --- a/src/laas_dashboard/urls.py +++ b/src/laas_dashboard/urls.py @@ -9,7 +9,8 @@ ############################################################################## -"""laas_dashboard URL Configuration +""" +laas_dashboard URL Configuration. The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/1.10/topics/http/urls/ diff --git a/src/notifier/manager.py b/src/notifier/manager.py index ee849a8..a5b7b9a 100644 --- a/src/notifier/manager.py +++ b/src/notifier/manager.py @@ -38,8 +38,8 @@ class NotificationHandler(object): @classmethod def booking_notify(cls, booking, template, titles): """ - Creates a notification for a booking owner and collaborators - using the template. + Create a notification for a booking owner and collaborators using the template. + titles is a list - the first is the title for the owner's notification, the last is the title for the collaborators' """ @@ -158,6 +158,8 @@ class NotificationHandler(object): @classmethod def task_updated(cls, task): """ + Notification of task changing. + called every time a lab updated info about a task. sends an email when 'task' changing state means a booking has just been fulfilled (all tasks done, servers ready to use) diff --git a/src/notifier/models.py b/src/notifier/models.py index 382d3a9..0af748b 100644 --- a/src/notifier/models.py +++ b/src/notifier/models.py @@ -27,10 +27,8 @@ class Notification(models.Model): class Emailed(models.Model): - """ - A simple record to remember who has already gotten an email - to avoid resending - """ + """A simple record to remember who has already gotten an email to avoid resending.""" + begin_booking = models.OneToOneField( Booking, null=True, @@ -49,4 +47,3 @@ class Emailed(models.Model): on_delete=models.CASCADE, related_name="over_mail" ) - diff --git a/src/notifier/tasks.py b/src/notifier/tasks.py index b45ab8e..474d64d 100644 --- a/src/notifier/tasks.py +++ b/src/notifier/tasks.py @@ -19,15 +19,15 @@ from notifier.manager import NotificationHandler @shared_task def notify_expiring(): - """ - Notify users if their booking is within 48 hours of expiring. - """ + """Notify users if their booking is within 48 hours of expiring.""" expire_time = timezone.now() + timezone.timedelta(hours=settings.EXPIRE_HOURS) # Don't email people about bookings that have started recently start_time = timezone.now() - timezone.timedelta(hours=settings.EXPIRE_LIFETIME) - bookings = Booking.objects.filter(end__lte=expire_time, + bookings = Booking.objects.filter( + end__lte=expire_time, end__gte=timezone.now(), - start__lte=start_time) + start__lte=start_time + ) for booking in bookings: if Emailed.objects.filter(almost_end_booking=booking).exists(): continue diff --git a/src/resource_inventory/idf_templater.py b/src/resource_inventory/idf_templater.py index bf6eda0..8f0f924 100644 --- a/src/resource_inventory/idf_templater.py +++ b/src/resource_inventory/idf_templater.py @@ -16,9 +16,8 @@ from resource_inventory.models import Vlan class IDFTemplater: - """ - Utility class to create a full IDF yaml file - """ + """Utility class to create a full Installer Descriptor File (IDF) yaml file.""" + net_names = ["admin", "mgmt", "private", "public"] bridge_names = { "admin": "br-admin", @@ -39,9 +38,7 @@ class IDFTemplater: } def makeIDF(self, booking): - """ - fills the installer descriptor file template with info about the resource - """ + """Fill the IDF template with info about the resource.""" template = "dashboard/idf.yaml" info = {} info['version'] = "0.1" diff --git a/src/resource_inventory/models.py b/src/resource_inventory/models.py index d152698..4bc9bf3 100644 --- a/src/resource_inventory/models.py +++ b/src/resource_inventory/models.py @@ -111,24 +111,20 @@ class Resource(models.Model): def get_configuration(self, state): """ + Get configuration of Resource. + Returns the desired configuration for this host as a JSON object as defined in the rest api spec. state is a ConfigState - TODO: single method, or different methods for hw, network, snapshot, etc? """ raise NotImplementedError("Must implement in concrete Resource classes") def reserve(self): - """ - Reserves this resource for its currently - assigned booking. - """ + """Reserve this resource for its currently assigned booking.""" raise NotImplementedError("Must implement in concrete Resource classes") def release(self): - """ - Makes this resource available again for new boookings - """ + """Make this resource available again for new boookings.""" raise NotImplementedError("Must implement in concrete Resource classes") @@ -170,15 +166,14 @@ class PhysicalNetwork(Resource): def get_configuration(self, state): """ - Returns the network configuration + Get the network configuration. + Collects info about each attached network interface and vlan, etc """ return {} def reserve(self): - """ - Reserves vlan(s) associated with this network - """ + """Reserve vlan(s) associated with this network.""" # vlan_manager = self.bundle.lab.vlan_manager return False @@ -329,9 +324,8 @@ class OPNFVRole(models.Model): class Image(models.Model): - """ - model for representing OS images / snapshots of hosts - """ + """Model for representing OS images / snapshots of hosts.""" + id = models.AutoField(primary_key=True) lab_id = models.IntegerField() # ID the lab who holds this image knows from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE) @@ -354,10 +348,8 @@ def get_sentinal_opnfv_role(): class HostConfiguration(models.Model): - """ - model to represent a complete configuration for a single - physical host - """ + """Model to represent a complete configuration for a single physical host.""" + id = models.AutoField(primary_key=True) host = models.ForeignKey(GenericHost, related_name="configuration", on_delete=models.CASCADE) image = models.ForeignKey(Image, on_delete=models.PROTECT) @@ -438,8 +430,7 @@ class Interface(models.Model): class OPNFV_SETTINGS(): - """ - This is a static configuration class - """ + """This is a static configuration class.""" + # all the required network types in PDF/IDF spec NETWORK_ROLES = ["public", "private", "admin", "mgmt"] diff --git a/src/resource_inventory/pdf_templater.py b/src/resource_inventory/pdf_templater.py index 7e91b87..51e3746 100644 --- a/src/resource_inventory/pdf_templater.py +++ b/src/resource_inventory/pdf_templater.py @@ -14,15 +14,11 @@ from resource_inventory.models import Host, InterfaceProfile class PDFTemplater: - """ - Utility class to create a full PDF yaml file - """ + """Utility class to create a full PDF yaml file.""" @classmethod def makePDF(cls, booking): - """ - fills the pod descriptor file template with info about the resource - """ + """Fill the pod descriptor file template with info about the resource.""" template = "dashboard/pdf.yaml" info = {} info['details'] = cls.get_pdf_details(booking.resource) @@ -33,9 +29,7 @@ class PDFTemplater: @classmethod def get_pdf_details(cls, resource): - """ - Info for the "details" section - """ + """Info for the "details" section.""" details = {} owner = "Anon" email = "email@mail.com" @@ -64,6 +58,7 @@ class PDFTemplater: @classmethod def get_jumphost(cls, booking): + """Return the host designated as the Jumphost for the booking.""" jumphost = None if booking.opnfv_config: jumphost_opnfv_config = booking.opnfv_config.host_opnfv_config.get( @@ -80,9 +75,7 @@ class PDFTemplater: @classmethod def get_pdf_jumphost(cls, booking): - """ - returns a dict of all the info for the "jumphost" section - """ + """Return a dict of all the info for the "jumphost" section.""" jumphost = cls.get_jumphost(booking) jumphost_info = cls.get_pdf_host(jumphost) jumphost_info['os'] = jumphost.config.image.os.name @@ -90,9 +83,7 @@ class PDFTemplater: @classmethod def get_pdf_nodes(cls, booking): - """ - returns a list of all the "nodes" (every host except jumphost) - """ + """Return a list of all the "nodes" (every host except jumphost).""" pdf_nodes = [] nodes = set(Host.objects.filter(bundle=booking.resource)) nodes.discard(cls.get_jumphost(booking)) @@ -105,8 +96,9 @@ class PDFTemplater: @classmethod def get_pdf_host(cls, host): """ - method to gather all needed info about a host - returns a dict + Gather all needed info about a host. + + returns a dictionary """ host_info = {} host_info['name'] = host.template.resource.name @@ -125,9 +117,7 @@ class PDFTemplater: @classmethod def get_pdf_host_node(cls, host): - """ - returns "node" info for a given host - """ + """Return "node" info for a given host.""" d = {} d['type'] = "baremetal" d['vendor'] = host.vendor @@ -148,9 +138,7 @@ class PDFTemplater: @classmethod def get_pdf_host_disk(cls, disk): - """ - returns a dict describing the given disk - """ + """Return a dict describing the given disk.""" disk_info = {} disk_info['name'] = disk.name disk_info['capacity'] = str(disk.size) + "G" @@ -161,9 +149,7 @@ class PDFTemplater: @classmethod def get_pdf_host_iface(cls, interface): - """ - returns a dict describing given interface - """ + """Return a dict describing given interface.""" iface_info = {} iface_info['features'] = "none" iface_info['mac_address'] = interface.mac_address @@ -179,9 +165,7 @@ class PDFTemplater: @classmethod def get_pdf_host_remote_management(cls, host): - """ - gives the remote params of the host - """ + """Get the remote params of the host.""" man = host.remote_management mgmt = {} mgmt['address'] = man.address diff --git a/src/resource_inventory/resource_manager.py b/src/resource_inventory/resource_manager.py index 7df4263..34c7be3 100644 --- a/src/resource_inventory/resource_manager.py +++ b/src/resource_inventory/resource_manager.py @@ -44,10 +44,10 @@ class ResourceManager: def hostsAvailable(self, grb): """ - This method will check if the given GenericResourceBundle - is available. No changes to the database - """ + Check if the given GenericResourceBundle is available. + No changes to the database + """ # count up hosts profile_count = {} for host in grb.getResources(): @@ -90,7 +90,10 @@ class ResourceManager: def convertResourceBundle(self, genericResourceBundle, config=None): """ - Takes in a GenericResourceBundle and 'converts' it into a ResourceBundle + Convert a GenericResourceBundle into a ResourceBundle. + + Takes in a genericResourceBundle and reserves all the + Resources needed and returns a completed ResourceBundle. """ resource_bundle = ResourceBundle.objects.create(template=genericResourceBundle) generic_hosts = genericResourceBundle.getResources() diff --git a/src/resource_inventory/urls.py b/src/resource_inventory/urls.py index a1eace7..a008176 100644 --- a/src/resource_inventory/urls.py +++ b/src/resource_inventory/urls.py @@ -9,7 +9,8 @@ ############################################################################## -"""laas_dashboard URL Configuration +""" +laas_dashboard URL Configuration. The `urlpatterns` list routes URLs to views. For more information please see: https://docs.djangoproject.com/en/1.10/topics/http/urls/ diff --git a/src/workflow/README b/src/workflow/README new file mode 100644 index 0000000..fb4b949 --- /dev/null +++ b/src/workflow/README @@ -0,0 +1,31 @@ +This app creates "workflows", which are long and complex interactions from the user. +Workflows are composed of multiple steps. At each step the user inputs some information. +The content of one step may impact following steps. + +The WorkflowStep object is the abstract type for all the workflow steps. +Important attributes and methods: + +template - the django template to use when rendering this step +valid - the status code from WorkflowStepStatus + +get_context() - returns a dictionary that is used when rendering this step's template + You should always call super's get_context and add / overwrite any data into that + dictionary + +post(data, user) - this method is called when the step is POST'd to. + data is from the request object, suitable for a Form's constructor + + +Repository +Each step has a reference to a shared repository (self.repo). +The repo is a key-value store that allows the steps to share data + +Steps render based on the current state of the repo. For example, a step +may get information about each host the user said they want and ask for additional +input for each machine. +Because the steps render based on what is in the repo, a user can easily go back to +a previous step and change some data. This data will change in the repo and +affect later steps accordingly. + +Everything stored in the repo is temporary. After a workflow has been completed, the repo +is translated into Django models and saved to the database. diff --git a/src/workflow/forms.py b/src/workflow/forms.py index 4d5e9e2..f7a20eb 100644 --- a/src/workflow/forms.py +++ b/src/workflow/forms.py @@ -65,7 +65,9 @@ class SearchableSelectMultipleField(forms.Field): items=None, queryset=None, show_from_noentry=True, show_x_results=-1, results_scrollable=False, selectable_limit=-1, placeholder="search here", name="searchable_select", initial=[], **kwargs): - """from the documentation: + """ + From the documentation. + # required -- Boolean that specifies whether the field is required. # True by default. # widget -- A Widget class, or instance of a Widget class, that should @@ -90,7 +92,6 @@ class SearchableSelectMultipleField(forms.Field): # label_suffix -- Suffix to be added to the label. Overrides # form's label_suffix. """ - self.widget = widget if self.widget is None: self.widget = SearchableSelectMultipleWidget( @@ -287,8 +288,9 @@ class FormUtils: @staticmethod def getLabData(multiple_hosts=False): """ - Gets all labs and thier host profiles and returns a serialized version the form can understand. - Should be rewritten with a related query to make it faster + Get all labs and thier host profiles, returns a serialized version the form can understand. + + Could be rewritten with a related query to make it faster """ # javascript truthy variables true = 1 diff --git a/src/workflow/models.py b/src/workflow/models.py index 99608f6..32ac39c 100644 --- a/src/workflow/models.py +++ b/src/workflow/models.py @@ -26,6 +26,15 @@ from booking.models import Booking class BookingAuthManager(): + """ + Verifies Booking Authorization. + + Class to verify that the user is allowed to book the requested resource + The user must input a url to the INFO.yaml file to prove that they are the ptl of + an approved project if they are booking a multi-node pod. + This class parses the url and checks the logged in user against the info file. + """ + LFN_PROJECTS = ["opnfv"] # TODO def parse_github_url(self, url): @@ -124,7 +133,9 @@ class BookingAuthManager(): def parse_url(self, info_url): """ - will return the PTL in the INFO file on success, or None + Parse the project URL. + + Gets the INFO.yaml file from the project and returns the PTL info. """ if "github" in info_url: return self.parse_github_url(info_url) @@ -137,6 +148,8 @@ class BookingAuthManager(): def booking_allowed(self, booking, repo): """ + Assert the current Booking Policy. + This is the method that will have to change whenever the booking policy changes in the Infra group / LFN. This is a nice isolation of that administration crap currently checks if the booking uses multiple servers. if it does, then the owner must be a PTL, @@ -158,6 +171,14 @@ class BookingAuthManager(): class WorkflowStepStatus(object): + """ + Poor man's enum for the status of a workflow step. + + The steps in a workflow are not completed (UNTOUCHED) + or they have been completed correctly (VALID) or they were filled out + incorrectly (INVALID) + """ + UNTOUCHED = 0 INVALID = 100 VALID = 200 diff --git a/src/workflow/resource_bundle_workflow.py b/src/workflow/resource_bundle_workflow.py index 2f4aa5d..f57476b 100644 --- a/src/workflow/resource_bundle_workflow.py +++ b/src/workflow/resource_bundle_workflow.py @@ -253,12 +253,13 @@ class Define_Nets(WorkflowStep): def decomposeXml(self, xmlString): """ + Translate XML into useable data. + This function takes in an xml doc from our front end and returns dictionaries that map cellIds to the xml nodes themselves. There is no unpacking of the xml objects, just grouping and organizing """ - connections = {} networks = {} hosts = {} diff --git a/src/workflow/sw_bundle_workflow.py b/src/workflow/sw_bundle_workflow.py index 4dc0b8e..ebd8c86 100644 --- a/src/workflow/sw_bundle_workflow.py +++ b/src/workflow/sw_bundle_workflow.py @@ -28,6 +28,8 @@ class Define_Software(WorkflowStep): def build_filter_data(self, hosts_data): """ + Build list of Images to filter out. + returns a 2D array of images to exclude based on the ordering of the passed hosts_data diff --git a/src/workflow/tests/test_steps.py b/src/workflow/tests/test_steps.py index 39b1f86..6101d4f 100644 --- a/src/workflow/tests/test_steps.py +++ b/src/workflow/tests/test_steps.py @@ -8,7 +8,8 @@ ############################################################################## """ -This file tests basic functionality of each step class +This file tests basic functionality of each step class. + More in depth case coverage of WorkflowStep.post() must happen elsewhere. """ @@ -28,9 +29,11 @@ from workflow.tests import test_fixtures class TestConfig: """ - Basic class to instantiate and hold reference + Basic class to instantiate and hold reference. + to models we will need often """ + def __init__(self, usr=None): self.lab = make_lab() self.user = usr or make_user() @@ -77,6 +80,8 @@ class StepTestCase(TestCase): def assertCorrectPostBehavior(self, post_data): """ + Stub for validating step behavior on POST request. + allows subclasses to override and make assertions about the side effects of self.step.post() post_data is the data passed into post() @@ -85,6 +90,8 @@ class StepTestCase(TestCase): def add_to_repo(self, repo): """ + Stub for modifying the step's repo. + This method is a hook that allows subclasses to modify the contents of the repo before the step is created. """ @@ -92,8 +99,8 @@ class StepTestCase(TestCase): def assertValidHtml(self, html_str): """ - This method should make sure that html_str is a valid - html fragment. + Assert that html_str is a valid html fragment. + However, I know of no good way of doing this in python """ self.assertTrue(isinstance(html_str, str)) diff --git a/src/workflow/tests/test_workflows.py b/src/workflow/tests/test_workflows.py index 293e43d..995d699 100644 --- a/src/workflow/tests/test_workflows.py +++ b/src/workflow/tests/test_workflows.py @@ -50,9 +50,7 @@ class WorkflowTestCase(TestCase): session.save() def render_steps(self): - """ - retrieves each step individually at /wf/workflow/step=<index> - """ + """Retrieve each step individually at /wf/workflow/step=<index>.""" for i in range(self.step_count): # renders the step itself, not in an iframe exception = None @@ -9,8 +9,7 @@ ############################################################################## # first, basic lint with flake8 -find . -type f -name "*.py" -not -name "manage.py" | xargs flake8 --count --ignore E501 - +find . -type f -name "*.py" -not -name "manage.py" -not -path "*/migrations/*" | xargs flake8 --count # this file should be executed from the dir it is in docker exec -it dg01 python manage.py test |