From e90f13e0413594d95e50256b1206ffd64217f2da Mon Sep 17 00:00:00 2001 From: Parker Berberian Date: Mon, 16 Mar 2020 10:10:30 -0400 Subject: Quick Deploy Fixes. Change-Id: I46d410af62e4962d235346ba56472aaacb9d3ff2 Signed-off-by: Parker Berberian --- src/api/models.py | 20 +++-- src/booking/quick_deployer.py | 100 ++++++++++++++++++--- src/dashboard/tasks.py | 40 ++------- src/dashboard/utils.py | 2 + src/notifier/manager.py | 4 +- .../migrations/0014_auto_20200305_1415.py | 18 ++++ src/resource_inventory/models.py | 9 +- src/resource_inventory/pdf_templater.py | 11 +-- src/resource_inventory/resource_manager.py | 21 ++--- src/templates/base/booking/quick_deploy.html | 8 +- src/workflow/forms.py | 2 +- 11 files changed, 154 insertions(+), 81 deletions(-) create mode 100644 src/resource_inventory/migrations/0014_auto_20200305_1415.py (limited to 'src') diff --git a/src/api/models.py b/src/api/models.py index de73a7a..d8e023b 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -331,7 +331,7 @@ class Job(models.Model): return {"id": self.id, "payload": d} def get_tasklist(self, status="all"): - if status == "all": + if status != "all": return JobTaskQuery.filter(job=self, status=status) return JobTaskQuery.filter(job=self) @@ -408,7 +408,7 @@ class BridgeConfig(models.Model): def to_dict(self): d = {} - hid = self.interfaces.first().host.labid + hid = ResourceQuery.get(interface__pk=self.interfaces.first().pk).labid d[hid] = {} for interface in self.interfaces.all(): d[hid][interface.mac_address] = [] @@ -611,7 +611,7 @@ class HardwareConfig(TaskConfig): def get_delta(self): return self.format_delta( - self.hosthardwarerelation.host.get_configuration(self.state), + self.hosthardwarerelation.get_resource().get_configuration(self.state), self.hosthardwarerelation.lab_token) @@ -623,7 +623,7 @@ class NetworkConfig(TaskConfig): def to_dict(self): d = {} - hid = self.hostnetworkrelation.host.labid + hid = self.hostnetworkrelation.resource_id d[hid] = {} for interface in self.interfaces.all(): d[hid][interface.mac_address] = [] @@ -652,7 +652,7 @@ class NetworkConfig(TaskConfig): def add_interface(self, interface): self.interfaces.add(interface) d = json.loads(self.delta) - hid = self.hostnetworkrelation.host.labid + hid = self.hostnetworkrelation.resource_id if hid not in d: d[hid] = {} d[hid][interface.mac_address] = [] @@ -809,6 +809,9 @@ class HostHardwareRelation(TaskRelation): raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource") super().save(*args, **kwargs) + def get_resource(self): + return ResourceQuery.get(labid=self.resource_id) + class HostNetworkRelation(TaskRelation): resource_id = models.CharField(max_length=200, default="default_id") @@ -827,6 +830,9 @@ class HostNetworkRelation(TaskRelation): raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource") super().save(*args, **kwargs) + def get_resource(self): + return ResourceQuery.get(labid=self.resource_id) + class SnapshotRelation(TaskRelation): snapshot = models.ForeignKey(Image, on_delete=models.CASCADE) @@ -1011,7 +1017,7 @@ class JobFactory(object): @classmethod def make_bridge_config(cls, booking): - if booking.resource.hosts.count() < 2: + if len(booking.resource.get_resources()) < 2: return None try: jumphost_config = ResourceOPNFVConfig.objects.filter( @@ -1049,7 +1055,7 @@ class JobFactory(object): opnfv_api_config.set_xdf(booking, False) opnfv_api_config.save() - for host in booking.resource.hosts.all(): + for host in booking.resource.get_resources(): opnfv_api_config.roles.add(host) software_config = SoftwareConfig.objects.create(opnfv=opnfv_api_config) software_relation = SoftwareRelation.objects.create(job=job, config=software_config) diff --git a/src/booking/quick_deployer.py b/src/booking/quick_deployer.py index 917f578..951ff47 100644 --- a/src/booking/quick_deployer.py +++ b/src/booking/quick_deployer.py @@ -22,6 +22,10 @@ from resource_inventory.models import ( OPNFVRole, OPNFVConfig, ResourceOPNFVConfig, + ResourceConfiguration, + NetworkConnection, + InterfaceConfiguration, + Network, ) from resource_inventory.resource_manager import ResourceManager from resource_inventory.pdf_templater import PDFTemplater @@ -56,13 +60,73 @@ def parse_resource_field(resource_json): return lab, template -def update_template(template, image, lab, hostname): +def update_template(old_template, image, hostname, user): """ - Update and copy a resource template to the user's profile. + Duplicate a template to the users account and update configured fields. - TODO: How, why, should we? + The dashboard presents users with preconfigured resource templates, + but the user might want to make small modifications, e.g hostname and + linux distro. So we copy the public template and create a private version + to the user's profile, and mark it temporary. When the booking ends the + new template is deleted """ - pass + name = user.username + "'s Copy of '" + old_template.name + "'" + num_copies = ResourceTemplate.objects.filter(name__startswith=name).count() + template = ResourceTemplate.objects.create( + name=name if num_copies == 0 else name + " (" + str(num_copies) + ")", + xml=old_template.xml, + owner=user, + lab=old_template.lab, + description=old_template.description, + public=False, + temporary=True + ) + + for old_network in old_template.networks.all(): + Network.objects.create( + name=old_network.name, + bundle=old_template, + is_public=False + ) + # We are assuming there is only one opnfv config per public resource template + old_opnfv = template.opnfv_config.first() + if old_opnfv: + opnfv_config = OPNFVConfig.objects.create( + installer=old_opnfv.installer, + scenario=old_opnfv.installer, + template=template, + name=old_opnfv.installer, + ) + # I am explicitly leaving opnfv_config.networks empty to avoid + # problems with duplicated / shared networks. In the quick deploy, + # there is never multiple networks anyway. This may have to change in the future + + for old_config in old_template.getConfigs(): + config = ResourceConfiguration.objects.create( + profile=old_config.profile, + image=image, + template=template + ) + + for old_iface_config in old_config.interface_configs.all(): + iface_config = InterfaceConfiguration.objects.create( + profile=old_iface_config.profile, + resource_config=config + ) + + for old_connection in old_iface_config.connections.all(): + iface_config.connections.add(NetworkConnection.objects.create( + network=template.networks.get(name=old_connection.network.name), + vlan_is_tagged=old_connection.vlan_is_tagged + )) + + for old_res_opnfv in old_config.resource_opnfv_config.all(): + if old_opnfv: + ResourceOPNFVConfig.objects.create( + role=old_opnfv.role, + resource_config=config, + opnfv_config=opnfv_config + ) def generate_opnfvconfig(scenario, installer, template): @@ -91,7 +155,7 @@ def generate_hostopnfv(hostconfig, opnfvconfig): def generate_resource_bundle(template): resource_manager = ResourceManager.getInstance() - resource_bundle = resource_manager.convertResourceBundle(template) + resource_bundle = resource_manager.instantiateTemplate(template) return resource_bundle @@ -101,7 +165,7 @@ def check_invariants(request, **kwargs): image = kwargs['image'] scenario = kwargs['scenario'] lab = kwargs['lab'] - host_profile = kwargs['host_profile'] + resource_template = kwargs['resource_template'] length = kwargs['length'] # check that image os is compatible with installer if installer in image.os.sup_installers.all(): @@ -112,8 +176,9 @@ def check_invariants(request, **kwargs): raise ValidationError("The chosen installer does not support the chosen scenario") if image.from_lab != lab: raise ValidationError("The chosen image is not available at the chosen hosting lab") - if image.host_type != host_profile: - raise ValidationError("The chosen image is not available for the chosen host type") + #TODO + #if image.host_type != host_profile: + # raise ValidationError("The chosen image is not available for the chosen host type") if not image.public and image.owner != request.user: raise ValidationError("You are not the owner of the chosen private image") if length < 1 or length > 21: @@ -152,7 +217,7 @@ def create_from_form(form, request): ResourceManager.getInstance().templateIsReservable(resource_template) - hconf = update_template(resource_template, image, lab, hostname) + hconf = update_template(resource_template, image, hostname, request.user) # if no installer provided, just create blank host opnfv_config = None @@ -213,10 +278,19 @@ def drop_filter(user): for image in images: image_filter[image.id] = { 'lab': 'lab_' + str(image.from_lab.lab_user.id), - 'host_profile': 'host_' + str(image.host_type.id), + 'host_profile': str(image.host_type.id), 'name': image.name } - return {'installer_filter': json.dumps(installer_filter), - 'scenario_filter': json.dumps(scenario_filter), - 'image_filter': json.dumps(image_filter)} + resource_filter = {} + templates = ResourceTemplate.objects.filter(Q(public=True) | Q(owner=user)) + for rt in templates: + profiles = [conf.profile for conf in rt.getConfigs()] + resource_filter["resource_" + str(rt.id)] = [str(p.id) for p in profiles] + + return { + 'installer_filter': json.dumps(installer_filter), + 'scenario_filter': json.dumps(scenario_filter), + 'image_filter': json.dumps(image_filter), + 'resource_profile_map': json.dumps(resource_filter), + } diff --git a/src/dashboard/tasks.py b/src/dashboard/tasks.py index ac4d36f..b980799 100644 --- a/src/dashboard/tasks.py +++ b/src/dashboard/tasks.py @@ -15,43 +15,15 @@ from booking.models import Booking from notifier.manager import NotificationHandler from api.models import Job, JobStatus, SoftwareRelation, HostHardwareRelation, HostNetworkRelation, AccessRelation from resource_inventory.resource_manager import ResourceManager +from resource_inventory.models import ConfigState @shared_task def booking_poll(): - def cleanup_hardware(qs): + def cleanup_resource_task(qs): for hostrelation in qs: - config = hostrelation.config - config.clear_delta() - config.power = "off" - config.save() - hostrelation.status = JobStatus.NEW - hostrelation.save() - - def cleanup_network(qs): - for hostrelation in qs: - network = hostrelation.config - network.interfaces.clear() - host = hostrelation.host - network.clear_delta() - vlans = [] - for interface in host.interfaces.all(): - for vlan in interface.config.all(): - if vlan.public: - try: - host.lab.vlan_manager.release_public_vlan(vlan.vlan_id) - except Exception: # will fail if we already released in this loop - pass - else: - vlans.append(vlan.vlan_id) - - # release all vlans - if len(vlans) > 0: - host.lab.vlan_manager.release_vlans(vlans) - - interface.config.clear() - network.add_interface(interface) - network.save() + hostrelation.config.state = ConfigState.CLEAN + hostrelation.config.save() hostrelation.status = JobStatus.NEW hostrelation.save() @@ -78,8 +50,8 @@ def booking_poll(): if not booking.job.complete: job = booking.job cleanup_software(SoftwareRelation.objects.filter(job=job)) - cleanup_hardware(HostHardwareRelation.objects.filter(job=job)) - cleanup_network(HostNetworkRelation.objects.filter(job=job)) + cleanup_resource_task(HostHardwareRelation.objects.filter(job=job)) + cleanup_resource_task(HostNetworkRelation.objects.filter(job=job)) cleanup_access(AccessRelation.objects.filter(job=job)) job.complete = True job.save() diff --git a/src/dashboard/utils.py b/src/dashboard/utils.py index 3d63366..d6b697a 100644 --- a/src/dashboard/utils.py +++ b/src/dashboard/utils.py @@ -34,6 +34,8 @@ class AbstractModelQuery(): for model in cls.model_list: result += list(model.objects.filter(*args, **kwargs)) + return result + @classmethod def get(cls, *args, **kwargs): try: diff --git a/src/notifier/manager.py b/src/notifier/manager.py index a5b7b9a..6d75a79 100644 --- a/src/notifier/manager.py +++ b/src/notifier/manager.py @@ -110,7 +110,7 @@ class NotificationHandler(object): @classmethod def email_booking_over(cls, booking): template_name = "notifier/email_ended.txt" - hostnames = [host.template.resource.name for host in booking.resource.hosts.all()] + hostnames = [host.name for host in booking.resource.getResources()] users = list(booking.collaborators.all()) users.append(booking.owner) for user in users: @@ -134,7 +134,7 @@ class NotificationHandler(object): @classmethod def email_booking_expiring(cls, booking): template_name = "notifier/email_expiring.txt" - hostnames = [host.template.resource.name for host in booking.resource.hosts.all()] + hostnames = [host.name for host in booking.resource.getResources()] users = list(booking.collaborators.all()) users.append(booking.owner) for user in users: diff --git a/src/resource_inventory/migrations/0014_auto_20200305_1415.py b/src/resource_inventory/migrations/0014_auto_20200305_1415.py new file mode 100644 index 0000000..6fcf4a6 --- /dev/null +++ b/src/resource_inventory/migrations/0014_auto_20200305_1415.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2020-03-05 14:15 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0013_auto_20200218_1536'), + ] + + operations = [ + migrations.RenameField( + model_name='resourcetemplate', + old_name='hidden', + new_name='temporary', + ), + ] diff --git a/src/resource_inventory/models.py b/src/resource_inventory/models.py index d11f71b..7115ece 100644 --- a/src/resource_inventory/models.py +++ b/src/resource_inventory/models.py @@ -161,7 +161,7 @@ class ResourceTemplate(models.Model): lab = models.ForeignKey(Lab, null=True, on_delete=models.SET_NULL, related_name="resourcetemplates") description = models.CharField(max_length=1000, default="") public = models.BooleanField(default=False) - hidden = models.BooleanField(default=False) + temporary = models.BooleanField(default=False) def getConfigs(self): return list(self.resourceConfigurations.all()) @@ -307,11 +307,12 @@ class Server(Resource): def get_configuration(self, state): ipmi = state == ConfigState.NEW power = "off" if state == ConfigState.CLEAN else "on" + image = self.config.image.lab_id if self.config else "unknown" return { "id": self.labid, - "image": self.config.image.lab_id, - "hostname": self.template.resource.name, + "image": image, + "hostname": self.name, "power": power, "ipmi_create": str(ipmi) } @@ -498,7 +499,7 @@ class Interface(models.Model): profile = models.ForeignKey(InterfaceProfile, on_delete=models.CASCADE) def __str__(self): - return self.mac_address + " on host " + str(self.host) + return self.mac_address + " on host " + str(self.profile.host.name) """ diff --git a/src/resource_inventory/pdf_templater.py b/src/resource_inventory/pdf_templater.py index 6844b09..367ba43 100644 --- a/src/resource_inventory/pdf_templater.py +++ b/src/resource_inventory/pdf_templater.py @@ -101,7 +101,7 @@ class PDFTemplater: returns a dictionary """ host_info = {} - host_info['name'] = host.template.resource.name + host_info['name'] = host.name host_info['node'] = cls.get_pdf_host_node(host) host_info['disks'] = [] for disk in host.profile.storageprofile.all(): @@ -153,13 +153,8 @@ class PDFTemplater: iface_info = {} iface_info['features'] = "none" iface_info['mac_address'] = interface.mac_address - iface_info['name'] = interface.name - speed = "unknown" - try: - profile = InterfaceProfile.objects.get(host=interface.host.profile, name=interface.name) - speed = str(int(profile.speed / 1000)) + "gb" - except Exception: - pass + iface_info['name'] = interface.profile.name + speed = str(int(interface.profile.speed / 1000)) + "gb" iface_info['speed'] = speed return iface_info diff --git a/src/resource_inventory/resource_manager.py b/src/resource_inventory/resource_manager.py index c8b2b05..4310f8c 100644 --- a/src/resource_inventory/resource_manager.py +++ b/src/resource_inventory/resource_manager.py @@ -35,7 +35,7 @@ class ResourceManager: def getAvailableResourceTemplates(self, lab, user): templates = ResourceTemplate.objects.filter(lab=lab) - templates.filter(Q(owner=user) | Q(public=True)) + templates = templates.filter(Q(owner=user) | Q(public=True)).filter(temporary=False) return templates def templateIsReservable(self, resource_template): @@ -65,10 +65,10 @@ class ResourceManager: resource.release() resourceBundle.delete() - def get_vlans(self, genericResourceBundle): + def get_vlans(self, resourceTemplate): networks = {} - vlan_manager = genericResourceBundle.lab.vlan_manager - for network in genericResourceBundle.networks.all(): + vlan_manager = resourceTemplate.lab.vlan_manager + for network in resourceTemplate.networks.all(): if network.is_public: public_net = vlan_manager.get_public_vlan() vlan_manager.reserve_public_vlan(public_net.vlan) @@ -108,12 +108,13 @@ class ResourceManager: return resource_bundle - def configureNetworking(self, host, vlan_map): - generic_interfaces = list(host.template.generic_interfaces.all()) - for int_num, physical_interface in enumerate(host.interfaces.all()): - generic_interface = generic_interfaces[int_num] + def configureNetworking(self, resource, vlan_map): + for physical_interface in resource.interfaces.all(): + iface_config = physical_interface.acts_as + if not iface_config: + continue physical_interface.config.clear() - for connection in generic_interface.connections.all(): + for connection in iface_config.connections.all(): physicalNetwork = PhysicalNetwork.objects.create( vlan_id=vlan_map[connection.network.name], generic_network=connection.network @@ -129,7 +130,7 @@ class ResourceManager: # private interface def acquireHost(self, resource_config): - resources = resource_config.profile.get_resources(lab=resource_config.lab, unreserved=True) + resources = resource_config.profile.get_resources(lab=resource_config.template.lab, unreserved=True) try: resource = resources[0] # TODO: should we randomize and 'load balance' the servers? resource.config = resource_config diff --git a/src/templates/base/booking/quick_deploy.html b/src/templates/base/booking/quick_deploy.html index 8570f25..d737b7d 100644 --- a/src/templates/base/booking/quick_deploy.html +++ b/src/templates/base/booking/quick_deploy.html @@ -77,17 +77,21 @@ var sup_image_dict = {{image_filter | safe}}; var sup_installer_dict = {{installer_filter | safe}}; var sup_scenario_dict = {{scenario_filter | safe}}; + var resource_profile_map = {{resource_profile_map | safe}}; function imageFilter() { var drop = document.getElementById("id_image"); var lab_pk = get_selected_value("lab"); - var host_pk = get_selected_value("host"); + var host_pk = get_selected_value("resource"); for (const childNode of drop.childNodes) { var image_object = sup_image_dict[childNode.value]; if (image_object) //weed out empty option { - childNode.disabled = !(image_object.host_profile == host_pk && image_object.lab == lab_pk); + const img_at_lab = image_object.lab == lab_pk; + const profiles = resource_profile_map[host_pk]; + const img_in_template = profiles && profiles.indexOf(image_object.host_profile) > -1 + childNode.disabled = !img_at_lab || !img_in_template; } } } diff --git a/src/workflow/forms.py b/src/workflow/forms.py index 37bc390..a8d3413 100644 --- a/src/workflow/forms.py +++ b/src/workflow/forms.py @@ -330,7 +330,7 @@ class FormUtils: 'selectable': true, 'follow': false, 'multiple': multiple_hosts, - 'class': 'host' + 'class': 'resource' } if multiple_hosts: resource_node['values'] = [] # place to store multiple values -- cgit 1.2.3-korg