From 23d35dc2c56b8c2b5496b6f0a5fc62066b22bbc7 Mon Sep 17 00:00:00 2001 From: Sawyer Bergeron Date: Fri, 29 Oct 2021 15:11:29 -0400 Subject: Add Cloud Init Support Squashed commit of the following: commit afcee3cad5c091e78e909b83f8df49accf1af5b6 Author: Sawyer Bergeron Date: Mon Oct 11 22:02:16 2021 +0000 Prod cobbler hotfixes Signed-off-by: Sawyer Bergeron Change-Id: I092bc6d85a3b2c77bfbe24f3af0d2b7a5f75a8c3 commit 5ce0a52b17e530436c298e1b581d37bac853f5a7 Author: Sawyer Bergeron Date: Thu Oct 7 17:14:01 2021 -0400 Manually merge CI files Signed-off-by: Sawyer Bergeron Change-Id: Ic63d5da699578007ef2f2cc373350ded06c66971 commit 5b70b8f1b8bbbe6aeec43b8d8dfdc6b7cc68bc9c Author: Sawyer Bergeron Date: Thu Sep 30 16:33:01 2021 -0400 Fixes for collaborator field Signed-off-by: Sawyer Bergeron Change-Id: I3dbdedf26fa84617ea7680a0f99e032d88f1ea98 Signed-off-by: Sawyer Bergeron commit 529b2521627b17142284c55c744812129edc71e8 Merge: d555513 e9d72ce Author: Sawyer Bergeron Date: Thu Sep 30 14:03:55 2021 +0000 Merge "Push cloud config content for generated files into userdata_raw" into cobbler commit d55551394df73645e49ae2ae3e730a9f1c6af81d Author: Sawyer Bergeron Date: Thu Sep 30 10:02:32 2021 -0400 Better error handling for quick deploy Change-Id: I03a725dfee9ce2f119d72ef940cd08df5aee3dcc Signed-off-by: Sawyer Bergeron commit e9d72ce78a85c6ff2f3f8591bcbf4115f97318d5 Author: Sawyer Bergeron Date: Tue Sep 28 19:11:49 2021 -0400 Push cloud config content for generated files into userdata_raw Signed-off-by: Sawyer Bergeron Change-Id: Ieb8bd9b8b172b6bf11062f67f41fc78154cc7c89 commit 95d39c60f7e8062cabc8c1665080a2d2c8904234 Author: Sawyer Bergeron Date: Sat Sep 25 16:18:12 2021 -0400 Allow for "pod specific" vlan allocation for LFEDGE allocation case Signed-off-by: Sawyer Bergeron Change-Id: I8b75410145027f43eaf6de7bd5f1813af38d3e7f Signed-off-by: Sawyer Bergeron commit 2ebb82b5f344de1e17abd70c51c4cce765761dd1 Author: Sawyer Bergeron Date: Thu Sep 23 16:37:43 2021 -0400 Fix collaborator field with recent changes Signed-off-by: Sawyer Bergeron Change-Id: Id305de9b1567adf103c47d5180b0b28ebfdf1b5e commit a819fc1df86721eda36eee89d0235c89b3159d6b Author: Sawyer Bergeron Date: Tue Sep 7 11:28:35 2021 -0400 Add user specified CI file entry Signed-off-by: Sawyer Bergeron Change-Id: Ia920130612da8fcde9d1a0d5dde7861904857162 Signed-off-by: Sawyer Bergeron commit d93346a716bde5237b7cfef5c10ea56e4922b59a Author: Adam Hassick Date: Tue Jul 27 13:05:16 2021 +0000 Make C-I serialization work with current netconf rules Signed-off-by: Sawyer Bergeron Change-Id: If967e5e1f268c5bee3ad4496847662cf4de1187c Signed-off-by: Sawyer Bergeron commit 6ffb1fdf6ce7825770148bada5a4c54899e4ed36 Author: Adam Hassick Date: Tue Jun 29 16:49:27 2021 -0400 Cobbler model changes, new endpoints Signed-off-by: Adam Hassick Change-Id: If0a94730e92747127cef121ec4930a4c8bae6c92 Signed-off-by: Sawyer Bergeron Signed-off-by: Adam Hassick commit 49e2b407003b69551ddafa851639e83ec42a5b09 Author: Jacob Hodgdon Date: Fri May 14 15:42:56 2021 -0400 Color fixes for rebrand Signed-off-by: Jacob Hodgdon Change-Id: I5cf4ede598afa377db7ecec17d8dfef085e130ac commit a908da441bf6efcdb289a46d0c2761840138b1a5 Author: Sawyer Bergeron Date: Tue Jun 8 11:15:56 2021 -0400 Draft for cloud-init file generation Signed-off-by: Sawyer Bergeron Signed-off-by: Sawyer Bergeron Change-Id: I07f3a4a1ab67531cba2cc7e3de22e9bb860706e1 Signed-off-by: Sawyer Bergeron Signed-off-by: Sawyer Bergeron Change-Id: I392505174cbc07214c31c42aab2474a748e47913 Signed-off-by: Sawyer Bergeron --- src/api/models.py | 188 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 186 insertions(+), 2 deletions(-) (limited to 'src/api/models.py') diff --git a/src/api/models.py b/src/api/models.py index d85f3e9..5928ea9 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -19,18 +19,22 @@ from django.utils import timezone import json import uuid +import yaml from booking.models import Booking from resource_inventory.models import ( Lab, ResourceProfile, Image, + Opsys, Interface, ResourceOPNFVConfig, RemoteInfo, OPNFVConfig, ConfigState, - ResourceQuery + ResourceQuery, + ResourceConfiguration, + CloudInitFile ) from resource_inventory.idf_templater import IDFTemplater from resource_inventory.pdf_templater import PDFTemplater @@ -84,6 +88,18 @@ class LabManager: def __init__(self, lab): self.lab = lab + def get_opsyss(self): + return Opsys.objects.filter(from_lab=self.lab) + + def get_images(self): + return Image.objects.filter(from_lab=self.lab) + + def get_image(self, image_id): + return Image.objects.filter(from_lab=self.lab, lab_id=image_id) + + def get_opsys(self, opsys_id): + return Opsys.objects.filter(from_lab=self.lab, lab_id=opsys_id) + def get_downtime(self): return Downtime.objects.filter(start__lt=timezone.now(), end__gt=timezone.now(), lab=self.lab) @@ -338,6 +354,157 @@ class LabManager: return profile_ser +class GeneratedCloudConfig(models.Model): + resource_id = models.CharField(max_length=200) + booking = models.ForeignKey(Booking, on_delete=models.CASCADE) + rconfig = models.ForeignKey(ResourceConfiguration, on_delete=models.CASCADE) + text = models.TextField(null=True, blank=True) + + def _normalize_username(self, username: str) -> str: + # TODO: make usernames posix compliant + return username + + def _get_ssh_string(self, username: str) -> str: + user = User.objects.get(username=username) + uprofile = user.userprofile + + ssh_file = uprofile.ssh_public_key + + escaped_file = ssh_file.open().read().decode(encoding="UTF-8").replace("\n", " ") + + return escaped_file + + def _serialize_users(self): + """ + returns the dictionary to be placed behind the `users` field of the toplevel c-i dict + """ + # conserves distro default user + user_array = ["default"] + + users = list(self.booking.collaborators.all()) + users.append(self.booking.owner) + for collaborator in users: + userdict = {} + + # TODO: validate if usernames are valid as linux usernames (and provide an override potentially) + userdict['name'] = self._normalize_username(collaborator.username) + + userdict['groups'] = "sudo" + userdict['sudo'] = "ALL=(ALL) NOPASSWD:ALL" + + userdict['ssh_authorized_keys'] = [self._get_ssh_string(collaborator.username)] + + user_array.append(userdict) + + # user_array.append({ + # "name": "opnfv", + # "passwd": "$6$k54L.vim1cLaEc4$5AyUIrufGlbtVBzuCWOlA1yV6QdD7Gr2MzwIs/WhuYR9ebSfh3Qlb7djkqzjwjxpnSAonK1YOabPP6NxUDccu.", + # "ssh_redirect_user": True, + # "sudo": "ALL=(ALL) NOPASSWD:ALL", + # "groups": "sudo", + # }) + + return user_array + + # TODO: make this configurable + def _serialize_sysinfo(self): + defuser = {} + defuser['name'] = 'opnfv' + defuser['plain_text_passwd'] = 'OPNFV_HOST' + defuser['home'] = '/home/opnfv' + defuser['shell'] = '/bin/bash' + defuser['lock_passwd'] = True + defuser['gecos'] = 'Lab Manager User' + defuser['groups'] = 'sudo' + + return {'default_user': defuser} + + # TODO: make this configurable + def _serialize_runcmds(self): + cmdlist = [] + + # have hosts run dhcp on boot + cmdlist.append(['sudo', 'dhclient', '-r']) + cmdlist.append(['sudo', 'dhclient']) + + return cmdlist + + def _serialize_netconf_v1(self): + # interfaces = {} # map from iface_name => dhcp_config + # vlans = {} # map from vlan_id => dhcp_config + + config_arr = [] + + for interface in self._resource().interfaces.all(): + interface_name = interface.profile.name + interface_mac = interface.mac_address + + iface_dict_entry = { + "type": "physical", + "name": interface_name, + "mac_address": interface_mac, + } + + for vlan in interface.config.all(): + if vlan.tagged: + vlan_dict_entry = {'type': 'vlan'} + vlan_dict_entry['name'] = str(interface_name) + "." + str(vlan.vlan_id) + vlan_dict_entry['vlan_link'] = str(interface_name) + vlan_dict_entry['vlan_id'] = int(vlan.vlan_id) + vlan_dict_entry['mac_address'] = str(interface_mac) + if vlan.public: + vlan_dict_entry["subnets"] = [{"type": "dhcp"}] + config_arr.append(vlan_dict_entry) + if (not vlan.tagged) and vlan.public: + iface_dict_entry["subnets"] = [{"type": "dhcp"}] + + # vlan_dict_entry['mtu'] = # TODO, determine override MTU if needed + + config_arr.append(iface_dict_entry) + + ns_dict = { + 'type': 'nameserver', + 'address': ['10.64.0.1', '8.8.8.8'] + } + + config_arr.append(ns_dict) + + full_dict = {'version': 1, 'config': config_arr} + + return full_dict + + @classmethod + def get(cls, booking_id: int, resource_lab_id: str, file_id: int): + return GeneratedCloudConfig.objects.get(resource_id=resource_lab_id, booking__id=booking_id, file_id=file_id) + + def _resource(self): + return ResourceQuery.get(labid=self.resource_id, lab=self.booking.lab) + + # def _get_facts(self): + # resource = self._resource() + + # hostname = self.rconfig.name + # iface_configs = for_config.interface_configs.all() + + def _to_dict(self): + main_dict = {} + + main_dict['users'] = self._serialize_users() + main_dict['network'] = self._serialize_netconf_v1() + main_dict['hostname'] = self.rconfig.name + + # add first startup commands + main_dict['runcmd'] = self._serialize_runcmds() + + # configure distro default user + main_dict['system_info'] = self._serialize_sysinfo() + + return main_dict + + def serialize(self) -> str: + return yaml.dump(self._to_dict()) + + class APILog(models.Model): user = models.ForeignKey(User, on_delete=models.PROTECT) call_time = models.DateTimeField(auto_now=True) @@ -761,6 +928,7 @@ class HardwareConfig(TaskConfig): return self.get_delta() def get_delta(self): + # TODO: grab the GeneratedCloudConfig urls from self.hosthardwarerelation.get_resource() return self.format_delta( self.hosthardwarerelation.get_resource().get_configuration(self.state), self.hosthardwarerelation.lab_token) @@ -813,7 +981,7 @@ class NetworkConfig(TaskConfig): class SnapshotConfig(TaskConfig): resource_id = models.CharField(max_length=200, default="default_id") - image = models.IntegerField(null=True) + image = models.CharField(max_length=200, null=True) # cobbler ID dashboard_id = models.IntegerField() delta = models.TextField(default="{}") @@ -1104,6 +1272,10 @@ class JobFactory(object): booking=booking, job=job ) + cls.makeGeneratedCloudConfigs( + resources=resources, + job=job + ) all_users = list(booking.collaborators.all()) all_users.append(booking.owner) cls.makeAccessConfig( @@ -1127,6 +1299,18 @@ class JobFactory(object): except Exception: continue + @classmethod + def makeGeneratedCloudConfigs(cls, resources=[], job=Job()): + for res in resources: + cif = GeneratedCloudConfig.objects.create(resource_id=res.labid, booking=job.booking, rconfig=res.config) + cif.save() + + cif = CloudInitFile.create(priority=0, text=cif.serialize()) + cif.save() + + res.config.cloud_init_files.add(cif) + res.config.save() + @classmethod def makeHardwareConfigs(cls, resources=[], job=Job()): """ -- cgit 1.2.3-korg