From 23d35dc2c56b8c2b5496b6f0a5fc62066b22bbc7 Mon Sep 17 00:00:00 2001 From: Sawyer Bergeron Date: Fri, 29 Oct 2021 15:11:29 -0400 Subject: Add Cloud Init Support Squashed commit of the following: commit afcee3cad5c091e78e909b83f8df49accf1af5b6 Author: Sawyer Bergeron Date: Mon Oct 11 22:02:16 2021 +0000 Prod cobbler hotfixes Signed-off-by: Sawyer Bergeron Change-Id: I092bc6d85a3b2c77bfbe24f3af0d2b7a5f75a8c3 commit 5ce0a52b17e530436c298e1b581d37bac853f5a7 Author: Sawyer Bergeron Date: Thu Oct 7 17:14:01 2021 -0400 Manually merge CI files Signed-off-by: Sawyer Bergeron Change-Id: Ic63d5da699578007ef2f2cc373350ded06c66971 commit 5b70b8f1b8bbbe6aeec43b8d8dfdc6b7cc68bc9c Author: Sawyer Bergeron Date: Thu Sep 30 16:33:01 2021 -0400 Fixes for collaborator field Signed-off-by: Sawyer Bergeron Change-Id: I3dbdedf26fa84617ea7680a0f99e032d88f1ea98 Signed-off-by: Sawyer Bergeron commit 529b2521627b17142284c55c744812129edc71e8 Merge: d555513 e9d72ce Author: Sawyer Bergeron Date: Thu Sep 30 14:03:55 2021 +0000 Merge "Push cloud config content for generated files into userdata_raw" into cobbler commit d55551394df73645e49ae2ae3e730a9f1c6af81d Author: Sawyer Bergeron Date: Thu Sep 30 10:02:32 2021 -0400 Better error handling for quick deploy Change-Id: I03a725dfee9ce2f119d72ef940cd08df5aee3dcc Signed-off-by: Sawyer Bergeron commit e9d72ce78a85c6ff2f3f8591bcbf4115f97318d5 Author: Sawyer Bergeron Date: Tue Sep 28 19:11:49 2021 -0400 Push cloud config content for generated files into userdata_raw Signed-off-by: Sawyer Bergeron Change-Id: Ieb8bd9b8b172b6bf11062f67f41fc78154cc7c89 commit 95d39c60f7e8062cabc8c1665080a2d2c8904234 Author: Sawyer Bergeron Date: Sat Sep 25 16:18:12 2021 -0400 Allow for "pod specific" vlan allocation for LFEDGE allocation case Signed-off-by: Sawyer Bergeron Change-Id: I8b75410145027f43eaf6de7bd5f1813af38d3e7f Signed-off-by: Sawyer Bergeron commit 2ebb82b5f344de1e17abd70c51c4cce765761dd1 Author: Sawyer Bergeron Date: Thu Sep 23 16:37:43 2021 -0400 Fix collaborator field with recent changes Signed-off-by: Sawyer Bergeron Change-Id: Id305de9b1567adf103c47d5180b0b28ebfdf1b5e commit a819fc1df86721eda36eee89d0235c89b3159d6b Author: Sawyer Bergeron Date: Tue Sep 7 11:28:35 2021 -0400 Add user specified CI file entry Signed-off-by: Sawyer Bergeron Change-Id: Ia920130612da8fcde9d1a0d5dde7861904857162 Signed-off-by: Sawyer Bergeron commit d93346a716bde5237b7cfef5c10ea56e4922b59a Author: Adam Hassick Date: Tue Jul 27 13:05:16 2021 +0000 Make C-I serialization work with current netconf rules Signed-off-by: Sawyer Bergeron Change-Id: If967e5e1f268c5bee3ad4496847662cf4de1187c Signed-off-by: Sawyer Bergeron commit 6ffb1fdf6ce7825770148bada5a4c54899e4ed36 Author: Adam Hassick Date: Tue Jun 29 16:49:27 2021 -0400 Cobbler model changes, new endpoints Signed-off-by: Adam Hassick Change-Id: If0a94730e92747127cef121ec4930a4c8bae6c92 Signed-off-by: Sawyer Bergeron Signed-off-by: Adam Hassick commit 49e2b407003b69551ddafa851639e83ec42a5b09 Author: Jacob Hodgdon Date: Fri May 14 15:42:56 2021 -0400 Color fixes for rebrand Signed-off-by: Jacob Hodgdon Change-Id: I5cf4ede598afa377db7ecec17d8dfef085e130ac commit a908da441bf6efcdb289a46d0c2761840138b1a5 Author: Sawyer Bergeron Date: Tue Jun 8 11:15:56 2021 -0400 Draft for cloud-init file generation Signed-off-by: Sawyer Bergeron Signed-off-by: Sawyer Bergeron Change-Id: I07f3a4a1ab67531cba2cc7e3de22e9bb860706e1 Signed-off-by: Sawyer Bergeron Signed-off-by: Sawyer Bergeron Change-Id: I392505174cbc07214c31c42aab2474a748e47913 Signed-off-by: Sawyer Bergeron --- .../migrations/0018_auto_20210630_1629.py | 101 ++++++++++++++++ .../migrations/0019_auto_20210701_1947.py | 43 +++++++ .../migrations/0020_cloudinitfile.py | 21 ++++ .../0021_resourceconfiguration_cloud_init_files.py | 18 +++ .../migrations/0022_auto_20210925_2028.py | 23 ++++ src/resource_inventory/models.py | 131 ++++++++++++++++++++- src/resource_inventory/resource_manager.py | 8 +- src/resource_inventory/tests/test_models.py | 2 +- 8 files changed, 337 insertions(+), 10 deletions(-) create mode 100644 src/resource_inventory/migrations/0018_auto_20210630_1629.py create mode 100644 src/resource_inventory/migrations/0019_auto_20210701_1947.py create mode 100644 src/resource_inventory/migrations/0020_cloudinitfile.py create mode 100644 src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py create mode 100644 src/resource_inventory/migrations/0022_auto_20210925_2028.py (limited to 'src/resource_inventory') diff --git a/src/resource_inventory/migrations/0018_auto_20210630_1629.py b/src/resource_inventory/migrations/0018_auto_20210630_1629.py new file mode 100644 index 0000000..19e53e4 --- /dev/null +++ b/src/resource_inventory/migrations/0018_auto_20210630_1629.py @@ -0,0 +1,101 @@ +# Generated by Django 2.2 on 2021-06-30 16:29 + +from django.db import migrations, models +import django.db.models.deletion +from account.models import Lab + + +def set_availability(apps, schema_editor): + models = [apps.get_model('resource_inventory', 'Image'), apps.get_model('resource_inventory', 'Opsys')] + + for model in models: + for obj in model.objects.all(): + obj.available = False + obj.obsolete = True + obj.save() + + +def set_rconfig_arch(apps, schema_editor): + rprofs = apps.get_model('resource_inventory', 'ResourceProfile') + + for rprof in rprofs.objects.all(): + rprof.architecture = rprof.cpuprofile.first().architecture + + +class Migration(migrations.Migration): + + dependencies = [ + ('account', '0009_auto_20210324_2107'), + ('resource_inventory', '0017_auto_20201218_1516'), + ] + + operations = [ + migrations.RemoveField( + model_name='image', + name='host_type', + ), + migrations.AlterField( + model_name='image', + name='lab_id', + field=models.CharField(default='none (retired)', max_length=100), + preserve_default=True, + ), + migrations.RemoveField( + model_name='opsys', + name='sup_installers', + ), + + migrations.AddField( + model_name='image', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50), + preserve_default=False, + ), + + migrations.AddField( + model_name='image', + name='available', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='image', + name='obsolete', + field=models.BooleanField(default=False), + ), + + migrations.AddField( + model_name='opsys', + name='available', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='opsys', + name='obsolete', + field=models.BooleanField(default=True), + ), + + migrations.RunPython(set_availability), + + migrations.AddField( + model_name='opsys', + name='lab_id', + field=models.CharField(default="none (retired)", max_length=100), + preserve_default=False, + ), + + migrations.AddField( + model_name='opsys', + name='from_lab', + field=models.ForeignKey(default=Lab.objects.first, on_delete=django.db.models.deletion.CASCADE, to='account.Lab'), + preserve_default=False, + ), + + migrations.AddField( + model_name='resourceprofile', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50), + preserve_default=False, + ), + + migrations.RunPython(set_rconfig_arch), + ] diff --git a/src/resource_inventory/migrations/0019_auto_20210701_1947.py b/src/resource_inventory/migrations/0019_auto_20210701_1947.py new file mode 100644 index 0000000..e64d174 --- /dev/null +++ b/src/resource_inventory/migrations/0019_auto_20210701_1947.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2 on 2021-07-01 19:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0018_auto_20210630_1629'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='lab_id', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='image', + name='name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='network', + name='name', + field=models.CharField(max_length=200), + ), + migrations.AlterField( + model_name='opsys', + name='available', + field=models.BooleanField(default=True), + ), + migrations.AlterField( + model_name='opsys', + name='obsolete', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='resourceprofile', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64')], max_length=50), + ), + ] diff --git a/src/resource_inventory/migrations/0020_cloudinitfile.py b/src/resource_inventory/migrations/0020_cloudinitfile.py new file mode 100644 index 0000000..198181c --- /dev/null +++ b/src/resource_inventory/migrations/0020_cloudinitfile.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2 on 2021-09-07 14:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0019_auto_20210701_1947'), + ] + + operations = [ + migrations.CreateModel( + name='CloudInitFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('text', models.TextField()), + ('priority', models.IntegerField()), + ], + ), + ] diff --git a/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py new file mode 100644 index 0000000..6b0befc --- /dev/null +++ b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2021-09-10 18:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0020_cloudinitfile'), + ] + + operations = [ + migrations.AddField( + model_name='resourceconfiguration', + name='cloud_init_files', + field=models.ManyToManyField(blank=True, to='resource_inventory.CloudInitFile'), + ), + ] diff --git a/src/resource_inventory/migrations/0022_auto_20210925_2028.py b/src/resource_inventory/migrations/0022_auto_20210925_2028.py new file mode 100644 index 0000000..2b0b902 --- /dev/null +++ b/src/resource_inventory/migrations/0022_auto_20210925_2028.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2 on 2021-09-25 20:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0021_resourceconfiguration_cloud_init_files'), + ] + + operations = [ + migrations.AddField( + model_name='resourcetemplate', + name='private_vlan_pool', + field=models.TextField(default=''), + ), + migrations.AddField( + model_name='resourcetemplate', + name='public_vlan_pool', + field=models.TextField(default=''), + ), + ] diff --git a/src/resource_inventory/models.py b/src/resource_inventory/models.py index 7fe479a..aefd5ce 100644 --- a/src/resource_inventory/models.py +++ b/src/resource_inventory/models.py @@ -9,10 +9,12 @@ ############################################################################## from django.contrib.auth.models import User + from django.core.exceptions import ValidationError from django.db import models from django.db.models import Q import traceback +import json import re from collections import Counter @@ -20,7 +22,6 @@ from collections import Counter from account.models import Lab from dashboard.utils import AbstractModelQuery - """ Profiles of resources hosted by labs. @@ -33,6 +34,10 @@ Profile models (e.g. an x86 server profile and armv8 server profile. class ResourceProfile(models.Model): id = models.AutoField(primary_key=True) name = models.CharField(max_length=200, unique=True) + architecture = models.CharField(max_length=50, choices=[ + ("x86_64", "x86_64"), + ("aarch64", "aarch64") + ]) description = models.TextField() labs = models.ManyToManyField(Lab, related_name="resourceprofiles") @@ -147,6 +152,24 @@ with varying degrees of abstraction. """ +class CloudInitFile(models.Model): + text = models.TextField() + + # higher priority is applied later, so "on top" of existing files + priority = models.IntegerField() + + @classmethod + def merge_strategy(cls): + return [ + {'name': 'list', 'settings': ['append']}, + {'name': 'dict', 'settings': ['recurse_list', 'replace']}, + ] + + @classmethod + def create(cls, text="", priority=0): + return CloudInitFile.objects.create(priority=priority, text=text) + + class ResourceTemplate(models.Model): """ Models a "template" of a complete, configured collection of resources that can be booked. @@ -167,6 +190,24 @@ class ResourceTemplate(models.Model): temporary = models.BooleanField(default=False) copy_of = models.ForeignKey("ResourceTemplate", blank=True, null=True, on_delete=models.SET_NULL) + # if these fields are empty ("") then they are implicitly "every vlan", + # otherwise we filter any allocations we try to instantiate against this list + # they should be represented as a json list of integers + private_vlan_pool = models.TextField(default="") + public_vlan_pool = models.TextField(default="") + + def private_vlan_pool_set(self): + if self.private_vlan_pool != "": + return set(json.loads(self.private_vlan_pool)) + else: + return None + + def public_vlan_pool_set(self): + if self.private_vlan_pool != "": + return set(json.loads(self.public_vlan_pool)) + else: + return None + def getConfigs(self): configs = self.resourceConfigurations.all() return list(configs) @@ -235,9 +276,14 @@ class ResourceConfiguration(models.Model): is_head_node = models.BooleanField(default=False) name = models.CharField(max_length=3000, default="opnfv_host") + cloud_init_files = models.ManyToManyField(CloudInitFile, blank=True) + def __str__(self): return str(self.name) + def ci_file_list(self): + return list(self.cloud_init_files.order_by("priority").all()) + def get_default_remote_info(): return RemoteInfo.objects.get_or_create( @@ -369,10 +415,43 @@ class Server(Resource): return isinstance(other, Server) and other.name == self.name +def is_serializable(data): + try: + json.dumps(data) + return True + except Exception: + return False + + class Opsys(models.Model): id = models.AutoField(primary_key=True) name = models.CharField(max_length=100) - sup_installers = models.ManyToManyField("Installer", blank=True) + lab_id = models.CharField(max_length=100) + obsolete = models.BooleanField(default=False) + available = models.BooleanField(default=True) # marked true by Cobbler if it exists there + from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE) + + indexes = [ + models.Index(fields=['cobbler_id']) + ] + + def new_from_data(data): + opsys = Opsys() + opsys.update(data) + return opsys + + def serialize(self): + d = {} + for field in vars(self): + attr = getattr(self, field) + if is_serializable(attr): + d[field] = attr + return d + + def update(self, data): + for field in vars(self): + if field in data: + setattr(self, field, data[field] if data[field] else getattr(self, field)) def __str__(self): return self.name @@ -382,18 +461,51 @@ class Image(models.Model): """Model for representing OS images / snapshots of hosts.""" id = models.AutoField(primary_key=True) - lab_id = models.IntegerField() # ID the lab who holds this image knows from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE) - name = models.CharField(max_length=200) + architecture = models.CharField(max_length=50, choices=[ + ("x86_64", "x86_64"), + ("aarch64", "aarch64"), + ("unknown", "unknown"), + ]) + lab_id = models.CharField(max_length=100) + name = models.CharField(max_length=100) owner = models.ForeignKey(User, null=True, on_delete=models.SET_NULL) public = models.BooleanField(default=True) - host_type = models.ForeignKey(ResourceProfile, on_delete=models.CASCADE) description = models.TextField() os = models.ForeignKey(Opsys, null=True, on_delete=models.CASCADE) + available = models.BooleanField(default=True) # marked True by cobbler if it exists there + obsolete = models.BooleanField(default=False) + + indexes = [ + models.Index(fields=['architecture']), + models.Index(fields=['cobbler_id']) + ] + def __str__(self): return self.name + def is_obsolete(self): + return self.obsolete or self.os.obsolete + + def serialize(self): + d = {} + for field in vars(self): + attr = getattr(self, field) + if is_serializable(attr): + d[field] = attr + return d + + def update(self, data): + for field in vars(self): + if field in data: + setattr(self, field, data[field] if data[field] else getattr(self, field)) + + def new_from_data(data): + img = Image() + img.update(data) + return img + def in_use(self): for resource in ResourceQuery.filter(config__image=self): if resource.is_reserved(): @@ -409,7 +521,7 @@ Networking configuration models class Network(models.Model): id = models.AutoField(primary_key=True) - name = models.CharField(max_length=100) + name = models.CharField(max_length=200) bundle = models.ForeignKey(ResourceTemplate, on_delete=models.CASCADE, related_name="networks") is_public = models.BooleanField() @@ -507,6 +619,13 @@ class NetworkRole(models.Model): network = models.ForeignKey(Network, on_delete=models.CASCADE) +def create_resource_ref_string(for_hosts: [str]) -> str: + # need to sort the list, then do dump + for_hosts.sort() + + return json.dumps(for_hosts) + + class OPNFVConfig(models.Model): id = models.AutoField(primary_key=True) installer = models.ForeignKey(Installer, on_delete=models.CASCADE) diff --git a/src/resource_inventory/resource_manager.py b/src/resource_inventory/resource_manager.py index 37bf33c..52af824 100644 --- a/src/resource_inventory/resource_manager.py +++ b/src/resource_inventory/resource_manager.py @@ -6,7 +6,8 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from __future__ import annotations + +from __future__ import annotations # noqa: F407 import re from typing import Optional @@ -85,12 +86,13 @@ class ResourceManager: vlan_manager = resourceTemplate.lab.vlan_manager for network in resourceTemplate.networks.all(): if network.is_public: - public_net = vlan_manager.get_public_vlan() + # already throws if can't get requested count, so can always expect public_net to be Some + public_net = vlan_manager.get_public_vlan(within=resourceTemplate.public_vlan_pool_set()) vlan_manager.reserve_public_vlan(public_net.vlan) networks[network.name] = public_net.vlan else: # already throws if can't get requested count, so can always index in @ 0 - vlans = vlan_manager.get_vlans(count=1) + vlans = vlan_manager.get_vlans(count=1, within=resourceTemplate.private_vlan_pool_set()) vlan_manager.reserve_vlans(vlans[0]) networks[network.name] = vlans[0] return networks diff --git a/src/resource_inventory/tests/test_models.py b/src/resource_inventory/tests/test_models.py index e1b2106..3f2d1d8 100644 --- a/src/resource_inventory/tests/test_models.py +++ b/src/resource_inventory/tests/test_models.py @@ -80,7 +80,7 @@ class ConfigUtil(): ) return Image.objects.create( - lab_id=0, + cobbler_id="profile1", from_lab=lab, name="an image for testing", owner=owner -- cgit 1.2.3-korg