diff options
Diffstat (limited to 'src/resource_inventory')
8 files changed, 337 insertions, 10 deletions
diff --git a/src/resource_inventory/migrations/0018_auto_20210630_1629.py b/src/resource_inventory/migrations/0018_auto_20210630_1629.py new file mode 100644 index 0000000..19e53e4 --- /dev/null +++ b/src/resource_inventory/migrations/0018_auto_20210630_1629.py @@ -0,0 +1,101 @@ +# Generated by Django 2.2 on 2021-06-30 16:29 + +from django.db import migrations, models +import django.db.models.deletion +from account.models import Lab + + +def set_availability(apps, schema_editor): + models = [apps.get_model('resource_inventory', 'Image'), apps.get_model('resource_inventory', 'Opsys')] + + for model in models: + for obj in model.objects.all(): + obj.available = False + obj.obsolete = True + obj.save() + + +def set_rconfig_arch(apps, schema_editor): + rprofs = apps.get_model('resource_inventory', 'ResourceProfile') + + for rprof in rprofs.objects.all(): + rprof.architecture = rprof.cpuprofile.first().architecture + + +class Migration(migrations.Migration): + + dependencies = [ + ('account', '0009_auto_20210324_2107'), + ('resource_inventory', '0017_auto_20201218_1516'), + ] + + operations = [ + migrations.RemoveField( + model_name='image', + name='host_type', + ), + migrations.AlterField( + model_name='image', + name='lab_id', + field=models.CharField(default='none (retired)', max_length=100), + preserve_default=True, + ), + migrations.RemoveField( + model_name='opsys', + name='sup_installers', + ), + + migrations.AddField( + model_name='image', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50), + preserve_default=False, + ), + + migrations.AddField( + model_name='image', + name='available', + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name='image', + name='obsolete', + field=models.BooleanField(default=False), + ), + + migrations.AddField( + model_name='opsys', + name='available', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='opsys', + name='obsolete', + field=models.BooleanField(default=True), + ), + + migrations.RunPython(set_availability), + + migrations.AddField( + model_name='opsys', + name='lab_id', + field=models.CharField(default="none (retired)", max_length=100), + preserve_default=False, + ), + + migrations.AddField( + model_name='opsys', + name='from_lab', + field=models.ForeignKey(default=Lab.objects.first, on_delete=django.db.models.deletion.CASCADE, to='account.Lab'), + preserve_default=False, + ), + + migrations.AddField( + model_name='resourceprofile', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50), + preserve_default=False, + ), + + migrations.RunPython(set_rconfig_arch), + ] diff --git a/src/resource_inventory/migrations/0019_auto_20210701_1947.py b/src/resource_inventory/migrations/0019_auto_20210701_1947.py new file mode 100644 index 0000000..e64d174 --- /dev/null +++ b/src/resource_inventory/migrations/0019_auto_20210701_1947.py @@ -0,0 +1,43 @@ +# Generated by Django 2.2 on 2021-07-01 19:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0018_auto_20210630_1629'), + ] + + operations = [ + migrations.AlterField( + model_name='image', + name='lab_id', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='image', + name='name', + field=models.CharField(max_length=100), + ), + migrations.AlterField( + model_name='network', + name='name', + field=models.CharField(max_length=200), + ), + migrations.AlterField( + model_name='opsys', + name='available', + field=models.BooleanField(default=True), + ), + migrations.AlterField( + model_name='opsys', + name='obsolete', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='resourceprofile', + name='architecture', + field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64')], max_length=50), + ), + ] diff --git a/src/resource_inventory/migrations/0020_cloudinitfile.py b/src/resource_inventory/migrations/0020_cloudinitfile.py new file mode 100644 index 0000000..198181c --- /dev/null +++ b/src/resource_inventory/migrations/0020_cloudinitfile.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2 on 2021-09-07 14:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0019_auto_20210701_1947'), + ] + + operations = [ + migrations.CreateModel( + name='CloudInitFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('text', models.TextField()), + ('priority', models.IntegerField()), + ], + ), + ] diff --git a/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py new file mode 100644 index 0000000..6b0befc --- /dev/null +++ b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2 on 2021-09-10 18:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0020_cloudinitfile'), + ] + + operations = [ + migrations.AddField( + model_name='resourceconfiguration', + name='cloud_init_files', + field=models.ManyToManyField(blank=True, to='resource_inventory.CloudInitFile'), + ), + ] diff --git a/src/resource_inventory/migrations/0022_auto_20210925_2028.py b/src/resource_inventory/migrations/0022_auto_20210925_2028.py new file mode 100644 index 0000000..2b0b902 --- /dev/null +++ b/src/resource_inventory/migrations/0022_auto_20210925_2028.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2 on 2021-09-25 20:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('resource_inventory', '0021_resourceconfiguration_cloud_init_files'), + ] + + operations = [ + migrations.AddField( + model_name='resourcetemplate', + name='private_vlan_pool', + field=models.TextField(default=''), + ), + migrations.AddField( + model_name='resourcetemplate', + name='public_vlan_pool', + field=models.TextField(default=''), + ), + ] diff --git a/src/resource_inventory/models.py b/src/resource_inventory/models.py index 7fe479a..aefd5ce 100644 --- a/src/resource_inventory/models.py +++ b/src/resource_inventory/models.py @@ -9,10 +9,12 @@ ############################################################################## from django.contrib.auth.models import User + from django.core.exceptions import ValidationError from django.db import models from django.db.models import Q import traceback +import json import re from collections import Counter @@ -20,7 +22,6 @@ from collections import Counter from account.models import Lab from dashboard.utils import AbstractModelQuery - """ Profiles of resources hosted by labs. @@ -33,6 +34,10 @@ Profile models (e.g. an x86 server profile and armv8 server profile. class ResourceProfile(models.Model): id = models.AutoField(primary_key=True) name = models.CharField(max_length=200, unique=True) + architecture = models.CharField(max_length=50, choices=[ + ("x86_64", "x86_64"), + ("aarch64", "aarch64") + ]) description = models.TextField() labs = models.ManyToManyField(Lab, related_name="resourceprofiles") @@ -147,6 +152,24 @@ with varying degrees of abstraction. """ +class CloudInitFile(models.Model): + text = models.TextField() + + # higher priority is applied later, so "on top" of existing files + priority = models.IntegerField() + + @classmethod + def merge_strategy(cls): + return [ + {'name': 'list', 'settings': ['append']}, + {'name': 'dict', 'settings': ['recurse_list', 'replace']}, + ] + + @classmethod + def create(cls, text="", priority=0): + return CloudInitFile.objects.create(priority=priority, text=text) + + class ResourceTemplate(models.Model): """ Models a "template" of a complete, configured collection of resources that can be booked. @@ -167,6 +190,24 @@ class ResourceTemplate(models.Model): temporary = models.BooleanField(default=False) copy_of = models.ForeignKey("ResourceTemplate", blank=True, null=True, on_delete=models.SET_NULL) + # if these fields are empty ("") then they are implicitly "every vlan", + # otherwise we filter any allocations we try to instantiate against this list + # they should be represented as a json list of integers + private_vlan_pool = models.TextField(default="") + public_vlan_pool = models.TextField(default="") + + def private_vlan_pool_set(self): + if self.private_vlan_pool != "": + return set(json.loads(self.private_vlan_pool)) + else: + return None + + def public_vlan_pool_set(self): + if self.private_vlan_pool != "": + return set(json.loads(self.public_vlan_pool)) + else: + return None + def getConfigs(self): configs = self.resourceConfigurations.all() return list(configs) @@ -235,9 +276,14 @@ class ResourceConfiguration(models.Model): is_head_node = models.BooleanField(default=False) name = models.CharField(max_length=3000, default="opnfv_host") + cloud_init_files = models.ManyToManyField(CloudInitFile, blank=True) + def __str__(self): return str(self.name) + def ci_file_list(self): + return list(self.cloud_init_files.order_by("priority").all()) + def get_default_remote_info(): return RemoteInfo.objects.get_or_create( @@ -369,10 +415,43 @@ class Server(Resource): return isinstance(other, Server) and other.name == self.name +def is_serializable(data): + try: + json.dumps(data) + return True + except Exception: + return False + + class Opsys(models.Model): id = models.AutoField(primary_key=True) name = models.CharField(max_length=100) - sup_installers = models.ManyToManyField("Installer", blank=True) + lab_id = models.CharField(max_length=100) + obsolete = models.BooleanField(default=False) + available = models.BooleanField(default=True) # marked true by Cobbler if it exists there + from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE) + + indexes = [ + models.Index(fields=['cobbler_id']) + ] + + def new_from_data(data): + opsys = Opsys() + opsys.update(data) + return opsys + + def serialize(self): + d = {} + for field in vars(self): + attr = getattr(self, field) + if is_serializable(attr): + d[field] = attr + return d + + def update(self, data): + for field in vars(self): + if field in data: + setattr(self, field, data[field] if data[field] else getattr(self, field)) def __str__(self): return self.name @@ -382,18 +461,51 @@ class Image(models.Model): """Model for representing OS images / snapshots of hosts.""" id = models.AutoField(primary_key=True) - lab_id = models.IntegerField() # ID the lab who holds this image knows from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE) - name = models.CharField(max_length=200) + architecture = models.CharField(max_length=50, choices=[ + ("x86_64", "x86_64"), + ("aarch64", "aarch64"), + ("unknown", "unknown"), + ]) + lab_id = models.CharField(max_length=100) + name = models.CharField(max_length=100) owner = models.ForeignKey(User, null=True, on_delete=models.SET_NULL) public = models.BooleanField(default=True) - host_type = models.ForeignKey(ResourceProfile, on_delete=models.CASCADE) description = models.TextField() os = models.ForeignKey(Opsys, null=True, on_delete=models.CASCADE) + available = models.BooleanField(default=True) # marked True by cobbler if it exists there + obsolete = models.BooleanField(default=False) + + indexes = [ + models.Index(fields=['architecture']), + models.Index(fields=['cobbler_id']) + ] + def __str__(self): return self.name + def is_obsolete(self): + return self.obsolete or self.os.obsolete + + def serialize(self): + d = {} + for field in vars(self): + attr = getattr(self, field) + if is_serializable(attr): + d[field] = attr + return d + + def update(self, data): + for field in vars(self): + if field in data: + setattr(self, field, data[field] if data[field] else getattr(self, field)) + + def new_from_data(data): + img = Image() + img.update(data) + return img + def in_use(self): for resource in ResourceQuery.filter(config__image=self): if resource.is_reserved(): @@ -409,7 +521,7 @@ Networking configuration models class Network(models.Model): id = models.AutoField(primary_key=True) - name = models.CharField(max_length=100) + name = models.CharField(max_length=200) bundle = models.ForeignKey(ResourceTemplate, on_delete=models.CASCADE, related_name="networks") is_public = models.BooleanField() @@ -507,6 +619,13 @@ class NetworkRole(models.Model): network = models.ForeignKey(Network, on_delete=models.CASCADE) +def create_resource_ref_string(for_hosts: [str]) -> str: + # need to sort the list, then do dump + for_hosts.sort() + + return json.dumps(for_hosts) + + class OPNFVConfig(models.Model): id = models.AutoField(primary_key=True) installer = models.ForeignKey(Installer, on_delete=models.CASCADE) diff --git a/src/resource_inventory/resource_manager.py b/src/resource_inventory/resource_manager.py index 37bf33c..52af824 100644 --- a/src/resource_inventory/resource_manager.py +++ b/src/resource_inventory/resource_manager.py @@ -6,7 +6,8 @@ # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -from __future__ import annotations + +from __future__ import annotations # noqa: F407 import re from typing import Optional @@ -85,12 +86,13 @@ class ResourceManager: vlan_manager = resourceTemplate.lab.vlan_manager for network in resourceTemplate.networks.all(): if network.is_public: - public_net = vlan_manager.get_public_vlan() + # already throws if can't get requested count, so can always expect public_net to be Some + public_net = vlan_manager.get_public_vlan(within=resourceTemplate.public_vlan_pool_set()) vlan_manager.reserve_public_vlan(public_net.vlan) networks[network.name] = public_net.vlan else: # already throws if can't get requested count, so can always index in @ 0 - vlans = vlan_manager.get_vlans(count=1) + vlans = vlan_manager.get_vlans(count=1, within=resourceTemplate.private_vlan_pool_set()) vlan_manager.reserve_vlans(vlans[0]) networks[network.name] = vlans[0] return networks diff --git a/src/resource_inventory/tests/test_models.py b/src/resource_inventory/tests/test_models.py index e1b2106..3f2d1d8 100644 --- a/src/resource_inventory/tests/test_models.py +++ b/src/resource_inventory/tests/test_models.py @@ -80,7 +80,7 @@ class ConfigUtil(): ) return Image.objects.create( - lab_id=0, + cobbler_id="profile1", from_lab=lab, name="an image for testing", owner=owner |