aboutsummaryrefslogtreecommitdiffstats
path: root/src/resource_inventory
diff options
context:
space:
mode:
Diffstat (limited to 'src/resource_inventory')
-rw-r--r--src/resource_inventory/migrations/0018_auto_20210630_1629.py101
-rw-r--r--src/resource_inventory/migrations/0019_auto_20210701_1947.py43
-rw-r--r--src/resource_inventory/migrations/0020_cloudinitfile.py21
-rw-r--r--src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py18
-rw-r--r--src/resource_inventory/migrations/0022_auto_20210925_2028.py23
-rw-r--r--src/resource_inventory/models.py131
-rw-r--r--src/resource_inventory/resource_manager.py8
-rw-r--r--src/resource_inventory/tests/test_models.py2
8 files changed, 337 insertions, 10 deletions
diff --git a/src/resource_inventory/migrations/0018_auto_20210630_1629.py b/src/resource_inventory/migrations/0018_auto_20210630_1629.py
new file mode 100644
index 0000000..19e53e4
--- /dev/null
+++ b/src/resource_inventory/migrations/0018_auto_20210630_1629.py
@@ -0,0 +1,101 @@
+# Generated by Django 2.2 on 2021-06-30 16:29
+
+from django.db import migrations, models
+import django.db.models.deletion
+from account.models import Lab
+
+
+def set_availability(apps, schema_editor):
+ models = [apps.get_model('resource_inventory', 'Image'), apps.get_model('resource_inventory', 'Opsys')]
+
+ for model in models:
+ for obj in model.objects.all():
+ obj.available = False
+ obj.obsolete = True
+ obj.save()
+
+
+def set_rconfig_arch(apps, schema_editor):
+ rprofs = apps.get_model('resource_inventory', 'ResourceProfile')
+
+ for rprof in rprofs.objects.all():
+ rprof.architecture = rprof.cpuprofile.first().architecture
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('account', '0009_auto_20210324_2107'),
+ ('resource_inventory', '0017_auto_20201218_1516'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='image',
+ name='host_type',
+ ),
+ migrations.AlterField(
+ model_name='image',
+ name='lab_id',
+ field=models.CharField(default='none (retired)', max_length=100),
+ preserve_default=True,
+ ),
+ migrations.RemoveField(
+ model_name='opsys',
+ name='sup_installers',
+ ),
+
+ migrations.AddField(
+ model_name='image',
+ name='architecture',
+ field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50),
+ preserve_default=False,
+ ),
+
+ migrations.AddField(
+ model_name='image',
+ name='available',
+ field=models.BooleanField(default=True),
+ ),
+ migrations.AddField(
+ model_name='image',
+ name='obsolete',
+ field=models.BooleanField(default=False),
+ ),
+
+ migrations.AddField(
+ model_name='opsys',
+ name='available',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AddField(
+ model_name='opsys',
+ name='obsolete',
+ field=models.BooleanField(default=True),
+ ),
+
+ migrations.RunPython(set_availability),
+
+ migrations.AddField(
+ model_name='opsys',
+ name='lab_id',
+ field=models.CharField(default="none (retired)", max_length=100),
+ preserve_default=False,
+ ),
+
+ migrations.AddField(
+ model_name='opsys',
+ name='from_lab',
+ field=models.ForeignKey(default=Lab.objects.first, on_delete=django.db.models.deletion.CASCADE, to='account.Lab'),
+ preserve_default=False,
+ ),
+
+ migrations.AddField(
+ model_name='resourceprofile',
+ name='architecture',
+ field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64'), ('unknown', 'unknown')], default='unknown', max_length=50),
+ preserve_default=False,
+ ),
+
+ migrations.RunPython(set_rconfig_arch),
+ ]
diff --git a/src/resource_inventory/migrations/0019_auto_20210701_1947.py b/src/resource_inventory/migrations/0019_auto_20210701_1947.py
new file mode 100644
index 0000000..e64d174
--- /dev/null
+++ b/src/resource_inventory/migrations/0019_auto_20210701_1947.py
@@ -0,0 +1,43 @@
+# Generated by Django 2.2 on 2021-07-01 19:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('resource_inventory', '0018_auto_20210630_1629'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='image',
+ name='lab_id',
+ field=models.CharField(max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='image',
+ name='name',
+ field=models.CharField(max_length=100),
+ ),
+ migrations.AlterField(
+ model_name='network',
+ name='name',
+ field=models.CharField(max_length=200),
+ ),
+ migrations.AlterField(
+ model_name='opsys',
+ name='available',
+ field=models.BooleanField(default=True),
+ ),
+ migrations.AlterField(
+ model_name='opsys',
+ name='obsolete',
+ field=models.BooleanField(default=False),
+ ),
+ migrations.AlterField(
+ model_name='resourceprofile',
+ name='architecture',
+ field=models.CharField(choices=[('x86_64', 'x86_64'), ('aarch64', 'aarch64')], max_length=50),
+ ),
+ ]
diff --git a/src/resource_inventory/migrations/0020_cloudinitfile.py b/src/resource_inventory/migrations/0020_cloudinitfile.py
new file mode 100644
index 0000000..198181c
--- /dev/null
+++ b/src/resource_inventory/migrations/0020_cloudinitfile.py
@@ -0,0 +1,21 @@
+# Generated by Django 2.2 on 2021-09-07 14:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('resource_inventory', '0019_auto_20210701_1947'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='CloudInitFile',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('text', models.TextField()),
+ ('priority', models.IntegerField()),
+ ],
+ ),
+ ]
diff --git a/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py
new file mode 100644
index 0000000..6b0befc
--- /dev/null
+++ b/src/resource_inventory/migrations/0021_resourceconfiguration_cloud_init_files.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2 on 2021-09-10 18:10
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('resource_inventory', '0020_cloudinitfile'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='resourceconfiguration',
+ name='cloud_init_files',
+ field=models.ManyToManyField(blank=True, to='resource_inventory.CloudInitFile'),
+ ),
+ ]
diff --git a/src/resource_inventory/migrations/0022_auto_20210925_2028.py b/src/resource_inventory/migrations/0022_auto_20210925_2028.py
new file mode 100644
index 0000000..2b0b902
--- /dev/null
+++ b/src/resource_inventory/migrations/0022_auto_20210925_2028.py
@@ -0,0 +1,23 @@
+# Generated by Django 2.2 on 2021-09-25 20:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('resource_inventory', '0021_resourceconfiguration_cloud_init_files'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='resourcetemplate',
+ name='private_vlan_pool',
+ field=models.TextField(default=''),
+ ),
+ migrations.AddField(
+ model_name='resourcetemplate',
+ name='public_vlan_pool',
+ field=models.TextField(default=''),
+ ),
+ ]
diff --git a/src/resource_inventory/models.py b/src/resource_inventory/models.py
index 7fe479a..aefd5ce 100644
--- a/src/resource_inventory/models.py
+++ b/src/resource_inventory/models.py
@@ -9,10 +9,12 @@
##############################################################################
from django.contrib.auth.models import User
+
from django.core.exceptions import ValidationError
from django.db import models
from django.db.models import Q
import traceback
+import json
import re
from collections import Counter
@@ -20,7 +22,6 @@ from collections import Counter
from account.models import Lab
from dashboard.utils import AbstractModelQuery
-
"""
Profiles of resources hosted by labs.
@@ -33,6 +34,10 @@ Profile models (e.g. an x86 server profile and armv8 server profile.
class ResourceProfile(models.Model):
id = models.AutoField(primary_key=True)
name = models.CharField(max_length=200, unique=True)
+ architecture = models.CharField(max_length=50, choices=[
+ ("x86_64", "x86_64"),
+ ("aarch64", "aarch64")
+ ])
description = models.TextField()
labs = models.ManyToManyField(Lab, related_name="resourceprofiles")
@@ -147,6 +152,24 @@ with varying degrees of abstraction.
"""
+class CloudInitFile(models.Model):
+ text = models.TextField()
+
+ # higher priority is applied later, so "on top" of existing files
+ priority = models.IntegerField()
+
+ @classmethod
+ def merge_strategy(cls):
+ return [
+ {'name': 'list', 'settings': ['append']},
+ {'name': 'dict', 'settings': ['recurse_list', 'replace']},
+ ]
+
+ @classmethod
+ def create(cls, text="", priority=0):
+ return CloudInitFile.objects.create(priority=priority, text=text)
+
+
class ResourceTemplate(models.Model):
"""
Models a "template" of a complete, configured collection of resources that can be booked.
@@ -167,6 +190,24 @@ class ResourceTemplate(models.Model):
temporary = models.BooleanField(default=False)
copy_of = models.ForeignKey("ResourceTemplate", blank=True, null=True, on_delete=models.SET_NULL)
+ # if these fields are empty ("") then they are implicitly "every vlan",
+ # otherwise we filter any allocations we try to instantiate against this list
+ # they should be represented as a json list of integers
+ private_vlan_pool = models.TextField(default="")
+ public_vlan_pool = models.TextField(default="")
+
+ def private_vlan_pool_set(self):
+ if self.private_vlan_pool != "":
+ return set(json.loads(self.private_vlan_pool))
+ else:
+ return None
+
+ def public_vlan_pool_set(self):
+ if self.private_vlan_pool != "":
+ return set(json.loads(self.public_vlan_pool))
+ else:
+ return None
+
def getConfigs(self):
configs = self.resourceConfigurations.all()
return list(configs)
@@ -235,9 +276,14 @@ class ResourceConfiguration(models.Model):
is_head_node = models.BooleanField(default=False)
name = models.CharField(max_length=3000, default="opnfv_host")
+ cloud_init_files = models.ManyToManyField(CloudInitFile, blank=True)
+
def __str__(self):
return str(self.name)
+ def ci_file_list(self):
+ return list(self.cloud_init_files.order_by("priority").all())
+
def get_default_remote_info():
return RemoteInfo.objects.get_or_create(
@@ -369,10 +415,43 @@ class Server(Resource):
return isinstance(other, Server) and other.name == self.name
+def is_serializable(data):
+ try:
+ json.dumps(data)
+ return True
+ except Exception:
+ return False
+
+
class Opsys(models.Model):
id = models.AutoField(primary_key=True)
name = models.CharField(max_length=100)
- sup_installers = models.ManyToManyField("Installer", blank=True)
+ lab_id = models.CharField(max_length=100)
+ obsolete = models.BooleanField(default=False)
+ available = models.BooleanField(default=True) # marked true by Cobbler if it exists there
+ from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE)
+
+ indexes = [
+ models.Index(fields=['cobbler_id'])
+ ]
+
+ def new_from_data(data):
+ opsys = Opsys()
+ opsys.update(data)
+ return opsys
+
+ def serialize(self):
+ d = {}
+ for field in vars(self):
+ attr = getattr(self, field)
+ if is_serializable(attr):
+ d[field] = attr
+ return d
+
+ def update(self, data):
+ for field in vars(self):
+ if field in data:
+ setattr(self, field, data[field] if data[field] else getattr(self, field))
def __str__(self):
return self.name
@@ -382,18 +461,51 @@ class Image(models.Model):
"""Model for representing OS images / snapshots of hosts."""
id = models.AutoField(primary_key=True)
- lab_id = models.IntegerField() # ID the lab who holds this image knows
from_lab = models.ForeignKey(Lab, on_delete=models.CASCADE)
- name = models.CharField(max_length=200)
+ architecture = models.CharField(max_length=50, choices=[
+ ("x86_64", "x86_64"),
+ ("aarch64", "aarch64"),
+ ("unknown", "unknown"),
+ ])
+ lab_id = models.CharField(max_length=100)
+ name = models.CharField(max_length=100)
owner = models.ForeignKey(User, null=True, on_delete=models.SET_NULL)
public = models.BooleanField(default=True)
- host_type = models.ForeignKey(ResourceProfile, on_delete=models.CASCADE)
description = models.TextField()
os = models.ForeignKey(Opsys, null=True, on_delete=models.CASCADE)
+ available = models.BooleanField(default=True) # marked True by cobbler if it exists there
+ obsolete = models.BooleanField(default=False)
+
+ indexes = [
+ models.Index(fields=['architecture']),
+ models.Index(fields=['cobbler_id'])
+ ]
+
def __str__(self):
return self.name
+ def is_obsolete(self):
+ return self.obsolete or self.os.obsolete
+
+ def serialize(self):
+ d = {}
+ for field in vars(self):
+ attr = getattr(self, field)
+ if is_serializable(attr):
+ d[field] = attr
+ return d
+
+ def update(self, data):
+ for field in vars(self):
+ if field in data:
+ setattr(self, field, data[field] if data[field] else getattr(self, field))
+
+ def new_from_data(data):
+ img = Image()
+ img.update(data)
+ return img
+
def in_use(self):
for resource in ResourceQuery.filter(config__image=self):
if resource.is_reserved():
@@ -409,7 +521,7 @@ Networking configuration models
class Network(models.Model):
id = models.AutoField(primary_key=True)
- name = models.CharField(max_length=100)
+ name = models.CharField(max_length=200)
bundle = models.ForeignKey(ResourceTemplate, on_delete=models.CASCADE, related_name="networks")
is_public = models.BooleanField()
@@ -507,6 +619,13 @@ class NetworkRole(models.Model):
network = models.ForeignKey(Network, on_delete=models.CASCADE)
+def create_resource_ref_string(for_hosts: [str]) -> str:
+ # need to sort the list, then do dump
+ for_hosts.sort()
+
+ return json.dumps(for_hosts)
+
+
class OPNFVConfig(models.Model):
id = models.AutoField(primary_key=True)
installer = models.ForeignKey(Installer, on_delete=models.CASCADE)
diff --git a/src/resource_inventory/resource_manager.py b/src/resource_inventory/resource_manager.py
index 37bf33c..52af824 100644
--- a/src/resource_inventory/resource_manager.py
+++ b/src/resource_inventory/resource_manager.py
@@ -6,7 +6,8 @@
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
-from __future__ import annotations
+
+from __future__ import annotations # noqa: F407
import re
from typing import Optional
@@ -85,12 +86,13 @@ class ResourceManager:
vlan_manager = resourceTemplate.lab.vlan_manager
for network in resourceTemplate.networks.all():
if network.is_public:
- public_net = vlan_manager.get_public_vlan()
+ # already throws if can't get requested count, so can always expect public_net to be Some
+ public_net = vlan_manager.get_public_vlan(within=resourceTemplate.public_vlan_pool_set())
vlan_manager.reserve_public_vlan(public_net.vlan)
networks[network.name] = public_net.vlan
else:
# already throws if can't get requested count, so can always index in @ 0
- vlans = vlan_manager.get_vlans(count=1)
+ vlans = vlan_manager.get_vlans(count=1, within=resourceTemplate.private_vlan_pool_set())
vlan_manager.reserve_vlans(vlans[0])
networks[network.name] = vlans[0]
return networks
diff --git a/src/resource_inventory/tests/test_models.py b/src/resource_inventory/tests/test_models.py
index e1b2106..3f2d1d8 100644
--- a/src/resource_inventory/tests/test_models.py
+++ b/src/resource_inventory/tests/test_models.py
@@ -80,7 +80,7 @@ class ConfigUtil():
)
return Image.objects.create(
- lab_id=0,
+ cobbler_id="profile1",
from_lab=lab,
name="an image for testing",
owner=owner