Merge "Test resource templates now use the same lab as the image generated alongside it."

6 files changed, 206 insertions, 81 deletions

diff --git a/src/api/migrations/0011_auto_20200218_1536.py b/src/api/migrations/0011_auto_20200218_1536.py
new file mode 100644
index 0000000..0fd7029
--- /dev/null
+++ b/src/api/migrations/0011_auto_20200218_1536.py
@@ -0,0 +1,29 @@
+# Generated by Django 2.2 on 2020-02-18 15:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0010_auto_20191219_2004'),
+        # ('resource_inventory', '0013_auto_20200218_1536')
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='hosthardwarerelation',
+            name='resource_id',
+            field=models.CharField(default='default_id', max_length=200),
+        ),
+        migrations.AddField(
+            model_name='hostnetworkrelation',
+            name='resource_id',
+            field=models.CharField(default='default_id', max_length=200),
+        ),
+        migrations.AddField(
+            model_name='snapshotconfig',
+            name='resource_id',
+            field=models.CharField(default='default_id', max_length=200),
+        ),
+    ]
diff --git a/src/api/migrations/0012_manual_20200218_1536.py b/src/api/migrations/0012_manual_20200218_1536.py
new file mode 100644
index 0000000..55befbd
--- /dev/null
+++ b/src/api/migrations/0012_manual_20200218_1536.py
@@ -0,0 +1,22 @@
+# Generated by Django 2.2 on 2020-02-18 15:36
+
+from django.db import migrations
+
+
+def set_resource_id(apps, schema_editor):
+    for cls in ["HostHardwareRelation", "HostNetworkRelation", "SnapshotConfig"]:
+        model = apps.get_model('api', cls)
+        for m in model.objects.all():
+            m.resource_id = m.host.labid
+            m.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0011_auto_20200218_1536'),
+    ]
+
+    operations = [
+        migrations.RunPython(set_resource_id),
+    ]
diff --git a/src/api/migrations/0013_manual_20200218_1536.py b/src/api/migrations/0013_manual_20200218_1536.py
new file mode 100644
index 0000000..0b76e84
--- /dev/null
+++ b/src/api/migrations/0013_manual_20200218_1536.py
@@ -0,0 +1,29 @@
+# Generated by Django 2.2 on 2020-02-18 15:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0012_manual_20200218_1536'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='hosthardwarerelation',
+            name='host',
+        ),
+        migrations.RemoveField(
+            model_name='hostnetworkrelation',
+            name='host',
+        ),
+        migrations.RemoveField(
+            model_name='snapshotconfig',
+            name='host',
+        ),
+        migrations.RemoveField(
+            model_name='opnfvapiconfig',
+            name='roles',
+        ),
+    ]
diff --git a/src/api/migrations/0014_manual_20200220.py b/src/api/migrations/0014_manual_20200220.py
new file mode 100644
index 0000000..2e2cd58
--- /dev/null
+++ b/src/api/migrations/0014_manual_20200220.py
@@ -0,0 +1,18 @@
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0013_manual_20200218_1536'),
+        ('resource_inventory', '0013_auto_20200218_1536')
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='opnfvapiconfig',
+            name='roles',
+            field=models.ManyToManyField(to='resource_inventory.ResourceOPNFVConfig'),
+        ),
+    ]
diff --git a/src/api/models.py b/src/api/models.py
index 1e5a2da..de73a7a 100644
--- a/src/api/models.py
+++ b/src/api/models.py
@@ -10,8 +10,9 @@
 
 from django.contrib.auth.models import User
 from django.db import models
-from django.core.exceptions import PermissionDenied
+from django.core.exceptions import PermissionDenied, ValidationError
 from django.shortcuts import get_object_or_404
+from django.http import HttpResponseNotFound
 from django.urls import reverse
 from django.utils import timezone
 
@@ -21,18 +22,19 @@ import uuid
 from booking.models import Booking
 from resource_inventory.models import (
     Lab,
-    HostProfile,
-    Host,
+    ResourceProfile,
     Image,
     Interface,
-    HostOPNFVConfig,
+    ResourceOPNFVConfig,
     RemoteInfo,
     OPNFVConfig,
-    ConfigState
+    ConfigState,
+    ResourceQuery
 )
 from resource_inventory.idf_templater import IDFTemplater
 from resource_inventory.pdf_templater import PDFTemplater
 from account.models import Downtime
+from dashboard.utils import AbstractModelQuery
 
 
 class JobStatus(object):
@@ -115,8 +117,11 @@ class LabManager(object):
         )
         return self.get_downtime_json()
 
-    def update_host_remote_info(self, data, host_id):
-        host = get_object_or_404(Host, labid=host_id, lab=self.lab)
+    def update_host_remote_info(self, data, res_id):
+        resource = ResourceQuery.filter(labid=res_id, lab=self.lab)
+        if len(resource) != 1:
+            return HttpResponseNotFound("Could not find single host with id " + str(res_id))
+        resource = resource[0]
         info = {}
         try:
             info['address'] = data['address']
@@ -127,7 +132,7 @@ class LabManager(object):
             info['versions'] = json.dumps(data['versions'])
         except Exception as e:
             return {"error": "invalid arguement: " + str(e)}
-        remote_info = host.remote_management
+        remote_info = resource.remote_management
         if "default" in remote_info.mac_address:
             remote_info = RemoteInfo()
         remote_info.address = info['address']
@@ -137,9 +142,9 @@ class LabManager(object):
         remote_info.type = info['type']
         remote_info.versions = info['versions']
         remote_info.save()
-        host.remote_management = remote_info
-        host.save()
-        booking = Booking.objects.get(resource=host.bundle)
+        resource.remote_management = remote_info
+        resource.save()
+        booking = Booking.objects.get(resource=resource.bundle)
         self.update_xdf(booking)
         return {"status": "success"}
 
@@ -163,41 +168,42 @@ class LabManager(object):
             "phone": self.lab.contact_phone,
             "email": self.lab.contact_email
         }
-        prof['host_count'] = []
-        for host in HostProfile.objects.filter(labs=self.lab):
-            count = Host.objects.filter(profile=host, lab=self.lab).count()
-            prof['host_count'].append(
-                {
-                    "type": host.name,
-                    "count": count
-                }
-            )
+        prof['host_count'] = [{
+            "type": profile.name,
+            "count": len(profile.get_resources(lab=self.lab))}
+            for profile in ResourceProfile.objects.filter(labs=self.lab)]
         return prof
 
     def get_inventory(self):
         inventory = {}
-        hosts = Host.objects.filter(lab=self.lab)
+        resources = ResourceQuery.filter(lab=self.lab)
         images = Image.objects.filter(from_lab=self.lab)
-        profiles = HostProfile.objects.filter(labs=self.lab)
-        inventory['hosts'] = self.serialize_hosts(hosts)
+        profiles = ResourceProfile.objects.filter(labs=self.lab)
+        inventory['resources'] = self.serialize_resources(resources)
         inventory['images'] = self.serialize_images(images)
         inventory['host_types'] = self.serialize_host_profiles(profiles)
         return inventory
 
     def get_host(self, hostname):
-        host = get_object_or_404(Host, labid=hostname, lab=self.lab)
+        resource = ResourceQuery.filter(labid=hostname, lab=self.lab)
+        if len(resource) != 1:
+            return HttpResponseNotFound("Could not find single host with id " + str(hostname))
+        resource = resource[0]
         return {
-            "booked": host.booked,
-            "working": host.working,
-            "type": host.profile.name
+            "booked": resource.booked,
+            "working": resource.working,
+            "type": resource.profile.name
         }
 
     def update_host(self, hostname, data):
-        host = get_object_or_404(Host, labid=hostname, lab=self.lab)
+        resource = ResourceQuery.filter(labid=hostname, lab=self.lab)
+        if len(resource) != 1:
+            return HttpResponseNotFound("Could not find single host with id " + str(hostname))
+        resource = resource[0]
         if "working" in data:
             working = data['working'] == "true"
-            host.working = working
-        host.save()
+            resource.working = working
+        resource.save()
         return self.get_host(hostname)
 
     def get_status(self):
@@ -237,20 +243,22 @@ class LabManager(object):
 
         return job_ser
 
-    def serialize_hosts(self, hosts):
+    def serialize_resources(self, resources):
+        # TODO: rewrite for Resource model
         host_ser = []
-        for host in hosts:
-            h = {}
-            h['interfaces'] = []
-            h['hostname'] = host.name
-            h['host_type'] = host.profile.name
-            for iface in host.interfaces.all():
-                eth = {}
-                eth['mac'] = iface.mac_address
-                eth['busaddr'] = iface.bus_address
-                eth['name'] = iface.name
-                eth['switchport'] = {"switch_name": iface.switch_name, "port_name": iface.port_name}
-                h['interfaces'].append(eth)
+        for res in resources:
+            r = {
+                'interfaces': [],
+                'hostname': res.name,
+                'host_type': res.profile.name
+            }
+            for iface in res.get_interfaces():
+                r['interfaces'].append({
+                    'mac': iface.mac_address,
+                    'busaddr': iface.bus_address,
+                    'name': iface.name,
+                    'switchport': {"switch_name": iface.switch_name, "port_name": iface.port_name}
+                })
         return host_ser
 
     def serialize_images(self, images):
@@ -265,7 +273,7 @@ class LabManager(object):
             )
         return images_ser
 
-    def serialize_host_profiles(self, profiles):
+    def serialize_resource_profiles(self, profiles):
         profile_ser = []
         for profile in profiles:
             p = {}
@@ -323,21 +331,9 @@ class Job(models.Model):
         return {"id": self.id, "payload": d}
 
     def get_tasklist(self, status="all"):
-        tasklist = []
-        clist = [
-            HostHardwareRelation,
-            AccessRelation,
-            HostNetworkRelation,
-            SoftwareRelation,
-            SnapshotRelation
-        ]
         if status == "all":
-            for cls in clist:
-                tasklist += list(cls.objects.filter(job=self))
-        else:
-            for cls in clist:
-                tasklist += list(cls.objects.filter(job=self).filter(status=status))
-        return tasklist
+            return JobTaskQuery.filter(job=self, status=status)
+        return JobTaskQuery.filter(job=self)
 
     def is_fulfilled(self):
         """
@@ -435,7 +431,7 @@ class OpnfvApiConfig(models.Model):
 
     installer = models.CharField(max_length=200)
     scenario = models.CharField(max_length=300)
-    roles = models.ManyToManyField(Host)
+    roles = models.ManyToManyField(ResourceOPNFVConfig)
     # pdf and idf are url endpoints, not the actual file
     pdf = models.CharField(max_length=100)
     idf = models.CharField(max_length=100)
@@ -632,6 +628,8 @@ class NetworkConfig(TaskConfig):
         for interface in self.interfaces.all():
             d[hid][interface.mac_address] = []
             for vlan in interface.config.all():
+                # TODO: should this come from the interface?
+                # e.g. will different interfaces for different resources need different configs?
                 d[hid][interface.mac_address].append({"vlan_id": vlan.vlan_id, "tagged": vlan.tagged})
 
         return d
@@ -665,7 +663,7 @@ class NetworkConfig(TaskConfig):
 
 class SnapshotConfig(TaskConfig):
 
-    host = models.ForeignKey(Host, null=True, on_delete=models.DO_NOTHING)
+    resource_id = models.CharField(max_length=200, default="default_id")
     image = models.IntegerField(null=True)
     dashboard_id = models.IntegerField()
     delta = models.TextField(default="{}")
@@ -718,6 +716,11 @@ class SnapshotConfig(TaskConfig):
         d['dashboard_id'] = self.dashboard_id
         self.delta = json.dumps(d)
 
+    def save(self, *args, **kwargs):
+        if len(ResourceQuery.filter(labid=self.resource_id)) != 1:
+            raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource")
+        super().save(*args, **kwargs)
+
 
 def get_task(task_id):
     for taskclass in [AccessRelation, SoftwareRelation, HostHardwareRelation, HostNetworkRelation, SnapshotRelation]:
@@ -787,7 +790,7 @@ class SoftwareRelation(TaskRelation):
 
 
 class HostHardwareRelation(TaskRelation):
-    host = models.ForeignKey(Host, on_delete=models.CASCADE)
+    resource_id = models.CharField(max_length=200, default="default_id")
     config = models.OneToOneField(HardwareConfig, on_delete=models.CASCADE)
     job_key = "hardware"
 
@@ -801,9 +804,14 @@ class HostHardwareRelation(TaskRelation):
         self.config.delete()
         return super(self.__class__, self).delete(*args, **kwargs)
 
+    def save(self, *args, **kwargs):
+        if len(ResourceQuery.filter(labid=self.resource_id)) != 1:
+            raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource")
+        super().save(*args, **kwargs)
+
 
 class HostNetworkRelation(TaskRelation):
-    host = models.ForeignKey(Host, on_delete=models.CASCADE)
+    resource_id = models.CharField(max_length=200, default="default_id")
     config = models.OneToOneField(NetworkConfig, on_delete=models.CASCADE)
     job_key = "network"
 
@@ -814,6 +822,11 @@ class HostNetworkRelation(TaskRelation):
         self.config.delete()
         return super(self.__class__, self).delete(*args, **kwargs)
 
+    def save(self, *args, **kwargs):
+        if len(ResourceQuery.filter(labid=self.resource_id)) != 1:
+            raise ValidationError("resource_id " + str(self.resource_id) + " does not refer to a single resource")
+        super().save(*args, **kwargs)
+
 
 class SnapshotRelation(TaskRelation):
     snapshot = models.ForeignKey(Image, on_delete=models.CASCADE)
@@ -876,18 +889,18 @@ class JobFactory(object):
     @classmethod
     def makeCompleteJob(cls, booking):
         """Create everything that is needed to fulfill the given booking."""
-        hosts = Host.objects.filter(bundle=booking.resource)
+        resources = booking.resource.get_resources()
         job = None
         try:
             job = Job.objects.get(booking=booking)
         except Exception:
             job = Job.objects.create(status=JobStatus.NEW, booking=booking)
         cls.makeHardwareConfigs(
-            hosts=hosts,
+            resources=resources,
             job=job
         )
         cls.makeNetworkConfigs(
-            hosts=hosts,
+            resources=resources,
             job=job
         )
         cls.makeSoftware(
@@ -911,29 +924,29 @@ class JobFactory(object):
                     job=job,
                     context={
                         "key": user.userprofile.ssh_public_key.open().read().decode(encoding="UTF-8"),
-                        "hosts": [host.labid for host in hosts]
+                        "hosts": [r.labid for r in resources]
                     }
                 )
             except Exception:
                 continue
 
     @classmethod
-    def makeHardwareConfigs(cls, hosts=[], job=Job()):
+    def makeHardwareConfigs(cls, resources=[], job=Job()):
         """
         Create and save HardwareConfig.
 
         Helper function to create the tasks related to
         configuring the hardware
         """
-        for host in hosts:
+        for res in resources:
             hardware_config = None
             try:
-                hardware_config = HardwareConfig.objects.get(relation__host=host)
+                hardware_config = HardwareConfig.objects.get(relation__host=res)
             except Exception:
                 hardware_config = HardwareConfig()
 
             relation = HostHardwareRelation()
-            relation.host = host
+            relation.resource_id = res.labid
             relation.job = job
             relation.config = hardware_config
             relation.config.save()
@@ -969,29 +982,30 @@ class JobFactory(object):
             config.save()
 
     @classmethod
-    def makeNetworkConfigs(cls, hosts=[], job=Job()):
+    def makeNetworkConfigs(cls, resources=[], job=Job()):
         """
         Create and save NetworkConfig.
 
         Helper function to create the tasks related to
         configuring the networking
         """
-        for host in hosts:
+        for res in resources:
             network_config = None
             try:
-                network_config = NetworkConfig.objects.get(relation__host=host)
+                network_config = NetworkConfig.objects.get(relation__host=res)
             except Exception:
                 network_config = NetworkConfig.objects.create()
 
             relation = HostNetworkRelation()
-            relation.host = host
+            relation.resource_id = res.labid
             relation.job = job
             network_config.save()
             relation.config = network_config
             relation.save()
             network_config.clear_delta()
 
-            for interface in host.interfaces.all():
+            # TODO: use get_interfaces() on resource
+            for interface in res.interfaces.all():
                 network_config.add_interface(interface)
             network_config.save()
 
@@ -1000,13 +1014,13 @@ class JobFactory(object):
         if booking.resource.hosts.count() < 2:
             return None
         try:
-            jumphost_config = HostOPNFVConfig.objects.filter(
+            jumphost_config = ResourceOPNFVConfig.objects.filter(
                 role__name__iexact="jumphost"
             )
-            jumphost = Host.objects.get(
+            jumphost = ResourceQuery.filter(
                 bundle=booking.resource,
-                config=jumphost_config.host_config
-            )
+                config=jumphost_config.resource_config
+            )[0]
         except Exception:
             return None
         br_config = BridgeConfig.objects.create(opnfv_config=booking.opnfv_config)
@@ -1040,3 +1054,16 @@ class JobFactory(object):
         software_config = SoftwareConfig.objects.create(opnfv=opnfv_api_config)
         software_relation = SoftwareRelation.objects.create(job=job, config=software_config)
         return software_relation
+
+
+JOB_TASK_CLASSLIST = [
+    HostHardwareRelation,
+    AccessRelation,
+    HostNetworkRelation,
+    SoftwareRelation,
+    SnapshotRelation
+]
+
+
+class JobTaskQuery(AbstractModelQuery):
+    model_list = JOB_TASK_CLASSLIST
diff --git a/src/api/serializers/booking_serializer.py b/src/api/serializers/booking_serializer.py
index 46a2348..993eb22 100644
--- a/src/api/serializers/booking_serializer.py
+++ b/src/api/serializers/booking_serializer.py
@@ -11,7 +11,7 @@
 from rest_framework import serializers
 
 from resource_inventory.models import (
-    HostConfiguration,
+    ResourceConfiguration,
     CpuProfile,
     DiskProfile,
     InterfaceProfile,
@@ -35,7 +35,7 @@ class BookingField(serializers.Field):
         host_configs = {}  # mapping hostname -> config
         networks = {}  # mapping vlan id -> network_hosts
         for host in booking.resource.hosts.all():
-            host_configs[host.name] = HostConfiguration.objects.get(host=host.template)
+            host_configs[host.name] = ResourceConfiguration.objects.get(host=host.template)
             if "jumphost" not in ser and host_configs[host.name].opnfvRole.name.lower() == "jumphost":
                 ser['jumphost'] = host.name
             # host is a Host model