From 986f474e540669fd9fb72810b3f31fa3f4c3e97a Mon Sep 17 00:00:00 2001 From: Sean Smith Date: Tue, 11 Aug 2020 10:41:27 -0400 Subject: Analytics changes Signed-off-by: Sean Smith Change-Id: Iaea350b3042f9c866939a9d1a79bdef1e165c1a7 Signed-off-by: Sawyer Bergeron --- src/analytics/__init__.py | 0 src/analytics/admin.py | 13 ++++ src/analytics/apps.py | 14 ++++ src/analytics/migrations/0001_initial.py | 22 ++++++ .../migrations/0002_auto_20201109_2149.py | 27 +++++++ src/analytics/migrations/__init__.py | 0 src/analytics/models.py | 30 ++++++++ src/analytics/tests.py | 10 +++ src/analytics/views.py | 10 +++ src/api/migrations/0016_auto_20201109_2149.py | 41 ++++++++++ src/api/models.py | 90 +++++++++++++++++++++- src/api/urls.py | 4 +- src/api/views.py | 20 +++++ src/dashboard/tasks.py | 20 ++++- src/laas_dashboard/settings.py | 5 ++ 15 files changed, 302 insertions(+), 4 deletions(-) create mode 100644 src/analytics/__init__.py create mode 100644 src/analytics/admin.py create mode 100644 src/analytics/apps.py create mode 100644 src/analytics/migrations/0001_initial.py create mode 100644 src/analytics/migrations/0002_auto_20201109_2149.py create mode 100644 src/analytics/migrations/__init__.py create mode 100644 src/analytics/models.py create mode 100644 src/analytics/tests.py create mode 100644 src/analytics/views.py create mode 100644 src/api/migrations/0016_auto_20201109_2149.py (limited to 'src') diff --git a/src/analytics/__init__.py b/src/analytics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/analytics/admin.py b/src/analytics/admin.py new file mode 100644 index 0000000..63f139f --- /dev/null +++ b/src/analytics/admin.py @@ -0,0 +1,13 @@ +############################################################################## +# Copyright (c) 2020 Sean Smith and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from django.contrib import admin +from analytics.models import ActiveVPNUser + +admin.site.register(ActiveVPNUser) diff --git a/src/analytics/apps.py b/src/analytics/apps.py new file mode 100644 index 0000000..fe1b11f --- /dev/null +++ b/src/analytics/apps.py @@ -0,0 +1,14 @@ +############################################################################## +# Copyright (c) 2020 Sean Smith and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from django.apps import AppConfig + + +class AnalyticsConfig(AppConfig): + name = 'analytics' diff --git a/src/analytics/migrations/0001_initial.py b/src/analytics/migrations/0001_initial.py new file mode 100644 index 0000000..05a7ec8 --- /dev/null +++ b/src/analytics/migrations/0001_initial.py @@ -0,0 +1,22 @@ +# Generated by Django 2.2 on 2020-08-10 20:10 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='ActiveVPNUsers', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('time_stamp', models.DateTimeField(auto_now_add=True)), + ('active_users', models.IntegerField()), + ], + ), + ] diff --git a/src/analytics/migrations/0002_auto_20201109_2149.py b/src/analytics/migrations/0002_auto_20201109_2149.py new file mode 100644 index 0000000..a845ff8 --- /dev/null +++ b/src/analytics/migrations/0002_auto_20201109_2149.py @@ -0,0 +1,27 @@ +# Generated by Django 2.2 on 2020-11-09 21:49 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('account', '0006_auto_20201109_1947'), + ('analytics', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='ActiveVPNUser', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('time_stamp', models.DateTimeField(auto_now_add=True)), + ('active_users', models.IntegerField()), + ('lab', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='account.Lab')), + ], + ), + migrations.DeleteModel( + name='ActiveVPNUsers', + ), + ] diff --git a/src/analytics/migrations/__init__.py b/src/analytics/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/analytics/models.py b/src/analytics/models.py new file mode 100644 index 0000000..10baa0c --- /dev/null +++ b/src/analytics/models.py @@ -0,0 +1,30 @@ +############################################################################## +# Copyright (c) 2020 Sean Smith and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from django.db import models +from account.models import Lab + + +class ActiveVPNUser(models.Model): + """ Keeps track of how many VPN Users are connected to Lab """ + time_stamp = models.DateTimeField(auto_now_add=True) + lab = models.ForeignKey(Lab, on_delete=models.CASCADE, null=False) + active_users = models.IntegerField() + + @classmethod + def create(cls, lab_name, active_users): + """ + This creates an Active VPN Users entry from + from lab_name as a string + """ + + lab = Lab.objects.get(name=lab_name) + avu = cls(lab=lab, active_users=active_users) + avu.save() + return avu diff --git a/src/analytics/tests.py b/src/analytics/tests.py new file mode 100644 index 0000000..d234f48 --- /dev/null +++ b/src/analytics/tests.py @@ -0,0 +1,10 @@ +############################################################################## +# Copyright (c) 2020 Sean Smith and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# from django.test import TestCase diff --git a/src/analytics/views.py b/src/analytics/views.py new file mode 100644 index 0000000..160bc59 --- /dev/null +++ b/src/analytics/views.py @@ -0,0 +1,10 @@ +############################################################################## +# Copyright (c) 2020 Sean Smith and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# from django.shortcuts import render diff --git a/src/api/migrations/0016_auto_20201109_2149.py b/src/api/migrations/0016_auto_20201109_2149.py new file mode 100644 index 0000000..a430659 --- /dev/null +++ b/src/api/migrations/0016_auto_20201109_2149.py @@ -0,0 +1,41 @@ +# Generated by Django 2.2 on 2020-11-09 21:49 + +import api.models +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('api', '0015_auto_20201109_1947'), + ] + + operations = [ + migrations.CreateModel( + name='ActiveUsersConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ], + ), + migrations.AddField( + model_name='job', + name='job_type', + field=models.CharField(choices=[('BOOK', 'Booking'), ('DATA', 'Analytics')], default='BOOK', max_length=4), + ), + migrations.CreateModel( + name='ActiveUsersRelation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.IntegerField(default=0)), + ('task_id', models.CharField(default=api.models.get_task_uuid, max_length=37)), + ('lab_token', models.CharField(default='null', max_length=50)), + ('message', models.TextField(default='')), + ('config', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='api.ActiveUsersConfig')), + ('job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='api.Job')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/src/api/models.py b/src/api/models.py index 9b9b778..527e66b 100644 --- a/src/api/models.py +++ b/src/api/models.py @@ -249,6 +249,15 @@ class LabManager(object): return self.serialize_jobs(jobs, status=JobStatus.DONE) + def get_analytics_job(self): + """ Get analytics job with status new """ + jobs = Job.objects.filter( + booking__lab=self.lab, + job_type='DATA' + ) + + return self.serialize_jobs(jobs, status=JobStatus.NEW) + def get_job(self, jobid): return Job.objects.get(pk=jobid).to_dict() @@ -339,9 +348,19 @@ class Job(models.Model): This is the class that is serialized and put into the api """ + JOB_TYPES = ( + ('BOOK', 'Booking'), + ('DATA', 'Analytics') + ) + booking = models.OneToOneField(Booking, on_delete=models.CASCADE, null=True) status = models.IntegerField(default=JobStatus.NEW) complete = models.BooleanField(default=False) + job_type = models.CharField( + max_length=4, + choices=JOB_TYPES, + default='BOOK' + ) def to_dict(self): d = {} @@ -449,6 +468,28 @@ class BridgeConfig(models.Model): return json.dumps(self.to_dict()) +class ActiveUsersConfig(models.Model): + """ + Task for getting active VPN users + + StackStorm needs no information to run this job + so this task is very bare, but neccessary to fit + job creation convention. + """ + + def clear_delta(self): + self.delta = '{}' + + def get_delta(self): + return json.loads(self.to_json()) + + def to_json(self): + return json.dumps(self.to_dict()) + + def to_dict(self): + return {} + + class OpnfvApiConfig(models.Model): installer = models.CharField(max_length=200) @@ -860,6 +901,14 @@ class SnapshotRelation(TaskRelation): return super(self.__class__, self).delete(*args, **kwargs) +class ActiveUsersRelation(TaskRelation): + config = models.OneToOneField(ActiveUsersConfig, on_delete=models.CASCADE) + job_key = "active users task" + + def type_str(self): + return "Active Users Task" + + class JobFactory(object): """This class creates all the API models (jobs, tasks, etc) needed to fulfill a booking.""" @@ -902,6 +951,44 @@ class JobFactory(object): config.set_host(host) config.save() + @classmethod + def makeActiveUsersTask(cls): + """ Append active users task to analytics job """ + config = ActiveUsersConfig() + relation = ActiveUsersRelation() + job = Job.objects.get(job_type='DATA') + + job.status = JobStatus.NEW + + relation.job = job + relation.config = config + relation.config.save() + relation.config = relation.config + relation.save() + config.save() + + @classmethod + def makeAnalyticsJob(cls, booking): + """ + Create the analytics job + + This will only run once since there will only be one analytics job. + All analytics tasks get appended to analytics job. + """ + + if len(Job.objects.filter(job_type='DATA')) > 0: + raise Exception("Cannot have more than one analytics job") + + if booking.resource: + raise Exception("Booking is not marker for analytics job, has resoure") + + job = Job() + job.booking = booking + job.job_type = 'DATA' + job.save() + + cls.makeActiveUsersTask() + @classmethod def makeCompleteJob(cls, booking): """Create everything that is needed to fulfill the given booking.""" @@ -1077,7 +1164,8 @@ JOB_TASK_CLASSLIST = [ AccessRelation, HostNetworkRelation, SoftwareRelation, - SnapshotRelation + SnapshotRelation, + ActiveUsersRelation ] diff --git a/src/api/urls.py b/src/api/urls.py index 0005d34..bae86ea 100644 --- a/src/api/urls.py +++ b/src/api/urls.py @@ -44,7 +44,8 @@ from api.views import ( get_idf, lab_users, lab_user, - GenerateTokenView + GenerateTokenView, + analytics_job ) urlpatterns = [ @@ -61,6 +62,7 @@ urlpatterns = [ path('labs//jobs/new', new_jobs), path('labs//jobs/current', current_jobs), path('labs//jobs/done', done_jobs), + path('labs//jobs/getByType/DATA', analytics_job), path('labs//users', lab_users), path('labs//users/', lab_user), url(r'^token$', GenerateTokenView.as_view(), name='generate_token'), diff --git a/src/api/views.py b/src/api/views.py index 75a0db3..2e5f33f 100644 --- a/src/api/views.py +++ b/src/api/views.py @@ -18,6 +18,7 @@ from django.http.response import JsonResponse, HttpResponse from rest_framework import viewsets from rest_framework.authtoken.models import Token from django.views.decorators.csrf import csrf_exempt +from django.core.exceptions import ObjectDoesNotExist from api.serializers.booking_serializer import BookingSerializer from api.serializers.old_serializers import UserSerializer @@ -26,6 +27,8 @@ from account.models import UserProfile from booking.models import Booking from api.models import LabManagerTracker, get_task from notifier.manager import NotificationHandler +from analytics.models import ActiveVPNUser +import json """ API views. @@ -176,6 +179,23 @@ def current_jobs(request, lab_name=""): return JsonResponse(lab_manager.get_current_jobs(), safe=False) +@csrf_exempt +def analytics_job(request, lab_name=""): + """ returns all jobs with type booking""" + lab_token = request.META.get('HTTP_AUTH_TOKEN') + lab_manager = LabManagerTracker.get(lab_name, lab_token) + if request.method == "GET": + return JsonResponse(lab_manager.get_analytics_job(), safe=False) + if request.method == "POST": + users = json.loads(request.body.decode('utf-8'))['active_users'] + try: + ActiveVPNUser.create(lab_name, users) + except ObjectDoesNotExist: + return JsonResponse('Lab does not exist!', safe=False) + return HttpResponse(status=200) + return HttpResponse(status=405) + + def lab_downtime(request, lab_name=""): lab_token = request.META.get('HTTP_AUTH_TOKEN') lab_manager = LabManagerTracker.get(lab_name, lab_token) diff --git a/src/dashboard/tasks.py b/src/dashboard/tasks.py index 50e64c8..8554f6c 100644 --- a/src/dashboard/tasks.py +++ b/src/dashboard/tasks.py @@ -13,7 +13,17 @@ from celery import shared_task from django.utils import timezone from booking.models import Booking from notifier.manager import NotificationHandler -from api.models import Job, JobStatus, SoftwareRelation, HostHardwareRelation, HostNetworkRelation, AccessRelation +from api.models import ( + Job, + JobStatus, + SoftwareRelation, + HostHardwareRelation, + HostNetworkRelation, + AccessRelation, + JobFactory +) + +from resource_inventory.resource_manager import ResourceManager from resource_inventory.models import ConfigState @@ -74,4 +84,10 @@ def free_hosts(): resource__isnull=False ) for booking in bookings: - booking.resource.release() + ResourceManager.getInstance().deleteResourceBundle(booking.resource) + + +@shared_task +def query_vpn_users(): + """ get active vpn users """ + JobFactory.makeActiveUsersTask() diff --git a/src/laas_dashboard/settings.py b/src/laas_dashboard/settings.py index 62fc9ec..86778c1 100644 --- a/src/laas_dashboard/settings.py +++ b/src/laas_dashboard/settings.py @@ -28,6 +28,7 @@ INSTALLED_APPS = [ 'notifier', 'workflow', 'api', + 'analytics', 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', @@ -209,6 +210,10 @@ CELERYBEAT_SCHEDULE = { 'task': 'notifier.tasks.notify_expiring', 'schedule': timedelta(hours=1) }, + 'query_vpn_users': { + 'task': 'dashboard.tasks.query_vpn_users', + 'schedule': timedelta(hours=1) + } } # Notifier Settings -- cgit 1.2.3-korg