From 66eb4d851e63d20031502ec0c96aaabe34c6fd32 Mon Sep 17 00:00:00 2001 From: maxbr Date: Fri, 19 Aug 2016 17:11:58 +0200 Subject: Implement periodic tasks JIRA: RELENG-12 The dashboard is now querying jenkins periodically and saving the results in the database. This fixes delays that were caused by calling the jenkins API. Signed-off-by: maxbr --- .../dashboard/fixtures/dashboard.json | 72 ++++++---------------- .../dashboard/migrations/0001_initial.py | 15 ++++- .../migrations/0002_auto_20160815_1511.py | 27 ++++++++ .../migrations/0003_auto_20160815_1512.py | 24 ++++++++ .../dashboard/migrations/0004_resource_slave.py | 23 +++++++ .../migrations/0005_remove_resource_slavename.py | 19 ++++++ tools/pharos-dashboard/dashboard/models.py | 8 ++- .../dashboard/templatetags/__init__.py | 0 .../dashboard/templatetags/jenkins_filters.py | 27 ++++++++ tools/pharos-dashboard/dashboard/views.py | 38 ++++-------- 10 files changed, 169 insertions(+), 84 deletions(-) create mode 100644 tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py create mode 100644 tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py create mode 100644 tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py create mode 100644 tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py create mode 100644 tools/pharos-dashboard/dashboard/templatetags/__init__.py create mode 100644 tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py (limited to 'tools/pharos-dashboard/dashboard') diff --git a/tools/pharos-dashboard/dashboard/fixtures/dashboard.json b/tools/pharos-dashboard/dashboard/fixtures/dashboard.json index f8c1fc13..d90e99b8 100644 --- a/tools/pharos-dashboard/dashboard/fixtures/dashboard.json +++ b/tools/pharos-dashboard/dashboard/fixtures/dashboard.json @@ -6,9 +6,7 @@ "name": "Linux Foundation POD 1", "slavename": "lf-pod1", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab" } }, { @@ -18,9 +16,7 @@ "name": "Linux Foundation POD 2", "slavename": "lf-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab" } }, { @@ -30,9 +26,7 @@ "name": "Ericsson POD 2", "slavename": "ericsson-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process" } }, { @@ -42,9 +36,7 @@ "name": "Intel POD 2", "slavename": "intel-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod2", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod2" } }, { @@ -54,9 +46,7 @@ "name": "Intel POD 5", "slavename": "intel-pod5", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod5", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod5" } }, { @@ -66,9 +56,7 @@ "name": "Intel POD 6", "slavename": "intel-pod6", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod6", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod6" } }, { @@ -78,9 +66,7 @@ "name": "Intel POD 8", "slavename": "intel-pod8", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod8", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod8" } }, { @@ -90,9 +76,7 @@ "name": "Huawei POD 1", "slavename": "huawei-pod1", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" } }, { @@ -102,9 +86,7 @@ "name": "Intel POD 3", "slavename": "intel-pod3", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod3", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod3" } }, { @@ -114,9 +96,7 @@ "name": "Dell POD 1", "slavename": "dell-pod1", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting" } }, { @@ -126,9 +106,7 @@ "name": "Dell POD 2", "slavename": "dell-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting" } }, { @@ -138,9 +116,7 @@ "name": "Orange POD 2", "slavename": "orange-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Opnfv-orange-pod2", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Opnfv-orange-pod2" } }, { @@ -150,9 +126,7 @@ "name": "Arm POD 1", "slavename": "arm-build1", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Enea-pharos-lab", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Enea-pharos-lab" } }, { @@ -162,9 +136,7 @@ "name": "Ericsson POD 1", "slavename": "ericsson-pod1", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process" } }, { @@ -174,9 +146,7 @@ "name": "Huawei POD 2", "slavename": "huawei-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" } }, { @@ -186,9 +156,7 @@ "name": "Huawei POD 3", "slavename": "huawei-pod3", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" } }, { @@ -198,9 +166,7 @@ "name": "Huawei POD 4", "slavename": "huawei-pod4", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" } }, { @@ -210,9 +176,7 @@ "name": "Intel POD 9", "slavename": "intel-pod9", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod9", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod9" } } ] diff --git a/tools/pharos-dashboard/dashboard/migrations/0001_initial.py b/tools/pharos-dashboard/dashboard/migrations/0001_initial.py index b93d8290..6343b463 100644 --- a/tools/pharos-dashboard/dashboard/migrations/0001_initial.py +++ b/tools/pharos-dashboard/dashboard/migrations/0001_initial.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.10 on 2016-08-12 09:51 +# Generated by Django 1.10 on 2016-08-15 12:19 from __future__ import unicode_literals +from django.conf import settings from django.db import migrations, models @@ -10,6 +11,7 @@ class Migration(migrations.Migration): initial = True dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ @@ -21,11 +23,18 @@ class Migration(migrations.Migration): ('slavename', models.CharField(blank=True, max_length=50, null=True)), ('description', models.CharField(blank=True, max_length=300, null=True)), ('url', models.CharField(blank=True, max_length=100, null=True)), - ('bookable', models.BooleanField(default=False)), - ('active', models.BooleanField(default=True)), + ('owners', models.ManyToManyField(to=settings.AUTH_USER_MODEL)), ], options={ 'db_table': 'resource', }, ), + migrations.CreateModel( + name='ResourceUtilization', + fields=[ + ('timestamp', models.DateTimeField(auto_created=True)), + ('id', models.AutoField(primary_key=True, serialize=False)), + ('pod_status', models.IntegerField()), + ], + ), ] diff --git a/tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py b/tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py new file mode 100644 index 00000000..67822a72 --- /dev/null +++ b/tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10 on 2016-08-15 15:11 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jenkins', '0002_auto_20160815_1226'), + ('dashboard', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='resource', + name='slavename', + ), + migrations.AddField( + model_name='resource', + name='slave', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.DO_NOTHING, to='jenkins.JenkinsSlave'), + preserve_default=False, + ), + ] diff --git a/tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py b/tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py new file mode 100644 index 00000000..53b4fcd4 --- /dev/null +++ b/tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10 on 2016-08-15 15:12 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dashboard', '0002_auto_20160815_1511'), + ] + + operations = [ + migrations.RemoveField( + model_name='resource', + name='slave', + ), + migrations.AddField( + model_name='resource', + name='slavename', + field=models.CharField(blank=True, max_length=50, null=True), + ), + ] diff --git a/tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py b/tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py new file mode 100644 index 00000000..82d45f0b --- /dev/null +++ b/tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10 on 2016-08-15 15:13 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jenkins', '0002_auto_20160815_1226'), + ('dashboard', '0003_auto_20160815_1512'), + ] + + operations = [ + migrations.AddField( + model_name='resource', + name='slave', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.DO_NOTHING, to='jenkins.JenkinsSlave'), + preserve_default=False, + ), + ] diff --git a/tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py b/tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py new file mode 100644 index 00000000..339f8c3f --- /dev/null +++ b/tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10 on 2016-08-15 15:17 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('dashboard', '0004_resource_slave'), + ] + + operations = [ + migrations.RemoveField( + model_name='resource', + name='slavename', + ), + ] diff --git a/tools/pharos-dashboard/dashboard/models.py b/tools/pharos-dashboard/dashboard/models.py index 973066b8..cb6b92b3 100644 --- a/tools/pharos-dashboard/dashboard/models.py +++ b/tools/pharos-dashboard/dashboard/models.py @@ -1,14 +1,17 @@ from django.contrib.auth.models import User from django.db import models +from django.utils import timezone + +from jenkins.models import JenkinsSlave class Resource(models.Model): id = models.AutoField(primary_key=True) name = models.CharField(max_length=100, unique=True) - slavename = models.CharField(max_length=50, blank=True, null=True) description = models.CharField(max_length=300, blank=True, null=True) url = models.CharField(max_length=100, blank=True, null=True) owners = models.ManyToManyField(User) + slave = models.ForeignKey(JenkinsSlave, on_delete=models.DO_NOTHING) class Meta: db_table = 'resource' @@ -16,6 +19,7 @@ class Resource(models.Model): def __str__(self): return self.name + class ResourceUtilization(models.Model): POD_STATUS = { 'online': 1, @@ -25,4 +29,4 @@ class ResourceUtilization(models.Model): id = models.AutoField(primary_key=True) timestamp = models.DateTimeField(auto_created=True) - pod_status = models.IntegerField() \ No newline at end of file + pod_status = models.IntegerField() diff --git a/tools/pharos-dashboard/dashboard/templatetags/__init__.py b/tools/pharos-dashboard/dashboard/templatetags/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py b/tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py new file mode 100644 index 00000000..f7e00a87 --- /dev/null +++ b/tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py @@ -0,0 +1,27 @@ +from django.template.defaultfilters import register + + +@register.filter +def jenkins_job_color(job_result): + if job_result == 'SUCCESS': + return '#5cb85c' + if job_result == 'FAILURE': + return '#d9534f' + if job_result == 'UNSTABLE': + return '#EDD62B' + return '#646F73' # job is still building + + +@register.filter +def jenkins_status_color(slave_status): + if slave_status == 'offline': + return '#d9534f' + if slave_status == 'online': + return '#5cb85c' + if slave_status == 'online / idle': + return '#5bc0de' + +@register.filter +def jenkins_job_blink(job_result): + if job_result == '': # job is still building + return 'class=blink_me' \ No newline at end of file diff --git a/tools/pharos-dashboard/dashboard/views.py b/tools/pharos-dashboard/dashboard/views.py index ed62b356..a4c0c4e9 100644 --- a/tools/pharos-dashboard/dashboard/views.py +++ b/tools/pharos-dashboard/dashboard/views.py @@ -1,19 +1,18 @@ +from datetime import timedelta from django.utils import timezone from django.views.generic import TemplateView from booking.models import Booking from dashboard.models import Resource from jenkins import adapter as jenkins +from jenkins.models import JenkinsSlave, JenkinsStatistic class JenkinsSlavesView(TemplateView): template_name = "dashboard/jenkins_slaves.html" def get_context_data(self, **kwargs): - slaves = jenkins.get_all_slaves() - for slave in slaves: - jenkins.parse_slave_data(slave, slave) - + slaves = JenkinsSlave.objects.all() context = super(JenkinsSlavesView, self).get_context_data(**kwargs) context.update({'title': "Jenkins Slaves", 'slaves': slaves}) return context @@ -23,15 +22,7 @@ class CIPodsView(TemplateView): template_name = "dashboard/ci_pods.html" def get_context_data(self, **kwargs): - resources = Resource.objects.filter().values() # get resources as a set of dicts - ci_pods = [] - for resource in resources: - if not jenkins.is_ci_slave(resource['slavename']): - continue - ci_slave = jenkins.get_slave(resource['slavename']) - jenkins.parse_slave_data(resource, ci_slave) - ci_pods.append(resource) - + ci_pods = Resource.objects.filter(slave__ci_slave=True) context = super(CIPodsView, self).get_context_data(**kwargs) context.update({'title': "CI Pods", 'ci_pods': ci_pods}) return context @@ -41,21 +32,18 @@ class DevelopmentPodsView(TemplateView): template_name = "dashboard/dev_pods.html" def get_context_data(self, **kwargs): - resources = Resource.objects.filter().values() # get resources as a set of dicts - dev_pods = [] + resources = Resource.objects.filter(slave__dev_pod=True) - current_bookings = Booking.objects.filter(start__lte=timezone.now()) - current_bookings = current_bookings.filter(end__gt=timezone.now()) + bookings = Booking.objects.filter(start__lte=timezone.now()) + bookings = bookings.filter(end__gt=timezone.now()) + dev_pods = [] for resource in resources: - if not jenkins.is_dev_pod(resource['slavename']): - continue - dev_pod = jenkins.get_slave(resource['slavename']) - jenkins.parse_slave_data(resource, dev_pod) - for booking in current_bookings: - if booking.resource.slavename == resource['slavename']: - resource['current_booking'] = booking - dev_pods.append(resource) + dev_pod = (resource, None) + for booking in bookings: + if booking.resource == resource: + dev_pod = (resource, booking) + dev_pods.append(dev_pod) context = super(DevelopmentPodsView, self).get_context_data(**kwargs) context.update({'title': "Development Pods", 'dev_pods': dev_pods}) -- cgit 1.2.3-korg