From 66eb4d851e63d20031502ec0c96aaabe34c6fd32 Mon Sep 17 00:00:00 2001 From: maxbr Date: Fri, 19 Aug 2016 17:11:58 +0200 Subject: Implement periodic tasks JIRA: RELENG-12 The dashboard is now querying jenkins periodically and saving the results in the database. This fixes delays that were caused by calling the jenkins API. Signed-off-by: maxbr --- tools/pharos-dashboard/account/middleware.py | 11 +++- .../account/migrations/0001_initial.py | 18 ++---- .../booking/migrations/0001_initial.py | 6 +- .../pharos-dashboard/booking/tests/test_models.py | 7 +- tools/pharos-dashboard/booking/tests/test_views.py | 4 +- tools/pharos-dashboard/celerybeat-schedule | Bin 0 -> 16384 bytes .../dashboard/fixtures/dashboard.json | 72 ++++++--------------- .../dashboard/migrations/0001_initial.py | 15 ++++- .../migrations/0002_auto_20160815_1511.py | 27 ++++++++ .../migrations/0003_auto_20160815_1512.py | 24 +++++++ .../dashboard/migrations/0004_resource_slave.py | 23 +++++++ .../migrations/0005_remove_resource_slavename.py | 19 ++++++ tools/pharos-dashboard/dashboard/models.py | 8 ++- .../dashboard/templatetags/__init__.py | 0 .../dashboard/templatetags/jenkins_filters.py | 27 ++++++++ tools/pharos-dashboard/dashboard/views.py | 38 ++++------- tools/pharos-dashboard/jenkins/adapter.py | 44 +++---------- .../jenkins/migrations/0001_initial.py | 52 +++++++++++++++ .../jenkins/migrations/0002_auto_20160815_1226.py | 20 ++++++ .../jenkins/migrations/__init__.py | 0 tools/pharos-dashboard/jenkins/models.py | 34 ++++++++++ tools/pharos-dashboard/jenkins/tasks.py | 39 +++++++++++ .../pharos-dashboard/pharos_dashboard/__init__.py | 3 + tools/pharos-dashboard/pharos_dashboard/celery.py | 20 ++++++ .../pharos-dashboard/pharos_dashboard/settings.py | 15 +++-- .../templates/dashboard/ci_pods.html | 26 ++++---- .../templates/dashboard/dev_pods.html | 19 +++--- .../templates/dashboard/jenkins_slaves.html | 13 ++-- 28 files changed, 410 insertions(+), 174 deletions(-) create mode 100644 tools/pharos-dashboard/celerybeat-schedule create mode 100644 tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py create mode 100644 tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py create mode 100644 tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py create mode 100644 tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py create mode 100644 tools/pharos-dashboard/dashboard/templatetags/__init__.py create mode 100644 tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py create mode 100644 tools/pharos-dashboard/jenkins/migrations/0001_initial.py create mode 100644 tools/pharos-dashboard/jenkins/migrations/0002_auto_20160815_1226.py create mode 100644 tools/pharos-dashboard/jenkins/migrations/__init__.py create mode 100644 tools/pharos-dashboard/jenkins/models.py create mode 100644 tools/pharos-dashboard/jenkins/tasks.py create mode 100644 tools/pharos-dashboard/pharos_dashboard/celery.py diff --git a/tools/pharos-dashboard/account/middleware.py b/tools/pharos-dashboard/account/middleware.py index f5170baa..6f7cac7a 100644 --- a/tools/pharos-dashboard/account/middleware.py +++ b/tools/pharos-dashboard/account/middleware.py @@ -1,7 +1,8 @@ -from django.core.exceptions import ObjectDoesNotExist from django.utils import timezone from django.utils.deprecation import MiddlewareMixin +from account.models import UserProfile + class TimezoneMiddleware(MiddlewareMixin): """ @@ -10,6 +11,12 @@ class TimezoneMiddleware(MiddlewareMixin): """ def process_request(self, request): if request.user.is_authenticated: - timezone.activate(request.user.userprofile.timezone) + try: + tz = request.user.userprofile.timezone + timezone.activate(tz) + except UserProfile.DoesNotExist: + UserProfile.objects.create(user=request.user) + tz = request.user.userprofile.timezone + timezone.activate(tz) else: timezone.deactivate() diff --git a/tools/pharos-dashboard/account/migrations/0001_initial.py b/tools/pharos-dashboard/account/migrations/0001_initial.py index 752d5171..4ff9510a 100644 --- a/tools/pharos-dashboard/account/migrations/0001_initial.py +++ b/tools/pharos-dashboard/account/migrations/0001_initial.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.10 on 2016-08-12 09:51 +# Generated by Django 1.10 on 2016-08-15 12:19 from __future__ import unicode_literals from django.conf import settings @@ -13,7 +13,6 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('dashboard', '0001_initial'), ] operations = [ @@ -21,7 +20,9 @@ class Migration(migrations.Migration): name='UserProfile', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('sshkey', models.CharField(max_length=1024)), + ('timezone', models.CharField(default='UTC', max_length=100)), + ('sshkey', models.CharField(max_length=2048)), + ('pgpkey', models.CharField(max_length=2048)), ('company', models.CharField(max_length=200)), ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], @@ -29,15 +30,4 @@ class Migration(migrations.Migration): 'db_table': 'user_profile', }, ), - migrations.CreateModel( - name='UserResource', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dashboard.Resource')), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), - ], - options={ - 'db_table': 'user_resource', - }, - ), ] diff --git a/tools/pharos-dashboard/booking/migrations/0001_initial.py b/tools/pharos-dashboard/booking/migrations/0001_initial.py index 57735eef..9706b812 100644 --- a/tools/pharos-dashboard/booking/migrations/0001_initial.py +++ b/tools/pharos-dashboard/booking/migrations/0001_initial.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.10 on 2016-08-12 09:51 +# Generated by Django 1.10 on 2016-08-15 12:19 from __future__ import unicode_literals from django.conf import settings @@ -12,8 +12,8 @@ class Migration(migrations.Migration): initial = True dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), ('dashboard', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ @@ -21,10 +21,8 @@ class Migration(migrations.Migration): name='Booking', fields=[ ('id', models.AutoField(primary_key=True, serialize=False)), - ('deleted', models.BooleanField(default=False)), ('start', models.DateTimeField()), ('end', models.DateTimeField()), - ('status', models.CharField(max_length=20)), ('purpose', models.CharField(max_length=300)), ('resource', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='dashboard.Resource')), ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), diff --git a/tools/pharos-dashboard/booking/tests/test_models.py b/tools/pharos-dashboard/booking/tests/test_models.py index e933f6e8..00f6b266 100644 --- a/tools/pharos-dashboard/booking/tests/test_models.py +++ b/tools/pharos-dashboard/booking/tests/test_models.py @@ -6,12 +6,15 @@ from django.utils import timezone from booking.models import Booking from dashboard.models import Resource +from jenkins.models import JenkinsSlave class BookingModelTestCase(TestCase): def setUp(self): - self.res1 = Resource.objects.create(name='res1', slavename='s1', description='x', url='x') - self.res2 = Resource.objects.create(name='res2', slavename='s2', description='x', url='x') + self.slave = JenkinsSlave.objects.create(name='test', url='test') + + self.res1 = Resource.objects.create(name='res1', slave=self.slave, description='x', url='x') + self.res2 = Resource.objects.create(name='res2', slave=self.slave, description='x', url='x') self.user1 = User.objects.create(username='user1') diff --git a/tools/pharos-dashboard/booking/tests/test_views.py b/tools/pharos-dashboard/booking/tests/test_views.py index f5b75d14..4f5ee8bd 100644 --- a/tools/pharos-dashboard/booking/tests/test_views.py +++ b/tools/pharos-dashboard/booking/tests/test_views.py @@ -12,12 +12,14 @@ from registration.forms import User from account.models import UserProfile from booking.models import Booking from dashboard.models import Resource +from jenkins.models import JenkinsSlave class BookingViewTestCase(TestCase): def setUp(self): self.client = Client() - self.res1 = Resource.objects.create(name='res1', slavename='s1', description='x', url='x') + self.slave = JenkinsSlave.objects.create(name='test', url='test') + self.res1 = Resource.objects.create(name='res1', slave=self.slave, description='x', url='x') self.user1 = User.objects.create(username='user1') self.user1.set_password('user1') self.user1profile = UserProfile.objects.create(user=self.user1) diff --git a/tools/pharos-dashboard/celerybeat-schedule b/tools/pharos-dashboard/celerybeat-schedule new file mode 100644 index 00000000..7e5fe853 Binary files /dev/null and b/tools/pharos-dashboard/celerybeat-schedule differ diff --git a/tools/pharos-dashboard/dashboard/fixtures/dashboard.json b/tools/pharos-dashboard/dashboard/fixtures/dashboard.json index f8c1fc13..d90e99b8 100644 --- a/tools/pharos-dashboard/dashboard/fixtures/dashboard.json +++ b/tools/pharos-dashboard/dashboard/fixtures/dashboard.json @@ -6,9 +6,7 @@ "name": "Linux Foundation POD 1", "slavename": "lf-pod1", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab" } }, { @@ -18,9 +16,7 @@ "name": "Linux Foundation POD 2", "slavename": "lf-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab" } }, { @@ -30,9 +26,7 @@ "name": "Ericsson POD 2", "slavename": "ericsson-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process" } }, { @@ -42,9 +36,7 @@ "name": "Intel POD 2", "slavename": "intel-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod2", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod2" } }, { @@ -54,9 +46,7 @@ "name": "Intel POD 5", "slavename": "intel-pod5", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod5", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod5" } }, { @@ -66,9 +56,7 @@ "name": "Intel POD 6", "slavename": "intel-pod6", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod6", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod6" } }, { @@ -78,9 +66,7 @@ "name": "Intel POD 8", "slavename": "intel-pod8", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod8", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod8" } }, { @@ -90,9 +76,7 @@ "name": "Huawei POD 1", "slavename": "huawei-pod1", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting", - "bookable": false, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" } }, { @@ -102,9 +86,7 @@ "name": "Intel POD 3", "slavename": "intel-pod3", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod3", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod3" } }, { @@ -114,9 +96,7 @@ "name": "Dell POD 1", "slavename": "dell-pod1", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting" } }, { @@ -126,9 +106,7 @@ "name": "Dell POD 2", "slavename": "dell-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting" } }, { @@ -138,9 +116,7 @@ "name": "Orange POD 2", "slavename": "orange-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Opnfv-orange-pod2", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Opnfv-orange-pod2" } }, { @@ -150,9 +126,7 @@ "name": "Arm POD 1", "slavename": "arm-build1", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Enea-pharos-lab", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Enea-pharos-lab" } }, { @@ -162,9 +136,7 @@ "name": "Ericsson POD 1", "slavename": "ericsson-pod1", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process" } }, { @@ -174,9 +146,7 @@ "name": "Huawei POD 2", "slavename": "huawei-pod2", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" } }, { @@ -186,9 +156,7 @@ "name": "Huawei POD 3", "slavename": "huawei-pod3", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" } }, { @@ -198,9 +166,7 @@ "name": "Huawei POD 4", "slavename": "huawei-pod4", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting" } }, { @@ -210,9 +176,7 @@ "name": "Intel POD 9", "slavename": "intel-pod9", "description": "Some description", - "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod9", - "bookable": true, - "active": true + "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod9" } } ] diff --git a/tools/pharos-dashboard/dashboard/migrations/0001_initial.py b/tools/pharos-dashboard/dashboard/migrations/0001_initial.py index b93d8290..6343b463 100644 --- a/tools/pharos-dashboard/dashboard/migrations/0001_initial.py +++ b/tools/pharos-dashboard/dashboard/migrations/0001_initial.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.10 on 2016-08-12 09:51 +# Generated by Django 1.10 on 2016-08-15 12:19 from __future__ import unicode_literals +from django.conf import settings from django.db import migrations, models @@ -10,6 +11,7 @@ class Migration(migrations.Migration): initial = True dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ @@ -21,11 +23,18 @@ class Migration(migrations.Migration): ('slavename', models.CharField(blank=True, max_length=50, null=True)), ('description', models.CharField(blank=True, max_length=300, null=True)), ('url', models.CharField(blank=True, max_length=100, null=True)), - ('bookable', models.BooleanField(default=False)), - ('active', models.BooleanField(default=True)), + ('owners', models.ManyToManyField(to=settings.AUTH_USER_MODEL)), ], options={ 'db_table': 'resource', }, ), + migrations.CreateModel( + name='ResourceUtilization', + fields=[ + ('timestamp', models.DateTimeField(auto_created=True)), + ('id', models.AutoField(primary_key=True, serialize=False)), + ('pod_status', models.IntegerField()), + ], + ), ] diff --git a/tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py b/tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py new file mode 100644 index 00000000..67822a72 --- /dev/null +++ b/tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10 on 2016-08-15 15:11 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jenkins', '0002_auto_20160815_1226'), + ('dashboard', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='resource', + name='slavename', + ), + migrations.AddField( + model_name='resource', + name='slave', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.DO_NOTHING, to='jenkins.JenkinsSlave'), + preserve_default=False, + ), + ] diff --git a/tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py b/tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py new file mode 100644 index 00000000..53b4fcd4 --- /dev/null +++ b/tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10 on 2016-08-15 15:12 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dashboard', '0002_auto_20160815_1511'), + ] + + operations = [ + migrations.RemoveField( + model_name='resource', + name='slave', + ), + migrations.AddField( + model_name='resource', + name='slavename', + field=models.CharField(blank=True, max_length=50, null=True), + ), + ] diff --git a/tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py b/tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py new file mode 100644 index 00000000..82d45f0b --- /dev/null +++ b/tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10 on 2016-08-15 15:13 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('jenkins', '0002_auto_20160815_1226'), + ('dashboard', '0003_auto_20160815_1512'), + ] + + operations = [ + migrations.AddField( + model_name='resource', + name='slave', + field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.DO_NOTHING, to='jenkins.JenkinsSlave'), + preserve_default=False, + ), + ] diff --git a/tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py b/tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py new file mode 100644 index 00000000..339f8c3f --- /dev/null +++ b/tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10 on 2016-08-15 15:17 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('dashboard', '0004_resource_slave'), + ] + + operations = [ + migrations.RemoveField( + model_name='resource', + name='slavename', + ), + ] diff --git a/tools/pharos-dashboard/dashboard/models.py b/tools/pharos-dashboard/dashboard/models.py index 973066b8..cb6b92b3 100644 --- a/tools/pharos-dashboard/dashboard/models.py +++ b/tools/pharos-dashboard/dashboard/models.py @@ -1,14 +1,17 @@ from django.contrib.auth.models import User from django.db import models +from django.utils import timezone + +from jenkins.models import JenkinsSlave class Resource(models.Model): id = models.AutoField(primary_key=True) name = models.CharField(max_length=100, unique=True) - slavename = models.CharField(max_length=50, blank=True, null=True) description = models.CharField(max_length=300, blank=True, null=True) url = models.CharField(max_length=100, blank=True, null=True) owners = models.ManyToManyField(User) + slave = models.ForeignKey(JenkinsSlave, on_delete=models.DO_NOTHING) class Meta: db_table = 'resource' @@ -16,6 +19,7 @@ class Resource(models.Model): def __str__(self): return self.name + class ResourceUtilization(models.Model): POD_STATUS = { 'online': 1, @@ -25,4 +29,4 @@ class ResourceUtilization(models.Model): id = models.AutoField(primary_key=True) timestamp = models.DateTimeField(auto_created=True) - pod_status = models.IntegerField() \ No newline at end of file + pod_status = models.IntegerField() diff --git a/tools/pharos-dashboard/dashboard/templatetags/__init__.py b/tools/pharos-dashboard/dashboard/templatetags/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py b/tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py new file mode 100644 index 00000000..f7e00a87 --- /dev/null +++ b/tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py @@ -0,0 +1,27 @@ +from django.template.defaultfilters import register + + +@register.filter +def jenkins_job_color(job_result): + if job_result == 'SUCCESS': + return '#5cb85c' + if job_result == 'FAILURE': + return '#d9534f' + if job_result == 'UNSTABLE': + return '#EDD62B' + return '#646F73' # job is still building + + +@register.filter +def jenkins_status_color(slave_status): + if slave_status == 'offline': + return '#d9534f' + if slave_status == 'online': + return '#5cb85c' + if slave_status == 'online / idle': + return '#5bc0de' + +@register.filter +def jenkins_job_blink(job_result): + if job_result == '': # job is still building + return 'class=blink_me' \ No newline at end of file diff --git a/tools/pharos-dashboard/dashboard/views.py b/tools/pharos-dashboard/dashboard/views.py index ed62b356..a4c0c4e9 100644 --- a/tools/pharos-dashboard/dashboard/views.py +++ b/tools/pharos-dashboard/dashboard/views.py @@ -1,19 +1,18 @@ +from datetime import timedelta from django.utils import timezone from django.views.generic import TemplateView from booking.models import Booking from dashboard.models import Resource from jenkins import adapter as jenkins +from jenkins.models import JenkinsSlave, JenkinsStatistic class JenkinsSlavesView(TemplateView): template_name = "dashboard/jenkins_slaves.html" def get_context_data(self, **kwargs): - slaves = jenkins.get_all_slaves() - for slave in slaves: - jenkins.parse_slave_data(slave, slave) - + slaves = JenkinsSlave.objects.all() context = super(JenkinsSlavesView, self).get_context_data(**kwargs) context.update({'title': "Jenkins Slaves", 'slaves': slaves}) return context @@ -23,15 +22,7 @@ class CIPodsView(TemplateView): template_name = "dashboard/ci_pods.html" def get_context_data(self, **kwargs): - resources = Resource.objects.filter().values() # get resources as a set of dicts - ci_pods = [] - for resource in resources: - if not jenkins.is_ci_slave(resource['slavename']): - continue - ci_slave = jenkins.get_slave(resource['slavename']) - jenkins.parse_slave_data(resource, ci_slave) - ci_pods.append(resource) - + ci_pods = Resource.objects.filter(slave__ci_slave=True) context = super(CIPodsView, self).get_context_data(**kwargs) context.update({'title': "CI Pods", 'ci_pods': ci_pods}) return context @@ -41,21 +32,18 @@ class DevelopmentPodsView(TemplateView): template_name = "dashboard/dev_pods.html" def get_context_data(self, **kwargs): - resources = Resource.objects.filter().values() # get resources as a set of dicts - dev_pods = [] + resources = Resource.objects.filter(slave__dev_pod=True) - current_bookings = Booking.objects.filter(start__lte=timezone.now()) - current_bookings = current_bookings.filter(end__gt=timezone.now()) + bookings = Booking.objects.filter(start__lte=timezone.now()) + bookings = bookings.filter(end__gt=timezone.now()) + dev_pods = [] for resource in resources: - if not jenkins.is_dev_pod(resource['slavename']): - continue - dev_pod = jenkins.get_slave(resource['slavename']) - jenkins.parse_slave_data(resource, dev_pod) - for booking in current_bookings: - if booking.resource.slavename == resource['slavename']: - resource['current_booking'] = booking - dev_pods.append(resource) + dev_pod = (resource, None) + for booking in bookings: + if booking.resource == resource: + dev_pod = (resource, booking) + dev_pods.append(dev_pod) context = super(DevelopmentPodsView, self).get_context_data(**kwargs) context.update({'title': "Development Pods", 'dev_pods': dev_pods}) diff --git a/tools/pharos-dashboard/jenkins/adapter.py b/tools/pharos-dashboard/jenkins/adapter.py index 06233af8..fabd5356 100644 --- a/tools/pharos-dashboard/jenkins/adapter.py +++ b/tools/pharos-dashboard/jenkins/adapter.py @@ -14,7 +14,7 @@ def get_json(url): response = requests.get(url) json = response.json() cache.set(url, json, 180) # cache result for 180 seconds - return response.json() + return json except requests.exceptions.RequestException as e: logger.exception(e) except ValueError as e: @@ -85,26 +85,22 @@ def is_dev_pod(slavename): return True return False -def parse_slave_data(slave_dict, slave): - slave_dict['status'] = get_slave_status(slave) - slave_dict['status_color'] = get_status_color(slave) - slave_dict['slaveurl'] = get_slave_url(slave) - job = get_jenkins_job(slave['displayName']) - if job is not None: - slave_dict['last_job'] = parse_job(job) - def parse_job(job): result = parse_job_string(job['lastBuild']['fullDisplayName']) + result['building'] = job['lastBuild']['building'] + result['result'] = '' + if not job['lastBuild']['building']: + result['result'] = job['lastBuild']['result'] result['url'] = job['url'] - result['color'] = get_job_color(job) - if job['lastBuild']['building']: - result['blink'] = 'class=blink_me' return result def parse_job_string(full_displayname): job = {} + job['scenario'] = '' + job['installer'] = '' + job['branch'] = '' tokens = re.split(r'[ -]', full_displayname) for i in range(len(tokens)): if tokens[i] == 'os': @@ -113,34 +109,10 @@ def parse_job_string(full_displayname): job['installer'] = tokens[i] elif tokens[i] in ['master', 'arno', 'brahmaputra', 'colorado']: job['branch'] = tokens[i] - tokens = full_displayname.split(' ') job['name'] = tokens[0] return job - -# TODO: use css -def get_job_color(job): - if job['lastBuild']['building'] is True: - return '#646F73' - result = job['lastBuild']['result'] - if result == 'SUCCESS': - return '#5cb85c' - if result == 'FAILURE': - return '#d9534f' - if result == 'UNSTABLE': - return '#EDD62B' - - -# TODO: use css -def get_status_color(slave): - if not slave['offline'] and slave['idle']: - return '#5bc0de' - if not slave['offline']: - return '#5cb85c' - return '#d9534f' - - def get_slave_url(slave): return 'https://build.opnfv.org/ci/computer/' + slave['displayName'] diff --git a/tools/pharos-dashboard/jenkins/migrations/0001_initial.py b/tools/pharos-dashboard/jenkins/migrations/0001_initial.py new file mode 100644 index 00000000..a9bb8d56 --- /dev/null +++ b/tools/pharos-dashboard/jenkins/migrations/0001_initial.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10 on 2016-08-15 12:19 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='JenkinsSlave', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('name', models.CharField(max_length=100, unique=True)), + ('status', models.CharField(default='offline', max_length=30)), + ('url', models.CharField(max_length=1024)), + ('ci_slave', models.BooleanField(default=False)), + ('dev_pod', models.BooleanField(default=False)), + ('building', models.BooleanField(default=False)), + ('last_job_name', models.CharField(default='', max_length=1024)), + ('last_job_url', models.CharField(default='', max_length=1024)), + ('last_job_scenario', models.CharField(default='', max_length=50)), + ('last_job_branch', models.CharField(default='', max_length=50)), + ('last_job_installer', models.CharField(default='', max_length=50)), + ('last_job_result', models.CharField(default='', max_length=30)), + ], + options={ + 'db_table': 'jenkins_slave', + }, + ), + migrations.CreateModel( + name='JenkinsStatistic', + fields=[ + ('timestamp', models.DateTimeField(auto_created=True)), + ('id', models.AutoField(primary_key=True, serialize=False)), + ('offline', models.BooleanField(default=False)), + ('idle', models.BooleanField(default=False)), + ('online', models.BooleanField(default=False)), + ('slave', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jenkins.JenkinsSlave')), + ], + options={ + 'db_table': 'jenkins_statistic', + }, + ), + ] diff --git a/tools/pharos-dashboard/jenkins/migrations/0002_auto_20160815_1226.py b/tools/pharos-dashboard/jenkins/migrations/0002_auto_20160815_1226.py new file mode 100644 index 00000000..f1cf7f99 --- /dev/null +++ b/tools/pharos-dashboard/jenkins/migrations/0002_auto_20160815_1226.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10 on 2016-08-15 12:26 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('jenkins', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='jenkinsstatistic', + name='timestamp', + field=models.DateTimeField(auto_now_add=True), + ), + ] diff --git a/tools/pharos-dashboard/jenkins/migrations/__init__.py b/tools/pharos-dashboard/jenkins/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/pharos-dashboard/jenkins/models.py b/tools/pharos-dashboard/jenkins/models.py new file mode 100644 index 00000000..f7d54c96 --- /dev/null +++ b/tools/pharos-dashboard/jenkins/models.py @@ -0,0 +1,34 @@ +from django.db import models + + +class JenkinsSlave(models.Model): + id = models.AutoField(primary_key=True) + name = models.CharField(max_length=100, unique=True) + status = models.CharField(max_length=30, default='offline') + url = models.CharField(max_length=1024) + ci_slave = models.BooleanField(default=False) + dev_pod = models.BooleanField(default=False) + + building = models.BooleanField(default=False) + + last_job_name = models.CharField(max_length=1024, default='') + last_job_url = models.CharField(max_length=1024, default='') + last_job_scenario = models.CharField(max_length=50, default='') + last_job_branch = models.CharField(max_length=50, default='') + last_job_installer = models.CharField(max_length=50, default='') + last_job_result = models.CharField(max_length=30, default='') + + class Meta: + db_table = 'jenkins_slave' + + +class JenkinsStatistic(models.Model): + id = models.AutoField(primary_key=True) + slave = models.ForeignKey(JenkinsSlave, on_delete=models.CASCADE) + offline = models.BooleanField(default=False) + idle = models.BooleanField(default=False) + online = models.BooleanField(default=False) + timestamp = models.DateTimeField(auto_now_add=True) + + class Meta: + db_table = 'jenkins_statistic' diff --git a/tools/pharos-dashboard/jenkins/tasks.py b/tools/pharos-dashboard/jenkins/tasks.py new file mode 100644 index 00000000..6998cf3b --- /dev/null +++ b/tools/pharos-dashboard/jenkins/tasks.py @@ -0,0 +1,39 @@ +from celery import shared_task + +from jenkins.models import JenkinsSlave, JenkinsStatistic +from .adapter import * + + +@shared_task +def sync_jenkins(): + update_jenkins_slaves() + + +def update_jenkins_slaves(): + jenkins_slaves = get_all_slaves() + for slave in jenkins_slaves: + jenkins_slave, created = JenkinsSlave.objects.get_or_create(name=slave['displayName'], + url=get_slave_url(slave)) + jenkins_slave.ci_slave = is_ci_slave(slave['displayName']) + jenkins_slave.dev_pod = is_dev_pod(slave['displayName']) + jenkins_slave.status = get_slave_status(slave) + + last_job = get_jenkins_job(jenkins_slave.name) + if last_job is not None: + last_job = parse_job(last_job) + jenkins_slave.last_job_name = last_job['name'] + jenkins_slave.last_job_url = last_job['url'] + jenkins_slave.last_job_scenario = last_job['scenario'] + jenkins_slave.last_job_branch = last_job['branch'] + jenkins_slave.last_job_installer = last_job['installer'] + jenkins_slave.last_job_result = last_job['result'] + jenkins_slave.save() + + jenkins_statistic = JenkinsStatistic(slave=jenkins_slave) + if jenkins_slave.status == 'online' or jenkins_slave.status == 'building': + jenkins_statistic.online = True + if jenkins_slave.status == 'offline': + jenkins_statistic.offline = True + if jenkins_slave.status == 'online / idle': + jenkins_statistic.idle = True + jenkins_statistic.save() diff --git a/tools/pharos-dashboard/pharos_dashboard/__init__.py b/tools/pharos-dashboard/pharos_dashboard/__init__.py index e69de29b..b6fc8176 100644 --- a/tools/pharos-dashboard/pharos_dashboard/__init__.py +++ b/tools/pharos-dashboard/pharos_dashboard/__init__.py @@ -0,0 +1,3 @@ +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app # noqa diff --git a/tools/pharos-dashboard/pharos_dashboard/celery.py b/tools/pharos-dashboard/pharos_dashboard/celery.py new file mode 100644 index 00000000..4cf6a7af --- /dev/null +++ b/tools/pharos-dashboard/pharos_dashboard/celery.py @@ -0,0 +1,20 @@ +import os + +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pharos_dashboard.settings') + +from django.conf import settings # noqa + +app = Celery('pharos_dashboard') + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) + + +@app.task(bind=True) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) \ No newline at end of file diff --git a/tools/pharos-dashboard/pharos_dashboard/settings.py b/tools/pharos-dashboard/pharos_dashboard/settings.py index b6e98991..77175016 100644 --- a/tools/pharos-dashboard/pharos_dashboard/settings.py +++ b/tools/pharos-dashboard/pharos_dashboard/settings.py @@ -15,7 +15,6 @@ import os # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ @@ -27,7 +26,6 @@ DEBUG = True ALLOWED_HOSTS = [] - # Application definition INSTALLED_APPS = [ @@ -43,6 +41,8 @@ INSTALLED_APPS = [ 'django.contrib.staticfiles', 'django.contrib.humanize', 'bootstrap3', + 'djcelery', + 'kombu.transport.django', ] MIDDLEWARE = [ @@ -77,7 +77,6 @@ TEMPLATES = [ WSGI_APPLICATION = 'pharos_dashboard.wsgi.application' - # Database # https://docs.djangoproject.com/en/1.10/ref/settings/#databases @@ -92,7 +91,6 @@ DATABASES = { } } - # Password validation # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators @@ -124,7 +122,6 @@ USE_L10N = True USE_TZ = True - # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.10/howto/static-files/ @@ -139,3 +136,11 @@ BOOTSTRAP3 = { } LOGIN_REDIRECT_URL = '/' + +import djcelery + +djcelery.setup_loader() +# django broker, NOT SAFE FOR PRODUCTION +BROKER_URL = 'django://' +CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler' + diff --git a/tools/pharos-dashboard/templates/dashboard/ci_pods.html b/tools/pharos-dashboard/templates/dashboard/ci_pods.html index d3e5ff62..2982a6ff 100644 --- a/tools/pharos-dashboard/templates/dashboard/ci_pods.html +++ b/tools/pharos-dashboard/templates/dashboard/ci_pods.html @@ -1,5 +1,6 @@ {% extends "dashboard/table.html" %} {% load staticfiles %} +{% load jenkins_filters %} {% block table %} @@ -20,23 +21,24 @@ {{ pod.name }} - {{ pod.slavename }} + {{ pod.slave.name }} - - {{ pod.status }} + + {{ pod.slave.status }} - - {{ pod.last_job.installer }} + + {{ pod.slave.last_job_installer }} - - {{ pod.last_job.scenario }} + + {{ pod.slave.last_job_scenario }} - - {{ pod.last_job.branch }} + + {{ pod.slave.last_job_branch }} - {{ pod.last_job.name }} + {{ pod.slave.last_job_name }} {% endfor %}` diff --git a/tools/pharos-dashboard/templates/dashboard/dev_pods.html b/tools/pharos-dashboard/templates/dashboard/dev_pods.html index f08e1d1f..532a3a11 100644 --- a/tools/pharos-dashboard/templates/dashboard/dev_pods.html +++ b/tools/pharos-dashboard/templates/dashboard/dev_pods.html @@ -1,5 +1,6 @@ {% extends "dashboard/table.html" %} {% load staticfiles %} +{% load jenkins_filters %} {% block table %} @@ -14,28 +15,28 @@ - {% for resource in dev_pods %} + {% for pod, booking in dev_pods %} - {{ resource.name }} + {{ pod.name }} - {{ resource.slavename }} + {{ pod.slave.name }} - {{ resource.current_booking.user.username }} + {{ booking.user.username }} - {{ resource.current_booking.end }} + {{ booking.end }} - {{ resource.current_booking.purpose }} + {{ booking.purpose }} - - {{ resource.status }} + + {{ pod.slave.status }} - + Book diff --git a/tools/pharos-dashboard/templates/dashboard/jenkins_slaves.html b/tools/pharos-dashboard/templates/dashboard/jenkins_slaves.html index 2d011b46..830ed198 100644 --- a/tools/pharos-dashboard/templates/dashboard/jenkins_slaves.html +++ b/tools/pharos-dashboard/templates/dashboard/jenkins_slaves.html @@ -1,6 +1,8 @@ {% extends "dashboard/table.html" %} {% load staticfiles %} +{% load jenkins_filters %} + {% block table %} @@ -13,14 +15,15 @@ {% for slave in slaves %} {{ slave.displayName }} + href={{ slave.url }}>{{ slave.name }} - + {{ slave.status }} - - {{ slave.last_job.name }} + + {{ slave.last_job_name }} {% endfor %} -- cgit 1.2.3-korg