summaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
authormaxbr <maxbr@mi.fu-berlin.de>2016-08-19 17:11:58 +0200
committermaxbr <maxbr@mi.fu-berlin.de>2016-08-19 17:11:58 +0200
commit66eb4d851e63d20031502ec0c96aaabe34c6fd32 (patch)
tree98248b6faf6b3c742efef258e34f06cc92d1a888 /tools
parent3b5ef3b0a88247eeafeee878de528aad71f9fd4b (diff)
Implement periodic tasks
JIRA: RELENG-12 The dashboard is now querying jenkins periodically and saving the results in the database. This fixes delays that were caused by calling the jenkins API. Signed-off-by: maxbr <maxbr@mi.fu-berlin.de>
Diffstat (limited to 'tools')
-rw-r--r--tools/pharos-dashboard/account/middleware.py11
-rw-r--r--tools/pharos-dashboard/account/migrations/0001_initial.py18
-rw-r--r--tools/pharos-dashboard/booking/migrations/0001_initial.py6
-rw-r--r--tools/pharos-dashboard/booking/tests/test_models.py7
-rw-r--r--tools/pharos-dashboard/booking/tests/test_views.py4
-rw-r--r--tools/pharos-dashboard/celerybeat-schedulebin0 -> 16384 bytes
-rw-r--r--tools/pharos-dashboard/dashboard/fixtures/dashboard.json72
-rw-r--r--tools/pharos-dashboard/dashboard/migrations/0001_initial.py15
-rw-r--r--tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py27
-rw-r--r--tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py24
-rw-r--r--tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py23
-rw-r--r--tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py19
-rw-r--r--tools/pharos-dashboard/dashboard/models.py8
-rw-r--r--tools/pharos-dashboard/dashboard/templatetags/__init__.py0
-rw-r--r--tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py27
-rw-r--r--tools/pharos-dashboard/dashboard/views.py38
-rw-r--r--tools/pharos-dashboard/jenkins/adapter.py44
-rw-r--r--tools/pharos-dashboard/jenkins/migrations/0001_initial.py52
-rw-r--r--tools/pharos-dashboard/jenkins/migrations/0002_auto_20160815_1226.py20
-rw-r--r--tools/pharos-dashboard/jenkins/migrations/__init__.py0
-rw-r--r--tools/pharos-dashboard/jenkins/models.py34
-rw-r--r--tools/pharos-dashboard/jenkins/tasks.py39
-rw-r--r--tools/pharos-dashboard/pharos_dashboard/__init__.py3
-rw-r--r--tools/pharos-dashboard/pharos_dashboard/celery.py20
-rw-r--r--tools/pharos-dashboard/pharos_dashboard/settings.py15
-rw-r--r--tools/pharos-dashboard/templates/dashboard/ci_pods.html26
-rw-r--r--tools/pharos-dashboard/templates/dashboard/dev_pods.html19
-rw-r--r--tools/pharos-dashboard/templates/dashboard/jenkins_slaves.html13
28 files changed, 410 insertions, 174 deletions
diff --git a/tools/pharos-dashboard/account/middleware.py b/tools/pharos-dashboard/account/middleware.py
index f5170baa..6f7cac7a 100644
--- a/tools/pharos-dashboard/account/middleware.py
+++ b/tools/pharos-dashboard/account/middleware.py
@@ -1,7 +1,8 @@
-from django.core.exceptions import ObjectDoesNotExist
from django.utils import timezone
from django.utils.deprecation import MiddlewareMixin
+from account.models import UserProfile
+
class TimezoneMiddleware(MiddlewareMixin):
"""
@@ -10,6 +11,12 @@ class TimezoneMiddleware(MiddlewareMixin):
"""
def process_request(self, request):
if request.user.is_authenticated:
- timezone.activate(request.user.userprofile.timezone)
+ try:
+ tz = request.user.userprofile.timezone
+ timezone.activate(tz)
+ except UserProfile.DoesNotExist:
+ UserProfile.objects.create(user=request.user)
+ tz = request.user.userprofile.timezone
+ timezone.activate(tz)
else:
timezone.deactivate()
diff --git a/tools/pharos-dashboard/account/migrations/0001_initial.py b/tools/pharos-dashboard/account/migrations/0001_initial.py
index 752d5171..4ff9510a 100644
--- a/tools/pharos-dashboard/account/migrations/0001_initial.py
+++ b/tools/pharos-dashboard/account/migrations/0001_initial.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Generated by Django 1.10 on 2016-08-12 09:51
+# Generated by Django 1.10 on 2016-08-15 12:19
from __future__ import unicode_literals
from django.conf import settings
@@ -13,7 +13,6 @@ class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
- ('dashboard', '0001_initial'),
]
operations = [
@@ -21,7 +20,9 @@ class Migration(migrations.Migration):
name='UserProfile',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('sshkey', models.CharField(max_length=1024)),
+ ('timezone', models.CharField(default='UTC', max_length=100)),
+ ('sshkey', models.CharField(max_length=2048)),
+ ('pgpkey', models.CharField(max_length=2048)),
('company', models.CharField(max_length=200)),
('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
@@ -29,15 +30,4 @@ class Migration(migrations.Migration):
'db_table': 'user_profile',
},
),
- migrations.CreateModel(
- name='UserResource',
- fields=[
- ('id', models.AutoField(primary_key=True, serialize=False)),
- ('resource', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='dashboard.Resource')),
- ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
- ],
- options={
- 'db_table': 'user_resource',
- },
- ),
]
diff --git a/tools/pharos-dashboard/booking/migrations/0001_initial.py b/tools/pharos-dashboard/booking/migrations/0001_initial.py
index 57735eef..9706b812 100644
--- a/tools/pharos-dashboard/booking/migrations/0001_initial.py
+++ b/tools/pharos-dashboard/booking/migrations/0001_initial.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-# Generated by Django 1.10 on 2016-08-12 09:51
+# Generated by Django 1.10 on 2016-08-15 12:19
from __future__ import unicode_literals
from django.conf import settings
@@ -12,8 +12,8 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
- migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('dashboard', '0001_initial'),
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
@@ -21,10 +21,8 @@ class Migration(migrations.Migration):
name='Booking',
fields=[
('id', models.AutoField(primary_key=True, serialize=False)),
- ('deleted', models.BooleanField(default=False)),
('start', models.DateTimeField()),
('end', models.DateTimeField()),
- ('status', models.CharField(max_length=20)),
('purpose', models.CharField(max_length=300)),
('resource', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='dashboard.Resource')),
('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
diff --git a/tools/pharos-dashboard/booking/tests/test_models.py b/tools/pharos-dashboard/booking/tests/test_models.py
index e933f6e8..00f6b266 100644
--- a/tools/pharos-dashboard/booking/tests/test_models.py
+++ b/tools/pharos-dashboard/booking/tests/test_models.py
@@ -6,12 +6,15 @@ from django.utils import timezone
from booking.models import Booking
from dashboard.models import Resource
+from jenkins.models import JenkinsSlave
class BookingModelTestCase(TestCase):
def setUp(self):
- self.res1 = Resource.objects.create(name='res1', slavename='s1', description='x', url='x')
- self.res2 = Resource.objects.create(name='res2', slavename='s2', description='x', url='x')
+ self.slave = JenkinsSlave.objects.create(name='test', url='test')
+
+ self.res1 = Resource.objects.create(name='res1', slave=self.slave, description='x', url='x')
+ self.res2 = Resource.objects.create(name='res2', slave=self.slave, description='x', url='x')
self.user1 = User.objects.create(username='user1')
diff --git a/tools/pharos-dashboard/booking/tests/test_views.py b/tools/pharos-dashboard/booking/tests/test_views.py
index f5b75d14..4f5ee8bd 100644
--- a/tools/pharos-dashboard/booking/tests/test_views.py
+++ b/tools/pharos-dashboard/booking/tests/test_views.py
@@ -12,12 +12,14 @@ from registration.forms import User
from account.models import UserProfile
from booking.models import Booking
from dashboard.models import Resource
+from jenkins.models import JenkinsSlave
class BookingViewTestCase(TestCase):
def setUp(self):
self.client = Client()
- self.res1 = Resource.objects.create(name='res1', slavename='s1', description='x', url='x')
+ self.slave = JenkinsSlave.objects.create(name='test', url='test')
+ self.res1 = Resource.objects.create(name='res1', slave=self.slave, description='x', url='x')
self.user1 = User.objects.create(username='user1')
self.user1.set_password('user1')
self.user1profile = UserProfile.objects.create(user=self.user1)
diff --git a/tools/pharos-dashboard/celerybeat-schedule b/tools/pharos-dashboard/celerybeat-schedule
new file mode 100644
index 00000000..7e5fe853
--- /dev/null
+++ b/tools/pharos-dashboard/celerybeat-schedule
Binary files differ
diff --git a/tools/pharos-dashboard/dashboard/fixtures/dashboard.json b/tools/pharos-dashboard/dashboard/fixtures/dashboard.json
index f8c1fc13..d90e99b8 100644
--- a/tools/pharos-dashboard/dashboard/fixtures/dashboard.json
+++ b/tools/pharos-dashboard/dashboard/fixtures/dashboard.json
@@ -6,9 +6,7 @@
"name": "Linux Foundation POD 1",
"slavename": "lf-pod1",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab",
- "bookable": false,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab"
}
},
{
@@ -18,9 +16,7 @@
"name": "Linux Foundation POD 2",
"slavename": "lf-pod2",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab",
- "bookable": false,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Lf+Lab"
}
},
{
@@ -30,9 +26,7 @@
"name": "Ericsson POD 2",
"slavename": "ericsson-pod2",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process",
- "bookable": false,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process"
}
},
{
@@ -42,9 +36,7 @@
"name": "Intel POD 2",
"slavename": "intel-pod2",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod2",
- "bookable": false,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod2"
}
},
{
@@ -54,9 +46,7 @@
"name": "Intel POD 5",
"slavename": "intel-pod5",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod5",
- "bookable": false,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod5"
}
},
{
@@ -66,9 +56,7 @@
"name": "Intel POD 6",
"slavename": "intel-pod6",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod6",
- "bookable": false,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod6"
}
},
{
@@ -78,9 +66,7 @@
"name": "Intel POD 8",
"slavename": "intel-pod8",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod8",
- "bookable": false,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod8"
}
},
{
@@ -90,9 +76,7 @@
"name": "Huawei POD 1",
"slavename": "huawei-pod1",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting",
- "bookable": false,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting"
}
},
{
@@ -102,9 +86,7 @@
"name": "Intel POD 3",
"slavename": "intel-pod3",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod3",
- "bookable": true,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod3"
}
},
{
@@ -114,9 +96,7 @@
"name": "Dell POD 1",
"slavename": "dell-pod1",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting",
- "bookable": true,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting"
}
},
{
@@ -126,9 +106,7 @@
"name": "Dell POD 2",
"slavename": "dell-pod2",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting",
- "bookable": true,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Dell+Hosting"
}
},
{
@@ -138,9 +116,7 @@
"name": "Orange POD 2",
"slavename": "orange-pod2",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Opnfv-orange-pod2",
- "bookable": true,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Opnfv-orange-pod2"
}
},
{
@@ -150,9 +126,7 @@
"name": "Arm POD 1",
"slavename": "arm-build1",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Enea-pharos-lab",
- "bookable": true,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Enea-pharos-lab"
}
},
{
@@ -162,9 +136,7 @@
"name": "Ericsson POD 1",
"slavename": "ericsson-pod1",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process",
- "bookable": true,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Ericsson+Hosting+and+Request+Process"
}
},
{
@@ -174,9 +146,7 @@
"name": "Huawei POD 2",
"slavename": "huawei-pod2",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting",
- "bookable": true,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting"
}
},
{
@@ -186,9 +156,7 @@
"name": "Huawei POD 3",
"slavename": "huawei-pod3",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting",
- "bookable": true,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting"
}
},
{
@@ -198,9 +166,7 @@
"name": "Huawei POD 4",
"slavename": "huawei-pod4",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting",
- "bookable": true,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Huawei+Hosting"
}
},
{
@@ -210,9 +176,7 @@
"name": "Intel POD 9",
"slavename": "intel-pod9",
"description": "Some description",
- "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod9",
- "bookable": true,
- "active": true
+ "url": "https://wiki.opnfv.org/display/pharos/Intel+Pod9"
}
}
]
diff --git a/tools/pharos-dashboard/dashboard/migrations/0001_initial.py b/tools/pharos-dashboard/dashboard/migrations/0001_initial.py
index b93d8290..6343b463 100644
--- a/tools/pharos-dashboard/dashboard/migrations/0001_initial.py
+++ b/tools/pharos-dashboard/dashboard/migrations/0001_initial.py
@@ -1,7 +1,8 @@
# -*- coding: utf-8 -*-
-# Generated by Django 1.10 on 2016-08-12 09:51
+# Generated by Django 1.10 on 2016-08-15 12:19
from __future__ import unicode_literals
+from django.conf import settings
from django.db import migrations, models
@@ -10,6 +11,7 @@ class Migration(migrations.Migration):
initial = True
dependencies = [
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
@@ -21,11 +23,18 @@ class Migration(migrations.Migration):
('slavename', models.CharField(blank=True, max_length=50, null=True)),
('description', models.CharField(blank=True, max_length=300, null=True)),
('url', models.CharField(blank=True, max_length=100, null=True)),
- ('bookable', models.BooleanField(default=False)),
- ('active', models.BooleanField(default=True)),
+ ('owners', models.ManyToManyField(to=settings.AUTH_USER_MODEL)),
],
options={
'db_table': 'resource',
},
),
+ migrations.CreateModel(
+ name='ResourceUtilization',
+ fields=[
+ ('timestamp', models.DateTimeField(auto_created=True)),
+ ('id', models.AutoField(primary_key=True, serialize=False)),
+ ('pod_status', models.IntegerField()),
+ ],
+ ),
]
diff --git a/tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py b/tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py
new file mode 100644
index 00000000..67822a72
--- /dev/null
+++ b/tools/pharos-dashboard/dashboard/migrations/0002_auto_20160815_1511.py
@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10 on 2016-08-15 15:11
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('jenkins', '0002_auto_20160815_1226'),
+ ('dashboard', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='resource',
+ name='slavename',
+ ),
+ migrations.AddField(
+ model_name='resource',
+ name='slave',
+ field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.DO_NOTHING, to='jenkins.JenkinsSlave'),
+ preserve_default=False,
+ ),
+ ]
diff --git a/tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py b/tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py
new file mode 100644
index 00000000..53b4fcd4
--- /dev/null
+++ b/tools/pharos-dashboard/dashboard/migrations/0003_auto_20160815_1512.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10 on 2016-08-15 15:12
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('dashboard', '0002_auto_20160815_1511'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='resource',
+ name='slave',
+ ),
+ migrations.AddField(
+ model_name='resource',
+ name='slavename',
+ field=models.CharField(blank=True, max_length=50, null=True),
+ ),
+ ]
diff --git a/tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py b/tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py
new file mode 100644
index 00000000..82d45f0b
--- /dev/null
+++ b/tools/pharos-dashboard/dashboard/migrations/0004_resource_slave.py
@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10 on 2016-08-15 15:13
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('jenkins', '0002_auto_20160815_1226'),
+ ('dashboard', '0003_auto_20160815_1512'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='resource',
+ name='slave',
+ field=models.ForeignKey(default=1, on_delete=django.db.models.deletion.DO_NOTHING, to='jenkins.JenkinsSlave'),
+ preserve_default=False,
+ ),
+ ]
diff --git a/tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py b/tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py
new file mode 100644
index 00000000..339f8c3f
--- /dev/null
+++ b/tools/pharos-dashboard/dashboard/migrations/0005_remove_resource_slavename.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10 on 2016-08-15 15:17
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('dashboard', '0004_resource_slave'),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='resource',
+ name='slavename',
+ ),
+ ]
diff --git a/tools/pharos-dashboard/dashboard/models.py b/tools/pharos-dashboard/dashboard/models.py
index 973066b8..cb6b92b3 100644
--- a/tools/pharos-dashboard/dashboard/models.py
+++ b/tools/pharos-dashboard/dashboard/models.py
@@ -1,14 +1,17 @@
from django.contrib.auth.models import User
from django.db import models
+from django.utils import timezone
+
+from jenkins.models import JenkinsSlave
class Resource(models.Model):
id = models.AutoField(primary_key=True)
name = models.CharField(max_length=100, unique=True)
- slavename = models.CharField(max_length=50, blank=True, null=True)
description = models.CharField(max_length=300, blank=True, null=True)
url = models.CharField(max_length=100, blank=True, null=True)
owners = models.ManyToManyField(User)
+ slave = models.ForeignKey(JenkinsSlave, on_delete=models.DO_NOTHING)
class Meta:
db_table = 'resource'
@@ -16,6 +19,7 @@ class Resource(models.Model):
def __str__(self):
return self.name
+
class ResourceUtilization(models.Model):
POD_STATUS = {
'online': 1,
@@ -25,4 +29,4 @@ class ResourceUtilization(models.Model):
id = models.AutoField(primary_key=True)
timestamp = models.DateTimeField(auto_created=True)
- pod_status = models.IntegerField() \ No newline at end of file
+ pod_status = models.IntegerField()
diff --git a/tools/pharos-dashboard/dashboard/templatetags/__init__.py b/tools/pharos-dashboard/dashboard/templatetags/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tools/pharos-dashboard/dashboard/templatetags/__init__.py
diff --git a/tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py b/tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py
new file mode 100644
index 00000000..f7e00a87
--- /dev/null
+++ b/tools/pharos-dashboard/dashboard/templatetags/jenkins_filters.py
@@ -0,0 +1,27 @@
+from django.template.defaultfilters import register
+
+
+@register.filter
+def jenkins_job_color(job_result):
+ if job_result == 'SUCCESS':
+ return '#5cb85c'
+ if job_result == 'FAILURE':
+ return '#d9534f'
+ if job_result == 'UNSTABLE':
+ return '#EDD62B'
+ return '#646F73' # job is still building
+
+
+@register.filter
+def jenkins_status_color(slave_status):
+ if slave_status == 'offline':
+ return '#d9534f'
+ if slave_status == 'online':
+ return '#5cb85c'
+ if slave_status == 'online / idle':
+ return '#5bc0de'
+
+@register.filter
+def jenkins_job_blink(job_result):
+ if job_result == '': # job is still building
+ return 'class=blink_me' \ No newline at end of file
diff --git a/tools/pharos-dashboard/dashboard/views.py b/tools/pharos-dashboard/dashboard/views.py
index ed62b356..a4c0c4e9 100644
--- a/tools/pharos-dashboard/dashboard/views.py
+++ b/tools/pharos-dashboard/dashboard/views.py
@@ -1,19 +1,18 @@
+from datetime import timedelta
from django.utils import timezone
from django.views.generic import TemplateView
from booking.models import Booking
from dashboard.models import Resource
from jenkins import adapter as jenkins
+from jenkins.models import JenkinsSlave, JenkinsStatistic
class JenkinsSlavesView(TemplateView):
template_name = "dashboard/jenkins_slaves.html"
def get_context_data(self, **kwargs):
- slaves = jenkins.get_all_slaves()
- for slave in slaves:
- jenkins.parse_slave_data(slave, slave)
-
+ slaves = JenkinsSlave.objects.all()
context = super(JenkinsSlavesView, self).get_context_data(**kwargs)
context.update({'title': "Jenkins Slaves", 'slaves': slaves})
return context
@@ -23,15 +22,7 @@ class CIPodsView(TemplateView):
template_name = "dashboard/ci_pods.html"
def get_context_data(self, **kwargs):
- resources = Resource.objects.filter().values() # get resources as a set of dicts
- ci_pods = []
- for resource in resources:
- if not jenkins.is_ci_slave(resource['slavename']):
- continue
- ci_slave = jenkins.get_slave(resource['slavename'])
- jenkins.parse_slave_data(resource, ci_slave)
- ci_pods.append(resource)
-
+ ci_pods = Resource.objects.filter(slave__ci_slave=True)
context = super(CIPodsView, self).get_context_data(**kwargs)
context.update({'title': "CI Pods", 'ci_pods': ci_pods})
return context
@@ -41,21 +32,18 @@ class DevelopmentPodsView(TemplateView):
template_name = "dashboard/dev_pods.html"
def get_context_data(self, **kwargs):
- resources = Resource.objects.filter().values() # get resources as a set of dicts
- dev_pods = []
+ resources = Resource.objects.filter(slave__dev_pod=True)
- current_bookings = Booking.objects.filter(start__lte=timezone.now())
- current_bookings = current_bookings.filter(end__gt=timezone.now())
+ bookings = Booking.objects.filter(start__lte=timezone.now())
+ bookings = bookings.filter(end__gt=timezone.now())
+ dev_pods = []
for resource in resources:
- if not jenkins.is_dev_pod(resource['slavename']):
- continue
- dev_pod = jenkins.get_slave(resource['slavename'])
- jenkins.parse_slave_data(resource, dev_pod)
- for booking in current_bookings:
- if booking.resource.slavename == resource['slavename']:
- resource['current_booking'] = booking
- dev_pods.append(resource)
+ dev_pod = (resource, None)
+ for booking in bookings:
+ if booking.resource == resource:
+ dev_pod = (resource, booking)
+ dev_pods.append(dev_pod)
context = super(DevelopmentPodsView, self).get_context_data(**kwargs)
context.update({'title': "Development Pods", 'dev_pods': dev_pods})
diff --git a/tools/pharos-dashboard/jenkins/adapter.py b/tools/pharos-dashboard/jenkins/adapter.py
index 06233af8..fabd5356 100644
--- a/tools/pharos-dashboard/jenkins/adapter.py
+++ b/tools/pharos-dashboard/jenkins/adapter.py
@@ -14,7 +14,7 @@ def get_json(url):
response = requests.get(url)
json = response.json()
cache.set(url, json, 180) # cache result for 180 seconds
- return response.json()
+ return json
except requests.exceptions.RequestException as e:
logger.exception(e)
except ValueError as e:
@@ -85,26 +85,22 @@ def is_dev_pod(slavename):
return True
return False
-def parse_slave_data(slave_dict, slave):
- slave_dict['status'] = get_slave_status(slave)
- slave_dict['status_color'] = get_status_color(slave)
- slave_dict['slaveurl'] = get_slave_url(slave)
- job = get_jenkins_job(slave['displayName'])
- if job is not None:
- slave_dict['last_job'] = parse_job(job)
-
def parse_job(job):
result = parse_job_string(job['lastBuild']['fullDisplayName'])
+ result['building'] = job['lastBuild']['building']
+ result['result'] = ''
+ if not job['lastBuild']['building']:
+ result['result'] = job['lastBuild']['result']
result['url'] = job['url']
- result['color'] = get_job_color(job)
- if job['lastBuild']['building']:
- result['blink'] = 'class=blink_me'
return result
def parse_job_string(full_displayname):
job = {}
+ job['scenario'] = ''
+ job['installer'] = ''
+ job['branch'] = ''
tokens = re.split(r'[ -]', full_displayname)
for i in range(len(tokens)):
if tokens[i] == 'os':
@@ -113,34 +109,10 @@ def parse_job_string(full_displayname):
job['installer'] = tokens[i]
elif tokens[i] in ['master', 'arno', 'brahmaputra', 'colorado']:
job['branch'] = tokens[i]
-
tokens = full_displayname.split(' ')
job['name'] = tokens[0]
return job
-
-# TODO: use css
-def get_job_color(job):
- if job['lastBuild']['building'] is True:
- return '#646F73'
- result = job['lastBuild']['result']
- if result == 'SUCCESS':
- return '#5cb85c'
- if result == 'FAILURE':
- return '#d9534f'
- if result == 'UNSTABLE':
- return '#EDD62B'
-
-
-# TODO: use css
-def get_status_color(slave):
- if not slave['offline'] and slave['idle']:
- return '#5bc0de'
- if not slave['offline']:
- return '#5cb85c'
- return '#d9534f'
-
-
def get_slave_url(slave):
return 'https://build.opnfv.org/ci/computer/' + slave['displayName']
diff --git a/tools/pharos-dashboard/jenkins/migrations/0001_initial.py b/tools/pharos-dashboard/jenkins/migrations/0001_initial.py
new file mode 100644
index 00000000..a9bb8d56
--- /dev/null
+++ b/tools/pharos-dashboard/jenkins/migrations/0001_initial.py
@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10 on 2016-08-15 12:19
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='JenkinsSlave',
+ fields=[
+ ('id', models.AutoField(primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=100, unique=True)),
+ ('status', models.CharField(default='offline', max_length=30)),
+ ('url', models.CharField(max_length=1024)),
+ ('ci_slave', models.BooleanField(default=False)),
+ ('dev_pod', models.BooleanField(default=False)),
+ ('building', models.BooleanField(default=False)),
+ ('last_job_name', models.CharField(default='', max_length=1024)),
+ ('last_job_url', models.CharField(default='', max_length=1024)),
+ ('last_job_scenario', models.CharField(default='', max_length=50)),
+ ('last_job_branch', models.CharField(default='', max_length=50)),
+ ('last_job_installer', models.CharField(default='', max_length=50)),
+ ('last_job_result', models.CharField(default='', max_length=30)),
+ ],
+ options={
+ 'db_table': 'jenkins_slave',
+ },
+ ),
+ migrations.CreateModel(
+ name='JenkinsStatistic',
+ fields=[
+ ('timestamp', models.DateTimeField(auto_created=True)),
+ ('id', models.AutoField(primary_key=True, serialize=False)),
+ ('offline', models.BooleanField(default=False)),
+ ('idle', models.BooleanField(default=False)),
+ ('online', models.BooleanField(default=False)),
+ ('slave', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jenkins.JenkinsSlave')),
+ ],
+ options={
+ 'db_table': 'jenkins_statistic',
+ },
+ ),
+ ]
diff --git a/tools/pharos-dashboard/jenkins/migrations/0002_auto_20160815_1226.py b/tools/pharos-dashboard/jenkins/migrations/0002_auto_20160815_1226.py
new file mode 100644
index 00000000..f1cf7f99
--- /dev/null
+++ b/tools/pharos-dashboard/jenkins/migrations/0002_auto_20160815_1226.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10 on 2016-08-15 12:26
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('jenkins', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='jenkinsstatistic',
+ name='timestamp',
+ field=models.DateTimeField(auto_now_add=True),
+ ),
+ ]
diff --git a/tools/pharos-dashboard/jenkins/migrations/__init__.py b/tools/pharos-dashboard/jenkins/migrations/__init__.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/tools/pharos-dashboard/jenkins/migrations/__init__.py
diff --git a/tools/pharos-dashboard/jenkins/models.py b/tools/pharos-dashboard/jenkins/models.py
new file mode 100644
index 00000000..f7d54c96
--- /dev/null
+++ b/tools/pharos-dashboard/jenkins/models.py
@@ -0,0 +1,34 @@
+from django.db import models
+
+
+class JenkinsSlave(models.Model):
+ id = models.AutoField(primary_key=True)
+ name = models.CharField(max_length=100, unique=True)
+ status = models.CharField(max_length=30, default='offline')
+ url = models.CharField(max_length=1024)
+ ci_slave = models.BooleanField(default=False)
+ dev_pod = models.BooleanField(default=False)
+
+ building = models.BooleanField(default=False)
+
+ last_job_name = models.CharField(max_length=1024, default='')
+ last_job_url = models.CharField(max_length=1024, default='')
+ last_job_scenario = models.CharField(max_length=50, default='')
+ last_job_branch = models.CharField(max_length=50, default='')
+ last_job_installer = models.CharField(max_length=50, default='')
+ last_job_result = models.CharField(max_length=30, default='')
+
+ class Meta:
+ db_table = 'jenkins_slave'
+
+
+class JenkinsStatistic(models.Model):
+ id = models.AutoField(primary_key=True)
+ slave = models.ForeignKey(JenkinsSlave, on_delete=models.CASCADE)
+ offline = models.BooleanField(default=False)
+ idle = models.BooleanField(default=False)
+ online = models.BooleanField(default=False)
+ timestamp = models.DateTimeField(auto_now_add=True)
+
+ class Meta:
+ db_table = 'jenkins_statistic'
diff --git a/tools/pharos-dashboard/jenkins/tasks.py b/tools/pharos-dashboard/jenkins/tasks.py
new file mode 100644
index 00000000..6998cf3b
--- /dev/null
+++ b/tools/pharos-dashboard/jenkins/tasks.py
@@ -0,0 +1,39 @@
+from celery import shared_task
+
+from jenkins.models import JenkinsSlave, JenkinsStatistic
+from .adapter import *
+
+
+@shared_task
+def sync_jenkins():
+ update_jenkins_slaves()
+
+
+def update_jenkins_slaves():
+ jenkins_slaves = get_all_slaves()
+ for slave in jenkins_slaves:
+ jenkins_slave, created = JenkinsSlave.objects.get_or_create(name=slave['displayName'],
+ url=get_slave_url(slave))
+ jenkins_slave.ci_slave = is_ci_slave(slave['displayName'])
+ jenkins_slave.dev_pod = is_dev_pod(slave['displayName'])
+ jenkins_slave.status = get_slave_status(slave)
+
+ last_job = get_jenkins_job(jenkins_slave.name)
+ if last_job is not None:
+ last_job = parse_job(last_job)
+ jenkins_slave.last_job_name = last_job['name']
+ jenkins_slave.last_job_url = last_job['url']
+ jenkins_slave.last_job_scenario = last_job['scenario']
+ jenkins_slave.last_job_branch = last_job['branch']
+ jenkins_slave.last_job_installer = last_job['installer']
+ jenkins_slave.last_job_result = last_job['result']
+ jenkins_slave.save()
+
+ jenkins_statistic = JenkinsStatistic(slave=jenkins_slave)
+ if jenkins_slave.status == 'online' or jenkins_slave.status == 'building':
+ jenkins_statistic.online = True
+ if jenkins_slave.status == 'offline':
+ jenkins_statistic.offline = True
+ if jenkins_slave.status == 'online / idle':
+ jenkins_statistic.idle = True
+ jenkins_statistic.save()
diff --git a/tools/pharos-dashboard/pharos_dashboard/__init__.py b/tools/pharos-dashboard/pharos_dashboard/__init__.py
index e69de29b..b6fc8176 100644
--- a/tools/pharos-dashboard/pharos_dashboard/__init__.py
+++ b/tools/pharos-dashboard/pharos_dashboard/__init__.py
@@ -0,0 +1,3 @@
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app # noqa
diff --git a/tools/pharos-dashboard/pharos_dashboard/celery.py b/tools/pharos-dashboard/pharos_dashboard/celery.py
new file mode 100644
index 00000000..4cf6a7af
--- /dev/null
+++ b/tools/pharos-dashboard/pharos_dashboard/celery.py
@@ -0,0 +1,20 @@
+import os
+
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pharos_dashboard.settings')
+
+from django.conf import settings # noqa
+
+app = Celery('pharos_dashboard')
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+
+
+@app.task(bind=True)
+def debug_task(self):
+ print('Request: {0!r}'.format(self.request)) \ No newline at end of file
diff --git a/tools/pharos-dashboard/pharos_dashboard/settings.py b/tools/pharos-dashboard/pharos_dashboard/settings.py
index b6e98991..77175016 100644
--- a/tools/pharos-dashboard/pharos_dashboard/settings.py
+++ b/tools/pharos-dashboard/pharos_dashboard/settings.py
@@ -15,7 +15,6 @@ import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
@@ -27,7 +26,6 @@ DEBUG = True
ALLOWED_HOSTS = []
-
# Application definition
INSTALLED_APPS = [
@@ -43,6 +41,8 @@ INSTALLED_APPS = [
'django.contrib.staticfiles',
'django.contrib.humanize',
'bootstrap3',
+ 'djcelery',
+ 'kombu.transport.django',
]
MIDDLEWARE = [
@@ -77,7 +77,6 @@ TEMPLATES = [
WSGI_APPLICATION = 'pharos_dashboard.wsgi.application'
-
# Database
# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
@@ -92,7 +91,6 @@ DATABASES = {
}
}
-
# Password validation
# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
@@ -124,7 +122,6 @@ USE_L10N = True
USE_TZ = True
-
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.10/howto/static-files/
@@ -139,3 +136,11 @@ BOOTSTRAP3 = {
}
LOGIN_REDIRECT_URL = '/'
+
+import djcelery
+
+djcelery.setup_loader()
+# django broker, NOT SAFE FOR PRODUCTION
+BROKER_URL = 'django://'
+CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'
+
diff --git a/tools/pharos-dashboard/templates/dashboard/ci_pods.html b/tools/pharos-dashboard/templates/dashboard/ci_pods.html
index d3e5ff62..2982a6ff 100644
--- a/tools/pharos-dashboard/templates/dashboard/ci_pods.html
+++ b/tools/pharos-dashboard/templates/dashboard/ci_pods.html
@@ -1,5 +1,6 @@
{% extends "dashboard/table.html" %}
{% load staticfiles %}
+{% load jenkins_filters %}
{% block table %}
<thead>
@@ -20,23 +21,24 @@
<a target='_blank' href={{ pod.url }}>{{ pod.name }}</a>
</th>
<th>
- <a target='_blank' href={{ pod.slaveurl }}>{{ pod.slavename }}</a>
+ <a target='_blank' href={{ pod.slave.url }}>{{ pod.slave.name }}</a>
</th>
- <th style="background-color:{{ pod.status_color }}">
- {{ pod.status }}
+ <th style="background-color:{{ pod.slave.status | jenkins_status_color }}">
+ {{ pod.slave.status }}
</th>
- <th {{ pod.last_job.blink }}>
- {{ pod.last_job.installer }}
+ <th {{ pod.slave.last_job_result | jenkins_job_blink }}>
+ {{ pod.slave.last_job_installer }}
</th>
- <th {{ pod.last_job.blink }}>
- {{ pod.last_job.scenario }}
+ <th {{ pod.slave.last_job_result | jenkins_job_blink }}>
+ {{ pod.slave.last_job_scenario }}
</th>
- <th {{ pod.last_job.blink }}>
- {{ pod.last_job.branch }}
+ <th {{ pod.slave.last_job_result | jenkins_job_blink }}>
+ {{ pod.slave.last_job_branch }}
</th>
- <th><a {{ pod.last_job.blink }} style="color:{{ pod.last_job.color }}"
- target='_blank'
- href={{ pod.last_job.url }}>{{ pod.last_job.name }}</a>
+ <th><a {{ pod.slave.last_job_result | jenkins_job_blink }}
+ style="color:{{ pod.slave.last_job_result | jenkins_job_color }}"
+ target='_blank'
+ href={{ pod.slave.last_job_url }}>{{ pod.slave.last_job_name }}</a>
</th>
</tr>
{% endfor %}`
diff --git a/tools/pharos-dashboard/templates/dashboard/dev_pods.html b/tools/pharos-dashboard/templates/dashboard/dev_pods.html
index f08e1d1f..532a3a11 100644
--- a/tools/pharos-dashboard/templates/dashboard/dev_pods.html
+++ b/tools/pharos-dashboard/templates/dashboard/dev_pods.html
@@ -1,5 +1,6 @@
{% extends "dashboard/table.html" %}
{% load staticfiles %}
+{% load jenkins_filters %}
{% block table %}
<thead>
@@ -14,28 +15,28 @@
</tr>
</thead>
<tbody>
- {% for resource in dev_pods %}
+ {% for pod, booking in dev_pods %}
<tr>
<th>
- <a target='_blank' href={{ resource.url }}>{{ resource.name }}</a>
+ <a target='_blank' href={{ pod.url }}>{{ pod.name }}</a>
</th>
<th>
- <a target='_blank' href={{ resource.slaveurl }}>{{ resource.slavename }}</a>
+ <a target='_blank' href={{ pod.slave.url }}>{{ pod.slave.name }}</a>
</th>
<th>
- {{ resource.current_booking.user.username }}
+ {{ booking.user.username }}
</th>
<th>
- {{ resource.current_booking.end }}
+ {{ booking.end }}
</th>
<th>
- {{ resource.current_booking.purpose }}
+ {{ booking.purpose }}
</th>
- <th style="background-color:{{ resource.status_color }}">
- {{ resource.status }}
+ <th style="background-color:{{ pod.slave.status | jenkins_status_color }}">
+ {{ pod.slave.status }}
</th>
<th>
- <a href="{% url 'booking:create' resource_id=resource.id %}" class="btn btn-primary">
+ <a href="{% url 'booking:create' resource_id=pod.id %}" class="btn btn-primary">
Book
</a>
</th>
diff --git a/tools/pharos-dashboard/templates/dashboard/jenkins_slaves.html b/tools/pharos-dashboard/templates/dashboard/jenkins_slaves.html
index 2d011b46..830ed198 100644
--- a/tools/pharos-dashboard/templates/dashboard/jenkins_slaves.html
+++ b/tools/pharos-dashboard/templates/dashboard/jenkins_slaves.html
@@ -1,6 +1,8 @@
{% extends "dashboard/table.html" %}
{% load staticfiles %}
+{% load jenkins_filters %}
+
{% block table %}
<thead>
<tr>
@@ -13,14 +15,15 @@
{% for slave in slaves %}
<tr>
<th><a target='_blank'
- href={{ slave.slaveurl }}>{{ slave.displayName }}</a>
+ href={{ slave.url }}>{{ slave.name }}</a>
</th>
- <th style="background-color:{{ slave.status_color }}">
+ <th style="background-color:{{ slave.status | jenkins_status_color }}">
{{ slave.status }}
</th>
- <th><a {{ slave.last_job.blink }} style="color:{{ slave.last_job.color }}"
- target="_blank" href={{ slave.last_job.url }}>
- {{ slave.last_job.name }}</a>
+ <th><a {{ slave.last_job_result | jenkins_job_blink }}
+ style="color:{{ slave.last_job_result | jenkins_job_color }}"
+ target="_blank" href={{ slave.last_job_url }}>
+ {{ slave.last_job_name }}</a>
</th>
</tr>
{% endfor %}