Implement periodic tasks

JIRA: RELENG-12

The dashboard is now querying jenkins periodically and saving the
results in the database. This fixes delays that were caused by calling
the jenkins API.

Signed-off-by: maxbr <maxbr@mi.fu-berlin.de>

6 files changed, 153 insertions, 36 deletions

diff --git a/pharos-dashboard/jenkins/adapter.py b/pharos-dashboard/jenkins/adapter.py
index 06233af..fabd535 100644
--- a/pharos-dashboard/jenkins/adapter.py
+++ b/pharos-dashboard/jenkins/adapter.py
@@ -14,7 +14,7 @@ def get_json(url):
             response = requests.get(url)
             json = response.json()
             cache.set(url, json, 180)  # cache result for 180 seconds
-            return response.json()
+            return json
         except requests.exceptions.RequestException as e:
             logger.exception(e)
         except ValueError as e:
@@ -85,26 +85,22 @@ def is_dev_pod(slavename):
         return True
     return False
 
-def parse_slave_data(slave_dict, slave):
-    slave_dict['status'] = get_slave_status(slave)
-    slave_dict['status_color'] = get_status_color(slave)
-    slave_dict['slaveurl'] = get_slave_url(slave)
-    job = get_jenkins_job(slave['displayName'])
-    if job is not None:
-        slave_dict['last_job'] = parse_job(job)
-
 
 def parse_job(job):
     result = parse_job_string(job['lastBuild']['fullDisplayName'])
+    result['building'] = job['lastBuild']['building']
+    result['result'] = ''
+    if not job['lastBuild']['building']:
+        result['result'] = job['lastBuild']['result']
     result['url'] = job['url']
-    result['color'] = get_job_color(job)
-    if job['lastBuild']['building']:
-        result['blink'] = 'class=blink_me'
     return result
 
 
 def parse_job_string(full_displayname):
     job = {}
+    job['scenario'] = ''
+    job['installer'] = ''
+    job['branch'] = ''
     tokens = re.split(r'[ -]', full_displayname)
     for i in range(len(tokens)):
         if tokens[i] == 'os':
@@ -113,34 +109,10 @@ def parse_job_string(full_displayname):
             job['installer'] = tokens[i]
         elif tokens[i] in ['master', 'arno', 'brahmaputra', 'colorado']:
             job['branch'] = tokens[i]
-
     tokens = full_displayname.split(' ')
     job['name'] = tokens[0]
     return job
 
-
-# TODO: use css
-def get_job_color(job):
-    if job['lastBuild']['building'] is True:
-        return '#646F73'
-    result = job['lastBuild']['result']
-    if result == 'SUCCESS':
-        return '#5cb85c'
-    if result == 'FAILURE':
-        return '#d9534f'
-    if result == 'UNSTABLE':
-        return '#EDD62B'
-
-
-# TODO: use css
-def get_status_color(slave):
-    if not slave['offline'] and slave['idle']:
-        return '#5bc0de'
-    if not slave['offline']:
-        return '#5cb85c'
-    return '#d9534f'
-
-
 def get_slave_url(slave):
     return 'https://build.opnfv.org/ci/computer/' + slave['displayName']
 
diff --git a/pharos-dashboard/jenkins/migrations/0001_initial.py b/pharos-dashboard/jenkins/migrations/0001_initial.py
new file mode 100644
index 0000000..a9bb8d5
--- /dev/null
+++ b/pharos-dashboard/jenkins/migrations/0001_initial.py
@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10 on 2016-08-15 12:19
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='JenkinsSlave',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=100, unique=True)),
+                ('status', models.CharField(default='offline', max_length=30)),
+                ('url', models.CharField(max_length=1024)),
+                ('ci_slave', models.BooleanField(default=False)),
+                ('dev_pod', models.BooleanField(default=False)),
+                ('building', models.BooleanField(default=False)),
+                ('last_job_name', models.CharField(default='', max_length=1024)),
+                ('last_job_url', models.CharField(default='', max_length=1024)),
+                ('last_job_scenario', models.CharField(default='', max_length=50)),
+                ('last_job_branch', models.CharField(default='', max_length=50)),
+                ('last_job_installer', models.CharField(default='', max_length=50)),
+                ('last_job_result', models.CharField(default='', max_length=30)),
+            ],
+            options={
+                'db_table': 'jenkins_slave',
+            },
+        ),
+        migrations.CreateModel(
+            name='JenkinsStatistic',
+            fields=[
+                ('timestamp', models.DateTimeField(auto_created=True)),
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('offline', models.BooleanField(default=False)),
+                ('idle', models.BooleanField(default=False)),
+                ('online', models.BooleanField(default=False)),
+                ('slave', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jenkins.JenkinsSlave')),
+            ],
+            options={
+                'db_table': 'jenkins_statistic',
+            },
+        ),
+    ]
diff --git a/pharos-dashboard/jenkins/migrations/0002_auto_20160815_1226.py b/pharos-dashboard/jenkins/migrations/0002_auto_20160815_1226.py
new file mode 100644
index 0000000..f1cf7f9
--- /dev/null
+++ b/pharos-dashboard/jenkins/migrations/0002_auto_20160815_1226.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10 on 2016-08-15 12:26
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jenkins', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='jenkinsstatistic',
+            name='timestamp',
+            field=models.DateTimeField(auto_now_add=True),
+        ),
+    ]
diff --git a/pharos-dashboard/jenkins/migrations/__init__.py b/pharos-dashboard/jenkins/migrations/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/pharos-dashboard/jenkins/migrations/__init__.py
diff --git a/pharos-dashboard/jenkins/models.py b/pharos-dashboard/jenkins/models.py
new file mode 100644
index 0000000..f7d54c9
--- /dev/null
+++ b/pharos-dashboard/jenkins/models.py
@@ -0,0 +1,34 @@
+from django.db import models
+
+
+class JenkinsSlave(models.Model):
+    id = models.AutoField(primary_key=True)
+    name = models.CharField(max_length=100, unique=True)
+    status = models.CharField(max_length=30, default='offline')
+    url = models.CharField(max_length=1024)
+    ci_slave = models.BooleanField(default=False)
+    dev_pod = models.BooleanField(default=False)
+
+    building = models.BooleanField(default=False)
+
+    last_job_name = models.CharField(max_length=1024, default='')
+    last_job_url = models.CharField(max_length=1024, default='')
+    last_job_scenario = models.CharField(max_length=50, default='')
+    last_job_branch = models.CharField(max_length=50, default='')
+    last_job_installer = models.CharField(max_length=50, default='')
+    last_job_result = models.CharField(max_length=30, default='')
+
+    class Meta:
+        db_table = 'jenkins_slave'
+
+
+class JenkinsStatistic(models.Model):
+    id = models.AutoField(primary_key=True)
+    slave = models.ForeignKey(JenkinsSlave, on_delete=models.CASCADE)
+    offline = models.BooleanField(default=False)
+    idle = models.BooleanField(default=False)
+    online = models.BooleanField(default=False)
+    timestamp = models.DateTimeField(auto_now_add=True)
+
+    class Meta:
+        db_table = 'jenkins_statistic'
diff --git a/pharos-dashboard/jenkins/tasks.py b/pharos-dashboard/jenkins/tasks.py
new file mode 100644
index 0000000..6998cf3
--- /dev/null
+++ b/pharos-dashboard/jenkins/tasks.py
@@ -0,0 +1,39 @@
+from celery import shared_task
+
+from jenkins.models import JenkinsSlave, JenkinsStatistic
+from .adapter import *
+
+
+@shared_task
+def sync_jenkins():
+    update_jenkins_slaves()
+
+
+def update_jenkins_slaves():
+    jenkins_slaves = get_all_slaves()
+    for slave in jenkins_slaves:
+        jenkins_slave, created = JenkinsSlave.objects.get_or_create(name=slave['displayName'],
+                                                                    url=get_slave_url(slave))
+        jenkins_slave.ci_slave = is_ci_slave(slave['displayName'])
+        jenkins_slave.dev_pod = is_dev_pod(slave['displayName'])
+        jenkins_slave.status = get_slave_status(slave)
+
+        last_job = get_jenkins_job(jenkins_slave.name)
+        if last_job is not None:
+            last_job = parse_job(last_job)
+            jenkins_slave.last_job_name = last_job['name']
+            jenkins_slave.last_job_url = last_job['url']
+            jenkins_slave.last_job_scenario = last_job['scenario']
+            jenkins_slave.last_job_branch = last_job['branch']
+            jenkins_slave.last_job_installer = last_job['installer']
+            jenkins_slave.last_job_result = last_job['result']
+        jenkins_slave.save()
+
+        jenkins_statistic = JenkinsStatistic(slave=jenkins_slave)
+        if jenkins_slave.status == 'online' or jenkins_slave.status == 'building':
+            jenkins_statistic.online = True
+        if jenkins_slave.status == 'offline':
+            jenkins_statistic.offline = True
+        if jenkins_slave.status == 'online / idle':
+            jenkins_statistic.idle = True
+        jenkins_statistic.save()