summaryrefslogtreecommitdiffstats
path: root/dashboard/src/jenkins
diff options
context:
space:
mode:
authorTrevor Bramwell <tbramwell@linuxfoundation.org>2017-09-22 12:23:36 -0700
committerTrevor Bramwell <tbramwell@linuxfoundation.org>2017-09-22 12:23:36 -0700
commit4b269fba0ca273dfa3acf44c9f5490f01e0c3d87 (patch)
tree909b5262b46306eb78327e535f2e14120e45116a /dashboard/src/jenkins
parentd46ab54583a6c20bfa5bea581f512474f488e788 (diff)
Rename pharos-dashboard and pharos-validator
As subdirectories of the pharos-tools repo, there is little need to keep the pharos prefix. Change-Id: Ica3d79411f409df638647300036c0664183c2725 Signed-off-by: Trevor Bramwell <tbramwell@linuxfoundation.org>
Diffstat (limited to 'dashboard/src/jenkins')
-rw-r--r--dashboard/src/jenkins/__init__.py10
-rw-r--r--dashboard/src/jenkins/adapter.py134
-rw-r--r--dashboard/src/jenkins/admin.py17
-rw-r--r--dashboard/src/jenkins/apps.py15
-rw-r--r--dashboard/src/jenkins/migrations/0001_initial.py53
-rw-r--r--dashboard/src/jenkins/migrations/__init__.py10
-rw-r--r--dashboard/src/jenkins/models.py62
-rw-r--r--dashboard/src/jenkins/tasks.py64
-rw-r--r--dashboard/src/jenkins/tests.py129
9 files changed, 494 insertions, 0 deletions
diff --git a/dashboard/src/jenkins/__init__.py b/dashboard/src/jenkins/__init__.py
new file mode 100644
index 0000000..b5914ce
--- /dev/null
+++ b/dashboard/src/jenkins/__init__.py
@@ -0,0 +1,10 @@
+##############################################################################
+# Copyright (c) 2016 Max Breitenfeldt and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
diff --git a/dashboard/src/jenkins/adapter.py b/dashboard/src/jenkins/adapter.py
new file mode 100644
index 0000000..edf502f
--- /dev/null
+++ b/dashboard/src/jenkins/adapter.py
@@ -0,0 +1,134 @@
+##############################################################################
+# Copyright (c) 2016 Max Breitenfeldt and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+import logging
+import re
+
+import requests
+from django.core.cache import cache
+
+logger = logging.getLogger(__name__)
+
+# TODO: implement caching decorator, cache get_* functions
+def get_json(url):
+ if cache.get(url) is None:
+ try:
+ response = requests.get(url)
+ json = response.json()
+ cache.set(url, json, 180) # cache result for 180 seconds
+ return json
+ except requests.exceptions.RequestException as e:
+ logger.exception(e)
+ except ValueError as e:
+ logger.exception(e)
+ else:
+ return cache.get(url)
+
+
+def get_all_slaves():
+ url = "https://build.opnfv.org/ci/computer/api/json?tree=computer[displayName,offline,idle]"
+ json = get_json(url)
+ if json is not None:
+ return json['computer'] # return list of dictionaries
+ return []
+
+
+def get_slave(slavename):
+ slaves = get_all_slaves()
+ for slave in slaves:
+ if slave['displayName'] == slavename:
+ return slave
+ return {}
+
+
+def get_ci_slaves():
+ url = "https://build.opnfv.org/ci/label/ci-pod/api/json?tree=nodes[nodeName,offline,idle]"
+ json = get_json(url)
+ if json is not None:
+ return json['nodes']
+ return []
+
+
+def get_all_jobs():
+ url = "https://build.opnfv.org/ci/api/json?tree=jobs[displayName,url,lastBuild[fullDisplayName,building,builtOn,timestamp,result]]"
+ json = get_json(url)
+ if json is not None:
+ return json['jobs'] # return list of dictionaries
+ return []
+
+
+def get_jenkins_job(slavename):
+ jobs = get_all_jobs()
+ max_time = 0
+ last_job = None
+ for job in jobs:
+ if job['lastBuild'] is not None:
+ if job['lastBuild']['builtOn'] == slavename:
+ if job['lastBuild']['building'] is True:
+ return job # return active build
+ if job['lastBuild']['timestamp'] > max_time:
+ last_job = job
+ max_time = job['lastBuild']['timestamp']
+ return last_job
+
+
+def is_ci_slave(slavename):
+ ci_slaves = get_ci_slaves()
+ for ci_slave in ci_slaves:
+ if ci_slave['nodeName'] == slavename:
+ return True
+ return False
+
+
+def is_dev_pod(slavename):
+ if is_ci_slave(slavename):
+ return False
+ if slavename.find('pod') != -1:
+ return True
+ return False
+
+
+def parse_job(job):
+ result = parse_job_string(job['lastBuild']['fullDisplayName'])
+ result['building'] = job['lastBuild']['building']
+ result['result'] = ''
+ if not job['lastBuild']['building']:
+ result['result'] = job['lastBuild']['result']
+ result['url'] = job['url']
+ return result
+
+
+def parse_job_string(full_displayname):
+ job = {}
+ job['scenario'] = ''
+ job['installer'] = ''
+ job['branch'] = ''
+ tokens = re.split(r'[ -]', full_displayname)
+ for i in range(len(tokens)):
+ if tokens[i] == 'os':
+ job['scenario'] = '-'.join(tokens[i: i + 4])
+ elif tokens[i] in ['fuel', 'joid', 'apex', 'compass']:
+ job['installer'] = tokens[i]
+ elif tokens[i] in ['master', 'arno', 'brahmaputra', 'colorado']:
+ job['branch'] = tokens[i]
+ tokens = full_displayname.split(' ')
+ job['name'] = tokens[0]
+ return job
+
+def get_slave_url(slave):
+ return 'https://build.opnfv.org/ci/computer/' + slave['displayName']
+
+
+def get_slave_status(slave):
+ if not slave['offline'] and slave['idle']:
+ return 'online / idle'
+ if not slave['offline']:
+ return 'online'
+ return 'offline'
diff --git a/dashboard/src/jenkins/admin.py b/dashboard/src/jenkins/admin.py
new file mode 100644
index 0000000..c499670
--- /dev/null
+++ b/dashboard/src/jenkins/admin.py
@@ -0,0 +1,17 @@
+##############################################################################
+# Copyright (c) 2016 Max Breitenfeldt and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from django.conf import settings
+from django.contrib import admin
+
+from jenkins.models import JenkinsSlave
+
+if settings.DEBUG:
+ admin.site.register(JenkinsSlave) \ No newline at end of file
diff --git a/dashboard/src/jenkins/apps.py b/dashboard/src/jenkins/apps.py
new file mode 100644
index 0000000..41faf60
--- /dev/null
+++ b/dashboard/src/jenkins/apps.py
@@ -0,0 +1,15 @@
+##############################################################################
+# Copyright (c) 2016 Max Breitenfeldt and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from django.apps import AppConfig
+
+
+class JenkinsConfig(AppConfig):
+ name = 'jenkins'
diff --git a/dashboard/src/jenkins/migrations/0001_initial.py b/dashboard/src/jenkins/migrations/0001_initial.py
new file mode 100644
index 0000000..b1c7889
--- /dev/null
+++ b/dashboard/src/jenkins/migrations/0001_initial.py
@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10 on 2016-11-03 13:33
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ initial = True
+
+ dependencies = [
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='JenkinsSlave',
+ fields=[
+ ('id', models.AutoField(primary_key=True, serialize=False)),
+ ('name', models.CharField(max_length=100, unique=True)),
+ ('status', models.CharField(default='offline', max_length=30)),
+ ('url', models.CharField(max_length=1024)),
+ ('ci_slave', models.BooleanField(default=False)),
+ ('dev_pod', models.BooleanField(default=False)),
+ ('building', models.BooleanField(default=False)),
+ ('last_job_name', models.CharField(default='', max_length=1024)),
+ ('last_job_url', models.CharField(default='', max_length=1024)),
+ ('last_job_scenario', models.CharField(default='', max_length=50)),
+ ('last_job_branch', models.CharField(default='', max_length=50)),
+ ('last_job_installer', models.CharField(default='', max_length=50)),
+ ('last_job_result', models.CharField(default='', max_length=30)),
+ ('active', models.BooleanField(default=False)),
+ ],
+ options={
+ 'db_table': 'jenkins_slave',
+ },
+ ),
+ migrations.CreateModel(
+ name='JenkinsStatistic',
+ fields=[
+ ('id', models.AutoField(primary_key=True, serialize=False)),
+ ('offline', models.BooleanField(default=False)),
+ ('idle', models.BooleanField(default=False)),
+ ('online', models.BooleanField(default=False)),
+ ('timestamp', models.DateTimeField(auto_now_add=True)),
+ ('slave', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jenkins.JenkinsSlave')),
+ ],
+ options={
+ 'db_table': 'jenkins_statistic',
+ },
+ ),
+ ]
diff --git a/dashboard/src/jenkins/migrations/__init__.py b/dashboard/src/jenkins/migrations/__init__.py
new file mode 100644
index 0000000..b5914ce
--- /dev/null
+++ b/dashboard/src/jenkins/migrations/__init__.py
@@ -0,0 +1,10 @@
+##############################################################################
+# Copyright (c) 2016 Max Breitenfeldt and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
diff --git a/dashboard/src/jenkins/models.py b/dashboard/src/jenkins/models.py
new file mode 100644
index 0000000..8254ff3
--- /dev/null
+++ b/dashboard/src/jenkins/models.py
@@ -0,0 +1,62 @@
+##############################################################################
+# Copyright (c) 2016 Max Breitenfeldt and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from django.db import models
+from django.utils import timezone
+
+
+class JenkinsSlave(models.Model):
+ id = models.AutoField(primary_key=True)
+ name = models.CharField(max_length=100, unique=True)
+ status = models.CharField(max_length=30, default='offline')
+ url = models.CharField(max_length=1024)
+ ci_slave = models.BooleanField(default=False)
+ dev_pod = models.BooleanField(default=False)
+
+ building = models.BooleanField(default=False)
+
+ last_job_name = models.CharField(max_length=1024, default='')
+ last_job_url = models.CharField(max_length=1024, default='')
+ last_job_scenario = models.CharField(max_length=50, default='')
+ last_job_branch = models.CharField(max_length=50, default='')
+ last_job_installer = models.CharField(max_length=50, default='')
+ last_job_result = models.CharField(max_length=30, default='')
+
+ active = models.BooleanField(default=False)
+
+ def get_utilization(self, timedelta):
+ """
+ Return a dictionary containing the count of idle, online and offline measurements in the time from
+ now-timedelta to now
+ """
+ utilization = {'idle': 0, 'online': 0, 'offline': 0}
+ statistics = self.jenkinsstatistic_set.filter(timestamp__gte=timezone.now() - timedelta)
+ utilization['idle'] = statistics.filter(idle=True).count()
+ utilization['online'] = statistics.filter(online=True).count()
+ utilization['offline'] = statistics.filter(offline=True).count()
+ return utilization
+
+ class Meta:
+ db_table = 'jenkins_slave'
+
+ def __str__(self):
+ return self.name
+
+
+class JenkinsStatistic(models.Model):
+ id = models.AutoField(primary_key=True)
+ slave = models.ForeignKey(JenkinsSlave, on_delete=models.CASCADE)
+ offline = models.BooleanField(default=False)
+ idle = models.BooleanField(default=False)
+ online = models.BooleanField(default=False)
+ timestamp = models.DateTimeField(auto_now_add=True)
+
+ class Meta:
+ db_table = 'jenkins_statistic'
diff --git a/dashboard/src/jenkins/tasks.py b/dashboard/src/jenkins/tasks.py
new file mode 100644
index 0000000..ea986c1
--- /dev/null
+++ b/dashboard/src/jenkins/tasks.py
@@ -0,0 +1,64 @@
+##############################################################################
+# Copyright (c) 2016 Max Breitenfeldt and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from celery import shared_task
+
+from dashboard.models import Resource
+from jenkins.models import JenkinsSlave, JenkinsStatistic
+from .adapter import *
+
+
+@shared_task
+def sync_jenkins():
+ update_jenkins_slaves()
+
+
+def update_jenkins_slaves():
+ JenkinsSlave.objects.all().update(active=False)
+
+ jenkins_slaves = get_all_slaves()
+ for slave in jenkins_slaves:
+ jenkins_slave, created = JenkinsSlave.objects.get_or_create(name=slave['displayName'],
+ url=get_slave_url(slave))
+ jenkins_slave.active = True
+ jenkins_slave.ci_slave = is_ci_slave(slave['displayName'])
+ jenkins_slave.dev_pod = is_dev_pod(slave['displayName'])
+ jenkins_slave.status = get_slave_status(slave)
+
+ # if this is a new slave and a pod, check if there is a resource for it, create one if not
+ if created and 'pod' in slave['displayName']:
+ # parse resource name from slave name
+ # naming example: orange-pod1, resource name: Orange POD 1
+ tokens = slave['displayName'].split('-')
+ name = tokens[0].capitalize() + ' POD '# company name
+ name += tokens[1][3:] # remove 'pod'
+ resource, created = Resource.objects.get_or_create(name=name)
+ resource.slave = jenkins_slave
+ resource.save()
+
+ last_job = get_jenkins_job(jenkins_slave.name)
+ if last_job is not None:
+ last_job = parse_job(last_job)
+ jenkins_slave.last_job_name = last_job['name']
+ jenkins_slave.last_job_url = last_job['url']
+ jenkins_slave.last_job_scenario = last_job['scenario']
+ jenkins_slave.last_job_branch = last_job['branch']
+ jenkins_slave.last_job_installer = last_job['installer']
+ jenkins_slave.last_job_result = last_job['result']
+ jenkins_slave.save()
+
+ jenkins_statistic = JenkinsStatistic(slave=jenkins_slave)
+ if jenkins_slave.status == 'online' or jenkins_slave.status == 'building':
+ jenkins_statistic.online = True
+ if jenkins_slave.status == 'offline':
+ jenkins_statistic.offline = True
+ if jenkins_slave.status == 'online / idle':
+ jenkins_statistic.idle = True
+ jenkins_statistic.save()
diff --git a/dashboard/src/jenkins/tests.py b/dashboard/src/jenkins/tests.py
new file mode 100644
index 0000000..3723cd3
--- /dev/null
+++ b/dashboard/src/jenkins/tests.py
@@ -0,0 +1,129 @@
+##############################################################################
+# Copyright (c) 2016 Max Breitenfeldt and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from datetime import timedelta
+from unittest import TestCase
+
+import jenkins.adapter as jenkins
+from jenkins.models import *
+
+
+# Tests that the data we get with the jenkinsadapter contains all the
+# data we need. These test will fail if;
+# - there is no internet connection
+# - the opnfv jenkins url has changed
+# - the jenkins api has changed
+# - jenkins is not set up / there is no data
+class JenkinsAdapterTestCase(TestCase):
+ def test_get_all_slaves(self):
+ slaves = jenkins.get_all_slaves()
+ self.assertTrue(len(slaves) > 0)
+ for slave in slaves:
+ self.assertTrue('displayName' in slave)
+ self.assertTrue('idle' in slave)
+ self.assertTrue('offline' in slave)
+
+ def test_get_slave(self):
+ slaves = jenkins.get_all_slaves()
+ self.assertEqual(slaves[0], jenkins.get_slave(slaves[0]['displayName']))
+ self.assertEqual({}, jenkins.get_slave('098f6bcd4621d373cade4e832627b4f6'))
+
+ def test_get_ci_slaves(self):
+ slaves = jenkins.get_ci_slaves()
+ self.assertTrue(len(slaves) > 0)
+ for slave in slaves:
+ self.assertTrue('nodeName' in slave)
+
+ def test_get_jenkins_job(self):
+ slaves = jenkins.get_ci_slaves()
+ job = None
+ for slave in slaves:
+ job = jenkins.get_jenkins_job(slave['nodeName'])
+ if job is not None:
+ break
+ # We need to test at least one job
+ self.assertNotEqual(job, None)
+
+ def test_get_all_jobs(self):
+ jobs = jenkins.get_all_jobs()
+ lastBuild = False
+ self.assertTrue(len(jobs) > 0)
+ for job in jobs:
+ self.assertTrue('displayName' in job)
+ self.assertTrue('url' in job)
+ self.assertTrue('lastBuild' in job)
+ if job['lastBuild'] is not None:
+ lastBuild = True
+ self.assertTrue('building' in job['lastBuild'])
+ self.assertTrue('fullDisplayName' in job['lastBuild'])
+ self.assertTrue('result' in job['lastBuild'])
+ self.assertTrue('timestamp' in job['lastBuild'])
+ self.assertTrue('builtOn' in job['lastBuild'])
+ self.assertTrue(lastBuild)
+
+ def test_parse_job(self):
+ job = {
+ "displayName": "apex-deploy-baremetal-os-nosdn-fdio-noha-colorado",
+ "url": "https://build.opnfv.org/ci/job/apex-deploy-baremetal-os-nosdn-fdio-noha-colorado/",
+ "lastBuild": {
+ "building": False,
+ "fullDisplayName": "apex-deploy-baremetal-os-nosdn-fdio-noha-colorado #37",
+ "result": "SUCCESS",
+ "timestamp": 1476283629917,
+ "builtOn": "lf-pod1"
+ }
+ }
+
+ job = jenkins.parse_job(job)
+ self.assertEqual(job['scenario'], 'os-nosdn-fdio-noha')
+ self.assertEqual(job['installer'], 'apex')
+ self.assertEqual(job['branch'], 'colorado')
+ self.assertEqual(job['result'], 'SUCCESS')
+ self.assertEqual(job['building'], False)
+ self.assertEqual(job['url'],
+ "https://build.opnfv.org/ci/job/apex-deploy-baremetal-os-nosdn-fdio-noha-colorado/")
+ self.assertEqual(job['name'],
+ 'apex-deploy-baremetal-os-nosdn-fdio-noha-colorado')
+
+ def test_get_slave_status(self):
+ slave = {
+ 'offline': True,
+ 'idle': False
+ }
+ self.assertEqual(jenkins.get_slave_status(slave), 'offline')
+ slave = {
+ 'offline': False,
+ 'idle': False
+ }
+ self.assertEqual(jenkins.get_slave_status(slave), 'online')
+ slave = {
+ 'offline': False,
+ 'idle': True
+ }
+ self.assertEqual(jenkins.get_slave_status(slave), 'online / idle')
+
+
+class JenkinsModelTestCase(TestCase):
+ def test_get_utilization(self):
+ jenkins_slave = JenkinsSlave.objects.create(name='test', status='offline', url='')
+ utilization = jenkins_slave.get_utilization(timedelta(weeks=1))
+ self.assertEqual(utilization['idle'], 0)
+ self.assertEqual(utilization['offline'], 0)
+ self.assertEqual(utilization['online'], 0)
+
+ for i in range(10):
+ JenkinsStatistic.objects.create(slave=jenkins_slave,
+ offline=True, idle=True,
+ online=True)
+
+ utilization = jenkins_slave.get_utilization(timedelta(weeks=1))
+ self.assertEqual(utilization['idle'], 10)
+ self.assertEqual(utilization['offline'], 10)
+ self.assertEqual(utilization['online'], 10)