diff options
Diffstat (limited to 'src/jenkins')
| -rw-r--r-- | src/jenkins/__init__.py | 10 | ||||
| -rw-r--r-- | src/jenkins/adapter.py | 134 | ||||
| -rw-r--r-- | src/jenkins/admin.py | 17 | ||||
| -rw-r--r-- | src/jenkins/apps.py | 15 | ||||
| -rw-r--r-- | src/jenkins/migrations/0001_initial.py | 53 | ||||
| -rw-r--r-- | src/jenkins/migrations/__init__.py | 10 | ||||
| -rw-r--r-- | src/jenkins/models.py | 62 | ||||
| -rw-r--r-- | src/jenkins/tasks.py | 64 | ||||
| -rw-r--r-- | src/jenkins/tests.py | 129 |
9 files changed, 494 insertions, 0 deletions
diff --git a/src/jenkins/__init__.py b/src/jenkins/__init__.py new file mode 100644 index 0000000..b5914ce --- /dev/null +++ b/src/jenkins/__init__.py @@ -0,0 +1,10 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + diff --git a/src/jenkins/adapter.py b/src/jenkins/adapter.py new file mode 100644 index 0000000..edf502f --- /dev/null +++ b/src/jenkins/adapter.py @@ -0,0 +1,134 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +import logging +import re + +import requests +from django.core.cache import cache + +logger = logging.getLogger(__name__) + +# TODO: implement caching decorator, cache get_* functions +def get_json(url): + if cache.get(url) is None: + try: + response = requests.get(url) + json = response.json() + cache.set(url, json, 180) # cache result for 180 seconds + return json + except requests.exceptions.RequestException as e: + logger.exception(e) + except ValueError as e: + logger.exception(e) + else: + return cache.get(url) + + +def get_all_slaves(): + url = "https://build.opnfv.org/ci/computer/api/json?tree=computer[displayName,offline,idle]" + json = get_json(url) + if json is not None: + return json['computer'] # return list of dictionaries + return [] + + +def get_slave(slavename): + slaves = get_all_slaves() + for slave in slaves: + if slave['displayName'] == slavename: + return slave + return {} + + +def get_ci_slaves(): + url = "https://build.opnfv.org/ci/label/ci-pod/api/json?tree=nodes[nodeName,offline,idle]" + json = get_json(url) + if json is not None: + return json['nodes'] + return [] + + +def get_all_jobs(): + url = "https://build.opnfv.org/ci/api/json?tree=jobs[displayName,url,lastBuild[fullDisplayName,building,builtOn,timestamp,result]]" + json = get_json(url) + if json is not None: + return json['jobs'] # return list of dictionaries + return [] + + +def get_jenkins_job(slavename): + jobs = get_all_jobs() + max_time = 0 + last_job = None + for job in jobs: + if job['lastBuild'] is not None: + if job['lastBuild']['builtOn'] == slavename: + if job['lastBuild']['building'] is True: + return job # return active build + if job['lastBuild']['timestamp'] > max_time: + last_job = job + max_time = job['lastBuild']['timestamp'] + return last_job + + +def is_ci_slave(slavename): + ci_slaves = get_ci_slaves() + for ci_slave in ci_slaves: + if ci_slave['nodeName'] == slavename: + return True + return False + + +def is_dev_pod(slavename): + if is_ci_slave(slavename): + return False + if slavename.find('pod') != -1: + return True + return False + + +def parse_job(job): + result = parse_job_string(job['lastBuild']['fullDisplayName']) + result['building'] = job['lastBuild']['building'] + result['result'] = '' + if not job['lastBuild']['building']: + result['result'] = job['lastBuild']['result'] + result['url'] = job['url'] + return result + + +def parse_job_string(full_displayname): + job = {} + job['scenario'] = '' + job['installer'] = '' + job['branch'] = '' + tokens = re.split(r'[ -]', full_displayname) + for i in range(len(tokens)): + if tokens[i] == 'os': + job['scenario'] = '-'.join(tokens[i: i + 4]) + elif tokens[i] in ['fuel', 'joid', 'apex', 'compass']: + job['installer'] = tokens[i] + elif tokens[i] in ['master', 'arno', 'brahmaputra', 'colorado']: + job['branch'] = tokens[i] + tokens = full_displayname.split(' ') + job['name'] = tokens[0] + return job + +def get_slave_url(slave): + return 'https://build.opnfv.org/ci/computer/' + slave['displayName'] + + +def get_slave_status(slave): + if not slave['offline'] and slave['idle']: + return 'online / idle' + if not slave['offline']: + return 'online' + return 'offline' diff --git a/src/jenkins/admin.py b/src/jenkins/admin.py new file mode 100644 index 0000000..c499670 --- /dev/null +++ b/src/jenkins/admin.py @@ -0,0 +1,17 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from django.conf import settings +from django.contrib import admin + +from jenkins.models import JenkinsSlave + +if settings.DEBUG: + admin.site.register(JenkinsSlave)
\ No newline at end of file diff --git a/src/jenkins/apps.py b/src/jenkins/apps.py new file mode 100644 index 0000000..41faf60 --- /dev/null +++ b/src/jenkins/apps.py @@ -0,0 +1,15 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from django.apps import AppConfig + + +class JenkinsConfig(AppConfig): + name = 'jenkins' diff --git a/src/jenkins/migrations/0001_initial.py b/src/jenkins/migrations/0001_initial.py new file mode 100644 index 0000000..b1c7889 --- /dev/null +++ b/src/jenkins/migrations/0001_initial.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.10 on 2016-11-03 13:33 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='JenkinsSlave', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('name', models.CharField(max_length=100, unique=True)), + ('status', models.CharField(default='offline', max_length=30)), + ('url', models.CharField(max_length=1024)), + ('ci_slave', models.BooleanField(default=False)), + ('dev_pod', models.BooleanField(default=False)), + ('building', models.BooleanField(default=False)), + ('last_job_name', models.CharField(default='', max_length=1024)), + ('last_job_url', models.CharField(default='', max_length=1024)), + ('last_job_scenario', models.CharField(default='', max_length=50)), + ('last_job_branch', models.CharField(default='', max_length=50)), + ('last_job_installer', models.CharField(default='', max_length=50)), + ('last_job_result', models.CharField(default='', max_length=30)), + ('active', models.BooleanField(default=False)), + ], + options={ + 'db_table': 'jenkins_slave', + }, + ), + migrations.CreateModel( + name='JenkinsStatistic', + fields=[ + ('id', models.AutoField(primary_key=True, serialize=False)), + ('offline', models.BooleanField(default=False)), + ('idle', models.BooleanField(default=False)), + ('online', models.BooleanField(default=False)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('slave', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jenkins.JenkinsSlave')), + ], + options={ + 'db_table': 'jenkins_statistic', + }, + ), + ] diff --git a/src/jenkins/migrations/__init__.py b/src/jenkins/migrations/__init__.py new file mode 100644 index 0000000..b5914ce --- /dev/null +++ b/src/jenkins/migrations/__init__.py @@ -0,0 +1,10 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + diff --git a/src/jenkins/models.py b/src/jenkins/models.py new file mode 100644 index 0000000..8254ff3 --- /dev/null +++ b/src/jenkins/models.py @@ -0,0 +1,62 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from django.db import models +from django.utils import timezone + + +class JenkinsSlave(models.Model): + id = models.AutoField(primary_key=True) + name = models.CharField(max_length=100, unique=True) + status = models.CharField(max_length=30, default='offline') + url = models.CharField(max_length=1024) + ci_slave = models.BooleanField(default=False) + dev_pod = models.BooleanField(default=False) + + building = models.BooleanField(default=False) + + last_job_name = models.CharField(max_length=1024, default='') + last_job_url = models.CharField(max_length=1024, default='') + last_job_scenario = models.CharField(max_length=50, default='') + last_job_branch = models.CharField(max_length=50, default='') + last_job_installer = models.CharField(max_length=50, default='') + last_job_result = models.CharField(max_length=30, default='') + + active = models.BooleanField(default=False) + + def get_utilization(self, timedelta): + """ + Return a dictionary containing the count of idle, online and offline measurements in the time from + now-timedelta to now + """ + utilization = {'idle': 0, 'online': 0, 'offline': 0} + statistics = self.jenkinsstatistic_set.filter(timestamp__gte=timezone.now() - timedelta) + utilization['idle'] = statistics.filter(idle=True).count() + utilization['online'] = statistics.filter(online=True).count() + utilization['offline'] = statistics.filter(offline=True).count() + return utilization + + class Meta: + db_table = 'jenkins_slave' + + def __str__(self): + return self.name + + +class JenkinsStatistic(models.Model): + id = models.AutoField(primary_key=True) + slave = models.ForeignKey(JenkinsSlave, on_delete=models.CASCADE) + offline = models.BooleanField(default=False) + idle = models.BooleanField(default=False) + online = models.BooleanField(default=False) + timestamp = models.DateTimeField(auto_now_add=True) + + class Meta: + db_table = 'jenkins_statistic' diff --git a/src/jenkins/tasks.py b/src/jenkins/tasks.py new file mode 100644 index 0000000..ea986c1 --- /dev/null +++ b/src/jenkins/tasks.py @@ -0,0 +1,64 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from celery import shared_task + +from dashboard.models import Resource +from jenkins.models import JenkinsSlave, JenkinsStatistic +from .adapter import * + + +@shared_task +def sync_jenkins(): + update_jenkins_slaves() + + +def update_jenkins_slaves(): + JenkinsSlave.objects.all().update(active=False) + + jenkins_slaves = get_all_slaves() + for slave in jenkins_slaves: + jenkins_slave, created = JenkinsSlave.objects.get_or_create(name=slave['displayName'], + url=get_slave_url(slave)) + jenkins_slave.active = True + jenkins_slave.ci_slave = is_ci_slave(slave['displayName']) + jenkins_slave.dev_pod = is_dev_pod(slave['displayName']) + jenkins_slave.status = get_slave_status(slave) + + # if this is a new slave and a pod, check if there is a resource for it, create one if not + if created and 'pod' in slave['displayName']: + # parse resource name from slave name + # naming example: orange-pod1, resource name: Orange POD 1 + tokens = slave['displayName'].split('-') + name = tokens[0].capitalize() + ' POD '# company name + name += tokens[1][3:] # remove 'pod' + resource, created = Resource.objects.get_or_create(name=name) + resource.slave = jenkins_slave + resource.save() + + last_job = get_jenkins_job(jenkins_slave.name) + if last_job is not None: + last_job = parse_job(last_job) + jenkins_slave.last_job_name = last_job['name'] + jenkins_slave.last_job_url = last_job['url'] + jenkins_slave.last_job_scenario = last_job['scenario'] + jenkins_slave.last_job_branch = last_job['branch'] + jenkins_slave.last_job_installer = last_job['installer'] + jenkins_slave.last_job_result = last_job['result'] + jenkins_slave.save() + + jenkins_statistic = JenkinsStatistic(slave=jenkins_slave) + if jenkins_slave.status == 'online' or jenkins_slave.status == 'building': + jenkins_statistic.online = True + if jenkins_slave.status == 'offline': + jenkins_statistic.offline = True + if jenkins_slave.status == 'online / idle': + jenkins_statistic.idle = True + jenkins_statistic.save() diff --git a/src/jenkins/tests.py b/src/jenkins/tests.py new file mode 100644 index 0000000..3723cd3 --- /dev/null +++ b/src/jenkins/tests.py @@ -0,0 +1,129 @@ +############################################################################## +# Copyright (c) 2016 Max Breitenfeldt and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from datetime import timedelta +from unittest import TestCase + +import jenkins.adapter as jenkins +from jenkins.models import * + + +# Tests that the data we get with the jenkinsadapter contains all the +# data we need. These test will fail if; +# - there is no internet connection +# - the opnfv jenkins url has changed +# - the jenkins api has changed +# - jenkins is not set up / there is no data +class JenkinsAdapterTestCase(TestCase): + def test_get_all_slaves(self): + slaves = jenkins.get_all_slaves() + self.assertTrue(len(slaves) > 0) + for slave in slaves: + self.assertTrue('displayName' in slave) + self.assertTrue('idle' in slave) + self.assertTrue('offline' in slave) + + def test_get_slave(self): + slaves = jenkins.get_all_slaves() + self.assertEqual(slaves[0], jenkins.get_slave(slaves[0]['displayName'])) + self.assertEqual({}, jenkins.get_slave('098f6bcd4621d373cade4e832627b4f6')) + + def test_get_ci_slaves(self): + slaves = jenkins.get_ci_slaves() + self.assertTrue(len(slaves) > 0) + for slave in slaves: + self.assertTrue('nodeName' in slave) + + def test_get_jenkins_job(self): + slaves = jenkins.get_ci_slaves() + job = None + for slave in slaves: + job = jenkins.get_jenkins_job(slave['nodeName']) + if job is not None: + break + # We need to test at least one job + self.assertNotEqual(job, None) + + def test_get_all_jobs(self): + jobs = jenkins.get_all_jobs() + lastBuild = False + self.assertTrue(len(jobs) > 0) + for job in jobs: + self.assertTrue('displayName' in job) + self.assertTrue('url' in job) + self.assertTrue('lastBuild' in job) + if job['lastBuild'] is not None: + lastBuild = True + self.assertTrue('building' in job['lastBuild']) + self.assertTrue('fullDisplayName' in job['lastBuild']) + self.assertTrue('result' in job['lastBuild']) + self.assertTrue('timestamp' in job['lastBuild']) + self.assertTrue('builtOn' in job['lastBuild']) + self.assertTrue(lastBuild) + + def test_parse_job(self): + job = { + "displayName": "apex-deploy-baremetal-os-nosdn-fdio-noha-colorado", + "url": "https://build.opnfv.org/ci/job/apex-deploy-baremetal-os-nosdn-fdio-noha-colorado/", + "lastBuild": { + "building": False, + "fullDisplayName": "apex-deploy-baremetal-os-nosdn-fdio-noha-colorado #37", + "result": "SUCCESS", + "timestamp": 1476283629917, + "builtOn": "lf-pod1" + } + } + + job = jenkins.parse_job(job) + self.assertEqual(job['scenario'], 'os-nosdn-fdio-noha') + self.assertEqual(job['installer'], 'apex') + self.assertEqual(job['branch'], 'colorado') + self.assertEqual(job['result'], 'SUCCESS') + self.assertEqual(job['building'], False) + self.assertEqual(job['url'], + "https://build.opnfv.org/ci/job/apex-deploy-baremetal-os-nosdn-fdio-noha-colorado/") + self.assertEqual(job['name'], + 'apex-deploy-baremetal-os-nosdn-fdio-noha-colorado') + + def test_get_slave_status(self): + slave = { + 'offline': True, + 'idle': False + } + self.assertEqual(jenkins.get_slave_status(slave), 'offline') + slave = { + 'offline': False, + 'idle': False + } + self.assertEqual(jenkins.get_slave_status(slave), 'online') + slave = { + 'offline': False, + 'idle': True + } + self.assertEqual(jenkins.get_slave_status(slave), 'online / idle') + + +class JenkinsModelTestCase(TestCase): + def test_get_utilization(self): + jenkins_slave = JenkinsSlave.objects.create(name='test', status='offline', url='') + utilization = jenkins_slave.get_utilization(timedelta(weeks=1)) + self.assertEqual(utilization['idle'], 0) + self.assertEqual(utilization['offline'], 0) + self.assertEqual(utilization['online'], 0) + + for i in range(10): + JenkinsStatistic.objects.create(slave=jenkins_slave, + offline=True, idle=True, + online=True) + + utilization = jenkins_slave.get_utilization(timedelta(weeks=1)) + self.assertEqual(utilization['idle'], 10) + self.assertEqual(utilization['offline'], 10) + self.assertEqual(utilization['online'], 10) |