diff options
Diffstat (limited to 'dashboard/src/jenkins')
| -rw-r--r-- | dashboard/src/jenkins/__init__.py | 10 | ||||
| -rw-r--r-- | dashboard/src/jenkins/adapter.py | 137 | ||||
| -rw-r--r-- | dashboard/src/jenkins/admin.py | 17 | ||||
| -rw-r--r-- | dashboard/src/jenkins/apps.py | 15 | ||||
| -rw-r--r-- | dashboard/src/jenkins/migrations/0001_initial.py | 53 | ||||
| -rw-r--r-- | dashboard/src/jenkins/migrations/__init__.py | 10 | ||||
| -rw-r--r-- | dashboard/src/jenkins/models.py | 62 | ||||
| -rw-r--r-- | dashboard/src/jenkins/tasks.py | 64 | ||||
| -rw-r--r-- | dashboard/src/jenkins/tests.py | 129 |
9 files changed, 0 insertions, 497 deletions
diff --git a/dashboard/src/jenkins/__init__.py b/dashboard/src/jenkins/__init__.py deleted file mode 100644 index b5914ce..0000000 --- a/dashboard/src/jenkins/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - diff --git a/dashboard/src/jenkins/adapter.py b/dashboard/src/jenkins/adapter.py deleted file mode 100644 index b48b868..0000000 --- a/dashboard/src/jenkins/adapter.py +++ /dev/null @@ -1,137 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -import logging -import re -from django.conf import settings - -import requests -from django.core.cache import cache - -logger = logging.getLogger(__name__) - -# TODO: implement caching decorator, cache get_* functions -def get_json(url): - if cache.get(url) is None: - try: - response = requests.get(url) - json = response.json() - cache.set(url, json, 180) # cache result for 180 seconds - return json - except requests.exceptions.RequestException as e: - logger.exception(e) - except ValueError as e: - logger.exception(e) - else: - return cache.get(url) - - -def get_all_slaves(): - url = settings.ALL_SLAVES_URL - json = get_json(url) - if json is not None: - return json['computer'] # return list of dictionaries - return [] - - - - -def get_slave(slavename): - slaves = get_all_slaves() - for slave in slaves: - if slave['displayName'] == slavename: - return slave - return {} - - -def get_ci_slaves(): - url = settings.CI_SLAVES_URL - json = get_json(url) - if json is not None: - return json['nodes'] - return [] - - -def get_all_jobs(): - url = settings.ALL_JOBS_URL - json = get_json(url) - if json is not None: - return json['jobs'] # return list of dictionaries - return [] - - -def get_jenkins_job(slavename): - jobs = get_all_jobs() - max_time = 0 - last_job = None - for job in jobs: - if job['lastBuild'] is not None: - if job['lastBuild']['builtOn'] == slavename: - if job['lastBuild']['building'] is True: - return job # return active build - if job['lastBuild']['timestamp'] > max_time: - last_job = job - max_time = job['lastBuild']['timestamp'] - return last_job - - -def is_ci_slave(slavename): - ci_slaves = get_ci_slaves() - for ci_slave in ci_slaves: - if ci_slave['nodeName'] == slavename: - return True - return False - - -def is_dev_pod(slavename): - if is_ci_slave(slavename): - return False - if slavename.find('pod') != -1: - return True - return False - - -def parse_job(job): - result = parse_job_string(job['lastBuild']['fullDisplayName']) - result['building'] = job['lastBuild']['building'] - result['result'] = '' - if not job['lastBuild']['building']: - result['result'] = job['lastBuild']['result'] - result['url'] = job['url'] - return result - - -def parse_job_string(full_displayname): - job = {} - job['scenario'] = '' - job['installer'] = '' - job['branch'] = '' - tokens = re.split(r'[ -]', full_displayname) - for i in range(len(tokens)): - if tokens[i] == 'os': - job['scenario'] = '-'.join(tokens[i: i + 4]) - elif tokens[i] in ['fuel', 'joid', 'apex', 'compass']: - job['installer'] = tokens[i] - elif tokens[i] in ['master', 'arno', 'brahmaputra', 'colorado']: - job['branch'] = tokens[i] - tokens = full_displayname.split(' ') - job['name'] = tokens[0] - return job - -def get_slave_url(slave): - return settings.GET_SLAVE_URL + slave['displayName'] - - -def get_slave_status(slave): - if not slave['offline'] and slave['idle']: - return 'online / idle' - if not slave['offline']: - return 'online' - return 'offline' diff --git a/dashboard/src/jenkins/admin.py b/dashboard/src/jenkins/admin.py deleted file mode 100644 index c499670..0000000 --- a/dashboard/src/jenkins/admin.py +++ /dev/null @@ -1,17 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from django.conf import settings -from django.contrib import admin - -from jenkins.models import JenkinsSlave - -if settings.DEBUG: - admin.site.register(JenkinsSlave)
\ No newline at end of file diff --git a/dashboard/src/jenkins/apps.py b/dashboard/src/jenkins/apps.py deleted file mode 100644 index 41faf60..0000000 --- a/dashboard/src/jenkins/apps.py +++ /dev/null @@ -1,15 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from django.apps import AppConfig - - -class JenkinsConfig(AppConfig): - name = 'jenkins' diff --git a/dashboard/src/jenkins/migrations/0001_initial.py b/dashboard/src/jenkins/migrations/0001_initial.py deleted file mode 100644 index b1c7889..0000000 --- a/dashboard/src/jenkins/migrations/0001_initial.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.10 on 2016-11-03 13:33 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='JenkinsSlave', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('name', models.CharField(max_length=100, unique=True)), - ('status', models.CharField(default='offline', max_length=30)), - ('url', models.CharField(max_length=1024)), - ('ci_slave', models.BooleanField(default=False)), - ('dev_pod', models.BooleanField(default=False)), - ('building', models.BooleanField(default=False)), - ('last_job_name', models.CharField(default='', max_length=1024)), - ('last_job_url', models.CharField(default='', max_length=1024)), - ('last_job_scenario', models.CharField(default='', max_length=50)), - ('last_job_branch', models.CharField(default='', max_length=50)), - ('last_job_installer', models.CharField(default='', max_length=50)), - ('last_job_result', models.CharField(default='', max_length=30)), - ('active', models.BooleanField(default=False)), - ], - options={ - 'db_table': 'jenkins_slave', - }, - ), - migrations.CreateModel( - name='JenkinsStatistic', - fields=[ - ('id', models.AutoField(primary_key=True, serialize=False)), - ('offline', models.BooleanField(default=False)), - ('idle', models.BooleanField(default=False)), - ('online', models.BooleanField(default=False)), - ('timestamp', models.DateTimeField(auto_now_add=True)), - ('slave', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jenkins.JenkinsSlave')), - ], - options={ - 'db_table': 'jenkins_statistic', - }, - ), - ] diff --git a/dashboard/src/jenkins/migrations/__init__.py b/dashboard/src/jenkins/migrations/__init__.py deleted file mode 100644 index b5914ce..0000000 --- a/dashboard/src/jenkins/migrations/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - diff --git a/dashboard/src/jenkins/models.py b/dashboard/src/jenkins/models.py deleted file mode 100644 index 8254ff3..0000000 --- a/dashboard/src/jenkins/models.py +++ /dev/null @@ -1,62 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from django.db import models -from django.utils import timezone - - -class JenkinsSlave(models.Model): - id = models.AutoField(primary_key=True) - name = models.CharField(max_length=100, unique=True) - status = models.CharField(max_length=30, default='offline') - url = models.CharField(max_length=1024) - ci_slave = models.BooleanField(default=False) - dev_pod = models.BooleanField(default=False) - - building = models.BooleanField(default=False) - - last_job_name = models.CharField(max_length=1024, default='') - last_job_url = models.CharField(max_length=1024, default='') - last_job_scenario = models.CharField(max_length=50, default='') - last_job_branch = models.CharField(max_length=50, default='') - last_job_installer = models.CharField(max_length=50, default='') - last_job_result = models.CharField(max_length=30, default='') - - active = models.BooleanField(default=False) - - def get_utilization(self, timedelta): - """ - Return a dictionary containing the count of idle, online and offline measurements in the time from - now-timedelta to now - """ - utilization = {'idle': 0, 'online': 0, 'offline': 0} - statistics = self.jenkinsstatistic_set.filter(timestamp__gte=timezone.now() - timedelta) - utilization['idle'] = statistics.filter(idle=True).count() - utilization['online'] = statistics.filter(online=True).count() - utilization['offline'] = statistics.filter(offline=True).count() - return utilization - - class Meta: - db_table = 'jenkins_slave' - - def __str__(self): - return self.name - - -class JenkinsStatistic(models.Model): - id = models.AutoField(primary_key=True) - slave = models.ForeignKey(JenkinsSlave, on_delete=models.CASCADE) - offline = models.BooleanField(default=False) - idle = models.BooleanField(default=False) - online = models.BooleanField(default=False) - timestamp = models.DateTimeField(auto_now_add=True) - - class Meta: - db_table = 'jenkins_statistic' diff --git a/dashboard/src/jenkins/tasks.py b/dashboard/src/jenkins/tasks.py deleted file mode 100644 index ea986c1..0000000 --- a/dashboard/src/jenkins/tasks.py +++ /dev/null @@ -1,64 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from celery import shared_task - -from dashboard.models import Resource -from jenkins.models import JenkinsSlave, JenkinsStatistic -from .adapter import * - - -@shared_task -def sync_jenkins(): - update_jenkins_slaves() - - -def update_jenkins_slaves(): - JenkinsSlave.objects.all().update(active=False) - - jenkins_slaves = get_all_slaves() - for slave in jenkins_slaves: - jenkins_slave, created = JenkinsSlave.objects.get_or_create(name=slave['displayName'], - url=get_slave_url(slave)) - jenkins_slave.active = True - jenkins_slave.ci_slave = is_ci_slave(slave['displayName']) - jenkins_slave.dev_pod = is_dev_pod(slave['displayName']) - jenkins_slave.status = get_slave_status(slave) - - # if this is a new slave and a pod, check if there is a resource for it, create one if not - if created and 'pod' in slave['displayName']: - # parse resource name from slave name - # naming example: orange-pod1, resource name: Orange POD 1 - tokens = slave['displayName'].split('-') - name = tokens[0].capitalize() + ' POD '# company name - name += tokens[1][3:] # remove 'pod' - resource, created = Resource.objects.get_or_create(name=name) - resource.slave = jenkins_slave - resource.save() - - last_job = get_jenkins_job(jenkins_slave.name) - if last_job is not None: - last_job = parse_job(last_job) - jenkins_slave.last_job_name = last_job['name'] - jenkins_slave.last_job_url = last_job['url'] - jenkins_slave.last_job_scenario = last_job['scenario'] - jenkins_slave.last_job_branch = last_job['branch'] - jenkins_slave.last_job_installer = last_job['installer'] - jenkins_slave.last_job_result = last_job['result'] - jenkins_slave.save() - - jenkins_statistic = JenkinsStatistic(slave=jenkins_slave) - if jenkins_slave.status == 'online' or jenkins_slave.status == 'building': - jenkins_statistic.online = True - if jenkins_slave.status == 'offline': - jenkins_statistic.offline = True - if jenkins_slave.status == 'online / idle': - jenkins_statistic.idle = True - jenkins_statistic.save() diff --git a/dashboard/src/jenkins/tests.py b/dashboard/src/jenkins/tests.py deleted file mode 100644 index 3723cd3..0000000 --- a/dashboard/src/jenkins/tests.py +++ /dev/null @@ -1,129 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Max Breitenfeldt and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -from datetime import timedelta -from unittest import TestCase - -import jenkins.adapter as jenkins -from jenkins.models import * - - -# Tests that the data we get with the jenkinsadapter contains all the -# data we need. These test will fail if; -# - there is no internet connection -# - the opnfv jenkins url has changed -# - the jenkins api has changed -# - jenkins is not set up / there is no data -class JenkinsAdapterTestCase(TestCase): - def test_get_all_slaves(self): - slaves = jenkins.get_all_slaves() - self.assertTrue(len(slaves) > 0) - for slave in slaves: - self.assertTrue('displayName' in slave) - self.assertTrue('idle' in slave) - self.assertTrue('offline' in slave) - - def test_get_slave(self): - slaves = jenkins.get_all_slaves() - self.assertEqual(slaves[0], jenkins.get_slave(slaves[0]['displayName'])) - self.assertEqual({}, jenkins.get_slave('098f6bcd4621d373cade4e832627b4f6')) - - def test_get_ci_slaves(self): - slaves = jenkins.get_ci_slaves() - self.assertTrue(len(slaves) > 0) - for slave in slaves: - self.assertTrue('nodeName' in slave) - - def test_get_jenkins_job(self): - slaves = jenkins.get_ci_slaves() - job = None - for slave in slaves: - job = jenkins.get_jenkins_job(slave['nodeName']) - if job is not None: - break - # We need to test at least one job - self.assertNotEqual(job, None) - - def test_get_all_jobs(self): - jobs = jenkins.get_all_jobs() - lastBuild = False - self.assertTrue(len(jobs) > 0) - for job in jobs: - self.assertTrue('displayName' in job) - self.assertTrue('url' in job) - self.assertTrue('lastBuild' in job) - if job['lastBuild'] is not None: - lastBuild = True - self.assertTrue('building' in job['lastBuild']) - self.assertTrue('fullDisplayName' in job['lastBuild']) - self.assertTrue('result' in job['lastBuild']) - self.assertTrue('timestamp' in job['lastBuild']) - self.assertTrue('builtOn' in job['lastBuild']) - self.assertTrue(lastBuild) - - def test_parse_job(self): - job = { - "displayName": "apex-deploy-baremetal-os-nosdn-fdio-noha-colorado", - "url": "https://build.opnfv.org/ci/job/apex-deploy-baremetal-os-nosdn-fdio-noha-colorado/", - "lastBuild": { - "building": False, - "fullDisplayName": "apex-deploy-baremetal-os-nosdn-fdio-noha-colorado #37", - "result": "SUCCESS", - "timestamp": 1476283629917, - "builtOn": "lf-pod1" - } - } - - job = jenkins.parse_job(job) - self.assertEqual(job['scenario'], 'os-nosdn-fdio-noha') - self.assertEqual(job['installer'], 'apex') - self.assertEqual(job['branch'], 'colorado') - self.assertEqual(job['result'], 'SUCCESS') - self.assertEqual(job['building'], False) - self.assertEqual(job['url'], - "https://build.opnfv.org/ci/job/apex-deploy-baremetal-os-nosdn-fdio-noha-colorado/") - self.assertEqual(job['name'], - 'apex-deploy-baremetal-os-nosdn-fdio-noha-colorado') - - def test_get_slave_status(self): - slave = { - 'offline': True, - 'idle': False - } - self.assertEqual(jenkins.get_slave_status(slave), 'offline') - slave = { - 'offline': False, - 'idle': False - } - self.assertEqual(jenkins.get_slave_status(slave), 'online') - slave = { - 'offline': False, - 'idle': True - } - self.assertEqual(jenkins.get_slave_status(slave), 'online / idle') - - -class JenkinsModelTestCase(TestCase): - def test_get_utilization(self): - jenkins_slave = JenkinsSlave.objects.create(name='test', status='offline', url='') - utilization = jenkins_slave.get_utilization(timedelta(weeks=1)) - self.assertEqual(utilization['idle'], 0) - self.assertEqual(utilization['offline'], 0) - self.assertEqual(utilization['online'], 0) - - for i in range(10): - JenkinsStatistic.objects.create(slave=jenkins_slave, - offline=True, idle=True, - online=True) - - utilization = jenkins_slave.get_utilization(timedelta(weeks=1)) - self.assertEqual(utilization['idle'], 10) - self.assertEqual(utilization['offline'], 10) - self.assertEqual(utilization['online'], 10) |