aboutsummaryrefslogtreecommitdiffstats
path: root/src/jenkins
diff options
context:
space:
mode:
authorParker Berberian <pberberian@iol.unh.edu>2018-10-10 16:06:47 -0400
committerParker Berberian <pberberian@iol.unh.edu>2018-10-15 13:16:11 -0400
commit1f3a770d2547848590f39e9d9b9bdffeb94eec14 (patch)
tree97222e5facd1a242d951c38482315057b5790d51 /src/jenkins
parent6d4019e59eda897384e9c00d1daf8b2ce87d128f (diff)
Lab as a Service 2.0
See changes here: https://wiki.opnfv.org/display/INF/Pharos+Laas Change-Id: I59ada5f98e70a28d7f8c14eab3239597e236ca26 Signed-off-by: Sawyer Bergeron <sbergeron@iol.unh.edu> Signed-off-by: Parker Berberian <pberberian@iol.unh.edu>
Diffstat (limited to 'src/jenkins')
-rw-r--r--src/jenkins/__init__.py10
-rw-r--r--src/jenkins/adapter.py137
-rw-r--r--src/jenkins/admin.py17
-rw-r--r--src/jenkins/apps.py15
-rw-r--r--src/jenkins/migrations/0001_initial.py53
-rw-r--r--src/jenkins/migrations/__init__.py10
-rw-r--r--src/jenkins/models.py62
-rw-r--r--src/jenkins/tasks.py64
-rw-r--r--src/jenkins/tests.py129
9 files changed, 0 insertions, 497 deletions
diff --git a/src/jenkins/__init__.py b/src/jenkins/__init__.py
deleted file mode 100644
index b5914ce..0000000
--- a/src/jenkins/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 Max Breitenfeldt and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
diff --git a/src/jenkins/adapter.py b/src/jenkins/adapter.py
deleted file mode 100644
index b48b868..0000000
--- a/src/jenkins/adapter.py
+++ /dev/null
@@ -1,137 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 Max Breitenfeldt and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-import logging
-import re
-from django.conf import settings
-
-import requests
-from django.core.cache import cache
-
-logger = logging.getLogger(__name__)
-
-# TODO: implement caching decorator, cache get_* functions
-def get_json(url):
- if cache.get(url) is None:
- try:
- response = requests.get(url)
- json = response.json()
- cache.set(url, json, 180) # cache result for 180 seconds
- return json
- except requests.exceptions.RequestException as e:
- logger.exception(e)
- except ValueError as e:
- logger.exception(e)
- else:
- return cache.get(url)
-
-
-def get_all_slaves():
- url = settings.ALL_SLAVES_URL
- json = get_json(url)
- if json is not None:
- return json['computer'] # return list of dictionaries
- return []
-
-
-
-
-def get_slave(slavename):
- slaves = get_all_slaves()
- for slave in slaves:
- if slave['displayName'] == slavename:
- return slave
- return {}
-
-
-def get_ci_slaves():
- url = settings.CI_SLAVES_URL
- json = get_json(url)
- if json is not None:
- return json['nodes']
- return []
-
-
-def get_all_jobs():
- url = settings.ALL_JOBS_URL
- json = get_json(url)
- if json is not None:
- return json['jobs'] # return list of dictionaries
- return []
-
-
-def get_jenkins_job(slavename):
- jobs = get_all_jobs()
- max_time = 0
- last_job = None
- for job in jobs:
- if job['lastBuild'] is not None:
- if job['lastBuild']['builtOn'] == slavename:
- if job['lastBuild']['building'] is True:
- return job # return active build
- if job['lastBuild']['timestamp'] > max_time:
- last_job = job
- max_time = job['lastBuild']['timestamp']
- return last_job
-
-
-def is_ci_slave(slavename):
- ci_slaves = get_ci_slaves()
- for ci_slave in ci_slaves:
- if ci_slave['nodeName'] == slavename:
- return True
- return False
-
-
-def is_dev_pod(slavename):
- if is_ci_slave(slavename):
- return False
- if slavename.find('pod') != -1:
- return True
- return False
-
-
-def parse_job(job):
- result = parse_job_string(job['lastBuild']['fullDisplayName'])
- result['building'] = job['lastBuild']['building']
- result['result'] = ''
- if not job['lastBuild']['building']:
- result['result'] = job['lastBuild']['result']
- result['url'] = job['url']
- return result
-
-
-def parse_job_string(full_displayname):
- job = {}
- job['scenario'] = ''
- job['installer'] = ''
- job['branch'] = ''
- tokens = re.split(r'[ -]', full_displayname)
- for i in range(len(tokens)):
- if tokens[i] == 'os':
- job['scenario'] = '-'.join(tokens[i: i + 4])
- elif tokens[i] in ['fuel', 'joid', 'apex', 'compass']:
- job['installer'] = tokens[i]
- elif tokens[i] in ['master', 'arno', 'brahmaputra', 'colorado']:
- job['branch'] = tokens[i]
- tokens = full_displayname.split(' ')
- job['name'] = tokens[0]
- return job
-
-def get_slave_url(slave):
- return settings.GET_SLAVE_URL + slave['displayName']
-
-
-def get_slave_status(slave):
- if not slave['offline'] and slave['idle']:
- return 'online / idle'
- if not slave['offline']:
- return 'online'
- return 'offline'
diff --git a/src/jenkins/admin.py b/src/jenkins/admin.py
deleted file mode 100644
index c499670..0000000
--- a/src/jenkins/admin.py
+++ /dev/null
@@ -1,17 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 Max Breitenfeldt and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-from django.conf import settings
-from django.contrib import admin
-
-from jenkins.models import JenkinsSlave
-
-if settings.DEBUG:
- admin.site.register(JenkinsSlave) \ No newline at end of file
diff --git a/src/jenkins/apps.py b/src/jenkins/apps.py
deleted file mode 100644
index 41faf60..0000000
--- a/src/jenkins/apps.py
+++ /dev/null
@@ -1,15 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 Max Breitenfeldt and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-from django.apps import AppConfig
-
-
-class JenkinsConfig(AppConfig):
- name = 'jenkins'
diff --git a/src/jenkins/migrations/0001_initial.py b/src/jenkins/migrations/0001_initial.py
deleted file mode 100644
index b1c7889..0000000
--- a/src/jenkins/migrations/0001_initial.py
+++ /dev/null
@@ -1,53 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by Django 1.10 on 2016-11-03 13:33
-from __future__ import unicode_literals
-
-from django.db import migrations, models
-import django.db.models.deletion
-
-
-class Migration(migrations.Migration):
-
- initial = True
-
- dependencies = [
- ]
-
- operations = [
- migrations.CreateModel(
- name='JenkinsSlave',
- fields=[
- ('id', models.AutoField(primary_key=True, serialize=False)),
- ('name', models.CharField(max_length=100, unique=True)),
- ('status', models.CharField(default='offline', max_length=30)),
- ('url', models.CharField(max_length=1024)),
- ('ci_slave', models.BooleanField(default=False)),
- ('dev_pod', models.BooleanField(default=False)),
- ('building', models.BooleanField(default=False)),
- ('last_job_name', models.CharField(default='', max_length=1024)),
- ('last_job_url', models.CharField(default='', max_length=1024)),
- ('last_job_scenario', models.CharField(default='', max_length=50)),
- ('last_job_branch', models.CharField(default='', max_length=50)),
- ('last_job_installer', models.CharField(default='', max_length=50)),
- ('last_job_result', models.CharField(default='', max_length=30)),
- ('active', models.BooleanField(default=False)),
- ],
- options={
- 'db_table': 'jenkins_slave',
- },
- ),
- migrations.CreateModel(
- name='JenkinsStatistic',
- fields=[
- ('id', models.AutoField(primary_key=True, serialize=False)),
- ('offline', models.BooleanField(default=False)),
- ('idle', models.BooleanField(default=False)),
- ('online', models.BooleanField(default=False)),
- ('timestamp', models.DateTimeField(auto_now_add=True)),
- ('slave', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='jenkins.JenkinsSlave')),
- ],
- options={
- 'db_table': 'jenkins_statistic',
- },
- ),
- ]
diff --git a/src/jenkins/migrations/__init__.py b/src/jenkins/migrations/__init__.py
deleted file mode 100644
index b5914ce..0000000
--- a/src/jenkins/migrations/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 Max Breitenfeldt and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
diff --git a/src/jenkins/models.py b/src/jenkins/models.py
deleted file mode 100644
index 8254ff3..0000000
--- a/src/jenkins/models.py
+++ /dev/null
@@ -1,62 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 Max Breitenfeldt and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-from django.db import models
-from django.utils import timezone
-
-
-class JenkinsSlave(models.Model):
- id = models.AutoField(primary_key=True)
- name = models.CharField(max_length=100, unique=True)
- status = models.CharField(max_length=30, default='offline')
- url = models.CharField(max_length=1024)
- ci_slave = models.BooleanField(default=False)
- dev_pod = models.BooleanField(default=False)
-
- building = models.BooleanField(default=False)
-
- last_job_name = models.CharField(max_length=1024, default='')
- last_job_url = models.CharField(max_length=1024, default='')
- last_job_scenario = models.CharField(max_length=50, default='')
- last_job_branch = models.CharField(max_length=50, default='')
- last_job_installer = models.CharField(max_length=50, default='')
- last_job_result = models.CharField(max_length=30, default='')
-
- active = models.BooleanField(default=False)
-
- def get_utilization(self, timedelta):
- """
- Return a dictionary containing the count of idle, online and offline measurements in the time from
- now-timedelta to now
- """
- utilization = {'idle': 0, 'online': 0, 'offline': 0}
- statistics = self.jenkinsstatistic_set.filter(timestamp__gte=timezone.now() - timedelta)
- utilization['idle'] = statistics.filter(idle=True).count()
- utilization['online'] = statistics.filter(online=True).count()
- utilization['offline'] = statistics.filter(offline=True).count()
- return utilization
-
- class Meta:
- db_table = 'jenkins_slave'
-
- def __str__(self):
- return self.name
-
-
-class JenkinsStatistic(models.Model):
- id = models.AutoField(primary_key=True)
- slave = models.ForeignKey(JenkinsSlave, on_delete=models.CASCADE)
- offline = models.BooleanField(default=False)
- idle = models.BooleanField(default=False)
- online = models.BooleanField(default=False)
- timestamp = models.DateTimeField(auto_now_add=True)
-
- class Meta:
- db_table = 'jenkins_statistic'
diff --git a/src/jenkins/tasks.py b/src/jenkins/tasks.py
deleted file mode 100644
index ea986c1..0000000
--- a/src/jenkins/tasks.py
+++ /dev/null
@@ -1,64 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 Max Breitenfeldt and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-from celery import shared_task
-
-from dashboard.models import Resource
-from jenkins.models import JenkinsSlave, JenkinsStatistic
-from .adapter import *
-
-
-@shared_task
-def sync_jenkins():
- update_jenkins_slaves()
-
-
-def update_jenkins_slaves():
- JenkinsSlave.objects.all().update(active=False)
-
- jenkins_slaves = get_all_slaves()
- for slave in jenkins_slaves:
- jenkins_slave, created = JenkinsSlave.objects.get_or_create(name=slave['displayName'],
- url=get_slave_url(slave))
- jenkins_slave.active = True
- jenkins_slave.ci_slave = is_ci_slave(slave['displayName'])
- jenkins_slave.dev_pod = is_dev_pod(slave['displayName'])
- jenkins_slave.status = get_slave_status(slave)
-
- # if this is a new slave and a pod, check if there is a resource for it, create one if not
- if created and 'pod' in slave['displayName']:
- # parse resource name from slave name
- # naming example: orange-pod1, resource name: Orange POD 1
- tokens = slave['displayName'].split('-')
- name = tokens[0].capitalize() + ' POD '# company name
- name += tokens[1][3:] # remove 'pod'
- resource, created = Resource.objects.get_or_create(name=name)
- resource.slave = jenkins_slave
- resource.save()
-
- last_job = get_jenkins_job(jenkins_slave.name)
- if last_job is not None:
- last_job = parse_job(last_job)
- jenkins_slave.last_job_name = last_job['name']
- jenkins_slave.last_job_url = last_job['url']
- jenkins_slave.last_job_scenario = last_job['scenario']
- jenkins_slave.last_job_branch = last_job['branch']
- jenkins_slave.last_job_installer = last_job['installer']
- jenkins_slave.last_job_result = last_job['result']
- jenkins_slave.save()
-
- jenkins_statistic = JenkinsStatistic(slave=jenkins_slave)
- if jenkins_slave.status == 'online' or jenkins_slave.status == 'building':
- jenkins_statistic.online = True
- if jenkins_slave.status == 'offline':
- jenkins_statistic.offline = True
- if jenkins_slave.status == 'online / idle':
- jenkins_statistic.idle = True
- jenkins_statistic.save()
diff --git a/src/jenkins/tests.py b/src/jenkins/tests.py
deleted file mode 100644
index 3723cd3..0000000
--- a/src/jenkins/tests.py
+++ /dev/null
@@ -1,129 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 Max Breitenfeldt and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-
-from datetime import timedelta
-from unittest import TestCase
-
-import jenkins.adapter as jenkins
-from jenkins.models import *
-
-
-# Tests that the data we get with the jenkinsadapter contains all the
-# data we need. These test will fail if;
-# - there is no internet connection
-# - the opnfv jenkins url has changed
-# - the jenkins api has changed
-# - jenkins is not set up / there is no data
-class JenkinsAdapterTestCase(TestCase):
- def test_get_all_slaves(self):
- slaves = jenkins.get_all_slaves()
- self.assertTrue(len(slaves) > 0)
- for slave in slaves:
- self.assertTrue('displayName' in slave)
- self.assertTrue('idle' in slave)
- self.assertTrue('offline' in slave)
-
- def test_get_slave(self):
- slaves = jenkins.get_all_slaves()
- self.assertEqual(slaves[0], jenkins.get_slave(slaves[0]['displayName']))
- self.assertEqual({}, jenkins.get_slave('098f6bcd4621d373cade4e832627b4f6'))
-
- def test_get_ci_slaves(self):
- slaves = jenkins.get_ci_slaves()
- self.assertTrue(len(slaves) > 0)
- for slave in slaves:
- self.assertTrue('nodeName' in slave)
-
- def test_get_jenkins_job(self):
- slaves = jenkins.get_ci_slaves()
- job = None
- for slave in slaves:
- job = jenkins.get_jenkins_job(slave['nodeName'])
- if job is not None:
- break
- # We need to test at least one job
- self.assertNotEqual(job, None)
-
- def test_get_all_jobs(self):
- jobs = jenkins.get_all_jobs()
- lastBuild = False
- self.assertTrue(len(jobs) > 0)
- for job in jobs:
- self.assertTrue('displayName' in job)
- self.assertTrue('url' in job)
- self.assertTrue('lastBuild' in job)
- if job['lastBuild'] is not None:
- lastBuild = True
- self.assertTrue('building' in job['lastBuild'])
- self.assertTrue('fullDisplayName' in job['lastBuild'])
- self.assertTrue('result' in job['lastBuild'])
- self.assertTrue('timestamp' in job['lastBuild'])
- self.assertTrue('builtOn' in job['lastBuild'])
- self.assertTrue(lastBuild)
-
- def test_parse_job(self):
- job = {
- "displayName": "apex-deploy-baremetal-os-nosdn-fdio-noha-colorado",
- "url": "https://build.opnfv.org/ci/job/apex-deploy-baremetal-os-nosdn-fdio-noha-colorado/",
- "lastBuild": {
- "building": False,
- "fullDisplayName": "apex-deploy-baremetal-os-nosdn-fdio-noha-colorado #37",
- "result": "SUCCESS",
- "timestamp": 1476283629917,
- "builtOn": "lf-pod1"
- }
- }
-
- job = jenkins.parse_job(job)
- self.assertEqual(job['scenario'], 'os-nosdn-fdio-noha')
- self.assertEqual(job['installer'], 'apex')
- self.assertEqual(job['branch'], 'colorado')
- self.assertEqual(job['result'], 'SUCCESS')
- self.assertEqual(job['building'], False)
- self.assertEqual(job['url'],
- "https://build.opnfv.org/ci/job/apex-deploy-baremetal-os-nosdn-fdio-noha-colorado/")
- self.assertEqual(job['name'],
- 'apex-deploy-baremetal-os-nosdn-fdio-noha-colorado')
-
- def test_get_slave_status(self):
- slave = {
- 'offline': True,
- 'idle': False
- }
- self.assertEqual(jenkins.get_slave_status(slave), 'offline')
- slave = {
- 'offline': False,
- 'idle': False
- }
- self.assertEqual(jenkins.get_slave_status(slave), 'online')
- slave = {
- 'offline': False,
- 'idle': True
- }
- self.assertEqual(jenkins.get_slave_status(slave), 'online / idle')
-
-
-class JenkinsModelTestCase(TestCase):
- def test_get_utilization(self):
- jenkins_slave = JenkinsSlave.objects.create(name='test', status='offline', url='')
- utilization = jenkins_slave.get_utilization(timedelta(weeks=1))
- self.assertEqual(utilization['idle'], 0)
- self.assertEqual(utilization['offline'], 0)
- self.assertEqual(utilization['online'], 0)
-
- for i in range(10):
- JenkinsStatistic.objects.create(slave=jenkins_slave,
- offline=True, idle=True,
- online=True)
-
- utilization = jenkins_slave.get_utilization(timedelta(weeks=1))
- self.assertEqual(utilization['idle'], 10)
- self.assertEqual(utilization['offline'], 10)
- self.assertEqual(utilization['online'], 10)