From 971df12c95bfa17459d042857e5200f4a7f60cd4 Mon Sep 17 00:00:00 2001 From: akhilbatra898 Date: Tue, 25 Jul 2017 03:30:18 +0530 Subject: Implement benchmark running and logging the output - Run benchmark by adding new tasks - Modify Task model to have log as log files - Add new urls for respective views - Module for running benchmarks in background - Updating status of tasks - Listing and detailed log of all tasks - Add corresponding html templates Change-Id: I244d1bb74949eeb470c738363f6917191e0f052d Signed-off-by: akhilbatra898 --- qtip/web/bench/forms.py | 19 +++++++ .../bench/migrations/0005_auto_20170720_2115.py | 35 ++++++++++++ .../bench/migrations/0006_auto_20170722_0135.py | 25 +++++++++ qtip/web/bench/models.py | 26 +++++++-- qtip/web/bench/urls.py | 3 ++ qtip/web/bench/utils.py | 27 ++++++++++ qtip/web/bench/views.py | 63 ++++++++++++++++++++-- qtip/web/templates/bench/run.html | 15 ++++++ qtip/web/templates/bench/task_detail.html | 17 ++++++ qtip/web/templates/bench/task_list.html | 22 ++++++++ 10 files changed, 243 insertions(+), 9 deletions(-) create mode 100644 qtip/web/bench/forms.py create mode 100644 qtip/web/bench/migrations/0005_auto_20170720_2115.py create mode 100644 qtip/web/bench/migrations/0006_auto_20170722_0135.py create mode 100644 qtip/web/bench/utils.py create mode 100644 qtip/web/templates/bench/run.html create mode 100644 qtip/web/templates/bench/task_detail.html create mode 100644 qtip/web/templates/bench/task_list.html (limited to 'qtip') diff --git a/qtip/web/bench/forms.py b/qtip/web/bench/forms.py new file mode 100644 index 00000000..d897aca7 --- /dev/null +++ b/qtip/web/bench/forms.py @@ -0,0 +1,19 @@ +############################################################################## +# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +from django import forms + +import models + + +class TaskForm(forms.ModelForm): + class Meta: + model = models.Task + fields = ['repo'] diff --git a/qtip/web/bench/migrations/0005_auto_20170720_2115.py b/qtip/web/bench/migrations/0005_auto_20170720_2115.py new file mode 100644 index 00000000..5bd81a2c --- /dev/null +++ b/qtip/web/bench/migrations/0005_auto_20170720_2115.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.3 on 2017-07-20 21:15 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bench', '0004_auto_20170715_0325'), + ] + + operations = [ + migrations.RenameField( + model_name='repo', + old_name='github_link', + new_name='git_link', + ), + migrations.AlterField( + model_name='task', + name='end_time', + field=models.DateTimeField(null=True), + ), + migrations.AlterField( + model_name='task', + name='log', + field=models.FilePathField(), + ), + migrations.AlterField( + model_name='task', + name='run_time', + field=models.DurationField(null=True), + ), + ] diff --git a/qtip/web/bench/migrations/0006_auto_20170722_0135.py b/qtip/web/bench/migrations/0006_auto_20170722_0135.py new file mode 100644 index 00000000..5d066702 --- /dev/null +++ b/qtip/web/bench/migrations/0006_auto_20170722_0135.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.3 on 2017-07-22 01:35 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('bench', '0005_auto_20170720_2115'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='status', + field=models.CharField(choices=[('P', 'Pending'), ('IP', 'In progress'), ('F', 'Finished')], default='P', max_length=20), + ), + migrations.AlterField( + model_name='task', + name='log', + field=models.FileField(upload_to='logs'), + ), + ] diff --git a/qtip/web/bench/models.py b/qtip/web/bench/models.py index 863af739..3f0439d9 100644 --- a/qtip/web/bench/models.py +++ b/qtip/web/bench/models.py @@ -18,15 +18,33 @@ from django.urls import reverse class Repo(models.Model): name = models.CharField(max_length=200, blank=False) - github_link = models.URLField(unique=True) + git_link = models.URLField(unique=True) def get_absolute_url(self): return reverse('repo_update', args=[self.pk]) + def __str__(self): + return "%s, %s" % (self.name, self.git_link) + class Task(models.Model): + TASK_STATUS_CHOICES = ( + ('P', 'Pending'), + ('IP', 'In progress'), + ('F', 'Finished') + ) + start_time = models.DateTimeField(auto_now_add=True) - end_time = models.DateTimeField() - run_time = models.DurationField() + status = models.CharField(choices=TASK_STATUS_CHOICES, default='P', max_length=20) + end_time = models.DateTimeField(null=True) + run_time = models.DurationField(null=True) repo = models.ForeignKey('Repo', on_delete=models.DO_NOTHING) - log = models.TextField() + log = models.FileField(upload_to='logs') + + def save(self, **kwargs): + if self.end_time: + self.run_time = self.end_time - self.start_time + super(Task, self).save(kwargs) + + def get_absolute_url(self): + return reverse('task_view', args=[self.pk]) diff --git a/qtip/web/bench/urls.py b/qtip/web/bench/urls.py index 3af4eb83..ae9738b6 100644 --- a/qtip/web/bench/urls.py +++ b/qtip/web/bench/urls.py @@ -18,4 +18,7 @@ urlpatterns = [ url('^', include('django.contrib.auth.urls')), url('^repos/$', views.ReposView.as_view(), name='repos'), url('^repos/(?P\d+)$', views.RepoUpdate.as_view(), name='repo_update'), + url('^run/$', views.Run.as_view(), name='run'), + url('^tasks/$', views.Logs.as_view(), name='tasks'), + url('^tasks/(?P\d+)$', views.TaskView.as_view(), name='task_view'), ] diff --git a/qtip/web/bench/utils.py b/qtip/web/bench/utils.py new file mode 100644 index 00000000..aac388e0 --- /dev/null +++ b/qtip/web/bench/utils.py @@ -0,0 +1,27 @@ +############################################################################## +# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import os +from subprocess import Popen, PIPE, STDOUT +import shutil + + +def run(repo): + if os.path.exists(repo.name): + shutil.rmtree(repo.name) + os.mkdir(repo.name) + os.chdir(repo.name) + output = Popen(("git clone %s ." % repo.git_link).split(), stdout=PIPE, stderr=STDOUT) + for line in output.stdout: + yield line + output.wait() + output = Popen("ansible-playbook run.yml".split(), stdout=PIPE, stderr=STDOUT) + for line in output.stdout: + yield line + os.chdir("../") diff --git a/qtip/web/bench/views.py b/qtip/web/bench/views.py index 786b67d5..6de5e4cc 100644 --- a/qtip/web/bench/views.py +++ b/qtip/web/bench/views.py @@ -9,16 +9,20 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals +from multiprocessing import Process from django.contrib.auth.mixins import LoginRequiredMixin -# from django.utils.decorators import method_decorator from django.views.generic.edit import CreateView, UpdateView +from django.views.generic.list import ListView +from django.views.generic.detail import DetailView +from django.views import View +from django.shortcuts import render, redirect +from django.core.files.base import ContentFile +from django.utils import timezone +import forms import models - -# from django.shortcuts import render - -# Create your views here. +import utils class ReposView(LoginRequiredMixin, CreateView): @@ -39,3 +43,52 @@ class RepoUpdate(LoginRequiredMixin, UpdateView): context = super(RepoUpdate, self).get_context_data(**kwargs) context["repos"] = self.model.objects.all() return context + + +class Run(LoginRequiredMixin, View): + template_name = 'bench/run.html' + form_class = forms.TaskForm + + def get(self, request): + task_form = self.form_class() + return render(request, self.template_name, {'form': task_form}) + + def post(self, request): + task_form = self.form_class(request.POST) + if task_form.is_valid(): + new_task = task_form.save() + new_task.log.save("run_%s.log" % new_task.pk, ContentFile('')) + p = Process(target=self.start_task, args=(new_task,)) + p.start() + return redirect('tasks') + p.join() + + def start_task(self, task): + task = models.Task.objects.get(pk=task.pk) + task.status = 'IP' + task.save() + with open(task.log.path, "a") as logfile: + for line in utils.run(task.repo): + logfile.write(line) + now = timezone.now() + task = models.Task.objects.get(pk=task.pk) + task.end_time = now + task.status = 'F' + task.save() + + +class Logs(LoginRequiredMixin, ListView): + model = models.Task + + +class TaskView(LoginRequiredMixin, DetailView): + model = models.Task + + def get_context_data(self, **kwargs): + context = super(TaskView, self).get_context_data(**kwargs) + try: + with open(context['object'].log.path, "r") as log_file: + context['log'] = log_file.read() + except ValueError: + context['log'] = "No log to show" + return context diff --git a/qtip/web/templates/bench/run.html b/qtip/web/templates/bench/run.html new file mode 100644 index 00000000..8eaa251b --- /dev/null +++ b/qtip/web/templates/bench/run.html @@ -0,0 +1,15 @@ +{% extends 'bench/base.html' %} +{% block content %} +
+

Repos

+ {% for repo in repos %} +
  • {{ repo.name }}
    • + {% endfor %} +
    +
    {% csrf_token %} + {{ form }} +
    + +
    +
    +{% endblock %} \ No newline at end of file diff --git a/qtip/web/templates/bench/task_detail.html b/qtip/web/templates/bench/task_detail.html new file mode 100644 index 00000000..d715022b --- /dev/null +++ b/qtip/web/templates/bench/task_detail.html @@ -0,0 +1,17 @@ +{% extends 'bench/base.html' %} +{% block content %} +
    +

    Task

    + + + + + +
    {{ object.repo}}{{ object.get_status_display }}{{ object.start_time }}{{ object.end_time }}{{ object.duration }}{{ object.log }}
    +
    +

    + {{ log | linebreaks }} +

    +
    +
    +{% endblock %} \ No newline at end of file diff --git a/qtip/web/templates/bench/task_list.html b/qtip/web/templates/bench/task_list.html new file mode 100644 index 00000000..188d6b81 --- /dev/null +++ b/qtip/web/templates/bench/task_list.html @@ -0,0 +1,22 @@ +{% extends 'bench/base.html' %} +{% block content %} +
    +

    Tasks Log

    + + + + + + + {% for task in object_list %} + + + + + + + + {% endfor %} +
    RepoStatusStart TimeEnd TimeRun time
    {{ task.repo }}{{ task.get_status_display }}{{ task.start_time }}{{ task.end_time }}{{ task.run_time }}
    + +{% endblock %} \ No newline at end of file -- cgit 1.2.3-korg