Implement benchmark running and logging the output

- Run benchmark by adding new tasks
- Modify Task model to have log as log files
- Add new urls for respective views
- Module for running benchmarks in background
- Updating status of tasks
- Listing and detailed log of all tasks
- Add corresponding html templates

Change-Id: I244d1bb74949eeb470c738363f6917191e0f052d
Signed-off-by: akhilbatra898 <akhil.batra@research.iiit.ac.in>

10 files changed, 243 insertions, 9 deletions

diff --git a/qtip/web/bench/forms.py b/qtip/web/bench/forms.py
new file mode 100644
index 00000000..d897aca7
--- /dev/null
+++ b/qtip/web/bench/forms.py
@@ -0,0 +1,19 @@
+##############################################################################
+# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+
+from django import forms
+
+import models
+
+
+class TaskForm(forms.ModelForm):
+    class Meta:
+        model = models.Task
+        fields = ['repo']
diff --git a/qtip/web/bench/migrations/0005_auto_20170720_2115.py b/qtip/web/bench/migrations/0005_auto_20170720_2115.py
new file mode 100644
index 00000000..5bd81a2c
--- /dev/null
+++ b/qtip/web/bench/migrations/0005_auto_20170720_2115.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.3 on 2017-07-20 21:15
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bench', '0004_auto_20170715_0325'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='repo',
+            old_name='github_link',
+            new_name='git_link',
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='end_time',
+            field=models.DateTimeField(null=True),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='log',
+            field=models.FilePathField(),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='run_time',
+            field=models.DurationField(null=True),
+        ),
+    ]
diff --git a/qtip/web/bench/migrations/0006_auto_20170722_0135.py b/qtip/web/bench/migrations/0006_auto_20170722_0135.py
new file mode 100644
index 00000000..5d066702
--- /dev/null
+++ b/qtip/web/bench/migrations/0006_auto_20170722_0135.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.3 on 2017-07-22 01:35
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bench', '0005_auto_20170720_2115'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='task',
+            name='status',
+            field=models.CharField(choices=[('P', 'Pending'), ('IP', 'In progress'), ('F', 'Finished')], default='P', max_length=20),
+        ),
+        migrations.AlterField(
+            model_name='task',
+            name='log',
+            field=models.FileField(upload_to='logs'),
+        ),
+    ]
diff --git a/qtip/web/bench/models.py b/qtip/web/bench/models.py
index 863af739..3f0439d9 100644
--- a/qtip/web/bench/models.py
+++ b/qtip/web/bench/models.py
@@ -18,15 +18,33 @@ from django.urls import reverse
 
 class Repo(models.Model):
     name = models.CharField(max_length=200, blank=False)
-    github_link = models.URLField(unique=True)
+    git_link = models.URLField(unique=True)
 
     def get_absolute_url(self):
         return reverse('repo_update', args=[self.pk])
 
+    def __str__(self):
+        return "%s, %s" % (self.name, self.git_link)
+
 
 class Task(models.Model):
+    TASK_STATUS_CHOICES = (
+        ('P', 'Pending'),
+        ('IP', 'In progress'),
+        ('F', 'Finished')
+    )
+
     start_time = models.DateTimeField(auto_now_add=True)
-    end_time = models.DateTimeField()
-    run_time = models.DurationField()
+    status = models.CharField(choices=TASK_STATUS_CHOICES, default='P', max_length=20)
+    end_time = models.DateTimeField(null=True)
+    run_time = models.DurationField(null=True)
     repo = models.ForeignKey('Repo', on_delete=models.DO_NOTHING)
-    log = models.TextField()
+    log = models.FileField(upload_to='logs')
+
+    def save(self, **kwargs):
+        if self.end_time:
+            self.run_time = self.end_time - self.start_time
+        super(Task, self).save(kwargs)
+
+    def get_absolute_url(self):
+        return reverse('task_view', args=[self.pk])
diff --git a/qtip/web/bench/urls.py b/qtip/web/bench/urls.py
index 3af4eb83..ae9738b6 100644
--- a/qtip/web/bench/urls.py
+++ b/qtip/web/bench/urls.py
@@ -18,4 +18,7 @@ urlpatterns = [
     url('^', include('django.contrib.auth.urls')),
     url('^repos/$', views.ReposView.as_view(), name='repos'),
     url('^repos/(?P<pk>\d+)$', views.RepoUpdate.as_view(), name='repo_update'),
+    url('^run/$', views.Run.as_view(), name='run'),
+    url('^tasks/$', views.Logs.as_view(), name='tasks'),
+    url('^tasks/(?P<pk>\d+)$', views.TaskView.as_view(), name='task_view'),
 ]
diff --git a/qtip/web/bench/utils.py b/qtip/web/bench/utils.py
new file mode 100644
index 00000000..aac388e0
--- /dev/null
+++ b/qtip/web/bench/utils.py
@@ -0,0 +1,27 @@
+##############################################################################
+# Copyright (c) 2017 akhil.batra@research.iiit.ac.in and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import os
+from subprocess import Popen, PIPE, STDOUT
+import shutil
+
+
+def run(repo):
+    if os.path.exists(repo.name):
+        shutil.rmtree(repo.name)
+    os.mkdir(repo.name)
+    os.chdir(repo.name)
+    output = Popen(("git clone %s ." % repo.git_link).split(), stdout=PIPE, stderr=STDOUT)
+    for line in output.stdout:
+        yield line
+    output.wait()
+    output = Popen("ansible-playbook run.yml".split(), stdout=PIPE, stderr=STDOUT)
+    for line in output.stdout:
+        yield line
+    os.chdir("../")
diff --git a/qtip/web/bench/views.py b/qtip/web/bench/views.py
index 786b67d5..6de5e4cc 100644
--- a/qtip/web/bench/views.py
+++ b/qtip/web/bench/views.py
@@ -9,16 +9,20 @@
 
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
+from multiprocessing import Process
 
 from django.contrib.auth.mixins import LoginRequiredMixin
-# from django.utils.decorators import method_decorator
 from django.views.generic.edit import CreateView, UpdateView
+from django.views.generic.list import ListView
+from django.views.generic.detail import DetailView
+from django.views import View
+from django.shortcuts import render, redirect
+from django.core.files.base import ContentFile
+from django.utils import timezone
 
+import forms
 import models
-
-# from django.shortcuts import render
-
-# Create your views here.
+import utils
 
 
 class ReposView(LoginRequiredMixin, CreateView):
@@ -39,3 +43,52 @@ class RepoUpdate(LoginRequiredMixin, UpdateView):
         context = super(RepoUpdate, self).get_context_data(**kwargs)
         context["repos"] = self.model.objects.all()
         return context
+
+
+class Run(LoginRequiredMixin, View):
+    template_name = 'bench/run.html'
+    form_class = forms.TaskForm
+
+    def get(self, request):
+        task_form = self.form_class()
+        return render(request, self.template_name, {'form': task_form})
+
+    def post(self, request):
+        task_form = self.form_class(request.POST)
+        if task_form.is_valid():
+            new_task = task_form.save()
+            new_task.log.save("run_%s.log" % new_task.pk, ContentFile(''))
+            p = Process(target=self.start_task, args=(new_task,))
+            p.start()
+            return redirect('tasks')
+            p.join()
+
+    def start_task(self, task):
+        task = models.Task.objects.get(pk=task.pk)
+        task.status = 'IP'
+        task.save()
+        with open(task.log.path, "a") as logfile:
+            for line in utils.run(task.repo):
+                logfile.write(line)
+        now = timezone.now()
+        task = models.Task.objects.get(pk=task.pk)
+        task.end_time = now
+        task.status = 'F'
+        task.save()
+
+
+class Logs(LoginRequiredMixin, ListView):
+    model = models.Task
+
+
+class TaskView(LoginRequiredMixin, DetailView):
+    model = models.Task
+
+    def get_context_data(self, **kwargs):
+        context = super(TaskView, self).get_context_data(**kwargs)
+        try:
+            with open(context['object'].log.path, "r") as log_file:
+                context['log'] = log_file.read()
+        except ValueError:
+            context['log'] = "No log to show"
+        return context
diff --git a/qtip/web/templates/bench/run.html b/qtip/web/templates/bench/run.html
new file mode 100644
index 00000000..8eaa251b
--- /dev/null
+++ b/qtip/web/templates/bench/run.html
@@ -0,0 +1,15 @@
+{% extends 'bench/base.html' %}
+{% block content %}
+    <div>
+        <h3>Repos</h3>
+        {% for repo in repos %}
+        <li>{{ repo.name }}</li>
+        {% endfor %}
+    </div>
+    <form role="form" method="post" action="">{% csrf_token %}
+        {{ form }}
+        <div class=input-field" style="margin:30px 20px">
+            <button  type="submit" value="Login"/>Submit</button>
+        </div>
+    </form>
+{% endblock %}
\ No newline at end of file
diff --git a/qtip/web/templates/bench/task_detail.html b/qtip/web/templates/bench/task_detail.html
new file mode 100644
index 00000000..d715022b
--- /dev/null
+++ b/qtip/web/templates/bench/task_detail.html
@@ -0,0 +1,17 @@
+{% extends 'bench/base.html' %}
+{% block content %}
+    <div>
+        <h3>Task</h3>
+        <table>
+            <tr>
+                <td>{{ object.repo}}</td><td>{{ object.get_status_display }}</td><td>{{ object.start_time }}</td>
+                <td>{{ object.end_time }}</td><td>{{ object.duration }}</td><td>{{ object.log }}</td>
+            </tr>
+        </table>
+        <div>
+            <p>
+                {{ log | linebreaks }}
+            </p>
+        </div>
+    </div>
+{% endblock %}
\ No newline at end of file
diff --git a/qtip/web/templates/bench/task_list.html b/qtip/web/templates/bench/task_list.html
new file mode 100644
index 00000000..188d6b81
--- /dev/null
+++ b/qtip/web/templates/bench/task_list.html
@@ -0,0 +1,22 @@
+{% extends 'bench/base.html' %}
+{% block content %}
+    <div>
+        <h3>Tasks Log</h3>
+        <table>
+            <th>Repo</th>
+            <th>Status</th>
+            <th>Start Time</th>
+            <th>End Time</th>
+            <th>Run time</th>
+            {% for task in object_list %}
+                <tr onclick="location.href='{% url 'task_view' task.id %}'">
+                    <td>{{ task.repo }}</td>
+                    <td>{{ task.get_status_display }}</td>
+                    <td>{{ task.start_time }}</td>
+                    <td>{{ task.end_time }}</td>
+                    <td>{{ task.run_time }}</td>
+                </tr>
+            {% endfor %}
+        <table>
+    </div>
+{% endblock %}
\ No newline at end of file