From 4e7e1d2ffea89950ae516a19997765daccf92664 Mon Sep 17 00:00:00 2001
From: Mark Beierl <mark.beierl@emc.com>
Date: Wed, 11 May 2016 15:38:26 -0600
Subject: Max test duration

Add a maximum time limit for any one given test so
that silly mistakes like block size 512, queue depth 1
does not end up running for 2 days

JIRA: STORPERF-53

Change-Id: I1f3e5046e1dd346fc5b28a7b935df2ee9aa6ba3d
Signed-off-by: Mark Beierl <mark.beierl@emc.com>
---
 storperf/test_executor.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

(limited to 'storperf/test_executor.py')

diff --git a/storperf/test_executor.py b/storperf/test_executor.py
index d1ad3ca..8230174 100644
--- a/storperf/test_executor.py
+++ b/storperf/test_executor.py
@@ -18,6 +18,8 @@ import copy
 import imp
 import logging
 import os
+import sched
+import time
 
 
 class UnknownWorkload(Exception):
@@ -31,6 +33,7 @@ class TestExecutor(object):
         self.workload_modules = []
         self.filename = None
         self.precondition = True
+        self.deadline = None
         self.warm = True
         self._queue_depths = [1, 4, 8]
         self._block_sizes = [512, 4096, 16384]
@@ -143,6 +146,10 @@ class TestExecutor(object):
 
     def terminate(self):
         self._terminated = True
+        return self.terminate_current_run()
+
+    def terminate_current_run(self):
+        self.logger.info("Terminating current run")
         terminated_hosts = []
         for workload in self._workload_executors:
             workload.terminate()
@@ -170,9 +177,17 @@ class TestExecutor(object):
             for blocksize in blocksizes:
                 for iodepth in iodepths:
 
+                    scheduler = sched.scheduler(time.time, time.sleep)
                     if self._terminated:
                         return
 
+                    if self.deadline is not None \
+                            and not workload_name.startswith("_"):
+                        event = scheduler.enter(self.deadline * 60, 1,
+                                                self.terminate_current_run, ())
+                        t = Thread(target=scheduler.run, args=())
+                        t.start()
+
                     workload.options['iodepth'] = str(iodepth)
                     workload.options['bs'] = str(blocksize)
 
@@ -192,6 +207,12 @@ class TestExecutor(object):
                     for slave_thread in slave_threads:
                         slave_thread.join()
 
+                    if not scheduler.empty():
+                        try:
+                            scheduler.cancel(event)
+                        except:
+                            pass
+
                     self._workload_executors = []
 
     def execute_on_node(self, workload):
-- 
cgit 1.2.3-korg