Prevent notifications after termination

Changes the event notification logic inside the FIO invoker
so that it no longer publishes events after termination.
Prevents false reports after steady state has been detected.

Change-Id: I694f77b6493b88820fe4f4cc7f634e3e62c45a9a
JIRA: STORPERF-105
Signed-off-by: mbeierl <mark.beierl@dell.com>

7 files changed, 128 insertions, 20 deletions

diff --git a/storperf/fio/fio_invoker.py b/storperf/fio/fio_invoker.py
index a201802..106696d 100644
--- a/storperf/fio/fio_invoker.py
+++ b/storperf/fio/fio_invoker.py
@@ -21,6 +21,7 @@ class FIOInvoker(object):
         self.event_callback_ids = set()
         self._remote_host = None
         self.callback_id = None
+        self.terminated = False
 
     @property
     def remote_host(self):
@@ -51,17 +52,19 @@ class FIOInvoker(object):
                         json_metric = json.loads(self.json_body)
                         self.json_body = ""
 
-                        for event_listener in self.event_listeners:
-                            try:
+                        if not self.terminated:
+                            for event_listener in self.event_listeners:
+                                try:
+                                    self.logger.debug(
+                                        "Event listener callback")
+                                    event_listener(
+                                        self.callback_id, json_metric)
+                                except Exception, e:
+                                    self.logger.exception(
+                                        "Notifying listener %s: %s",
+                                        self.callback_id, e)
                                 self.logger.debug(
-                                    "Event listener callback")
-                                event_listener(self.callback_id, json_metric)
-                            except Exception, e:
-                                self.logger.exception(
-                                    "Notifying listener %s: %s",
-                                    self.callback_id, e)
-                            self.logger.debug(
-                                "Event listener callback complete")
+                                    "Event listener callback complete")
                 except Exception, e:
                     self.logger.error("Error parsing JSON: %s", e)
         except IOError:
@@ -128,6 +131,7 @@ class FIOInvoker(object):
 
     def terminate(self):
         self.logger.debug("Terminating fio on " + self.remote_host)
+        self.terminated = True
 
         ssh = paramiko.SSHClient()
         ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
diff --git a/storperf/utilities/data_handler.py b/storperf/utilities/data_handler.py
index e38f502..a4c9ae4 100644
--- a/storperf/utilities/data_handler.py
+++ b/storperf/utilities/data_handler.py
@@ -125,7 +125,7 @@ class DataHandler(object):
 
         for item in series:
             elapsed = (item[0] - start_time)
-            sample_number = (elapsed / 60) + 1
+            sample_number = int(round(float(elapsed) / 60))
             normalized_series.append([sample_number, item[1]])
 
         return normalized_series
diff --git a/storperf/utilities/math.py b/storperf/utilities/math.py
index 4ddddca..8e04134 100644
--- a/storperf/utilities/math.py
+++ b/storperf/utilities/math.py
@@ -6,6 +6,7 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
+import copy
 
 
 def slope(data_series):
@@ -20,7 +21,7 @@ def slope(data_series):
     [[x1,y1], [x2,y2], ..., [xm,ym]].
     If this data pattern were to change, the data_treatement function
     should be adjusted to ensure compatibility with the rest of the
-    Steady State Dectection module.
+    Steady State Detection module.
     """
 
     # In the particular case of an empty data series
@@ -28,6 +29,7 @@ def slope(data_series):
         beta2 = None
 
     else:  # The general case
+        data_series = copy.deepcopy(data_series)
         m = len(data_series)
         # To make sure at least one element is a float number so the result
         # of the algorithm be a float number
diff --git a/tests/__init__.py b/tests/__init__.py
index 73334c7..230494c 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -6,3 +6,6 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
+import logging
+
+logging.basicConfig(level=logging.DEBUG)
diff --git a/tests/fio_tests/__init__.py b/tests/fio_tests/__init__.py
new file mode 100644
index 0000000..df29e18
--- /dev/null
+++ b/tests/fio_tests/__init__.py
@@ -0,0 +1,11 @@
+##############################################################################
+# Copyright (c) 2017 Dell EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+import logging
+
+logging.basicConfig(level=logging.DEBUG)
diff --git a/tests/fio_tests/fio_invoker_test.py b/tests/fio_tests/fio_invoker_test.py
new file mode 100644
index 0000000..4672651
--- /dev/null
+++ b/tests/fio_tests/fio_invoker_test.py
@@ -0,0 +1,88 @@
+##############################################################################
+# Copyright (c) 2017 Dell EMC and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+from StringIO import StringIO
+import json
+import unittest
+
+from storperf.fio.fio_invoker import FIOInvoker
+
+
+class Test(unittest.TestCase):
+
+    simple_dictionary = {'Key': 'Value'}
+
+    def exceptional_event(self, callback_id, metric):
+        self.exception_called = True
+        raise Exception
+
+    def event(self, callback_id, metric):
+        self.metric = metric
+
+    def setUp(self):
+        self.exception_called = False
+        self.metric = None
+        self.fio_invoker = FIOInvoker()
+
+    def testStdoutValidJSON(self):
+        self.fio_invoker.register(self.event)
+        string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
+
+        output = StringIO(string + "\n")
+        self.fio_invoker.stdout_handler(output)
+
+        self.assertEqual(self.simple_dictionary, self.metric)
+
+    def testStdoutValidJSONWithFIOOutput(self):
+        self.fio_invoker.register(self.event)
+        string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
+        terminating = "fio: terminating on signal 2\n"
+        output = StringIO(terminating + string + "\n")
+        self.fio_invoker.stdout_handler(output)
+
+        self.assertEqual(self.simple_dictionary, self.metric)
+
+    def testStdoutNoJSON(self):
+        self.fio_invoker.register(self.event)
+        string = "{'key': 'value'}"
+
+        output = StringIO(string + "\n")
+        self.fio_invoker.stdout_handler(output)
+
+        self.assertEqual(None, self.metric)
+
+    def testStdoutInvalidJSON(self):
+        self.fio_invoker.register(self.event)
+        string = "{'key':\n}"
+
+        output = StringIO(string + "\n")
+        self.fio_invoker.stdout_handler(output)
+
+        self.assertEqual(None, self.metric)
+
+    def testStdoutAfterTerminated(self):
+        self.fio_invoker.register(self.event)
+        string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
+
+        self.fio_invoker.terminated = True
+        output = StringIO(string + "\n")
+        self.fio_invoker.stdout_handler(output)
+
+        self.assertEqual(None, self.metric)
+
+    def testStdoutCallbackException(self):
+        self.fio_invoker.register(self.exceptional_event)
+        self.fio_invoker.register(self.event)
+        string = json.dumps(self.simple_dictionary, indent=4, sort_keys=True)
+
+        output = StringIO(string + "\n")
+        self.fio_invoker.stdout_handler(output)
+
+        self.assertEqual(self.simple_dictionary, self.metric)
+        self.assertEqual(self.exception_called, True)
diff --git a/tests/utilities_tests/data_handler_test.py b/tests/utilities_tests/data_handler_test.py
index 93b0b97..333bed0 100644
--- a/tests/utilities_tests/data_handler_test.py
+++ b/tests/utilities_tests/data_handler_test.py
@@ -167,7 +167,7 @@ class DataHandlerTest(unittest.TestCase):
              [1480456050, 217.75]]
         mock_graphite_db.return_value = series
         mock_time.return_value = series[-1][0] + 10
-        expected_slope = 11.48297119140625
+        expected_slope = 12.292030334472656
         expected_range = 17.78
         expected_average = 212.49777777777774
 
@@ -222,19 +222,19 @@ class DataHandlerTest(unittest.TestCase):
                   [4804560400, 219.28],
                   [4804560500, 217.75]]
         report_data = [[2, 205.345],
-                       [4, 201.59],
-                       [6, 205.76],
+                       [3, 201.59],
+                       [5, 205.76],
                        [7, 205.76],
-                       [9, 205.76],
-                       [11, 205.76],
+                       [8, 205.76],
+                       [10, 205.76],
                        [12, 205.76],
                        [22, 219.37],
-                       [24, 219.28],
-                       [26, 217.75]]
+                       [23, 219.28],
+                       [25, 217.75]]
         mock_graphite_db.return_value = series
         mock_time.return_value = 4804560500 + 10
 
-        expected_slope = 0.7318639667704995
+        expected_slope = 0.7419522662249607
         expected_range = 17.78
         expected_average = 209.2135