Update sla check for scenarios

This patch modify the question that SLA check result is not complete.

JIRA: YARDSTICK-172

Change-Id: I10438390baee92caf00dbfcdbdb833823ff8ce31
Signed-off-by: houjingwen <houjingwen@huawei.com>

6 files changed, 79 insertions, 34 deletions

diff --git a/tests/unit/benchmark/scenarios/compute/test_cyclictest.py b/tests/unit/benchmark/scenarios/compute/test_cyclictest.py
index 3791b4a76..28dc4d6b3 100644
--- a/tests/unit/benchmark/scenarios/compute/test_cyclictest.py
+++ b/tests/unit/benchmark/scenarios/compute/test_cyclictest.py
@@ -51,12 +51,14 @@ class CyclictestTestCase(unittest.TestCase):
         args = {
             "options": options,
         }
+        result = {}
+
         c.server = mock_ssh.SSH()
 
         sample_output = '{"min": 100, "avg": 500, "max": 1000}'
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
 
-        result = c.run(args)
+        c.run(args, result)
         expected_result = json.loads(sample_output)
         self.assertEqual(result, expected_result)
 
@@ -80,12 +82,14 @@ class CyclictestTestCase(unittest.TestCase):
             "options": options,
             "sla": sla
         }
+        result = {}
+
         c.server = mock_ssh.SSH()
 
         sample_output = '{"min": 100, "avg": 500, "max": 1000}'
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
 
-        result = c.run(args)
+        c.run(args, result)
         expected_result = json.loads(sample_output)
         self.assertEqual(result, expected_result)
 
@@ -96,11 +100,13 @@ class CyclictestTestCase(unittest.TestCase):
             "options": {},
             "sla": {"max_min_latency": 10}
         }
+        result = {}
+
         c.server = mock_ssh.SSH()
         sample_output = '{"min": 100, "avg": 500, "max": 1000}'
 
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, c.run, args)
+        self.assertRaises(AssertionError, c.run, args, result)
 
     def test_cyclictest_unsuccessful_sla_avg_latency(self, mock_ssh):
 
@@ -109,11 +115,13 @@ class CyclictestTestCase(unittest.TestCase):
             "options": {},
             "sla": {"max_avg_latency": 10}
         }
+        result = {}
+
         c.server = mock_ssh.SSH()
         sample_output = '{"min": 100, "avg": 500, "max": 1000}'
 
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, c.run, args)
+        self.assertRaises(AssertionError, c.run, args, result)
 
     def test_cyclictest_unsuccessful_sla_max_latency(self, mock_ssh):
 
@@ -122,11 +130,13 @@ class CyclictestTestCase(unittest.TestCase):
             "options": {},
             "sla": {"max_max_latency": 10}
         }
+        result = {}
+
         c.server = mock_ssh.SSH()
         sample_output = '{"min": 100, "avg": 500, "max": 1000}'
 
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, c.run, args)
+        self.assertRaises(AssertionError, c.run, args, result)
 
     def test_cyclictest_unsuccessful_script_error(self, mock_ssh):
 
@@ -148,10 +158,12 @@ class CyclictestTestCase(unittest.TestCase):
             "options": options,
             "sla": sla
         }
+        result = {}
+
         c.server = mock_ssh.SSH()
 
         mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR')
-        self.assertRaises(RuntimeError, c.run, args)
+        self.assertRaises(RuntimeError, c.run, args, result)
 
 
 def main():
diff --git a/tests/unit/benchmark/scenarios/networking/test_iperf3.py b/tests/unit/benchmark/scenarios/networking/test_iperf3.py
index 8b0da655b..2ec73ebd2 100644
--- a/tests/unit/benchmark/scenarios/networking/test_iperf3.py
+++ b/tests/unit/benchmark/scenarios/networking/test_iperf3.py
@@ -67,11 +67,12 @@ class IperfTestCase(unittest.TestCase):
 
         options = {}
         args = {'options': options}
+        result = {}
 
         sample_output = self._read_sample_output(self.output_name_tcp)
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
         expected_result = json.loads(sample_output)
-        result = p.run(args)
+        p.run(args, result)
         self.assertEqual(result, expected_result)
 
     def test_iperf_successful_sla(self, mock_ssh):
@@ -85,11 +86,12 @@ class IperfTestCase(unittest.TestCase):
             'options': options,
             'sla': {'bytes_per_second': 15000000}
         }
+        result = {}
 
         sample_output = self._read_sample_output(self.output_name_tcp)
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
         expected_result = json.loads(sample_output)
-        result = p.run(args)
+        p.run(args, result)
         self.assertEqual(result, expected_result)
 
     def test_iperf_unsuccessful_sla(self, mock_ssh):
@@ -103,10 +105,11 @@ class IperfTestCase(unittest.TestCase):
             'options': options,
             'sla': {'bytes_per_second': 25000000}
         }
+        result = {}
 
         sample_output = self._read_sample_output(self.output_name_tcp)
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, args)
+        self.assertRaises(AssertionError, p.run, args, result)
 
     def test_iperf_successful_sla_jitter(self, mock_ssh):
 
@@ -119,11 +122,12 @@ class IperfTestCase(unittest.TestCase):
             'options': options,
             'sla': {'jitter': 10}
         }
+        result = {}
 
         sample_output = self._read_sample_output(self.output_name_udp)
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
         expected_result = json.loads(sample_output)
-        result = p.run(args)
+        p.run(args, result)
         self.assertEqual(result, expected_result)
 
     def test_iperf_unsuccessful_sla_jitter(self, mock_ssh):
@@ -137,10 +141,11 @@ class IperfTestCase(unittest.TestCase):
             'options': options,
             'sla': {'jitter': 0.0001}
         }
+        result = {}
 
         sample_output = self._read_sample_output(self.output_name_udp)
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, args)
+        self.assertRaises(AssertionError, p.run, args, result)
 
     def test_iperf_unsuccessful_script_error(self, mock_ssh):
 
@@ -150,9 +155,10 @@ class IperfTestCase(unittest.TestCase):
 
         options = {}
         args = {'options': options}
+        result = {}
 
         mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR')
-        self.assertRaises(RuntimeError, p.run, args)
+        self.assertRaises(RuntimeError, p.run, args, result)
 
     def _read_sample_output(self,filename):
         curr_path = os.path.dirname(os.path.abspath(__file__))
diff --git a/tests/unit/benchmark/scenarios/networking/test_netperf.py b/tests/unit/benchmark/scenarios/networking/test_netperf.py
index d5c19918b..4bb5983c3 100755
--- a/tests/unit/benchmark/scenarios/networking/test_netperf.py
+++ b/tests/unit/benchmark/scenarios/networking/test_netperf.py
@@ -48,11 +48,12 @@ class NetperfTestCase(unittest.TestCase):
 
         options = {}
         args = {'options': options}
+        result = {}
 
         sample_output = self._read_sample_output()
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
         expected_result = json.loads(sample_output)
-        result = p.run(args)
+        p.run(args, result)
         self.assertEqual(result, expected_result)
 
     def test_netperf_successful_sla(self, mock_ssh):
@@ -66,11 +67,12 @@ class NetperfTestCase(unittest.TestCase):
             'options': options,
             'sla': {'mean_latency': 100}
         }
+        result = {}
 
         sample_output = self._read_sample_output()
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
         expected_result = json.loads(sample_output)
-        result = p.run(args)
+        p.run(args, result)
         self.assertEqual(result, expected_result)
 
     def test_netperf_unsuccessful_sla(self, mock_ssh):
@@ -84,10 +86,11 @@ class NetperfTestCase(unittest.TestCase):
             'options': options,
             'sla': {'mean_latency': 5}
         }
+        result = {}
 
         sample_output = self._read_sample_output()
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, args)
+        self.assertRaises(AssertionError, p.run, args, result)
 
     def test_netperf_unsuccessful_script_error(self, mock_ssh):
 
@@ -97,9 +100,10 @@ class NetperfTestCase(unittest.TestCase):
 
         options = {}
         args = {'options': options}
+        result = {}
 
         mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR')
-        self.assertRaises(RuntimeError, p.run, args)
+        self.assertRaises(RuntimeError, p.run, args, result)
 
     def _read_sample_output(self):
         curr_path = os.path.dirname(os.path.abspath(__file__))
diff --git a/tests/unit/benchmark/scenarios/networking/test_ping.py b/tests/unit/benchmark/scenarios/networking/test_ping.py
index d930adcee..b2c5b9859 100644
--- a/tests/unit/benchmark/scenarios/networking/test_ping.py
+++ b/tests/unit/benchmark/scenarios/networking/test_ping.py
@@ -35,10 +35,11 @@ class PingTestCase(unittest.TestCase):
             'options': {'packetsize': 200},
             'ipaddr': '172.16.0.138'
             }
+        result = {}
 
         mock_ssh.SSH().execute.return_value = (0, '100', '')
-        result = p.run(args)
-        self.assertEqual(result, float(mock_ssh.SSH().execute.return_value[1]))
+        p.run(args, result)
+        self.assertEqual(result, {'rtt': 100.0})
 
     @mock.patch('yardstick.benchmark.scenarios.networking.ping.ssh')
     def test_ping_successful_sla(self, mock_ssh):
@@ -50,10 +51,11 @@ class PingTestCase(unittest.TestCase):
             'ipaddr': '172.16.0.138',
             'sla': {'max_rtt': 150}
             }
+        result = {}
 
         mock_ssh.SSH().execute.return_value = (0, '100', '')
-        result = p.run(args)
-        self.assertEqual(result, float(mock_ssh.SSH().execute.return_value[1]))
+        p.run(args, result)
+        self.assertEqual(result, {'rtt': 100.0})
 
     @mock.patch('yardstick.benchmark.scenarios.networking.ping.ssh')
     def test_ping_unsuccessful_sla(self, mock_ssh):
@@ -65,9 +67,10 @@ class PingTestCase(unittest.TestCase):
             'ipaddr': '172.16.0.138',
             'sla': {'max_rtt': 50}
             }
+        result = {}
 
         mock_ssh.SSH().execute.return_value = (0, '100', '')
-        self.assertRaises(AssertionError, p.run, args)
+        self.assertRaises(AssertionError, p.run, args, result)
 
     @mock.patch('yardstick.benchmark.scenarios.networking.ping.ssh')
     def test_ping_unsuccessful_script_error(self, mock_ssh):
@@ -79,9 +82,10 @@ class PingTestCase(unittest.TestCase):
             'ipaddr': '172.16.0.138',
             'sla': {'max_rtt': 50}
             }
+        result = {}
 
         mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR')
-        self.assertRaises(RuntimeError, p.run, args)
+        self.assertRaises(RuntimeError, p.run, args, result)
 
 
 def main():
diff --git a/tests/unit/benchmark/scenarios/networking/test_pktgen.py b/tests/unit/benchmark/scenarios/networking/test_pktgen.py
index a20382cb7..ae4481f0e 100644
--- a/tests/unit/benchmark/scenarios/networking/test_pktgen.py
+++ b/tests/unit/benchmark/scenarios/networking/test_pktgen.py
@@ -113,6 +113,8 @@ class PktgenTestCase(unittest.TestCase):
             'options': {'packetsize': 60, 'number_of_ports': 10},
             'ipaddr': '172.16.0.139'
         }
+        result = {}
+
         p.server = mock_ssh.SSH()
         p.client = mock_ssh.SSH()
 
@@ -124,7 +126,7 @@ class PktgenTestCase(unittest.TestCase):
             "packets_sent": 149776, "flows": 110}'
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
 
-        result = p.run(args)
+        p.run(args, result)
         expected_result = json.loads(sample_output)
         expected_result["packets_received"] = 149300
         self.assertEqual(result, expected_result)
@@ -137,6 +139,7 @@ class PktgenTestCase(unittest.TestCase):
             'ipaddr': '172.16.0.139',
             'sla': {'max_ppm': 10000}
         }
+        result = {}
         p.server = mock_ssh.SSH()
         p.client = mock_ssh.SSH()
 
@@ -148,7 +151,7 @@ class PktgenTestCase(unittest.TestCase):
             "packets_sent": 149776, "flows": 110}'
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
 
-        result = p.run(args)
+        p.run(args, result)
         expected_result = json.loads(sample_output)
         expected_result["packets_received"] = 149300
         self.assertEqual(result, expected_result)
@@ -161,6 +164,8 @@ class PktgenTestCase(unittest.TestCase):
             'ipaddr': '172.16.0.139',
             'sla': {'max_ppm': 1000}
         }
+        result = {}
+
         p.server = mock_ssh.SSH()
         p.client = mock_ssh.SSH()
 
@@ -171,7 +176,7 @@ class PktgenTestCase(unittest.TestCase):
         sample_output = '{"packets_per_second": 9753, "errors": 0, \
             "packets_sent": 149776, "flows": 110}'
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, args)
+        self.assertRaises(AssertionError, p.run, args, result)
 
     def test_pktgen_unsuccessful_script_error(self, mock_ssh):
 
@@ -181,11 +186,13 @@ class PktgenTestCase(unittest.TestCase):
             'ipaddr': '172.16.0.139',
             'sla': {'max_ppm': 1000}
         }
+        result = {}
+
         p.server = mock_ssh.SSH()
         p.client = mock_ssh.SSH()
 
         mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR')
-        self.assertRaises(RuntimeError, p.run, args)
+        self.assertRaises(RuntimeError, p.run, args, result)
 
 
 def main():
diff --git a/tests/unit/benchmark/scenarios/storage/test_fio.py b/tests/unit/benchmark/scenarios/storage/test_fio.py
index 6d38e9c53..b47aed968 100644
--- a/tests/unit/benchmark/scenarios/storage/test_fio.py
+++ b/tests/unit/benchmark/scenarios/storage/test_fio.py
@@ -60,12 +60,14 @@ class FioTestCase(unittest.TestCase):
             'ramp_time': 10
         }
         args = {'options': options}
+        result = {}
+
         p.client = mock_ssh.SSH()
 
         sample_output = self._read_sample_output(self.sample_output['rw'])
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
 
-        result = p.run(args)
+        p.run(args, result)
 
         expected_result = '{"read_bw": 83888, "read_iops": 20972,' \
             '"read_lat": 236.8, "write_bw": 84182, "write_iops": 21045,'\
@@ -83,12 +85,14 @@ class FioTestCase(unittest.TestCase):
             'ramp_time': 10
         }
         args = {'options': options}
+        result = {}
+
         p.client = mock_ssh.SSH()
 
         sample_output = self._read_sample_output(self.sample_output['read'])
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
 
-        result = p.run(args)
+        p.run(args, result)
 
         expected_result = '{"read_bw": 36113, "read_iops": 9028,' \
             '"read_lat": 108.7}'
@@ -105,12 +109,14 @@ class FioTestCase(unittest.TestCase):
             'ramp_time': 10
         }
         args = {'options': options}
+        result = {}
+
         p.client = mock_ssh.SSH()
 
         sample_output = self._read_sample_output(self.sample_output['write'])
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
 
-        result = p.run(args)
+        p.run(args, result)
 
         expected_result = '{"write_bw": 35107, "write_iops": 8776,'\
             '"write_lat": 111.74}'
@@ -130,13 +136,14 @@ class FioTestCase(unittest.TestCase):
             'options': options,
             'sla': {'write_lat': 300.1}
         }
+        result = {}
 
         p.client = mock_ssh.SSH()
 
         sample_output = self._read_sample_output(self.sample_output['rw'])
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
 
-        result = p.run(args)
+        p.run(args, result)
 
         expected_result = '{"read_bw": 83888, "read_iops": 20972,' \
             '"read_lat": 236.8, "write_bw": 84182, "write_iops": 21045,'\
@@ -158,12 +165,13 @@ class FioTestCase(unittest.TestCase):
             'options': options,
             'sla': {'write_lat': 200.1}
         }
+        result = {}
 
         p.client = mock_ssh.SSH()
 
         sample_output = self._read_sample_output(self.sample_output['rw'])
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, args)
+        self.assertRaises(AssertionError, p.run, args, result)
 
     def test_fio_successful_bw_iops_sla(self, mock_ssh):
 
@@ -178,13 +186,14 @@ class FioTestCase(unittest.TestCase):
             'options': options,
             'sla': {'read_iops': 20000}
         }
+        result = {}
 
         p.client = mock_ssh.SSH()
 
         sample_output = self._read_sample_output(self.sample_output['rw'])
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
 
-        result = p.run(args)
+        p.run(args, result)
 
         expected_result = '{"read_bw": 83888, "read_iops": 20972,' \
             '"read_lat": 236.8, "write_bw": 84182, "write_iops": 21045,'\
@@ -205,12 +214,13 @@ class FioTestCase(unittest.TestCase):
             'options': options,
             'sla': {'read_iops': 30000}
         }
+        result = {}
 
         p.client = mock_ssh.SSH()
 
         sample_output = self._read_sample_output(self.sample_output['rw'])
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, args)
+        self.assertRaises(AssertionError, p.run, args, result)
 
     def test_fio_unsuccessful_script_error(self, mock_ssh):
 
@@ -222,10 +232,12 @@ class FioTestCase(unittest.TestCase):
             'ramp_time': 10
         }
         args = {'options': options}
+        result = {}
+
         p.client = mock_ssh.SSH()
 
         mock_ssh.SSH().execute.return_value = (1, '', 'FOOBAR')
-        self.assertRaises(RuntimeError, p.run, args)
+        self.assertRaises(RuntimeError, p.run, args, result)
 
     def _read_sample_output(self, file_name):
         curr_path = os.path.dirname(os.path.abspath(__file__))