Merge "Convert SLA asserts to raises"

16 files changed, 65 insertions, 25 deletions

diff --git a/yardstick/tests/unit/benchmark/scenarios/availability/test_scenario_general.py b/yardstick/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
index d1172d5a6..cd065c961 100644
--- a/yardstick/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
+++ b/yardstick/tests/unit/benchmark/scenarios/availability/test_scenario_general.py
@@ -11,6 +11,7 @@ import mock
 import unittest
 
 from yardstick.benchmark.scenarios.availability import scenario_general
+from yardstick.common import exceptions as y_exc
 
 class ScenarioGeneralTestCase(unittest.TestCase):
 
@@ -59,6 +60,14 @@ class ScenarioGeneralTestCase(unittest.TestCase):
         self.instance.director.verify.return_value = False
         self.instance.director.data = {}
         ret = {}
-        self.assertRaises(AssertionError, self.instance.run, ret)
+        self.assertRaises(y_exc.SLAValidationError, self.instance.run, ret)
+        self.instance.teardown()
+        self.assertEqual(ret['sla_pass'], 0)
+
+    def test_scenario_general_case_service_not_found_fail(self):
+        self.instance.director.verify.return_value = True
+        self.instance.director.data = {"general-attacker": 0}
+        ret = {}
+        self.assertRaises(y_exc.SLAValidationError, self.instance.run, ret)
         self.instance.teardown()
         self.assertEqual(ret['sla_pass'], 0)
diff --git a/yardstick/tests/unit/benchmark/scenarios/availability/test_serviceha.py b/yardstick/tests/unit/benchmark/scenarios/availability/test_serviceha.py
index dd656fbd5..cf1e76d7a 100644
--- a/yardstick/tests/unit/benchmark/scenarios/availability/test_serviceha.py
+++ b/yardstick/tests/unit/benchmark/scenarios/availability/test_serviceha.py
@@ -11,6 +11,7 @@ import mock
 import unittest
 
 from yardstick.benchmark.scenarios.availability import serviceha
+from yardstick.common import exceptions as y_exc
 
 
 class ServicehaTestCase(unittest.TestCase):
@@ -71,5 +72,21 @@ class ServicehaTestCase(unittest.TestCase):
         mock_monitor.MonitorMgr().verify_SLA.return_value = False
 
         ret = {}
-        self.assertRaises(AssertionError, p.run, ret)
+        self.assertRaises(y_exc.SLAValidationError, p.run, ret)
+        self.assertEqual(ret['sla_pass'], 0)
+
+    @mock.patch.object(serviceha, 'baseattacker')
+    @mock.patch.object(serviceha, 'basemonitor')
+    def test__serviceha_run_service_not_found_sla_error(self, mock_monitor,
+                                                        *args):
+        p = serviceha.ServiceHA(self.args, self.ctx)
+
+        p.setup()
+        self.assertTrue(p.setup_done)
+        p.data["kill-process"] = 0
+
+        mock_monitor.MonitorMgr().verify_SLA.return_value = True
+
+        ret = {}
+        self.assertRaises(y_exc.SLAValidationError, p.run, ret)
         self.assertEqual(ret['sla_pass'], 0)
diff --git a/yardstick/tests/unit/benchmark/scenarios/compute/test_cyclictest.py b/yardstick/tests/unit/benchmark/scenarios/compute/test_cyclictest.py
index f24ec24ec..4fadde4dc 100644
--- a/yardstick/tests/unit/benchmark/scenarios/compute/test_cyclictest.py
+++ b/yardstick/tests/unit/benchmark/scenarios/compute/test_cyclictest.py
@@ -17,6 +17,7 @@ import mock
 from oslo_serialization import jsonutils
 
 from yardstick.benchmark.scenarios.compute import cyclictest
+from yardstick.common import exceptions as y_exc
 
 
 @mock.patch('yardstick.benchmark.scenarios.compute.cyclictest.ssh')
@@ -122,7 +123,7 @@ class CyclictestTestCase(unittest.TestCase):
         sample_output = '{"min": 100, "avg": 500, "max": 1000}'
 
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, c.run, result)
+        self.assertRaises(y_exc.SLAValidationError, c.run, result)
 
     def test_cyclictest_unsuccessful_sla_avg_latency(self, mock_ssh):
 
@@ -136,7 +137,7 @@ class CyclictestTestCase(unittest.TestCase):
         sample_output = '{"min": 100, "avg": 500, "max": 1000}'
 
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, c.run, result)
+        self.assertRaises(y_exc.SLAValidationError, c.run, result)
 
     def test_cyclictest_unsuccessful_sla_max_latency(self, mock_ssh):
 
@@ -150,7 +151,7 @@ class CyclictestTestCase(unittest.TestCase):
         sample_output = '{"min": 100, "avg": 500, "max": 1000}'
 
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, c.run, result)
+        self.assertRaises(y_exc.SLAValidationError, c.run, result)
 
     def test_cyclictest_unsuccessful_script_error(self, mock_ssh):
 
diff --git a/yardstick/tests/unit/benchmark/scenarios/compute/test_lmbench.py b/yardstick/tests/unit/benchmark/scenarios/compute/test_lmbench.py
index 9640ce000..c4ac347f4 100644
--- a/yardstick/tests/unit/benchmark/scenarios/compute/test_lmbench.py
+++ b/yardstick/tests/unit/benchmark/scenarios/compute/test_lmbench.py
@@ -17,6 +17,7 @@ import mock
 from oslo_serialization import jsonutils
 
 from yardstick.benchmark.scenarios.compute import lmbench
+from yardstick.common import exceptions as y_exc
 
 
 # pylint: disable=unused-argument
@@ -144,7 +145,7 @@ class LmbenchTestCase(unittest.TestCase):
 
         sample_output = '[{"latency": 37.5, "size": 0.00049}]'
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, l.run, self.result)
+        self.assertRaises(y_exc.SLAValidationError, l.run, self.result)
 
     def test_unsuccessful_bandwidth_run_sla(self, mock_ssh):
 
@@ -162,7 +163,7 @@ class LmbenchTestCase(unittest.TestCase):
 
         sample_output = '{"size(MB)": 0.262144, "bandwidth(MBps)": 9925.5}'
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, l.run, self.result)
+        self.assertRaises(y_exc.SLAValidationError, l.run, self.result)
 
     def test_successful_latency_for_cache_run_sla(self, mock_ssh):
 
diff --git a/yardstick/tests/unit/benchmark/scenarios/compute/test_qemumigrate.py b/yardstick/tests/unit/benchmark/scenarios/compute/test_qemumigrate.py
index 03003d01f..02040ca01 100644
--- a/yardstick/tests/unit/benchmark/scenarios/compute/test_qemumigrate.py
+++ b/yardstick/tests/unit/benchmark/scenarios/compute/test_qemumigrate.py
@@ -17,6 +17,7 @@ import mock
 from oslo_serialization import jsonutils
 
 from yardstick.benchmark.scenarios.compute import qemu_migrate
+from yardstick.common import exceptions as y_exc
 
 
 @mock.patch('yardstick.benchmark.scenarios.compute.qemu_migrate.ssh')
@@ -116,7 +117,7 @@ class QemuMigrateTestCase(unittest.TestCase):
         sample_output = '{"totaltime": 15, "downtime": 2, "setuptime": 1}'
 
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, q.run, result)
+        self.assertRaises(y_exc.SLAValidationError, q.run, result)
 
     def test_qemu_migrate_unsuccessful_sla_downtime(self, mock_ssh):
 
@@ -129,7 +130,7 @@ class QemuMigrateTestCase(unittest.TestCase):
         sample_output = '{"totaltime": 15, "downtime": 2, "setuptime": 1}'
 
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, q.run, result)
+        self.assertRaises(y_exc.SLAValidationError, q.run, result)
 
     def test_qemu_migrate_unsuccessful_sla_setuptime(self, mock_ssh):
 
@@ -142,7 +143,7 @@ class QemuMigrateTestCase(unittest.TestCase):
         sample_output = '{"totaltime": 15, "downtime": 2, "setuptime": 1}'
 
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, q.run, result)
+        self.assertRaises(y_exc.SLAValidationError, q.run, result)
 
     def test_qemu_migrate_unsuccessful_script_error(self, mock_ssh):
 
diff --git a/yardstick/tests/unit/benchmark/scenarios/compute/test_ramspeed.py b/yardstick/tests/unit/benchmark/scenarios/compute/test_ramspeed.py
index dcc0e810d..9e055befe 100644
--- a/yardstick/tests/unit/benchmark/scenarios/compute/test_ramspeed.py
+++ b/yardstick/tests/unit/benchmark/scenarios/compute/test_ramspeed.py
@@ -18,6 +18,7 @@ from oslo_serialization import jsonutils
 
 from yardstick.common import utils
 from yardstick.benchmark.scenarios.compute import ramspeed
+from yardstick.common import exceptions as y_exc
 
 
 @mock.patch('yardstick.benchmark.scenarios.compute.ramspeed.ssh')
@@ -146,7 +147,7 @@ class RamspeedTestCase(unittest.TestCase):
  "Block_size(kb)": 16384, "Bandwidth(MBps)": 14128.94}, {"Test_type":\
  "INTEGER & WRITING", "Block_size(kb)": 32768, "Bandwidth(MBps)": 8340.85}]}'
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, r.run, self.result)
+        self.assertRaises(y_exc.SLAValidationError, r.run, self.result)
 
     def test_ramspeed_unsuccessful_script_error(self, mock_ssh):
         options = {
@@ -219,7 +220,7 @@ class RamspeedTestCase(unittest.TestCase):
  "Bandwidth(MBps)": 1300.27}, {"Test_type": "INTEGER AVERAGE:",\
  "Bandwidth(MBps)": 2401.58}]}'
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, r.run, self.result)
+        self.assertRaises(y_exc.SLAValidationError, r.run, self.result)
 
     def test_ramspeed_unsuccessful_unknown_type_run(self, mock_ssh):
         options = {
diff --git a/yardstick/tests/unit/benchmark/scenarios/compute/test_unixbench.py b/yardstick/tests/unit/benchmark/scenarios/compute/test_unixbench.py
index 6339a2dcd..e4a8d6e26 100644
--- a/yardstick/tests/unit/benchmark/scenarios/compute/test_unixbench.py
+++ b/yardstick/tests/unit/benchmark/scenarios/compute/test_unixbench.py
@@ -17,6 +17,7 @@ import mock
 from oslo_serialization import jsonutils
 
 from yardstick.benchmark.scenarios.compute import unixbench
+from yardstick.common import exceptions as y_exc
 
 
 @mock.patch('yardstick.benchmark.scenarios.compute.unixbench.ssh')
@@ -122,7 +123,7 @@ class UnixbenchTestCase(unittest.TestCase):
         sample_output = '{"single_score":"200.7","parallel_score":"4395.9"}'
 
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, u.run, result)
+        self.assertRaises(y_exc.SLAValidationError, u.run, result)
 
     def test_unixbench_unsuccessful_sla_parallel_score(self, mock_ssh):
 
@@ -137,7 +138,7 @@ class UnixbenchTestCase(unittest.TestCase):
         sample_output = '{"signle_score":"2251.7","parallel_score":"3395.9"}'
 
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, u.run, result)
+        self.assertRaises(y_exc.SLAValidationError, u.run, result)
 
     def test_unixbench_unsuccessful_script_error(self, mock_ssh):
 
diff --git a/yardstick/tests/unit/benchmark/scenarios/networking/test_iperf3.py b/yardstick/tests/unit/benchmark/scenarios/networking/test_iperf3.py
index 74144afd5..2190e9337 100644
--- a/yardstick/tests/unit/benchmark/scenarios/networking/test_iperf3.py
+++ b/yardstick/tests/unit/benchmark/scenarios/networking/test_iperf3.py
@@ -19,6 +19,7 @@ from oslo_serialization import jsonutils
 
 from yardstick.common import utils
 from yardstick.benchmark.scenarios.networking import iperf3
+from yardstick.common import exceptions as y_exc
 
 
 @mock.patch('yardstick.benchmark.scenarios.networking.iperf3.ssh')
@@ -118,7 +119,7 @@ class IperfTestCase(unittest.TestCase):
 
         sample_output = self._read_sample_output(self.output_name_tcp)
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, result)
+        self.assertRaises(y_exc.SLAValidationError, p.run, result)
 
     def test_iperf_successful_sla_jitter(self, mock_ssh):
         options = {"protocol": "udp", "bandwidth": "20m"}
@@ -152,7 +153,7 @@ class IperfTestCase(unittest.TestCase):
 
         sample_output = self._read_sample_output(self.output_name_udp)
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, result)
+        self.assertRaises(y_exc.SLAValidationError, p.run, result)
 
     def test_iperf_successful_tcp_protocal(self, mock_ssh):
         options = {"protocol": "tcp", "nodelay": "yes"}
diff --git a/yardstick/tests/unit/benchmark/scenarios/networking/test_netperf.py b/yardstick/tests/unit/benchmark/scenarios/networking/test_netperf.py
index 5907562c2..a7abcd98a 100755
--- a/yardstick/tests/unit/benchmark/scenarios/networking/test_netperf.py
+++ b/yardstick/tests/unit/benchmark/scenarios/networking/test_netperf.py
@@ -18,6 +18,7 @@ import mock
 from oslo_serialization import jsonutils
 
 from yardstick.benchmark.scenarios.networking import netperf
+from yardstick.common import exceptions as y_exc
 
 
 @mock.patch('yardstick.benchmark.scenarios.networking.netperf.ssh')
@@ -98,7 +99,7 @@ class NetperfTestCase(unittest.TestCase):
 
         sample_output = self._read_sample_output()
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, result)
+        self.assertRaises(y_exc.SLAValidationError, p.run, result)
 
     def test_netperf_unsuccessful_script_error(self, mock_ssh):
 
diff --git a/yardstick/tests/unit/benchmark/scenarios/networking/test_netperf_node.py b/yardstick/tests/unit/benchmark/scenarios/networking/test_netperf_node.py
index 956a9c078..a577dba59 100755
--- a/yardstick/tests/unit/benchmark/scenarios/networking/test_netperf_node.py
+++ b/yardstick/tests/unit/benchmark/scenarios/networking/test_netperf_node.py
@@ -19,6 +19,7 @@ import mock
 from oslo_serialization import jsonutils
 
 from yardstick.benchmark.scenarios.networking import netperf_node
+from yardstick.common import exceptions as y_exc
 
 
 @mock.patch('yardstick.benchmark.scenarios.networking.netperf_node.ssh')
@@ -98,7 +99,7 @@ class NetperfNodeTestCase(unittest.TestCase):
 
         sample_output = self._read_sample_output()
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, result)
+        self.assertRaises(y_exc.SLAValidationError, p.run, result)
 
     def test_netperf_node_unsuccessful_script_error(self, mock_ssh):
 
diff --git a/yardstick/tests/unit/benchmark/scenarios/networking/test_ping.py b/yardstick/tests/unit/benchmark/scenarios/networking/test_ping.py
index 4adfab120..559e0599e 100644
--- a/yardstick/tests/unit/benchmark/scenarios/networking/test_ping.py
+++ b/yardstick/tests/unit/benchmark/scenarios/networking/test_ping.py
@@ -14,6 +14,7 @@ import mock
 import unittest
 
 from yardstick.benchmark.scenarios.networking import ping
+from yardstick.common import exceptions as y_exc
 
 
 class PingTestCase(unittest.TestCase):
@@ -74,7 +75,7 @@ class PingTestCase(unittest.TestCase):
         p = ping.Ping(args, self.ctx)
 
         mock_ssh.SSH.from_node().execute.return_value = (0, '100', '')
-        self.assertRaises(AssertionError, p.run, result)
+        self.assertRaises(y_exc.SLAValidationError, p.run, result)
 
     @mock.patch('yardstick.benchmark.scenarios.networking.ping.ssh')
     def test_ping_unsuccessful_script_error(self, mock_ssh):
diff --git a/yardstick/tests/unit/benchmark/scenarios/networking/test_ping6.py b/yardstick/tests/unit/benchmark/scenarios/networking/test_ping6.py
index 4662c8537..ad5217a14 100644
--- a/yardstick/tests/unit/benchmark/scenarios/networking/test_ping6.py
+++ b/yardstick/tests/unit/benchmark/scenarios/networking/test_ping6.py
@@ -14,6 +14,7 @@ import mock
 import unittest
 
 from yardstick.benchmark.scenarios.networking import ping6
+from yardstick.common import exceptions as y_exc
 
 
 class PingTestCase(unittest.TestCase):
@@ -98,7 +99,7 @@ class PingTestCase(unittest.TestCase):
         p = ping6.Ping6(args, self.ctx)
         p.client = mock_ssh.SSH.from_node()
         mock_ssh.SSH.from_node().execute.side_effect = [(0, 'host1', ''), (0, 100, '')]
-        self.assertRaises(AssertionError, p.run, result)
+        self.assertRaises(y_exc.SLAValidationError, p.run, result)
 
     @mock.patch('yardstick.benchmark.scenarios.networking.ping6.ssh')
     def test_ping_unsuccessful_script_error(self, mock_ssh):
diff --git a/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen.py b/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen.py
index 6aea03aee..ea0deab3e 100644
--- a/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen.py
+++ b/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen.py
@@ -13,6 +13,7 @@ import unittest
 from oslo_serialization import jsonutils
 
 from yardstick.benchmark.scenarios.networking import pktgen
+from yardstick.common import exceptions as y_exc
 
 
 @mock.patch('yardstick.benchmark.scenarios.networking.pktgen.ssh')
@@ -176,7 +177,7 @@ class PktgenTestCase(unittest.TestCase):
         sample_output = '{"packets_per_second": 9753, "errors": 0, \
             "packets_sent": 149776, "packetsize": 60, "flows": 110}'
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, result)
+        self.assertRaises(y_exc.SLAValidationError, p.run, result)
 
     def test_pktgen_unsuccessful_script_error(self, mock_ssh):
 
diff --git a/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen_dpdk.py b/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen_dpdk.py
index 976087148..b141591f7 100644
--- a/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen_dpdk.py
+++ b/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen_dpdk.py
@@ -12,6 +12,7 @@ import unittest
 
 import yardstick.common.utils as utils
 from yardstick.benchmark.scenarios.networking import pktgen_dpdk
+from yardstick.common import exceptions as y_exc
 
 
 class PktgenDPDKLatencyTestCase(unittest.TestCase):
@@ -162,7 +163,7 @@ class PktgenDPDKLatencyTestCase(unittest.TestCase):
 
         sample_output = '100\n110\n112\n130\n149\n150\n90\n150\n200\n162\n'
         self.mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, result)
+        self.assertRaises(y_exc.SLAValidationError, p.run, result)
 
     def test_pktgen_dpdk_unsuccessful_script_error(self):
 
diff --git a/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen_dpdk_throughput.py b/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen_dpdk_throughput.py
index e90fb07c7..39392e4bb 100644
--- a/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen_dpdk_throughput.py
+++ b/yardstick/tests/unit/benchmark/scenarios/networking/test_pktgen_dpdk_throughput.py
@@ -16,6 +16,7 @@ from oslo_serialization import jsonutils
 import mock
 
 from yardstick.benchmark.scenarios.networking import pktgen_dpdk_throughput
+from yardstick.common import exceptions as y_exc
 
 
 # pylint: disable=unused-argument
@@ -131,7 +132,7 @@ class PktgenDPDKTestCase(unittest.TestCase):
         sample_output = '{"packets_per_second": 9753, "errors": 0, \
             "packets_sent": 149776, "flows": 110}'
         mock_ssh.SSH().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, result)
+        self.assertRaises(y_exc.SLAValidationError, p.run, result)
 
     def test_pktgen_dpdk_throughput_unsuccessful_script_error(
             self, mock_ssh):
diff --git a/yardstick/tests/unit/benchmark/scenarios/storage/test_fio.py b/yardstick/tests/unit/benchmark/scenarios/storage/test_fio.py
index f149cee69..6e69ddc6d 100644
--- a/yardstick/tests/unit/benchmark/scenarios/storage/test_fio.py
+++ b/yardstick/tests/unit/benchmark/scenarios/storage/test_fio.py
@@ -18,6 +18,7 @@ import mock
 from oslo_serialization import jsonutils
 
 from yardstick.benchmark.scenarios.storage import fio
+from yardstick.common import exceptions as y_exc
 
 
 @mock.patch('yardstick.benchmark.scenarios.storage.fio.ssh')
@@ -203,7 +204,7 @@ class FioTestCase(unittest.TestCase):
 
         sample_output = self._read_sample_output(self.sample_output['rw'])
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, result)
+        self.assertRaises(y_exc.SLAValidationError, p.run, result)
 
     def test_fio_successful_bw_iops_sla(self, mock_ssh):
 
@@ -252,7 +253,7 @@ class FioTestCase(unittest.TestCase):
 
         sample_output = self._read_sample_output(self.sample_output['rw'])
         mock_ssh.SSH.from_node().execute.return_value = (0, sample_output, '')
-        self.assertRaises(AssertionError, p.run, result)
+        self.assertRaises(y_exc.SLAValidationError, p.run, result)
 
     def test_fio_unsuccessful_script_error(self, mock_ssh):