Merge changes from topic 'convert-sla-asserts-to-raises'

* changes:
  Replace MagicMocks with Mocks from SearchRunner UTs
  Add vsperf_dpdk SLA validation unit test cases
  Cleanup vsperf unit tests

2 files changed, 184 insertions, 60 deletions

diff --git a/yardstick/tests/unit/benchmark/scenarios/networking/test_vsperf.py b/yardstick/tests/unit/benchmark/scenarios/networking/test_vsperf.py
index 419605b26..a606543e5 100644
--- a/yardstick/tests/unit/benchmark/scenarios/networking/test_vsperf.py
+++ b/yardstick/tests/unit/benchmark/scenarios/networking/test_vsperf.py
@@ -12,31 +12,26 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Unittest for yardstick.benchmark.scenarios.networking.vsperf.Vsperf
-
-from __future__ import absolute_import
-try:
-    from unittest import mock
-except ImportError:
-    import mock
+import mock
 import unittest
+import subprocess
+import yardstick.ssh as ssh
 
 from yardstick.benchmark.scenarios.networking import vsperf
+from yardstick import exceptions as y_exc
 
 
-@mock.patch('yardstick.benchmark.scenarios.networking.vsperf.subprocess')
-@mock.patch('yardstick.benchmark.scenarios.networking.vsperf.ssh')
 class VsperfTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.ctx = {
+        self.context_cfg = {
             "host": {
                 "ip": "10.229.47.137",
                 "user": "ubuntu",
                 "password": "ubuntu",
             },
         }
-        self.args = {
+        self.scenario_cfg = {
             'options': {
                 'testname': 'p2p_rfc2544_continuous',
                 'traffic_type': 'continuous',
@@ -57,70 +52,154 @@ class VsperfTestCase(unittest.TestCase):
             }
         }
 
-    def test_vsperf_setup(self, mock_ssh, mock_subprocess):
-        p = vsperf.Vsperf(self.args, self.ctx)
-        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
-        mock_subprocess.call().execute.return_value = None
+        self._mock_SSH = mock.patch.object(ssh, 'SSH')
+        self.mock_SSH = self._mock_SSH.start()
+        self.mock_SSH.from_node().execute.return_value = (0, '', '')
+
+        self._mock_subprocess_call = mock.patch.object(subprocess, 'call')
+        self.mock_subprocess_call = self._mock_subprocess_call.start()
+        self.mock_subprocess_call.return_value = None
+
+        self.addCleanup(self._stop_mock)
+
+        self.scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
+
+    def _stop_mock(self):
+        self._mock_SSH.stop()
+        self._mock_subprocess_call.stop()
+
+    def test_setup(self):
+        self.scenario.setup()
+        self.assertIsNotNone(self.scenario.client)
+        self.assertTrue(self.scenario.setup_done)
+
+    def test_setup_tg_port_not_set(self):
+        del self.scenario_cfg['options']['trafficgen_port1']
+        del self.scenario_cfg['options']['trafficgen_port2']
+        scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
+        scenario.setup()
+
+        self.mock_subprocess_call.assert_called_once_with(
+            'setup_yardstick.sh setup', shell=True)
+        self.assertIsNone(scenario.tg_port1)
+        self.assertIsNone(scenario.tg_port2)
+        self.assertIsNotNone(scenario.client)
+        self.assertTrue(scenario.setup_done)
+
+    def test_setup_no_setup_script(self):
+        del self.scenario_cfg['options']['setup_script']
+        scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
+        scenario.setup()
+
+        self.mock_subprocess_call.assert_has_calls(
+            (mock.call('sudo bash -c "ovs-vsctl add-port br-ex eth1"',
+                       shell=True),
+             mock.call('sudo bash -c "ovs-vsctl add-port br-ex eth3"',
+                       shell=True)))
+        self.assertEqual(2, self.mock_subprocess_call.call_count)
+        self.assertIsNone(scenario.setup_script)
+        self.assertIsNotNone(scenario.client)
+        self.assertTrue(scenario.setup_done)
+
+    def test_run_ok(self):
+        self.scenario.setup()
+
+        self.mock_SSH.from_node().execute.return_value = (
+            0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
 
-        p.setup()
-        self.assertIsNotNone(p.client)
-        self.assertTrue(p.setup_done)
+        result = {}
+        self.scenario.run(result)
 
-    def test_vsperf_teardown(self, mock_ssh, mock_subprocess):
-        p = vsperf.Vsperf(self.args, self.ctx)
+        self.assertEqual(result['throughput_rx_fps'], '14797660.000')
 
-        # setup() specific mocks
-        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
-        mock_subprocess.call().execute.return_value = None
+    def test_run_ok_setup_not_done(self):
+        self.mock_SSH.from_node().execute.return_value = (
+            0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
 
-        p.setup()
-        self.assertIsNotNone(p.client)
-        self.assertTrue(p.setup_done)
+        result = {}
+        self.scenario.run(result)
 
-        p.teardown()
-        self.assertFalse(p.setup_done)
+        self.assertTrue(self.scenario.setup_done)
+        self.assertEqual(result['throughput_rx_fps'], '14797660.000')
 
-    def test_vsperf_run_ok(self, mock_ssh, mock_subprocess):
-        p = vsperf.Vsperf(self.args, self.ctx)
+    def test_run_failed_vsperf_execution(self):
+        self.mock_SSH.from_node().execute.side_effect = ((0, '', ''),
+                                                         (1, '', ''))
 
-        # setup() specific mocks
-        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
-        mock_subprocess.call().execute.return_value = None
+        with self.assertRaises(RuntimeError):
+            self.scenario.run({})
+        self.assertEqual(self.mock_SSH.from_node().execute.call_count, 2)
 
-        # run() specific mocks
-        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
-        mock_ssh.SSH.from_node().execute.return_value = (
-            0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
+    def test_run_failed_csv_report(self):
+        self.mock_SSH.from_node().execute.side_effect = ((0, '', ''),
+                                                         (0, '', ''),
+                                                         (1, '', ''))
 
-        result = {}
-        p.run(result)
+        with self.assertRaises(RuntimeError):
+            self.scenario.run({})
+        self.assertEqual(self.mock_SSH.from_node().execute.call_count, 3)
 
-        self.assertEqual(result['throughput_rx_fps'], '14797660.000')
+    def test_run_sla_fail(self):
+        self.mock_SSH.from_node().execute.return_value = (
+            0, 'throughput_rx_fps\r\n123456.000\r\n', '')
 
-    def test_vsperf_run_falied_vsperf_execution(self, mock_ssh,
-                                                mock_subprocess):
-        p = vsperf.Vsperf(self.args, self.ctx)
+        with self.assertRaises(y_exc.SLAValidationError) as raised:
+            self.scenario.run({})
 
-        # setup() specific mocks
-        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
-        mock_subprocess.call().execute.return_value = None
+        self.assertTrue('VSPERF_throughput_rx_fps(123456.000000) < '
+                        'SLA_throughput_rx_fps(500000.000000)'
+                        in str(raised.exception))
 
-        # run() specific mocks
-        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+    def test_run_sla_fail_metric_not_collected(self):
+        self.mock_SSH.from_node().execute.return_value = (
+            0, 'nonexisting_metric\r\n14797660.000\r\n', '')
 
-        result = {}
-        self.assertRaises(RuntimeError, p.run, result)
+        with self.assertRaises(y_exc.SLAValidationError) as raised:
+            self.scenario.run({})
 
-    def test_vsperf_run_falied_csv_report(self, mock_ssh, mock_subprocess):
-        p = vsperf.Vsperf(self.args, self.ctx)
+        self.assertTrue('throughput_rx_fps was not collected by VSPERF'
+                        in str(raised.exception))
 
-        # setup() specific mocks
-        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
-        mock_subprocess.call().execute.return_value = None
+    def test_run_sla_fail_metric_not_defined_in_sla(self):
+        del self.scenario_cfg['sla']['throughput_rx_fps']
+        scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
+        scenario.setup()
 
-        # run() specific mocks
-        mock_ssh.SSH.from_node().execute.return_value = (0, '', '')
-        mock_ssh.SSH.from_node().execute.return_value = (1, '', '')
+        self.mock_SSH.from_node().execute.return_value = (
+            0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
 
-        result = {}
-        self.assertRaises(RuntimeError, p.run, result)
+        with self.assertRaises(y_exc.SLAValidationError) as raised:
+            scenario.run({})
+        self.assertTrue('throughput_rx_fps is not defined in SLA'
+                        in str(raised.exception))
+
+    def test_teardown(self):
+        self.scenario.setup()
+        self.assertIsNotNone(self.scenario.client)
+        self.assertTrue(self.scenario.setup_done)
+
+        self.scenario.teardown()
+        self.assertFalse(self.scenario.setup_done)
+
+    def test_teardown_tg_port_not_set(self):
+        del self.scenario_cfg['options']['trafficgen_port1']
+        del self.scenario_cfg['options']['trafficgen_port2']
+        scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
+        scenario.teardown()
+
+        self.mock_subprocess_call.assert_called_once_with(
+            'setup_yardstick.sh teardown', shell=True)
+        self.assertFalse(scenario.setup_done)
+
+    def test_teardown_no_setup_script(self):
+        del self.scenario_cfg['options']['setup_script']
+        scenario = vsperf.Vsperf(self.scenario_cfg, self.context_cfg)
+        scenario.teardown()
+
+        self.mock_subprocess_call.assert_has_calls(
+            (mock.call('sudo bash -c "ovs-vsctl del-port br-ex eth1"',
+                       shell=True),
+             mock.call('sudo bash -c "ovs-vsctl del-port br-ex eth3"',
+                       shell=True)))
+        self.assertEqual(2, self.mock_subprocess_call.call_count)
+        self.assertFalse(scenario.setup_done)
diff --git a/yardstick/tests/unit/benchmark/scenarios/networking/test_vsperf_dpdk.py b/yardstick/tests/unit/benchmark/scenarios/networking/test_vsperf_dpdk.py
index 1d2278e21..c05d2ced2 100644
--- a/yardstick/tests/unit/benchmark/scenarios/networking/test_vsperf_dpdk.py
+++ b/yardstick/tests/unit/benchmark/scenarios/networking/test_vsperf_dpdk.py
@@ -19,6 +19,7 @@ import mock
 import unittest
 
 from yardstick.benchmark.scenarios.networking import vsperf_dpdk
+from yardstick import exceptions as y_exc
 
 
 class VsperfDPDKTestCase(unittest.TestCase):
@@ -211,3 +212,47 @@ class VsperfDPDKTestCase(unittest.TestCase):
 
         result = {}
         self.assertRaises(RuntimeError, self.scenario.run, result)
+
+    @mock.patch.object(time, 'sleep')
+    @mock.patch.object(subprocess, 'check_output')
+    def test_vsperf_run_sla_fail(self, *args):
+        self.scenario.setup()
+
+        self.mock_ssh.SSH.from_node().execute.return_value = (
+            0, 'throughput_rx_fps\r\n123456.000\r\n', '')
+
+        with self.assertRaises(y_exc.SLAValidationError) as raised:
+            self.scenario.run({})
+
+        self.assertIn('VSPERF_throughput_rx_fps(123456.000000) < '
+                      'SLA_throughput_rx_fps(500000.000000)',
+                      str(raised.exception))
+
+    @mock.patch.object(time, 'sleep')
+    @mock.patch.object(subprocess, 'check_output')
+    def test_vsperf_run_sla_fail_metric_not_collected(self, *args):
+        self.scenario.setup()
+
+        self.mock_ssh.SSH.from_node().execute.return_value = (
+            0, 'nonexisting_metric\r\n123456.000\r\n', '')
+
+        with self.assertRaises(y_exc.SLAValidationError) as raised:
+            self.scenario.run({})
+
+        self.assertIn('throughput_rx_fps was not collected by VSPERF',
+                      str(raised.exception))
+
+    @mock.patch.object(time, 'sleep')
+    @mock.patch.object(subprocess, 'check_output')
+    def test_vsperf_run_sla_fail_sla_not_defined(self, *args):
+        del self.scenario.scenario_cfg['sla']['throughput_rx_fps']
+        self.scenario.setup()
+
+        self.mock_ssh.SSH.from_node().execute.return_value = (
+            0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
+
+        with self.assertRaises(y_exc.SLAValidationError) as raised:
+            self.scenario.run({})
+
+        self.assertIn('throughput_rx_fps is not defined in SLA',
+                      str(raised.exception))