NSB PROX test hang fixes

The PROX tests were hanging in the duration
runner.

These are fixes for various errors:

raise error in collect_kpi if VNF is down

move prox dpdk_rebind after collectd stop
fix dpdk nicbind rebind to group by drivers

prox: raise error in collect_kpi if the VNF is down
prox: add VNF_TYPE for consistency

sample_vnf: debug and fix kill_vnf
pkill is not matching some executable names,
add some debug process dumps and try switching
back to killall until we can find the issue

sample_vnf: add default timeout, so we can override
default 3600 SSH timeout

collect_kpi is the point at which we check
the VNFs and TGs for failures or exits

queues are the problem make sure we aren't silently blocking on
non-empty queues by canceling join thread in subprocess

fixup duration runner to close queues
and other attempt to stop duration runner
from hanging

VnfdHelper: memoize port_num

resource: fail if ssh can't connect
at the end of 3600 second test our ssh connection
is dead, so we can't actually stop collectd
unless we reconnect

fix stop() logic to ignore ssh errors

Change-Id: I6c8e682a80cb9d00362e2fef4a46df080f304e55
Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>

1 files changed, 25 insertions, 11 deletions

diff --git a/tests/unit/network_services/nfvi/test_resource.py b/tests/unit/network_services/nfvi/test_resource.py
index 4fc6d7773..799cc202b 100644
--- a/tests/unit/network_services/nfvi/test_resource.py
+++ b/tests/unit/network_services/nfvi/test_resource.py
@@ -14,6 +14,8 @@
 
 from __future__ import absolute_import
 import unittest
+
+import errno
 import mock
 
 from yardstick.network_services.nfvi.resource import ResourceProfile
@@ -88,7 +90,7 @@ class TestResourceProfile(unittest.TestCase):
         with mock.patch("yardstick.ssh.AutoConnectSSH") as ssh:
             self.ssh_mock = mock.Mock(autospec=ssh.SSH)
             self.ssh_mock.execute = \
-                mock.Mock(return_value=(0, {}, ""))
+                mock.Mock(return_value=(0, "", ""))
             ssh.from_node.return_value = self.ssh_mock
 
             mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
@@ -105,7 +107,12 @@ class TestResourceProfile(unittest.TestCase):
 
     def test_check_if_sa_running(self):
         self.assertEqual(self.resource_profile.check_if_sa_running("collectd"),
-                         (True, {}))
+                         (0, ""))
+
+    def test_check_if_sa_running_excetion(self):
+        with mock.patch.object(self.resource_profile.connection, "execute") as mock_execute:
+            mock_execute.side_effect = OSError(errno.ECONNRESET, "error")
+            self.assertEqual(self.resource_profile.check_if_sa_running("collectd"), (1, None))
 
     def test_get_cpu_data(self):
         reskey = ["", "cpufreq", "cpufreq-0"]
@@ -127,7 +134,6 @@ class TestResourceProfile(unittest.TestCase):
             self.assertIsNone(
                 self.resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
 
-
     @mock.patch("yardstick.network_services.nfvi.resource.open")
     @mock.patch("yardstick.network_services.nfvi.resource.os")
     def test__provide_config_file(self, mock_open, mock_os):
@@ -141,13 +147,18 @@ class TestResourceProfile(unittest.TestCase):
         self.resource_profile._provide_config_file("/opt/nsb_bin", "collectd.conf", kwargs)
         self.ssh_mock.execute.assert_called_once()
 
-
     @mock.patch("yardstick.network_services.nfvi.resource.open")
     def test_initiate_systemagent(self, mock_open):
         self.resource_profile._start_collectd = mock.Mock()
         self.assertIsNone(
             self.resource_profile.initiate_systemagent("/opt/nsb_bin"))
 
+    @mock.patch("yardstick.network_services.nfvi.resource.open")
+    def test_initiate_systemagent_raise(self, mock_open):
+        self.resource_profile._start_collectd = mock.Mock(side_effect=RuntimeError)
+        with self.assertRaises(RuntimeError):
+            self.resource_profile.initiate_systemagent("/opt/nsb_bin")
+
     def test__parse_hugepages(self):
         reskey = ["cpu", "cpuFreq"]
         value = "timestamp:12345"
@@ -173,7 +184,7 @@ class TestResourceProfile(unittest.TestCase):
         self.assertEqual({'ovs/stats': '45'}, res)
 
     def test_parse_collectd_result(self):
-        res = self.resource_profile.parse_collectd_result({}, [0, 1, 2])
+        res = self.resource_profile.parse_collectd_result({})
         expected_result = {'cpu': {}, 'dpdkstat': {}, 'hugepages': {},
                            'memory': {}, 'ovs_stats': {}, 'timestamp': '',
                            'intel_pmu': {},
@@ -186,7 +197,7 @@ class TestResourceProfile(unittest.TestCase):
                                                                      "ipc",
                                                                      "1234",
                                                                      ""])
-        res = self.resource_profile.parse_collectd_result(metric, [0, 1, 2])
+        res = self.resource_profile.parse_collectd_result(metric)
         expected_result = {'cpu': {1: {'ipc': '1234'}}, 'dpdkstat': {}, 'hugepages': {},
                            'memory': {}, 'ovs_stats': {}, 'timestamp': '',
                            'intel_pmu': {},
@@ -195,7 +206,7 @@ class TestResourceProfile(unittest.TestCase):
 
     def test_parse_collectd_result_memory(self):
         metric = {"nsb_stats/memory/bw": "101"}
-        res = self.resource_profile.parse_collectd_result(metric, [0, 1, 2])
+        res = self.resource_profile.parse_collectd_result(metric)
         expected_result = {'cpu': {}, 'dpdkstat': {}, 'hugepages': {},
                            'memory': {'bw': '101'}, 'ovs_stats': {}, 'timestamp': '',
                            'intel_pmu': {},
@@ -205,9 +216,8 @@ class TestResourceProfile(unittest.TestCase):
     def test_parse_collectd_result_hugepage(self):
         # amqp returns bytes
         metric = {b"nsb_stats/hugepages/free": b"101"}
-        self.resource_profile.parse_hugepages = \
-        mock.Mock(return_value={"free": "101"})
-        res = self.resource_profile.parse_collectd_result(metric, [0, 1, 2])
+        self.resource_profile.parse_hugepages = mock.Mock(return_value={"free": "101"})
+        res = self.resource_profile.parse_collectd_result(metric)
         expected_result = {'cpu': {}, 'dpdkstat': {}, 'hugepages': {'free': '101'},
                            'memory': {}, 'ovs_stats': {}, 'timestamp': '',
                            'intel_pmu': {},
@@ -224,7 +234,7 @@ class TestResourceProfile(unittest.TestCase):
             mock.Mock(return_value={"memory": "101"})
         self.resource_profile.parse_ovs_stats = \
             mock.Mock(return_value={"tx": "101"})
-        res = self.resource_profile.parse_collectd_result(metric, [0, 1, 2])
+        res = self.resource_profile.parse_collectd_result(metric)
         expected_result = {'cpu': {}, 'dpdkstat': {'tx': '101'}, 'hugepages': {},
                            'memory': {}, 'ovs_stats': {'tx': '101'}, 'timestamp': '',
                            'intel_pmu': {},
@@ -258,5 +268,9 @@ class TestResourceProfile(unittest.TestCase):
     def test_stop(self):
         self.assertIsNone(self.resource_profile.stop())
 
+    def test_stop(self):
+        self.resource_profile.amqp_client = mock.MagicMock()
+        self.assertIsNone(self.resource_profile.stop())
+
 if __name__ == '__main__':
     unittest.main()