diff options
30 files changed, 440 insertions, 130 deletions
diff --git a/tests/unit/benchmark/scenarios/networking/test_vnf_generic.py b/tests/unit/benchmark/scenarios/networking/test_vnf_generic.py index 5b15daca4..016608a21 100644 --- a/tests/unit/benchmark/scenarios/networking/test_vnf_generic.py +++ b/tests/unit/benchmark/scenarios/networking/test_vnf_generic.py @@ -445,6 +445,13 @@ class TestNetworkServiceTestCase(unittest.TestCase): self.assertIsNotNone( self.s.load_vnf_models(self.scenario_cfg, self.context_cfg)) + def test_load_vnf_models_no_model(self): + vnf = mock.Mock(autospec=GenericVNF) + self.s.get_vnf_impl = mock.Mock(return_value=vnf) + + self.assertIsNotNone( + self.s.load_vnf_models(self.scenario_cfg, self.context_cfg)) + def test_map_topology_to_infrastructure(self): with mock.patch("yardstick.ssh.SSH") as ssh: ssh_mock = mock.Mock(autospec=ssh.SSH) @@ -568,6 +575,35 @@ class TestNetworkServiceTestCase(unittest.TestCase): mock.Mock(return_value=TRAFFIC_PROFILE) self.assertEqual(None, self.s.setup()) + def test_setup_exception(self): + with mock.patch("yardstick.ssh.SSH") as ssh: + ssh_mock = mock.Mock(autospec=ssh.SSH) + ssh_mock.execute = \ + mock.Mock(return_value=(0, SYS_CLASS_NET + IP_ADDR_SHOW, "")) + ssh.from_node.return_value = ssh_mock + + tgen = mock.Mock(autospec=GenericTrafficGen) + tgen.traffic_finished = True + verified_dict = {"verified": True} + tgen.verify_traffic = lambda x: verified_dict + tgen.terminate = mock.Mock(return_value=True) + tgen.name = "tgen__1" + vnf = mock.Mock(autospec=GenericVNF) + vnf.runs_traffic = False + vnf.instantiate.side_effect = RuntimeError("error during instantiate") + vnf.terminate = mock.Mock(return_value=True) + self.s.vnfs = [tgen, vnf] + self.s.traffic_profile = mock.Mock() + self.s.collector = mock.Mock(autospec=Collector) + self.s.collector.get_kpi = \ + mock.Mock(return_value={tgen.name: verified_dict}) + self.s.map_topology_to_infrastructure = mock.Mock(return_value=0) + self.s.load_vnf_models = mock.Mock(return_value=self.s.vnfs) + self.s._fill_traffic_profile = \ + mock.Mock(return_value=TRAFFIC_PROFILE) + with self.assertRaises(RuntimeError): + self.s.setup() + def test__get_traffic_profile(self): self.scenario_cfg["traffic_profile"] = \ self._get_file_abspath("ipv4_throughput_vpe.yaml") @@ -594,8 +630,8 @@ class TestNetworkServiceTestCase(unittest.TestCase): def test_teardown(self): vnf = mock.Mock(autospec=GenericVNF) - vnf.terminate = \ - mock.Mock(return_value=True) + vnf.terminate = mock.Mock(return_value=True) + vnf.name = str(vnf) self.s.vnfs = [vnf] self.s.traffic_profile = mock.Mock() self.s.collector = mock.Mock(autospec=Collector) @@ -603,6 +639,18 @@ class TestNetworkServiceTestCase(unittest.TestCase): mock.Mock(return_value=True) self.assertIsNone(self.s.teardown()) + def test_teardown_exception(self): + vnf = mock.Mock(autospec=GenericVNF) + vnf.terminate = mock.Mock(side_effect=RuntimeError("error duing terminate")) + vnf.name = str(vnf) + self.s.vnfs = [vnf] + self.s.traffic_profile = mock.Mock() + self.s.collector = mock.Mock(autospec=Collector) + self.s.collector.stop = \ + mock.Mock(return_value=True) + with self.assertRaises(RuntimeError): + self.s.teardown() + SAMPLE_NETDEVS = { 'enp11s0': { 'address': '0a:de:ad:be:ef:f5', diff --git a/tests/unit/common/test_process.py b/tests/unit/common/test_process.py new file mode 100644 index 000000000..5eee55bcc --- /dev/null +++ b/tests/unit/common/test_process.py @@ -0,0 +1,46 @@ +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +import mock + +from yardstick.common import process + + +class ProcessTestcase(unittest.TestCase): + def test_check_if_procces_failed_None(self): + p = mock.MagicMock(**{"exitcode": None, "name": "debug"}) + process.check_if_process_failed(p) + + def test_check_if_procces_failed_0(self): + p = mock.MagicMock(**{"exitcode": 0, "name": "debug"}) + process.check_if_process_failed(p) + + def test_check_if_procces_failed_1(self): + p = mock.MagicMock(**{"exitcode": 1, "name": "debug"}) + with self.assertRaises(RuntimeError): + process.check_if_process_failed(p) + + +@mock.patch("yardstick.common.process.multiprocessing") +class TerminateChildrenTestcase(unittest.TestCase): + def test_some_children(self, mock_multiprocessing): + p1 = mock.MagicMock() + p2 = mock.MagicMock() + mock_multiprocessing.active_children.return_value = [p1, p2] + process.terminate_children() + + def test_no_children(self, mock_multiprocessing): + mock_multiprocessing.active_children.return_value = [] + process.terminate_children() diff --git a/tests/unit/network_services/collector/test_subscriber.py b/tests/unit/network_services/collector/test_subscriber.py index 260f0bb27..f324f627d 100644 --- a/tests/unit/network_services/collector/test_subscriber.py +++ b/tests/unit/network_services/collector/test_subscriber.py @@ -45,6 +45,7 @@ class CollectorTestCase(unittest.TestCase): NODES = { 'node1': {}, 'node2': { + 'ip': '1.2.3.4', 'collectd': { 'plugins': {'abc': 12, 'def': 34}, 'interval': 987, diff --git a/tests/unit/network_services/helpers/test_dpdkbindnic_helper.py b/tests/unit/network_services/helpers/test_dpdkbindnic_helper.py index dbd8396c8..0f1cf7d92 100644 --- a/tests/unit/network_services/helpers/test_dpdkbindnic_helper.py +++ b/tests/unit/network_services/helpers/test_dpdkbindnic_helper.py @@ -16,19 +16,19 @@ import mock import unittest -from yardstick.network_services.helpers.dpdknicbind_helper import DpdkBindHelper -from yardstick.network_services.helpers.dpdknicbind_helper import DpdkBindHelperException -from yardstick.network_services.helpers.dpdknicbind_helper import NETWORK_KERNEL -from yardstick.network_services.helpers.dpdknicbind_helper import NETWORK_DPDK -from yardstick.network_services.helpers.dpdknicbind_helper import CRYPTO_KERNEL -from yardstick.network_services.helpers.dpdknicbind_helper import CRYPTO_DPDK -from yardstick.network_services.helpers.dpdknicbind_helper import NETWORK_OTHER -from yardstick.network_services.helpers.dpdknicbind_helper import CRYPTO_OTHER +from yardstick.network_services.helpers.dpdkbindnic_helper import DpdkBindHelper +from yardstick.network_services.helpers.dpdkbindnic_helper import DpdkBindHelperException +from yardstick.network_services.helpers.dpdkbindnic_helper import NETWORK_KERNEL +from yardstick.network_services.helpers.dpdkbindnic_helper import NETWORK_DPDK +from yardstick.network_services.helpers.dpdkbindnic_helper import CRYPTO_KERNEL +from yardstick.network_services.helpers.dpdkbindnic_helper import CRYPTO_DPDK +from yardstick.network_services.helpers.dpdkbindnic_helper import NETWORK_OTHER +from yardstick.network_services.helpers.dpdkbindnic_helper import CRYPTO_OTHER pass -class MyTestDpdkBindHelper(unittest.TestCase): +class TestDpdkBindHelper(unittest.TestCase): EXAMPLE_OUTPUT = """ Network devices using DPDK-compatible driver @@ -204,17 +204,31 @@ Other crypto devices def test_bind(self): conn = mock.Mock() conn.execute = mock.Mock(return_value=(0, '', '')) - conn.provision_tool = mock.Mock(return_value='/opt/nsb_bin/dpdk_nic_bind.py') + conn.provision_tool = mock.Mock(return_value='/opt/nsb_bin/dpdk-devbind.py') dpdk_bind_helper = DpdkBindHelper(conn) dpdk_bind_helper.read_status = mock.Mock() dpdk_bind_helper.bind(['0000:00:03.0', '0000:00:04.0'], 'my_driver') - conn.execute.assert_called_with('sudo /opt/nsb_bin/dpdk_nic_bind.py --force ' + conn.execute.assert_called_with('sudo /opt/nsb_bin/dpdk-devbind.py --force ' '-b my_driver 0000:00:03.0 0000:00:04.0') dpdk_bind_helper.read_status.assert_called_once() + def test_bind_single_pci(self): + conn = mock.Mock() + conn.execute = mock.Mock(return_value=(0, '', '')) + conn.provision_tool = mock.Mock(return_value='/opt/nsb_bin/dpdk-devbind.py') + + dpdk_bind_helper = DpdkBindHelper(conn) + dpdk_bind_helper.read_status = mock.Mock() + + dpdk_bind_helper.bind('0000:00:03.0', 'my_driver') + + conn.execute.assert_called_with('sudo /opt/nsb_bin/dpdk-devbind.py --force ' + '-b my_driver 0000:00:03.0') + dpdk_bind_helper.read_status.assert_called_once() + def test_rebind_drivers(self): conn = mock.Mock() @@ -222,14 +236,14 @@ Other crypto devices dpdk_bind_helper.bind = mock.Mock() dpdk_bind_helper.used_drivers = { - '0000:05:00.0': 'd1', - '0000:05:01.0': 'd3', + 'd1': ['0000:05:00.0'], + 'd3': ['0000:05:01.0', '0000:05:02.0'], } dpdk_bind_helper.rebind_drivers() - dpdk_bind_helper.bind.assert_any_call('0000:05:00.0', 'd1', True) - dpdk_bind_helper.bind.assert_any_call('0000:05:01.0', 'd3', True) + dpdk_bind_helper.bind.assert_any_call(['0000:05:00.0'], 'd1', True) + dpdk_bind_helper.bind.assert_any_call(['0000:05:01.0', '0000:05:02.0'], 'd3', True) def test_save_used_drivers(self): conn = mock.Mock() @@ -239,9 +253,8 @@ Other crypto devices dpdk_bind_helper.save_used_drivers() expected = { - '0000:00:04.0': 'igb_uio', - '0000:00:05.0': 'igb_uio', - '0000:00:03.0': 'virtio-pci', + 'igb_uio': ['0000:00:04.0', '0000:00:05.0'], + 'virtio-pci': ['0000:00:03.0'], } - self.assertEqual(expected, dpdk_bind_helper.used_drivers) + self.assertDictEqual(expected, dpdk_bind_helper.used_drivers) diff --git a/tests/unit/network_services/nfvi/test_resource.py b/tests/unit/network_services/nfvi/test_resource.py index 4fc6d7773..799cc202b 100644 --- a/tests/unit/network_services/nfvi/test_resource.py +++ b/tests/unit/network_services/nfvi/test_resource.py @@ -14,6 +14,8 @@ from __future__ import absolute_import import unittest + +import errno import mock from yardstick.network_services.nfvi.resource import ResourceProfile @@ -88,7 +90,7 @@ class TestResourceProfile(unittest.TestCase): with mock.patch("yardstick.ssh.AutoConnectSSH") as ssh: self.ssh_mock = mock.Mock(autospec=ssh.SSH) self.ssh_mock.execute = \ - mock.Mock(return_value=(0, {}, "")) + mock.Mock(return_value=(0, "", "")) ssh.from_node.return_value = self.ssh_mock mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface'] @@ -105,7 +107,12 @@ class TestResourceProfile(unittest.TestCase): def test_check_if_sa_running(self): self.assertEqual(self.resource_profile.check_if_sa_running("collectd"), - (True, {})) + (0, "")) + + def test_check_if_sa_running_excetion(self): + with mock.patch.object(self.resource_profile.connection, "execute") as mock_execute: + mock_execute.side_effect = OSError(errno.ECONNRESET, "error") + self.assertEqual(self.resource_profile.check_if_sa_running("collectd"), (1, None)) def test_get_cpu_data(self): reskey = ["", "cpufreq", "cpufreq-0"] @@ -127,7 +134,6 @@ class TestResourceProfile(unittest.TestCase): self.assertIsNone( self.resource_profile._prepare_collectd_conf("/opt/nsb_bin")) - @mock.patch("yardstick.network_services.nfvi.resource.open") @mock.patch("yardstick.network_services.nfvi.resource.os") def test__provide_config_file(self, mock_open, mock_os): @@ -141,13 +147,18 @@ class TestResourceProfile(unittest.TestCase): self.resource_profile._provide_config_file("/opt/nsb_bin", "collectd.conf", kwargs) self.ssh_mock.execute.assert_called_once() - @mock.patch("yardstick.network_services.nfvi.resource.open") def test_initiate_systemagent(self, mock_open): self.resource_profile._start_collectd = mock.Mock() self.assertIsNone( self.resource_profile.initiate_systemagent("/opt/nsb_bin")) + @mock.patch("yardstick.network_services.nfvi.resource.open") + def test_initiate_systemagent_raise(self, mock_open): + self.resource_profile._start_collectd = mock.Mock(side_effect=RuntimeError) + with self.assertRaises(RuntimeError): + self.resource_profile.initiate_systemagent("/opt/nsb_bin") + def test__parse_hugepages(self): reskey = ["cpu", "cpuFreq"] value = "timestamp:12345" @@ -173,7 +184,7 @@ class TestResourceProfile(unittest.TestCase): self.assertEqual({'ovs/stats': '45'}, res) def test_parse_collectd_result(self): - res = self.resource_profile.parse_collectd_result({}, [0, 1, 2]) + res = self.resource_profile.parse_collectd_result({}) expected_result = {'cpu': {}, 'dpdkstat': {}, 'hugepages': {}, 'memory': {}, 'ovs_stats': {}, 'timestamp': '', 'intel_pmu': {}, @@ -186,7 +197,7 @@ class TestResourceProfile(unittest.TestCase): "ipc", "1234", ""]) - res = self.resource_profile.parse_collectd_result(metric, [0, 1, 2]) + res = self.resource_profile.parse_collectd_result(metric) expected_result = {'cpu': {1: {'ipc': '1234'}}, 'dpdkstat': {}, 'hugepages': {}, 'memory': {}, 'ovs_stats': {}, 'timestamp': '', 'intel_pmu': {}, @@ -195,7 +206,7 @@ class TestResourceProfile(unittest.TestCase): def test_parse_collectd_result_memory(self): metric = {"nsb_stats/memory/bw": "101"} - res = self.resource_profile.parse_collectd_result(metric, [0, 1, 2]) + res = self.resource_profile.parse_collectd_result(metric) expected_result = {'cpu': {}, 'dpdkstat': {}, 'hugepages': {}, 'memory': {'bw': '101'}, 'ovs_stats': {}, 'timestamp': '', 'intel_pmu': {}, @@ -205,9 +216,8 @@ class TestResourceProfile(unittest.TestCase): def test_parse_collectd_result_hugepage(self): # amqp returns bytes metric = {b"nsb_stats/hugepages/free": b"101"} - self.resource_profile.parse_hugepages = \ - mock.Mock(return_value={"free": "101"}) - res = self.resource_profile.parse_collectd_result(metric, [0, 1, 2]) + self.resource_profile.parse_hugepages = mock.Mock(return_value={"free": "101"}) + res = self.resource_profile.parse_collectd_result(metric) expected_result = {'cpu': {}, 'dpdkstat': {}, 'hugepages': {'free': '101'}, 'memory': {}, 'ovs_stats': {}, 'timestamp': '', 'intel_pmu': {}, @@ -224,7 +234,7 @@ class TestResourceProfile(unittest.TestCase): mock.Mock(return_value={"memory": "101"}) self.resource_profile.parse_ovs_stats = \ mock.Mock(return_value={"tx": "101"}) - res = self.resource_profile.parse_collectd_result(metric, [0, 1, 2]) + res = self.resource_profile.parse_collectd_result(metric) expected_result = {'cpu': {}, 'dpdkstat': {'tx': '101'}, 'hugepages': {}, 'memory': {}, 'ovs_stats': {'tx': '101'}, 'timestamp': '', 'intel_pmu': {}, @@ -258,5 +268,9 @@ class TestResourceProfile(unittest.TestCase): def test_stop(self): self.assertIsNone(self.resource_profile.stop()) + def test_stop(self): + self.resource_profile.amqp_client = mock.MagicMock() + self.assertIsNone(self.resource_profile.stop()) + if __name__ == '__main__': unittest.main() diff --git a/tests/unit/network_services/vnf_generic/vnf/test_cgnapt_vnf.py b/tests/unit/network_services/vnf_generic/vnf/test_cgnapt_vnf.py index 0a4c12446..b0ef1da91 100644 --- a/tests/unit/network_services/vnf_generic/vnf/test_cgnapt_vnf.py +++ b/tests/unit/network_services/vnf_generic/vnf/test_cgnapt_vnf.py @@ -41,7 +41,7 @@ TEST_FILE_YAML = 'nsb_test_case.yaml' SSH_HELPER = 'yardstick.network_services.vnf_generic.vnf.sample_vnf.VnfSshHelper' -name = 'vnf__1' +name = 'vnf__0' class TestCgnaptApproxSetupEnvHelper(unittest.TestCase): @@ -162,7 +162,7 @@ class TestCgnaptApproxVnf(unittest.TestCase): 'rfc2544': { 'allowed_drop_rate': '0.8 - 1', }, - 'vnf__1': { + 'vnf__0': { 'napt': 'dynamic', 'vnf_config': { 'lb_config': 'SW', @@ -172,6 +172,10 @@ class TestCgnaptApproxVnf(unittest.TestCase): 'worker_threads': 1, }, }, + 'flow': {'count': 1, + 'dst_ip': [{'tg__1': 'xe0'}], + 'public_ip': [''], + 'src_ip': [{'tg__0': 'xe0'}]}, }, 'task_id': 'a70bdf4a-8e67-47a3-9dc1-273c14506eb7', 'task_path': '/tmp', @@ -185,15 +189,11 @@ class TestCgnaptApproxVnf(unittest.TestCase): 'type': 'Duration', }, 'traffic_profile': 'ipv4_throughput_acl.yaml', - 'traffic_options': { - 'flow': 'ipv4_Packets_acl.yaml', - 'imix': 'imix_voice.yaml', - }, - 'type': 'ISB', + 'type': 'NSPerf', 'nodes': { - 'tg__2': 'trafficgen_2.yardstick', 'tg__1': 'trafficgen_1.yardstick', - 'vnf__1': 'vnf.yardstick', + 'tg__0': 'trafficgen_0.yardstick', + 'vnf__0': 'vnf.yardstick', }, 'topology': 'vpe-tg-topology-baremetal.yaml', } @@ -253,9 +253,9 @@ class TestCgnaptApproxVnf(unittest.TestCase): 'password': 'r00t', 'VNF model': 'tg_rfc2544_tpl.yaml', 'user': 'root'}, - 'vnf__1': + 'vnf__0': {'name': 'vnf.yardstick', - 'vnfd-id-ref': 'vnf__1', + 'vnfd-id-ref': 'vnf__0', 'ip': '1.2.1.1', 'interfaces': {'xe0': {'local_iface_name': 'ens786f0', @@ -318,6 +318,8 @@ class TestCgnaptApproxVnf(unittest.TestCase): vnfd = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0] cgnapt_approx_vnf = CgnaptApproxVnf(name, vnfd) + cgnapt_approx_vnf._vnf_process = mock.MagicMock( + **{"is_alive.return_value": True, "exitcode": None}) cgnapt_approx_vnf.q_in = mock.MagicMock() cgnapt_approx_vnf.q_out = mock.MagicMock() cgnapt_approx_vnf.q_out.qsize = mock.Mock(return_value=0) @@ -388,7 +390,7 @@ class TestCgnaptApproxVnf(unittest.TestCase): 'rules': ""}} cgnapt_approx_vnf.q_out.put("pipeline>") cgnapt_vnf.WAIT_TIME = 3 - self.scenario_cfg.update({"nodes": {"vnf__1": ""}}) + self.scenario_cfg.update({"nodes": {"vnf__0": ""}}) self.assertIsNone(cgnapt_approx_vnf.instantiate(self.scenario_cfg, self.context_cfg)) diff --git a/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py b/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py index e4319d602..54540ada1 100644 --- a/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py +++ b/tests/unit/network_services/vnf_generic/vnf/test_prox_helpers.py @@ -17,12 +17,10 @@ from __future__ import absolute_import -import copy import os import socket import unittest from itertools import repeat, chain -from contextlib import contextmanager import mock from tests.unit import STL_MOCKS @@ -1501,7 +1499,9 @@ class TestProxResourceHelper(unittest.TestCase): helper = ProxResourceHelper(mock.MagicMock()) helper.resource = resource = mock.MagicMock() + resource.check_if_sa_running.return_value = 0, '1234' resource.amqp_collect_nfvi_kpi.return_value = 543 + resource.check_if_sa_running.return_value = (0, None) expected = {'core': 543} result = helper.collect_collectd_kpi() @@ -1513,7 +1513,9 @@ class TestProxResourceHelper(unittest.TestCase): helper._result = {'z': 123} helper.resource = resource = mock.MagicMock() + resource.check_if_sa_running.return_value = 0, '1234' resource.amqp_collect_nfvi_kpi.return_value = 543 + resource.check_if_sa_running.return_value = (0, None) queue.empty.return_value = False queue.get.return_value = {'a': 789} @@ -1800,7 +1802,7 @@ class TestProxProfileHelper(unittest.TestCase): def test_latency_cores(self): resource_helper = mock.MagicMock() - resource_helper.setup_helper.prox_config_data= [] + resource_helper.setup_helper.prox_config_data = [] helper = ProxProfileHelper(resource_helper) helper._cpu_topology = [] @@ -1977,7 +1979,6 @@ class TestProxProfileHelper(unittest.TestCase): ]), ] - client = mock.MagicMock() client.hz.return_value = 2 client.port_stats.return_value = tuple(range(12)) diff --git a/tests/unit/network_services/vnf_generic/vnf/test_prox_vnf.py b/tests/unit/network_services/vnf_generic/vnf/test_prox_vnf.py index c88b1528c..09060ff57 100644 --- a/tests/unit/network_services/vnf_generic/vnf/test_prox_vnf.py +++ b/tests/unit/network_services/vnf_generic/vnf/test_prox_vnf.py @@ -85,7 +85,7 @@ class TestProxApproxVnf(unittest.TestCase): 'vpci': '0000:05:00.0', 'local_ip': '152.16.100.19', 'type': 'PCI-PASSTHROUGH', - 'vld_id': '', + 'vld_id': 'downlink_0', 'ifname': 'xe1', 'netmask': '255.255.255.0', 'dpdk_port_num': 0, @@ -104,8 +104,8 @@ class TestProxApproxVnf(unittest.TestCase): 'vpci': '0000:05:00.1', 'local_ip': '152.16.40.19', 'type': 'PCI-PASSTHROUGH', - 'vld_id': '', - 'ifname': 'xe3', + 'vld_id': 'uplink_0', + 'ifname': 'xe1', 'driver': "i40e", 'netmask': '255.255.255.0', 'dpdk_port_num': 1, @@ -365,6 +365,7 @@ class TestProxApproxVnf(unittest.TestCase): prox_approx_vnf = ProxApproxVnf(NAME, deepcopy(self.VNFD0)) prox_approx_vnf.resource_helper = resource_helper prox_approx_vnf.vnfd_helper['vdu'][0]['external-interface'] = [] + prox_approx_vnf.vnfd_helper.port_pairs.interfaces = [] with self.assertRaises(RuntimeError): prox_approx_vnf.collect_kpi() diff --git a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py index 77c92e0fb..4d53f0989 100644 --- a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py +++ b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py @@ -833,13 +833,13 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase): dpdk_setup_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, scenario_helper) dpdk_setup_helper.dpdk_bind_helper.bind = mock.Mock() dpdk_setup_helper.dpdk_bind_helper.used_drivers = { - '0000:05:00.0': 'd1', - '0000:05:01.0': 'd3', + 'd1': ['0000:05:00.0'], + 'd3': ['0000:05:01.0'], } self.assertIsNone(dpdk_setup_helper.tear_down()) - dpdk_setup_helper.dpdk_bind_helper.bind.assert_any_call('0000:05:00.0', 'd1', True) - dpdk_setup_helper.dpdk_bind_helper.bind.assert_any_call('0000:05:01.0', 'd3', True) + dpdk_setup_helper.dpdk_bind_helper.bind.assert_any_call(['0000:05:00.0'], 'd1', True) + dpdk_setup_helper.dpdk_bind_helper.bind.assert_any_call(['0000:05:01.0'], 'd3', True) class TestResourceHelper(unittest.TestCase): @@ -2130,6 +2130,7 @@ class TestSampleVNFTrafficGen(unittest.TestCase): def test_terminate(self): sample_vnf_tg = SampleVNFTrafficGen('tg1', self.VNFD_0) sample_vnf_tg._traffic_process = mock.Mock() + sample_vnf_tg._tg_process = mock.Mock() sample_vnf_tg.terminate() diff --git a/tests/unit/network_services/vnf_generic/vnf/test_tg_prox.py b/tests/unit/network_services/vnf_generic/vnf/test_tg_prox.py index bdda149c5..23d448c5e 100644 --- a/tests/unit/network_services/vnf_generic/vnf/test_tg_prox.py +++ b/tests/unit/network_services/vnf_generic/vnf/test_tg_prox.py @@ -426,6 +426,8 @@ class TestProxTrafficGen(unittest.TestCase): prox_traffic_gen._traffic_process.terminate = mock.Mock() prox_traffic_gen.ssh_helper = mock.MagicMock() prox_traffic_gen.setup_helper = mock.MagicMock() - prox_traffic_gen._vnf_wrapper.setup_helper = mock.MagicMock() prox_traffic_gen.resource_helper = mock.MagicMock() + prox_traffic_gen._vnf_wrapper.setup_helper = mock.MagicMock() + prox_traffic_gen._vnf_wrapper._vnf_process = mock.MagicMock() + prox_traffic_gen._vnf_wrapper.resource_helper = mock.MagicMock() self.assertEqual(None, prox_traffic_gen.terminate()) diff --git a/tests/unit/network_services/vnf_generic/vnf/test_udp_replay.py b/tests/unit/network_services/vnf_generic/vnf/test_udp_replay.py index b75ed6764..39b37c606 100644 --- a/tests/unit/network_services/vnf_generic/vnf/test_udp_replay.py +++ b/tests/unit/network_services/vnf_generic/vnf/test_udp_replay.py @@ -426,6 +426,7 @@ class TestUdpReplayApproxVnf(unittest.TestCase): udp_replay_approx_vnf = UdpReplayApproxVnf(NAME, self.VNFD_0) udp_replay_approx_vnf._build_config = mock.MagicMock() udp_replay_approx_vnf.queue_wrapper = mock.MagicMock() + udp_replay_approx_vnf.scenario_helper = mock.MagicMock() udp_replay_approx_vnf._run() diff --git a/tests/unit/network_services/vnf_generic/vnf/test_vpe_vnf.py b/tests/unit/network_services/vnf_generic/vnf/test_vpe_vnf.py index 3813aaa21..4103d7825 100644 --- a/tests/unit/network_services/vnf_generic/vnf/test_vpe_vnf.py +++ b/tests/unit/network_services/vnf_generic/vnf/test_vpe_vnf.py @@ -565,8 +565,9 @@ class TestVpeApproxVnf(unittest.TestCase): mock_ssh(ssh) resource = mock.Mock(autospec=ResourceProfile) - resource.check_if_sa_running.return_value = False, 'error' + resource.check_if_sa_running.return_value = 1, '' resource.amqp_collect_nfvi_kpi.return_value = {'foo': 234} + resource.check_if_sa_running.return_value = (1, None) vpe_approx_vnf = VpeApproxVnf(NAME, self.VNFD_0) vpe_approx_vnf.q_in = mock.MagicMock() @@ -588,7 +589,7 @@ class TestVpeApproxVnf(unittest.TestCase): mock_ssh(ssh) resource = mock.Mock(autospec=ResourceProfile) - resource.check_if_sa_running.return_value = True, 'good' + resource.check_if_sa_running.return_value = 0, '1234' resource.amqp_collect_nfvi_kpi.return_value = {'foo': 234} vpe_approx_vnf = VpeApproxVnf(NAME, self.VNFD_0) diff --git a/yardstick/benchmark/runners/arithmetic.py b/yardstick/benchmark/runners/arithmetic.py index 3ff064ae1..6aaaed888 100755 --- a/yardstick/benchmark/runners/arithmetic.py +++ b/yardstick/benchmark/runners/arithmetic.py @@ -191,7 +191,9 @@ class ArithmeticRunner(base.Runner): __execution_type__ = 'Arithmetic' def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, context_cfg, self.aborted, self.output_queue)) diff --git a/yardstick/benchmark/runners/base.py b/yardstick/benchmark/runners/base.py index 3ecf67736..13718d793 100755 --- a/yardstick/benchmark/runners/base.py +++ b/yardstick/benchmark/runners/base.py @@ -17,13 +17,14 @@ # rally/rally/benchmark/runners/base.py from __future__ import absolute_import -import importlib + import logging import multiprocessing import subprocess import time import traceback +import importlib from six.moves.queue import Empty @@ -36,7 +37,6 @@ log = logging.getLogger(__name__) def _execute_shell_command(command): """execute shell script with error handling""" exitcode = 0 - output = [] try: output = subprocess.check_output(command, shell=True) except Exception: diff --git a/yardstick/benchmark/runners/duration.py b/yardstick/benchmark/runners/duration.py index 2cb2600c8..fbf72a74c 100644 --- a/yardstick/benchmark/runners/duration.py +++ b/yardstick/benchmark/runners/duration.py @@ -138,7 +138,9 @@ If the scenario ends before the time has elapsed, it will be started again. __execution_type__ = 'Duration' def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, context_cfg, self.aborted, self.output_queue)) diff --git a/yardstick/benchmark/runners/dynamictp.py b/yardstick/benchmark/runners/dynamictp.py index 01e76c6f4..63bfc823a 100755 --- a/yardstick/benchmark/runners/dynamictp.py +++ b/yardstick/benchmark/runners/dynamictp.py @@ -19,11 +19,12 @@ """A runner that searches for the max throughput with binary search """ -import os -import multiprocessing import logging -import traceback +import multiprocessing import time +import traceback + +import os from yardstick.benchmark.runners import base @@ -65,8 +66,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, max_throuput_found = False sequence = 0 - last_min_data = {} - last_min_data['packets_per_second'] = 0 + last_min_data = {'packets_per_second': 0} while True: sequence += 1 @@ -125,7 +125,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, queue.put(record) max_throuput_found = True - if (errors) or aborted.is_set() or max_throuput_found: + if errors or aborted.is_set() or max_throuput_found: LOG.info("worker END") break @@ -155,7 +155,7 @@ def _worker_process(queue, cls, method_name, scenario_cfg, class IterationRunner(base.Runner): - '''Run a scenario to find the max throughput + """Run a scenario to find the max throughput If the scenario ends before the time has elapsed, it will be started again. @@ -168,11 +168,13 @@ If the scenario ends before the time has elapsed, it will be started again. type: int unit: pps default: 1000 pps - ''' + """ __execution_type__ = 'Dynamictp' def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, context_cfg, self.aborted)) diff --git a/yardstick/benchmark/runners/iteration.py b/yardstick/benchmark/runners/iteration.py index 88158eed3..cb0424377 100644 --- a/yardstick/benchmark/runners/iteration.py +++ b/yardstick/benchmark/runners/iteration.py @@ -20,11 +20,13 @@ """ from __future__ import absolute_import -import os -import multiprocessing + import logging -import traceback +import multiprocessing import time +import traceback + +import os from yardstick.benchmark.runners import base @@ -88,9 +90,9 @@ def _worker_process(queue, cls, method_name, scenario_cfg, scenario_cfg['options']['rate'] -= delta sequence = 1 continue - except Exception as e: + except Exception: errors = traceback.format_exc() - LOG.exception(e) + LOG.exception("") else: if result: # add timeout for put so we don't block test @@ -151,7 +153,9 @@ If the scenario ends before the time has elapsed, it will be started again. __execution_type__ = 'Iteration' def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, context_cfg, self.aborted, self.output_queue)) diff --git a/yardstick/benchmark/runners/search.py b/yardstick/benchmark/runners/search.py index 5948763a7..8037329b5 100644 --- a/yardstick/benchmark/runners/search.py +++ b/yardstick/benchmark/runners/search.py @@ -20,15 +20,16 @@ """ from __future__ import absolute_import -import os -import multiprocessing + import logging -import traceback +import multiprocessing import time - -from collections import Mapping +import traceback from contextlib import contextmanager from itertools import takewhile + +import os +from collections import Mapping from six.moves import zip from yardstick.benchmark.runners import base @@ -173,7 +174,9 @@ If the scenario ends before the time has elapsed, it will be started again. break def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=self._worker_run, args=(cls, method, scenario_cfg, context_cfg)) self.process.start() diff --git a/yardstick/benchmark/runners/sequence.py b/yardstick/benchmark/runners/sequence.py index f08ca5dde..d6e3f7109 100644 --- a/yardstick/benchmark/runners/sequence.py +++ b/yardstick/benchmark/runners/sequence.py @@ -21,11 +21,13 @@ The input value in the sequence is specified in a list in the input file. """ from __future__ import absolute_import -import os -import multiprocessing + import logging -import traceback +import multiprocessing import time +import traceback + +import os from yardstick.benchmark.runners import base @@ -140,7 +142,9 @@ class SequenceRunner(base.Runner): __execution_type__ = 'Sequence' def _run_benchmark(self, cls, method, scenario_cfg, context_cfg): + name = "{}-{}-{}".format(self.__execution_type__, scenario_cfg.get("type"), os.getpid()) self.process = multiprocessing.Process( + name=name, target=_worker_process, args=(self.result_queue, cls, method, scenario_cfg, context_cfg, self.aborted, self.output_queue)) diff --git a/yardstick/benchmark/scenarios/networking/vnf_generic.py b/yardstick/benchmark/scenarios/networking/vnf_generic.py index d9cc0eac1..d85125230 100644 --- a/yardstick/benchmark/scenarios/networking/vnf_generic.py +++ b/yardstick/benchmark/scenarios/networking/vnf_generic.py @@ -30,6 +30,7 @@ from collections import defaultdict from yardstick.benchmark.scenarios import base from yardstick.common.constants import LOG_DIR +from yardstick.common.process import terminate_children from yardstick.common.utils import import_modules_from_package, itersubclasses from yardstick.common.yaml_loader import yaml_load from yardstick.network_services.collector.subscriber import Collector @@ -596,7 +597,8 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ vnf.instantiate(self.scenario_cfg, self.context_cfg) LOG.info("Waiting for %s to instantiate", vnf.name) vnf.wait_for_instantiate() - except RuntimeError: + except: + LOG.exception("") for vnf in self.vnfs: vnf.terminate() raise @@ -635,7 +637,19 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ :return """ - self.collector.stop() - for vnf in self.vnfs: - LOG.info("Stopping %s", vnf.name) - vnf.terminate() + try: + try: + self.collector.stop() + for vnf in self.vnfs: + LOG.info("Stopping %s", vnf.name) + vnf.terminate() + LOG.debug("all VNFs terminated: %s", ", ".join(vnf.name for vnf in self.vnfs)) + finally: + terminate_children() + except Exception: + # catch any exception in teardown and convert to simple exception + # never pass exceptions back to multiprocessing, because some exceptions can + # be unpicklable + # https://bugs.python.org/issue9400 + LOG.exception("") + raise RuntimeError("Error in teardown") diff --git a/yardstick/common/process.py b/yardstick/common/process.py new file mode 100644 index 000000000..812ddea94 --- /dev/null +++ b/yardstick/common/process.py @@ -0,0 +1,47 @@ +# Copyright (c) 2017 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import multiprocessing + +import os + +LOG = logging.getLogger(__name__) + + +def check_if_process_failed(proc, timeout=1): + if proc is not None: + proc.join(timeout) + # Only abort if the process aborted + if proc.exitcode is not None and proc.exitcode > 0: + raise RuntimeError("{} exited with status {}".format(proc.name, proc.exitcode)) + + +def terminate_children(timeout=3): + current_proccess = multiprocessing.current_process() + active_children = multiprocessing.active_children() + if not active_children: + LOG.debug("no children to terminate") + return + for child in active_children: + LOG.debug("%s %s %s, child: %s %s", current_proccess.name, current_proccess.pid, + os.getpid(), child, child.pid) + LOG.debug("joining %s", child) + child.join(timeout) + child.terminate() + active_children = multiprocessing.active_children() + if not active_children: + LOG.debug("no children to terminate") + for child in active_children: + LOG.debug("%s %s %s, after terminate child: %s %s", current_proccess.name, + current_proccess.pid, os.getpid(), child, child.pid) diff --git a/yardstick/network_services/collector/subscriber.py b/yardstick/network_services/collector/subscriber.py index db52e0b45..d560e1d42 100644 --- a/yardstick/network_services/collector/subscriber.py +++ b/yardstick/network_services/collector/subscriber.py @@ -74,11 +74,12 @@ class Collector(object): # Result example: # {"VNF1: { "tput" : [1000, 999] }, "VNF2": { "latency": 100 }} LOG.debug("collect KPI for %s", node_name) - if not resource.check_if_sa_running("collectd")[0]: + if resource.check_if_sa_running("collectd")[0] != 0: continue try: results[node_name] = {"core": resource.amqp_collect_nfvi_kpi()} + LOG.debug("%s collect KPIs %s", node_name, results[node_name]['core']) except Exception: LOG.exception("") return results diff --git a/yardstick/network_services/helpers/dpdknicbind_helper.py b/yardstick/network_services/helpers/dpdkbindnic_helper.py index 605d08d38..c07613147 100644 --- a/yardstick/network_services/helpers/dpdknicbind_helper.py +++ b/yardstick/network_services/helpers/dpdkbindnic_helper.py @@ -11,9 +11,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import logging + import re import itertools +import six + NETWORK_KERNEL = 'network_kernel' NETWORK_DPDK = 'network_dpdk' NETWORK_OTHER = 'network_other' @@ -22,6 +26,9 @@ CRYPTO_DPDK = 'crypto_dpdk' CRYPTO_OTHER = 'crypto_other' +LOG = logging.getLogger(__name__) + + class DpdkBindHelperException(Exception): pass @@ -123,23 +130,31 @@ class DpdkBindHelper(object): @property def interface_driver_map(self): return {interface['vpci']: interface['driver'] - for interface in itertools.chain(*self.dpdk_status.values())} + for interface in itertools.chain.from_iterable(self.dpdk_status.values())} def read_status(self): return self.parse_dpdk_status_output(self._dpdk_execute(self._status_cmd)[1]) - def bind(self, pci, driver, force=True): + def bind(self, pci_addresses, driver, force=True): + # accept single PCI or list of PCI + if isinstance(pci_addresses, six.string_types): + pci_addresses = [pci_addresses] cmd = self.DPDK_BIND_CMD.format(dpdk_nic_bind=self._dpdk_nic_bind, driver=driver, - vpci=' '.join(list(pci)), + vpci=' '.join(list(pci_addresses)), force='--force' if force else '') + LOG.debug(cmd) self._dpdk_execute(cmd) # update the inner status dict self.read_status() def save_used_drivers(self): - self.used_drivers = self.interface_driver_map + # invert the map, so we can bind by driver type + self.used_drivers = {} + # sort for stabililty + for vpci, driver in sorted(self.interface_driver_map.items()): + self.used_drivers.setdefault(driver, []).append(vpci) def rebind_drivers(self, force=True): - for vpci, driver in self.used_drivers.items(): - self.bind(vpci, driver, force) + for driver, vpcis in self.used_drivers.items(): + self.bind(vpcis, driver, force) diff --git a/yardstick/network_services/nfvi/resource.py b/yardstick/network_services/nfvi/resource.py index 7e8334c73..e3d0e3bca 100644 --- a/yardstick/network_services/nfvi/resource.py +++ b/yardstick/network_services/nfvi/resource.py @@ -19,6 +19,7 @@ from __future__ import print_function import logging from itertools import chain +import errno import jinja2 import os import os.path @@ -72,7 +73,6 @@ class ResourceProfile(object): self.timeout = timeout self.enable = True - self.cores = validate_non_string_sequence(cores, default=[]) self._queue = multiprocessing.Queue() self.amqp_client = None self.port_names = validate_non_string_sequence(port_names, default=[]) @@ -83,8 +83,16 @@ class ResourceProfile(object): def check_if_sa_running(self, process): """ verify if system agent is running """ - status, pid, _ = self.connection.execute("pgrep -f %s" % process) - return status == 0, pid + try: + err, pid, _ = self.connection.execute("pgrep -f %s" % process) + # strip whitespace + return err, pid.strip() + except OSError as e: + if e.errno in {errno.ECONNRESET}: + # if we can't connect to check, then we won't be able to connect to stop it + LOG.exception("can't connect to host to check collectd status") + return 1, None + raise def run_collectd_amqp(self): """ run amqp consumer to collect the NFVi data """ @@ -137,7 +145,7 @@ class ResourceProfile(object): def parse_intel_pmu_stats(cls, key, value): return {''.join(str(v) for v in key): value.split(":")[1]} - def parse_collectd_result(self, metrics, core_list): + def parse_collectd_result(self, metrics): """ convert collectd data into json""" result = { "cpu": {}, @@ -161,8 +169,7 @@ class ResourceProfile(object): if "cpu" in res_key0 or "intel_rdt" in res_key0: cpu_key, name, metric, testcase = \ self.get_cpu_data(res_key0, res_key1, value) - if cpu_key in core_list: - result["cpu"].setdefault(cpu_key, {}).update({name: metric}) + result["cpu"].setdefault(cpu_key, {}).update({name: metric}) elif "memory" in res_key0: result["memory"].update({res_key1: value.split(":")[0]}) @@ -189,8 +196,9 @@ class ResourceProfile(object): def amqp_process_for_nfvi_kpi(self): """ amqp collect and return nfvi kpis """ if self.amqp_client is None and self.enable: - self.amqp_client = \ - multiprocessing.Process(target=self.run_collectd_amqp) + self.amqp_client = multiprocessing.Process( + name="AmqpClient-{}-{}".format(self.mgmt['ip'], os.getpid()), + target=self.run_collectd_amqp) self.amqp_client.start() def amqp_collect_nfvi_kpi(self): @@ -201,7 +209,7 @@ class ResourceProfile(object): metric = {} while not self._queue.empty(): metric.update(self._queue.get()) - msg = self.parse_collectd_result(metric, self.cores) + msg = self.parse_collectd_result(metric) return msg def _provide_config_file(self, config_file_path, nfvi_cfg, template_kwargs): @@ -265,8 +273,8 @@ class ResourceProfile(object): connection.execute("sudo rabbitmqctl authenticate_user admin admin") connection.execute("sudo rabbitmqctl set_permissions -p / admin '.*' '.*' '.*'") - LOG.debug("Start collectd service.....") - connection.execute("sudo %s" % collectd_path) + LOG.debug("Start collectd service..... %s second timeout", self.timeout) + connection.execute("sudo %s" % collectd_path, timeout=self.timeout) LOG.debug("Done") def initiate_systemagent(self, bin_path): @@ -292,13 +300,18 @@ class ResourceProfile(object): LOG.debug("Stop resource monitor...") if self.amqp_client is not None: + # we proper and try to join first + self.amqp_client.join(3) self.amqp_client.terminate() + LOG.debug("Check if %s is running", agent) status, pid = self.check_if_sa_running(agent) - if status == 0: + LOG.debug("status %s pid %s", status, pid) + if status != 0: return - self.connection.execute('sudo kill -9 %s' % pid) - self.connection.execute('sudo pkill -9 %s' % agent) + if pid: + self.connection.execute('sudo kill -9 "%s"' % pid) + self.connection.execute('sudo pkill -9 "%s"' % agent) self.connection.execute('sudo service rabbitmq-server stop') self.connection.execute("sudo rabbitmqctl stop_app") diff --git a/yardstick/network_services/vnf_generic/vnf/base.py b/yardstick/network_services/vnf_generic/vnf/base.py index 67634a79c..7391633af 100644 --- a/yardstick/network_services/vnf_generic/vnf/base.py +++ b/yardstick/network_services/vnf_generic/vnf/base.py @@ -64,6 +64,8 @@ class VnfdHelper(dict): def __init__(self, *args, **kwargs): super(VnfdHelper, self).__init__(*args, **kwargs) self.port_pairs = PortPairs(self['vdu'][0]['external-interface']) + # port num is not present until binding so we have to memoize + self._port_num_map = {} @property def mgmt_interface(self): @@ -91,12 +93,14 @@ class VnfdHelper(dict): virtual_intf = interface["virtual-interface"] if virtual_intf[key] == value: return interface + raise KeyError() def find_interface(self, **kwargs): key, value = next(iter(kwargs.items())) for interface in self.interfaces: if interface[key] == value: return interface + raise KeyError() # hide dpdk_port_num key so we can abstract def find_interface_by_port(self, port): @@ -105,6 +109,7 @@ class VnfdHelper(dict): # we have to convert to int to compare if int(virtual_intf['dpdk_port_num']) == port: return interface + raise KeyError() def port_num(self, port): # we need interface name -> DPDK port num (PMD ID) -> LINK ID @@ -118,7 +123,8 @@ class VnfdHelper(dict): intf = port else: intf = self.find_interface(name=port) - return int(intf["virtual-interface"]["dpdk_port_num"]) + return self._port_num_map.setdefault(intf["name"], + int(intf["virtual-interface"]["dpdk_port_num"])) def port_nums(self, intfs): return [self.port_num(i) for i in intfs] diff --git a/yardstick/network_services/vnf_generic/vnf/prox_helpers.py b/yardstick/network_services/vnf_generic/vnf/prox_helpers.py index ba4d44c41..d9fe9a948 100644 --- a/yardstick/network_services/vnf_generic/vnf/prox_helpers.py +++ b/yardstick/network_services/vnf_generic/vnf/prox_helpers.py @@ -364,6 +364,7 @@ class ProxSocketHelper(object): """ send data to the remote instance """ LOG.debug("Sending data to socket: [%s]", to_send.rstrip('\n')) try: + # TODO: sendall will block, we need a timeout self._sock.sendall(to_send.encode('utf-8')) except: pass @@ -593,6 +594,8 @@ _LOCAL_OBJECT = object() class ProxDpdkVnfSetupEnvHelper(DpdkVnfSetupEnvHelper): # the actual app is lowercase APP_NAME = 'prox' + # not used for Prox but added for consistency + VNF_TYPE = "PROX" LUA_PARAMETER_NAME = "" LUA_PARAMETER_PEER = { @@ -609,6 +612,8 @@ class ProxDpdkVnfSetupEnvHelper(DpdkVnfSetupEnvHelper): self._prox_config_data = None self.additional_files = {} self.config_queue = Queue() + # allow_exit_without_flush + self.config_queue.cancel_join_thread() self._global_section = None @property diff --git a/yardstick/network_services/vnf_generic/vnf/prox_vnf.py b/yardstick/network_services/vnf_generic/vnf/prox_vnf.py index 2ac6ea412..3bfca19aa 100644 --- a/yardstick/network_services/vnf_generic/vnf/prox_vnf.py +++ b/yardstick/network_services/vnf_generic/vnf/prox_vnf.py @@ -16,9 +16,10 @@ import errno import logging +from yardstick.common.process import check_if_process_failed from yardstick.network_services.vnf_generic.vnf.prox_helpers import ProxDpdkVnfSetupEnvHelper from yardstick.network_services.vnf_generic.vnf.prox_helpers import ProxResourceHelper -from yardstick.network_services.vnf_generic.vnf.sample_vnf import SampleVNF +from yardstick.network_services.vnf_generic.vnf.sample_vnf import SampleVNF, PROCESS_JOIN_TIMEOUT LOG = logging.getLogger(__name__) @@ -51,10 +52,18 @@ class ProxApproxVnf(SampleVNF): try: return self.resource_helper.execute(cmd, *args, **kwargs) except OSError as e: - if not ignore_errors or e.errno not in {errno.EPIPE, errno.ESHUTDOWN}: + if e.errno in {errno.EPIPE, errno.ESHUTDOWN, errno.ECONNRESET}: + if ignore_errors: + LOG.debug("ignoring vnf_execute exception %s for command %s", e, cmd) + else: + raise + else: raise def collect_kpi(self): + # we can't get KPIs if the VNF is down + check_if_process_failed(self._vnf_process) + if self.resource_helper is None: result = { "packets_in": 0, @@ -64,12 +73,13 @@ class ProxApproxVnf(SampleVNF): } return result - intf_count = len(self.vnfd_helper.interfaces) - if intf_count not in {1, 2, 4}: + # use all_ports so we only use ports matched in topology + port_count = len(self.vnfd_helper.port_pairs.all_ports) + if port_count not in {1, 2, 4}: raise RuntimeError("Failed ..Invalid no of ports .. " "1, 2 or 4 ports only supported at this time") - port_stats = self.vnf_execute('port_stats', range(intf_count)) + port_stats = self.vnf_execute('port_stats', range(port_count)) try: rx_total = port_stats[6] tx_total = port_stats[7] @@ -94,13 +104,20 @@ class ProxApproxVnf(SampleVNF): self.setup_helper.tear_down() def terminate(self): + # stop collectd first or we get pika errors? + self.resource_helper.stop_collect() # try to quit with socket commands - self.vnf_execute("stop_all") - self.vnf_execute("quit") - # hopefully quit succeeds and socket closes, so ignore force_quit socket errors - self.vnf_execute("force_quit", _ignore_errors=True) - if self._vnf_process: - self._vnf_process.terminate() + # pkill is not matching, debug with pgrep + self.ssh_helper.execute("sudo pgrep -lax %s" % self.setup_helper.APP_NAME) + self.ssh_helper.execute("sudo ps aux | grep -i %s" % self.setup_helper.APP_NAME) + if self._vnf_process.is_alive(): + self.vnf_execute("stop_all") + self.vnf_execute("quit") + # hopefully quit succeeds and socket closes, so ignore force_quit socket errors + self.vnf_execute("force_quit", _ignore_errors=True) self.setup_helper.kill_vnf() self._tear_down() - self.resource_helper.stop_collect() + if self._vnf_process is not None: + LOG.debug("joining before terminate %s", self._vnf_process.name) + self._vnf_process.join(PROCESS_JOIN_TIMEOUT) + self._vnf_process.terminate() diff --git a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py index bbaae39e3..114153dab 100644 --- a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py +++ b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py @@ -29,10 +29,11 @@ from six.moves import cStringIO from yardstick.benchmark.contexts.base import Context from yardstick.benchmark.scenarios.networking.vnf_generic import find_relative_file +from yardstick.common.process import check_if_process_failed from yardstick.network_services.helpers.cpu import CpuSysCores from yardstick.network_services.helpers.samplevnf_helper import PortPairs from yardstick.network_services.helpers.samplevnf_helper import MultiPortConfig -from yardstick.network_services.helpers.dpdknicbind_helper import DpdkBindHelper +from yardstick.network_services.helpers.dpdkbindnic_helper import DpdkBindHelper from yardstick.network_services.nfvi.resource import ResourceProfile from yardstick.network_services.vnf_generic.vnf.base import GenericVNF from yardstick.network_services.vnf_generic.vnf.base import QueueFileWrapper @@ -51,6 +52,8 @@ LOG = logging.getLogger(__name__) REMOTE_TMP = "/tmp" +DEFAULT_VNF_TIMEOUT = 3600 +PROCESS_JOIN_TIMEOUT = 3 class VnfSshHelper(AutoConnectSSH): @@ -290,8 +293,12 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper): return resource def kill_vnf(self): + # pkill is not matching, debug with pgrep + self.ssh_helper.execute("sudo pgrep -lax %s" % self.APP_NAME) + self.ssh_helper.execute("sudo ps aux | grep -i %s" % self.APP_NAME) # have to use exact match - self.ssh_helper.execute("sudo pkill -x %s" % self.APP_NAME) + # try using killall to match + self.ssh_helper.execute("sudo killall %s" % self.APP_NAME) def _setup_dpdk(self): """ setup dpdk environment needed for vnf to run """ @@ -330,8 +337,10 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper): port_names = (intf["name"] for intf in ports) collectd_options = self.get_collectd_options() plugins = collectd_options.get("plugins", {}) + # we must set timeout to be the same as the VNF otherwise KPIs will die before VNF return ResourceProfile(self.vnfd_helper.mgmt_interface, port_names=port_names, cores=cores, - plugins=plugins, interval=collectd_options.get("interval")) + plugins=plugins, interval=collectd_options.get("interval"), + timeout=self.scenario_helper.timeout) def _detect_and_bind_drivers(self): interfaces = self.vnfd_helper.interfaces @@ -386,7 +395,7 @@ class ResourceHelper(object): def _collect_resource_kpi(self): result = {} status = self.resource.check_if_sa_running("collectd")[0] - if status: + if status == 0: result = self.resource.amqp_collect_nfvi_kpi() result = {"core": result} @@ -672,12 +681,18 @@ class ScenarioHelper(object): def topology(self): return self.scenario_cfg['topology'] + @property + def timeout(self): + return self.options.get('timeout', DEFAULT_VNF_TIMEOUT) + class SampleVNF(GenericVNF): """ Class providing file-like API for generic VNF implementation """ VNF_PROMPT = "pipeline>" WAIT_TIME = 1 + APP_NAME = "SampleVNF" + # we run the VNF interactively, so the ssh command will timeout after this long def __init__(self, name, vnfd, setup_env_helper_type=None, resource_helper_type=None): super(SampleVNF, self).__init__(name, vnfd) @@ -760,7 +775,8 @@ class SampleVNF(GenericVNF): def _start_vnf(self): self.queue_wrapper = QueueFileWrapper(self.q_in, self.q_out, self.VNF_PROMPT) - self._vnf_process = Process(target=self._run) + name = "{}-{}-{}".format(self.name, self.APP_NAME, os.getpid()) + self._vnf_process = Process(name=name, target=self._run) self._vnf_process.start() def _vnf_up_post(self): @@ -810,6 +826,7 @@ class SampleVNF(GenericVNF): 'stdout': self.queue_wrapper, 'keep_stdin_open': True, 'pty': True, + 'timeout': self.scenario_helper.timeout, } def _build_config(self): @@ -842,11 +859,15 @@ class SampleVNF(GenericVNF): def terminate(self): self.vnf_execute("quit") - if self._vnf_process: - self._vnf_process.terminate() self.setup_helper.kill_vnf() self._tear_down() self.resource_helper.stop_collect() + if self._vnf_process is not None: + # be proper and join first before we kill + LOG.debug("joining before terminate %s", self._vnf_process.name) + self._vnf_process.join(PROCESS_JOIN_TIMEOUT) + self._vnf_process.terminate() + # no terminate children here because we share processes with tg def get_stats(self, *args, **kwargs): """ @@ -860,6 +881,8 @@ class SampleVNF(GenericVNF): return out def collect_kpi(self): + # we can't get KPIs if the VNF is down + check_if_process_failed(self._vnf_process) stats = self.get_stats() m = re.search(self.COLLECT_KPI, stats, re.MULTILINE) if m: @@ -884,7 +907,6 @@ class SampleVNFTrafficGen(GenericTrafficGen): def __init__(self, name, vnfd, setup_env_helper_type=None, resource_helper_type=None): super(SampleVNFTrafficGen, self).__init__(name, vnfd) self.bin_path = get_nsb_option('bin_path', '') - self.name = "tgen__1" # name in topology file self.scenario_helper = ScenarioHelper(self.name) self.ssh_helper = VnfSshHelper(self.vnfd_helper.mgmt_interface, self.bin_path, wait=True) @@ -916,7 +938,8 @@ class SampleVNFTrafficGen(GenericTrafficGen): self.resource_helper.setup() LOG.info("Starting %s server...", self.APP_NAME) - self._tg_process = Process(target=self._start_server) + name = "{}-{}-{}".format(self.name, self.APP_NAME, os.getpid()) + self._tg_process = Process(name=name, target=self._start_server) self._tg_process.start() def wait_for_instantiate(self): @@ -952,7 +975,9 @@ class SampleVNFTrafficGen(GenericTrafficGen): :param traffic_profile: :return: True/False """ - self._traffic_process = Process(target=self._traffic_runner, + name = "{}-{}-{}-{}".format(self.name, self.APP_NAME, traffic_profile.__class__.__name__, + os.getpid()) + self._traffic_process = Process(name=name, target=self._traffic_runner, args=(traffic_profile,)) self._traffic_process.start() # Wait for traffic process to start @@ -983,6 +1008,9 @@ class SampleVNFTrafficGen(GenericTrafficGen): pass def collect_kpi(self): + # check if the tg processes have exited + for proc in (self._tg_process, self._traffic_process): + check_if_process_failed(proc) result = self.resource_helper.collect_kpi() LOG.debug("%s collect KPIs %s", self.APP_NAME, result) return result @@ -993,5 +1021,15 @@ class SampleVNFTrafficGen(GenericTrafficGen): :return: True/False """ self.traffic_finished = True + # we must kill client before we kill the server, or the client will raise exception if self._traffic_process is not None: + # be proper and try to join before terminating + LOG.debug("joining before terminate %s", self._traffic_process.name) + self._traffic_process.join(PROCESS_JOIN_TIMEOUT) self._traffic_process.terminate() + if self._tg_process is not None: + # be proper and try to join before terminating + LOG.debug("joining before terminate %s", self._tg_process.name) + self._tg_process.join(PROCESS_JOIN_TIMEOUT) + self._tg_process.terminate() + # no terminate children here because we share processes with vnf diff --git a/yardstick/network_services/vnf_generic/vnf/udp_replay.py b/yardstick/network_services/vnf_generic/vnf/udp_replay.py index 6b7779782..a57f53bc7 100644 --- a/yardstick/network_services/vnf_generic/vnf/udp_replay.py +++ b/yardstick/network_services/vnf_generic/vnf/udp_replay.py @@ -15,6 +15,7 @@ from __future__ import absolute_import import logging +from yardstick.common.process import check_if_process_failed from yardstick.network_services.vnf_generic.vnf.sample_vnf import SampleVNF from yardstick.network_services.vnf_generic.vnf.sample_vnf import DpdkVnfSetupEnvHelper from yardstick.network_services.vnf_generic.vnf.sample_vnf import ClientResourceHelper @@ -107,6 +108,8 @@ class UdpReplayApproxVnf(SampleVNF): def collect_kpi(self): def get_sum(offset): return sum(int(i) for i in split_stats[offset::5]) + # we can't get KPIs if the VNF is down + check_if_process_failed(self._vnf_process) number_of_ports = len(self.vnfd_helper.port_pairs.all_ports) diff --git a/yardstick/network_services/vnf_generic/vnf/vpe_vnf.py b/yardstick/network_services/vnf_generic/vnf/vpe_vnf.py index 5f1c4d4d3..c02c0eb27 100644 --- a/yardstick/network_services/vnf_generic/vnf/vpe_vnf.py +++ b/yardstick/network_services/vnf_generic/vnf/vpe_vnf.py @@ -24,6 +24,7 @@ import posixpath from six.moves import configparser, zip +from yardstick.common.process import check_if_process_failed from yardstick.network_services.helpers.samplevnf_helper import PortPairs from yardstick.network_services.pipeline import PipelineRules from yardstick.network_services.vnf_generic.vnf.sample_vnf import SampleVNF, DpdkVnfSetupEnvHelper @@ -278,6 +279,8 @@ class VpeApproxVnf(SampleVNF): raise NotImplementedError def collect_kpi(self): + # we can't get KPIs if the VNF is down + check_if_process_failed(self._vnf_process) result = { 'pkt_in_up_stream': 0, 'pkt_drop_up_stream': 0, |