aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRoss Brattain <ross.b.brattain@intel.com>2017-10-04 13:38:43 -0700
committerRoss Brattain <ross.b.brattain@intel.com>2018-03-01 08:21:53 -0800
commit6f7dd8ff0e6358ef958426a6baeae5deee1a57d8 (patch)
treebb3a46048a7ccd7d45b45352824d18612a25f7d3
parent9316c6c49957f2d8c680ed8acfaccac9070ed2f4 (diff)
NSB: move interface probe to VNF, and attempt driver-only probe first
If no devices are present we can't detect MAC address so we can't match Heat ports to interfaces. If only the driver is missing we can try to probe the driver using lspci. We can use lspci to ask the kernel what driver it should use for the PCI device. If we can't probe at all because the device is already bound, we can use dpkd-devind to find all the PCI address we care about and create a map with PCI device and real kernel driver. Then we can dpdk force rebind to the kernel driver. Once we have rebound to the kernel driver we can detect MAC address and all the other attributes that are required. Fix VnfSshHelper to allow override of wait timeout And a bunch of other refactors that got swept up in this JIRA: YARDSTICK-835 Change-Id: I14cb657ed289a77941d048345d06ced5b5d5da52 Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>
Diffstat
-rw-r--r--tests/unit/network_services/helpers/test_dpdkbindnic_helper.py402
-rw-r--r--tests/unit/network_services/vnf_generic/vnf/test_prox_vnf.py19
-rw-r--r--tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py27
-rw-r--r--tests/unit/network_services/vnf_generic/vnf/test_tg_ping.py2
-rw-r--r--yardstick/benchmark/scenarios/networking/vnf_generic.py183
-rw-r--r--yardstick/common/utils.py15
-rw-r--r--yardstick/error.py48
-rw-r--r--yardstick/network_services/constants.py17
-rw-r--r--yardstick/network_services/helpers/dpdkbindnic_helper.py313
-rw-r--r--yardstick/network_services/utils.py1
-rw-r--r--yardstick/network_services/vnf_generic/vnf/prox_vnf.py3
-rw-r--r--yardstick/network_services/vnf_generic/vnf/sample_vnf.py82
-rw-r--r--yardstick/network_services/vnf_generic/vnf/tg_rfc2544_ixia.py4
-rw-r--r--yardstick/network_services/vnf_generic/vnf/vnf_ssh_helper.py61
-rw-r--r--yardstick/ssh.py68
-rw-r--r--yardstick/tests/unit/benchmark/scenarios/networking/test_vnf_generic.py189
-rw-r--r--yardstick/tests/unit/common/test_utils.py62
-rw-r--r--yardstick/tests/unit/test_ssh.py54
18 files changed, 1068 insertions, 482 deletions
diff --git a/tests/unit/network_services/helpers/test_dpdkbindnic_helper.py b/tests/unit/network_services/helpers/test_dpdkbindnic_helper.py
index e30aee854..cc980640c 100644
--- a/tests/unit/network_services/helpers/test_dpdkbindnic_helper.py
+++ b/tests/unit/network_services/helpers/test_dpdkbindnic_helper.py
@@ -16,6 +16,14 @@
import mock
import unittest
+
+import os
+
+from yardstick.error import IncorrectConfig, SSHError
+from yardstick.error import IncorrectNodeSetup
+from yardstick.error import IncorrectSetup
+from yardstick.network_services.helpers.dpdkbindnic_helper import DpdkInterface
+from yardstick.network_services.helpers.dpdkbindnic_helper import DpdkNode
from yardstick.network_services.helpers.dpdkbindnic_helper import DpdkBindHelper
from yardstick.network_services.helpers.dpdkbindnic_helper import DpdkBindHelperException
from yardstick.network_services.helpers.dpdkbindnic_helper import NETWORK_KERNEL
@@ -26,7 +34,269 @@ from yardstick.network_services.helpers.dpdkbindnic_helper import NETWORK_OTHER
from yardstick.network_services.helpers.dpdkbindnic_helper import CRYPTO_OTHER
+NAME = "tg_0"
+
+
+class TestDpdkInterface(unittest.TestCase):
+
+ SAMPLE_NETDEVS = {
+ 'enp11s0': {
+ 'address': '0a:de:ad:be:ef:f5',
+ 'device': '0x1533',
+ 'driver': 'igb',
+ 'ifindex': '2',
+ 'interface_name': 'enp11s0',
+ 'operstate': 'down',
+ 'pci_bus_id': '0000:0b:00.0',
+ 'subsystem_device': '0x1533',
+ 'subsystem_vendor': '0x15d9',
+ 'vendor': '0x8086'
+ },
+ 'lan': {
+ 'address': '0a:de:ad:be:ef:f4',
+ 'device': '0x153a',
+ 'driver': 'e1000e',
+ 'ifindex': '3',
+ 'interface_name': 'lan',
+ 'operstate': 'up',
+ 'pci_bus_id': '0000:00:19.0',
+ 'subsystem_device': '0x153a',
+ 'subsystem_vendor': '0x15d9',
+ 'vendor': '0x8086'
+ }
+ }
+
+ SAMPLE_VM_NETDEVS = {
+ 'eth1': {
+ 'address': 'fa:de:ad:be:ef:5b',
+ 'device': '0x0001',
+ 'driver': 'virtio_net',
+ 'ifindex': '3',
+ 'interface_name': 'eth1',
+ 'operstate': 'down',
+ 'pci_bus_id': '0000:00:04.0',
+ 'vendor': '0x1af4'
+ }
+ }
+
+ def test_parse_netdev_info(self):
+ output = """\
+/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/ifindex:2
+/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/address:0a:de:ad:be:ef:f5
+/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/operstate:down
+/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/device/vendor:0x8086
+/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/device/device:0x1533
+/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/device/subsystem_vendor:0x15d9
+/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/device/subsystem_device:0x1533
+/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/driver:igb
+/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/pci_bus_id:0000:0b:00.0
+/sys/devices/pci0000:00/0000:00:19.0/net/lan/ifindex:3
+/sys/devices/pci0000:00/0000:00:19.0/net/lan/address:0a:de:ad:be:ef:f4
+/sys/devices/pci0000:00/0000:00:19.0/net/lan/operstate:up
+/sys/devices/pci0000:00/0000:00:19.0/net/lan/device/vendor:0x8086
+/sys/devices/pci0000:00/0000:00:19.0/net/lan/device/device:0x153a
+/sys/devices/pci0000:00/0000:00:19.0/net/lan/device/subsystem_vendor:0x15d9
+/sys/devices/pci0000:00/0000:00:19.0/net/lan/device/subsystem_device:0x153a
+/sys/devices/pci0000:00/0000:00:19.0/net/lan/driver:e1000e
+/sys/devices/pci0000:00/0000:00:19.0/net/lan/pci_bus_id:0000:00:19.0
+"""
+ res = DpdkBindHelper.parse_netdev_info(output)
+ self.assertDictEqual(res, self.SAMPLE_NETDEVS)
+
+ def test_parse_netdev_info_virtio(self):
+ output = """\
+/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/ifindex:3
+/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/address:fa:de:ad:be:ef:5b
+/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/operstate:down
+/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/device/vendor:0x1af4
+/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/device/device:0x0001
+/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/driver:virtio_net
+"""
+ res = DpdkBindHelper.parse_netdev_info(output)
+ self.assertDictEqual(res, self.SAMPLE_VM_NETDEVS)
+
+ def test_probe_missing_values(self):
+ mock_dpdk_node = mock.Mock()
+ mock_dpdk_node.netdevs = self.SAMPLE_NETDEVS.copy()
+
+ interface = {'local_mac': '0a:de:ad:be:ef:f5'}
+ dpdk_intf = DpdkInterface(mock_dpdk_node, interface)
+
+ dpdk_intf.probe_missing_values()
+ self.assertEqual(interface['vpci'], '0000:0b:00.0')
+
+ interface['local_mac'] = '0a:de:ad:be:ef:f4'
+ dpdk_intf.probe_missing_values()
+ self.assertEqual(interface['vpci'], '0000:00:19.0')
+
+ def test_probe_missing_values_no_update(self):
+ mock_dpdk_node = mock.Mock()
+ mock_dpdk_node.netdevs = self.SAMPLE_NETDEVS.copy()
+ del mock_dpdk_node.netdevs['enp11s0']['driver']
+ del mock_dpdk_node.netdevs['lan']['driver']
+
+ interface = {'local_mac': '0a:de:ad:be:ef:f5'}
+ dpdk_intf = DpdkInterface(mock_dpdk_node, interface)
+
+ dpdk_intf.probe_missing_values()
+ self.assertNotIn('vpci', interface)
+ self.assertNotIn('driver', interface)
+
+ def test_probe_missing_values_negative(self):
+ mock_dpdk_node = mock.Mock()
+ mock_dpdk_node.netdevs.values.side_effect = IncorrectNodeSetup
+
+ interface = {'local_mac': '0a:de:ad:be:ef:f5'}
+ dpdk_intf = DpdkInterface(mock_dpdk_node, interface)
+
+ with self.assertRaises(IncorrectConfig):
+ dpdk_intf.probe_missing_values()
+
+
+class TestDpdkNode(unittest.TestCase):
+
+ INTERFACES = [
+ {'name': 'name1',
+ 'virtual-interface': {
+ 'local_mac': 404,
+ 'vpci': 'pci10',
+ }},
+ {'name': 'name2',
+ 'virtual-interface': {
+ 'local_mac': 404,
+ 'vpci': 'pci2',
+ }},
+ {'name': 'name3',
+ 'virtual-interface': {
+ 'local_mac': 404,
+ 'vpci': 'some-pci1',
+ }},
+ ]
+
+ def test_probe_dpdk_drivers(self):
+ mock_ssh_helper = mock.Mock()
+ mock_ssh_helper.execute.return_value = 0, '', ''
+
+ interfaces = [
+ {'name': 'name1',
+ 'virtual-interface': {
+ 'local_mac': 404,
+ 'vpci': 'pci10',
+ }},
+ {'name': 'name2',
+ 'virtual-interface': {
+ 'local_mac': 404,
+ 'vpci': 'pci2',
+ }},
+ {'name': 'name3',
+ 'virtual-interface': {
+ 'local_mac': 404,
+ 'vpci': 'some-pci1',
+ }},
+ ]
+
+ dpdk_node = DpdkNode(NAME, interfaces, mock_ssh_helper)
+ dpdk_helper = dpdk_node.dpdk_helper
+
+ dpdk_helper.probe_real_kernel_drivers = mock.Mock()
+ dpdk_helper.real_kernel_interface_driver_map = {
+ 'pci1': 'driver1',
+ 'pci2': 'driver2',
+ 'pci3': 'driver3',
+ 'pci4': 'driver1',
+ 'pci6': 'driver3',
+ }
+
+ dpdk_node._probe_dpdk_drivers()
+ self.assertNotIn('driver', interfaces[0]['virtual-interface'])
+ self.assertEqual(interfaces[1]['virtual-interface']['driver'], 'driver2')
+ self.assertEqual(interfaces[2]['virtual-interface']['driver'], 'driver1')
+
+ def test_check(self):
+ def update():
+ if not mock_force_rebind.called:
+ raise IncorrectConfig
+
+ interfaces[0]['virtual-interface'].update({
+ 'vpci': '0000:01:02.1',
+ 'local_ip': '10.20.30.40',
+ 'netmask': '255.255.0.0',
+ 'driver': 'ixgbe',
+ })
+
+ mock_ssh_helper = mock.Mock()
+ mock_ssh_helper.execute.return_value = 0, '', ''
+
+ interfaces = [
+ {'name': 'name1',
+ 'virtual-interface': {
+ 'local_mac': 404,
+ }},
+ ]
+
+ dpdk_node = DpdkNode(NAME, interfaces, mock_ssh_helper)
+ dpdk_node._probe_missing_values = mock_probe_missing = mock.Mock(side_effect=update)
+ dpdk_node._force_rebind = mock_force_rebind = mock.Mock()
+
+ self.assertIsNone(dpdk_node.check())
+ self.assertEqual(mock_probe_missing.call_count, 2)
+
+ @mock.patch('yardstick.network_services.helpers.dpdkbindnic_helper.DpdkInterface')
+ def test_check_negative(self, mock_intf_type):
+ mock_ssh_helper = mock.Mock()
+ mock_ssh_helper.execute.return_value = 0, '', ''
+
+ mock_intf_type().check.side_effect = SSHError
+
+ dpdk_node = DpdkNode(NAME, self.INTERFACES, mock_ssh_helper)
+
+ with self.assertRaises(IncorrectSetup):
+ dpdk_node.check()
+
+ def test_probe_netdevs(self):
+ mock_ssh_helper = mock.Mock()
+ mock_ssh_helper.execute.return_value = 0, '', ''
+
+ expected = {'key1': 500, 'key2': 'hello world'}
+ update = {'key1': 1000, 'key3': []}
+
+ dpdk_node = DpdkNode(NAME, self.INTERFACES, mock_ssh_helper)
+ dpdk_helper = dpdk_node.dpdk_helper
+ dpdk_helper.find_net_devices = mock.Mock(side_effect=[expected, update])
+
+ self.assertDictEqual(dpdk_node.netdevs, {})
+ dpdk_node._probe_netdevs()
+ self.assertDictEqual(dpdk_node.netdevs, expected)
+
+ expected = {'key1': 1000, 'key2': 'hello world', 'key3': []}
+ dpdk_node._probe_netdevs()
+ self.assertDictEqual(dpdk_node.netdevs, expected)
+
+ def test_probe_netdevs_setup_negative(self):
+ mock_ssh_helper = mock.Mock()
+ mock_ssh_helper.execute.return_value = 0, '', ''
+
+ dpdk_node = DpdkNode(NAME, self.INTERFACES, mock_ssh_helper)
+ dpdk_helper = dpdk_node.dpdk_helper
+ dpdk_helper.find_net_devices = mock.Mock(side_effect=DpdkBindHelperException)
+
+ with self.assertRaises(DpdkBindHelperException):
+ dpdk_node._probe_netdevs()
+
+ def test_force_rebind(self):
+ mock_ssh_helper = mock.Mock()
+ mock_ssh_helper.execute.return_value = 0, '', ''
+
+ dpdk_node = DpdkNode(NAME, self.INTERFACES, mock_ssh_helper)
+ dpdk_helper = dpdk_node.dpdk_helper
+ dpdk_helper.force_dpdk_rebind = mock_helper_func = mock.Mock()
+
+ dpdk_node._force_rebind()
+ self.assertEqual(mock_helper_func.call_count, 1)
+
+
class TestDpdkBindHelper(unittest.TestCase):
+ bin_path = "/opt/nsb_bin"
EXAMPLE_OUTPUT = """
Network devices using DPDK-compatible driver
@@ -111,13 +381,15 @@ Other crypto devices
def test___init__(self):
conn = mock.Mock()
conn.provision_tool = mock.Mock(return_value='path_to_tool')
+ conn.join_bin_path.return_value = os.path.join(self.bin_path, DpdkBindHelper.DPDK_DEVBIND)
dpdk_bind_helper = DpdkBindHelper(conn)
self.assertEqual(conn, dpdk_bind_helper.ssh_helper)
self.assertEqual(self.CLEAN_STATUS, dpdk_bind_helper.dpdk_status)
self.assertIsNone(dpdk_bind_helper.status_nic_row_re)
- self.assertIsNone(dpdk_bind_helper._dpdk_devbind)
+ self.assertEqual(dpdk_bind_helper.dpdk_devbind,
+ os.path.join(self.bin_path, dpdk_bind_helper.DPDK_DEVBIND))
self.assertIsNone(dpdk_bind_helper._status_cmd_attr)
def test__dpdk_execute(self):
@@ -125,8 +397,7 @@ Other crypto devices
conn.execute = mock.Mock(return_value=(0, 'output', 'error'))
conn.provision_tool = mock.Mock(return_value='tool_path')
dpdk_bind_helper = DpdkBindHelper(conn)
- self.assertEqual((0, 'output', 'error'),
- dpdk_bind_helper._dpdk_execute('command'))
+ self.assertEqual((0, 'output', 'error'), dpdk_bind_helper._dpdk_execute('command'))
def test__dpdk_execute_failure(self):
conn = mock.Mock()
@@ -141,7 +412,7 @@ Other crypto devices
dpdk_bind_helper = DpdkBindHelper(conn)
- dpdk_bind_helper._addline(NETWORK_KERNEL, self.ONE_INPUT_LINE)
+ dpdk_bind_helper._add_line(NETWORK_KERNEL, self.ONE_INPUT_LINE)
self.assertIsNotNone(dpdk_bind_helper.dpdk_status)
self.assertEqual(self.ONE_INPUT_LINE_PARSED, dpdk_bind_helper.dpdk_status[NETWORK_KERNEL])
@@ -161,11 +432,35 @@ Other crypto devices
dpdk_bind_helper = DpdkBindHelper(conn)
- dpdk_bind_helper.parse_dpdk_status_output(self.EXAMPLE_OUTPUT)
+ dpdk_bind_helper._parse_dpdk_status_output(self.EXAMPLE_OUTPUT)
self.maxDiff = None
self.assertEqual(self.PARSED_EXAMPLE, dpdk_bind_helper.dpdk_status)
+ def test_kernel_bound_pci_addresses(self):
+ mock_ssh_helper = mock.Mock()
+ mock_ssh_helper.execute.return_value = 0, '', ''
+
+ expected = ['a', 'b', 3]
+
+ dpdk_helper = DpdkBindHelper(mock_ssh_helper)
+ dpdk_helper.dpdk_status = {
+ NETWORK_DPDK: [{'vpci': 4}, {'vpci': 5}, {'vpci': 'g'}],
+ NETWORK_KERNEL: [{'vpci': 'a'}, {'vpci': 'b'}, {'vpci': 3}],
+ CRYPTO_DPDK: [],
+ }
+
+ result = dpdk_helper.kernel_bound_pci_addresses
+ self.assertEqual(result, expected)
+
+ def test_find_net_devices_negative(self):
+ mock_ssh_helper = mock.Mock()
+ mock_ssh_helper.execute.return_value = 1, 'error', 'debug'
+
+ dpdk_helper = DpdkBindHelper(mock_ssh_helper)
+
+ self.assertDictEqual(dpdk_helper.find_net_devices(), {})
+
def test_read_status(self):
conn = mock.Mock()
conn.execute = mock.Mock(return_value=(0, self.EXAMPLE_OUTPUT, ''))
@@ -180,7 +475,7 @@ Other crypto devices
dpdk_bind_helper = DpdkBindHelper(conn)
- dpdk_bind_helper.parse_dpdk_status_output(self.EXAMPLE_OUTPUT)
+ dpdk_bind_helper._parse_dpdk_status_output(self.EXAMPLE_OUTPUT)
self.assertEqual(['0000:00:04.0', '0000:00:05.0'],
dpdk_bind_helper._get_bound_pci_addresses(NETWORK_DPDK))
@@ -192,18 +487,18 @@ Other crypto devices
dpdk_bind_helper = DpdkBindHelper(conn)
- dpdk_bind_helper.parse_dpdk_status_output(self.EXAMPLE_OUTPUT)
+ dpdk_bind_helper._parse_dpdk_status_output(self.EXAMPLE_OUTPUT)
self.assertEqual({'0000:00:04.0': 'igb_uio',
- '0000:00:03.0': 'virtio-pci',
- '0000:00:05.0': 'igb_uio',
- },
- dpdk_bind_helper.interface_driver_map)
+ '0000:00:03.0': 'virtio-pci',
+ '0000:00:05.0': 'igb_uio',
+ },
+ dpdk_bind_helper.interface_driver_map)
def test_bind(self):
conn = mock.Mock()
conn.execute = mock.Mock(return_value=(0, '', ''))
- conn.provision_tool = mock.Mock(return_value='/opt/nsb_bin/dpdk-devbind.py')
+ conn.join_bin_path.return_value = os.path.join(self.bin_path, DpdkBindHelper.DPDK_DEVBIND)
dpdk_bind_helper = DpdkBindHelper(conn)
dpdk_bind_helper.read_status = mock.Mock()
@@ -217,7 +512,7 @@ Other crypto devices
def test_bind_single_pci(self):
conn = mock.Mock()
conn.execute = mock.Mock(return_value=(0, '', ''))
- conn.provision_tool = mock.Mock(return_value='/opt/nsb_bin/dpdk-devbind.py')
+ conn.join_bin_path.return_value = os.path.join(self.bin_path, DpdkBindHelper.DPDK_DEVBIND)
dpdk_bind_helper = DpdkBindHelper(conn)
dpdk_bind_helper.read_status = mock.Mock()
@@ -257,3 +552,84 @@ Other crypto devices
}
self.assertDictEqual(expected, dpdk_bind_helper.used_drivers)
+
+ def test_force_dpdk_rebind(self):
+ mock_ssh_helper = mock.Mock()
+ mock_ssh_helper.execute.return_value = 0, '', ''
+
+ dpdk_helper = DpdkBindHelper(mock_ssh_helper, 'driver2')
+ dpdk_helper.dpdk_status = {
+ NETWORK_DPDK: [
+ {
+ 'vpci': 'pci1',
+ },
+ {
+ 'vpci': 'pci3',
+ },
+ {
+ 'vpci': 'pci6',
+ },
+ {
+ 'vpci': 'pci3',
+ },
+ ]
+ }
+ dpdk_helper.real_kernel_interface_driver_map = {
+ 'pci1': 'real_driver1',
+ 'pci2': 'real_driver2',
+ 'pci3': 'real_driver1',
+ 'pci4': 'real_driver4',
+ 'pci6': 'real_driver6',
+ }
+ dpdk_helper.load_dpdk_driver = mock.Mock()
+ dpdk_helper.read_status = mock.Mock()
+ dpdk_helper.save_real_kernel_interface_driver_map = mock.Mock()
+ dpdk_helper.save_used_drivers = mock.Mock()
+ dpdk_helper.bind = mock_bind = mock.Mock()
+
+ dpdk_helper.force_dpdk_rebind()
+ self.assertEqual(mock_bind.call_count, 2)
+
+ def test_save_real_kernel_drivers(self):
+ mock_ssh_helper = mock.Mock()
+ mock_ssh_helper.execute.return_value = 0, '', ''
+
+ dpdk_helper = DpdkBindHelper(mock_ssh_helper)
+ dpdk_helper.real_kernel_drivers = {
+ 'abc': '123',
+ }
+ dpdk_helper.real_kernel_interface_driver_map = {
+ 'abc': 'AAA',
+ 'def': 'DDD',
+ 'abs': 'AAA',
+ 'ghi': 'GGG',
+ }
+
+ # save_used_drivers must be called before save_real_kernel_drivers can be
+ with self.assertRaises(AttributeError):
+ dpdk_helper.save_real_kernel_drivers()
+
+ dpdk_helper.save_used_drivers()
+
+ expected_used_drivers = {
+ 'AAA': ['abc', 'abs'],
+ 'DDD': ['def'],
+ 'GGG': ['ghi'],
+ }
+ dpdk_helper.save_real_kernel_drivers()
+ self.assertDictEqual(dpdk_helper.used_drivers, expected_used_drivers)
+ self.assertDictEqual(dpdk_helper.real_kernel_drivers, {})
+
+ def test_get_real_kernel_driver(self):
+ mock_ssh_helper = mock.Mock()
+ mock_ssh_helper.execute.side_effect = [
+ (0, 'non-matching text', ''),
+ (0, 'pre Kernel modules: real_driver1', ''),
+ (0, 'before Ethernet middle Virtio network device after', ''),
+ ]
+
+ dpdk_helper = DpdkBindHelper(mock_ssh_helper)
+
+ self.assertIsNone(dpdk_helper.get_real_kernel_driver('abc'))
+ self.assertEqual(dpdk_helper.get_real_kernel_driver('abc'), 'real_driver1')
+ self.assertEqual(dpdk_helper.get_real_kernel_driver('abc'), DpdkBindHelper.VIRTIO_DRIVER)
diff --git a/tests/unit/network_services/vnf_generic/vnf/test_prox_vnf.py b/tests/unit/network_services/vnf_generic/vnf/test_prox_vnf.py
index 08be4865b..46786a304 100644
--- a/tests/unit/network_services/vnf_generic/vnf/test_prox_vnf.py
+++ b/tests/unit/network_services/vnf_generic/vnf/test_prox_vnf.py
@@ -379,6 +379,25 @@ class TestProxApproxVnf(unittest.TestCase):
file_path = os.path.join(curr_path, filename)
return file_path
+ @mock.patch('yardstick.common.utils.open', create=True)
+ @mock.patch('yardstick.benchmark.scenarios.networking.vnf_generic.open', create=True)
+ @mock.patch('yardstick.network_services.helpers.iniparser.open', create=True)
+ @mock.patch(SSH_HELPER)
+ def test_run_prox(self, ssh, *_):
+ mock_ssh(ssh)
+
+ prox_approx_vnf = ProxApproxVnf(NAME, self.VNFD0)
+ prox_approx_vnf.scenario_helper.scenario_cfg = self.SCENARIO_CFG
+ prox_approx_vnf.ssh_helper.join_bin_path.return_value = '/tool_path12/tool_file34'
+ prox_approx_vnf.setup_helper.remote_path = 'configs/file56.cfg'
+
+ expected = "sudo bash -c 'cd /tool_path12; " \
+ "/tool_path12/tool_file34 -o cli -t -f /tmp/l3-swap-2.cfg '"
+
+ prox_approx_vnf._run()
+ result = prox_approx_vnf.ssh_helper.run.call_args[0][0]
+ self.assertEqual(result, expected)
+
@mock.patch(SSH_HELPER)
def bad_test_instantiate(self, *args):
prox_approx_vnf = ProxApproxVnf(NAME, self.VNFD0)
diff --git a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
index f8f8eb604..b2e3fd85b 100644
--- a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
+++ b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python
-# Copyright (c) 2017 Intel Corporation
+# Copyright (c) 2017-2018 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -40,7 +40,7 @@ stl_patch.start()
if stl_patch:
from yardstick.network_services.vnf_generic.vnf import sample_vnf
- from yardstick.network_services.vnf_generic.vnf.sample_vnf import VnfSshHelper
+ from yardstick.network_services.vnf_generic.vnf.vnf_ssh_helper import VnfSshHelper
from yardstick.network_services.vnf_generic.vnf.sample_vnf import SampleVNFDeployHelper
from yardstick.network_services.vnf_generic.vnf.sample_vnf import ScenarioHelper
from yardstick.network_services.vnf_generic.vnf.sample_vnf import ResourceHelper
@@ -616,7 +616,9 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
ssh_helper = mock.Mock()
ssh_helper.execute = execute
- dpdk_vnf_setup_env_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, mock.Mock())
+ scenario_helper = mock.Mock()
+ scenario_helper.nodes = [None, None]
+ dpdk_vnf_setup_env_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, scenario_helper)
dpdk_vnf_setup_env_helper._validate_cpu_cfg = mock.Mock(return_value=[])
with mock.patch.object(dpdk_vnf_setup_env_helper, '_setup_dpdk'):
@@ -638,21 +640,6 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
mock.call('lsmod | grep -i igb_uio')
])
- def test__setup_dpdk_igb_uio_not_loaded(self):
- ssh_helper = mock.Mock()
- ssh_helper.execute = mock.Mock()
- ssh_helper.execute.side_effect = [(0, 0, 0), (1, 0, 0)]
- dpdk_setup_helper = DpdkVnfSetupEnvHelper(mock.ANY, ssh_helper, mock.ANY)
- with mock.patch.object(dpdk_setup_helper, '_setup_hugepages') as \
- mock_setup_hp:
- with self.assertRaises(y_exceptions.DPDKSetupDriverError):
- dpdk_setup_helper._setup_dpdk()
- mock_setup_hp.assert_called_once()
- ssh_helper.execute.assert_has_calls([
- mock.call('sudo modprobe uio && sudo modprobe igb_uio'),
- mock.call('lsmod | grep -i igb_uio')
- ])
-
@mock.patch('yardstick.ssh.SSH')
def test__setup_resources(self, _):
vnfd_helper = VnfdHelper(deepcopy(self.VNFD_0))
@@ -690,6 +677,7 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
# ssh_helper.execute = mock.Mock(return_value = (0, 'text', ''))
# ssh_helper.execute.return_value = 0, 'output', ''
scenario_helper = mock.Mock()
+ scenario_helper.nodes = [None, None]
rv = ['0000:05:00.1', '0000:05:00.0']
dpdk_setup_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, scenario_helper)
@@ -708,6 +696,7 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
vnfd_helper = VnfdHelper(self.VNFD_0)
ssh_helper = mock.Mock()
scenario_helper = mock.Mock()
+ scenario_helper.nodes = [None, None]
dpdk_setup_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, scenario_helper)
dpdk_setup_helper.dpdk_bind_helper.bind = mock.Mock()
dpdk_setup_helper.dpdk_bind_helper.used_drivers = {
@@ -1386,7 +1375,7 @@ class TestSampleVNFDeployHelper(unittest.TestCase):
@mock.patch('yardstick.network_services.vnf_generic.vnf.sample_vnf.time')
@mock.patch('subprocess.check_output')
- def test_deploy_vnfs_disabled(self, *args):
+ def test_deploy_vnfs_disabled(self, *_):
vnfd_helper = mock.Mock()
ssh_helper = mock.Mock()
ssh_helper.join_bin_path.return_value = 'joined_path'
diff --git a/tests/unit/network_services/vnf_generic/vnf/test_tg_ping.py b/tests/unit/network_services/vnf_generic/vnf/test_tg_ping.py
index fb26f20b5..ed2274e79 100644
--- a/tests/unit/network_services/vnf_generic/vnf/test_tg_ping.py
+++ b/tests/unit/network_services/vnf_generic/vnf/test_tg_ping.py
@@ -36,7 +36,7 @@ if stl_patch:
from yardstick.network_services.vnf_generic.vnf.tg_ping import PingTrafficGen
from yardstick.network_services.vnf_generic.vnf.tg_ping import PingResourceHelper
from yardstick.network_services.vnf_generic.vnf.tg_ping import PingSetupEnvHelper
- from yardstick.network_services.vnf_generic.vnf.sample_vnf import VnfSshHelper
+ from yardstick.network_services.vnf_generic.vnf.vnf_ssh_helper import VnfSshHelper
class TestPingResourceHelper(unittest.TestCase):
diff --git a/yardstick/benchmark/scenarios/networking/vnf_generic.py b/yardstick/benchmark/scenarios/networking/vnf_generic.py
index 45c151b59..0e4785294 100644
--- a/yardstick/benchmark/scenarios/networking/vnf_generic.py
+++ b/yardstick/benchmark/scenarios/networking/vnf_generic.py
@@ -12,19 +12,19 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from collections import defaultdict
import copy
import logging
+
import ipaddress
from itertools import chain
import os
-import re
import sys
import six
import yaml
from yardstick.benchmark.scenarios import base as scenario_base
+from yardstick.error import IncorrectConfig
from yardstick.common.constants import LOG_DIR
from yardstick.common.process import terminate_children
from yardstick.common import utils
@@ -36,58 +36,12 @@ from yardstick.network_services.traffic_profile import base as tprofile_base
from yardstick.network_services.utils import get_nsb_option
from yardstick import ssh
-
traffic_profile.register_modules()
LOG = logging.getLogger(__name__)
-class SSHError(Exception):
- """Class handles ssh connection error exception"""
- pass
-
-
-class SSHTimeout(SSHError):
- """Class handles ssh connection timeout exception"""
- pass
-
-
-class IncorrectConfig(Exception):
- """Class handles incorrect configuration during setup"""
- pass
-
-
-class IncorrectSetup(Exception):
- """Class handles incorrect setup during setup"""
- pass
-
-
-class SshManager(object):
- def __init__(self, node, timeout=120):
- super(SshManager, self).__init__()
- self.node = node
- self.conn = None
- self.timeout = timeout
-
- def __enter__(self):
- """
- args -> network device mappings
- returns -> ssh connection ready to be used
- """
- try:
- self.conn = ssh.SSH.from_node(self.node)
- self.conn.wait(timeout=self.timeout)
- except SSHError as error:
- LOG.info("connect failed to %s, due to %s", self.node["ip"], error)
- # self.conn defaults to None
- return self.conn
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- if self.conn:
- self.conn.close()
-
-
class NetworkServiceTestCase(scenario_base.Scenario):
"""Class handles Generic framework to do pre-deployment VNF &
Network service testing """
@@ -104,6 +58,7 @@ class NetworkServiceTestCase(scenario_base.Scenario):
self.collector = None
self.traffic_profile = None
self.node_netdevs = {}
+ self.bin_path = get_nsb_option('bin_path', '')
def _get_ip_flow_range(self, ip_start_range):
@@ -212,16 +167,10 @@ class NetworkServiceTestCase(scenario_base.Scenario):
for vnfd in self.topology["constituent-vnfd"]
if vnf_id == vnfd["member-vnf-index"]), None)
- @staticmethod
- def get_vld_networks(networks):
- # network name is vld_id
- vld_map = {}
- for name, n in networks.items():
- try:
- vld_map[n['vld_id']] = n
- except KeyError:
- vld_map[name] = n
- return vld_map
+ def _find_vnfd_from_vnf_idx(self, vnf_id):
+ return next((vnfd
+ for vnfd in self.topology["constituent-vnfd"]
+ if vnf_id == vnfd["member-vnf-index"]), None)
@staticmethod
def find_node_if(nodes, name, if_name, vld_id):
@@ -273,7 +222,9 @@ class NetworkServiceTestCase(scenario_base.Scenario):
node1_if["peer_ifname"] = node0_if_name
# just load the network
- vld_networks = self.get_vld_networks(self.context_cfg["networks"])
+ vld_networks = {n.get('vld_id', name): n for name, n in
+ self.context_cfg["networks"].items()}
+
node0_if["network"] = vld_networks.get(vld["id"], {})
node1_if["network"] = vld_networks.get(vld["id"], {})
@@ -312,10 +263,6 @@ class NetworkServiceTestCase(scenario_base.Scenario):
node0_if["peer_intf"] = node1_copy
node1_if["peer_intf"] = node0_copy
- def _find_vnfd_from_vnf_idx(self, vnf_idx):
- return next((vnfd for vnfd in self.topology["constituent-vnfd"]
- if vnf_idx == vnfd["member-vnf-index"]), None)
-
def _update_context_with_topology(self):
for vnfd in self.topology["constituent-vnfd"]:
vnf_idx = vnfd["member-vnf-index"]
@@ -323,43 +270,6 @@ class NetworkServiceTestCase(scenario_base.Scenario):
vnfd = self._find_vnfd_from_vnf_idx(vnf_idx)
self.context_cfg["nodes"][vnf_name].update(vnfd)
- def _probe_netdevs(self, node, node_dict, timeout=120):
- try:
- return self.node_netdevs[node]
- except KeyError:
- pass
-
- netdevs = {}
- cmd = "PATH=$PATH:/sbin:/usr/sbin ip addr show"
-
- with SshManager(node_dict, timeout=timeout) as conn:
- if conn:
- exit_status = conn.execute(cmd)[0]
- if exit_status != 0:
- raise IncorrectSetup("Node's %s lacks ip tool." % node)
- exit_status, stdout, _ = conn.execute(
- self.FIND_NETDEVICE_STRING)
- if exit_status != 0:
- raise IncorrectSetup(
- "Cannot find netdev info in sysfs" % node)
- netdevs = node_dict['netdevs'] = self.parse_netdev_info(stdout)
-
- self.node_netdevs[node] = netdevs
- return netdevs
-
- @classmethod
- def _probe_missing_values(cls, netdevs, network):
-
- mac_lower = network['local_mac'].lower()
- for netdev in netdevs.values():
- if netdev['address'].lower() != mac_lower:
- continue
- network.update({
- 'driver': netdev['driver'],
- 'vpci': netdev['pci_bus_id'],
- 'ifindex': netdev['ifindex'],
- })
-
def _generate_pod_yaml(self):
context_yaml = os.path.join(LOG_DIR, "pod-{}.yaml".format(self.scenario_cfg['task_id']))
# convert OrderedDict to a list
@@ -385,84 +295,16 @@ class NetworkServiceTestCase(scenario_base.Scenario):
pass
return new_node
- TOPOLOGY_REQUIRED_KEYS = frozenset({
- "vpci", "local_ip", "netmask", "local_mac", "driver"})
-
def map_topology_to_infrastructure(self):
""" This method should verify if the available resources defined in pod.yaml
match the topology.yaml file.
:return: None. Side effect: context_cfg is updated
"""
- num_nodes = len(self.context_cfg["nodes"])
- # OpenStack instance creation time is probably proportional to the number
- # of instances
- timeout = 120 * num_nodes
- for node, node_dict in self.context_cfg["nodes"].items():
-
- for network in node_dict["interfaces"].values():
- missing = self.TOPOLOGY_REQUIRED_KEYS.difference(network)
- if not missing:
- continue
-
- # only ssh probe if there are missing values
- # ssh probe won't work on Ixia, so we had better define all our values
- try:
- netdevs = self._probe_netdevs(node, node_dict, timeout=timeout)
- except (SSHError, SSHTimeout):
- raise IncorrectConfig(
- "Unable to probe missing interface fields '%s', on node %s "
- "SSH Error" % (', '.join(missing), node))
- try:
- self._probe_missing_values(netdevs, network)
- except KeyError:
- pass
- else:
- missing = self.TOPOLOGY_REQUIRED_KEYS.difference(
- network)
- if missing:
- raise IncorrectConfig(
- "Require interface fields '%s' not found, topology file "
- "corrupted" % ', '.join(missing))
-
- # we have to generate pod.yaml here so we have vpci and driver
- self._generate_pod_yaml()
# 3. Use topology file to find connections & resolve dest address
self._resolve_topology()
self._update_context_with_topology()
- FIND_NETDEVICE_STRING = (
- r"""find /sys/devices/pci* -type d -name net -exec sh -c '{ grep -sH ^ \
- $1/ifindex $1/address $1/operstate $1/device/vendor $1/device/device \
- $1/device/subsystem_vendor $1/device/subsystem_device ; \
- printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \
- ' sh \{\}/* \;
- """)
-
- BASE_ADAPTER_RE = re.compile(
- '^/sys/devices/(.*)/net/([^/]*)/([^:]*):(.*)$', re.M)
-
- @classmethod
- def parse_netdev_info(cls, stdout):
- network_devices = defaultdict(dict)
- matches = cls.BASE_ADAPTER_RE.findall(stdout)
- for bus_path, interface_name, name, value in matches:
- dirname, bus_id = os.path.split(bus_path)
- if 'virtio' in bus_id:
- # for some stupid reason VMs include virtio1/
- # in PCI device path
- bus_id = os.path.basename(dirname)
- # remove extra 'device/' from 'device/vendor,
- # device/subsystem_vendor', etc.
- if 'device/' in name:
- name = name.split('/')[1]
- network_devices[interface_name][name] = value
- network_devices[interface_name][
- 'interface_name'] = interface_name
- network_devices[interface_name]['pci_bus_id'] = bus_id
- # convert back to regular dict
- return dict(network_devices)
-
@classmethod
def get_vnf_impl(cls, vnf_model_id):
""" Find the implementing class from vnf_model["vnf"]["name"] field
@@ -530,7 +372,7 @@ class NetworkServiceTestCase(scenario_base.Scenario):
context_cfg = self.context_cfg
vnfs = []
- # we assume OrderedDict for consistenct in instantiation
+ # we assume OrderedDict for consistency in instantiation
for node_name, node in context_cfg["nodes"].items():
LOG.debug(node)
try:
@@ -589,6 +431,9 @@ class NetworkServiceTestCase(scenario_base.Scenario):
vnf.terminate()
raise
+ # we have to generate pod.yaml here after VNF has probed so we know vpci and driver
+ self._generate_pod_yaml()
+
# 3. Run experiment
# Start listeners first to avoid losing packets
for traffic_gen in traffic_runners:
diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py
index 3f2d546fc..357f66be8 100644
--- a/yardstick/common/utils.py
+++ b/yardstick/common/utils.py
@@ -93,6 +93,21 @@ def import_modules_from_package(package, raise_exception=False):
logger.exception('Unable to import module %s', module_name)
+NON_NONE_DEFAULT = object()
+
+
+def get_key_with_default(data, key, default=NON_NONE_DEFAULT):
+ value = data.get(key, default)
+ if value is NON_NONE_DEFAULT:
+ raise KeyError(key)
+ return value
+
+
+def make_dict_from_map(data, key_map):
+ return {dest_key: get_key_with_default(data, src_key, default)
+ for dest_key, (src_key, default) in key_map.items()}
+
+
def makedirs(d):
try:
os.makedirs(d)
diff --git a/yardstick/error.py b/yardstick/error.py
new file mode 100644
index 000000000..9b84de1af
--- /dev/null
+++ b/yardstick/error.py
@@ -0,0 +1,48 @@
+# Copyright (c) 2016-2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+class SSHError(Exception):
+ """Class handles ssh connection error exception"""
+ pass
+
+
+class SSHTimeout(SSHError):
+ """Class handles ssh connection timeout exception"""
+ pass
+
+
+class IncorrectConfig(Exception):
+ """Class handles incorrect configuration during setup"""
+ pass
+
+
+class IncorrectSetup(Exception):
+ """Class handles incorrect setup during setup"""
+ pass
+
+
+class IncorrectNodeSetup(IncorrectSetup):
+ """Class handles incorrect setup during setup"""
+ pass
+
+
+class ErrorClass(object):
+
+ def __init__(self, *args, **kwargs):
+ if 'test' not in kwargs:
+ raise RuntimeError
+
+ def __getattr__(self, item):
+ raise AttributeError
diff --git a/yardstick/network_services/constants.py b/yardstick/network_services/constants.py
new file mode 100644
index 000000000..79951e353
--- /dev/null
+++ b/yardstick/network_services/constants.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2016-2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+REMOTE_TMP = "/tmp"
+DEFAULT_VNF_TIMEOUT = 3600
+PROCESS_JOIN_TIMEOUT = 3
diff --git a/yardstick/network_services/helpers/dpdkbindnic_helper.py b/yardstick/network_services/helpers/dpdkbindnic_helper.py
index 8c44b26c2..05b822c2e 100644
--- a/yardstick/network_services/helpers/dpdkbindnic_helper.py
+++ b/yardstick/network_services/helpers/dpdkbindnic_helper.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2016-2017 Intel Corporation
+# Copyright (c) 2016-2018 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,11 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
+import os
import re
-import itertools
+from collections import defaultdict
+from itertools import chain
-import six
+from yardstick.common.utils import validate_non_string_sequence
+from yardstick.error import IncorrectConfig
+from yardstick.error import IncorrectSetup
+from yardstick.error import IncorrectNodeSetup
+from yardstick.error import SSHTimeout
+from yardstick.error import SSHError
NETWORK_KERNEL = 'network_kernel'
NETWORK_DPDK = 'network_dpdk'
@@ -25,7 +32,6 @@ CRYPTO_KERNEL = 'crypto_kernel'
CRYPTO_DPDK = 'crypto_dpdk'
CRYPTO_OTHER = 'crypto_other'
-
LOG = logging.getLogger(__name__)
@@ -33,6 +39,166 @@ class DpdkBindHelperException(Exception):
pass
+class DpdkInterface(object):
+ TOPOLOGY_REQUIRED_KEYS = frozenset({
+ "vpci", "local_ip", "netmask", "local_mac", "driver"})
+
+ def __init__(self, dpdk_node, interface):
+ super(DpdkInterface, self).__init__()
+ self.dpdk_node = dpdk_node
+ self.interface = interface
+
+ try:
+ assert self.local_mac
+ except (AssertionError, KeyError):
+ raise IncorrectConfig
+
+ @property
+ def local_mac(self):
+ return self.interface['local_mac']
+
+ @property
+ def mac_lower(self):
+ return self.local_mac.lower()
+
+ @property
+ def missing_fields(self):
+ return self.TOPOLOGY_REQUIRED_KEYS.difference(self.interface)
+
+ @staticmethod
+ def _detect_socket(netdev):
+ try:
+ socket = netdev['numa_node']
+ except KeyError:
+ # Where is this documented?
+ # It seems for dual-sockets systems the second socket PCI bridge
+ # will have an address > 0x0f, e.g.
+ # Bridge PCI->PCI (P#524320 busid=0000:80:02.0 id=8086:6f04
+ if netdev['pci_bus_id'][5] == "0":
+ socket = 0
+ else:
+ # this doesn't handle quad-sockets
+ # TODO: fix this for quad-socket
+ socket = 1
+ return socket
+
+ def probe_missing_values(self):
+ try:
+ for netdev in self.dpdk_node.netdevs.values():
+ if netdev['address'].lower() == self.mac_lower:
+ socket = self._detect_socket(netdev)
+ self.interface.update({
+ 'vpci': netdev['pci_bus_id'],
+ 'driver': netdev['driver'],
+ 'socket': socket,
+ # don't need ifindex
+ })
+
+ except KeyError:
+ # if we don't find all the keys then don't update
+ pass
+
+ except (IncorrectNodeSetup, SSHError, SSHTimeout):
+ raise IncorrectConfig(
+ "Unable to probe missing interface fields '%s', on node %s "
+ "SSH Error" % (', '.join(self.missing_fields), self.dpdk_node.node_key))
+
+
+class DpdkNode(object):
+
+ def __init__(self, node_name, interfaces, ssh_helper, timeout=120):
+ super(DpdkNode, self).__init__()
+ self.interfaces = interfaces
+ self.ssh_helper = ssh_helper
+ self.node_key = node_name
+ self.timeout = timeout
+ self._dpdk_helper = None
+ self.netdevs = {}
+
+ try:
+ self.dpdk_interfaces = {intf['name']: DpdkInterface(self, intf['virtual-interface'])
+ for intf in self.interfaces}
+ except IncorrectConfig:
+ template = "MAC address is required for all interfaces, missing on: {}"
+ errors = (intf['name'] for intf in self.interfaces if
+ 'local_mac' not in intf['virtual-interface'])
+ raise IncorrectSetup(template.format(", ".join(errors)))
+
+ @property
+ def dpdk_helper(self):
+ if not isinstance(self._dpdk_helper, DpdkBindHelper):
+ self._dpdk_helper = DpdkBindHelper(self.ssh_helper)
+ return self._dpdk_helper
+
+ @property
+ def _interface_missing_iter(self):
+ return chain.from_iterable(self._interface_missing_map.values())
+
+ @property
+ def _interface_missing_map(self):
+ return {name: intf.missing_fields for name, intf in self.dpdk_interfaces.items()}
+
+ def _probe_netdevs(self):
+ self.netdevs.update(self.dpdk_helper.find_net_devices())
+
+ def _force_rebind(self):
+ return self.dpdk_helper.force_dpdk_rebind()
+
+ def _probe_dpdk_drivers(self):
+ self.dpdk_helper.probe_real_kernel_drivers()
+ for pci, driver in self.dpdk_helper.real_kernel_interface_driver_map.items():
+ for intf in self.interfaces:
+ vintf = intf['virtual-interface']
+ # stupid substring matches
+ # don't use netdev use interface
+ if vintf['vpci'].endswith(pci):
+ vintf['driver'] = driver
+ # we can't update netdevs because we may not have netdev info
+
+ def _probe_missing_values(self):
+ for intf in self.dpdk_interfaces.values():
+ intf.probe_missing_values()
+
+ def check(self):
+ # only ssh probe if there are missing values
+ # ssh probe won't work on Ixia, so we had better define all our values
+ try:
+ missing_fields_set = set(self._interface_missing_iter)
+
+ # if we are only missing driver then maybe we can get kernel module
+ # this requires vpci
+ if missing_fields_set == {'driver'}:
+ self._probe_dpdk_drivers()
+ # we can't reprobe missing values because we may not have netdev info
+
+ # if there are any other missing then we have to netdev probe
+ if missing_fields_set.difference({'driver'}):
+ self._probe_netdevs()
+ try:
+ self._probe_missing_values()
+ except IncorrectConfig:
+ # ignore for now
+ pass
+
+ # check again and verify we have all the fields
+ if set(self._interface_missing_iter):
+ # last chance fallback, rebind everything and probe
+ # this probably won't work
+ self._force_rebind()
+ self._probe_netdevs()
+ self._probe_missing_values()
+
+ errors = ("{} missing: {}".format(name, ", ".join(missing_fields)) for
+ name, missing_fields in self._interface_missing_map.items() if
+ missing_fields)
+ errors = "\n".join(errors)
+ if errors:
+ raise IncorrectSetup(errors)
+
+ finally:
+ self._dpdk_helper = None
+
+
class DpdkBindHelper(object):
DPDK_STATUS_CMD = "{dpdk_devbind} --status"
DPDK_BIND_CMD = "sudo {dpdk_devbind} {force} -b {driver} {vpci}"
@@ -42,6 +208,8 @@ class DpdkBindHelper(object):
SKIP_RE = re.compile('(====|<none>|^$)')
NIC_ROW_FIELDS = ['vpci', 'dev_type', 'iface', 'driver', 'unused', 'active']
+ UIO_DRIVER = "uio"
+
HEADER_DICT_PAIRS = [
(re.compile('^Network.*DPDK.*$'), NETWORK_DPDK),
(re.compile('^Network.*kernel.*$'), NETWORK_KERNEL),
@@ -51,6 +219,42 @@ class DpdkBindHelper(object):
(re.compile('^Other crypto.*$'), CRYPTO_OTHER),
]
+ FIND_NETDEVICE_STRING = r"""\
+find /sys/devices/pci* -type d -name net -exec sh -c '{ grep -sH ^ \
+$1/ifindex $1/address $1/operstate $1/device/vendor $1/device/device \
+$1/device/subsystem_vendor $1/device/subsystem_device $1/device/numa_node ; \
+printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \
+' sh \{\}/* \;
+"""
+
+ BASE_ADAPTER_RE = re.compile('^/sys/devices/(.*)/net/([^/]*)/([^:]*):(.*)$', re.M)
+ DPDK_DEVBIND = "dpdk-devbind.py"
+
+ @classmethod
+ def parse_netdev_info(cls, stdout):
+ network_devices = defaultdict(dict)
+ match_iter = (match.groups() for match in cls.BASE_ADAPTER_RE.finditer(stdout))
+ for bus_path, interface_name, name, value in match_iter:
+ dir_name, bus_id = os.path.split(bus_path)
+ if 'virtio' in bus_id:
+ # for some stupid reason VMs include virtio1/
+ # in PCI device path
+ bus_id = os.path.basename(dir_name)
+
+ # remove extra 'device/' from 'device/vendor,
+ # device/subsystem_vendor', etc.
+ if 'device' in name:
+ name = name.split('/')[1]
+
+ network_devices[interface_name].update({
+ name: value,
+ 'interface_name': interface_name,
+ 'pci_bus_id': bus_id,
+ })
+
+ # convert back to regular dict
+ return dict(network_devices)
+
def clean_status(self):
self.dpdk_status = {
NETWORK_KERNEL: [],
@@ -61,11 +265,17 @@ class DpdkBindHelper(object):
CRYPTO_OTHER: [],
}
- def __init__(self, ssh_helper):
+ # TODO: add support for driver other than igb_uio
+ def __init__(self, ssh_helper, dpdk_driver="igb_uio"):
+ self.ssh_helper = ssh_helper
+ self.real_kernel_interface_driver_map = {}
+ self.dpdk_driver = dpdk_driver
self.dpdk_status = None
self.status_nic_row_re = None
- self._dpdk_devbind = None
+ self.dpdk_devbind = self.ssh_helper.join_bin_path(self.DPDK_DEVBIND)
self._status_cmd_attr = None
+ self.used_drivers = None
+ self.real_kernel_drivers = {}
self.ssh_helper = ssh_helper
self.clean_status()
@@ -73,15 +283,16 @@ class DpdkBindHelper(object):
def _dpdk_execute(self, *args, **kwargs):
res = self.ssh_helper.execute(*args, **kwargs)
if res[0] != 0:
- raise DpdkBindHelperException('{} command failed with rc={}'.format(
- self.dpdk_devbind, res[0]))
+ template = '{} command failed with rc={}'
+ raise DpdkBindHelperException(template.format(self.dpdk_devbind, res[0]))
return res
- @property
- def dpdk_devbind(self):
- if self._dpdk_devbind is None:
- self._dpdk_devbind = self.ssh_helper.provision_tool(tool_file="dpdk-devbind.py")
- return self._dpdk_devbind
+ def load_dpdk_driver(self):
+ cmd_template = "sudo modprobe {} && sudo modprobe {}"
+ self.ssh_helper.execute(cmd_template.format(self.UIO_DRIVER, self.dpdk_driver))
+
+ def check_dpdk_driver(self):
+ return self.ssh_helper.execute("lsmod | grep -i {}".format(self.dpdk_driver))[0]
@property
def _status_cmd(self):
@@ -89,12 +300,14 @@ class DpdkBindHelper(object):
self._status_cmd_attr = self.DPDK_STATUS_CMD.format(dpdk_devbind=self.dpdk_devbind)
return self._status_cmd_attr
- def _addline(self, active_list, line):
+ def _add_line(self, active_list, line):
if active_list is None:
return
+
res = self.NIC_ROW_RE.match(line)
if res is None:
return
+
new_data = {k: v for k, v in zip(self.NIC_ROW_FIELDS, res.groups())}
new_data['active'] = bool(new_data['active'])
self.dpdk_status[active_list].append(new_data)
@@ -106,14 +319,14 @@ class DpdkBindHelper(object):
return a_dict
return active_dict
- def parse_dpdk_status_output(self, input):
+ def _parse_dpdk_status_output(self, output):
active_dict = None
self.clean_status()
- for a_row in input.splitlines():
+ for a_row in output.splitlines():
if self.SKIP_RE.match(a_row):
continue
active_dict = self._switch_active_dict(a_row, active_dict)
- self._addline(active_dict, a_row)
+ self._add_line(active_dict, a_row)
return self.dpdk_status
def _get_bound_pci_addresses(self, active_dict):
@@ -130,31 +343,85 @@ class DpdkBindHelper(object):
@property
def interface_driver_map(self):
return {interface['vpci']: interface['driver']
- for interface in itertools.chain.from_iterable(self.dpdk_status.values())}
+ for interface in chain.from_iterable(self.dpdk_status.values())}
def read_status(self):
- return self.parse_dpdk_status_output(self._dpdk_execute(self._status_cmd)[1])
+ return self._parse_dpdk_status_output(self._dpdk_execute(self._status_cmd)[1])
+
+ def find_net_devices(self):
+ exit_status, stdout, _ = self.ssh_helper.execute(self.FIND_NETDEVICE_STRING)
+ if exit_status != 0:
+ return {}
+
+ return self.parse_netdev_info(stdout)
def bind(self, pci_addresses, driver, force=True):
- # accept single PCI or list of PCI
- if isinstance(pci_addresses, six.string_types):
- pci_addresses = [pci_addresses]
+ # accept single PCI or sequence of PCI
+ pci_addresses = validate_non_string_sequence(pci_addresses, [pci_addresses])
+
cmd = self.DPDK_BIND_CMD.format(dpdk_devbind=self.dpdk_devbind,
driver=driver,
vpci=' '.join(list(pci_addresses)),
force='--force' if force else '')
LOG.debug(cmd)
self._dpdk_execute(cmd)
+
# update the inner status dict
self.read_status()
+ def probe_real_kernel_drivers(self):
+ self.read_status()
+ self.save_real_kernel_interface_driver_map()
+
+ def force_dpdk_rebind(self):
+ self.load_dpdk_driver()
+ self.read_status()
+ self.save_real_kernel_interface_driver_map()
+ self.save_used_drivers()
+
+ real_driver_map = {}
+ # only rebind devices that are bound to DPDK
+ for pci in self.dpdk_bound_pci_addresses:
+ # messy
+ real_driver = self.real_kernel_interface_driver_map[pci]
+ real_driver_map.setdefault(real_driver, []).append(pci)
+ for real_driver, pcis in real_driver_map.items():
+ self.bind(pcis, real_driver, force=True)
+
def save_used_drivers(self):
# invert the map, so we can bind by driver type
self.used_drivers = {}
- # sort for stabililty
+ # sort for stability
for vpci, driver in sorted(self.interface_driver_map.items()):
self.used_drivers.setdefault(driver, []).append(vpci)
+ KERNEL_DRIVER_RE = re.compile(r"Kernel modules: (\S+)", re.M)
+ VIRTIO_DRIVER_RE = re.compile(r"Ethernet.*Virtio network device", re.M)
+ VIRTIO_DRIVER = "virtio-pci"
+
+ def save_real_kernel_drivers(self):
+ # invert the map, so we can bind by driver type
+ self.real_kernel_drivers = {}
+ # sort for stability
+ for vpci, driver in sorted(self.real_kernel_interface_driver_map.items()):
+ self.used_drivers.setdefault(driver, []).append(vpci)
+
+ def get_real_kernel_driver(self, pci):
+ out = self.ssh_helper.execute('lspci -k -s %s' % pci)[1]
+ match = self.KERNEL_DRIVER_RE.search(out)
+ if match:
+ return match.group(1)
+
+ match = self.VIRTIO_DRIVER_RE.search(out)
+ if match:
+ return self.VIRTIO_DRIVER
+
+ return None
+
+ def save_real_kernel_interface_driver_map(self):
+ iter1 = ((pci, self.get_real_kernel_driver(pci)) for pci in self.interface_driver_map)
+ self.real_kernel_interface_driver_map = {pci: driver for pci, driver in iter1 if driver}
+
def rebind_drivers(self, force=True):
for driver, vpcis in self.used_drivers.items():
self.bind(vpcis, driver, force)
diff --git a/yardstick/network_services/utils.py b/yardstick/network_services/utils.py
index 7a1815eb9..4b987fafe 100644
--- a/yardstick/network_services/utils.py
+++ b/yardstick/network_services/utils.py
@@ -121,7 +121,6 @@ def provision_tool(connection, tool_path, tool_file=None):
tool_path = get_nsb_option('tool_path')
if tool_file:
tool_path = os.path.join(tool_path, tool_file)
- bin_path = get_nsb_option("bin_path")
exit_status = connection.execute("which %s > /dev/null 2>&1" % tool_path)[0]
if exit_status == 0:
return encodeutils.safe_decode(tool_path, incoming='utf-8').rstrip()
diff --git a/yardstick/network_services/vnf_generic/vnf/prox_vnf.py b/yardstick/network_services/vnf_generic/vnf/prox_vnf.py
index ee7735972..2cdb3f904 100644
--- a/yardstick/network_services/vnf_generic/vnf/prox_vnf.py
+++ b/yardstick/network_services/vnf_generic/vnf/prox_vnf.py
@@ -21,7 +21,8 @@ import time
from yardstick.common.process import check_if_process_failed
from yardstick.network_services.vnf_generic.vnf.prox_helpers import ProxDpdkVnfSetupEnvHelper
from yardstick.network_services.vnf_generic.vnf.prox_helpers import ProxResourceHelper
-from yardstick.network_services.vnf_generic.vnf.sample_vnf import SampleVNF, PROCESS_JOIN_TIMEOUT
+from yardstick.network_services.vnf_generic.vnf.sample_vnf import SampleVNF
+from yardstick.network_services.constants import PROCESS_JOIN_TIMEOUT
LOG = logging.getLogger(__name__)
diff --git a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
index d6249a874..f16b4142d 100644
--- a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
+++ b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
@@ -1,4 +1,4 @@
-# Copyright (c) 2016-2017 Intel Corporation
+# Copyright (c) 2016-2018 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
from collections import Mapping
import logging
from multiprocessing import Queue, Value, Process
+
import os
import posixpath
import re
@@ -23,7 +24,6 @@ import subprocess
import time
import six
-from six.moves import cStringIO
from trex_stl_lib.trex_stl_client import LoggerApi
from trex_stl_lib.trex_stl_client import STLClient
@@ -32,64 +32,23 @@ from yardstick.benchmark.contexts.base import Context
from yardstick.common import exceptions as y_exceptions
from yardstick.common.process import check_if_process_failed
from yardstick.common import utils
-from yardstick.network_services.helpers.dpdkbindnic_helper import DpdkBindHelper
-from yardstick.network_services.helpers.samplevnf_helper import PortPairs
+from yardstick.network_services.constants import DEFAULT_VNF_TIMEOUT
+from yardstick.network_services.constants import PROCESS_JOIN_TIMEOUT
+from yardstick.network_services.constants import REMOTE_TMP
+from yardstick.network_services.helpers.dpdkbindnic_helper import DpdkBindHelper, DpdkNode
from yardstick.network_services.helpers.samplevnf_helper import MultiPortConfig
+from yardstick.network_services.helpers.samplevnf_helper import PortPairs
from yardstick.network_services.nfvi.resource import ResourceProfile
from yardstick.network_services.utils import get_nsb_option
-from yardstick.network_services.vnf_generic.vnf.base import GenericVNF
from yardstick.network_services.vnf_generic.vnf.base import GenericTrafficGen
+from yardstick.network_services.vnf_generic.vnf.base import GenericVNF
from yardstick.network_services.vnf_generic.vnf.base import QueueFileWrapper
-from yardstick.ssh import AutoConnectSSH
-
+from yardstick.network_services.vnf_generic.vnf.vnf_ssh_helper import VnfSshHelper
-DPDK_VERSION = "dpdk-16.07"
LOG = logging.getLogger(__name__)
-REMOTE_TMP = "/tmp"
-DEFAULT_VNF_TIMEOUT = 3600
-PROCESS_JOIN_TIMEOUT = 3
-
-
-class VnfSshHelper(AutoConnectSSH):
-
- def __init__(self, node, bin_path, wait=None):
- self.node = node
- kwargs = self.args_from_node(self.node)
- if wait:
- kwargs.setdefault('wait', wait)
-
- super(VnfSshHelper, self).__init__(**kwargs)
- self.bin_path = bin_path
-
- @staticmethod
- def get_class():
- # must return static class name, anything else refers to the calling class
- # i.e. the subclass, not the superclass
- return VnfSshHelper
-
- def copy(self):
- # this copy constructor is different from SSH classes, since it uses node
- return self.get_class()(self.node, self.bin_path)
-
- def upload_config_file(self, prefix, content):
- cfg_file = os.path.join(REMOTE_TMP, prefix)
- LOG.debug(content)
- file_obj = cStringIO(content)
- self.put_file_obj(file_obj, cfg_file)
- return cfg_file
-
- def join_bin_path(self, *args):
- return os.path.join(self.bin_path, *args)
-
- def provision_tool(self, tool_path=None, tool_file=None):
- if tool_path is None:
- tool_path = self.bin_path
- return super(VnfSshHelper, self).provision_tool(tool_path, tool_file)
-
-
class SetupEnvHelper(object):
CFG_CONFIG = os.path.join(REMOTE_TMP, "sample_config")
@@ -245,7 +204,6 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper):
def setup_vnf_environment(self):
self._setup_dpdk()
- self.bound_pci = [v['virtual-interface']["vpci"] for v in self.vnfd_helper.interfaces]
self.kill_vnf()
# bind before _setup_resources so we can use dpdk_port_num
self._detect_and_bind_drivers()
@@ -263,10 +221,11 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper):
def _setup_dpdk(self):
"""Setup DPDK environment needed for VNF to run"""
self._setup_hugepages()
- self.ssh_helper.execute('sudo modprobe uio && sudo modprobe igb_uio')
- exit_status = self.ssh_helper.execute('lsmod | grep -i igb_uio')[0]
- if exit_status:
- raise y_exceptions.DPDKSetupDriverError()
+ self.dpdk_bind_helper.load_dpdk_driver()
+
+ exit_status = self.dpdk_bind_helper.check_dpdk_driver()
+ if exit_status == 0:
+ return
def get_collectd_options(self):
options = self.scenario_helper.all_options.get("collectd", {})
@@ -293,9 +252,22 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper):
plugins=plugins, interval=collectd_options.get("interval"),
timeout=self.scenario_helper.timeout)
+ def _check_interface_fields(self):
+ num_nodes = len(self.scenario_helper.nodes)
+ # OpenStack instance creation time is probably proportional to the number
+ # of instances
+ timeout = 120 * num_nodes
+ dpdk_node = DpdkNode(self.scenario_helper.name, self.vnfd_helper.interfaces,
+ self.ssh_helper, timeout)
+ dpdk_node.check()
+
def _detect_and_bind_drivers(self):
interfaces = self.vnfd_helper.interfaces
+ self._check_interface_fields()
+ # check for bound after probe
+ self.bound_pci = [v['virtual-interface']["vpci"] for v in interfaces]
+
self.dpdk_bind_helper.read_status()
self.dpdk_bind_helper.save_used_drivers()
diff --git a/yardstick/network_services/vnf_generic/vnf/tg_rfc2544_ixia.py b/yardstick/network_services/vnf_generic/vnf/tg_rfc2544_ixia.py
index 068b19d8d..265d0b7a9 100644
--- a/yardstick/network_services/vnf_generic/vnf/tg_rfc2544_ixia.py
+++ b/yardstick/network_services/vnf_generic/vnf/tg_rfc2544_ixia.py
@@ -20,11 +20,11 @@ import logging
import sys
from yardstick.common import utils
+from yardstick import error
from yardstick.network_services.vnf_generic.vnf.sample_vnf import SampleVNFTrafficGen
from yardstick.network_services.vnf_generic.vnf.sample_vnf import ClientResourceHelper
from yardstick.network_services.vnf_generic.vnf.sample_vnf import Rfc2544ResourceHelper
-
LOG = logging.getLogger(__name__)
WAIT_AFTER_CFG_LOAD = 10
@@ -36,7 +36,7 @@ sys.path.append(IXNET_LIB)
try:
from IxNet import IxNextgen
except ImportError:
- IxNextgen = utils.ErrorClass
+ IxNextgen = error.ErrorClass
class IxiaRfc2544Helper(Rfc2544ResourceHelper):
diff --git a/yardstick/network_services/vnf_generic/vnf/vnf_ssh_helper.py b/yardstick/network_services/vnf_generic/vnf/vnf_ssh_helper.py
new file mode 100644
index 000000000..8e02cf3ac
--- /dev/null
+++ b/yardstick/network_services/vnf_generic/vnf/vnf_ssh_helper.py
@@ -0,0 +1,61 @@
+# Copyright (c) 2016-2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+
+from six.moves import StringIO
+
+from yardstick.network_services.constants import REMOTE_TMP
+from yardstick.ssh import AutoConnectSSH
+
+LOG = logging.getLogger(__name__)
+
+
+class VnfSshHelper(AutoConnectSSH):
+
+ def __init__(self, node, bin_path, wait=None):
+ self.node = node
+ kwargs = self.args_from_node(self.node)
+ if wait:
+ # if wait is defined here we want to override
+ kwargs['wait'] = wait
+
+ super(VnfSshHelper, self).__init__(**kwargs)
+ self.bin_path = bin_path
+
+ @staticmethod
+ def get_class():
+ # must return static class name, anything else refers to the calling class
+ # i.e. the subclass, not the superclass
+ return VnfSshHelper
+
+ def copy(self):
+ # this copy constructor is different from SSH classes, since it uses node
+ return self.get_class()(self.node, self.bin_path)
+
+ def upload_config_file(self, prefix, content):
+ cfg_file = os.path.join(REMOTE_TMP, prefix)
+ LOG.debug(content)
+ file_obj = StringIO(content)
+ self.put_file_obj(file_obj, cfg_file)
+ return cfg_file
+
+ def join_bin_path(self, *args):
+ return os.path.join(self.bin_path, *args)
+
+ def provision_tool(self, tool_path=None, tool_file=None):
+ if tool_path is None:
+ tool_path = self.bin_path
+ return super(VnfSshHelper, self).provision_tool(tool_path, tool_file)
diff --git a/yardstick/ssh.py b/yardstick/ssh.py
index 6ddf327f2..d7adc0d05 100644
--- a/yardstick/ssh.py
+++ b/yardstick/ssh.py
@@ -78,7 +78,7 @@ from oslo_utils import encodeutils
from scp import SCPClient
import six
-from yardstick.common.utils import try_int
+from yardstick.common.utils import try_int, NON_NONE_DEFAULT, make_dict_from_map
from yardstick.network_services.utils import provision_tool
@@ -102,6 +102,7 @@ class SSH(object):
"""Represent ssh connection."""
SSH_PORT = paramiko.config.SSH_PORT
+ DEFAULT_WAIT_TIMEOUT = 120
@staticmethod
def gen_keys(key_filename, bit_count=2048):
@@ -120,6 +121,18 @@ class SSH(object):
# i.e. the subclass, not the superclass
return SSH
+ @classmethod
+ def get_arg_key_map(cls):
+ return {
+ 'user': ('user', NON_NONE_DEFAULT),
+ 'host': ('ip', NON_NONE_DEFAULT),
+ 'port': ('ssh_port', cls.SSH_PORT),
+ 'pkey': ('pkey', None),
+ 'key_filename': ('key_filename', None),
+ 'password': ('password', None),
+ 'name': ('name', None),
+ }
+
def __init__(self, user, host, port=None, pkey=None,
key_filename=None, password=None, name=None):
"""Initialize SSH client.
@@ -137,6 +150,7 @@ class SSH(object):
else:
self.log = logging.getLogger(__name__)
+ self.wait_timeout = self.DEFAULT_WAIT_TIMEOUT
self.user = user
self.host = host
# everybody wants to debug this in the caller, do it here instead
@@ -162,16 +176,9 @@ class SSH(object):
overrides = {}
if defaults is None:
defaults = {}
+
params = ChainMap(overrides, node, defaults)
- return {
- 'user': params['user'],
- 'host': params['ip'],
- 'port': params.get('ssh_port', cls.SSH_PORT),
- 'pkey': params.get('pkey'),
- 'key_filename': params.get('key_filename'),
- 'password': params.get('password'),
- 'name': params.get('name'),
- }
+ return make_dict_from_map(params, cls.get_arg_key_map())
@classmethod
def from_node(cls, node, overrides=None, defaults=None):
@@ -186,7 +193,7 @@ class SSH(object):
return key_class.from_private_key(key)
except paramiko.SSHException as e:
errors.append(e)
- raise SSHError("Invalid pkey: %s" % (errors))
+ raise SSHError("Invalid pkey: %s" % errors)
@property
def is_connected(self):
@@ -287,7 +294,7 @@ class SSH(object):
while True:
# Block until data can be read/write.
- r, w, e = select.select([session], writes, [session], 1)
+ e = select.select([session], writes, [session], 1)[2]
if session.recv_ready():
data = encodeutils.safe_decode(session.recv(4096), 'utf-8')
@@ -361,17 +368,20 @@ class SSH(object):
stderr.seek(0)
return exit_status, stdout.read(), stderr.read()
- def wait(self, timeout=120, interval=1):
+ def wait(self, timeout=None, interval=1):
"""Wait for the host will be available via ssh."""
- start_time = time.time()
+ if timeout is None:
+ timeout = self.wait_timeout
+
+ end_time = time.time() + timeout
while True:
try:
return self.execute("uname")
except (socket.error, SSHError) as e:
self.log.debug("Ssh is still unavailable: %r", e)
time.sleep(interval)
- if time.time() > (start_time + timeout):
- raise SSHTimeout("Timeout waiting for '%s'", self.host)
+ if time.time() > end_time:
+ raise SSHTimeout("Timeout waiting for '%s'" % self.host)
def put(self, files, remote_path=b'.', recursive=False):
client = self._get_client()
@@ -447,24 +457,40 @@ class SSH(object):
class AutoConnectSSH(SSH):
+ @classmethod
+ def get_arg_key_map(cls):
+ arg_key_map = super(AutoConnectSSH, cls).get_arg_key_map()
+ arg_key_map['wait'] = ('wait', True)
+ return arg_key_map
+
# always wait or we will get OpenStack SSH errors
def __init__(self, user, host, port=None, pkey=None,
key_filename=None, password=None, name=None, wait=True):
super(AutoConnectSSH, self).__init__(user, host, port, pkey, key_filename, password, name)
- self._wait = wait
+ if wait and wait is not True:
+ self.wait_timeout = int(wait)
def _make_dict(self):
data = super(AutoConnectSSH, self)._make_dict()
data.update({
- 'wait': self._wait
+ 'wait': self.wait_timeout
})
return data
def _connect(self):
if not self.is_connected:
- self._get_client()
- if self._wait:
- self.wait()
+ interval = 1
+ timeout = self.wait_timeout
+
+ end_time = time.time() + timeout
+ while True:
+ try:
+ return self._get_client()
+ except (socket.error, SSHError) as e:
+ self.log.debug("Ssh is still unavailable: %r", e)
+ time.sleep(interval)
+ if time.time() > end_time:
+ raise SSHTimeout("Timeout waiting for '%s'" % self.host)
def drop_connection(self):
""" Don't close anything, just force creation of a new client """
diff --git a/yardstick/tests/unit/benchmark/scenarios/networking/test_vnf_generic.py b/yardstick/tests/unit/benchmark/scenarios/networking/test_vnf_generic.py
index 83db6ae01..c7a29f27e 100644
--- a/yardstick/tests/unit/benchmark/scenarios/networking/test_vnf_generic.py
+++ b/yardstick/tests/unit/benchmark/scenarios/networking/test_vnf_generic.py
@@ -13,12 +13,10 @@
# limitations under the License.
from copy import deepcopy
-import errno
import os
import sys
import mock
-import six
import unittest
from yardstick import tests
@@ -26,8 +24,9 @@ from yardstick.common import utils
from yardstick.network_services.collector.subscriber import Collector
from yardstick.network_services.traffic_profile import base
from yardstick.network_services.vnf_generic import vnfdgen
-from yardstick.network_services.vnf_generic.vnf.base import \
- GenericTrafficGen, GenericVNF
+from yardstick.error import IncorrectConfig
+from yardstick.network_services.vnf_generic.vnf.base import GenericTrafficGen
+from yardstick.network_services.vnf_generic.vnf.base import GenericVNF
stl_patch = mock.patch.dict(sys.modules, tests.STL_MOCKS)
@@ -355,16 +354,6 @@ class TestNetworkServiceTestCase(unittest.TestCase):
file_path = os.path.join(curr_path, filename)
return file_path
- def test_ssh_manager(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, SYS_CLASS_NET + IP_ADDR_SHOW, ""))
- ssh.from_node.return_value = ssh_mock
- for node_dict in self.context_cfg["nodes"].values():
- with vnf_generic.SshManager(node_dict) as conn:
- self.assertIsNotNone(conn)
-
def test___init__(self):
assert self.topology
@@ -462,12 +451,7 @@ class TestNetworkServiceTestCase(unittest.TestCase):
self.s.load_vnf_models(self.scenario_cfg, self.context_cfg))
def test_map_topology_to_infrastructure(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, SYS_CLASS_NET + IP_ADDR_SHOW, ""))
- ssh.from_node.return_value = ssh_mock
- self.s.map_topology_to_infrastructure()
+ self.s.map_topology_to_infrastructure()
nodes = self.context_cfg["nodes"]
self.assertEqual('../../vnf_descriptors/tg_rfc2544_tpl.yaml',
@@ -476,26 +460,29 @@ class TestNetworkServiceTestCase(unittest.TestCase):
nodes['vnf__1']['VNF model'])
def test_map_topology_to_infrastructure_insufficient_nodes(self):
- del self.context_cfg['nodes']['vnf__1']
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(1, SYS_CLASS_NET + IP_ADDR_SHOW, ""))
- ssh.from_node.return_value = ssh_mock
+ cfg = deepcopy(self.context_cfg)
+ del cfg['nodes']['vnf__1']
- with self.assertRaises(vnf_generic.IncorrectConfig):
+ cfg_patch = mock.patch.object(self.s, 'context_cfg', cfg)
+ with cfg_patch:
+ with self.assertRaises(IncorrectConfig):
self.s.map_topology_to_infrastructure()
def test_map_topology_to_infrastructure_config_invalid(self):
- cfg = dict(self.context_cfg)
+ ssh_mock = mock.Mock()
+ ssh_mock.execute.return_value = 0, SYS_CLASS_NET + IP_ADDR_SHOW, ""
+
+ cfg = deepcopy(self.s.context_cfg)
+
+ # delete all, we don't know which will come first
del cfg['nodes']['vnf__1']['interfaces']['xe0']['local_mac']
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, SYS_CLASS_NET + IP_ADDR_SHOW, ""))
- ssh.from_node.return_value = ssh_mock
+ del cfg['nodes']['vnf__1']['interfaces']['xe1']['local_mac']
+ del cfg['nodes']['tg__1']['interfaces']['xe0']['local_mac']
+ del cfg['nodes']['tg__1']['interfaces']['xe1']['local_mac']
- with self.assertRaises(vnf_generic.IncorrectConfig):
+ config_patch = mock.patch.object(self.s, 'context_cfg', cfg)
+ with config_patch:
+ with self.assertRaises(IncorrectConfig):
self.s.map_topology_to_infrastructure()
def test__resolve_topology_invalid_config(self):
@@ -691,137 +678,3 @@ class TestNetworkServiceTestCase(unittest.TestCase):
mock.Mock(return_value=True)
with self.assertRaises(RuntimeError):
self.s.teardown()
-
- SAMPLE_NETDEVS = {
- 'enp11s0': {
- 'address': '0a:de:ad:be:ef:f5',
- 'device': '0x1533',
- 'driver': 'igb',
- 'ifindex': '2',
- 'interface_name': 'enp11s0',
- 'operstate': 'down',
- 'pci_bus_id': '0000:0b:00.0',
- 'subsystem_device': '0x1533',
- 'subsystem_vendor': '0x15d9',
- 'vendor': '0x8086'
- },
- 'lan': {
- 'address': '0a:de:ad:be:ef:f4',
- 'device': '0x153a',
- 'driver': 'e1000e',
- 'ifindex': '3',
- 'interface_name': 'lan',
- 'operstate': 'up',
- 'pci_bus_id': '0000:00:19.0',
- 'subsystem_device': '0x153a',
- 'subsystem_vendor': '0x15d9',
- 'vendor': '0x8086'
- }
- }
-
- SAMPLE_VM_NETDEVS = {
- 'eth1': {
- 'address': 'fa:de:ad:be:ef:5b',
- 'device': '0x0001',
- 'driver': 'virtio_net',
- 'ifindex': '3',
- 'interface_name': 'eth1',
- 'operstate': 'down',
- 'pci_bus_id': '0000:00:04.0',
- 'vendor': '0x1af4'
- }
- }
-
- def test_parse_netdev_info(self):
- output = """\
-/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/ifindex:2
-/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/address:0a:de:ad:be:ef:f5
-/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/operstate:down
-/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/device/vendor:0x8086
-/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/device/device:0x1533
-/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/device/subsystem_vendor:0x15d9
-/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/device/subsystem_device:0x1533
-/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/driver:igb
-/sys/devices/pci0000:00/0000:00:1c.3/0000:0b:00.0/net/enp11s0/pci_bus_id:0000:0b:00.0
-/sys/devices/pci0000:00/0000:00:19.0/net/lan/ifindex:3
-/sys/devices/pci0000:00/0000:00:19.0/net/lan/address:0a:de:ad:be:ef:f4
-/sys/devices/pci0000:00/0000:00:19.0/net/lan/operstate:up
-/sys/devices/pci0000:00/0000:00:19.0/net/lan/device/vendor:0x8086
-/sys/devices/pci0000:00/0000:00:19.0/net/lan/device/device:0x153a
-/sys/devices/pci0000:00/0000:00:19.0/net/lan/device/subsystem_vendor:0x15d9
-/sys/devices/pci0000:00/0000:00:19.0/net/lan/device/subsystem_device:0x153a
-/sys/devices/pci0000:00/0000:00:19.0/net/lan/driver:e1000e
-/sys/devices/pci0000:00/0000:00:19.0/net/lan/pci_bus_id:0000:00:19.0
-"""
- res = vnf_generic.NetworkServiceTestCase.parse_netdev_info(output)
- assert res == self.SAMPLE_NETDEVS
-
- def test_parse_netdev_info_virtio(self):
- output = """\
-/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/ifindex:3
-/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/address:fa:de:ad:be:ef:5b
-/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/operstate:down
-/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/device/vendor:0x1af4
-/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/device/device:0x0001
-/sys/devices/pci0000:00/0000:00:04.0/virtio1/net/eth1/driver:virtio_net
-"""
- res = vnf_generic.NetworkServiceTestCase.parse_netdev_info(output)
- assert res == self.SAMPLE_VM_NETDEVS
-
- def test_probe_missing_values(self):
- netdevs = self.SAMPLE_NETDEVS.copy()
- network = {'local_mac': '0a:de:ad:be:ef:f5'}
- vnf_generic.NetworkServiceTestCase._probe_missing_values(netdevs,
- network)
- assert network['vpci'] == '0000:0b:00.0'
-
- network = {'local_mac': '0a:de:ad:be:ef:f4'}
- vnf_generic.NetworkServiceTestCase._probe_missing_values(netdevs,
- network)
- assert network['vpci'] == '0000:00:19.0'
-
- @mock.patch.object(six.moves.builtins, 'open')
- def test_open_relative_path(self, mock_open):
- # NOTE(ralonsoh): the mocked function is not properly used and tested.
- mock_open_result = mock_open()
- mock_open_call_count = 1 # initial call to get result
- self.assertEqual(utils.open_relative_file('foo', 'bar'),
- mock_open_result)
-
- mock_open_call_count += 1 # one more call expected
- self.assertEqual(mock_open.call_count, mock_open_call_count)
- self.assertIn('foo', mock_open.call_args_list[-1][0][0])
- self.assertNotIn('bar', mock_open.call_args_list[-1][0][0])
-
- def open_effect(*args, **kwargs):
- if kwargs.get('name', args[0]) == os.path.join('bar', 'foo'):
- return mock_open_result
- raise IOError(errno.ENOENT, 'not found')
-
- mock_open.side_effect = open_effect
- self.assertEqual(utils.open_relative_file('foo', 'bar'),
- mock_open_result)
-
- mock_open_call_count += 2 # two more calls expected
- self.assertEqual(mock_open.call_count, mock_open_call_count)
- self.assertIn('foo', mock_open.call_args_list[-1][0][0])
- self.assertIn('bar', mock_open.call_args_list[-1][0][0])
-
- # test an IOError of type ENOENT
- mock_open.side_effect = IOError(errno.ENOENT, 'not found')
- with self.assertRaises(IOError):
- # the second call still raises
- utils.open_relative_file('foo', 'bar')
-
- mock_open_call_count += 2 # two more calls expected
- self.assertEqual(mock_open.call_count, mock_open_call_count)
- self.assertIn('foo', mock_open.call_args_list[-1][0][0])
- self.assertIn('bar', mock_open.call_args_list[-1][0][0])
-
- # test an IOError other than ENOENT
- mock_open.side_effect = IOError(errno.EBUSY, 'busy')
- with self.assertRaises(IOError):
- utils.open_relative_file('foo', 'bar')
-
- mock_open_call_count += 1 # one more call expected
- self.assertEqual(mock_open.call_count, mock_open_call_count)
diff --git a/yardstick/tests/unit/common/test_utils.py b/yardstick/tests/unit/common/test_utils.py
index b4907addc..e71d0ff0f 100644
--- a/yardstick/tests/unit/common/test_utils.py
+++ b/yardstick/tests/unit/common/test_utils.py
@@ -20,6 +20,7 @@ import unittest
import yardstick
from yardstick import ssh
+import yardstick.error
from yardstick.common import utils
from yardstick.common import constants
@@ -126,6 +127,63 @@ class CommonUtilTestCase(unittest.TestCase):
("=".join(item) for item in sorted(flattened_data.items())))
self.assertEqual(result, line)
+ def test_get_key_with_default_negative(self):
+ with self.assertRaises(KeyError):
+ utils.get_key_with_default({}, 'key1')
+
+ @mock.patch('yardstick.common.utils.open', create=True)
+ def test_(self, mock_open):
+ mock_open.side_effect = IOError
+
+ with self.assertRaises(IOError):
+ utils.find_relative_file('my/path', 'task/path')
+
+ self.assertEqual(mock_open.call_count, 2)
+
+ @mock.patch('yardstick.common.utils.open', create=True)
+ def test_open_relative_path(self, mock_open):
+ mock_open_result = mock_open()
+ mock_open_call_count = 1 # initial call to get result
+
+ self.assertEqual(utils.open_relative_file('foo', 'bar'), mock_open_result)
+
+ mock_open_call_count += 1 # one more call expected
+ self.assertEqual(mock_open.call_count, mock_open_call_count)
+ self.assertIn('foo', mock_open.call_args_list[-1][0][0])
+ self.assertNotIn('bar', mock_open.call_args_list[-1][0][0])
+
+ def open_effect(*args, **kwargs):
+ if kwargs.get('name', args[0]) == os.path.join('bar', 'foo'):
+ return mock_open_result
+ raise IOError(errno.ENOENT, 'not found')
+
+ mock_open.side_effect = open_effect
+ self.assertEqual(utils.open_relative_file('foo', 'bar'), mock_open_result)
+
+ mock_open_call_count += 2 # two more calls expected
+ self.assertEqual(mock_open.call_count, mock_open_call_count)
+ self.assertIn('foo', mock_open.call_args_list[-1][0][0])
+ self.assertIn('bar', mock_open.call_args_list[-1][0][0])
+
+ # test an IOError of type ENOENT
+ mock_open.side_effect = IOError(errno.ENOENT, 'not found')
+ with self.assertRaises(IOError):
+ # the second call still raises
+ utils.open_relative_file('foo', 'bar')
+
+ mock_open_call_count += 2 # two more calls expected
+ self.assertEqual(mock_open.call_count, mock_open_call_count)
+ self.assertIn('foo', mock_open.call_args_list[-1][0][0])
+ self.assertIn('bar', mock_open.call_args_list[-1][0][0])
+
+ # test an IOError other than ENOENT
+ mock_open.side_effect = IOError(errno.EBUSY, 'busy')
+ with self.assertRaises(IOError):
+ utils.open_relative_file('foo', 'bar')
+
+ mock_open_call_count += 1 # one more call expected
+ self.assertEqual(mock_open.call_count, mock_open_call_count)
+
class TestMacAddressToHex(unittest.TestCase):
@@ -931,9 +989,9 @@ class TestUtils(unittest.TestCase):
def test_error_class(self):
with self.assertRaises(RuntimeError):
- utils.ErrorClass()
+ yardstick.error.ErrorClass()
- error_instance = utils.ErrorClass(test='')
+ error_instance = yardstick.error.ErrorClass(test='')
with self.assertRaises(AttributeError):
error_instance.get_name()
diff --git a/yardstick/tests/unit/test_ssh.py b/yardstick/tests/unit/test_ssh.py
index dbaae8c37..615783f3e 100644
--- a/yardstick/tests/unit/test_ssh.py
+++ b/yardstick/tests/unit/test_ssh.py
@@ -21,12 +21,13 @@ import os
import socket
import unittest
from io import StringIO
+from itertools import count
import mock
from oslo_utils import encodeutils
from yardstick import ssh
-from yardstick.ssh import SSHError
+from yardstick.ssh import SSHError, SSHTimeout
from yardstick.ssh import SSH
from yardstick.ssh import AutoConnectSSH
@@ -508,13 +509,45 @@ class SSHRunTestCase(unittest.TestCase):
class TestAutoConnectSSH(unittest.TestCase):
- def test__connect_with_wait(self):
- auto_connect_ssh = AutoConnectSSH('user1', 'host1', wait=True)
- auto_connect_ssh._get_client = mock.Mock()
- auto_connect_ssh.wait = mock_wait = mock.Mock()
+ def test__connect_loop(self):
+ auto_connect_ssh = AutoConnectSSH('user1', 'host1', wait=0)
+ auto_connect_ssh._get_client = mock__get_client = mock.Mock()
auto_connect_ssh._connect()
- self.assertEqual(mock_wait.call_count, 1)
+ self.assertEqual(mock__get_client.call_count, 1)
+
+ def test___init___negative(self):
+ with self.assertRaises(TypeError):
+ AutoConnectSSH('user1', 'host1', wait=['wait'])
+
+ with self.assertRaises(ValueError):
+ AutoConnectSSH('user1', 'host1', wait='wait')
+
+ @mock.patch('yardstick.ssh.time')
+ def test__connect_loop_ssh_error(self, mock_time):
+ mock_time.time.side_effect = count()
+
+ auto_connect_ssh = AutoConnectSSH('user1', 'host1', wait=10)
+ auto_connect_ssh._get_client = mock__get_client = mock.Mock()
+ mock__get_client.side_effect = SSHError
+
+ with self.assertRaises(SSHTimeout):
+ auto_connect_ssh._connect()
+
+ self.assertEqual(mock_time.time.call_count, 12)
+
+ def test_get_file_obj(self):
+ auto_connect_ssh = AutoConnectSSH('user1', 'host1', wait=10)
+ auto_connect_ssh._get_client = mock__get_client = mock.Mock()
+ mock_client = mock__get_client()
+ mock_open_sftp = mock_client.open_sftp()
+ mock_sftp = mock.Mock()
+ mock_open_sftp.__enter__ = mock.Mock(return_value=mock_sftp)
+ mock_open_sftp.__exit__ = mock.Mock()
+
+ auto_connect_ssh.get_file_obj('remote/path', mock.Mock())
+
+ self.assertEqual(mock_sftp.getfo.call_count, 1)
def test__make_dict(self):
auto_connect_ssh = AutoConnectSSH('user1', 'host1')
@@ -527,7 +560,7 @@ class TestAutoConnectSSH(unittest.TestCase):
'key_filename': None,
'password': None,
'name': None,
- 'wait': True,
+ 'wait': AutoConnectSSH.DEFAULT_WAIT_TIMEOUT,
}
result = auto_connect_ssh._make_dict()
self.assertDictEqual(result, expected)
@@ -537,6 +570,13 @@ class TestAutoConnectSSH(unittest.TestCase):
self.assertEqual(auto_connect_ssh.get_class(), AutoConnectSSH)
+ def test_drop_connection(self):
+ auto_connect_ssh = AutoConnectSSH('user1', 'host1')
+ self.assertFalse(auto_connect_ssh._client)
+ auto_connect_ssh._client = True
+ auto_connect_ssh.drop_connection()
+ self.assertFalse(auto_connect_ssh._client)
+
@mock.patch('yardstick.ssh.SCPClient')
def test_put(self, mock_scp_client_type):
auto_connect_ssh = AutoConnectSSH('user1', 'host1')