From 4459d4475e5d9d0c60145072741513f0ea39c39c Mon Sep 17 00:00:00 2001 From: treyad Date: Tue, 20 Nov 2018 01:40:07 -0800 Subject: Support bind driver for QAT HW cards Load QAT driver Initialize QAT VFs JIRA: YARDSTICK-1504 Change-Id: I1f3768d5436bfaccac5f6226ba19131c4662a08d Signed-off-by: treyad --- .../network_services/helpers/dpdkbindnic_helper.py | 18 ++-- .../network_services/vnf_generic/vnf/ipsec_vnf.py | 8 ++ .../vnf_generic/vnf/vpp_helpers.py | 58 +++++++++++ .../vnf_generic/vnf/test_ipsec_vnf.py | 45 ++++++++ .../vnf_generic/vnf/test_vpp_helpers.py | 116 +++++++++++++++++++++ 5 files changed, 238 insertions(+), 7 deletions(-) (limited to 'yardstick') diff --git a/yardstick/network_services/helpers/dpdkbindnic_helper.py b/yardstick/network_services/helpers/dpdkbindnic_helper.py index 0fc63f88d..33a5e8c1d 100644 --- a/yardstick/network_services/helpers/dpdkbindnic_helper.py +++ b/yardstick/network_services/helpers/dpdkbindnic_helper.py @@ -13,7 +13,6 @@ # limitations under the License. import logging import os - import re from collections import defaultdict from itertools import chain @@ -21,7 +20,6 @@ from itertools import chain from yardstick.common import exceptions from yardstick.common.utils import validate_non_string_sequence - NETWORK_KERNEL = 'network_kernel' NETWORK_DPDK = 'network_dpdk' NETWORK_OTHER = 'network_other' @@ -288,12 +286,18 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \ raise DpdkBindHelperException(template.format(self.dpdk_devbind, res[0])) return res - def load_dpdk_driver(self): + def load_dpdk_driver(self, dpdk_driver=None): + if dpdk_driver is None: + dpdk_driver = self.dpdk_driver cmd_template = "sudo modprobe {} && sudo modprobe {}" - self.ssh_helper.execute(cmd_template.format(self.UIO_DRIVER, self.dpdk_driver)) - - def check_dpdk_driver(self): - return self.ssh_helper.execute("lsmod | grep -i {}".format(self.dpdk_driver))[0] + self.ssh_helper.execute( + cmd_template.format(self.UIO_DRIVER, dpdk_driver)) + + def check_dpdk_driver(self, dpdk_driver=None): + if dpdk_driver is None: + dpdk_driver = self.dpdk_driver + return \ + self.ssh_helper.execute("lsmod | grep -i {}".format(dpdk_driver))[0] @property def _status_cmd(self): diff --git a/yardstick/network_services/vnf_generic/vnf/ipsec_vnf.py b/yardstick/network_services/vnf_generic/vnf/ipsec_vnf.py index 9aedefec1..1961ac1b1 100644 --- a/yardstick/network_services/vnf_generic/vnf/ipsec_vnf.py +++ b/yardstick/network_services/vnf_generic/vnf/ipsec_vnf.py @@ -161,6 +161,14 @@ class VipsecApproxSetupEnvHelper(VppSetupEnvHelper): self).setup_vnf_environment() self.start_vpp_service() + # for QAT device DH895xCC, the number of VFs is required as 32 + if self._get_crypto_type() == 'HW_cryptodev': + sriov_numvfs = self.get_sriov_numvfs( + self.find_encrypted_data_interface()["vpci"]) + if sriov_numvfs != 32: + self.crypto_device_init( + self.find_encrypted_data_interface()["vpci"], 32) + self._update_vnfd_helper(self.sys_cores.get_cpu_layout()) self.update_vpp_interface_data() self.iface_update_numa() diff --git a/yardstick/network_services/vnf_generic/vnf/vpp_helpers.py b/yardstick/network_services/vnf_generic/vnf/vpp_helpers.py index 4e67b3c65..fe8e7b2ba 100644 --- a/yardstick/network_services/vnf_generic/vnf/vpp_helpers.py +++ b/yardstick/network_services/vnf_generic/vnf/vpp_helpers.py @@ -195,6 +195,7 @@ class VppSetupEnvHelper(DpdkVnfSetupEnvHelper): CFG_CONFIG = "/etc/vpp/startup.conf" CFG_SCRIPT = "" PIPELINE_COMMAND = "" + QAT_DRIVER = "qat_dh895xcc" VNF_TYPE = "IPSEC" VAT_BIN_NAME = 'vpp_api_test' @@ -246,6 +247,63 @@ class VppSetupEnvHelper(DpdkVnfSetupEnvHelper): except (KeyError, ValueError): return None + def crypto_device_init(self, pci_addr, numvfs): + # QAT device must be re-bound to kernel driver before initialization. + self.dpdk_bind_helper.load_dpdk_driver(self.QAT_DRIVER) + + # Stop VPP to prevent deadlock. + self.kill_vnf() + + current_driver = self.get_pci_dev_driver(pci_addr.replace(':', r'\:')) + if current_driver is not None: + self.pci_driver_unbind(pci_addr) + + # Bind to kernel driver. + self.dpdk_bind_helper.bind(pci_addr, self.QAT_DRIVER.replace('qat_', '')) + + # Initialize QAT VFs. + if numvfs > 0: + self.set_sriov_numvfs(pci_addr, numvfs) + + def get_sriov_numvfs(self, pf_pci_addr): + command = 'cat /sys/bus/pci/devices/{pci}/sriov_numvfs'. \ + format(pci=pf_pci_addr.replace(':', r'\:')) + _, stdout, _ = self.ssh_helper.execute(command) + try: + return int(stdout) + except ValueError: + LOG.debug('Reading sriov_numvfs info failed') + return 0 + + def set_sriov_numvfs(self, pf_pci_addr, numvfs=0): + command = "sh -c 'echo {num} | tee /sys/bus/pci/devices/{pci}/sriov_numvfs'". \ + format(num=numvfs, pci=pf_pci_addr.replace(':', r'\:')) + self.ssh_helper.execute(command) + + def pci_driver_unbind(self, pci_addr): + command = "sh -c 'echo {pci} | tee /sys/bus/pci/devices/{pcie}/driver/unbind'". \ + format(pci=pci_addr, pcie=pci_addr.replace(':', r'\:')) + self.ssh_helper.execute(command) + + def get_pci_dev_driver(self, pci_addr): + cmd = 'lspci -vmmks {0}'.format(pci_addr) + ret_code, stdout, _ = self.ssh_helper.execute(cmd) + if int(ret_code): + raise RuntimeError("'{0}' failed".format(cmd)) + for line in stdout.splitlines(): + if not line: + continue + name = None + value = None + try: + name, value = line.split("\t", 1) + except ValueError: + if name == "Driver:": + return None + if name == 'Driver:': + return value + return None + def vpp_create_ipsec_tunnels(self, if1_ip_addr, if2_ip_addr, if_name, n_tunnels, n_connections, crypto_alg, crypto_key, integ_alg, integ_key, addrs_ip, diff --git a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_ipsec_vnf.py b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_ipsec_vnf.py index 8b245b3bf..00dc4a5d1 100644 --- a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_ipsec_vnf.py +++ b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_ipsec_vnf.py @@ -1368,6 +1368,51 @@ class TestVipsecApproxSetupEnvHelper(unittest.TestCase): 'xe1', 'vpp_sw_index')) self.assertGreaterEqual(ssh_helper.execute.call_count, 4) + @mock.patch.object(utils, 'setup_hugepages') + def test_setup_vnf_environment_hw(self, *args): + vnfd_helper = VnfdHelper( + TestVipsecApproxVnf.VNFD['vnfd:vnfd-catalog']['vnfd'][0]) + ssh_helper = mock.Mock() + ssh_helper.execute.return_value = 0, '0', '' + scenario_helper = mock.Mock() + scenario_helper.nodes = [None, None] + scenario_helper.options = self.OPTIONS_HW + scenario_helper.all_options = self.ALL_OPTIONS + + ipsec_approx_setup_helper = VipsecApproxSetupEnvHelper(vnfd_helper, + ssh_helper, + scenario_helper) + with mock.patch.object(cpu.CpuSysCores, 'get_cpu_layout') as \ + mock_get_cpu_layout, \ + mock.patch.object(ipsec_approx_setup_helper, + 'execute_script_json_out') as \ + mock_execute_script_json_out: + mock_get_cpu_layout.return_value = self.CPU_LAYOUT + mock_execute_script_json_out.return_value = str( + self.VPP_INTERFACES_DUMP).replace("\'", "\"") + self.assertIsInstance( + ipsec_approx_setup_helper.setup_vnf_environment(), + ResourceProfile) + self.assertEqual(0, + ipsec_approx_setup_helper.get_value_by_interface_key( + 'xe0', 'numa_node')) + self.assertEqual('TenGigabitEthernetff/6/0', + ipsec_approx_setup_helper.get_value_by_interface_key( + 'xe0', 'vpp_name')) + self.assertEqual(1, + ipsec_approx_setup_helper.get_value_by_interface_key( + 'xe0', 'vpp_sw_index')) + self.assertEqual(0, + ipsec_approx_setup_helper.get_value_by_interface_key( + 'xe1', 'numa_node')) + self.assertEqual('VirtualFunctionEthernetff/7/0', + ipsec_approx_setup_helper.get_value_by_interface_key( + 'xe1', 'vpp_name')) + self.assertEqual(2, + ipsec_approx_setup_helper.get_value_by_interface_key( + 'xe1', 'vpp_sw_index')) + self.assertGreaterEqual(ssh_helper.execute.call_count, 4) + def test_calculate_frame_size(self): vnfd_helper = VnfdHelper( TestVipsecApproxVnf.VNFD['vnfd:vnfd-catalog']['vnfd'][0]) diff --git a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_vpp_helpers.py b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_vpp_helpers.py index c3066d7fe..cca604f43 100644 --- a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_vpp_helpers.py +++ b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_vpp_helpers.py @@ -837,6 +837,122 @@ class TestVppSetupEnvHelper(unittest.TestCase): self.assertIsNone(vpp_setup_env_helper.get_value_by_interface_key( 'xe2', 'vpp-err')) + def test_crypto_device_init(self): + vnfd_helper = VnfdHelper(self.VNFD_0) + ssh_helper = mock.Mock() + + scenario_helper = mock.Mock() + vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper, + scenario_helper) + vpp_setup_env_helper.dpdk_bind_helper.load_dpdk_driver = mock.Mock() + vpp_setup_env_helper.dpdk_bind_helper.bind = mock.Mock() + + vpp_setup_env_helper.kill_vnf = mock.Mock() + vpp_setup_env_helper.pci_driver_unbind = mock.Mock() + + with mock.patch.object(vpp_setup_env_helper, 'get_pci_dev_driver') as \ + mock_get_pci_dev_driver, \ + mock.patch.object(vpp_setup_env_helper, 'set_sriov_numvfs') as \ + mock_set_sriov_numvfs: + mock_get_pci_dev_driver.return_value = 'igb_uio' + self.assertIsNone( + vpp_setup_env_helper.crypto_device_init('0000:ff:06.0', 32)) + mock_set_sriov_numvfs.assert_called() + + def test_get_sriov_numvfs(self): + vnfd_helper = VnfdHelper(self.VNFD_0) + ssh_helper = mock.Mock() + ssh_helper.execute.return_value = 0, '32', '' + + scenario_helper = mock.Mock() + vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper, + scenario_helper) + self.assertEqual(32, + vpp_setup_env_helper.get_sriov_numvfs('0000:ff:06.0')) + + def test_get_sriov_numvfs_error(self): + vnfd_helper = VnfdHelper(self.VNFD_0) + ssh_helper = mock.Mock() + ssh_helper.execute.return_value = 0, 'err', '' + + scenario_helper = mock.Mock() + vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper, + scenario_helper) + self.assertEqual(0, + vpp_setup_env_helper.get_sriov_numvfs('0000:ff:06.0')) + + def test_set_sriov_numvfs(self): + vnfd_helper = VnfdHelper(self.VNFD_0) + ssh_helper = mock.Mock() + + scenario_helper = mock.Mock() + vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper, + scenario_helper) + vpp_setup_env_helper.set_sriov_numvfs('0000:ff:06.0') + self.assertEqual(ssh_helper.execute.call_count, 1) + + def test_pci_driver_unbind(self): + vnfd_helper = VnfdHelper(self.VNFD_0) + ssh_helper = mock.Mock() + + scenario_helper = mock.Mock() + vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper, + scenario_helper) + vpp_setup_env_helper.pci_driver_unbind('0000:ff:06.0') + self.assertEqual(ssh_helper.execute.call_count, 1) + + def test_get_pci_dev_driver(self): + vnfd_helper = VnfdHelper(self.VNFD_0) + ssh_helper = mock.Mock() + ssh_helper.execute.return_value = \ + 0, 'Slot: ff:07.0\n' \ + 'Class: Ethernet controller\n' \ + 'Vendor: Intel Corporation\n' \ + 'Device: 82599 Ethernet Controller Virtual Function\n' \ + 'SVendor: Intel Corporation\n' \ + 'SDevice: 82599 Ethernet Controller Virtual Function\n' \ + 'Rev: 01\n' \ + 'Driver: igb_uio\n' \ + 'Module: ixgbevf', '' + scenario_helper = mock.Mock() + vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper, + scenario_helper) + self.assertEqual('igb_uio', vpp_setup_env_helper.get_pci_dev_driver( + '0000:ff:06.0')) + + def test_get_pci_dev_driver_error(self): + vnfd_helper = VnfdHelper(self.VNFD_0) + ssh_helper = mock.Mock() + ssh_helper.execute.return_value = 1, 'err', '' + scenario_helper = mock.Mock() + vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper, + scenario_helper) + with self.assertRaises(RuntimeError) as raised: + vpp_setup_env_helper.get_pci_dev_driver( + '0000:ff:06.0') + + self.assertIn("'lspci -vmmks 0000:ff:06.0' failed", + str(raised.exception)) + + def test_get_pci_dev_driver_output_error(self): + vnfd_helper = VnfdHelper(self.VNFD_0) + ssh_helper = mock.Mock() + ssh_helper.execute.return_value = \ + 0, 'Slot: ff:07.0\n' \ + '\n\t' \ + 'Vendor: Intel Corporation\n' \ + 'Device: 82599 Ethernet Controller Virtual Function\n' \ + 'SVendor: Intel Corporation\n' \ + 'SDevice: 82599 Ethernet Controller Virtual Function\n' \ + 'Rev: 01\n' \ + 'Driver_err: igb_uio\n' \ + 'Module: ixgbevf', '' + scenario_helper = mock.Mock() + vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper, + scenario_helper) + self.assertIsNone( + vpp_setup_env_helper.get_pci_dev_driver('0000:ff:06.0')) + def test_vpp_create_ipsec_tunnels(self): vnfd_helper = VnfdHelper(self.VNFD_0) ssh_helper = mock.Mock() -- cgit 1.2.3-korg