Support bind driver for QAT HW cards

Load QAT driver
Initialize QAT VFs

JIRA: YARDSTICK-1504

Change-Id: I1f3768d5436bfaccac5f6226ba19131c4662a08d
Signed-off-by: treyad <treyad@viosoft.com>

5 files changed, 238 insertions, 7 deletions

diff --git a/yardstick/network_services/helpers/dpdkbindnic_helper.py b/yardstick/network_services/helpers/dpdkbindnic_helper.py
index 0fc63f88d..33a5e8c1d 100644
--- a/yardstick/network_services/helpers/dpdkbindnic_helper.py
+++ b/yardstick/network_services/helpers/dpdkbindnic_helper.py
@@ -13,7 +13,6 @@
 # limitations under the License.
 import logging
 import os
-
 import re
 from collections import defaultdict
 from itertools import chain
@@ -21,7 +20,6 @@ from itertools import chain
 from yardstick.common import exceptions
 from yardstick.common.utils import validate_non_string_sequence
 
-
 NETWORK_KERNEL = 'network_kernel'
 NETWORK_DPDK = 'network_dpdk'
 NETWORK_OTHER = 'network_other'
@@ -288,12 +286,18 @@ printf "%s/driver:" $1 ; basename $(readlink -s $1/device/driver); } \
             raise DpdkBindHelperException(template.format(self.dpdk_devbind, res[0]))
         return res
 
-    def load_dpdk_driver(self):
+    def load_dpdk_driver(self, dpdk_driver=None):
+        if dpdk_driver is None:
+            dpdk_driver = self.dpdk_driver
         cmd_template = "sudo modprobe {} && sudo modprobe {}"
-        self.ssh_helper.execute(cmd_template.format(self.UIO_DRIVER, self.dpdk_driver))
-
-    def check_dpdk_driver(self):
-        return self.ssh_helper.execute("lsmod | grep -i {}".format(self.dpdk_driver))[0]
+        self.ssh_helper.execute(
+            cmd_template.format(self.UIO_DRIVER, dpdk_driver))
+
+    def check_dpdk_driver(self, dpdk_driver=None):
+        if dpdk_driver is None:
+            dpdk_driver = self.dpdk_driver
+        return \
+        self.ssh_helper.execute("lsmod | grep -i {}".format(dpdk_driver))[0]
 
     @property
     def _status_cmd(self):
diff --git a/yardstick/network_services/vnf_generic/vnf/ipsec_vnf.py b/yardstick/network_services/vnf_generic/vnf/ipsec_vnf.py
index 9aedefec1..1961ac1b1 100644
--- a/yardstick/network_services/vnf_generic/vnf/ipsec_vnf.py
+++ b/yardstick/network_services/vnf_generic/vnf/ipsec_vnf.py
@@ -161,6 +161,14 @@ class VipsecApproxSetupEnvHelper(VppSetupEnvHelper):
                          self).setup_vnf_environment()
 
         self.start_vpp_service()
+        # for QAT device DH895xCC, the number of VFs is required as 32
+        if self._get_crypto_type() == 'HW_cryptodev':
+            sriov_numvfs = self.get_sriov_numvfs(
+                self.find_encrypted_data_interface()["vpci"])
+            if sriov_numvfs != 32:
+                self.crypto_device_init(
+                    self.find_encrypted_data_interface()["vpci"], 32)
+
         self._update_vnfd_helper(self.sys_cores.get_cpu_layout())
         self.update_vpp_interface_data()
         self.iface_update_numa()
diff --git a/yardstick/network_services/vnf_generic/vnf/vpp_helpers.py b/yardstick/network_services/vnf_generic/vnf/vpp_helpers.py
index 4e67b3c65..fe8e7b2ba 100644
--- a/yardstick/network_services/vnf_generic/vnf/vpp_helpers.py
+++ b/yardstick/network_services/vnf_generic/vnf/vpp_helpers.py
@@ -195,6 +195,7 @@ class VppSetupEnvHelper(DpdkVnfSetupEnvHelper):
     CFG_CONFIG = "/etc/vpp/startup.conf"
     CFG_SCRIPT = ""
     PIPELINE_COMMAND = ""
+    QAT_DRIVER = "qat_dh895xcc"
     VNF_TYPE = "IPSEC"
     VAT_BIN_NAME = 'vpp_api_test'
 
@@ -246,6 +247,63 @@ class VppSetupEnvHelper(DpdkVnfSetupEnvHelper):
         except (KeyError, ValueError):
             return None
 
+    def crypto_device_init(self, pci_addr, numvfs):
+        # QAT device must be re-bound to kernel driver before initialization.
+        self.dpdk_bind_helper.load_dpdk_driver(self.QAT_DRIVER)
+
+        # Stop VPP to prevent deadlock.
+        self.kill_vnf()
+
+        current_driver = self.get_pci_dev_driver(pci_addr.replace(':', r'\:'))
+        if current_driver is not None:
+            self.pci_driver_unbind(pci_addr)
+
+        # Bind to kernel driver.
+        self.dpdk_bind_helper.bind(pci_addr, self.QAT_DRIVER.replace('qat_', ''))
+
+        # Initialize QAT VFs.
+        if numvfs > 0:
+            self.set_sriov_numvfs(pci_addr, numvfs)
+
+    def get_sriov_numvfs(self, pf_pci_addr):
+        command = 'cat /sys/bus/pci/devices/{pci}/sriov_numvfs'. \
+            format(pci=pf_pci_addr.replace(':', r'\:'))
+        _, stdout, _ = self.ssh_helper.execute(command)
+        try:
+            return int(stdout)
+        except ValueError:
+            LOG.debug('Reading sriov_numvfs info failed')
+            return 0
+
+    def set_sriov_numvfs(self, pf_pci_addr, numvfs=0):
+        command = "sh -c 'echo {num} | tee /sys/bus/pci/devices/{pci}/sriov_numvfs'". \
+            format(num=numvfs, pci=pf_pci_addr.replace(':', r'\:'))
+        self.ssh_helper.execute(command)
+
+    def pci_driver_unbind(self, pci_addr):
+        command = "sh -c 'echo {pci} | tee /sys/bus/pci/devices/{pcie}/driver/unbind'". \
+            format(pci=pci_addr, pcie=pci_addr.replace(':', r'\:'))
+        self.ssh_helper.execute(command)
+
+    def get_pci_dev_driver(self, pci_addr):
+        cmd = 'lspci -vmmks {0}'.format(pci_addr)
+        ret_code, stdout, _ = self.ssh_helper.execute(cmd)
+        if int(ret_code):
+            raise RuntimeError("'{0}' failed".format(cmd))
+        for line in stdout.splitlines():
+            if not line:
+                continue
+            name = None
+            value = None
+            try:
+                name, value = line.split("\t", 1)
+            except ValueError:
+                if name == "Driver:":
+                    return None
+            if name == 'Driver:':
+                return value
+        return None
+
     def vpp_create_ipsec_tunnels(self, if1_ip_addr, if2_ip_addr, if_name,
                                  n_tunnels, n_connections, crypto_alg,
                                  crypto_key, integ_alg, integ_key, addrs_ip,
diff --git a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_ipsec_vnf.py b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_ipsec_vnf.py
index 8b245b3bf..00dc4a5d1 100644
--- a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_ipsec_vnf.py
+++ b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_ipsec_vnf.py
@@ -1368,6 +1368,51 @@ class TestVipsecApproxSetupEnvHelper(unittest.TestCase):
                              'xe1', 'vpp_sw_index'))
         self.assertGreaterEqual(ssh_helper.execute.call_count, 4)
 
+    @mock.patch.object(utils, 'setup_hugepages')
+    def test_setup_vnf_environment_hw(self, *args):
+        vnfd_helper = VnfdHelper(
+            TestVipsecApproxVnf.VNFD['vnfd:vnfd-catalog']['vnfd'][0])
+        ssh_helper = mock.Mock()
+        ssh_helper.execute.return_value = 0, '0', ''
+        scenario_helper = mock.Mock()
+        scenario_helper.nodes = [None, None]
+        scenario_helper.options = self.OPTIONS_HW
+        scenario_helper.all_options = self.ALL_OPTIONS
+
+        ipsec_approx_setup_helper = VipsecApproxSetupEnvHelper(vnfd_helper,
+                                                               ssh_helper,
+                                                               scenario_helper)
+        with mock.patch.object(cpu.CpuSysCores, 'get_cpu_layout') as \
+                mock_get_cpu_layout, \
+                mock.patch.object(ipsec_approx_setup_helper,
+                                  'execute_script_json_out') as \
+                        mock_execute_script_json_out:
+            mock_get_cpu_layout.return_value = self.CPU_LAYOUT
+            mock_execute_script_json_out.return_value = str(
+                self.VPP_INTERFACES_DUMP).replace("\'", "\"")
+            self.assertIsInstance(
+                ipsec_approx_setup_helper.setup_vnf_environment(),
+                ResourceProfile)
+        self.assertEqual(0,
+                         ipsec_approx_setup_helper.get_value_by_interface_key(
+                             'xe0', 'numa_node'))
+        self.assertEqual('TenGigabitEthernetff/6/0',
+                         ipsec_approx_setup_helper.get_value_by_interface_key(
+                             'xe0', 'vpp_name'))
+        self.assertEqual(1,
+                         ipsec_approx_setup_helper.get_value_by_interface_key(
+                             'xe0', 'vpp_sw_index'))
+        self.assertEqual(0,
+                         ipsec_approx_setup_helper.get_value_by_interface_key(
+                             'xe1', 'numa_node'))
+        self.assertEqual('VirtualFunctionEthernetff/7/0',
+                         ipsec_approx_setup_helper.get_value_by_interface_key(
+                             'xe1', 'vpp_name'))
+        self.assertEqual(2,
+                         ipsec_approx_setup_helper.get_value_by_interface_key(
+                             'xe1', 'vpp_sw_index'))
+        self.assertGreaterEqual(ssh_helper.execute.call_count, 4)
+
     def test_calculate_frame_size(self):
         vnfd_helper = VnfdHelper(
             TestVipsecApproxVnf.VNFD['vnfd:vnfd-catalog']['vnfd'][0])
diff --git a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_vpp_helpers.py b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_vpp_helpers.py
index c3066d7fe..cca604f43 100644
--- a/yardstick/tests/unit/network_services/vnf_generic/vnf/test_vpp_helpers.py
+++ b/yardstick/tests/unit/network_services/vnf_generic/vnf/test_vpp_helpers.py
@@ -837,6 +837,122 @@ class TestVppSetupEnvHelper(unittest.TestCase):
         self.assertIsNone(vpp_setup_env_helper.get_value_by_interface_key(
             'xe2', 'vpp-err'))
 
+    def test_crypto_device_init(self):
+        vnfd_helper = VnfdHelper(self.VNFD_0)
+        ssh_helper = mock.Mock()
+
+        scenario_helper = mock.Mock()
+        vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper,
+                                                 scenario_helper)
+        vpp_setup_env_helper.dpdk_bind_helper.load_dpdk_driver = mock.Mock()
+        vpp_setup_env_helper.dpdk_bind_helper.bind = mock.Mock()
+
+        vpp_setup_env_helper.kill_vnf = mock.Mock()
+        vpp_setup_env_helper.pci_driver_unbind = mock.Mock()
+
+        with mock.patch.object(vpp_setup_env_helper, 'get_pci_dev_driver') as \
+                mock_get_pci_dev_driver, \
+                mock.patch.object(vpp_setup_env_helper, 'set_sriov_numvfs') as \
+                        mock_set_sriov_numvfs:
+            mock_get_pci_dev_driver.return_value = 'igb_uio'
+            self.assertIsNone(
+                vpp_setup_env_helper.crypto_device_init('0000:ff:06.0', 32))
+            mock_set_sriov_numvfs.assert_called()
+
+    def test_get_sriov_numvfs(self):
+        vnfd_helper = VnfdHelper(self.VNFD_0)
+        ssh_helper = mock.Mock()
+        ssh_helper.execute.return_value = 0, '32', ''
+
+        scenario_helper = mock.Mock()
+        vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper,
+                                                 scenario_helper)
+        self.assertEqual(32,
+                         vpp_setup_env_helper.get_sriov_numvfs('0000:ff:06.0'))
+
+    def test_get_sriov_numvfs_error(self):
+        vnfd_helper = VnfdHelper(self.VNFD_0)
+        ssh_helper = mock.Mock()
+        ssh_helper.execute.return_value = 0, 'err', ''
+
+        scenario_helper = mock.Mock()
+        vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper,
+                                                 scenario_helper)
+        self.assertEqual(0,
+                         vpp_setup_env_helper.get_sriov_numvfs('0000:ff:06.0'))
+
+    def test_set_sriov_numvfs(self):
+        vnfd_helper = VnfdHelper(self.VNFD_0)
+        ssh_helper = mock.Mock()
+
+        scenario_helper = mock.Mock()
+        vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper,
+                                                 scenario_helper)
+        vpp_setup_env_helper.set_sriov_numvfs('0000:ff:06.0')
+        self.assertEqual(ssh_helper.execute.call_count, 1)
+
+    def test_pci_driver_unbind(self):
+        vnfd_helper = VnfdHelper(self.VNFD_0)
+        ssh_helper = mock.Mock()
+
+        scenario_helper = mock.Mock()
+        vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper,
+                                                 scenario_helper)
+        vpp_setup_env_helper.pci_driver_unbind('0000:ff:06.0')
+        self.assertEqual(ssh_helper.execute.call_count, 1)
+
+    def test_get_pci_dev_driver(self):
+        vnfd_helper = VnfdHelper(self.VNFD_0)
+        ssh_helper = mock.Mock()
+        ssh_helper.execute.return_value = \
+            0, 'Slot:	ff:07.0\n' \
+               'Class:	Ethernet controller\n' \
+               'Vendor:	Intel Corporation\n' \
+               'Device:	82599 Ethernet Controller Virtual Function\n' \
+               'SVendor:	Intel Corporation\n' \
+               'SDevice:	82599 Ethernet Controller Virtual Function\n' \
+               'Rev:	01\n' \
+               'Driver:	igb_uio\n' \
+               'Module:	ixgbevf', ''
+        scenario_helper = mock.Mock()
+        vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper,
+                                                 scenario_helper)
+        self.assertEqual('igb_uio', vpp_setup_env_helper.get_pci_dev_driver(
+            '0000:ff:06.0'))
+
+    def test_get_pci_dev_driver_error(self):
+        vnfd_helper = VnfdHelper(self.VNFD_0)
+        ssh_helper = mock.Mock()
+        ssh_helper.execute.return_value = 1, 'err', ''
+        scenario_helper = mock.Mock()
+        vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper,
+                                                 scenario_helper)
+        with self.assertRaises(RuntimeError) as raised:
+            vpp_setup_env_helper.get_pci_dev_driver(
+                '0000:ff:06.0')
+
+        self.assertIn("'lspci -vmmks 0000:ff:06.0' failed",
+                      str(raised.exception))
+
+    def test_get_pci_dev_driver_output_error(self):
+        vnfd_helper = VnfdHelper(self.VNFD_0)
+        ssh_helper = mock.Mock()
+        ssh_helper.execute.return_value = \
+            0, 'Slot:	ff:07.0\n' \
+               '\n\t' \
+               'Vendor:	Intel Corporation\n' \
+               'Device:	82599 Ethernet Controller Virtual Function\n' \
+               'SVendor:	Intel Corporation\n' \
+               'SDevice:	82599 Ethernet Controller Virtual Function\n' \
+               'Rev:	01\n' \
+               'Driver_err:	igb_uio\n' \
+               'Module:	ixgbevf', ''
+        scenario_helper = mock.Mock()
+        vpp_setup_env_helper = VppSetupEnvHelper(vnfd_helper, ssh_helper,
+                                                 scenario_helper)
+        self.assertIsNone(
+            vpp_setup_env_helper.get_pci_dev_driver('0000:ff:06.0'))
+
     def test_vpp_create_ipsec_tunnels(self):
         vnfd_helper = VnfdHelper(self.VNFD_0)
         ssh_helper = mock.Mock()