NSB: fix port topology

Add a new PortPair class to resolve the
topology into list of public and private ports.

Before we were calculating public/private in multiple
locations and using different conventions.

In addition for all the DPDK test we need to use the DPDK
port number and no rely on interface ordering or interface naming
conventions.

We used to use  xe0 -> 0, xe1 -> 1, etc.  This is not the DPDK port
number.

Use the new dpdknicbind_helper class to parse the output of
dpdk-devbind.py to find the actual DPDK port number at runtime.

We then use this DPDK port number to correctly calculate the
port_mask_hex.

The port mask maps the DPDK port num (PMD ID) to the LINK ID
used in the pipeline config

We also need to make sure we only use the interfaces matched to the
topology and not use all the interfaces, because in some cases we will
have unused interfaces.  In particular TRex always requires an even
number of interfaces, so for single port TRex tests we have to create
the second port and not use it.

Thus we had to modify the traffic generator stats code to only dump
stats for used ports and no unused ports.

Ixia was using interface ordering to map to Ixia ports, instead we use
the dpdk_port_num which must be hardcoded for Ixia.

Renamed traffic_profile.execute to traffic_profile.execute_traffic so
we can trace the code easier.

We pass the port used by the traffic profile to generate_samples so we
don't get stats for unused ports.

Fixed up vPE config creation and bring up issues.

Fixed up CGNAPT and UDP_Replay to work correctly.

Tested with 4-port scale-out

Change-Id: I2e4f328bff2904108081e92a4bf712333fa73869
Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>
Signed-off-by: Edward MacGillivray <edward.s.macgillivray@intel.com>

1 files changed, 145 insertions, 0 deletions

diff --git a/yardstick/network_services/helpers/dpdknicbind_helper.py b/yardstick/network_services/helpers/dpdknicbind_helper.py
new file mode 100644
index 000000000..605d08d38
--- /dev/null
+++ b/yardstick/network_services/helpers/dpdknicbind_helper.py
@@ -0,0 +1,145 @@
+# Copyright (c) 2016-2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+import itertools
+
+NETWORK_KERNEL = 'network_kernel'
+NETWORK_DPDK = 'network_dpdk'
+NETWORK_OTHER = 'network_other'
+CRYPTO_KERNEL = 'crypto_kernel'
+CRYPTO_DPDK = 'crypto_dpdk'
+CRYPTO_OTHER = 'crypto_other'
+
+
+class DpdkBindHelperException(Exception):
+    pass
+
+
+class DpdkBindHelper(object):
+    DPDK_STATUS_CMD = "{dpdk_nic_bind} --status"
+    DPDK_BIND_CMD = "sudo {dpdk_nic_bind} {force} -b {driver} {vpci}"
+
+    NIC_ROW_RE = re.compile("([^ ]+) '([^']+)' (?:if=([^ ]+) )?drv=([^ ]+) "
+                            "unused=([^ ]*)(?: (\*Active\*))?")
+    SKIP_RE = re.compile('(====|<none>|^$)')
+    NIC_ROW_FIELDS = ['vpci', 'dev_type', 'iface', 'driver', 'unused', 'active']
+
+    HEADER_DICT_PAIRS = [
+        (re.compile('^Network.*DPDK.*$'), NETWORK_DPDK),
+        (re.compile('^Network.*kernel.*$'), NETWORK_KERNEL),
+        (re.compile('^Other network.*$'), NETWORK_OTHER),
+        (re.compile('^Crypto.*DPDK.*$'), CRYPTO_DPDK),
+        (re.compile('^Crypto.*kernel$'), CRYPTO_KERNEL),
+        (re.compile('^Other crypto.*$'), CRYPTO_OTHER),
+    ]
+
+    def clean_status(self):
+        self.dpdk_status = {
+            NETWORK_KERNEL: [],
+            NETWORK_DPDK: [],
+            CRYPTO_KERNEL: [],
+            CRYPTO_DPDK: [],
+            NETWORK_OTHER: [],
+            CRYPTO_OTHER: [],
+        }
+
+    def __init__(self, ssh_helper):
+        self.dpdk_status = None
+        self.status_nic_row_re = None
+        self._dpdk_nic_bind_attr = None
+        self._status_cmd_attr = None
+
+        self.ssh_helper = ssh_helper
+        self.clean_status()
+
+    def _dpdk_execute(self, *args, **kwargs):
+        res = self.ssh_helper.execute(*args, **kwargs)
+        if res[0] != 0:
+            raise DpdkBindHelperException('{} command failed with rc={}'.format(
+                self._dpdk_nic_bind, res[0]))
+        return res
+
+    @property
+    def _dpdk_nic_bind(self):
+        if self._dpdk_nic_bind_attr is None:
+            self._dpdk_nic_bind_attr = self.ssh_helper.provision_tool(tool_file="dpdk-devbind.py")
+        return self._dpdk_nic_bind_attr
+
+    @property
+    def _status_cmd(self):
+        if self._status_cmd_attr is None:
+            self._status_cmd_attr = self.DPDK_STATUS_CMD.format(dpdk_nic_bind=self._dpdk_nic_bind)
+        return self._status_cmd_attr
+
+    def _addline(self, active_list, line):
+        if active_list is None:
+            return
+        res = self.NIC_ROW_RE.match(line)
+        if res is None:
+            return
+        new_data = {k: v for k, v in zip(self.NIC_ROW_FIELDS, res.groups())}
+        new_data['active'] = bool(new_data['active'])
+        self.dpdk_status[active_list].append(new_data)
+
+    @classmethod
+    def _switch_active_dict(cls, a_row, active_dict):
+        for regexp, a_dict in cls.HEADER_DICT_PAIRS:
+            if regexp.match(a_row):
+                return a_dict
+        return active_dict
+
+    def parse_dpdk_status_output(self, input):
+        active_dict = None
+        self.clean_status()
+        for a_row in input.splitlines():
+            if self.SKIP_RE.match(a_row):
+                continue
+            active_dict = self._switch_active_dict(a_row, active_dict)
+            self._addline(active_dict, a_row)
+        return self.dpdk_status
+
+    def _get_bound_pci_addresses(self, active_dict):
+        return [iface['vpci'] for iface in self.dpdk_status[active_dict]]
+
+    @property
+    def dpdk_bound_pci_addresses(self):
+        return self._get_bound_pci_addresses(NETWORK_DPDK)
+
+    @property
+    def kernel_bound_pci_addresses(self):
+        return self._get_bound_pci_addresses(NETWORK_KERNEL)
+
+    @property
+    def interface_driver_map(self):
+        return {interface['vpci']: interface['driver']
+                for interface in itertools.chain(*self.dpdk_status.values())}
+
+    def read_status(self):
+        return self.parse_dpdk_status_output(self._dpdk_execute(self._status_cmd)[1])
+
+    def bind(self, pci, driver, force=True):
+        cmd = self.DPDK_BIND_CMD.format(dpdk_nic_bind=self._dpdk_nic_bind,
+                                        driver=driver,
+                                        vpci=' '.join(list(pci)),
+                                        force='--force' if force else '')
+        self._dpdk_execute(cmd)
+        # update the inner status dict
+        self.read_status()
+
+    def save_used_drivers(self):
+        self.used_drivers = self.interface_driver_map
+
+    def rebind_drivers(self, force=True):
+        for vpci, driver in self.used_drivers.items():
+            self.bind(vpci, driver, force)