Setup OVS-DPDK Standalone Context

This patch performs following tasks
- getting dpdk_nic_bind path
- setup ovs,dpdk ports,vhostuserports,
- creation of vm,
- apache2 licence
- test cases for ovsdpdk
- Changes to standalone context - update unit test cases

Change-Id: I54e4062eb440b8677625e4abe6e0579d9fd54d41
Signed-off-by: Bindya N <bindya.narayan@intel.com>

3 files changed, 404 insertions, 18 deletions

diff --git a/yardstick/benchmark/contexts/ovsdpdk.py b/yardstick/benchmark/contexts/ovsdpdk.py
new file mode 100644
index 000000000..86610305e
--- /dev/null
+++ b/yardstick/benchmark/contexts/ovsdpdk.py
@@ -0,0 +1,369 @@
+# Copyright (c) 2016-2017 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+import os
+import yaml
+import time
+import glob
+import itertools
+import logging
+from yardstick import ssh
+from yardstick.benchmark.contexts.standalone import StandaloneContext
+
+BIN_PATH = "/opt/isb_bin/"
+DPDK_NIC_BIND = "dpdk_nic_bind.py"
+
+log = logging.getLogger(__name__)
+
+VM_TEMPLATE = """
+<domain type='kvm'>
+  <name>vm1</name>
+  <uuid>18230c0c-635d-4c50-b2dc-a213d30acb34</uuid>
+  <memory unit='KiB'>20971520</memory>
+  <currentMemory unit="KiB">20971520</currentMemory>
+  <memoryBacking>
+    <hugepages/>
+  </memoryBacking>
+  <vcpu placement='static'>20</vcpu>
+  <os>
+    <type arch='x86_64' machine='pc'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <features>
+    <acpi/>
+    <apic/>
+  </features>
+  <cpu match="exact" mode='host-model'>
+    <model fallback='allow'/>
+    <topology sockets='1' cores='10' threads='2'/>
+  </cpu>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/qemu-system-x86_64</emulator>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2' cache='none'/>
+      <source file="{vm_image}"/>
+      <target dev='vda' bus='virtio'/>
+      <address bus="0x00" domain="0x0000"
+      function="0x0" slot="0x04" type="pci" />
+    </disk>
+    <!--disk type='dir' device='disk'>
+      <driver name='qemu' type='fat'/>
+      <source dir='/opt/isb_bin/dpdk'/>
+      <target dev='vdb' bus='virtio'/>
+      <readonly/>
+    </disk-->
+    <interface type="bridge">
+      <mac address="00:00:00:ab:cd:ef" />
+      <source bridge="br-int" />
+    </interface>
+    <interface type='vhostuser'>
+      <mac address='00:00:00:00:00:01'/>
+      <source type='unix' path='/usr/local/var/run/openvswitch/dpdkvhostuser0' mode='client'/>
+       <model type='virtio'/>
+      <driver queues='4'>
+        <host mrg_rxbuf='off'/>
+      </driver>
+    </interface>
+    <interface type='vhostuser'>
+      <mac address='00:00:00:00:00:02'/>
+      <source type='unix' path='/usr/local/var/run/openvswitch/dpdkvhostuser1' mode='client'/>
+      <model type='virtio'/>
+      <driver queues='4'>
+        <host mrg_rxbuf='off'/>
+      </driver>
+    </interface>
+    <serial type='pty'>
+      <target port='0'/>
+    </serial>
+    <console type='pty'>
+      <target type='serial' port='0'/>
+    </console>
+    <graphics autoport="yes" listen="0.0.0.0" port="1" type="vnc" />
+  </devices>
+</domain>
+"""
+
+
+class Ovsdpdk(StandaloneContext):
+    def __init__(self):
+        self.name = None
+        self.file_path = None
+        self.nodes = []
+        self.vm_deploy = False
+        self.ovs = []
+        self.first_run = True
+        self.dpdk_nic_bind = BIN_PATH + DPDK_NIC_BIND
+        self.user = ""
+        self.ssh_ip = ""
+        self.passwd = ""
+        self.ssh_port = ""
+        self.auth_type = ""
+
+    def init(self):
+        '''initializes itself'''
+        log.debug("In init")
+        self.parse_pod_and_get_data()
+
+    def parse_pod_and_get_data(self, file_path):
+        self.file_path = file_path
+        print("parsing pod file: {0}".format(self.file_path))
+        try:
+            with open(self.file_path) as stream:
+                cfg = yaml.load(stream)
+        except IOError:
+            print("File {0} does not exist".format(self.file_path))
+            raise
+
+        self.ovs.extend([node for node in cfg["nodes"]
+                         if node["role"] == "Ovsdpdk"])
+        self.user = self.ovs[0]['user']
+        self.ssh_ip = self.ovs[0]['ip']
+        if self.ovs[0]['auth_type'] == "password":
+            self.passwd = self.ovs[0]['password']
+        else:
+            self.ssh_port = self.ovs[0]['ssh_port']
+            self.key_filename = self.ovs[0]['key_filename']
+
+    def ssh_remote_machine(self):
+        if self.ovs[0]['auth_type'] == "password":
+            self.connection = ssh.SSH(
+                self.user,
+                self.ssh_ip,
+                password=self.passwd)
+            self.connection.wait()
+        else:
+            if self.ssh_port is not None:
+                ssh_port = self.ssh_port
+            else:
+                ssh_port = ssh.DEFAULT_PORT
+            self.connection = ssh.SSH(
+                self.user,
+                self.ssh_ip,
+                port=ssh_port,
+                key_filename=self.key_filename)
+            self.connection.wait()
+
+    def get_nic_details(self):
+        nic_details = {}
+        nic_details['interface'] = {}
+        nic_details['pci'] = self.ovs[0]['phy_ports']
+        nic_details['phy_driver'] = self.ovs[0]['phy_driver']
+        nic_details['vports_mac'] = self.ovs[0]['vports_mac']
+        #    Make sure that ports are bound to kernel drivers e.g. i40e/ixgbe
+        for i, _ in enumerate(nic_details['pci']):
+            err, out, _ = self.connection.execute(
+                "{dpdk_nic_bind} --force -b {driver} {port}".format(
+                    dpdk_nic_bind=self.dpdk_nic_bind,
+                    driver=self.ovs[0]['phy_driver'],
+                    port=self.ovs[0]['phy_ports'][i]))
+            err, out, _ = self.connection.execute(
+                "lshw -c network -businfo | grep '{port}'".format(
+                    port=self.ovs[0]['phy_ports'][i]))
+            a = out.split()[1]
+            err, out, _ = self.connection.execute(
+                "ip -s link show {interface}".format(
+                    interface=out.split()[1]))
+            nic_details['interface'][i] = str(a)
+        print("{0}".format(nic_details))
+        return nic_details
+
+    def install_req_libs(self):
+        if self.first_run:
+            err, out, _ = self.connection.execute("apt-get update")
+            print("{0}".format(out))
+            err, out, _ = self.connection.execute(
+                "apt-get -y install qemu-kvm libvirt-bin")
+            print("{0}".format(out))
+            err, out, _ = self.connection.execute(
+                "apt-get -y install libvirt-dev  bridge-utils numactl")
+            print("{0}".format(out))
+            self.first_run = False
+
+    def setup_ovs(self, vpcis):
+        self.connection.execute("/usr/bin/chmod 0666 /dev/vfio/*")
+        self.connection.execute("/usr/bin/chmod a+x /dev/vfio")
+        self.connection.execute("pkill -9 ovs")
+        self.connection.execute("ps -ef | grep ovs | grep -v grep | "
+                                "awk '{print $2}' | xargs -r kill -9")
+        self.connection.execute("killall -r 'ovs*'")
+        self.connection.execute(
+            "mkdir -p {0}/etc/openvswitch".format(self.ovs[0]["vpath"]))
+        self.connection.execute(
+            "mkdir -p {0}/var/run/openvswitch".format(self.ovs[0]["vpath"]))
+        self.connection.execute(
+            "rm {0}/etc/openvswitch/conf.db".format(self.ovs[0]["vpath"]))
+        self.connection.execute(
+            "ovsdb-tool create {0}/etc/openvswitch/conf.db "
+            "{0}/share/openvswitch/"
+            "vswitch.ovsschema".format(self.ovs[0]["vpath"]))
+        self.connection.execute("modprobe vfio-pci")
+        self.connection.execute("chmod a+x /dev/vfio")
+        self.connection.execute("chmod 0666 /dev/vfio/*")
+        for vpci in vpcis:
+            self.connection.execute(
+                "/opt/isb_bin/dpdk_nic_bind.py "
+                "--bind=vfio-pci {0}".format(vpci))
+
+    def start_ovs_serverswitch(self):
+            self.connection.execute("mkdir -p /usr/local/var/run/openvswitch")
+            self.connection.execute(
+                "ovsdb-server --remote=punix:"
+                "/usr/local/var/run/openvswitch/db.sock  --pidfile --detach")
+            self.connection.execute(
+                "ovs-vsctl --no-wait set "
+                "Open_vSwitch . other_config:dpdk-init=true")
+            self.connection.execute(
+                "ovs-vsctl --no-wait set "
+                "Open_vSwitch . other_config:dpdk-lcore-mask=0x3")
+            self.connection.execute(
+                "ovs-vsctl --no-wait set "
+                "Open_vSwitch . other_config:dpdk-socket-mem='2048,0'")
+            self.connection.execute(
+                "ovs-vswitchd unix:{0}/"
+                "var/run/openvswitch/db.sock --pidfile --detach "
+                "--log-file=/var/log/openvswitch/"
+                "ovs-vswitchd.log".format(
+                    self.ovs[0]["vpath"]))
+            self.connection.execute(
+                "ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=2C")
+
+    def setup_ovs_bridge(self):
+        self.connection.execute("ovs-vsctl del-br br0")
+        self.connection.execute(
+            "rm -rf /usr/local/var/run/openvswitch/dpdkvhostuser*")
+        self.connection.execute(
+            "ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev")
+        self.connection.execute(
+            "ovs-vsctl add-port br0 dpdk0 -- set Interface dpdk0 type=dpdk")
+        self.connection.execute(
+            "ovs-vsctl add-port br0 dpdk1 -- set Interface dpdk1 type=dpdk")
+        self.connection.execute(
+            "ovs-vsctl add-port br0 dpdkvhostuser0 -- set Interface "
+            "dpdkvhostuser0 type=dpdkvhostuser")
+        self.connection.execute("ovs-vsctl add-port br0 dpdkvhostuser1 "
+                                "-- set Interface dpdkvhostuser1 "
+                                "type=dpdkvhostuser")
+        self.connection.execute(
+            "chmod 0777 {0}/var/run/"
+            "openvswitch/dpdkvhostuser*".format(self.ovs[0]["vpath"]))
+
+    def add_oflows(self):
+        self.connection.execute("ovs-ofctl del-flows br0")
+        for flow in self.ovs[0]["flow"]:
+            self.connection.execute(flow)
+            self.connection.execute("ovs-ofctl dump-flows br0")
+            self.connection.execute(
+                "ovs-vsctl set Interface dpdk0 options:n_rxq=4")
+            self.connection.execute(
+                "ovs-vsctl set Interface dpdk1 options:n_rxq=4")
+
+    def setup_ovs_context(self, pcis, nic_details, host_driver):
+
+        ''' 1: Setup vm_ovs.xml to launch VM.'''
+        cfg_ovs = '/tmp/vm_ovs.xml'
+        vm_ovs_xml = VM_TEMPLATE.format(vm_image=self.ovs[0]["images"])
+        with open(cfg_ovs, 'w') as f:
+            f.write(vm_ovs_xml)
+
+        ''' 2: Create and start the VM'''
+        self.connection.put(cfg_ovs, cfg_ovs)
+        time.sleep(10)
+        err, out = self.check_output("virsh list --name | grep -i vm1")
+        if out == "vm1":
+            print("VM is already present")
+        else:
+            ''' FIXME: launch through libvirt'''
+            print("virsh create ...")
+            err, out, _ = self.connection.execute(
+                "virsh create /tmp/vm_ovs.xml")
+            time.sleep(10)
+            print("err : {0}".format(err))
+            print("{0}".format(_))
+            print("out : {0}".format(out))
+
+        ''' 3: Tuning for better performace.'''
+        self.pin_vcpu(pcis)
+        self.connection.execute(
+            "echo 1 > /sys/module/kvm/parameters/"
+            "allow_unsafe_assigned_interrupts")
+        self.connection.execute(
+            "echo never > /sys/kernel/mm/transparent_hugepage/enabled")
+        print("After tuning performance ...")
+
+    ''' This is roughly compatible with check_output function in subprocess
+     module which is only available in python 2.7.'''
+    def check_output(self, cmd, stderr=None):
+        '''Run a command and capture its output'''
+        err, out, _ = self.connection.execute(cmd)
+        return err, out
+
+    def read_from_file(self, filename):
+        data = ""
+        with open(filename, 'r') as the_file:
+            data = the_file.read()
+        return data
+
+    def write_to_file(self, filename, content):
+        with open(filename, 'w') as the_file:
+            the_file.write(content)
+
+    def pin_vcpu(self, pcis):
+        nodes = self.get_numa_nodes()
+        print("{0}".format(nodes))
+        num_nodes = len(nodes)
+        for i in range(0, 10):
+            self.connection.execute(
+                "virsh vcpupin vm1 {0} {1}".format(
+                    i, nodes[str(num_nodes - 1)][i]))
+
+    def get_numa_nodes(self):
+        nodes_sysfs = glob.iglob("/sys/devices/system/node/node*")
+        nodes = {}
+        for node_sysfs in nodes_sysfs:
+            num = os.path.basename(node_sysfs).replace("node", "")
+            with open(os.path.join(node_sysfs, "cpulist")) as cpulist_file:
+                cpulist = cpulist_file.read().strip()
+                print("cpulist: {0}".format(cpulist))
+            nodes[num] = self.split_cpu_list(cpulist)
+        print("nodes: {0}".format(nodes))
+        return nodes
+
+    def split_cpu_list(self, cpu_list):
+        if cpu_list:
+            ranges = cpu_list.split(',')
+            bounds = ([int(b) for b in r.split('-')] for r in ranges)
+            range_objects =\
+                (range(bound[0], bound[1] + 1 if len(bound) == 2
+                 else bound[0] + 1) for bound in bounds)
+
+            return sorted(itertools.chain.from_iterable(range_objects))
+        else:
+            return []
+
+    def destroy_vm(self):
+        host_driver = self.ovs[0]['phy_driver']
+        err, out = self.check_output("virsh list --name | grep -i vm1")
+        print("{0}".format(out))
+        if err == 0:
+            self.connection.execute("virsh shutdown vm1")
+            self.connection.execute("virsh destroy vm1")
+            self.check_output("rmmod {0}".format(host_driver))[1].splitlines()
+            self.check_output("modprobe {0}".format(host_driver))[
+                1].splitlines()
+        else:
+            print("error : ", err)
diff --git a/yardstick/benchmark/contexts/sriov.py b/yardstick/benchmark/contexts/sriov.py
index 5dc27bfae..da143cc4b 100644
--- a/yardstick/benchmark/contexts/sriov.py
+++ b/yardstick/benchmark/contexts/sriov.py
@@ -134,6 +134,7 @@ class Sriov(StandaloneContext):
         self.auth_type = ""
 
     def init(self):
+        log.debug("In init")
         self.parse_pod_and_get_data(self.file_path)
 
     def parse_pod_and_get_data(self, file_path):
@@ -146,8 +147,6 @@ class Sriov(StandaloneContext):
             log.error("File {0} does not exist".format(self.file_path))
             raise
 
-        self.nodes.extend([node for node in cfg["nodes"]
-                           if node["role"] != "Sriov"])
         self.sriov.extend([node for node in cfg["nodes"]
                            if node["role"] == "Sriov"])
         self.user = self.sriov[0]['user']
diff --git a/yardstick/benchmark/contexts/standalone.py b/yardstick/benchmark/contexts/standalone.py
index e921437d6..2bc1f3755 100644
--- a/yardstick/benchmark/contexts/standalone.py
+++ b/yardstick/benchmark/contexts/standalone.py
@@ -18,6 +18,7 @@ import logging
 import errno
 import collections
 import yaml
+import time
 
 from yardstick.benchmark.contexts.base import Context
 from yardstick.common.constants import YARDSTICK_ROOT_PATH
@@ -74,20 +75,24 @@ class StandaloneContext(Context):
             else:
                 raise
 
-        self.nodes.extend(cfg["nodes"])
-        self.nfvi_node.extend([node for node in cfg["nodes"]
-                               if node["role"] == "nfvi_node"])
-        # add optional static network definition
-        self.networks.update(cfg.get("networks", {}))
+        self.vm_deploy = attrs.get("vm_deploy", True)
+        self.nodes.extend([node for node in cfg["nodes"]
+                           if str(node["role"]) != "Sriov" and
+                           str(node["role"]) != "Ovsdpdk"])
         for node in cfg["nodes"]:
             if str(node["role"]) == "Sriov":
                 self.nfvi_node.extend([node for node in cfg["nodes"]
                                        if str(node["role"]) == "Sriov"])
-            if str(node["role"]) == "ovs-dpdk":
+            if str(node["role"]) == "Ovsdpdk":
+                self.nfvi_node.extend([node for node in cfg["nodes"]
+                                       if str(node["role"]) == "Ovsdpdk"])
                 LOG.info("{0}".format(node["role"]))
             else:
                 LOG.debug("Node role is other than SRIOV and OVS")
         self.nfvi_obj = self.get_nfvi_obj()
+        # add optional static network definition
+        self.networks.update(cfg.get("networks", {}))
+        self.nfvi_obj = self.get_nfvi_obj()
         LOG.debug("Nodes: %r", self.nodes)
         LOG.debug("NFVi Node: %r", self.nfvi_node)
         LOG.debug("Networks: %r", self.networks)
@@ -96,21 +101,39 @@ class StandaloneContext(Context):
         """don't need to deploy"""
 
         # Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
+        if not self.vm_deploy:
+            return
+
+        #    Todo: NFVi deploy (sriov, vswitch, ovs etc) based on the config.
         self.nfvi_obj.ssh_remote_machine()
         if self.nfvi_obj.first_run is True:
             self.nfvi_obj.install_req_libs()
 
         nic_details = self.nfvi_obj.get_nic_details()
         print("{0}".format(nic_details))
-        self.nfvi_obj.setup_sriov_context(
-            self.nfvi_obj.sriov[0]['phy_ports'],
-            nic_details,
-            self.nfvi_obj.sriov[0]['phy_driver'])
-        pass
+
+        if self.nfvi_node[0]["role"] == "Sriov":
+            self.nfvi_obj.setup_sriov_context(
+                self.nfvi_obj.sriov[0]['phy_ports'],
+                nic_details,
+                self.nfvi_obj.sriov[0]['phy_driver'])
+        if self.nfvi_node[0]["role"] == "Ovsdpdk":
+            self.nfvi_obj.setup_ovs(self.nfvi_obj.ovs[0]["phy_ports"])
+            self.nfvi_obj.start_ovs_serverswitch()
+            time.sleep(5)
+            self.nfvi_obj.setup_ovs_bridge()
+            self.nfvi_obj.add_oflows()
+            self.nfvi_obj.setup_ovs_context(
+                self.nfvi_obj.ovs[0]['phy_ports'],
+                nic_details,
+                self.nfvi_obj.ovs[0]['phy_driver'])
+            pass
 
     def undeploy(self):
         """don't need to undeploy"""
 
+        if not self.vm_deploy:
+            return
         # Todo: NFVi undeploy (sriov, vswitch, ovs etc) based on the config.
         # self.nfvi_obj = self.get_nfvi_obj()
         self.nfvi_obj.ssh_remote_machine()
@@ -123,16 +146,12 @@ class StandaloneContext(Context):
         Keyword arguments:
         attr_name -- A name for a server listed in nodes config file
         """
-
         if isinstance(attr_name, collections.Mapping):
             return None
-
         if self.name != attr_name.split(".")[1]:
             return None
-
         node_name = attr_name.split(".")[0]
         matching_nodes = (n for n in self.nodes if n["name"] == node_name)
-
         try:
             # A clone is created in order to avoid affecting the
             # original one.
@@ -147,7 +166,6 @@ class StandaloneContext(Context):
         else:
             raise ValueError("Duplicate nodes!!! Nodes: %s %s",
                              (matching_nodes, duplicate))
-
         node["name"] = attr_name
         return node