Tools: Deployment and TestControl Containers

This patch add containerization of VSPERF support.
The patch facilitates creation of 4 containers:
1. Interactive Deployment
2. Auto Deployment
3. Interactive TestControl
4. Auto TestControl.
The patch also includes a minimal client to work with interactive
containers.
The docs folder provides detailed documentation.
Fixed pylint errors in libs folder.
Removed proto built python files, and added the build process in
prepare.sh.
Stability improvements for Auto versions of deployment and testcontrol.
Enhance client with 'mode' feature, where client can run either to do
only deploy/only test or both.
Add sample configuration file for client
Fixed few typos - as suggested by AL.

JIRA: VSPERF-594
Signed-off-by: Sridhar K. N. Rao <sridhar.rao@spirent.com>
Change-Id: Id40b02960f71a7f9183d9a53955e2483117fb9e2

4 files changed, 462 insertions, 0 deletions

diff --git a/tools/docker/deployment/auto/controller/vsperf/__init__.py b/tools/docker/deployment/auto/controller/vsperf/__init__.py
new file mode 100644
index 00000000..ad0ebec3
--- /dev/null
+++ b/tools/docker/deployment/auto/controller/vsperf/__init__.py
@@ -0,0 +1 @@
+#### Empty
diff --git a/tools/docker/deployment/auto/controller/vsperf/collectd.conf b/tools/docker/deployment/auto/controller/vsperf/collectd.conf
new file mode 100644
index 00000000..9cefc8c5
--- /dev/null
+++ b/tools/docker/deployment/auto/controller/vsperf/collectd.conf
@@ -0,0 +1,49 @@
+Hostname "pod12-node4"
+Interval     1
+LoadPlugin intel_rdt
+LoadPlugin processes
+LoadPlugin interface
+LoadPlugin network
+LoadPlugin ovs_stats
+LoadPlugin cpu
+LoadPlugin memory
+LoadPlugin csv
+#LoadPlugin dpdkstat
+##############################################################################
+# Plugin configuration                                                       #
+##############################################################################
+<Plugin processes>
+        ProcessMatch "ovs-vswitchd" "ovs-vswitchd"
+        ProcessMatch "ovsdb-server" "ovsdb-server"
+        ProcessMatch "collectd" "collectd"
+</Plugin>
+<Plugin network>
+        Server "10.10.120.22" "25826"
+</Plugin>
+
+<Plugin ovs_stats>
+  Port "6640"
+  Address "127.0.0.1"
+  Socket "/usr/local/var/run/openvswitch/db.sock"
+  Bridges "vsperf-br0"
+</Plugin>
+
+<Plugin "intel_rdt">
+  Cores "2" "4-5" "6-7" "8" "9" "22" "23" "24" "25" "26" "27"
+</Plugin>
+
+<Plugin csv>
+       DataDir "/tmp/csv"
+       StoreRates false
+</Plugin>
+
+#<Plugin dpdkstat>
+#  <EAL>
+#    Coremask "0x1"
+#    MemoryChannels "4"
+#    FilePrefix "rte"
+#  </EAL>
+#  SharedMemObj "dpdk_collectd_stats_0"
+#  EnabledPortMask 0xffff
+#</Plugin>
+
diff --git a/tools/docker/deployment/auto/controller/vsperf/trex_cfg.yaml b/tools/docker/deployment/auto/controller/vsperf/trex_cfg.yaml
new file mode 100644
index 00000000..8bb8e341
--- /dev/null
+++ b/tools/docker/deployment/auto/controller/vsperf/trex_cfg.yaml
@@ -0,0 +1,20 @@
+- port_limit    : 2         
+  version       : 2
+  interfaces    : ["81:00.0", "81:00.1"]   # list of the interfaces to bind # node 4
+  port_bandwidth_gb : 10 #10G nics
+  port_info       :  # set eth mac addr
+          - dest_mac        :   "3c:fd:fe:b4:41:09"  # port 0
+            src_mac         :   "3c:fd:fe:b4:41:08"
+          - dest_mac        :   "3c:fd:fe:b4:41:08"  # port 1
+            src_mac         :   "3c:fd:fe:b4:41:09"
+  platform :
+        master_thread_id  : 17
+        latency_thread_id : 16
+        dual_if   :
+             - socket   : 1
+               threads  : [22,23,24,25,26,27]
+             - socket   : 0
+               threads  : [10,11,12,13,14,15]
+
+
+
diff --git a/tools/docker/deployment/auto/controller/vsperf/vsperf_controller.py b/tools/docker/deployment/auto/controller/vsperf/vsperf_controller.py
new file mode 100644
index 00000000..b6865272
--- /dev/null
+++ b/tools/docker/deployment/auto/controller/vsperf/vsperf_controller.py
@@ -0,0 +1,392 @@
+# Copyright 2018-19 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+VSPERF_deploy_auto
+"""
+# pylint: disable=W0603
+
+import os
+import sys
+from utils import ssh
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+DUT_IP = os.getenv('DUT_IP_ADDRESS')
+DUT_USER = os.getenv('DUT_USERNAME')
+DUT_PWD = os.getenv('DUT_PASSWORD')
+
+TGEN_IP = os.getenv('TGEN_IP_ADDRESS')
+TGEN_USER = os.getenv('TGEN_USERNAME')
+TGEN_PWD = os.getenv('TGEN_PASSWORD')
+TGEN_PARAM = os.getenv('TGEN_PARAMS')
+
+HPMAX = int(os.getenv('HUGEPAGE_MAX'))
+HPREQUESTED = int(os.getenv('HUGEPAGE_REQUESTED'))
+
+SANITY = str(os.getenv('SANITY_CHECK'))#
+
+DUT_CLIENT = None
+TGEN_CLIENT = None
+
+
+def host_connect():
+    """
+    Handle host connectivity to DUT
+    """
+    global DUT_CLIENT
+    DUT_CLIENT = ssh.SSH(host=DUT_IP, user=DUT_USER, password=DUT_PWD)
+    print("DUT-Host Successfully Connected .........................................[OK] \n ")
+
+def tgen_connect():
+    """
+    Handle Tgen Connection to Trex
+    """
+    global TGEN_CLIENT
+    TGEN_CLIENT = ssh.SSH(host=TGEN_IP, user=TGEN_USER, password=TGEN_PWD)
+    print("Traffic Generator Successfully Connected ...............................[OK] \n ")
+
+
+def vsperf_install():
+    """
+    Perform actual installation
+    """
+    vsperf_check_command = "source ~/vsperfenv/bin/activate ; "
+    vsperf_check_command += "cd vswitchperf && ./vsperf --help"
+    vsperf_check_cmd_result = str(DUT_CLIENT.execute(vsperf_check_command)[1])
+    vsperf_verify_list = [
+        'usage',
+        'positional arguments',
+        'optional arguments',
+        'test selection options',
+        'test behavior options']
+    for idx, i in enumerate(vsperf_verify_list, start=1):
+        if str(i) in vsperf_check_cmd_result:
+            if idx < 5:
+                continue
+            elif idx == 5:
+                print(
+                    "VSPERF is Already Installed on DUT-Host..........................."\
+                    ".......[OK]\n")
+            else:
+                download_cmd = "git clone https://gerrit.opnfv.org/gerrit/vswitchperf"
+                DUT_CLIENT.run(download_cmd)
+                install_cmd = "cd vswitchperf/systems ; "
+                install_cmd += "echo '{}' | sudo -S ./build_base_machine.sh ".\
+                                format(DUT_PWD)
+                DUT_CLIENT.run(install_cmd)
+                print(
+                    "Vsperf Installed on DUT-Host ....................................[OK]\n")
+
+
+def tgen_install():
+    """
+    Install T-rex traffic gen on TGen
+    """
+    kill_cmd = "pkill -f ./t-rex"
+    TGEN_CLIENT.send_command(kill_cmd)
+    tgen_start_check = "cd trex/scripts && ./t-rex-64 -f cap2/dns.yaml -d 100 -m 1 --nc"
+    tgen_start_cmd_result = int(TGEN_CLIENT.execute(tgen_start_check)[0])
+    if tgen_start_cmd_result == 0:
+        print(
+            "The Host has T-rex Installed....................................[OK]\n")
+    else:
+        download_cmd = "git clone https://github.com/cisco-system-traffic-generator/trex-core trex"
+        TGEN_CLIENT.run(download_cmd)
+        install_cmd = "cd trex-core/linux_dpdk ; ./b configure ; ./b build"
+        TGEN_CLIENT.run(install_cmd)
+        print(
+            "The Host has now T-rex Installed...........................[OK]\n")
+
+def upload_tgen_config_file():
+    """
+    Upload Tgen Config File on T-rex
+    """
+    localpath = '/usr/src/app/vsperf/trex_cfg.yaml'
+    if not os.path.exists(localpath):
+        print("TGEN config File does not exist................[Failed]")
+        return
+    remotepath = '~/trex_cfg.yaml'
+    check_trex_config_cmd = "echo {} | sudo -S find /etc -maxdepth 1 -name '{}'".format(
+        TGEN_PWD, remotepath[2:])
+    check_test_result = str(TGEN_CLIENT.execute(check_trex_config_cmd)[1])
+    if remotepath[2:] in check_test_result:
+        DUT_CLIENT.run("rm -f /etc/{}".format(remotepath[2:]))
+    TGEN_CLIENT.put_file(localpath, remotepath)
+    TGEN_CLIENT.run(
+        "echo {} | sudo -S mv ~/{} /etc/".format(TGEN_PWD, remotepath[2:]), pty=True)
+    print(
+        "T-rex Configuration File Uploaded on TGen-Host...........................[OK]\n")
+
+
+def install_collectd():
+    """
+    installation of the collectd
+    """
+    check_collectd_config_cmd = "find /opt -maxdepth 1 -name 'collectd'"
+    check_test_result = str(DUT_CLIENT.execute(check_collectd_config_cmd)[1])
+    if "collectd" in check_test_result:
+        print(
+            'Collectd Installed Successfully on DUT-Host..............................[OK]\n')
+    else:
+        download_cmd = "git clone https://github.com/collectd/collectd.git"
+        DUT_CLIENT.run(download_cmd)
+        build_cmd = "cd collectd ; "
+        build_cmd += "./build.sh"
+        DUT_CLIENT.run(build_cmd)
+        config_cmd = "cd collectd ; ./configure --enable-syslog --enable-logfile "
+        config_cmd += "--enable-hugepages --enable-debug ; "
+        DUT_CLIENT.run(config_cmd)
+        install_cmd = "cd collectd ; make ; "
+        install_cmd += "echo '{}' | sudo -S make install".format(DUT_PWD)
+        DUT_CLIENT.run(install_cmd, pty=True)
+        print(
+            'Collectd Installed Successfully on DUT-Host.............................[OK]\n ')
+
+
+def collectd_upload_config():
+    """
+    Upload Configuration file of Collectd on DUT
+    """
+    localpath = '/usr/src/app/vsperf/collectd.conf'
+    if not os.path.exists(localpath):
+        print("Collectd config File does not exist.......................[Failed]")
+        return
+    remotepath = '~/collectd.conf'
+    collectd_config_cmd = "echo {} | sudo -S find /opt/collectd/etc -maxdepth 1 -name '{}'".\
+                           format(DUT_PWD, remotepath[2:])
+    check_test_result = str(DUT_CLIENT.execute(collectd_config_cmd)[1])
+    if remotepath[2:] in check_test_result:
+        DUT_CLIENT.run(
+            "echo {} | sudo -S rm -f /opt/collectd/etc/{}".format(DUT_PWD, remotepath[2:]))
+    DUT_CLIENT.put_file(localpath, remotepath)
+    DUT_CLIENT.run("echo {} | sudo -S mv ~/{} /opt/collectd/etc/".\
+                  format(DUT_PWD, remotepath[2:]), pty=True)
+    print(
+        "Collectd Configuration File Uploaded on DUT-Host.........................[OK]\n ")
+
+def start_tgen():
+    """
+    It will start the Traffic generetor
+    """
+    kill_cmd = "pkill -f ./t-rex"
+    TGEN_CLIENT.send_command(kill_cmd)
+    run_cmd = "cd trex_2.37/scripts && "
+    run_cmd += "screen ./t-rex-64 "
+    run_cmd += TGEN_PARAM
+    TGEN_CLIENT.send_command(run_cmd)
+    print(
+        "T-Rex Successfully running...............................................[OK]\n")
+
+
+def dut_hugepage_config():
+    """
+    Configure the DUT system hugepage parameter from client
+    """
+    if not HPMAX or not HPREQUESTED:
+        print("HPMAX and HPREQUESTED not defined ...................[Failed]")
+        return
+    hugepage_cmd = "echo '{}' | sudo -S mkdir -p /mnt/huge ; ".format(
+        DUT_PWD)
+    hugepage_cmd += "echo '{}' | sudo -S mount -t hugetlbfs nodev /mnt/huge".format(
+        DUT_PWD)
+    DUT_CLIENT.run(hugepage_cmd, pty=True)
+    hp_nr_cmd = "cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages"
+    hp_free_cmd = "cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/free_hugepages"
+    hp_nr = int(DUT_CLIENT.execute(hp_nr_cmd)[1])
+    hp_free = int(DUT_CLIENT.execute(hp_free_cmd)[1])
+    if hp_free <= HPREQUESTED:
+        hp_nr_new = hp_nr + (HPREQUESTED - hp_free)
+        if hp_nr_new > HPMAX:
+            hp_nr_new = HPMAX
+
+    nr_hugepage_cmd = "echo '{}' | sudo -S bash -c \"echo 'vm.nr_hugepages={}' >> ".\
+                       format(DUT_PWD, hp_nr_new)
+    nr_hugepage_cmd += "/etc/sysctl.conf\""
+    DUT_CLIENT.run(nr_hugepage_cmd, pty=True)
+
+    dict_cmd = "cat /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages"
+    dict_check = int(DUT_CLIENT.execute(dict_cmd)[0])
+    if dict_check == 0:
+        node1_hugepage_cmd = "echo '{}' | sudo -s bash -c \"echo 0 > ".format(DUT_PWD)
+        node1_hugepage_cmd += "/sys/devices/system/node/node1/hugepages"
+        node1_hugepage_cmd += "/hugepages-2048kB/nr_hugepages\""
+        DUT_CLIENT.run(node1_hugepage_cmd, pty=True)
+    print("DUT-Host system configured with {} No of Hugepages.....................[OK] \n ".\
+          format(hp_nr_new))
+
+
+def sanity_nic_check():
+    """
+    Check either NIC PCI ids are Correctly placed or not
+    """
+    trex_conf_path = "cat /etc/trex_cfg.yaml | grep interfaces"
+    trex_conf_read = TGEN_CLIENT.execute(trex_conf_path)[1]
+    nic_pid_ids_list = [trex_conf_read.split("\"")[1], trex_conf_read.split("\"")[3]]
+    trex_nic_pic_id_cmd = "lspci | egrep -i --color 'network|ethernet'"
+    trex_nic_pic_id = str(TGEN_CLIENT.execute(trex_nic_pic_id_cmd)[1]).split('\n')
+    acheck = 0
+    for k in trex_nic_pic_id:
+        for j in nic_pid_ids_list:
+            if j in k:
+                acheck += 1
+            else:
+                pass
+    if acheck == 2:
+        print("Both the NIC PCI Ids are Correctly"\
+            " configured on TGen-Host...............[OK]\n")
+    else:
+        print("You configured NIC PCI Ids Wrong in "\
+            "TGen-Host............................[OK]\n")
+
+
+def sanity_collectd_check():
+    """
+    Check and verify collectd is able to run and start properly
+    """
+    check_collectd_cmd = "find /opt -maxdepth 1 -name 'collectd'"
+    check_test_result = str(DUT_CLIENT.execute(check_collectd_cmd)[1])
+    if "collectd" in check_test_result:
+        check_collectd_run_cmd = "echo {} | sudo -S service collectd start".format(
+            DUT_PWD)
+        DUT_CLIENT.run(check_collectd_run_cmd, pty=True)
+        check_collectd_status_cmd = "ps aux | grep collectd"
+        check_collectd_status = str(
+            DUT_CLIENT.execute(check_collectd_status_cmd)[1])
+        if "/sbin/collectd" in check_collectd_status:
+            print(
+                "Collectd is working Fine ................................................[OK] \n ")
+        else:
+            print(
+                "Collectd Fail to Start, Install correctly before running Test....[Failed]\n ")
+    else:
+        print(
+            "Collectd is not installed yet........................................[Failed]\n")
+
+def sanity_vsperf_check():
+    """
+    We have to make sure that VSPERF install correctly
+    """
+    if not DUT_CLIENT:
+        print("The Client is disconnected................................[Failed]")
+        return
+    vsperf_check_cmd = "source ~/vsperfenv/bin/activate ; cd vswitchperf && ./vsperf --help"
+    vsperf_check_cmd_result = str(DUT_CLIENT.execute(vsperf_check_cmd)[1])
+    vsperf_verify_list = [
+        'usage',
+        'positional arguments',
+        'optional arguments',
+        'test selection options',
+        'test behavior options']
+    for idx, i in enumerate(vsperf_verify_list, start=1):
+        if str(i) in vsperf_check_cmd_result:
+            if idx < 5:
+                continue
+            elif idx == 5:
+                print(
+                    "VSPERF Installed Correctly and Working fine.........................."\
+                    "....[OK]\n")
+            else:
+                print(
+                    "VSPERF Does Not Installed Correctly , INSTALL IT AGAIN........[Critical]\n")
+        else:
+            print(
+                "VSPERF Does Not Installed Correctly , INSTALL IT AGAIN............[Critical]\n")
+            break
+
+
+def sanity_tgen_conn_dut_check():
+    """
+    We should confirm the DUT connectivity with the Tgen and Traffic Generator is working or not
+    """
+    if not DUT_CLIENT or not TGEN_CLIENT:
+        print("The Client is disconnected................................[Failed]")
+        return
+    tgen_connectivity_check_cmd = "ping {} -c 1".format(TGEN_IP)
+    tgen_connectivity_check_result = int(
+        DUT_CLIENT.execute(tgen_connectivity_check_cmd)[0])
+    if tgen_connectivity_check_result == 0:
+        print(
+            "DUT-Host is successfully reachable to Traffic Generator Host.............[OK]\n")
+    else:
+        print(
+            "DUT-host is unsuccessful to reach the Traffic Generator Host..............[Failed]")
+        print(
+            "Make sure to establish connection before running Test...............[Critical]\n")
+
+
+def sanity_tgen_check():
+    """
+    It will check Trex properly running or not
+    """
+    if not TGEN_CLIENT:
+        print("The Client is disconnected................................[Failed]")
+        return
+    tgen_start_cmd_check = "cd trex/scripts &&"
+    tgen_start_cmd_check += " ./t-rex-64 -f cap2/dns.yaml -d 100 -m 1 --nc"
+    tgen_start_cmd_result = int(TGEN_CLIENT.execute(tgen_start_cmd_check)[0])
+    if tgen_start_cmd_result == 0:
+        print(
+            "TGen-Host successfully running........................................[OK]\n")
+    else:
+        print("TGen-Host is unable to start t-rex ..................[Failed]")
+        print("Make sure you install t-rex correctly ...............[Critical]\n")
+
+
+def dut_vsperf_test_availability():
+    """
+    Before running test we have to make sure there is no other test running
+    """
+    vsperf_ava_cmd = "ps -ef | grep -v grep | grep ./vsperf | awk '{print $2}'"
+    vsperf_ava_result = len(
+        (DUT_CLIENT.execute(vsperf_ava_cmd)[1]).split("\n"))
+    if vsperf_ava_result == 1:
+        print("DUT-Host is available for performing VSPERF Test\n\
+            You can perform Test!")
+    else:
+        print("DUT-Host is busy right now, Wait for some time\n\
+            Always Check availability before Running Test!\n")
+
+if DUT_IP:
+    host_connect()
+if not DUT_CLIENT:
+    print('Failed to connect to DUT ...............[Critical]')
+    sys.exit()
+else:
+    vsperf_install()
+    install_collectd()
+    collectd_upload_config()
+    dut_hugepage_config()
+    dut_vsperf_test_availability()
+if TGEN_IP:
+    tgen_connect()
+if not TGEN_CLIENT:
+    print('Failed to connect to TGEN_HOST.............[Critical]')
+    sys.exit()
+else:
+    tgen_install()
+    upload_tgen_config_file()
+    sanity_nic_check()
+    start_tgen()
+
+print("\n\nIF you are getting any Failed or Critical message!!!\n" \
+      "Please follow this steps:\n"
+      "1. Make necessory changes before running VSPERF TEST\n"\
+      "2. Re-Run the auto deployment container")
+
+if SANITY and 'yes' in SANITY.lower():
+    sanity_collectd_check()
+    sanity_vsperf_check()
+    sanity_tgen_check()
+    sanity_tgen_conn_dut_check()