Merge "Tools: Deployment and TestControl Containers"opnfv-9.0.0stable/iruya

6 files changed, 499 insertions, 0 deletions

diff --git a/tools/docker/deployment/auto/controller/Dockerfile b/tools/docker/deployment/auto/controller/Dockerfile
new file mode 100644
index 00000000..e849d8f2
--- /dev/null
+++ b/tools/docker/deployment/auto/controller/Dockerfile
@@ -0,0 +1,23 @@
+FROM python:3.6
+LABEL maintainer="sridhar.rao@spirent.com"
+
+ENV GRPC_PYTHON_VERSION 1.4.0
+RUN apt-get update && apt-get -y install python3-pip && apt-get -y install openssh-server
+RUN pip3 install grpcio==${GRPC_PYTHON_VERSION} grpcio-tools==${GRPC_PYTHON_VERSION}
+RUN pip3 install paramiko
+RUN pip3 install chainmap
+RUN pip3 install oslo.utils
+RUN pip3 install scp
+
+WORKDIR /usr/src/app
+
+COPY ./vsperf ./vsperf
+
+VOLUME ["/usr/src/app/vsperf"]
+
+EXPOSE 50051
+
+CMD ["python3", "./vsperf/vsperf_controller.py"]
+
+#CMD tail -f /dev/null
+
diff --git a/tools/docker/deployment/auto/controller/list.env b/tools/docker/deployment/auto/controller/list.env
new file mode 100644
index 00000000..ab4404b7
--- /dev/null
+++ b/tools/docker/deployment/auto/controller/list.env
@@ -0,0 +1,14 @@
+DUT_IP_ADDRESS=10.10.120.24
+DUT_USERNAME=opnfv
+DUT_PASSWORD=opnfv
+
+TGEN_IP_ADDRESS=10.10.120.25
+TGEN_USERNAME=root
+TGEN_PASSWORD=P@ssw0rd
+TGEN_PARAMS= -i --no-scapy-server --nc --no-watchdog
+
+HUGEPAGE_MAX=8192
+HUGEPAGE_REQUESTED=1024
+
+SANITY_CHECK=NO
+
diff --git a/tools/docker/deployment/auto/controller/vsperf/__init__.py b/tools/docker/deployment/auto/controller/vsperf/__init__.py
new file mode 100644
index 00000000..ad0ebec3
--- /dev/null
+++ b/tools/docker/deployment/auto/controller/vsperf/__init__.py
@@ -0,0 +1 @@
+#### Empty
diff --git a/tools/docker/deployment/auto/controller/vsperf/collectd.conf b/tools/docker/deployment/auto/controller/vsperf/collectd.conf
new file mode 100644
index 00000000..9cefc8c5
--- /dev/null
+++ b/tools/docker/deployment/auto/controller/vsperf/collectd.conf
@@ -0,0 +1,49 @@
+Hostname "pod12-node4"
+Interval     1
+LoadPlugin intel_rdt
+LoadPlugin processes
+LoadPlugin interface
+LoadPlugin network
+LoadPlugin ovs_stats
+LoadPlugin cpu
+LoadPlugin memory
+LoadPlugin csv
+#LoadPlugin dpdkstat
+##############################################################################
+# Plugin configuration                                                       #
+##############################################################################
+<Plugin processes>
+        ProcessMatch "ovs-vswitchd" "ovs-vswitchd"
+        ProcessMatch "ovsdb-server" "ovsdb-server"
+        ProcessMatch "collectd" "collectd"
+</Plugin>
+<Plugin network>
+        Server "10.10.120.22" "25826"
+</Plugin>
+
+<Plugin ovs_stats>
+  Port "6640"
+  Address "127.0.0.1"
+  Socket "/usr/local/var/run/openvswitch/db.sock"
+  Bridges "vsperf-br0"
+</Plugin>
+
+<Plugin "intel_rdt">
+  Cores "2" "4-5" "6-7" "8" "9" "22" "23" "24" "25" "26" "27"
+</Plugin>
+
+<Plugin csv>
+       DataDir "/tmp/csv"
+       StoreRates false
+</Plugin>
+
+#<Plugin dpdkstat>
+#  <EAL>
+#    Coremask "0x1"
+#    MemoryChannels "4"
+#    FilePrefix "rte"
+#  </EAL>
+#  SharedMemObj "dpdk_collectd_stats_0"
+#  EnabledPortMask 0xffff
+#</Plugin>
+
diff --git a/tools/docker/deployment/auto/controller/vsperf/trex_cfg.yaml b/tools/docker/deployment/auto/controller/vsperf/trex_cfg.yaml
new file mode 100644
index 00000000..8bb8e341
--- /dev/null
+++ b/tools/docker/deployment/auto/controller/vsperf/trex_cfg.yaml
@@ -0,0 +1,20 @@
+- port_limit    : 2         
+  version       : 2
+  interfaces    : ["81:00.0", "81:00.1"]   # list of the interfaces to bind # node 4
+  port_bandwidth_gb : 10 #10G nics
+  port_info       :  # set eth mac addr
+          - dest_mac        :   "3c:fd:fe:b4:41:09"  # port 0
+            src_mac         :   "3c:fd:fe:b4:41:08"
+          - dest_mac        :   "3c:fd:fe:b4:41:08"  # port 1
+            src_mac         :   "3c:fd:fe:b4:41:09"
+  platform :
+        master_thread_id  : 17
+        latency_thread_id : 16
+        dual_if   :
+             - socket   : 1
+               threads  : [22,23,24,25,26,27]
+             - socket   : 0
+               threads  : [10,11,12,13,14,15]
+
+
+
diff --git a/tools/docker/deployment/auto/controller/vsperf/vsperf_controller.py b/tools/docker/deployment/auto/controller/vsperf/vsperf_controller.py
new file mode 100644
index 00000000..b6865272
--- /dev/null
+++ b/tools/docker/deployment/auto/controller/vsperf/vsperf_controller.py
@@ -0,0 +1,392 @@
+# Copyright 2018-19 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+VSPERF_deploy_auto
+"""
+# pylint: disable=W0603
+
+import os
+import sys
+from utils import ssh
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+DUT_IP = os.getenv('DUT_IP_ADDRESS')
+DUT_USER = os.getenv('DUT_USERNAME')
+DUT_PWD = os.getenv('DUT_PASSWORD')
+
+TGEN_IP = os.getenv('TGEN_IP_ADDRESS')
+TGEN_USER = os.getenv('TGEN_USERNAME')
+TGEN_PWD = os.getenv('TGEN_PASSWORD')
+TGEN_PARAM = os.getenv('TGEN_PARAMS')
+
+HPMAX = int(os.getenv('HUGEPAGE_MAX'))
+HPREQUESTED = int(os.getenv('HUGEPAGE_REQUESTED'))
+
+SANITY = str(os.getenv('SANITY_CHECK'))#
+
+DUT_CLIENT = None
+TGEN_CLIENT = None
+
+
+def host_connect():
+    """
+    Handle host connectivity to DUT
+    """
+    global DUT_CLIENT
+    DUT_CLIENT = ssh.SSH(host=DUT_IP, user=DUT_USER, password=DUT_PWD)
+    print("DUT-Host Successfully Connected .........................................[OK] \n ")
+
+def tgen_connect():
+    """
+    Handle Tgen Connection to Trex
+    """
+    global TGEN_CLIENT
+    TGEN_CLIENT = ssh.SSH(host=TGEN_IP, user=TGEN_USER, password=TGEN_PWD)
+    print("Traffic Generator Successfully Connected ...............................[OK] \n ")
+
+
+def vsperf_install():
+    """
+    Perform actual installation
+    """
+    vsperf_check_command = "source ~/vsperfenv/bin/activate ; "
+    vsperf_check_command += "cd vswitchperf && ./vsperf --help"
+    vsperf_check_cmd_result = str(DUT_CLIENT.execute(vsperf_check_command)[1])
+    vsperf_verify_list = [
+        'usage',
+        'positional arguments',
+        'optional arguments',
+        'test selection options',
+        'test behavior options']
+    for idx, i in enumerate(vsperf_verify_list, start=1):
+        if str(i) in vsperf_check_cmd_result:
+            if idx < 5:
+                continue
+            elif idx == 5:
+                print(
+                    "VSPERF is Already Installed on DUT-Host..........................."\
+                    ".......[OK]\n")
+            else:
+                download_cmd = "git clone https://gerrit.opnfv.org/gerrit/vswitchperf"
+                DUT_CLIENT.run(download_cmd)
+                install_cmd = "cd vswitchperf/systems ; "
+                install_cmd += "echo '{}' | sudo -S ./build_base_machine.sh ".\
+                                format(DUT_PWD)
+                DUT_CLIENT.run(install_cmd)
+                print(
+                    "Vsperf Installed on DUT-Host ....................................[OK]\n")
+
+
+def tgen_install():
+    """
+    Install T-rex traffic gen on TGen
+    """
+    kill_cmd = "pkill -f ./t-rex"
+    TGEN_CLIENT.send_command(kill_cmd)
+    tgen_start_check = "cd trex/scripts && ./t-rex-64 -f cap2/dns.yaml -d 100 -m 1 --nc"
+    tgen_start_cmd_result = int(TGEN_CLIENT.execute(tgen_start_check)[0])
+    if tgen_start_cmd_result == 0:
+        print(
+            "The Host has T-rex Installed....................................[OK]\n")
+    else:
+        download_cmd = "git clone https://github.com/cisco-system-traffic-generator/trex-core trex"
+        TGEN_CLIENT.run(download_cmd)
+        install_cmd = "cd trex-core/linux_dpdk ; ./b configure ; ./b build"
+        TGEN_CLIENT.run(install_cmd)
+        print(
+            "The Host has now T-rex Installed...........................[OK]\n")
+
+def upload_tgen_config_file():
+    """
+    Upload Tgen Config File on T-rex
+    """
+    localpath = '/usr/src/app/vsperf/trex_cfg.yaml'
+    if not os.path.exists(localpath):
+        print("TGEN config File does not exist................[Failed]")
+        return
+    remotepath = '~/trex_cfg.yaml'
+    check_trex_config_cmd = "echo {} | sudo -S find /etc -maxdepth 1 -name '{}'".format(
+        TGEN_PWD, remotepath[2:])
+    check_test_result = str(TGEN_CLIENT.execute(check_trex_config_cmd)[1])
+    if remotepath[2:] in check_test_result:
+        DUT_CLIENT.run("rm -f /etc/{}".format(remotepath[2:]))
+    TGEN_CLIENT.put_file(localpath, remotepath)
+    TGEN_CLIENT.run(
+        "echo {} | sudo -S mv ~/{} /etc/".format(TGEN_PWD, remotepath[2:]), pty=True)
+    print(
+        "T-rex Configuration File Uploaded on TGen-Host...........................[OK]\n")
+
+
+def install_collectd():
+    """
+    installation of the collectd
+    """
+    check_collectd_config_cmd = "find /opt -maxdepth 1 -name 'collectd'"
+    check_test_result = str(DUT_CLIENT.execute(check_collectd_config_cmd)[1])
+    if "collectd" in check_test_result:
+        print(
+            'Collectd Installed Successfully on DUT-Host..............................[OK]\n')
+    else:
+        download_cmd = "git clone https://github.com/collectd/collectd.git"
+        DUT_CLIENT.run(download_cmd)
+        build_cmd = "cd collectd ; "
+        build_cmd += "./build.sh"
+        DUT_CLIENT.run(build_cmd)
+        config_cmd = "cd collectd ; ./configure --enable-syslog --enable-logfile "
+        config_cmd += "--enable-hugepages --enable-debug ; "
+        DUT_CLIENT.run(config_cmd)
+        install_cmd = "cd collectd ; make ; "
+        install_cmd += "echo '{}' | sudo -S make install".format(DUT_PWD)
+        DUT_CLIENT.run(install_cmd, pty=True)
+        print(
+            'Collectd Installed Successfully on DUT-Host.............................[OK]\n ')
+
+
+def collectd_upload_config():
+    """
+    Upload Configuration file of Collectd on DUT
+    """
+    localpath = '/usr/src/app/vsperf/collectd.conf'
+    if not os.path.exists(localpath):
+        print("Collectd config File does not exist.......................[Failed]")
+        return
+    remotepath = '~/collectd.conf'
+    collectd_config_cmd = "echo {} | sudo -S find /opt/collectd/etc -maxdepth 1 -name '{}'".\
+                           format(DUT_PWD, remotepath[2:])
+    check_test_result = str(DUT_CLIENT.execute(collectd_config_cmd)[1])
+    if remotepath[2:] in check_test_result:
+        DUT_CLIENT.run(
+            "echo {} | sudo -S rm -f /opt/collectd/etc/{}".format(DUT_PWD, remotepath[2:]))
+    DUT_CLIENT.put_file(localpath, remotepath)
+    DUT_CLIENT.run("echo {} | sudo -S mv ~/{} /opt/collectd/etc/".\
+                  format(DUT_PWD, remotepath[2:]), pty=True)
+    print(
+        "Collectd Configuration File Uploaded on DUT-Host.........................[OK]\n ")
+
+def start_tgen():
+    """
+    It will start the Traffic generetor
+    """
+    kill_cmd = "pkill -f ./t-rex"
+    TGEN_CLIENT.send_command(kill_cmd)
+    run_cmd = "cd trex_2.37/scripts && "
+    run_cmd += "screen ./t-rex-64 "
+    run_cmd += TGEN_PARAM
+    TGEN_CLIENT.send_command(run_cmd)
+    print(
+        "T-Rex Successfully running...............................................[OK]\n")
+
+
+def dut_hugepage_config():
+    """
+    Configure the DUT system hugepage parameter from client
+    """
+    if not HPMAX or not HPREQUESTED:
+        print("HPMAX and HPREQUESTED not defined ...................[Failed]")
+        return
+    hugepage_cmd = "echo '{}' | sudo -S mkdir -p /mnt/huge ; ".format(
+        DUT_PWD)
+    hugepage_cmd += "echo '{}' | sudo -S mount -t hugetlbfs nodev /mnt/huge".format(
+        DUT_PWD)
+    DUT_CLIENT.run(hugepage_cmd, pty=True)
+    hp_nr_cmd = "cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages"
+    hp_free_cmd = "cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/free_hugepages"
+    hp_nr = int(DUT_CLIENT.execute(hp_nr_cmd)[1])
+    hp_free = int(DUT_CLIENT.execute(hp_free_cmd)[1])
+    if hp_free <= HPREQUESTED:
+        hp_nr_new = hp_nr + (HPREQUESTED - hp_free)
+        if hp_nr_new > HPMAX:
+            hp_nr_new = HPMAX
+
+    nr_hugepage_cmd = "echo '{}' | sudo -S bash -c \"echo 'vm.nr_hugepages={}' >> ".\
+                       format(DUT_PWD, hp_nr_new)
+    nr_hugepage_cmd += "/etc/sysctl.conf\""
+    DUT_CLIENT.run(nr_hugepage_cmd, pty=True)
+
+    dict_cmd = "cat /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages"
+    dict_check = int(DUT_CLIENT.execute(dict_cmd)[0])
+    if dict_check == 0:
+        node1_hugepage_cmd = "echo '{}' | sudo -s bash -c \"echo 0 > ".format(DUT_PWD)
+        node1_hugepage_cmd += "/sys/devices/system/node/node1/hugepages"
+        node1_hugepage_cmd += "/hugepages-2048kB/nr_hugepages\""
+        DUT_CLIENT.run(node1_hugepage_cmd, pty=True)
+    print("DUT-Host system configured with {} No of Hugepages.....................[OK] \n ".\
+          format(hp_nr_new))
+
+
+def sanity_nic_check():
+    """
+    Check either NIC PCI ids are Correctly placed or not
+    """
+    trex_conf_path = "cat /etc/trex_cfg.yaml | grep interfaces"
+    trex_conf_read = TGEN_CLIENT.execute(trex_conf_path)[1]
+    nic_pid_ids_list = [trex_conf_read.split("\"")[1], trex_conf_read.split("\"")[3]]
+    trex_nic_pic_id_cmd = "lspci | egrep -i --color 'network|ethernet'"
+    trex_nic_pic_id = str(TGEN_CLIENT.execute(trex_nic_pic_id_cmd)[1]).split('\n')
+    acheck = 0
+    for k in trex_nic_pic_id:
+        for j in nic_pid_ids_list:
+            if j in k:
+                acheck += 1
+            else:
+                pass
+    if acheck == 2:
+        print("Both the NIC PCI Ids are Correctly"\
+            " configured on TGen-Host...............[OK]\n")
+    else:
+        print("You configured NIC PCI Ids Wrong in "\
+            "TGen-Host............................[OK]\n")
+
+
+def sanity_collectd_check():
+    """
+    Check and verify collectd is able to run and start properly
+    """
+    check_collectd_cmd = "find /opt -maxdepth 1 -name 'collectd'"
+    check_test_result = str(DUT_CLIENT.execute(check_collectd_cmd)[1])
+    if "collectd" in check_test_result:
+        check_collectd_run_cmd = "echo {} | sudo -S service collectd start".format(
+            DUT_PWD)
+        DUT_CLIENT.run(check_collectd_run_cmd, pty=True)
+        check_collectd_status_cmd = "ps aux | grep collectd"
+        check_collectd_status = str(
+            DUT_CLIENT.execute(check_collectd_status_cmd)[1])
+        if "/sbin/collectd" in check_collectd_status:
+            print(
+                "Collectd is working Fine ................................................[OK] \n ")
+        else:
+            print(
+                "Collectd Fail to Start, Install correctly before running Test....[Failed]\n ")
+    else:
+        print(
+            "Collectd is not installed yet........................................[Failed]\n")
+
+def sanity_vsperf_check():
+    """
+    We have to make sure that VSPERF install correctly
+    """
+    if not DUT_CLIENT:
+        print("The Client is disconnected................................[Failed]")
+        return
+    vsperf_check_cmd = "source ~/vsperfenv/bin/activate ; cd vswitchperf && ./vsperf --help"
+    vsperf_check_cmd_result = str(DUT_CLIENT.execute(vsperf_check_cmd)[1])
+    vsperf_verify_list = [
+        'usage',
+        'positional arguments',
+        'optional arguments',
+        'test selection options',
+        'test behavior options']
+    for idx, i in enumerate(vsperf_verify_list, start=1):
+        if str(i) in vsperf_check_cmd_result:
+            if idx < 5:
+                continue
+            elif idx == 5:
+                print(
+                    "VSPERF Installed Correctly and Working fine.........................."\
+                    "....[OK]\n")
+            else:
+                print(
+                    "VSPERF Does Not Installed Correctly , INSTALL IT AGAIN........[Critical]\n")
+        else:
+            print(
+                "VSPERF Does Not Installed Correctly , INSTALL IT AGAIN............[Critical]\n")
+            break
+
+
+def sanity_tgen_conn_dut_check():
+    """
+    We should confirm the DUT connectivity with the Tgen and Traffic Generator is working or not
+    """
+    if not DUT_CLIENT or not TGEN_CLIENT:
+        print("The Client is disconnected................................[Failed]")
+        return
+    tgen_connectivity_check_cmd = "ping {} -c 1".format(TGEN_IP)
+    tgen_connectivity_check_result = int(
+        DUT_CLIENT.execute(tgen_connectivity_check_cmd)[0])
+    if tgen_connectivity_check_result == 0:
+        print(
+            "DUT-Host is successfully reachable to Traffic Generator Host.............[OK]\n")
+    else:
+        print(
+            "DUT-host is unsuccessful to reach the Traffic Generator Host..............[Failed]")
+        print(
+            "Make sure to establish connection before running Test...............[Critical]\n")
+
+
+def sanity_tgen_check():
+    """
+    It will check Trex properly running or not
+    """
+    if not TGEN_CLIENT:
+        print("The Client is disconnected................................[Failed]")
+        return
+    tgen_start_cmd_check = "cd trex/scripts &&"
+    tgen_start_cmd_check += " ./t-rex-64 -f cap2/dns.yaml -d 100 -m 1 --nc"
+    tgen_start_cmd_result = int(TGEN_CLIENT.execute(tgen_start_cmd_check)[0])
+    if tgen_start_cmd_result == 0:
+        print(
+            "TGen-Host successfully running........................................[OK]\n")
+    else:
+        print("TGen-Host is unable to start t-rex ..................[Failed]")
+        print("Make sure you install t-rex correctly ...............[Critical]\n")
+
+
+def dut_vsperf_test_availability():
+    """
+    Before running test we have to make sure there is no other test running
+    """
+    vsperf_ava_cmd = "ps -ef | grep -v grep | grep ./vsperf | awk '{print $2}'"
+    vsperf_ava_result = len(
+        (DUT_CLIENT.execute(vsperf_ava_cmd)[1]).split("\n"))
+    if vsperf_ava_result == 1:
+        print("DUT-Host is available for performing VSPERF Test\n\
+            You can perform Test!")
+    else:
+        print("DUT-Host is busy right now, Wait for some time\n\
+            Always Check availability before Running Test!\n")
+
+if DUT_IP:
+    host_connect()
+if not DUT_CLIENT:
+    print('Failed to connect to DUT ...............[Critical]')
+    sys.exit()
+else:
+    vsperf_install()
+    install_collectd()
+    collectd_upload_config()
+    dut_hugepage_config()
+    dut_vsperf_test_availability()
+if TGEN_IP:
+    tgen_connect()
+if not TGEN_CLIENT:
+    print('Failed to connect to TGEN_HOST.............[Critical]')
+    sys.exit()
+else:
+    tgen_install()
+    upload_tgen_config_file()
+    sanity_nic_check()
+    start_tgen()
+
+print("\n\nIF you are getting any Failed or Critical message!!!\n" \
+      "Please follow this steps:\n"
+      "1. Make necessory changes before running VSPERF TEST\n"\
+      "2. Re-Run the auto deployment container")
+
+if SANITY and 'yes' in SANITY.lower():
+    sanity_collectd_check()
+    sanity_vsperf_check()
+    sanity_tgen_check()
+    sanity_tgen_conn_dut_check()