diff options
Diffstat (limited to 'tools/docker/deployment/auto/controller/vsperf')
4 files changed, 462 insertions, 0 deletions
diff --git a/tools/docker/deployment/auto/controller/vsperf/__init__.py b/tools/docker/deployment/auto/controller/vsperf/__init__.py new file mode 100644 index 00000000..ad0ebec3 --- /dev/null +++ b/tools/docker/deployment/auto/controller/vsperf/__init__.py @@ -0,0 +1 @@ +#### Empty diff --git a/tools/docker/deployment/auto/controller/vsperf/collectd.conf b/tools/docker/deployment/auto/controller/vsperf/collectd.conf new file mode 100644 index 00000000..9cefc8c5 --- /dev/null +++ b/tools/docker/deployment/auto/controller/vsperf/collectd.conf @@ -0,0 +1,49 @@ +Hostname "pod12-node4" +Interval 1 +LoadPlugin intel_rdt +LoadPlugin processes +LoadPlugin interface +LoadPlugin network +LoadPlugin ovs_stats +LoadPlugin cpu +LoadPlugin memory +LoadPlugin csv +#LoadPlugin dpdkstat +############################################################################## +# Plugin configuration # +############################################################################## +<Plugin processes> + ProcessMatch "ovs-vswitchd" "ovs-vswitchd" + ProcessMatch "ovsdb-server" "ovsdb-server" + ProcessMatch "collectd" "collectd" +</Plugin> +<Plugin network> + Server "10.10.120.22" "25826" +</Plugin> + +<Plugin ovs_stats> + Port "6640" + Address "127.0.0.1" + Socket "/usr/local/var/run/openvswitch/db.sock" + Bridges "vsperf-br0" +</Plugin> + +<Plugin "intel_rdt"> + Cores "2" "4-5" "6-7" "8" "9" "22" "23" "24" "25" "26" "27" +</Plugin> + +<Plugin csv> + DataDir "/tmp/csv" + StoreRates false +</Plugin> + +#<Plugin dpdkstat> +# <EAL> +# Coremask "0x1" +# MemoryChannels "4" +# FilePrefix "rte" +# </EAL> +# SharedMemObj "dpdk_collectd_stats_0" +# EnabledPortMask 0xffff +#</Plugin> + diff --git a/tools/docker/deployment/auto/controller/vsperf/trex_cfg.yaml b/tools/docker/deployment/auto/controller/vsperf/trex_cfg.yaml new file mode 100644 index 00000000..8bb8e341 --- /dev/null +++ b/tools/docker/deployment/auto/controller/vsperf/trex_cfg.yaml @@ -0,0 +1,20 @@ +- port_limit : 2 + version : 2 + interfaces : ["81:00.0", "81:00.1"] # list of the interfaces to bind # node 4 + port_bandwidth_gb : 10 #10G nics + port_info : # set eth mac addr + - dest_mac : "3c:fd:fe:b4:41:09" # port 0 + src_mac : "3c:fd:fe:b4:41:08" + - dest_mac : "3c:fd:fe:b4:41:08" # port 1 + src_mac : "3c:fd:fe:b4:41:09" + platform : + master_thread_id : 17 + latency_thread_id : 16 + dual_if : + - socket : 1 + threads : [22,23,24,25,26,27] + - socket : 0 + threads : [10,11,12,13,14,15] + + + diff --git a/tools/docker/deployment/auto/controller/vsperf/vsperf_controller.py b/tools/docker/deployment/auto/controller/vsperf/vsperf_controller.py new file mode 100644 index 00000000..b6865272 --- /dev/null +++ b/tools/docker/deployment/auto/controller/vsperf/vsperf_controller.py @@ -0,0 +1,392 @@ +# Copyright 2018-19 Spirent Communications. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +VSPERF_deploy_auto +""" +# pylint: disable=W0603 + +import os +import sys +from utils import ssh + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + +DUT_IP = os.getenv('DUT_IP_ADDRESS') +DUT_USER = os.getenv('DUT_USERNAME') +DUT_PWD = os.getenv('DUT_PASSWORD') + +TGEN_IP = os.getenv('TGEN_IP_ADDRESS') +TGEN_USER = os.getenv('TGEN_USERNAME') +TGEN_PWD = os.getenv('TGEN_PASSWORD') +TGEN_PARAM = os.getenv('TGEN_PARAMS') + +HPMAX = int(os.getenv('HUGEPAGE_MAX')) +HPREQUESTED = int(os.getenv('HUGEPAGE_REQUESTED')) + +SANITY = str(os.getenv('SANITY_CHECK'))# + +DUT_CLIENT = None +TGEN_CLIENT = None + + +def host_connect(): + """ + Handle host connectivity to DUT + """ + global DUT_CLIENT + DUT_CLIENT = ssh.SSH(host=DUT_IP, user=DUT_USER, password=DUT_PWD) + print("DUT-Host Successfully Connected .........................................[OK] \n ") + +def tgen_connect(): + """ + Handle Tgen Connection to Trex + """ + global TGEN_CLIENT + TGEN_CLIENT = ssh.SSH(host=TGEN_IP, user=TGEN_USER, password=TGEN_PWD) + print("Traffic Generator Successfully Connected ...............................[OK] \n ") + + +def vsperf_install(): + """ + Perform actual installation + """ + vsperf_check_command = "source ~/vsperfenv/bin/activate ; " + vsperf_check_command += "cd vswitchperf && ./vsperf --help" + vsperf_check_cmd_result = str(DUT_CLIENT.execute(vsperf_check_command)[1]) + vsperf_verify_list = [ + 'usage', + 'positional arguments', + 'optional arguments', + 'test selection options', + 'test behavior options'] + for idx, i in enumerate(vsperf_verify_list, start=1): + if str(i) in vsperf_check_cmd_result: + if idx < 5: + continue + elif idx == 5: + print( + "VSPERF is Already Installed on DUT-Host..........................."\ + ".......[OK]\n") + else: + download_cmd = "git clone https://gerrit.opnfv.org/gerrit/vswitchperf" + DUT_CLIENT.run(download_cmd) + install_cmd = "cd vswitchperf/systems ; " + install_cmd += "echo '{}' | sudo -S ./build_base_machine.sh ".\ + format(DUT_PWD) + DUT_CLIENT.run(install_cmd) + print( + "Vsperf Installed on DUT-Host ....................................[OK]\n") + + +def tgen_install(): + """ + Install T-rex traffic gen on TGen + """ + kill_cmd = "pkill -f ./t-rex" + TGEN_CLIENT.send_command(kill_cmd) + tgen_start_check = "cd trex/scripts && ./t-rex-64 -f cap2/dns.yaml -d 100 -m 1 --nc" + tgen_start_cmd_result = int(TGEN_CLIENT.execute(tgen_start_check)[0]) + if tgen_start_cmd_result == 0: + print( + "The Host has T-rex Installed....................................[OK]\n") + else: + download_cmd = "git clone https://github.com/cisco-system-traffic-generator/trex-core trex" + TGEN_CLIENT.run(download_cmd) + install_cmd = "cd trex-core/linux_dpdk ; ./b configure ; ./b build" + TGEN_CLIENT.run(install_cmd) + print( + "The Host has now T-rex Installed...........................[OK]\n") + +def upload_tgen_config_file(): + """ + Upload Tgen Config File on T-rex + """ + localpath = '/usr/src/app/vsperf/trex_cfg.yaml' + if not os.path.exists(localpath): + print("TGEN config File does not exist................[Failed]") + return + remotepath = '~/trex_cfg.yaml' + check_trex_config_cmd = "echo {} | sudo -S find /etc -maxdepth 1 -name '{}'".format( + TGEN_PWD, remotepath[2:]) + check_test_result = str(TGEN_CLIENT.execute(check_trex_config_cmd)[1]) + if remotepath[2:] in check_test_result: + DUT_CLIENT.run("rm -f /etc/{}".format(remotepath[2:])) + TGEN_CLIENT.put_file(localpath, remotepath) + TGEN_CLIENT.run( + "echo {} | sudo -S mv ~/{} /etc/".format(TGEN_PWD, remotepath[2:]), pty=True) + print( + "T-rex Configuration File Uploaded on TGen-Host...........................[OK]\n") + + +def install_collectd(): + """ + installation of the collectd + """ + check_collectd_config_cmd = "find /opt -maxdepth 1 -name 'collectd'" + check_test_result = str(DUT_CLIENT.execute(check_collectd_config_cmd)[1]) + if "collectd" in check_test_result: + print( + 'Collectd Installed Successfully on DUT-Host..............................[OK]\n') + else: + download_cmd = "git clone https://github.com/collectd/collectd.git" + DUT_CLIENT.run(download_cmd) + build_cmd = "cd collectd ; " + build_cmd += "./build.sh" + DUT_CLIENT.run(build_cmd) + config_cmd = "cd collectd ; ./configure --enable-syslog --enable-logfile " + config_cmd += "--enable-hugepages --enable-debug ; " + DUT_CLIENT.run(config_cmd) + install_cmd = "cd collectd ; make ; " + install_cmd += "echo '{}' | sudo -S make install".format(DUT_PWD) + DUT_CLIENT.run(install_cmd, pty=True) + print( + 'Collectd Installed Successfully on DUT-Host.............................[OK]\n ') + + +def collectd_upload_config(): + """ + Upload Configuration file of Collectd on DUT + """ + localpath = '/usr/src/app/vsperf/collectd.conf' + if not os.path.exists(localpath): + print("Collectd config File does not exist.......................[Failed]") + return + remotepath = '~/collectd.conf' + collectd_config_cmd = "echo {} | sudo -S find /opt/collectd/etc -maxdepth 1 -name '{}'".\ + format(DUT_PWD, remotepath[2:]) + check_test_result = str(DUT_CLIENT.execute(collectd_config_cmd)[1]) + if remotepath[2:] in check_test_result: + DUT_CLIENT.run( + "echo {} | sudo -S rm -f /opt/collectd/etc/{}".format(DUT_PWD, remotepath[2:])) + DUT_CLIENT.put_file(localpath, remotepath) + DUT_CLIENT.run("echo {} | sudo -S mv ~/{} /opt/collectd/etc/".\ + format(DUT_PWD, remotepath[2:]), pty=True) + print( + "Collectd Configuration File Uploaded on DUT-Host.........................[OK]\n ") + +def start_tgen(): + """ + It will start the Traffic generetor + """ + kill_cmd = "pkill -f ./t-rex" + TGEN_CLIENT.send_command(kill_cmd) + run_cmd = "cd trex_2.37/scripts && " + run_cmd += "screen ./t-rex-64 " + run_cmd += TGEN_PARAM + TGEN_CLIENT.send_command(run_cmd) + print( + "T-Rex Successfully running...............................................[OK]\n") + + +def dut_hugepage_config(): + """ + Configure the DUT system hugepage parameter from client + """ + if not HPMAX or not HPREQUESTED: + print("HPMAX and HPREQUESTED not defined ...................[Failed]") + return + hugepage_cmd = "echo '{}' | sudo -S mkdir -p /mnt/huge ; ".format( + DUT_PWD) + hugepage_cmd += "echo '{}' | sudo -S mount -t hugetlbfs nodev /mnt/huge".format( + DUT_PWD) + DUT_CLIENT.run(hugepage_cmd, pty=True) + hp_nr_cmd = "cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages" + hp_free_cmd = "cat /sys/devices/system/node/node0/hugepages/hugepages-2048kB/free_hugepages" + hp_nr = int(DUT_CLIENT.execute(hp_nr_cmd)[1]) + hp_free = int(DUT_CLIENT.execute(hp_free_cmd)[1]) + if hp_free <= HPREQUESTED: + hp_nr_new = hp_nr + (HPREQUESTED - hp_free) + if hp_nr_new > HPMAX: + hp_nr_new = HPMAX + + nr_hugepage_cmd = "echo '{}' | sudo -S bash -c \"echo 'vm.nr_hugepages={}' >> ".\ + format(DUT_PWD, hp_nr_new) + nr_hugepage_cmd += "/etc/sysctl.conf\"" + DUT_CLIENT.run(nr_hugepage_cmd, pty=True) + + dict_cmd = "cat /sys/devices/system/node/node1/hugepages/hugepages-2048kB/nr_hugepages" + dict_check = int(DUT_CLIENT.execute(dict_cmd)[0]) + if dict_check == 0: + node1_hugepage_cmd = "echo '{}' | sudo -s bash -c \"echo 0 > ".format(DUT_PWD) + node1_hugepage_cmd += "/sys/devices/system/node/node1/hugepages" + node1_hugepage_cmd += "/hugepages-2048kB/nr_hugepages\"" + DUT_CLIENT.run(node1_hugepage_cmd, pty=True) + print("DUT-Host system configured with {} No of Hugepages.....................[OK] \n ".\ + format(hp_nr_new)) + + +def sanity_nic_check(): + """ + Check either NIC PCI ids are Correctly placed or not + """ + trex_conf_path = "cat /etc/trex_cfg.yaml | grep interfaces" + trex_conf_read = TGEN_CLIENT.execute(trex_conf_path)[1] + nic_pid_ids_list = [trex_conf_read.split("\"")[1], trex_conf_read.split("\"")[3]] + trex_nic_pic_id_cmd = "lspci | egrep -i --color 'network|ethernet'" + trex_nic_pic_id = str(TGEN_CLIENT.execute(trex_nic_pic_id_cmd)[1]).split('\n') + acheck = 0 + for k in trex_nic_pic_id: + for j in nic_pid_ids_list: + if j in k: + acheck += 1 + else: + pass + if acheck == 2: + print("Both the NIC PCI Ids are Correctly"\ + " configured on TGen-Host...............[OK]\n") + else: + print("You configured NIC PCI Ids Wrong in "\ + "TGen-Host............................[OK]\n") + + +def sanity_collectd_check(): + """ + Check and verify collectd is able to run and start properly + """ + check_collectd_cmd = "find /opt -maxdepth 1 -name 'collectd'" + check_test_result = str(DUT_CLIENT.execute(check_collectd_cmd)[1]) + if "collectd" in check_test_result: + check_collectd_run_cmd = "echo {} | sudo -S service collectd start".format( + DUT_PWD) + DUT_CLIENT.run(check_collectd_run_cmd, pty=True) + check_collectd_status_cmd = "ps aux | grep collectd" + check_collectd_status = str( + DUT_CLIENT.execute(check_collectd_status_cmd)[1]) + if "/sbin/collectd" in check_collectd_status: + print( + "Collectd is working Fine ................................................[OK] \n ") + else: + print( + "Collectd Fail to Start, Install correctly before running Test....[Failed]\n ") + else: + print( + "Collectd is not installed yet........................................[Failed]\n") + +def sanity_vsperf_check(): + """ + We have to make sure that VSPERF install correctly + """ + if not DUT_CLIENT: + print("The Client is disconnected................................[Failed]") + return + vsperf_check_cmd = "source ~/vsperfenv/bin/activate ; cd vswitchperf && ./vsperf --help" + vsperf_check_cmd_result = str(DUT_CLIENT.execute(vsperf_check_cmd)[1]) + vsperf_verify_list = [ + 'usage', + 'positional arguments', + 'optional arguments', + 'test selection options', + 'test behavior options'] + for idx, i in enumerate(vsperf_verify_list, start=1): + if str(i) in vsperf_check_cmd_result: + if idx < 5: + continue + elif idx == 5: + print( + "VSPERF Installed Correctly and Working fine.........................."\ + "....[OK]\n") + else: + print( + "VSPERF Does Not Installed Correctly , INSTALL IT AGAIN........[Critical]\n") + else: + print( + "VSPERF Does Not Installed Correctly , INSTALL IT AGAIN............[Critical]\n") + break + + +def sanity_tgen_conn_dut_check(): + """ + We should confirm the DUT connectivity with the Tgen and Traffic Generator is working or not + """ + if not DUT_CLIENT or not TGEN_CLIENT: + print("The Client is disconnected................................[Failed]") + return + tgen_connectivity_check_cmd = "ping {} -c 1".format(TGEN_IP) + tgen_connectivity_check_result = int( + DUT_CLIENT.execute(tgen_connectivity_check_cmd)[0]) + if tgen_connectivity_check_result == 0: + print( + "DUT-Host is successfully reachable to Traffic Generator Host.............[OK]\n") + else: + print( + "DUT-host is unsuccessful to reach the Traffic Generator Host..............[Failed]") + print( + "Make sure to establish connection before running Test...............[Critical]\n") + + +def sanity_tgen_check(): + """ + It will check Trex properly running or not + """ + if not TGEN_CLIENT: + print("The Client is disconnected................................[Failed]") + return + tgen_start_cmd_check = "cd trex/scripts &&" + tgen_start_cmd_check += " ./t-rex-64 -f cap2/dns.yaml -d 100 -m 1 --nc" + tgen_start_cmd_result = int(TGEN_CLIENT.execute(tgen_start_cmd_check)[0]) + if tgen_start_cmd_result == 0: + print( + "TGen-Host successfully running........................................[OK]\n") + else: + print("TGen-Host is unable to start t-rex ..................[Failed]") + print("Make sure you install t-rex correctly ...............[Critical]\n") + + +def dut_vsperf_test_availability(): + """ + Before running test we have to make sure there is no other test running + """ + vsperf_ava_cmd = "ps -ef | grep -v grep | grep ./vsperf | awk '{print $2}'" + vsperf_ava_result = len( + (DUT_CLIENT.execute(vsperf_ava_cmd)[1]).split("\n")) + if vsperf_ava_result == 1: + print("DUT-Host is available for performing VSPERF Test\n\ + You can perform Test!") + else: + print("DUT-Host is busy right now, Wait for some time\n\ + Always Check availability before Running Test!\n") + +if DUT_IP: + host_connect() +if not DUT_CLIENT: + print('Failed to connect to DUT ...............[Critical]') + sys.exit() +else: + vsperf_install() + install_collectd() + collectd_upload_config() + dut_hugepage_config() + dut_vsperf_test_availability() +if TGEN_IP: + tgen_connect() +if not TGEN_CLIENT: + print('Failed to connect to TGEN_HOST.............[Critical]') + sys.exit() +else: + tgen_install() + upload_tgen_config_file() + sanity_nic_check() + start_tgen() + +print("\n\nIF you are getting any Failed or Critical message!!!\n" \ + "Please follow this steps:\n" + "1. Make necessory changes before running VSPERF TEST\n"\ + "2. Re-Run the auto deployment container") + +if SANITY and 'yes' in SANITY.lower(): + sanity_collectd_check() + sanity_vsperf_check() + sanity_tgen_check() + sanity_tgen_conn_dut_check() |