# Copyright 2018-19 Spirent Communications. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ VSPERF-controller """ # Fetching Environment Variable for controller, You can configure or # modifies list.env file for setting your environment variable. #pylint: disable=global-statement,no-else-continue #pylint: disable=too-many-branches import os import time import math import ast from utils import ssh _ONE_DAY_IN_SECONDS = 60 * 60 * 24 TIMER = float() DUT_IP = os.getenv('DUT_IP_ADDRESS') DUT_USER = os.getenv('DUT_USERNAME') DUT_PWD = os.getenv('DUT_PASSWORD') TGEN_IP = os.getenv('TGEN_IP_ADDRESS') VSPERF_TEST = os.getenv('VSPERF_TESTS') VSPERF_CONF = os.getenv('VSPERF_CONFFILE') VSPERF_TRAFFICGEN_MODE = str(os.getenv('VSPERF_TRAFFICGEN_MODE')) START_COLLECTD = os.getenv('START_COLLECTD') START_BEATS = os.getenv('START_BEATS') CLEAN_UP = os.getenv('CLEAN_UP') DUT_CLIENT = None TGEN_CLIENT = None SANITY_CHECK_DONE_LIST = list() def host_connect(): """ Handle host connectivity to DUT """ global DUT_CLIENT DUT_CLIENT = ssh.SSH(host=DUT_IP, user=DUT_USER, password=DUT_PWD) print("DUT Successfully Connected ..............................................[OK] \n ") def upload_test_config_file(): """ #Upload Test Config File on DUT """ localpath = '/usr/src/app/vsperf/vsperf.conf' if not os.path.exists(localpath): print("VSPERF Test config File does not exists.......................[Failed]") return remotepath = '~/vsperf.conf' check_test_config_cmd = "find ~/ -maxdepth 1 -name '{}'".format( remotepath[2:]) check_test_result = str(DUT_CLIENT.execute(check_test_config_cmd)[1]) if remotepath[2:] in check_test_result: DUT_CLIENT.run("rm -f {}".format(remotepath[2:])) DUT_CLIENT.put_file(localpath, remotepath) check_test_config_cmd_1= "find ~/ -maxdepth 1 -name '{}'".format( remotepath[2:]) check_test_result_1= str(DUT_CLIENT.execute(check_test_config_cmd)[1]) if remotepath[2:] in check_test_result_1: print( "Test Configuration File Uploaded on DUT-Host.............................[OK] \n ") else: print("VSPERF Test config file upload failed.....................................[Critical]") def start_beats(): """ Start fileBeats on DUT """ run_cmd = "echo '{}' | sudo -S service filebeat start".format(DUT_PWD) DUT_CLIENT.run(run_cmd, pty=True) print( "Beats are started on DUT-Host............................................[OK] \n") def start_collectd(): """ start the collectd """ run_cmd = "echo '{}' | sudo -S service collectd start".format(DUT_PWD) DUT_CLIENT.run(run_cmd, pty=True) print( "Collectd is started on DUT-Host............................................[OK] \n") def run_vsperf_test(): """ Here we will perform the actual vsperf test """ global TIMER rmv_cmd = "cd /mnt/huge && echo {} | sudo -S rm -rf *".format(DUT_PWD) DUT_CLIENT.run(rmv_cmd, pty=True) cmd = "source ~/vsperfenv/bin/activate ; " #cmd = "scl enable python33 bash ; " cmd += "cd vswitchperf && " cmd += "./vsperf " if VSPERF_CONF: cmd += "--conf-file ~/vsperf.conf " if "yes" in VSPERF_TRAFFICGEN_MODE.lower(): cmd += "--mode trafficgen" vsperf_test_list = VSPERF_TEST.split(",") print(vsperf_test_list) for test in vsperf_test_list: atest = cmd atest += test DUT_CLIENT.run(atest, pty=True) print( "Test Successfully running................................................[OK]\n ") def test_status(): """ Chechk for the test status after performing test """ testtype_list = VSPERF_TEST.split(",") num_test = len(testtype_list) test_success = [] test_failed = [] testtype_list_len = len(testtype_list) for test in testtype_list: passed_minutes = 5 latest_result_cmd = "find /tmp -mindepth 1 -type d -cmin -{} -printf '%f'".format( passed_minutes) test_result_dir = str( (DUT_CLIENT.execute(latest_result_cmd)[1]).split('find')[0]) test_date_cmd = "date +%F" test_date = str(DUT_CLIENT.execute(test_date_cmd)[1]).replace("\n", "") if test_date in test_result_dir: testcase_check_cmd = "cd /tmp && cd `ls -t | grep results | head" testcase_check_cmd += " -{} | tail -1` && find . -maxdepth 1 -name '*{}*'".\ format(testtype_list_len, test) testcase_check_output = str( DUT_CLIENT.execute(testcase_check_cmd)[1]).split('\n', 2) check = 0 for i in testcase_check_output: if (".csv" in i) or (".md" in i) or (".rst" in i): check += 1 if check == 3: test_success.append(test) else: test_failed.append(test) testtype_list_len -= 1 if num_test == len(test_success): print("All Test Successfully Completed on DUT-Host Results... [OK]") elif not test_success: print("All Test Failed on DUT-Host \nResults... [Failed]") else: print( "Only {} Test failed Results ... [Failed]\n"\ "All other Test Successfully Completed on DUT-Host Results... [OK] ".\ format(test_failed)) def vsperf_remove(): """ Actual removal of the VSPERF """ vsperf_rm_cmd = "echo '{}' | sudo -S rm -r ~/vswitchperf".format(DUT_PWD) DUT_CLIENT.run(vsperf_rm_cmd) vsperfenv_rm_cmd = "echo '{}' | sudo -S rm -r -f ~/vsperfenv".\ format(DUT_PWD) DUT_CLIENT.run(vsperfenv_rm_cmd) def remove_uploaded_config(): """ Remove all the uploaded configuration files """ vconfig_rm_cmd = "rm ~/vsperf.conf" DUT_CLIENT.run(vconfig_rm_cmd) cdconfig_rm_cmd = "echo '{}' | sudo -S rm /opt/collectd/etc/collectd.conf".\ format(DUT_PWD) DUT_CLIENT.run(cdconfig_rm_cmd) def result_folders_remove(): """ Remove result folder on DUT """ remove_cmd = "rm -r /tmp/*results*" DUT_CLIENT.run(remove_cmd) def collectd_remove(): """ Remove collectd from DUT """ collectd_dwn_rm_cmd = "echo '{}' | sudo -S rm -r -f ~/collectd".format( DUT_PWD) DUT_CLIENT.run(collectd_dwn_rm_cmd) collectd_rm_cmd = "echo '{}' | sudo -S rm -r -f /opt/collectd".format( DUT_PWD) DUT_CLIENT.run(collectd_rm_cmd) def terminate_vsperf(): """ Terminate the VSPERF and kill processes """ stress_kill_cmd = "echo '{}' | sudo -S pkill stress &> /dev/null".format( DUT_PWD) python3_kill_cmd = "echo '{}' | sudo -S pkill python3 &> /dev/null".format( DUT_PWD) qemu_kill_cmd = "echo '{}' | sudo -S killall -9 qemu-system-x86_64 &> /dev/null".format( DUT_PWD) DUT_CLIENT.run(stress_kill_cmd) DUT_CLIENT.run(python3_kill_cmd) DUT_CLIENT.run(qemu_kill_cmd) # sometimes qemu resists to terminate, so wait a bit and kill it again qemu_check_cmd = "pgrep qemu-system-x86_64" qemu_cmd_response = DUT_CLIENT.execute(qemu_check_cmd)[1] if qemu_cmd_response != '': time.sleep(5) DUT_CLIENT.run(qemu_kill_cmd) time.sleep(5) ovs_kill_cmd = "echo '{}' | sudo pkill ovs-vswitchd &> /dev/null".format( DUT_PWD) ovsdb_kill_cmd = "echo '{}' | sudo pkill ovsdb-server &> /dev/null".format( DUT_PWD) vppctl_kill_cmd = "echo '{}' | sudo pkill vppctl &> /dev/null".format( DUT_PWD) vpp_kill_cmd = "echo '{}' | sudo pkill vpp &> /dev/null".format(DUT_PWD) vpp_cmd = "echo '{}' | sudo pkill -9 vpp &> /dev/null".format(DUT_PWD) DUT_CLIENT.run(ovs_kill_cmd) time.sleep(1) DUT_CLIENT.run(ovsdb_kill_cmd) time.sleep(1) DUT_CLIENT.run(vppctl_kill_cmd) time.sleep(1) DUT_CLIENT.run(vpp_kill_cmd) time.sleep(1) DUT_CLIENT.run(vpp_cmd) time.sleep(1) print( "All the VSPERF related process terminated successfully..............[OK]") def sanity_collectd_check(): """ Check and verify collectd is able to run and start properly """ global SANITY_CHECK_DONE_LIST check_collectd_cmd = "find /opt -maxdepth 1 -name 'collectd'" check_test_result = str(DUT_CLIENT.execute(check_collectd_cmd)[1]) if "collectd" in check_test_result: check_collectd_run_cmd = "echo {} | sudo -S service collectd start".format( DUT_PWD) DUT_CLIENT.run(check_collectd_run_cmd, pty=True) check_collectd_status_cmd = "ps aux | grep collectd" check_collectd_status = str( DUT_CLIENT.execute(check_collectd_status_cmd)[1]) if "/sbin/collectd" in check_collectd_status: SANITY_CHECK_DONE_LIST.append(int(1)) print( "Collectd is working Fine ................................................[OK] \n ") else: print( "Collectd Fail to Start, Install correctly before running Test....[Failed]\n ") else: print( "Collectd is not installed yet........................................[Failed]\n") def sanity_vnf_path(): """ Check if VNF image is available on the configured path in Test Config File """ # fetch the VNF path we placed in vsperf.conf file global SANITY_CHECK_DONE_LIST vsperf_conf_path = open('/usr/src/app/vsperf/vsperf.conf') vsperf_conf_read = vsperf_conf_path.readlines() for i in vsperf_conf_read: if 'GUEST_IMAGE' in i: vnf_image_path = i.split("'")[1] vnf_path_check_cmd = "find {}".format(vnf_image_path) vnf_path_check_result = str( DUT_CLIENT.execute(vnf_path_check_cmd)[1]) if vnf_image_path in vnf_path_check_result: SANITY_CHECK_DONE_LIST.append(int(2)) print( "Test Configratuion file has Correct VNF path information on DUT-Host.." \ "...[OK]\n ") else: print( "Test Configuration file has incorrect VNF path information......" \ "....[FAILED]\n") def sanity_vsperf_check(): """ We have to make sure that VSPERF is installed correctly """ global SANITY_CHECK_DONE_LIST vsperf_check_command = "source ~/vsperfenv/bin/activate ; cd vswitchperf* && ./vsperf --help" vsperf_check_cmd_result = str(DUT_CLIENT.execute(vsperf_check_command)[1]) vsperf_verify_list = [ 'usage', 'positional arguments', 'optional arguments', 'test selection options', 'test behavior options'] for idx, i in enumerate(vsperf_verify_list, start=1): if str(i) in vsperf_check_cmd_result: if idx < 5: continue elif idx == 5: SANITY_CHECK_DONE_LIST.append(int(3)) print("VSPERF Installed Correctly and Working fine......................." \ ".......[OK]\n") else: print( "VSPERF DID Not Installed Correctly , INSTALL IT AGAIN...........[Critical]\n") else: print( "VSPERF DID Not Installed Correctly , INSTALL IT AGAIN................[Critical]\n") break def variable_from_test_config(aparameter): """This function can be use to read any configuration paramter from vsperf.conf""" read_cmd = 'cat ~/vsperf.conf | grep "{}"'.format(aparameter) read_cmd_output = str(DUT_CLIENT.execute(read_cmd)[1]) print(read_cmd_output) if not read_cmd_output or '#' in read_cmd_output: return 0 return read_cmd_output.split("=")[1].strip() def cpumask2coreids(mask): """conver mask to coreids""" intmask = int(mask, 16) i = 1 coreids = [] while i < intmask: if i & intmask: coreids.append(str(math.frexp(i)[1]-1)) i = i << 1 return coreids def sanity_cpu_allocation_check(): """It will check the cpu allocation before run test""" global SANITY_CHECK_DONE_LIST read_setting_cmd = "source vsperfenv/bin/activate ; cd vswitchperf* && " read_setting_cmd += './vsperf --list-settings' default_vsperf_settings = ast.literal_eval(str(DUT_CLIENT.execute(read_setting_cmd)[1])) default_cpu_map = default_vsperf_settings["VSWITCH_VHOST_CPU_MAP"] default_vswitch_pmd_cpu_mask = str(default_vsperf_settings["VSWITCH_PMD_CPU_MASK"]) default_vswitch_vhost_cpu_map = [str(x) for x in default_cpu_map] vswitch_pmd_cpu_mask = variable_from_test_config("VSWITCH_PMD_CPU_MASK") vswitch_cpu_map = (variable_from_test_config("VSWITCH_VHOST_CPU_MAP")) vswitch_vhost_cpu_map = 0 if vswitch_cpu_map != 0: vswitch_vhost_cpu_map = [str(x) for x in ast.literal_eval(vswitch_cpu_map)] if vswitch_pmd_cpu_mask == 0 and vswitch_vhost_cpu_map == 0: print("CPU allocation Check Done,"\ "\nNo vswitch_pmd_cpu_mask or vswitch_vhost_cpu_map assign in test config file\n" \ "Using Default Settings ..................................................[OK]\n") elif vswitch_pmd_cpu_mask != 0 and vswitch_vhost_cpu_map == 0: core_id = cpumask2coreids(vswitch_pmd_cpu_mask) print(core_id) if len(default_vswitch_vhost_cpu_map) >= len(core_id): if all(elem in default_vswitch_vhost_cpu_map for elem in core_id): print("CPU allocation properly done on DUT-Host.................[OK]\n") else: print("CPU allocation not done properly on DUT-Host............[Failed]\n") else: print("CPU allocation not done properly on DUT-Host............[Failed]\n") elif vswitch_pmd_cpu_mask == 0 and vswitch_vhost_cpu_map != 0: core_id_1 = cpumask2coreids(default_vswitch_pmd_cpu_mask) print(core_id_1) if len(vswitch_vhost_cpu_map) >= len(core_id_1): if all(elem in vswitch_vhost_cpu_map for elem in core_id_1): print("CPU allocation properly done on DUT-Host.................[OK]\n") else: print("CPU allocation not done properly on DUT-Host............[Failed]\n") else: print("CPU allocation not done properly on DUT-Host............[Failed]\n") else: core_id_2 = cpumask2coreids(vswitch_pmd_cpu_mask) print(core_id_2) if len(vswitch_vhost_cpu_map) >= len(core_id_2): if all(elem in vswitch_vhost_cpu_map for elem in core_id_2): print("CPU allocation properly done on DUT-Host.................[OK]\n") else: print("CPU allocation not done properly on DUT-Host............[Failed]\n") else: print("CPU allocation not done properly on DUT-Host............[Failed]\n") def sanity_dut_conn_tgen_check(): """ We should confirm the DUT connectivity with the Tgen and Traffic Generator is working or not """ global SANITY_CHECK_DONE_LIST tgen_connectivity_check_cmd = "ping {} -c 1".format(TGEN_IP) tgen_connectivity_check_result = int(DUT_CLIENT.execute(tgen_connectivity_check_cmd)[0]) if tgen_connectivity_check_result == 0: SANITY_CHECK_DONE_LIST.append(int(5)) print( "DUT-Host is successfully reachable to Traffic Generator Host.............[OK]\n") else: print( "DUT-host is unsuccessful to reach the Traffic Generator Host.............[Failed]") print( "Make sure to establish connection before running Test...............[Critical]\n") if DUT_IP: host_connect() if not DUT_CLIENT: print('Failed to connect to DUT ...............[Critical]') sys.exit() else: upload_test_config_file() sanity_vnf_path() sanity_cpu_allocation_check() sanity_collectd_check() sanity_vsperf_check() sanity_dut_conn_tgen_check() if "yes" in START_COLLECTD.lower(): start_collectd() if "yes" in START_BEATS.lower(): start_beats() if 'v' in VSPERF_TEST: if len(SANITY_CHECK_DONE_LIST) != 4: print("Certain Sanity Checks Failed\n" \ "You can make changes based on the outputs and run" \ "the testcontrol auto container again") else: run_vsperf_test() test_status() else: if len(SANITY_CHECK_DONE_LIST) != 3: print("Certain Sanity Checks Failed\n" \ "You can make changes based on the outputs and run" \ "the testcontrol auto container again") else: run_vsperf_test() test_status() if "yes" in CLEAN_UP.lower(): vsperf_remove() remove_uploaded_config() result_folders_remove() collectd_remove() terminate_vsperf()