Tools: Deployment and TestControl Containers

This patch add containerization of VSPERF support.
The patch facilitates creation of 4 containers:
1. Interactive Deployment
2. Auto Deployment
3. Interactive TestControl
4. Auto TestControl.
The patch also includes a minimal client to work with interactive
containers.
The docs folder provides detailed documentation.
Fixed pylint errors in libs folder.
Removed proto built python files, and added the build process in
prepare.sh.
Stability improvements for Auto versions of deployment and testcontrol.
Enhance client with 'mode' feature, where client can run either to do
only deploy/only test or both.
Add sample configuration file for client
Fixed few typos - as suggested by AL.

JIRA: VSPERF-594
Signed-off-by: Sridhar K. N. Rao <sridhar.rao@spirent.com>
Change-Id: Id40b02960f71a7f9183d9a53955e2483117fb9e2

5 files changed, 527 insertions, 0 deletions

diff --git a/tools/docker/testcontrol/auto/controller/Dockerfile b/tools/docker/testcontrol/auto/controller/Dockerfile
new file mode 100644
index 00000000..4fbf7294
--- /dev/null
+++ b/tools/docker/testcontrol/auto/controller/Dockerfile
@@ -0,0 +1,23 @@
+FROM python:3.6
+LABEL maintainer="sridhar.rao@spirent.com"
+
+ENV GRPC_PYTHON_VERSION 1.4.0
+RUN apt-get update && apt-get -y install python3-pip && apt-get -y install openssh-server
+RUN pip3 install grpcio==${GRPC_PYTHON_VERSION} grpcio-tools==${GRPC_PYTHON_VERSION}
+RUN pip3 install paramiko
+RUN pip3 install chainmap
+RUN pip3 install oslo.utils
+RUN pip3 install scp
+
+WORKDIR /usr/src/app
+
+COPY ./vsperf ./vsperf
+
+VOLUME ["/usr/src/app/vsperf"]
+
+EXPOSE 50052
+
+CMD ["python3", "./vsperf/vsperf_controller.py"]
+
+#CMD tail -f /dev/null
+
diff --git a/tools/docker/testcontrol/auto/controller/list.env b/tools/docker/testcontrol/auto/controller/list.env
new file mode 100644
index 00000000..2883021b
--- /dev/null
+++ b/tools/docker/testcontrol/auto/controller/list.env
@@ -0,0 +1,13 @@
+DUT_IP_ADDRESS=10.10.120.24
+DUT_USERNAME=opnfv
+DUT_PASSWORD=opnfv
+
+TGEN_IP_ADDRESS=10.10.120.25
+
+VSPERF_TESTS=phy2phy_tput,pvp_tput
+VSPERF_CONFFILE=vsperf.conf
+
+VSPERF_TRAFFICGEN_MODE=NO
+
+CLEAN_UP=NO
+
diff --git a/tools/docker/testcontrol/auto/controller/vsperf/__init__.py b/tools/docker/testcontrol/auto/controller/vsperf/__init__.py
new file mode 100644
index 00000000..ad0ebec3
--- /dev/null
+++ b/tools/docker/testcontrol/auto/controller/vsperf/__init__.py
@@ -0,0 +1 @@
+#### Empty
diff --git a/tools/docker/testcontrol/auto/controller/vsperf/vsperf.conf b/tools/docker/testcontrol/auto/controller/vsperf/vsperf.conf
new file mode 100644
index 00000000..50d40f49
--- /dev/null
+++ b/tools/docker/testcontrol/auto/controller/vsperf/vsperf.conf
@@ -0,0 +1,21 @@
+VSWITCH_BRIDGE_NAME = 'vsperf-br0'
+WHITELIST_NICS = ['02:00.0', '02:00.1']
+TRAFFICGEN = 'Trex'
+TRAFFICGEN_TREX_HOST_IP_ADDR = '10.10.120.25'
+TRAFFICGEN_TREX_USER = 'root'
+TRAFFICGEN_TREX_BASE_DIR = '/root/trex_2.37/scripts/'
+TRAFFICGEN_TREX_LINE_SPEED_GBPS = '10'
+TRAFFICGEN_TREX_PORT1 = '0000:81:00.0'
+TRAFFICGEN_TREX_PORT2 = '0000:81:00.1'
+TRAFFICGEN_TREX_PROMISCUOUS = False
+TRAFFICGEN_DURATION=1
+TRAFFICGEN_LOSSRATE=0
+TRAFFICGEN_RFC2544_TESTS=10
+#TRAFFICGEN_PKT_SIZES=(64,128,256,512,1024,1280,1518)
+TRAFFICGEN_PKT_SIZES=(64,)
+GUEST_TESTPMD_FWD_MODE = ['io']
+GUEST_IMAGE = ['/home/opnfv/vnfs/vloop-vnf-ubuntu-18.04_20180920.qcow2']
+TRAFFICGEN_TREX_LATENCY_PPS = 1000
+TRAFFICGEN_TREX_RFC2544_BINARY_SEARCH_LOSS_VERIFICATION = True
+TRAFFICGEN_TREX_RFC2544_MAX_REPEAT = 2
+
diff --git a/tools/docker/testcontrol/auto/controller/vsperf/vsperf_controller.py b/tools/docker/testcontrol/auto/controller/vsperf/vsperf_controller.py
new file mode 100644
index 00000000..1b088fea
--- /dev/null
+++ b/tools/docker/testcontrol/auto/controller/vsperf/vsperf_controller.py
@@ -0,0 +1,469 @@
+# Copyright 2018-19 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+VSPERF-controller
+"""
+
+# Fetching Environment Variable for controller, You can configure or
+# modifies list.env file for setting your environment variable.
+
+#pylint: disable=global-statement,no-else-continue
+#pylint: disable=too-many-branches
+
+import os
+import time
+import math
+import ast
+from utils import ssh
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+TIMER = float()
+
+
+DUT_IP = os.getenv('DUT_IP_ADDRESS')
+DUT_USER = os.getenv('DUT_USERNAME')
+DUT_PWD = os.getenv('DUT_PASSWORD')
+
+TGEN_IP = os.getenv('TGEN_IP_ADDRESS')
+
+VSPERF_TEST = os.getenv('VSPERF_TESTS')
+VSPERF_CONF = os.getenv('VSPERF_CONFFILE')
+VSPERF_TRAFFICGEN_MODE = str(os.getenv('VSPERF_TRAFFICGEN_MODE'))
+
+START_COLLECTD = os.getenv('START_COLLECTD')
+START_BEATS = os.getenv('START_BEATS')
+CLEAN_UP = os.getenv('CLEAN_UP')
+
+DUT_CLIENT = None
+TGEN_CLIENT = None
+SANITY_CHECK_DONE_LIST = list()
+
+
+def host_connect():
+    """
+    Handle host connectivity to DUT
+    """
+    global DUT_CLIENT
+    DUT_CLIENT = ssh.SSH(host=DUT_IP, user=DUT_USER, password=DUT_PWD)
+    print("DUT Successfully Connected ..............................................[OK] \n ")
+
+def upload_test_config_file():
+    """
+    #Upload Test Config File on DUT
+    """
+    localpath = '/usr/src/app/vsperf/vsperf.conf'
+    if not os.path.exists(localpath):
+        print("VSPERF Test config File does not exists.......................[Failed]")
+        return
+    remotepath = '~/vsperf.conf'
+    check_test_config_cmd = "find ~/ -maxdepth 1 -name '{}'".format(
+        remotepath[2:])
+    check_test_result = str(DUT_CLIENT.execute(check_test_config_cmd)[1])
+    if remotepath[2:] in check_test_result:
+        DUT_CLIENT.run("rm -f {}".format(remotepath[2:]))
+    DUT_CLIENT.put_file(localpath, remotepath)
+    check_test_config_cmd_1= "find ~/ -maxdepth 1 -name '{}'".format(
+        remotepath[2:])
+    check_test_result_1= str(DUT_CLIENT.execute(check_test_config_cmd)[1])
+    if remotepath[2:] in check_test_result_1:
+    	print(
+        "Test Configuration File Uploaded on DUT-Host.............................[OK] \n ")
+    else:
+    	print("VSPERF Test config file upload failed.....................................[Critical]")
+
+def start_beats():
+    """
+    Start fileBeats on DUT
+    """
+    run_cmd = "echo '{}' | sudo -S service filebeat start".format(DUT_PWD)
+    DUT_CLIENT.run(run_cmd, pty=True)
+    print(
+        "Beats are started on DUT-Host............................................[OK] \n")
+
+def start_collectd():
+    """
+    start the collectd
+    """
+    run_cmd = "echo '{}' | sudo -S service collectd start".format(DUT_PWD)
+    DUT_CLIENT.run(run_cmd, pty=True)
+    print(
+        "Collectd is started on DUT-Host............................................[OK] \n")
+
+def run_vsperf_test():
+    """
+    Here we will perform the actual vsperf test
+    """
+    global TIMER
+    rmv_cmd = "cd /mnt/huge && echo {} | sudo -S rm -rf *".format(DUT_PWD)
+    DUT_CLIENT.run(rmv_cmd, pty=True)
+    cmd = "source ~/vsperfenv/bin/activate ; "
+    #cmd = "scl enable python33 bash ; "
+    cmd += "cd vswitchperf && "
+    cmd += "./vsperf "
+    if VSPERF_CONF:
+        cmd += "--conf-file ~/vsperf.conf "
+    if "yes" in VSPERF_TRAFFICGEN_MODE.lower():
+        cmd += "--mode trafficgen"
+    vsperf_test_list = VSPERF_TEST.split(",")
+    print(vsperf_test_list)
+    for test in vsperf_test_list:
+        atest = cmd
+        atest += test
+        DUT_CLIENT.run(atest, pty=True)
+    print(
+        "Test Successfully running................................................[OK]\n ")
+
+
+def test_status():
+    """
+    Chechk for the test status after performing test
+    """
+    testtype_list = VSPERF_TEST.split(",")
+    num_test = len(testtype_list)
+    test_success = []
+    test_failed = []
+    testtype_list_len = len(testtype_list)
+    for test in testtype_list:
+        passed_minutes = 5
+        latest_result_cmd = "find /tmp -mindepth 1 -type d -cmin -{} -printf '%f'".format(
+            passed_minutes)
+        test_result_dir = str(
+            (DUT_CLIENT.execute(latest_result_cmd)[1]).split('find')[0])
+        test_date_cmd = "date +%F"
+        test_date = str(DUT_CLIENT.execute(test_date_cmd)[1]).replace("\n", "")
+        if test_date in test_result_dir:
+            testcase_check_cmd = "cd /tmp && cd `ls -t | grep results | head"
+            testcase_check_cmd += " -{} | tail -1` && find . -maxdepth 1 -name '*{}*'".\
+                                  format(testtype_list_len, test)
+            testcase_check_output = str(
+                DUT_CLIENT.execute(testcase_check_cmd)[1]).split('\n', 2)
+            check = 0
+            for i in testcase_check_output:
+                if (".csv" in i) or (".md" in i) or (".rst" in i):
+                    check += 1
+            if check == 3:
+                test_success.append(test)
+            else:
+                test_failed.append(test)
+            testtype_list_len -= 1
+    if num_test == len(test_success):
+        print("All Test Successfully Completed on DUT-Host   Results... [OK]")
+    elif not test_success:
+        print("All Test Failed on DUT-Host \nResults... [Failed]")
+    else:
+        print(
+            "Only {} Test failed    Results ... [Failed]\n"\
+            "All other Test Successfully Completed on DUT-Host     Results... [OK] ".\
+            format(test_failed))
+
+
+def vsperf_remove():
+    """
+    Actual removal of the VSPERF
+    """
+    vsperf_rm_cmd = "echo '{}' | sudo -S rm -r ~/vswitchperf".format(DUT_PWD)
+    DUT_CLIENT.run(vsperf_rm_cmd)
+    vsperfenv_rm_cmd = "echo '{}' | sudo -S rm -r -f ~/vsperfenv".\
+                        format(DUT_PWD)
+    DUT_CLIENT.run(vsperfenv_rm_cmd)
+
+
+def remove_uploaded_config():
+    """
+    Remove all the uploaded configuration files
+    """
+    vconfig_rm_cmd = "rm ~/vsperf.conf"
+    DUT_CLIENT.run(vconfig_rm_cmd)
+    cdconfig_rm_cmd = "echo '{}' | sudo -S rm /opt/collectd/etc/collectd.conf".\
+                       format(DUT_PWD)
+    DUT_CLIENT.run(cdconfig_rm_cmd)
+
+
+def result_folders_remove():
+    """
+    Remove result folder on DUT
+    """
+    remove_cmd = "rm -r /tmp/*results*"
+    DUT_CLIENT.run(remove_cmd)
+
+
+def collectd_remove():
+    """
+    Remove collectd from DUT
+    """
+    collectd_dwn_rm_cmd = "echo '{}' | sudo -S rm -r -f ~/collectd".format(
+        DUT_PWD)
+    DUT_CLIENT.run(collectd_dwn_rm_cmd)
+    collectd_rm_cmd = "echo '{}' | sudo -S rm -r -f /opt/collectd".format(
+        DUT_PWD)
+    DUT_CLIENT.run(collectd_rm_cmd)
+
+
+def terminate_vsperf():
+    """
+    Terminate the VSPERF and kill processes
+    """
+    stress_kill_cmd = "echo '{}' | sudo -S pkill stress &> /dev/null".format(
+        DUT_PWD)
+    python3_kill_cmd = "echo '{}' | sudo -S pkill python3 &> /dev/null".format(
+        DUT_PWD)
+    qemu_kill_cmd = "echo '{}' | sudo -S killall -9 qemu-system-x86_64 &> /dev/null".format(
+        DUT_PWD)
+    DUT_CLIENT.run(stress_kill_cmd)
+    DUT_CLIENT.run(python3_kill_cmd)
+    DUT_CLIENT.run(qemu_kill_cmd)
+
+    # sometimes qemu resists to terminate, so wait a bit and kill it again
+    qemu_check_cmd = "pgrep qemu-system-x86_64"
+    qemu_cmd_response = DUT_CLIENT.execute(qemu_check_cmd)[1]
+
+    if qemu_cmd_response != '':
+        time.sleep(5)
+        DUT_CLIENT.run(qemu_kill_cmd)
+        time.sleep(5)
+
+    ovs_kill_cmd = "echo '{}' | sudo pkill ovs-vswitchd &> /dev/null".format(
+        DUT_PWD)
+    ovsdb_kill_cmd = "echo '{}' | sudo pkill ovsdb-server &> /dev/null".format(
+        DUT_PWD)
+    vppctl_kill_cmd = "echo '{}' | sudo pkill vppctl &> /dev/null".format(
+        DUT_PWD)
+    vpp_kill_cmd = "echo '{}' | sudo pkill vpp &> /dev/null".format(DUT_PWD)
+    vpp_cmd = "echo '{}' | sudo pkill -9 vpp &> /dev/null".format(DUT_PWD)
+
+    DUT_CLIENT.run(ovs_kill_cmd)
+    time.sleep(1)
+    DUT_CLIENT.run(ovsdb_kill_cmd)
+    time.sleep(1)
+    DUT_CLIENT.run(vppctl_kill_cmd)
+    time.sleep(1)
+    DUT_CLIENT.run(vpp_kill_cmd)
+    time.sleep(1)
+    DUT_CLIENT.run(vpp_cmd)
+    time.sleep(1)
+
+    print(
+        "All the VSPERF related process terminated successfully..............[OK]")
+
+
+def sanity_collectd_check():
+    """
+    Check and verify collectd is able to run and start properly
+    """
+    global SANITY_CHECK_DONE_LIST
+    check_collectd_cmd = "find /opt -maxdepth 1 -name 'collectd'"
+    check_test_result = str(DUT_CLIENT.execute(check_collectd_cmd)[1])
+    if "collectd" in check_test_result:
+        check_collectd_run_cmd = "echo {} | sudo -S service collectd start".format(
+            DUT_PWD)
+        DUT_CLIENT.run(check_collectd_run_cmd, pty=True)
+        check_collectd_status_cmd = "ps aux | grep collectd"
+        check_collectd_status = str(
+            DUT_CLIENT.execute(check_collectd_status_cmd)[1])
+        if "/sbin/collectd" in check_collectd_status:
+            SANITY_CHECK_DONE_LIST.append(int(1))
+            print(
+                "Collectd is working Fine ................................................[OK] \n ")
+        else:
+            print(
+                "Collectd Fail to Start, Install correctly before running Test....[Failed]\n ")
+    else:
+        print(
+            "Collectd is not installed yet........................................[Failed]\n")
+
+
+def sanity_vnf_path():
+    """
+    Check if VNF image is available on the configured path in Test Config File
+    """
+    # fetch the VNF path we placed in vsperf.conf file
+    global SANITY_CHECK_DONE_LIST
+    vsperf_conf_path = open('/usr/src/app/vsperf/vsperf.conf')
+    vsperf_conf_read = vsperf_conf_path.readlines()
+    for i in vsperf_conf_read:
+        if 'GUEST_IMAGE' in i:
+            vnf_image_path = i.split("'")[1]
+            vnf_path_check_cmd = "find {}".format(vnf_image_path)
+            vnf_path_check_result = str(
+                DUT_CLIENT.execute(vnf_path_check_cmd)[1])
+            if vnf_image_path in vnf_path_check_result:
+                SANITY_CHECK_DONE_LIST.append(int(2))
+                print(
+                    "Test Configratuion file has Correct VNF path information on DUT-Host.." \
+                    "...[OK]\n ")
+            else:
+                print(
+                    "Test Configuration file has incorrect VNF path information......" \
+                    "....[FAILED]\n")
+
+def sanity_vsperf_check():
+    """
+    We have to make sure that VSPERF is installed correctly
+    """
+    global SANITY_CHECK_DONE_LIST
+    vsperf_check_command = "source ~/vsperfenv/bin/activate ; cd vswitchperf* && ./vsperf --help"
+    vsperf_check_cmd_result = str(DUT_CLIENT.execute(vsperf_check_command)[1])
+    vsperf_verify_list = [
+        'usage',
+        'positional arguments',
+        'optional arguments',
+        'test selection options',
+        'test behavior options']
+    for idx, i in enumerate(vsperf_verify_list, start=1):
+        if str(i) in vsperf_check_cmd_result:
+            if idx < 5:
+                continue
+            elif idx == 5:
+                SANITY_CHECK_DONE_LIST.append(int(3))
+                print("VSPERF Installed Correctly and Working fine......................." \
+                    ".......[OK]\n")
+            else:
+                print(
+                    "VSPERF DID Not Installed Correctly , INSTALL IT AGAIN...........[Critical]\n")
+        else:
+            print(
+                "VSPERF DID Not Installed Correctly , INSTALL IT AGAIN................[Critical]\n")
+            break
+
+def variable_from_test_config(aparameter):
+    """This function can be use to read any configuration paramter from vsperf.conf"""
+    read_cmd = 'cat ~/vsperf.conf | grep "{}"'.format(aparameter)
+    read_cmd_output = str(DUT_CLIENT.execute(read_cmd)[1])
+    print(read_cmd_output)
+    if not read_cmd_output or '#' in read_cmd_output:
+        return 0
+    return read_cmd_output.split("=")[1].strip()
+
+def cpumask2coreids(mask):
+    """conver mask to coreids"""
+    intmask = int(mask, 16)
+    i = 1
+    coreids = []
+    while i < intmask:
+        if i & intmask:
+            coreids.append(str(math.frexp(i)[1]-1))
+        i = i << 1
+    return coreids
+
+def sanity_cpu_allocation_check():
+    """It will check the cpu allocation before run test"""
+    global SANITY_CHECK_DONE_LIST
+    read_setting_cmd = "source vsperfenv/bin/activate ; cd vswitchperf* && "
+    read_setting_cmd += './vsperf --list-settings'
+    default_vsperf_settings = ast.literal_eval(str(DUT_CLIENT.execute(read_setting_cmd)[1]))
+    default_cpu_map = default_vsperf_settings["VSWITCH_VHOST_CPU_MAP"]
+    default_vswitch_pmd_cpu_mask = str(default_vsperf_settings["VSWITCH_PMD_CPU_MASK"])
+    default_vswitch_vhost_cpu_map = [str(x) for x in default_cpu_map]
+    vswitch_pmd_cpu_mask = variable_from_test_config("VSWITCH_PMD_CPU_MASK")
+    vswitch_cpu_map = (variable_from_test_config("VSWITCH_VHOST_CPU_MAP"))
+    vswitch_vhost_cpu_map = 0
+    if vswitch_cpu_map != 0:
+        vswitch_vhost_cpu_map = [str(x) for x in  ast.literal_eval(vswitch_cpu_map)]
+
+    if vswitch_pmd_cpu_mask == 0 and vswitch_vhost_cpu_map == 0:
+        print("CPU allocation Check Done,"\
+            "\nNo vswitch_pmd_cpu_mask or vswitch_vhost_cpu_map assign in test config file\n" \
+            "Using Default Settings ..................................................[OK]\n")
+    elif vswitch_pmd_cpu_mask != 0 and vswitch_vhost_cpu_map == 0:
+        core_id = cpumask2coreids(vswitch_pmd_cpu_mask)
+        print(core_id)
+        if len(default_vswitch_vhost_cpu_map) >= len(core_id):
+            if all(elem in default_vswitch_vhost_cpu_map  for elem in core_id):
+                print("CPU allocation properly done on DUT-Host.................[OK]\n")
+            else:
+                print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+        else:
+            print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+    elif vswitch_pmd_cpu_mask == 0 and vswitch_vhost_cpu_map != 0:
+        core_id_1 = cpumask2coreids(default_vswitch_pmd_cpu_mask)
+        print(core_id_1)
+        if len(vswitch_vhost_cpu_map) >= len(core_id_1):
+            if all(elem in vswitch_vhost_cpu_map  for elem in core_id_1):
+                print("CPU allocation properly done on DUT-Host.................[OK]\n")
+            else:
+                print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+        else:
+            print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+    else:
+        core_id_2 = cpumask2coreids(vswitch_pmd_cpu_mask)
+        print(core_id_2)
+        if len(vswitch_vhost_cpu_map) >= len(core_id_2):
+            if all(elem in vswitch_vhost_cpu_map  for elem in core_id_2):
+                print("CPU allocation properly done on DUT-Host.................[OK]\n")
+            else:
+                print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+        else:
+            print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+
+
+
+def sanity_dut_conn_tgen_check():
+    """
+    We should confirm the DUT connectivity with the Tgen and Traffic Generator is working or not
+    """
+    global SANITY_CHECK_DONE_LIST
+    tgen_connectivity_check_cmd = "ping {} -c 1".format(TGEN_IP)
+    tgen_connectivity_check_result = int(DUT_CLIENT.execute(tgen_connectivity_check_cmd)[0])
+    if tgen_connectivity_check_result == 0:
+        SANITY_CHECK_DONE_LIST.append(int(5))
+        print(
+            "DUT-Host is successfully reachable to Traffic Generator Host.............[OK]\n")
+    else:
+        print(
+            "DUT-host is unsuccessful to reach the Traffic Generator Host.............[Failed]")
+        print(
+            "Make sure to establish connection before running Test...............[Critical]\n")
+
+if DUT_IP:
+    host_connect()
+if not DUT_CLIENT:
+    print('Failed to connect to DUT ...............[Critical]')
+    sys.exit()
+else:
+    upload_test_config_file()
+    sanity_vnf_path()
+    sanity_cpu_allocation_check()
+    sanity_collectd_check()
+    sanity_vsperf_check()
+    sanity_dut_conn_tgen_check()
+    if "yes" in START_COLLECTD.lower():
+        start_collectd()
+    if "yes" in START_BEATS.lower():
+        start_beats()
+
+if 'v' in VSPERF_TEST:
+    if len(SANITY_CHECK_DONE_LIST) != 4:
+        print("Certain Sanity Checks Failed\n" \
+              "You can make changes based on the outputs and run" \
+              "the testcontrol auto container again")
+    else:
+        run_vsperf_test()
+        test_status()
+else:
+    if len(SANITY_CHECK_DONE_LIST) != 3:
+        print("Certain Sanity Checks Failed\n" \
+              "You can make changes based on the outputs and run" \
+              "the testcontrol auto container again")
+    else:
+        run_vsperf_test()
+        test_status()
+
+
+if "yes" in CLEAN_UP.lower():
+    vsperf_remove()
+    remove_uploaded_config()
+    result_folders_remove()
+    collectd_remove()
+    terminate_vsperf()