Tools: Deployment and TestControl Containers

This patch add containerization of VSPERF support.
The patch facilitates creation of 4 containers:
1. Interactive Deployment
2. Auto Deployment
3. Interactive TestControl
4. Auto TestControl.
The patch also includes a minimal client to work with interactive
containers.
The docs folder provides detailed documentation.
Fixed pylint errors in libs folder.
Removed proto built python files, and added the build process in
prepare.sh.
Stability improvements for Auto versions of deployment and testcontrol.
Enhance client with 'mode' feature, where client can run either to do
only deploy/only test or both.
Add sample configuration file for client
Fixed few typos - as suggested by AL.

JIRA: VSPERF-594
Signed-off-by: Sridhar K. N. Rao <sridhar.rao@spirent.com>
Change-Id: Id40b02960f71a7f9183d9a53955e2483117fb9e2

11 files changed, 1299 insertions, 0 deletions

diff --git a/tools/docker/testcontrol/auto/controller/Dockerfile b/tools/docker/testcontrol/auto/controller/Dockerfile
new file mode 100644
index 00000000..4fbf7294
--- /dev/null
+++ b/tools/docker/testcontrol/auto/controller/Dockerfile
@@ -0,0 +1,23 @@
+FROM python:3.6
+LABEL maintainer="sridhar.rao@spirent.com"
+
+ENV GRPC_PYTHON_VERSION 1.4.0
+RUN apt-get update && apt-get -y install python3-pip && apt-get -y install openssh-server
+RUN pip3 install grpcio==${GRPC_PYTHON_VERSION} grpcio-tools==${GRPC_PYTHON_VERSION}
+RUN pip3 install paramiko
+RUN pip3 install chainmap
+RUN pip3 install oslo.utils
+RUN pip3 install scp
+
+WORKDIR /usr/src/app
+
+COPY ./vsperf ./vsperf
+
+VOLUME ["/usr/src/app/vsperf"]
+
+EXPOSE 50052
+
+CMD ["python3", "./vsperf/vsperf_controller.py"]
+
+#CMD tail -f /dev/null
+
diff --git a/tools/docker/testcontrol/auto/controller/list.env b/tools/docker/testcontrol/auto/controller/list.env
new file mode 100644
index 00000000..2883021b
--- /dev/null
+++ b/tools/docker/testcontrol/auto/controller/list.env
@@ -0,0 +1,13 @@
+DUT_IP_ADDRESS=10.10.120.24
+DUT_USERNAME=opnfv
+DUT_PASSWORD=opnfv
+
+TGEN_IP_ADDRESS=10.10.120.25
+
+VSPERF_TESTS=phy2phy_tput,pvp_tput
+VSPERF_CONFFILE=vsperf.conf
+
+VSPERF_TRAFFICGEN_MODE=NO
+
+CLEAN_UP=NO
+
diff --git a/tools/docker/testcontrol/auto/controller/vsperf/__init__.py b/tools/docker/testcontrol/auto/controller/vsperf/__init__.py
new file mode 100644
index 00000000..ad0ebec3
--- /dev/null
+++ b/tools/docker/testcontrol/auto/controller/vsperf/__init__.py
@@ -0,0 +1 @@
+#### Empty
diff --git a/tools/docker/testcontrol/auto/controller/vsperf/vsperf.conf b/tools/docker/testcontrol/auto/controller/vsperf/vsperf.conf
new file mode 100644
index 00000000..50d40f49
--- /dev/null
+++ b/tools/docker/testcontrol/auto/controller/vsperf/vsperf.conf
@@ -0,0 +1,21 @@
+VSWITCH_BRIDGE_NAME = 'vsperf-br0'
+WHITELIST_NICS = ['02:00.0', '02:00.1']
+TRAFFICGEN = 'Trex'
+TRAFFICGEN_TREX_HOST_IP_ADDR = '10.10.120.25'
+TRAFFICGEN_TREX_USER = 'root'
+TRAFFICGEN_TREX_BASE_DIR = '/root/trex_2.37/scripts/'
+TRAFFICGEN_TREX_LINE_SPEED_GBPS = '10'
+TRAFFICGEN_TREX_PORT1 = '0000:81:00.0'
+TRAFFICGEN_TREX_PORT2 = '0000:81:00.1'
+TRAFFICGEN_TREX_PROMISCUOUS = False
+TRAFFICGEN_DURATION=1
+TRAFFICGEN_LOSSRATE=0
+TRAFFICGEN_RFC2544_TESTS=10
+#TRAFFICGEN_PKT_SIZES=(64,128,256,512,1024,1280,1518)
+TRAFFICGEN_PKT_SIZES=(64,)
+GUEST_TESTPMD_FWD_MODE = ['io']
+GUEST_IMAGE = ['/home/opnfv/vnfs/vloop-vnf-ubuntu-18.04_20180920.qcow2']
+TRAFFICGEN_TREX_LATENCY_PPS = 1000
+TRAFFICGEN_TREX_RFC2544_BINARY_SEARCH_LOSS_VERIFICATION = True
+TRAFFICGEN_TREX_RFC2544_MAX_REPEAT = 2
+
diff --git a/tools/docker/testcontrol/auto/controller/vsperf/vsperf_controller.py b/tools/docker/testcontrol/auto/controller/vsperf/vsperf_controller.py
new file mode 100644
index 00000000..1b088fea
--- /dev/null
+++ b/tools/docker/testcontrol/auto/controller/vsperf/vsperf_controller.py
@@ -0,0 +1,469 @@
+# Copyright 2018-19 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+VSPERF-controller
+"""
+
+# Fetching Environment Variable for controller, You can configure or
+# modifies list.env file for setting your environment variable.
+
+#pylint: disable=global-statement,no-else-continue
+#pylint: disable=too-many-branches
+
+import os
+import time
+import math
+import ast
+from utils import ssh
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+TIMER = float()
+
+
+DUT_IP = os.getenv('DUT_IP_ADDRESS')
+DUT_USER = os.getenv('DUT_USERNAME')
+DUT_PWD = os.getenv('DUT_PASSWORD')
+
+TGEN_IP = os.getenv('TGEN_IP_ADDRESS')
+
+VSPERF_TEST = os.getenv('VSPERF_TESTS')
+VSPERF_CONF = os.getenv('VSPERF_CONFFILE')
+VSPERF_TRAFFICGEN_MODE = str(os.getenv('VSPERF_TRAFFICGEN_MODE'))
+
+START_COLLECTD = os.getenv('START_COLLECTD')
+START_BEATS = os.getenv('START_BEATS')
+CLEAN_UP = os.getenv('CLEAN_UP')
+
+DUT_CLIENT = None
+TGEN_CLIENT = None
+SANITY_CHECK_DONE_LIST = list()
+
+
+def host_connect():
+    """
+    Handle host connectivity to DUT
+    """
+    global DUT_CLIENT
+    DUT_CLIENT = ssh.SSH(host=DUT_IP, user=DUT_USER, password=DUT_PWD)
+    print("DUT Successfully Connected ..............................................[OK] \n ")
+
+def upload_test_config_file():
+    """
+    #Upload Test Config File on DUT
+    """
+    localpath = '/usr/src/app/vsperf/vsperf.conf'
+    if not os.path.exists(localpath):
+        print("VSPERF Test config File does not exists.......................[Failed]")
+        return
+    remotepath = '~/vsperf.conf'
+    check_test_config_cmd = "find ~/ -maxdepth 1 -name '{}'".format(
+        remotepath[2:])
+    check_test_result = str(DUT_CLIENT.execute(check_test_config_cmd)[1])
+    if remotepath[2:] in check_test_result:
+        DUT_CLIENT.run("rm -f {}".format(remotepath[2:]))
+    DUT_CLIENT.put_file(localpath, remotepath)
+    check_test_config_cmd_1= "find ~/ -maxdepth 1 -name '{}'".format(
+        remotepath[2:])
+    check_test_result_1= str(DUT_CLIENT.execute(check_test_config_cmd)[1])
+    if remotepath[2:] in check_test_result_1:
+    	print(
+        "Test Configuration File Uploaded on DUT-Host.............................[OK] \n ")
+    else:
+    	print("VSPERF Test config file upload failed.....................................[Critical]")
+
+def start_beats():
+    """
+    Start fileBeats on DUT
+    """
+    run_cmd = "echo '{}' | sudo -S service filebeat start".format(DUT_PWD)
+    DUT_CLIENT.run(run_cmd, pty=True)
+    print(
+        "Beats are started on DUT-Host............................................[OK] \n")
+
+def start_collectd():
+    """
+    start the collectd
+    """
+    run_cmd = "echo '{}' | sudo -S service collectd start".format(DUT_PWD)
+    DUT_CLIENT.run(run_cmd, pty=True)
+    print(
+        "Collectd is started on DUT-Host............................................[OK] \n")
+
+def run_vsperf_test():
+    """
+    Here we will perform the actual vsperf test
+    """
+    global TIMER
+    rmv_cmd = "cd /mnt/huge && echo {} | sudo -S rm -rf *".format(DUT_PWD)
+    DUT_CLIENT.run(rmv_cmd, pty=True)
+    cmd = "source ~/vsperfenv/bin/activate ; "
+    #cmd = "scl enable python33 bash ; "
+    cmd += "cd vswitchperf && "
+    cmd += "./vsperf "
+    if VSPERF_CONF:
+        cmd += "--conf-file ~/vsperf.conf "
+    if "yes" in VSPERF_TRAFFICGEN_MODE.lower():
+        cmd += "--mode trafficgen"
+    vsperf_test_list = VSPERF_TEST.split(",")
+    print(vsperf_test_list)
+    for test in vsperf_test_list:
+        atest = cmd
+        atest += test
+        DUT_CLIENT.run(atest, pty=True)
+    print(
+        "Test Successfully running................................................[OK]\n ")
+
+
+def test_status():
+    """
+    Chechk for the test status after performing test
+    """
+    testtype_list = VSPERF_TEST.split(",")
+    num_test = len(testtype_list)
+    test_success = []
+    test_failed = []
+    testtype_list_len = len(testtype_list)
+    for test in testtype_list:
+        passed_minutes = 5
+        latest_result_cmd = "find /tmp -mindepth 1 -type d -cmin -{} -printf '%f'".format(
+            passed_minutes)
+        test_result_dir = str(
+            (DUT_CLIENT.execute(latest_result_cmd)[1]).split('find')[0])
+        test_date_cmd = "date +%F"
+        test_date = str(DUT_CLIENT.execute(test_date_cmd)[1]).replace("\n", "")
+        if test_date in test_result_dir:
+            testcase_check_cmd = "cd /tmp && cd `ls -t | grep results | head"
+            testcase_check_cmd += " -{} | tail -1` && find . -maxdepth 1 -name '*{}*'".\
+                                  format(testtype_list_len, test)
+            testcase_check_output = str(
+                DUT_CLIENT.execute(testcase_check_cmd)[1]).split('\n', 2)
+            check = 0
+            for i in testcase_check_output:
+                if (".csv" in i) or (".md" in i) or (".rst" in i):
+                    check += 1
+            if check == 3:
+                test_success.append(test)
+            else:
+                test_failed.append(test)
+            testtype_list_len -= 1
+    if num_test == len(test_success):
+        print("All Test Successfully Completed on DUT-Host   Results... [OK]")
+    elif not test_success:
+        print("All Test Failed on DUT-Host \nResults... [Failed]")
+    else:
+        print(
+            "Only {} Test failed    Results ... [Failed]\n"\
+            "All other Test Successfully Completed on DUT-Host     Results... [OK] ".\
+            format(test_failed))
+
+
+def vsperf_remove():
+    """
+    Actual removal of the VSPERF
+    """
+    vsperf_rm_cmd = "echo '{}' | sudo -S rm -r ~/vswitchperf".format(DUT_PWD)
+    DUT_CLIENT.run(vsperf_rm_cmd)
+    vsperfenv_rm_cmd = "echo '{}' | sudo -S rm -r -f ~/vsperfenv".\
+                        format(DUT_PWD)
+    DUT_CLIENT.run(vsperfenv_rm_cmd)
+
+
+def remove_uploaded_config():
+    """
+    Remove all the uploaded configuration files
+    """
+    vconfig_rm_cmd = "rm ~/vsperf.conf"
+    DUT_CLIENT.run(vconfig_rm_cmd)
+    cdconfig_rm_cmd = "echo '{}' | sudo -S rm /opt/collectd/etc/collectd.conf".\
+                       format(DUT_PWD)
+    DUT_CLIENT.run(cdconfig_rm_cmd)
+
+
+def result_folders_remove():
+    """
+    Remove result folder on DUT
+    """
+    remove_cmd = "rm -r /tmp/*results*"
+    DUT_CLIENT.run(remove_cmd)
+
+
+def collectd_remove():
+    """
+    Remove collectd from DUT
+    """
+    collectd_dwn_rm_cmd = "echo '{}' | sudo -S rm -r -f ~/collectd".format(
+        DUT_PWD)
+    DUT_CLIENT.run(collectd_dwn_rm_cmd)
+    collectd_rm_cmd = "echo '{}' | sudo -S rm -r -f /opt/collectd".format(
+        DUT_PWD)
+    DUT_CLIENT.run(collectd_rm_cmd)
+
+
+def terminate_vsperf():
+    """
+    Terminate the VSPERF and kill processes
+    """
+    stress_kill_cmd = "echo '{}' | sudo -S pkill stress &> /dev/null".format(
+        DUT_PWD)
+    python3_kill_cmd = "echo '{}' | sudo -S pkill python3 &> /dev/null".format(
+        DUT_PWD)
+    qemu_kill_cmd = "echo '{}' | sudo -S killall -9 qemu-system-x86_64 &> /dev/null".format(
+        DUT_PWD)
+    DUT_CLIENT.run(stress_kill_cmd)
+    DUT_CLIENT.run(python3_kill_cmd)
+    DUT_CLIENT.run(qemu_kill_cmd)
+
+    # sometimes qemu resists to terminate, so wait a bit and kill it again
+    qemu_check_cmd = "pgrep qemu-system-x86_64"
+    qemu_cmd_response = DUT_CLIENT.execute(qemu_check_cmd)[1]
+
+    if qemu_cmd_response != '':
+        time.sleep(5)
+        DUT_CLIENT.run(qemu_kill_cmd)
+        time.sleep(5)
+
+    ovs_kill_cmd = "echo '{}' | sudo pkill ovs-vswitchd &> /dev/null".format(
+        DUT_PWD)
+    ovsdb_kill_cmd = "echo '{}' | sudo pkill ovsdb-server &> /dev/null".format(
+        DUT_PWD)
+    vppctl_kill_cmd = "echo '{}' | sudo pkill vppctl &> /dev/null".format(
+        DUT_PWD)
+    vpp_kill_cmd = "echo '{}' | sudo pkill vpp &> /dev/null".format(DUT_PWD)
+    vpp_cmd = "echo '{}' | sudo pkill -9 vpp &> /dev/null".format(DUT_PWD)
+
+    DUT_CLIENT.run(ovs_kill_cmd)
+    time.sleep(1)
+    DUT_CLIENT.run(ovsdb_kill_cmd)
+    time.sleep(1)
+    DUT_CLIENT.run(vppctl_kill_cmd)
+    time.sleep(1)
+    DUT_CLIENT.run(vpp_kill_cmd)
+    time.sleep(1)
+    DUT_CLIENT.run(vpp_cmd)
+    time.sleep(1)
+
+    print(
+        "All the VSPERF related process terminated successfully..............[OK]")
+
+
+def sanity_collectd_check():
+    """
+    Check and verify collectd is able to run and start properly
+    """
+    global SANITY_CHECK_DONE_LIST
+    check_collectd_cmd = "find /opt -maxdepth 1 -name 'collectd'"
+    check_test_result = str(DUT_CLIENT.execute(check_collectd_cmd)[1])
+    if "collectd" in check_test_result:
+        check_collectd_run_cmd = "echo {} | sudo -S service collectd start".format(
+            DUT_PWD)
+        DUT_CLIENT.run(check_collectd_run_cmd, pty=True)
+        check_collectd_status_cmd = "ps aux | grep collectd"
+        check_collectd_status = str(
+            DUT_CLIENT.execute(check_collectd_status_cmd)[1])
+        if "/sbin/collectd" in check_collectd_status:
+            SANITY_CHECK_DONE_LIST.append(int(1))
+            print(
+                "Collectd is working Fine ................................................[OK] \n ")
+        else:
+            print(
+                "Collectd Fail to Start, Install correctly before running Test....[Failed]\n ")
+    else:
+        print(
+            "Collectd is not installed yet........................................[Failed]\n")
+
+
+def sanity_vnf_path():
+    """
+    Check if VNF image is available on the configured path in Test Config File
+    """
+    # fetch the VNF path we placed in vsperf.conf file
+    global SANITY_CHECK_DONE_LIST
+    vsperf_conf_path = open('/usr/src/app/vsperf/vsperf.conf')
+    vsperf_conf_read = vsperf_conf_path.readlines()
+    for i in vsperf_conf_read:
+        if 'GUEST_IMAGE' in i:
+            vnf_image_path = i.split("'")[1]
+            vnf_path_check_cmd = "find {}".format(vnf_image_path)
+            vnf_path_check_result = str(
+                DUT_CLIENT.execute(vnf_path_check_cmd)[1])
+            if vnf_image_path in vnf_path_check_result:
+                SANITY_CHECK_DONE_LIST.append(int(2))
+                print(
+                    "Test Configratuion file has Correct VNF path information on DUT-Host.." \
+                    "...[OK]\n ")
+            else:
+                print(
+                    "Test Configuration file has incorrect VNF path information......" \
+                    "....[FAILED]\n")
+
+def sanity_vsperf_check():
+    """
+    We have to make sure that VSPERF is installed correctly
+    """
+    global SANITY_CHECK_DONE_LIST
+    vsperf_check_command = "source ~/vsperfenv/bin/activate ; cd vswitchperf* && ./vsperf --help"
+    vsperf_check_cmd_result = str(DUT_CLIENT.execute(vsperf_check_command)[1])
+    vsperf_verify_list = [
+        'usage',
+        'positional arguments',
+        'optional arguments',
+        'test selection options',
+        'test behavior options']
+    for idx, i in enumerate(vsperf_verify_list, start=1):
+        if str(i) in vsperf_check_cmd_result:
+            if idx < 5:
+                continue
+            elif idx == 5:
+                SANITY_CHECK_DONE_LIST.append(int(3))
+                print("VSPERF Installed Correctly and Working fine......................." \
+                    ".......[OK]\n")
+            else:
+                print(
+                    "VSPERF DID Not Installed Correctly , INSTALL IT AGAIN...........[Critical]\n")
+        else:
+            print(
+                "VSPERF DID Not Installed Correctly , INSTALL IT AGAIN................[Critical]\n")
+            break
+
+def variable_from_test_config(aparameter):
+    """This function can be use to read any configuration paramter from vsperf.conf"""
+    read_cmd = 'cat ~/vsperf.conf | grep "{}"'.format(aparameter)
+    read_cmd_output = str(DUT_CLIENT.execute(read_cmd)[1])
+    print(read_cmd_output)
+    if not read_cmd_output or '#' in read_cmd_output:
+        return 0
+    return read_cmd_output.split("=")[1].strip()
+
+def cpumask2coreids(mask):
+    """conver mask to coreids"""
+    intmask = int(mask, 16)
+    i = 1
+    coreids = []
+    while i < intmask:
+        if i & intmask:
+            coreids.append(str(math.frexp(i)[1]-1))
+        i = i << 1
+    return coreids
+
+def sanity_cpu_allocation_check():
+    """It will check the cpu allocation before run test"""
+    global SANITY_CHECK_DONE_LIST
+    read_setting_cmd = "source vsperfenv/bin/activate ; cd vswitchperf* && "
+    read_setting_cmd += './vsperf --list-settings'
+    default_vsperf_settings = ast.literal_eval(str(DUT_CLIENT.execute(read_setting_cmd)[1]))
+    default_cpu_map = default_vsperf_settings["VSWITCH_VHOST_CPU_MAP"]
+    default_vswitch_pmd_cpu_mask = str(default_vsperf_settings["VSWITCH_PMD_CPU_MASK"])
+    default_vswitch_vhost_cpu_map = [str(x) for x in default_cpu_map]
+    vswitch_pmd_cpu_mask = variable_from_test_config("VSWITCH_PMD_CPU_MASK")
+    vswitch_cpu_map = (variable_from_test_config("VSWITCH_VHOST_CPU_MAP"))
+    vswitch_vhost_cpu_map = 0
+    if vswitch_cpu_map != 0:
+        vswitch_vhost_cpu_map = [str(x) for x in  ast.literal_eval(vswitch_cpu_map)]
+
+    if vswitch_pmd_cpu_mask == 0 and vswitch_vhost_cpu_map == 0:
+        print("CPU allocation Check Done,"\
+            "\nNo vswitch_pmd_cpu_mask or vswitch_vhost_cpu_map assign in test config file\n" \
+            "Using Default Settings ..................................................[OK]\n")
+    elif vswitch_pmd_cpu_mask != 0 and vswitch_vhost_cpu_map == 0:
+        core_id = cpumask2coreids(vswitch_pmd_cpu_mask)
+        print(core_id)
+        if len(default_vswitch_vhost_cpu_map) >= len(core_id):
+            if all(elem in default_vswitch_vhost_cpu_map  for elem in core_id):
+                print("CPU allocation properly done on DUT-Host.................[OK]\n")
+            else:
+                print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+        else:
+            print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+    elif vswitch_pmd_cpu_mask == 0 and vswitch_vhost_cpu_map != 0:
+        core_id_1 = cpumask2coreids(default_vswitch_pmd_cpu_mask)
+        print(core_id_1)
+        if len(vswitch_vhost_cpu_map) >= len(core_id_1):
+            if all(elem in vswitch_vhost_cpu_map  for elem in core_id_1):
+                print("CPU allocation properly done on DUT-Host.................[OK]\n")
+            else:
+                print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+        else:
+            print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+    else:
+        core_id_2 = cpumask2coreids(vswitch_pmd_cpu_mask)
+        print(core_id_2)
+        if len(vswitch_vhost_cpu_map) >= len(core_id_2):
+            if all(elem in vswitch_vhost_cpu_map  for elem in core_id_2):
+                print("CPU allocation properly done on DUT-Host.................[OK]\n")
+            else:
+                print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+        else:
+            print("CPU allocation not done properly on DUT-Host............[Failed]\n")
+
+
+
+def sanity_dut_conn_tgen_check():
+    """
+    We should confirm the DUT connectivity with the Tgen and Traffic Generator is working or not
+    """
+    global SANITY_CHECK_DONE_LIST
+    tgen_connectivity_check_cmd = "ping {} -c 1".format(TGEN_IP)
+    tgen_connectivity_check_result = int(DUT_CLIENT.execute(tgen_connectivity_check_cmd)[0])
+    if tgen_connectivity_check_result == 0:
+        SANITY_CHECK_DONE_LIST.append(int(5))
+        print(
+            "DUT-Host is successfully reachable to Traffic Generator Host.............[OK]\n")
+    else:
+        print(
+            "DUT-host is unsuccessful to reach the Traffic Generator Host.............[Failed]")
+        print(
+            "Make sure to establish connection before running Test...............[Critical]\n")
+
+if DUT_IP:
+    host_connect()
+if not DUT_CLIENT:
+    print('Failed to connect to DUT ...............[Critical]')
+    sys.exit()
+else:
+    upload_test_config_file()
+    sanity_vnf_path()
+    sanity_cpu_allocation_check()
+    sanity_collectd_check()
+    sanity_vsperf_check()
+    sanity_dut_conn_tgen_check()
+    if "yes" in START_COLLECTD.lower():
+        start_collectd()
+    if "yes" in START_BEATS.lower():
+        start_beats()
+
+if 'v' in VSPERF_TEST:
+    if len(SANITY_CHECK_DONE_LIST) != 4:
+        print("Certain Sanity Checks Failed\n" \
+              "You can make changes based on the outputs and run" \
+              "the testcontrol auto container again")
+    else:
+        run_vsperf_test()
+        test_status()
+else:
+    if len(SANITY_CHECK_DONE_LIST) != 3:
+        print("Certain Sanity Checks Failed\n" \
+              "You can make changes based on the outputs and run" \
+              "the testcontrol auto container again")
+    else:
+        run_vsperf_test()
+        test_status()
+
+
+if "yes" in CLEAN_UP.lower():
+    vsperf_remove()
+    remove_uploaded_config()
+    result_folders_remove()
+    collectd_remove()
+    terminate_vsperf()
diff --git a/tools/docker/testcontrol/auto/docker-compose.yml b/tools/docker/testcontrol/auto/docker-compose.yml
new file mode 100644
index 00000000..50c528a6
--- /dev/null
+++ b/tools/docker/testcontrol/auto/docker-compose.yml
@@ -0,0 +1,22 @@
+version: '2'
+
+services:
+  testcontrol:
+    build:
+      context: ./controller
+    volumes:
+      - ./controller/vsperf:/vsperf
+    env_file:
+      - ./controller/list.env
+    ports:
+      - 50052
+ 
+
+    
+
+
+    
+    
+
+
+
diff --git a/tools/docker/testcontrol/interactive/controller/Dockerfile b/tools/docker/testcontrol/interactive/controller/Dockerfile
new file mode 100644
index 00000000..16cf59fd
--- /dev/null
+++ b/tools/docker/testcontrol/interactive/controller/Dockerfile
@@ -0,0 +1,22 @@
+FROM python:3.6
+LABEL maintainer="sridhar.rao@spirent.com"
+
+ENV GRPC_PYTHON_VERSION 1.4.0
+RUN apt-get update && apt-get -y install python3-pip
+RUN pip3 install grpcio==${GRPC_PYTHON_VERSION} grpcio-tools==${GRPC_PYTHON_VERSION}
+RUN pip3 install paramiko
+RUN pip3 install chainmap
+RUN pip3 install oslo.utils
+RUN pip3 install scp
+
+WORKDIR /usr/src/app
+
+COPY ./vsperf ./vsperf
+
+VOLUME ["/usr/src/app/vsperf"]
+
+EXPOSE 50052
+
+CMD ["python3", "./vsperf/vsperf_controller.py"]
+
+#CMD tail -f /dev/null
diff --git a/tools/docker/testcontrol/interactive/controller/vsperf/__init__.py b/tools/docker/testcontrol/interactive/controller/vsperf/__init__.py
new file mode 100644
index 00000000..ad0ebec3
--- /dev/null
+++ b/tools/docker/testcontrol/interactive/controller/vsperf/__init__.py
@@ -0,0 +1 @@
+#### Empty
diff --git a/tools/docker/testcontrol/interactive/controller/vsperf/output.txt b/tools/docker/testcontrol/interactive/controller/vsperf/output.txt
new file mode 100644
index 00000000..912c877b
--- /dev/null
+++ b/tools/docker/testcontrol/interactive/controller/vsperf/output.txt
@@ -0,0 +1 @@
+[INFO ]  2019-08-27 18:09:46,085 : (root) - Overall test report written to "/tmp/results_2019-08-27_18-08-53/OvsDpdkVhost_test_report.rst"

diff --git a/tools/docker/testcontrol/interactive/controller/vsperf/vsperf_controller.py b/tools/docker/testcontrol/interactive/controller/vsperf/vsperf_controller.py
new file mode 100644
index 00000000..d1c3838d
--- /dev/null
+++ b/tools/docker/testcontrol/interactive/controller/vsperf/vsperf_controller.py
@@ -0,0 +1,706 @@
+# Copyright 2018-19 Spirent Communications.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=R0904
+# pylint: disable=R0902
+# twenty-two is reasonable in this script
+
+"""
+VSPER docker-controller.
+"""
+
+import io
+import time
+import ast
+import math
+
+from concurrent import futures
+
+import grpc
+from proto import vsperf_pb2
+from proto import vsperf_pb2_grpc
+from utils import ssh
+
+_ONE_DAY_IN_SECONDS = 60 * 60 * 24
+
+
+# pylint: disable=too-few-public-methods,no-self-use
+class PseudoFile(io.RawIOBase):
+    """
+    Handle ssh command output.
+    """
+
+    def write(self, chunk):
+        """
+        Write to file
+        """
+        if "error" in chunk:
+            return
+        with open("./output.txt", "w") as fref:
+            fref.write(chunk)
+
+
+class VsperfController(vsperf_pb2_grpc.ControllerServicer):
+    """
+    Main Controller Class
+    """
+
+    def __init__(self):
+        """
+        Initialization
+        """
+        self.client = None
+        self.dut_check = None
+        self.dut = None
+        self.user = None
+        self.pwd = None
+        self.vsperf_conf = None
+        self.tgen_client = None
+        self.tgen_check = None
+        self.tgen = None
+        self.tgen_user = None
+        self.tgenpwd = None
+        self.tgen_conf = None
+        self.scenario = None
+        self.testcase = None
+        self.tgen_ip_address = None
+        self.testtype = None
+        self.trex_conf = None
+        self.trex_params = None
+        self.conffile = None
+        self.tests_run_check = None
+        self.tgen_start_check = None
+        # Default TGen is T-Rex
+        self.trex_conffile = "trex_cfg.yml"
+        self.collectd_conffile = "collectd.conf"
+        self.test_upload_check = 0
+        self.sanity_check_done_list = list()
+
+    def setup(self):
+        """
+        Performs Setup of the client.
+        """
+        # Just connect to VM.
+        self.client = ssh.SSH(host=self.dut, user=self.user,
+                              password=self.pwd)
+        self.client.wait()
+
+    def upload_config(self):
+        """
+        Perform file upload.
+        """
+        # self.client._put_file_shell(self.conffile, '~/vsperf.conf')
+        self.client.put_file(self.conffile, '~/{}'.format(self.conffile))
+        print("No")
+
+    def run_test(self):
+        """
+        Run test
+        """
+        # Sometimes hugepage store in /mnt/huge in order to free up the
+        # hugepage removing this stored hugepage is necessory
+        rmv_cmd = "cd /mnt/huge && echo {} | sudo -S rm -rf *".format(self.pwd)
+        self.client.run(rmv_cmd, pty=True)
+        cmd = "source ~/vsperfenv/bin/activate ; "
+        #cmd = "scl enable python33 bash ; "
+        cmd += "cd vswitchperf* && "
+        cmd += "./vsperf "
+        if self.vsperf_conf:
+            cmd += "--conf-file ~/{} ".format(self.conffile)
+            # cmd += self.conffile
+        cmd += self.scenario
+        with PseudoFile() as pref:
+            self.client.run(cmd, stdout=pref, pty=True, timeout=0)
+
+    def TestStatus(self, request, context):
+        """
+        Chechk for the test status after performing test
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        if self.tests_run_check != 1:
+            return vsperf_pb2.StatusReply(message="No test have ran yet. [!]")
+        testtype_list = request.testtype.split(",")
+        test_success = []
+        test_failed = []
+        testtype_list_len = len(testtype_list)
+        for test in testtype_list:
+            #latest_result_cmd = "find /tmp -mindepth 1 -type d -cmin -5 -printf '%f'"
+            test_result_dir = str((self.client.\
+                              execute("find /tmp -mindepth 1 -type d -cmin -5 -printf '%f'")[1]).\
+                              split('find')[0])
+            #test_date_cmd = "date +%F"
+            test_date = str(self.client.execute("date +%F")[1]).replace("\n", "")
+            if test_date in test_result_dir:
+                testcase_check_cmd = "cd /tmp && cd `ls -t | grep results | head -{} | tail -1`".\
+                                      format(testtype_list_len)
+                testcase_check_cmd += " && find . -maxdepth 1 -name '*{}*'".\
+                                        format(test)
+                testcase_check_output = str(self.client.execute(testcase_check_cmd)[1]).\
+                                        split('\n', 2)
+                check = 0
+                for i in testcase_check_output:
+                    if (".csv" in i) or (".md" in i) or (".rst" in i):
+                        check += 1
+                if check == 3:
+                    test_success.append(test)
+                else:
+                    test_failed.append(test)
+                testtype_list_len -= 1
+        if len(testtype_list) == len(test_success):
+            return vsperf_pb2.StatusReply(message="All Test Successfully Completed on DUT-Host" \
+                                          "\nResults... [OK]")
+        if not test_success:
+            return vsperf_pb2.StatusReply(
+                message="All Test Failed on DUT-Host \nResults... [Failed]")
+        return vsperf_pb2.StatusReply(message="Only {} Test failed   Results ... [Failed]\n"\
+                "All other Test Successfully Completed on DUT-Host     Results... [OK] ".\
+                format(test_failed))
+
+    def HostConnect(self, request, context):
+        """
+        Handle host connectivity command from client
+        """
+        self.dut = request.ip
+        self.user = request.uname
+        self.pwd = request.pwd
+        self.setup()
+        check_cmd = "ls -l"
+        self.dut_check = int(self.client.execute(check_cmd)[0])
+        return vsperf_pb2.StatusReply(message="Successfully Connected")
+
+    def save_chunks_to_file(self, chunks, filename):
+        """
+        Write the output to file
+        """
+        with open(filename, 'w+') as fref:
+            fref.write(chunks)
+
+    def UploadConfigFile(self, request, context):
+        """
+        Handle upload config-file command from client
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        chunks = request.Content
+        filename = request.Filename
+        self.conffile = filename
+        self.save_chunks_to_file(chunks, filename)
+        # This is chechking if vsperf.conf already exist first remove that and
+        # then upload the new file.
+        check_test_config_cmd = "find ~/ -maxdepth 1 -name {}".format(filename)
+        check_test_result = str(self.client.execute(check_test_config_cmd)[1])
+        if "{}".format(filename) in check_test_result:
+            self.client.run("rm -f {}".format(filename))
+        self.upload_config()
+        self.test_upload_check = 1
+        print("Hello")
+        return vsperf_pb2.UploadStatus(Message="Successfully Uploaded", Code=1)
+
+    def StartTest(self, request, context):
+        """
+        Handle start-test command from client
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        sanity_dict = {1:"Check installed VSPERF",
+                       2:"Check Test Config's VNF path is available on DUT-Host",
+                       3:"Check NIC PCIs is available on Traffic Generator",
+                       4:"Check CPU allocation on DUT-Host",
+                       5:"Check installed Collectd",
+                       6:"Check Connection between DUT-Host and Traffic Generator Host"}
+        sanity_dict_option_list = list(sanity_dict.keys())
+        remaining_sanity = [item for item in sanity_dict_option_list if item not in \
+                            self.sanity_check_done_list]
+        if remaining_sanity:
+            sanity_return_msg = ""
+            for i_sanity in remaining_sanity:
+                sanity_return_msg += sanity_dict[i_sanity] + "\n"
+            return vsperf_pb2.StatusReply(message="The following sanity checks are either not"\
+                    " performed yet or Does not satisfy test requirements" \
+                    "\n{}".format(sanity_return_msg))
+        if self.test_upload_check == 0:
+            return vsperf_pb2.StatusReply(message="Test File is not uploaded yet [!] " \
+                         "\nUpload Test Configuration File.")
+        if self.tgen_start_check != 1:
+            return vsperf_pb2.StatusReply(message="Traffic Generator has not started yet [!]")
+        self.vsperf_conf = request.conffile
+        self.testtype = request.testtype
+        testtype_list = self.testtype.split(",")
+        self.tests_run_check = 1
+        for test in testtype_list:
+            self.scenario = test
+            self.run_test()
+        return vsperf_pb2.StatusReply(message="Test Successfully Completed")
+
+###### Traffic Generator Related functions ####
+    def TGenHostConnect(self, request, context):
+        """
+        Connect to TGen-Node
+        """
+        self.tgen = request.ip
+        self.tgen_user = request.uname
+        self.tgenpwd = request.pwd
+        self.tgen_setup()
+        check_tgen_cmd = "ls"
+        self.tgen_check = int(self.tgen_client.execute(check_tgen_cmd)[0])
+        return vsperf_pb2.StatusReply(message="Successfully Connected")
+
+    def tgen_setup(self):
+        """
+        Setup the T-Gen Client
+        """
+        # Just connect to VM.
+        self.tgen_client = ssh.SSH(host=self.tgen, user=self.tgen_user,
+                                   password=self.tgenpwd)
+        self.tgen_client.wait()
+
+    def StartBeats(self, request, context):
+        """
+        Start fileBeats on DUT
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        run_cmd = "echo '{}' | sudo -S service filebeat start".format(self.pwd)
+        #run_cmd = "sudo service filebeat start"
+        self.client.run(run_cmd, pty=True)
+        return vsperf_pb2.StatusReply(message="Beats are started on DUT-Host")
+
+    def DUTvsperfTestAvailability(self, request, context):
+        """
+        Before running test we have to make sure there is no other test running
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        vsperf_ava_cmd = "ps -ef | grep -v grep | grep ./vsperf | awk '{print $2}'"
+        vsperf_ava_result = len((self.client.execute(vsperf_ava_cmd)[1]).split("\n"))
+        if vsperf_ava_result == 1:
+            return vsperf_pb2.StatusReply(message="DUT-Host is available for performing" \
+                                                  " VSPERF Test\nYou can perform Test!")
+        return vsperf_pb2.StatusReply(message="DUT-Host is busy right now, Wait for some time\n\
+            Always Check availability before Running Test!")
+
+
+###Clean-UP process related functions####
+
+
+    def vsperf_remove(self):
+        """
+        Actual removal of the VSPERF
+        """
+        vsperf_rm_cmd = "echo '{}' | sudo -S rm -r ~/vswitchperf".format(
+            self.pwd)
+        self.client.run(vsperf_rm_cmd, pty=True)
+        vsperfenv_rm_cmd = "echo '{}' | sudo -S rm -r -f ~/vsperfenv".format(
+            self.pwd)
+        self.client.run(vsperfenv_rm_cmd, pty=True)
+
+    def remove_uploaded_config(self):
+        """
+        Remove all the uploaded test configuration file
+        """
+        vconfig_rm_cmd = "rm ~/{}".format(self.conffile)
+        self.client.run(vconfig_rm_cmd, pty=True)
+
+    def result_folder_remove(self):
+        """
+        Remove result folder on DUT
+        """
+        remove_cmd = "rm -r /tmp/*results*"
+        self.client.run(remove_cmd, pty=True)
+
+    def collectd_remove(self):
+        """
+        Remove collectd from DUT
+        """
+        collectd_dwn_rm_cmd = "echo '{}' | sudo -S rm -r -f ~/collectd".format(
+            self.pwd)
+        self.client.run(collectd_dwn_rm_cmd, pty=True)
+        collectd_rm_cmd = "echo '{}' | sudo -S rm -r -f /opt/collectd".format(
+            self.pwd)
+        self.client.run(collectd_rm_cmd, pty=True)
+
+    def RemoveVsperf(self, request, context):
+        """
+        Handle VSPERF removal command from client
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        self.vsperf_remove()
+        return vsperf_pb2.StatusReply(message="Successfully VSPERF Removed")
+
+    def TerminateVsperf(self, request, context):
+        """
+        Terminate the VSPERF and kill processes
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        stress_kill_cmd = "pkill stress"
+        python3_kill_cmd = "pkill python3"
+        qemu_kill_cmd = "killall -9 qemu-system-x86_64"
+        self.client.send_command(stress_kill_cmd)
+        self.client.send_command(python3_kill_cmd)
+        self.client.send_command(qemu_kill_cmd)
+
+        # sometimes qemu resists to terminate, so wait a bit and kill it again
+        qemu_check_cmd = "pgrep qemu-system-x86_64"
+        qemu_cmd_response = self.client.execute(qemu_check_cmd)[1]
+
+        if qemu_cmd_response != '':
+            time.sleep(5)
+            self.client.send_command(qemu_kill_cmds)
+            time.sleep(5)
+
+        ovs_kill_cmd = "pkill ovs-vswitchd"
+        ovsdb_kill_cmd = "pkill ovsdb-server"
+        vppctl_kill_cmd = "pkill vppctl"
+        vpp_kill_cmd = "pkill vpp"
+        vpp_cmd = "pkill -9".format(self.pwd)
+
+        self.client.send_command(ovs_kill_cmd)
+        time.sleep(1)
+        self.client.send_command(ovsdb_kill_cmd)
+        time.sleep(1)
+        self.client.send_command(vppctl_kill_cmd)
+        time.sleep(1)
+        self.client.send_command(vpp_kill_cmd)
+        time.sleep(1)
+        self.client.send_command(vpp_cmd)
+        time.sleep(1)
+
+        return vsperf_pb2.StatusReply(
+            message="All the VSPERF related process terminated successfully")
+
+    def RemoveResultFolder(self, request, context):
+        """
+        Handle result folder removal command from client
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        self.result_folder_remove()
+        return vsperf_pb2.StatusReply(
+            message="Successfully VSPERF Results Removed")
+
+    def RemoveUploadedConfig(self, request, context):
+        """
+        Handle all configuration file removal command from client
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        if self.tgen_check != 0:
+            return vsperf_pb2.StatusReply(message="TGen-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " TGen-Host.")
+        if self.test_upload_check == 0:
+            return vsperf_pb2.StatusReply(message="Test File is not uploaded yet [!] " \
+                         "\nUpload Test Configuration File.")
+        self.remove_uploaded_config()
+        return vsperf_pb2.StatusReply(
+            message="Successfully All Uploaded Config Files Removed")
+
+    def RemoveCollectd(self, request, context):
+        """
+        Handle collectd removal command from client
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        self.collectd_remove()
+        return vsperf_pb2.StatusReply(
+            message="Successfully Collectd Removed From DUT-Host")
+
+    def RemoveEverything(self, request, context):
+        """
+        Handle of removing everything from DUT command from client
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        if self.tgen_check != 0:
+            return vsperf_pb2.StatusReply(message="TGen-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " TGen-Host.")
+        self.vsperf_remove()
+        self.result_folder_remove()
+        self.remove_uploaded_config()
+        self.collectd_remove()
+        return vsperf_pb2.StatusReply(
+            message="Successfully Everything Removed From DUT-Host")
+
+    def StartTGen(self, request, context):
+        """
+        Handle start-Tgen command from client
+        """
+        if self.tgen_check != 0:
+            return vsperf_pb2.StatusReply(message="TGen-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " TGen-Host.")
+        self.trex_params = request.params
+        run_cmd = "cd trex_2.37/scripts ; "
+        run_cmd += "./t-rex-64 "
+        run_cmd += self.trex_params
+        self.tgen_client.send_command(run_cmd)
+        self.tgen_start_check = 1
+        return vsperf_pb2.StatusReply(message="T-Rex Successfully running...")
+
+    def SanityCollectdCheck(self, request, context):
+        """
+        Check and verify collectd is able to run and start properly
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        check_collectd_cmd = "find /opt -maxdepth 1 -name 'collectd'"
+        check_test_result = str(self.client.execute(check_collectd_cmd)[1])
+        if "collectd" in check_test_result:
+            check_collectd_run_cmd = "echo {} | sudo -S service collectd start".format(self.pwd)
+            self.client.run(check_collectd_run_cmd, pty=True)
+            check_collectd_status_cmd = "ps aux | grep collectd"
+            check_collectd_status = str(self.client.execute(check_collectd_status_cmd)[1])
+            if "/sbin/collectd" in check_collectd_status:
+                self.sanity_check_done_list.append(int(5))
+                return vsperf_pb2.StatusReply(message="Collectd is working Fine")
+            return vsperf_pb2.StatusReply(message="Collectd Fail to Start, \
+                                                   Install correctly before running Test")
+        return vsperf_pb2.StatusReply(message="Collectd is not installed yet.")
+
+    def SanityVNFpath(self, request, context):
+        """
+        Check if VNF image available on the mention path in Test Config File
+        """
+        # fetch the VNF path we placed in vsperf.conf file
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        if self.test_upload_check == 0:
+            return vsperf_pb2.StatusReply(message="Test File is not uploaded yet [!] " \
+                         "\nUpload Test Configuration File.")
+        vsperf_conf_path = 'cat ~/{} | grep "GUEST_IMAGE"'.format(self.conffile)
+        vsperf_conf_read = self.client.execute(vsperf_conf_path)[1]
+        vnf_image_path = vsperf_conf_read.split("'")[1]
+        vnf_path_check_cmd = "find {}".format(vnf_image_path)
+        vfn_path_check_result = str(self.client.execute(vnf_path_check_cmd)[1])
+        if vnf_image_path in vfn_path_check_result:
+            self.sanity_check_done_list.append(int(2))
+            return vsperf_pb2.StatusReply(message="Test Configratuion file has Correct "\
+                "VNF path information on DUT-Host.....[OK]")
+        return vsperf_pb2.StatusReply(message='Test Configuration file has wrongly placed VNF '\
+            'path information \n'\
+            'VNF is not available on DUT-Host................................[Failed]\n ')
+
+    def SanityVSPERFCheck(self, request, context):
+        """
+        We have to make sure that VSPERF install correctly
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        vsperf_check_command = "source ~/vsperfenv/bin/activate ; cd vswitchperf* && "
+        vsperf_check_command += "./vsperf --help"
+        vsperf_check_cmd_result = str(self.client.execute(vsperf_check_command)[1])
+        vsperf_verify_list = [
+            'usage',
+            'positional arguments',
+            'optional arguments',
+            'test selection options',
+            'test behavior options']
+        for idx, i in enumerate(vsperf_verify_list, start=1):
+            if str(i) in vsperf_check_cmd_result:
+                if idx < 5:
+                    continue
+                elif idx == 5:
+                    self.sanity_check_done_list.append(int(1))
+                    return vsperf_pb2.StatusReply(
+                        message="VSPERF Installed Correctly and Working fine")
+            return vsperf_pb2.StatusReply(message="VSPERF Does Not Installed Correctly ," \
+                                          "INSTALL IT AGAIN..............[Critical]")
+        return vsperf_pb2.StatusReply(message="VSPERF Does Not Installed Correctly ," \
+                                      "INSTALL IT AGAIN..............[Critical]")
+
+    def SanityNICCheck(self, request, context):
+        """
+        Check either NIC PCI ids are Correctly placed or not
+        """
+        if self.tgen_check != 0:
+            return vsperf_pb2.StatusReply(message="TGen-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " TGen-Host.")
+        trex_conf_path = "cat /etc/trex_cfg.yaml | grep interfaces"
+        trex_conf_read = self.tgen_client.execute(trex_conf_path)[1]
+        nic_pid_ids_list = [trex_conf_read.split("\"")[1], trex_conf_read.split("\"")[3]]
+        trex_nic_pic_id_cmd = "lspci | egrep -i --color 'network|ethernet'"
+        trex_nic_pic_id = str(self.tgen_client.execute(trex_nic_pic_id_cmd)[1]).split('\n')
+        acheck = 0
+        for k in trex_nic_pic_id:
+            for j in nic_pid_ids_list:
+                if j in k:
+                    acheck += 1
+                else:
+                    pass
+        if acheck == 2:
+            self.sanity_check_done_list.append(int(3))
+            return vsperf_pb2.StatusReply(message="Both the NIC PCI Ids are Correctly "\
+                "configured on TGen-Host..............")
+        return vsperf_pb2.StatusReply(message="You configured NIC PCI Ids Wrong in "\
+                "TGen-Host............................[OK]\n")
+
+    def SanityTgenConnDUTCheck(self, request, context):
+        """
+        We should confirm the DUT connectivity with the Tgen and Traffic Generator is working or not
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        self.tgen_ip_address = request.ip
+        tgen_connectivity_check_cmd = "ping {} -c 1".format(
+            self.tgen_ip_address)
+        tgen_connectivity_check_result = int(
+            self.client.execute(tgen_connectivity_check_cmd)[0])
+        if tgen_connectivity_check_result == 0:
+            self.sanity_check_done_list.append(int(6))
+            return vsperf_pb2.StatusReply(
+                message="DUT-Host is successfully reachable to Traffic Generator......")
+        return vsperf_pb2.StatusReply(message="DUT-Host is unsuccessful to reach the \
+                                      Traffic Generator \nMake sure to establish connection \
+                                      between DUT-Host and TGen-Host before running Test\
+                                       ............... ")
+
+    def variable_from_test_config(self, aparameter):
+        """This function can be use to read any configuration paramter from vsperf.conf"""
+        read_cmd = 'cat ~/{} | grep "{}"'.format(aparameter, self.conffile)
+        read_cmd_output = str(self.client.execute(read_cmd)[1])
+        print(read_cmd_output)
+        if not read_cmd_output or '#' in read_cmd_output:
+            return 0
+        return read_cmd_output.split("=")[1].strip()
+
+    def cpumask2coreids(self, mask):
+        """conver mask to coreids"""
+        intmask = int(mask, 16)
+        i = 1
+        coreids = []
+        while i < intmask:
+            if i & intmask:
+                coreids.append(str(math.frexp(i)[1]-1))
+            i = i << 1
+        return coreids
+
+    def cpu_allocation_check(self, list1, list2):
+        """compare to cpu_map list"""
+        if len(list1) >= len(list2):
+            if all(elem in list1  for elem in list2):
+                self.sanity_check_done_list.append(int(4))
+                return vsperf_pb2.StatusReply(message="CPU allocation properly done on" \
+                    " DUT-Host.................[OK]")
+            return vsperf_pb2.StatusReply(message="CPU allocation not done properly on " \
+                "DUT-Host............[Failed]")
+        return vsperf_pb2.StatusReply(message="CPU allocation not done properly on" \
+            " DUT-Host............[Failed]")
+
+    def SanityCPUAllocationCheck(self, request, context):
+        """
+        check for cpu-allocation on DUT-Host
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        if self.test_upload_check == 0:
+            return vsperf_pb2.StatusReply(message="Test File is not uploaded yet [!] " \
+                         "\nUpload Test Configuration File.")
+        read_setting_cmd = "source vsperfenv/bin/activate ; cd vswitchperf* && "
+        read_setting_cmd += './vsperf --list-settings'
+        default_vsperf_settings = ast.literal_eval(str(self.client.execute(read_setting_cmd)[1]))
+        default_cpu_map = default_vsperf_settings["VSWITCH_VHOST_CPU_MAP"]
+        default_vswitch_pmd_cpu_mask = str(default_vsperf_settings["VSWITCH_PMD_CPU_MASK"])
+        default_vswitch_vhost_cpu_map = [str(x) for x in default_cpu_map]
+        vswitch_pmd_cpu_mask = self.variable_from_test_config("VSWITCH_PMD_CPU_MASK")
+        vswitch_cpu_map = (self.variable_from_test_config("VSWITCH_VHOST_CPU_MAP"))
+        vswitch_vhost_cpu_map = 0
+
+        if vswitch_cpu_map != 0:
+            vswitch_vhost_cpu_map = [str(x) for x in  ast.literal_eval(vswitch_cpu_map)]
+
+        if vswitch_pmd_cpu_mask == 0 and vswitch_vhost_cpu_map == 0:
+            self.sanity_check_done_list.append(int(4))
+            return vsperf_pb2.StatusReply(message="CPU allocation Check Done,"\
+                "\nNo vswitch_pmd_cpu_mask or vswitch_vhost_cpu_map assign in test " \
+                "configuration file.\nUsing Default Settings..[OK]\n")
+        if vswitch_pmd_cpu_mask != 0 and vswitch_vhost_cpu_map == 0:
+            core_id = self.cpumask2coreids(vswitch_pmd_cpu_mask)
+            return self.cpu_allocation_check(default_vswitch_vhost_cpu_map, core_id)
+        if vswitch_pmd_cpu_mask == 0 and vswitch_vhost_cpu_map != 0:
+            core_id_1 = self.cpumask2coreids(default_vswitch_pmd_cpu_mask)
+            return self.cpu_allocation_check(vswitch_vhost_cpu_map, core_id_1)
+        core_id_2 = self.cpumask2coreids(vswitch_pmd_cpu_mask)
+        return self.cpu_allocation_check(vswitch_vhost_cpu_map, core_id_2)
+
+    def GetVSPERFConffromDUT(self, request, context):
+        """
+        This will extract the vsperf test configuration from DUT-Host
+        """
+        if self.dut_check != 0:
+            return vsperf_pb2.StatusReply(message="DUT-Host is not Connected [!]" \
+                                                   "\nMake sure to establish connection with" \
+                                                   " DUT-Host.")
+        if self.test_upload_check == 0:
+            return vsperf_pb2.StatusReply(message="Test File is not uploaded yet [!] " \
+                         "\nUpload Test Configuration File.")
+        read_cmd = "cat ~/{}".format(self.conffile)
+        read_cmd_output = str(self.client.execute(read_cmd)[1])
+        return vsperf_pb2.StatusReply(message="{}".format(read_cmd_output))
+
+
+def serve():
+    """
+    Start servicing the client
+    """
+    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+    vsperf_pb2_grpc.add_ControllerServicer_to_server(
+        VsperfController(), server)
+    server.add_insecure_port('[::]:50052')
+    server.start()
+    try:
+        while True:
+            time.sleep(_ONE_DAY_IN_SECONDS)
+    except (SystemExit, KeyboardInterrupt, MemoryError, RuntimeError):
+        server.stop(0)
+
+
+if __name__ == "__main__":
+    serve()
diff --git a/tools/docker/testcontrol/interactive/docker-compose.yml b/tools/docker/testcontrol/interactive/docker-compose.yml
new file mode 100644
index 00000000..431de124
--- /dev/null
+++ b/tools/docker/testcontrol/interactive/docker-compose.yml
@@ -0,0 +1,20 @@
+version: '2'
+
+services:
+  testcontrol:
+    build:
+      context: ./controller
+    volumes:
+      - ./controller/vsperf:/vsperf
+    ports:
+      - 50052:50052
+ 
+
+    
+
+
+    
+    
+
+
+