dovetail tool: remove installer and CI related variables

JIRA: DOVETAIL-371

1. remove all the envs in functest_config.yml and yardstick_config.yml
2. remove all relative cmds in cmd_config.yml
3. remove all the envs when running a new functest/yardstick container
4. add some variables just for functest pushing results to db
5. get the external network name for yardstick
6. make --debug/-d just control the log level shown on the screen
7. set requests=2.10.0 since python-openstackclient needs
   'requests!=2.12.2,!=2.13.0,>=2.10.0'

Change-Id: I9f941e09d067fc95a14c3c9be1374f41df4a9a16
Signed-off-by: xudan <xudan16@huawei.com>

9 files changed, 60 insertions, 101 deletions

diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml
index 05567709..2ae0402e 100644
--- a/dovetail/conf/cmd_config.yml
+++ b/dovetail/conf/cmd_config.yml
@@ -15,30 +15,6 @@ cli:
 
   options:
     config:
-      SUT_TYPE:
-        flags:
-          - '--SUT_TYPE'
-          - '-t'
-        path:
-          - 'functest/envs'
-          - 'yardstick/envs'
-        help: 'Installer type of the system under test (SUT).'
-      SUT_IP:
-        flags:
-          - '--SUT_IP'
-          - '-i'
-        path:
-          - 'functest/envs'
-          - 'yardstick/envs'
-        help: 'IP of the system under test (SUT).'
-      debug:
-        flags:
-          - '--debug'
-          - '-d'
-        path:
-          - 'functest/envs'
-          - 'yardstick/envs'
-        help: 'True for showing debug log on screen.'
       yard_tag:
         flags:
           - '--yard_tag'
@@ -71,3 +47,9 @@ cli:
           - '--testarea'
         default: 'full'
         help: 'compliance testarea within testsuite'
+      debug:
+        flags:
+          - '--debug'
+          - '-d'
+        is_flag: 'True'
+        help: 'Flag for showing debug log on screen.'
diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml
index 9d2c0cf3..9521617e 100644
--- a/dovetail/conf/dovetail_config.yml
+++ b/dovetail/conf/dovetail_config.yml
@@ -57,12 +57,6 @@ test_project:
   - 'functest'
 
 validate_input:
-  valid_sut_type:
-    - 'compass'
-    - 'fuel'
-    - 'joid'
-    - 'apex'
-
   valid_docker_tag:
     - 'stable'
     - 'latest'
diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml
index 2fca713d..8195c547 100644
--- a/dovetail/conf/functest_config.yml
+++ b/dovetail/conf/functest_config.yml
@@ -2,11 +2,6 @@
 functest:
   image_name: opnfv/functest
   docker_tag: latest
-  envs: '-e INSTALLER_TYPE=compass -e INSTALLER_IP=192.168.200.2
-         -e NODE_NAME=dovetail-pod -e DEPLOY_SCENARIO=ha_nosdn
-         -e BUILD_TAG=daily-dovetail-001 -e CI_DEBUG=true
-         -e DEPLOY_TYPE=baremetal
-         -e RESULTS_STORE=file:///home/opnfv/functest/results/functest_result.json'
   opts: '-id --privileged=true'
   pre_condition:
     - 'echo test for precondition in functest'
diff --git a/dovetail/conf/yardstick_config.yml b/dovetail/conf/yardstick_config.yml
index 029100cd..a70749d9 100644
--- a/dovetail/conf/yardstick_config.yml
+++ b/dovetail/conf/yardstick_config.yml
@@ -2,10 +2,6 @@
 yardstick:
   image_name: opnfv/yardstick
   docker_tag: latest
-  envs: '-e INSTALLER_TYPE=compass -e INSTALLER_IP=192.168.200.2
-         -e NODE_NAME=dovetail-pod -e DEPLOY_SCENARIO=ha_nosdn
-         -e BUILD_TAG=dovetail -e CI_DEBUG=true -e DEPLOY_TYPE=baremetal
-         -e EXTERNAL_NETWORK=ext-net'
   opts: '-id --privileged=true'
   pre_condition:
     - 'source /home/opnfv/repos/yardstick/tests/ci/prepare_env.sh &&
diff --git a/dovetail/container.py b/dovetail/container.py
index 4f6975fc..c90ff387 100644
--- a/dovetail/container.py
+++ b/dovetail/container.py
@@ -45,7 +45,6 @@ class Container(object):
         sshkey = "-v /root/.ssh/id_rsa:/root/.ssh/id_rsa "
         dovetail_config = dt_cfg.dovetail_config
         docker_image = cls.get_docker_image(type)
-        envs = dovetail_config[type]['envs']
         opts = dovetail_config[type]['opts']
 
         # credentials file openrc.sh is neccessary
@@ -57,6 +56,27 @@ class Container(object):
             cls.logger.error("File %s is not exist", dovetail_config['openrc'])
             return None
 
+        # This is used for showing the debug logs of the upstream projects
+        envs = ' -e CI_DEBUG=true'
+
+        # These are all just used by Functest's function push_results_to_db
+        if type.lower() == "functest":
+            ins_type = " -e INSTALLER_TYPE=vendor-specific"
+            scenario = " -e DEPLOY_SCENARIO=default"
+            node = " -e NODE_NAME=default"
+            tag = " -e BUILD_TAG=daily-master-001"
+
+            envs = "%s %s %s %s %s" % (envs, ins_type, scenario, node, tag)
+
+        if type.lower() == "yardstick":
+            ext_net = dt_utils.get_ext_net_name(dovetail_config['openrc'],
+                                                cls.logger)
+            if ext_net:
+                envs = "%s%s%s" % (envs, " -e EXTERNAL_NETWORK=", ext_net)
+            else:
+                cls.logger.error("Can't find any external network.")
+                return None
+
         result_volume = ' -v %s:%s ' % (dovetail_config['result_dir'],
                                         dovetail_config[type]['result']['dir'])
         cmd = 'sudo docker run %s %s %s %s %s %s /bin/bash' % \
diff --git a/dovetail/run.py b/dovetail/run.py
index 1fb0a89a..05f48501 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -82,13 +82,6 @@ def validate_input(input_dict, check_dict, logger):
         logger.error("yard_tag can't be %s, valid in %s", yard_tag, valid_tag)
         raise SystemExit(1)
 
-    # for 'SUT_TYPE' option
-    sut_type = input_dict['sut_type']
-    valid_type = check_dict['valid_sut_type']
-    if sut_type is not None and sut_type not in valid_type:
-        logger.error("SUT_TYPE can't be %s, valid in %s", sut_type, valid_type)
-        raise SystemExit(1)
-
 
 def filter_config(input_dict, logger):
     cli_dict = dt_cfg.dovetail_config['cli']
@@ -151,7 +144,7 @@ def main(*args, **kwargs):
     """Dovetail compliance test entry!"""
     clean_results_dir()
     if kwargs['debug']:
-        os.environ['DEBUG'] = kwargs['debug']
+        os.environ['DEBUG'] = 'true'
     create_logs()
     logger = dt_logger.Logger('run').getLogger()
     logger.info('================================================')
@@ -162,10 +155,6 @@ def main(*args, **kwargs):
 
     if configs is not None:
         dt_cfg.update_config(configs)
-    logger.info('Your new envs for functest: %s',
-                dt_cfg.dovetail_config['functest']['envs'])
-    logger.info('Your new envs for yardstick: %s',
-                dt_cfg.dovetail_config['yardstick']['envs'])
 
     testarea = kwargs['testarea']
     testsuite_validation = False
diff --git a/dovetail/utils/dovetail_config.py b/dovetail/utils/dovetail_config.py
index 1efb4d8f..f8193e5d 100644
--- a/dovetail/utils/dovetail_config.py
+++ b/dovetail/utils/dovetail_config.py
@@ -9,19 +9,12 @@
 
 import yaml
 import os
-import re
 
 
 class DovetailConfig(object):
 
     dovetail_config = {}
 
-    CMD_NAME_TRANS = {
-        'SUT_TYPE': 'INSTALLER_TYPE',
-        'SUT_IP': 'INSTALLER_IP',
-        'DEBUG': 'CI_DEBUG',
-    }
-
     @classmethod
     def load_config_files(cls):
         curr_path = os.path.dirname(os.path.abspath(__file__))
@@ -39,23 +32,6 @@ class DovetailConfig(object):
             cmd_yml = yaml.safe_load(f)
             cls.dovetail_config['cli'] = cmd_yml[cmd_yml.keys()[0]]
 
-    @classmethod
-    def cmd_name_trans(cls, cmd_name):
-        key = cmd_name.upper()
-        return cls.CMD_NAME_TRANS.get(key, key)
-
-    # Analyze the kind of the giving path,
-    # return true for env path,
-    # return false for non_env path.
-    @classmethod
-    def is_env_path(cls, path):
-        if len(path) == 2:
-            test_project = cls.dovetail_config['test_project']
-            if path[0] in test_project and path[1] == 'envs':
-                return True
-        else:
-            return False
-
     # update dovetail_config dict with the giving path.
     # if path is in the dovetail_config dict, its value will be replaced.
     # if path is not in the dict, it will be added as a new item of the dict.
@@ -66,29 +42,7 @@ class DovetailConfig(object):
             for item in value['path']:
                 path_list.append([(k.strip()) for k in item.split('/')])
             for path in path_list:
-                if cls.is_env_path(path):
-                    cls.update_envs(key, path, value['value'])
-                else:
-                    cls.update_non_envs(path, value['value'])
-
-    @classmethod
-    def update_envs(cls, key, path, value):
-        key = cls.cmd_name_trans(key)
-        if not value and key in os.environ:
-            value = os.environ[key]
-        if value:
-            cls.update_config_envs(path[0], key, value)
-
-    @classmethod
-    def update_config_envs(cls, validate_type, key, value):
-        envs = cls.dovetail_config[validate_type]['envs']
-        old_value = re.findall(r'\s+%s=(.*?)(\s+|$)' % key, envs)
-        if old_value == []:
-            envs += ' -e ' + key + '=' + value
-        else:
-            envs = envs.replace(old_value[0][0], value)
-        cls.dovetail_config[validate_type]['envs'] = envs
-        return envs
+                cls.update_non_envs(path, value['value'])
 
     @staticmethod
     def set_leaf_dict(dic, path, value):
diff --git a/dovetail/utils/dovetail_utils.py b/dovetail/utils/dovetail_utils.py
index a54081f5..32d334e8 100644
--- a/dovetail/utils/dovetail_utils.py
+++ b/dovetail/utils/dovetail_utils.py
@@ -10,6 +10,8 @@
 #
 
 import sys
+import os
+import re
 import subprocess
 from collections import Mapping, Set, Sequence
 
@@ -96,6 +98,31 @@ def get_obj_by_path(obj, dst_path):
             return obj
 
 
+def source_env(env_file):
+    with open(env_file, 'r') as f:
+        lines = f.readlines()
+    for line in lines:
+        for match in re.findall(r"export (.*)=(.*)", line):
+            match = (match[0].strip('\"'), match[1].strip('\"'))
+            match = (match[0].strip('\''), match[1].strip('\''))
+            os.environ.update({match[0]: match[1]})
+
+
+def get_ext_net_name(env_file, logger=None):
+    source_env(env_file)
+    cmd_check = "openstack network list"
+    ret, msg = exec_cmd(cmd_check, logger)
+    if ret:
+        logger.error("The credentials info in %s is invalid." % env_file)
+        return None
+    cmd = "openstack network list --long | grep 'External' | head -1 | \
+           awk '{print $4}'"
+    ret, msg = exec_cmd(cmd, logger)
+    if not ret:
+        return msg
+    return None
+
+
 def show_progress_bar(length):
     max_len = 50
     length %= max_len
diff --git a/requirements.txt b/requirements.txt
index 8c4577ca..7b9b7d9e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,5 +1,7 @@
-pbr>=1.8
+Click
 Jinja2>=2.6
+pbr>=1.8
 PyYAML>=3.10
+python-openstackclient==3.9.0
+requests==2.10.0
 six>=1.9.0
-Click