dovetail tool: improve cli to update non-envs automatically

1. modify cmd_config.yml
  a. change 'env' and 'non-env' into 'config' and 'control'
  b. add subsection 'path' for each block in 'config'
  c. delete 'DEPLOY_SCENARIO' and 'DEPLOY_TYPE'
  d. use 'func_tag' and 'yard_tag' to control container's tag
2. if cmd in config
  a. if its path is envs, it will be used to update envs in dovetail_config
  b. if its path is non_envs, it will be used to update other value in the dict

JIRA: DOVETAIL-70

Change-Id: I1cf9bfcd1a19294a390a85bce6e458ce50672b7f
Signed-off-by: xudan <xudan16@huawei.com>

4 files changed, 154 insertions, 67 deletions

diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml
index 35c000ef..e2159ca7 100644
--- a/dovetail/conf/cmd_config.yml
+++ b/dovetail/conf/cmd_config.yml
@@ -1,42 +1,58 @@
 cli:
   arguments:
-    envs:
+    config:
       # This is a simple example of arguments.
       # Dovetail has no need of this kind of parameters currently.
       # The arguments must be given orderly at the run-time.
       #
       # docker_tag:
       #   flags: 'docker_tag'
-    non-envs:
+      #   path:
+      #     - 'functest/docker_tag'
+      #     - 'yardstick/docker_tag'
+    control:
 
   options:
-    envs:
+    config:
       SUT_TYPE:
         flags:
           - '--SUT_TYPE'
           - '-t'
+        path:
+          - 'functest/envs'
+          - 'yardstick/envs'
         help: 'Installer type of the system under test (SUT).'
       SUT_IP:
         flags:
           - '--SUT_IP'
           - '-i'
+        path:
+          - 'functest/envs'
+          - 'yardstick/envs'
         help: 'IP of the system under test (SUT).'
-      DEPLOY_SCENARIO:
-        flags:
-          - '--DEPLOY_SCENARIO'
-          - '-S'
-        help: 'DEPLOY_SCENARIO of the system under test (SUT).'
-      DEPLOY_TYPE:
-        flags:
-          - '--DEPLOY_TYPE'
-          - '-T'
-        help: 'DEPLOY_TYPE of the system under test (SUT).'
       CON_DEBUG:
         flags:
           - '--CON_DEBUG'
           - '-c'
+        path:
+          - 'functest/envs'
+          - 'yardstick/envs'
         help: 'True for showing debug log in functest/yardstick container.'
-    non-envs:
+      yard_tag:
+        flags:
+          - '--yard_tag'
+          - '-y'
+        path:
+          - 'yardstick/docker_tag'
+        help: 'Overwrite tag for yardstick docker container (e.g. stable or latest)'
+      func_tag:
+        flags:
+          - '--func_tag'
+          - '-f'
+        path:
+          - 'functest/docker_tag'
+        help: 'Overwrite tag for functest docker container (e.g. stable or latest)'
+    control:
       testsuite:
         flags:
           - '--testsuite'
@@ -47,11 +63,6 @@ cli:
           - '--testarea'
         default: 'full'
         help: 'compliance testarea within testsuite'
-      tag:
-        flags:
-          - '--tag'
-          - '-o'
-        help: 'Overwrite tags for each docker container (e.g. "functest:stable,yardstick:latest")'
       debug:
         flags:
           - '--debug'
diff --git a/dovetail/conf/dovetail_config.py b/dovetail/conf/dovetail_config.py
index 6cf3f7af..1087bdc5 100644
--- a/dovetail/conf/dovetail_config.py
+++ b/dovetail/conf/dovetail_config.py
@@ -43,15 +43,40 @@ class DovetailConfig:
         key = cmd_name.upper()
         return cls.CMD_NAME_TRANS.get(key, key)
 
+    # Analyze the kind of the giving path,
+    # return true for env path,
+    # return false for non_env path.
     @classmethod
-    def update_envs(cls, options):
-        for item in options:
-            key = cls.cmd_name_trans(item)
-            if not options[item] and key in os.environ:
-                options[item] = os.environ[key]
-            if options[item]:
-                cls.update_config_envs('functest', key, options[item])
-                cls.update_config_envs('yardstick', key, options[item])
+    def is_env_path(cls, path):
+        if len(path) == 2:
+            test_project = cls.dovetail_config['test_project']
+            if path[0] in test_project and path[1] == 'envs':
+                return True
+        else:
+            return False
+
+    # update dovetail_config dict with the giving path.
+    # if path is in the dovetail_config dict, its value will be replaced.
+    # if path is not in the dict, it will be added as a new item of the dict.
+    @classmethod
+    def update_config(cls, config_dict):
+        for key, value in config_dict.items():
+            path_list = []
+            for item in value['path']:
+                path_list.append([(k.strip()) for k in item.split('/')])
+            for path in path_list:
+                if cls.is_env_path(path):
+                    cls.update_envs(key, path, value['value'])
+                else:
+                    cls.update_non_envs(path, value['value'])
+
+    @classmethod
+    def update_envs(cls, key, path, value):
+        key = cls.cmd_name_trans(key)
+        if not value and key in os.environ:
+            value = os.environ[key]
+        if value:
+            cls.update_config_envs(path[0], key, value)
 
     @classmethod
     def update_config_envs(cls, validate_type, key, value):
@@ -63,3 +88,14 @@ class DovetailConfig:
             envs = envs.replace(old_value[0][0], value)
         cls.dovetail_config[validate_type]['envs'] = envs
         return envs
+
+    @staticmethod
+    def set_leaf_dict(dic, path, value):
+        for key in path[:-1]:
+            dic = dic.setdefault(key, {})
+        dic[path[-1]] = value
+
+    @classmethod
+    def update_non_envs(cls, path, value):
+        if value:
+            cls.set_leaf_dict(cls.dovetail_config, path, value)
diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml
index 5264f140..86ac6cf6 100644
--- a/dovetail/conf/dovetail_config.yml
+++ b/dovetail/conf/dovetail_config.yml
@@ -32,4 +32,18 @@ include_config:
   - functest_config.yml
   - yardstick_config.yml
 
+test_project:
+  - 'yardstick'
+  - 'functest'
+
+validate_input:
+  valid_sut_type:
+    - 'compass'
+    - 'fuel'
+    - 'joid'
+    - 'apex'
+
+  valid_docker_tag:
+    - 'stable'
+    - 'latest'
 
diff --git a/dovetail/run.py b/dovetail/run.py
index 52a350e5..c0cc872c 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -9,8 +9,8 @@
 
 
 import click
-import sys
 import os
+import copy
 
 import utils.dovetail_logger as dt_logger
 import utils.dovetail_utils as dt_utils
@@ -31,14 +31,6 @@ def load_testsuite(testsuite):
     return Testsuite.get(testsuite)
 
 
-def set_container_tags(option_str):
-    for script_tag_opt in option_str.split(','):
-        option_str = script_tag_opt.split(':')
-        validate_type = option_str[0].strip()
-        script_tag = option_str[1].strip()
-        dt_cfg.dovetail_config[validate_type]['docker_tag'] = script_tag
-
-
 def load_testcase():
     Testcase.load()
 
@@ -74,23 +66,56 @@ def run_test(testsuite, testarea, logger):
     return duration
 
 
-def validate_options(input_dict, logger):
-    # for 'tag' option
-    for key, value in input_dict.items():
-        if key == 'tag' and value is not None:
-            for tag in value.split(','):
-                if len(tag.split(':')) != 2:
-                    logger.error('TAGS option must be "<image>:<tag>,..."')
-                    sys.exit(1)
-
-
-def filter_env_options(input_dict):
-    envs_options = {}
-    for key, value in input_dict.items():
-        key = key.upper()
-        if key in dt_cfg.dovetail_config['cli']['options']['envs']:
-            envs_options[key] = value
-    return envs_options
+def validate_input(input_dict, check_dict, logger):
+    # for 'func_tag' and 'yard_tag' options
+    func_tag = input_dict['func_tag']
+    yard_tag = input_dict['yard_tag']
+    valid_tag = check_dict['valid_docker_tag']
+    if func_tag is not None and func_tag not in valid_tag:
+        logger.error("func_tag can't be %s, valid in %s", func_tag, valid_tag)
+        raise SystemExit(1)
+    if yard_tag is not None and yard_tag not in valid_tag:
+        logger.error("yard_tag can't be %s, valid in %s", yard_tag, valid_tag)
+        raise SystemExit(1)
+
+    # for 'SUT_TYPE' option
+    sut_type = input_dict['sut_type']
+    valid_type = check_dict['valid_sut_type']
+    if sut_type is not None and sut_type not in valid_type:
+        logger.error("SUT_TYPE can't be %s, valid in %s", sut_type, valid_type)
+        raise SystemExit(1)
+
+
+def filter_config(input_dict, logger):
+    cli_dict = dt_cfg.dovetail_config['cli']
+    configs = {}
+    for key in cli_dict:
+        if not cli_dict[key]:
+            continue
+        try:
+            cli_config = cli_dict[key]['config']
+            if cli_config is None:
+                continue
+        except KeyError:
+            continue
+        for key, value in input_dict.items():
+            for config_key, config_value in cli_config.items():
+                value_dict = {}
+                value_dict['value'] = value
+                try:
+                    value_dict['path'] = config_value['path']
+                    if key == config_key:
+                        configs[key] = value_dict
+                        break
+                    if key.upper() == config_key:
+                        configs[key.upper()] = value_dict
+                        break
+                except KeyError as e:
+                    logger.exception('%s lacks subsection %s', config_key, e)
+                    raise SystemExit(1)
+    if not configs:
+        return None
+    return configs
 
 
 def create_logs():
@@ -115,7 +140,7 @@ def clean_results_dir():
             dt_utils.exec_cmd(cmd, exit_on_error=False)
         else:
             print "result_dir in dovetail_config.yml is not a directory."
-            sys.exit(-1)
+            raise SystemExit(1)
 
 
 def main(*args, **kwargs):
@@ -128,17 +153,16 @@ def main(*args, **kwargs):
     logger.info('================================================')
     logger.info('Dovetail compliance: %s!' % (kwargs['testsuite']))
     logger.info('================================================')
-    validate_options(kwargs, logger)
-    envs_options = filter_env_options(kwargs)
-    dt_cfg.update_envs(envs_options)
+    validate_input(kwargs, dt_cfg.dovetail_config['validate_input'], logger)
+    configs = filter_config(kwargs, logger)
+
+    if configs is not None:
+        dt_cfg.update_config(configs)
     logger.info('Your new envs for functest: %s' %
                 dt_cfg.dovetail_config['functest']['envs'])
     logger.info('Your new envs for yardstick: %s' %
                 dt_cfg.dovetail_config['yardstick']['envs'])
 
-    if 'tag' in kwargs and kwargs['tag'] is not None:
-        set_container_tags(kwargs['tag'])
-
     testarea = kwargs['testarea']
     testsuite_validation = False
     testarea_validation = False
@@ -158,21 +182,23 @@ def main(*args, **kwargs):
 
 
 dt_cfg.load_config_files()
-
+dovetail_config = copy.deepcopy(dt_cfg.dovetail_config)
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
-if dt_cfg.dovetail_config['cli']['options'] is not None:
-    for key, value in dt_cfg.dovetail_config['cli']['options'].items():
+if dovetail_config['cli']['options'] is not None:
+    for key, value in dovetail_config['cli']['options'].items():
         if value is not None:
             for k, v in value.items():
                 flags = v['flags']
-                del v['flags']
+                v.pop('flags')
+                v.pop('path', None)
                 main = click.option(*flags, **v)(main)
-if dt_cfg.dovetail_config['cli']['arguments'] is not None:
-    for key, value in dt_cfg.dovetail_config['cli']['arguments'].items():
+if dovetail_config['cli']['arguments'] is not None:
+    for key, value in dovetail_config['cli']['arguments'].items():
         if value is not None:
             for k, v in value.items():
                 flags = v['flags']
-                del v['flags']
+                v.pop('flags')
+                v.pop('path', None)
                 main = click.argument(flags, **v)(main)
 main = click.command(context_settings=CONTEXT_SETTINGS)(main)