[dovetail tool] support shell scripts for testcase validation

JIRA: DOVETAIL-46

1. for now a testcase has two kinds of validation types(functest, yardstick),
and it is not enough to check the complete funcionality
2. add new validation type(shell) for extra validation of the test case to make
result more accurate and more convincing.

Change-Id: I5c049a71f11cca71a7914f8af704c1983aba3dca
Signed-off-by: Leo Wang <grakiss.wanglei@huawei.com>

8 files changed, 170 insertions, 75 deletions

diff --git a/dovetail/conf/cmd_config.yml b/dovetail/conf/cmd_config.yml
index 35c000ef..e2159ca7 100644
--- a/dovetail/conf/cmd_config.yml
+++ b/dovetail/conf/cmd_config.yml
@@ -1,42 +1,58 @@
 cli:
   arguments:
-    envs:
+    config:
       # This is a simple example of arguments.
       # Dovetail has no need of this kind of parameters currently.
       # The arguments must be given orderly at the run-time.
       #
       # docker_tag:
       #   flags: 'docker_tag'
-    non-envs:
+      #   path:
+      #     - 'functest/docker_tag'
+      #     - 'yardstick/docker_tag'
+    control:
 
   options:
-    envs:
+    config:
       SUT_TYPE:
         flags:
           - '--SUT_TYPE'
           - '-t'
+        path:
+          - 'functest/envs'
+          - 'yardstick/envs'
         help: 'Installer type of the system under test (SUT).'
       SUT_IP:
         flags:
           - '--SUT_IP'
           - '-i'
+        path:
+          - 'functest/envs'
+          - 'yardstick/envs'
         help: 'IP of the system under test (SUT).'
-      DEPLOY_SCENARIO:
-        flags:
-          - '--DEPLOY_SCENARIO'
-          - '-S'
-        help: 'DEPLOY_SCENARIO of the system under test (SUT).'
-      DEPLOY_TYPE:
-        flags:
-          - '--DEPLOY_TYPE'
-          - '-T'
-        help: 'DEPLOY_TYPE of the system under test (SUT).'
       CON_DEBUG:
         flags:
           - '--CON_DEBUG'
           - '-c'
+        path:
+          - 'functest/envs'
+          - 'yardstick/envs'
         help: 'True for showing debug log in functest/yardstick container.'
-    non-envs:
+      yard_tag:
+        flags:
+          - '--yard_tag'
+          - '-y'
+        path:
+          - 'yardstick/docker_tag'
+        help: 'Overwrite tag for yardstick docker container (e.g. stable or latest)'
+      func_tag:
+        flags:
+          - '--func_tag'
+          - '-f'
+        path:
+          - 'functest/docker_tag'
+        help: 'Overwrite tag for functest docker container (e.g. stable or latest)'
+    control:
       testsuite:
         flags:
           - '--testsuite'
@@ -47,11 +63,6 @@ cli:
           - '--testarea'
         default: 'full'
         help: 'compliance testarea within testsuite'
-      tag:
-        flags:
-          - '--tag'
-          - '-o'
-        help: 'Overwrite tags for each docker container (e.g. "functest:stable,yardstick:latest")'
       debug:
         flags:
           - '--debug'
diff --git a/dovetail/conf/dovetail_config.py b/dovetail/conf/dovetail_config.py
index 6cf3f7af..1087bdc5 100644
--- a/dovetail/conf/dovetail_config.py
+++ b/dovetail/conf/dovetail_config.py
@@ -43,15 +43,40 @@ class DovetailConfig:
         key = cmd_name.upper()
         return cls.CMD_NAME_TRANS.get(key, key)
 
+    # Analyze the kind of the giving path,
+    # return true for env path,
+    # return false for non_env path.
     @classmethod
-    def update_envs(cls, options):
-        for item in options:
-            key = cls.cmd_name_trans(item)
-            if not options[item] and key in os.environ:
-                options[item] = os.environ[key]
-            if options[item]:
-                cls.update_config_envs('functest', key, options[item])
-                cls.update_config_envs('yardstick', key, options[item])
+    def is_env_path(cls, path):
+        if len(path) == 2:
+            test_project = cls.dovetail_config['test_project']
+            if path[0] in test_project and path[1] == 'envs':
+                return True
+        else:
+            return False
+
+    # update dovetail_config dict with the giving path.
+    # if path is in the dovetail_config dict, its value will be replaced.
+    # if path is not in the dict, it will be added as a new item of the dict.
+    @classmethod
+    def update_config(cls, config_dict):
+        for key, value in config_dict.items():
+            path_list = []
+            for item in value['path']:
+                path_list.append([(k.strip()) for k in item.split('/')])
+            for path in path_list:
+                if cls.is_env_path(path):
+                    cls.update_envs(key, path, value['value'])
+                else:
+                    cls.update_non_envs(path, value['value'])
+
+    @classmethod
+    def update_envs(cls, key, path, value):
+        key = cls.cmd_name_trans(key)
+        if not value and key in os.environ:
+            value = os.environ[key]
+        if value:
+            cls.update_config_envs(path[0], key, value)
 
     @classmethod
     def update_config_envs(cls, validate_type, key, value):
@@ -63,3 +88,14 @@ class DovetailConfig:
             envs = envs.replace(old_value[0][0], value)
         cls.dovetail_config[validate_type]['envs'] = envs
         return envs
+
+    @staticmethod
+    def set_leaf_dict(dic, path, value):
+        for key in path[:-1]:
+            dic = dic.setdefault(key, {})
+        dic[path[-1]] = value
+
+    @classmethod
+    def update_non_envs(cls, path, value):
+        if value:
+            cls.set_leaf_dict(cls.dovetail_config, path, value)
diff --git a/dovetail/conf/dovetail_config.yml b/dovetail/conf/dovetail_config.yml
index 5264f140..86ac6cf6 100644
--- a/dovetail/conf/dovetail_config.yml
+++ b/dovetail/conf/dovetail_config.yml
@@ -32,4 +32,18 @@ include_config:
   - functest_config.yml
   - yardstick_config.yml
 
+test_project:
+  - 'yardstick'
+  - 'functest'
+
+validate_input:
+  valid_sut_type:
+    - 'compass'
+    - 'fuel'
+    - 'joid'
+    - 'apex'
+
+  valid_docker_tag:
+    - 'stable'
+    - 'latest'
 
diff --git a/dovetail/conf/functest_config.yml b/dovetail/conf/functest_config.yml
index 72cdb0dd..682d19bf 100644
--- a/dovetail/conf/functest_config.yml
+++ b/dovetail/conf/functest_config.yml
@@ -11,7 +11,7 @@ functest:
     - 'functest env prepare'
     - 'functest testcase run {{validate_testcase}}'
   post_condition:
-    - ''
+    - 'echo test for postcondition'
   result:
     dir: '/home/opnfv/functest/results'
     store_type: 'file'
diff --git a/dovetail/container.py b/dovetail/container.py
index af05dcd5..87174727 100644
--- a/dovetail/container.py
+++ b/dovetail/container.py
@@ -77,5 +77,7 @@ class Container:
 
     @classmethod
     def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False):
+        if sub_cmd == "":
+            return
         cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd)
         dt_utils.exec_cmd(cmd, cls.logger, exit_on_error)
diff --git a/dovetail/run.py b/dovetail/run.py
index 52a350e5..c0cc872c 100755
--- a/dovetail/run.py
+++ b/dovetail/run.py
@@ -9,8 +9,8 @@
 
 
 import click
-import sys
 import os
+import copy
 
 import utils.dovetail_logger as dt_logger
 import utils.dovetail_utils as dt_utils
@@ -31,14 +31,6 @@ def load_testsuite(testsuite):
     return Testsuite.get(testsuite)
 
 
-def set_container_tags(option_str):
-    for script_tag_opt in option_str.split(','):
-        option_str = script_tag_opt.split(':')
-        validate_type = option_str[0].strip()
-        script_tag = option_str[1].strip()
-        dt_cfg.dovetail_config[validate_type]['docker_tag'] = script_tag
-
-
 def load_testcase():
     Testcase.load()
 
@@ -74,23 +66,56 @@ def run_test(testsuite, testarea, logger):
     return duration
 
 
-def validate_options(input_dict, logger):
-    # for 'tag' option
-    for key, value in input_dict.items():
-        if key == 'tag' and value is not None:
-            for tag in value.split(','):
-                if len(tag.split(':')) != 2:
-                    logger.error('TAGS option must be "<image>:<tag>,..."')
-                    sys.exit(1)
-
-
-def filter_env_options(input_dict):
-    envs_options = {}
-    for key, value in input_dict.items():
-        key = key.upper()
-        if key in dt_cfg.dovetail_config['cli']['options']['envs']:
-            envs_options[key] = value
-    return envs_options
+def validate_input(input_dict, check_dict, logger):
+    # for 'func_tag' and 'yard_tag' options
+    func_tag = input_dict['func_tag']
+    yard_tag = input_dict['yard_tag']
+    valid_tag = check_dict['valid_docker_tag']
+    if func_tag is not None and func_tag not in valid_tag:
+        logger.error("func_tag can't be %s, valid in %s", func_tag, valid_tag)
+        raise SystemExit(1)
+    if yard_tag is not None and yard_tag not in valid_tag:
+        logger.error("yard_tag can't be %s, valid in %s", yard_tag, valid_tag)
+        raise SystemExit(1)
+
+    # for 'SUT_TYPE' option
+    sut_type = input_dict['sut_type']
+    valid_type = check_dict['valid_sut_type']
+    if sut_type is not None and sut_type not in valid_type:
+        logger.error("SUT_TYPE can't be %s, valid in %s", sut_type, valid_type)
+        raise SystemExit(1)
+
+
+def filter_config(input_dict, logger):
+    cli_dict = dt_cfg.dovetail_config['cli']
+    configs = {}
+    for key in cli_dict:
+        if not cli_dict[key]:
+            continue
+        try:
+            cli_config = cli_dict[key]['config']
+            if cli_config is None:
+                continue
+        except KeyError:
+            continue
+        for key, value in input_dict.items():
+            for config_key, config_value in cli_config.items():
+                value_dict = {}
+                value_dict['value'] = value
+                try:
+                    value_dict['path'] = config_value['path']
+                    if key == config_key:
+                        configs[key] = value_dict
+                        break
+                    if key.upper() == config_key:
+                        configs[key.upper()] = value_dict
+                        break
+                except KeyError as e:
+                    logger.exception('%s lacks subsection %s', config_key, e)
+                    raise SystemExit(1)
+    if not configs:
+        return None
+    return configs
 
 
 def create_logs():
@@ -115,7 +140,7 @@ def clean_results_dir():
             dt_utils.exec_cmd(cmd, exit_on_error=False)
         else:
             print "result_dir in dovetail_config.yml is not a directory."
-            sys.exit(-1)
+            raise SystemExit(1)
 
 
 def main(*args, **kwargs):
@@ -128,17 +153,16 @@ def main(*args, **kwargs):
     logger.info('================================================')
     logger.info('Dovetail compliance: %s!' % (kwargs['testsuite']))
     logger.info('================================================')
-    validate_options(kwargs, logger)
-    envs_options = filter_env_options(kwargs)
-    dt_cfg.update_envs(envs_options)
+    validate_input(kwargs, dt_cfg.dovetail_config['validate_input'], logger)
+    configs = filter_config(kwargs, logger)
+
+    if configs is not None:
+        dt_cfg.update_config(configs)
     logger.info('Your new envs for functest: %s' %
                 dt_cfg.dovetail_config['functest']['envs'])
     logger.info('Your new envs for yardstick: %s' %
                 dt_cfg.dovetail_config['yardstick']['envs'])
 
-    if 'tag' in kwargs and kwargs['tag'] is not None:
-        set_container_tags(kwargs['tag'])
-
     testarea = kwargs['testarea']
     testsuite_validation = False
     testarea_validation = False
@@ -158,21 +182,23 @@ def main(*args, **kwargs):
 
 
 dt_cfg.load_config_files()
-
+dovetail_config = copy.deepcopy(dt_cfg.dovetail_config)
 CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help'])
-if dt_cfg.dovetail_config['cli']['options'] is not None:
-    for key, value in dt_cfg.dovetail_config['cli']['options'].items():
+if dovetail_config['cli']['options'] is not None:
+    for key, value in dovetail_config['cli']['options'].items():
         if value is not None:
             for k, v in value.items():
                 flags = v['flags']
-                del v['flags']
+                v.pop('flags')
+                v.pop('path', None)
                 main = click.option(*flags, **v)(main)
-if dt_cfg.dovetail_config['cli']['arguments'] is not None:
-    for key, value in dt_cfg.dovetail_config['cli']['arguments'].items():
+if dovetail_config['cli']['arguments'] is not None:
+    for key, value in dovetail_config['cli']['arguments'].items():
         if value is not None:
             for k, v in value.items():
                 flags = v['flags']
-                del v['flags']
+                v.pop('flags')
+                v.pop('path', None)
                 main = click.argument(flags, **v)(main)
 main = click.command(context_settings=CONTEXT_SETTINGS)(main)
 
diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py
index bc0e4679..8a95b1f7 100644
--- a/dovetail/test_runner.py
+++ b/dovetail/test_runner.py
@@ -38,7 +38,7 @@ class DockerRunner(object):
 
         if not self.testcase.prepare_cmd():
             self.logger.error('failed to prepare testcase:%s',
-                              self.testcase.name())
+                              self.testcase.name)
         else:
             for cmd in self.testcase.cmds:
                 Container.exec_cmd(container_id, cmd)
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index af8b325e..6f2d76de 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -23,7 +23,6 @@ class Testcase(object):
 
     def __init__(self, testcase_yaml):
         self.testcase = testcase_yaml.values()[0]
-        # self.logger.debug('testcase:%s', self.testcase)
         self.testcase['passed'] = False
         self.cmds = []
         self.sub_testcase_status = {}
@@ -35,8 +34,17 @@ class Testcase(object):
 
     def prepare_cmd(self):
         try:
-            self.cmds = self.testcase['validate']['cmds']
-            return True
+            for cmd in self.testcase['validate']['cmds']:
+                cmd_lines = Parser.parse_cmd(cmd, self)
+                if not cmd_lines:
+                    return False
+                # self.logger.debug('cmd_lines:%s', cmd_lines)
+                self.cmds.append(cmd_lines)
+            self.logger.debug('cmds:%s', self.cmds)
+            if len(self.cmds) > 0:
+                return True
+            else:
+                return False
         except KeyError:
             return False
 
@@ -182,8 +190,6 @@ class FunctestTestcase(Testcase):
     def prepare_cmd(self):
         ret = super(FunctestTestcase, self).prepare_cmd()
         if not ret:
-            return False
-        else:
             for cmd in \
                 dt_cfg.dovetail_config[self.name]['cmds']:
                 cmd_lines = Parser.parse_cmd(cmd, self)
@@ -191,7 +197,7 @@ class FunctestTestcase(Testcase):
                     return False
                 self.logger.debug('cmd_lines:%s', cmd_lines)
                 self.cmds.append(cmd_lines)
-            return True
+        return True
 
 
 class YardstickTestcase(Testcase):