Decoupling Bottlenecks Docker config items from source code

There are some special Docker setting items when creating
Bottlenecks Docker Cntainers.
Move the special settings to config files.
Disable the function to push results to DB.

JIRA: DOVETAIL-645

Change-Id: Icc033222edf74c0fe4853dede2794f4226fd4252
Signed-off-by: xudan <xudan16@huawei.com>

5 files changed, 88 insertions, 106 deletions

diff --git a/dovetail/container.py b/dovetail/container.py
index 833f1ec9..1465cb5c 100644
--- a/dovetail/container.py
+++ b/dovetail/container.py
@@ -69,23 +69,6 @@ class Container(object):
             return None
 
     @classmethod
-    def set_bottlenecks_config(cls, testcase_name):
-        dovetail_config = dt_cfg.dovetail_config
-        yard_tag = dovetail_config['yardstick']['docker_tag']
-        docker_vol = '-v /var/run/docker.sock:/var/run/docker.sock'
-        env = ('-e Yardstick_TAG={} -e OUTPUT_FILE={}.out'
-               .format(yard_tag, testcase_name))
-        insecure = os.getenv("OS_INSECURE")
-        if insecure and insecure.lower() == 'true':
-            env = env + " -e OS_CACERT=False "
-
-        report = ""
-        if dovetail_config['report_dest'].startswith("http"):
-            report = ("-e BOTTLENECKS_DB_TARGET={}"
-                      .format(dovetail_config['report_dest']))
-        return "{} {} {}".format(docker_vol, env, report)
-
-    @classmethod
     def set_vnftest_config(cls):
         dovetail_config = dt_cfg.dovetail_config
 
@@ -103,17 +86,21 @@ class Container(object):
         return "%s %s" % (log_vol, key_vol)
 
     @classmethod
-    def create(cls, type, testcase_name, docker_image):
+    def create(cls, valid_type, testcase_name, docker_image):
         dovetail_config = dt_cfg.dovetail_config
-        opts = dovetail_config[type]['opts']
+        project_cfg = dovetail_config[valid_type]
 
         # credentials file openrc.sh is neccessary
-        openrc = cls.openrc_volume(type)
+        openrc = cls.openrc_volume(valid_type)
         if not openrc:
             return None
 
-        opts = dt_cfg.get_opts(type)
-        envs = dt_cfg.get_envs(type)
+        opts = dt_utils.get_value_from_dict('opts', project_cfg)
+        envs = dt_utils.get_value_from_dict('envs', project_cfg)
+        volumes = dt_utils.get_value_from_dict('volumes', project_cfg)
+        opts = ' ' if not opts else opts
+        envs = ' ' if not envs else envs
+        volumes = ' ' if not volumes else ' '.join(volumes)
 
         # CI_DEBUG is used for showing the debug logs of the upstream projects
         # BUILD_TAG is the unique id for this test
@@ -127,14 +114,13 @@ class Container(object):
 
         hosts_config = dt_utils.get_hosts_info(cls.logger)
 
-        # This part will be totally removed after remove the 3 functions
+        # This part will be totally removed after remove the 4 functions
         # set_functest_config has been removed
-        # set_yardstick_config
-        # set_bottlenecks_config
+        # set_yardstick_config has been removed
+        # set_bottlenecks_config has been removed
+        # set_vnftest_config
         config = " "
-        if type.lower() == "bottlenecks":
-            config = cls.set_bottlenecks_config(testcase_name)
-        if type.lower() == "vnftest":
+        if valid_type.lower() == "vnftest":
             config = cls.set_vnftest_config()
         if not config:
             return None
@@ -142,7 +128,7 @@ class Container(object):
         # for refstack, support user self_defined configuration
         config_volume = \
             ' -v %s:%s ' % (os.getenv("DOVETAIL_HOME"),
-                            dovetail_config[type]['config']['dir'])
+                            project_cfg['config']['dir'])
 
         cacert_volume = ""
         https_enabled = dt_utils.check_https_enabled(cls.logger)
@@ -162,23 +148,24 @@ class Container(object):
                 return None
 
         images_volume = ''
-        if dovetail_config[type]['config'].get('images', None):
+        if project_cfg['config'].get('images', None):
             images_volume = '-v {}:{}'.format(
                 dovetail_config['images_dir'],
-                dovetail_config[type]['config']['images'])
+                project_cfg['config']['images'])
 
         result_volume = ' -v %s:%s ' % (dovetail_config['result_dir'],
-                                        dovetail_config[type]['result']['dir'])
-        cmd = 'sudo docker run {opts} {envs} {config} {hosts_config} ' \
-              '{openrc} {cacert_volume} {config_volume} {result_volume} ' \
-              '{images_volume} {docker_image} /bin/bash'.format(**locals())
+                                        project_cfg['result']['dir'])
+        cmd = 'sudo docker run {opts} {envs} {volumes} {config} ' \
+              '{hosts_config} {openrc} {cacert_volume} {config_volume} ' \
+              '{result_volume} {images_volume} {docker_image} /bin/bash' \
+              .format(**locals())
         dt_utils.exec_cmd(cmd, cls.logger)
         ret, container_id = \
             dt_utils.exec_cmd("sudo docker ps | grep " + docker_image +
                               " | awk '{print $1}' | head -1", cls.logger)
-        cls.container_list[type] = container_id
+        cls.container_list[valid_type] = container_id
 
-        if type.lower() == 'vnftest':
+        if valid_type.lower() == 'vnftest':
             cls.set_vnftest_conf_file(container_id)
 
         return container_id
diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py
index 28988de8..f43b4739 100644
--- a/dovetail/test_runner.py
+++ b/dovetail/test_runner.py
@@ -118,6 +118,50 @@ class DockerRunner(object):
     def save_logs(self):
         pass
 
+    @staticmethod
+    def _render(task_template, **kwargs):
+        return jinja2.Template(task_template).render(**kwargs)
+
+    @staticmethod
+    def _add_testcase_info(testcase, config_item=None):
+        if not config_item:
+            config_item = {}
+        config_item['validate_testcase'] = testcase.validate_testcase()
+        config_item['testcase'] = testcase.name()
+        config_item['os_insecure'] = os.getenv("OS_INSECURE")
+        return config_item
+
+    def _update_config(self, testcase):
+        config_item = None
+        pod_file = os.path.join(dt_cfg.dovetail_config['config_dir'],
+                                dt_cfg.dovetail_config['pod_file'])
+        config_file = os.path.join(constants.CONF_PATH, self.config_file_name)
+        pod_info = dt_utils.read_yaml_file(pod_file, self.logger)
+        task_template = dt_utils.read_plain_file(config_file, self.logger)
+        if not task_template:
+            return None
+        if pod_info:
+            try:
+                process_info = pod_info['process_info']
+            except KeyError as e:
+                process_info = None
+        else:
+            process_info = None
+        if process_info:
+            for item in process_info:
+                try:
+                    if item['testcase_name'] == testcase.name():
+                        config_item = self._add_testcase_info(testcase, item)
+                        break
+                except KeyError as e:
+                    self.logger.error('Need key {} in {}'.format(e, item))
+        if not config_item:
+            config_item = self._add_testcase_info(testcase)
+        full_task = self._render(task_template, **config_item)
+        full_task_yaml = yaml.load(full_task)
+        dt_cfg.dovetail_config.update(full_task_yaml)
+        return dt_cfg.dovetail_config
+
 
 class FunctestRunner(DockerRunner):
 
@@ -156,55 +200,17 @@ class YardstickRunner(DockerRunner):
     def __init__(self, testcase):
         self.type = 'yardstick'
         super(YardstickRunner, self).__init__(testcase)
-        self._update_yardstick_config(testcase)
-
-    @staticmethod
-    def _render(task_template, **kwargs):
-        return jinja2.Template(task_template).render(**kwargs)
-
-    @staticmethod
-    def _add_testcase_info(testcase, config_item=None):
-        if not config_item:
-            config_item = {}
-        config_item['validate_testcase'] = testcase.validate_testcase()
-        config_item['testcase'] = testcase.name()
-        config_item['os_insecure'] = os.getenv("OS_INSECURE")
-        return config_item
-
-    def _update_yardstick_config(self, testcase):
-        config_item = None
-        pod_file = os.path.join(dt_cfg.dovetail_config['config_dir'],
-                                dt_cfg.dovetail_config['pod_file'])
-        config_file = os.path.join(constants.CONF_PATH, self.config_file_name)
-        pod_info = dt_utils.read_yaml_file(pod_file, self.logger)
-        task_template = dt_utils.read_plain_file(config_file, self.logger)
-        if not (pod_info and task_template):
-            return None
-        try:
-            process_info = pod_info['process_info']
-        except KeyError as e:
-            process_info = None
-        if process_info:
-            for item in process_info:
-                try:
-                    if item['testcase_name'] == testcase.name():
-                        config_item = self._add_testcase_info(testcase, item)
-                        break
-                except KeyError as e:
-                    self.logger.error('Need key {} in {}'.format(e, item))
-        if not config_item:
-            config_item = self._add_testcase_info(testcase)
-        full_task = self._render(task_template, **config_item)
-        full_task_yaml = yaml.load(full_task)
-        dt_cfg.dovetail_config.update(full_task_yaml)
-        return dt_cfg.dovetail_config
+        self._update_config(testcase)
 
 
 class BottlenecksRunner(DockerRunner):
 
+    config_file_name = 'bottlenecks_config.yml'
+
     def __init__(self, testcase):
         self.type = 'bottlenecks'
         super(BottlenecksRunner, self).__init__(testcase)
+        self._update_config(testcase)
 
 
 class ShellRunner(object):
diff --git a/dovetail/testcase.py b/dovetail/testcase.py
index 866c33b8..221d07f4 100644
--- a/dovetail/testcase.py
+++ b/dovetail/testcase.py
@@ -325,14 +325,6 @@ class BottlenecksTestcase(Testcase):
     def __init__(self, testcase_yaml):
         super(BottlenecksTestcase, self).__init__(testcase_yaml)
         self.type = 'bottlenecks'
-        self._update_cmds()
-
-    def _update_cmds(self):
-        if dt_cfg.dovetail_config['report_dest'].startswith("http"):
-            try:
-                self.testcase['validate']['cmds'][0] += ' --report'
-            except KeyError:
-                return
 
 
 class ShellTestcase(Testcase):
diff --git a/dovetail/utils/dovetail_config.py b/dovetail/utils/dovetail_config.py
index f582d4d7..394bcf73 100644
--- a/dovetail/utils/dovetail_config.py
+++ b/dovetail/utils/dovetail_config.py
@@ -22,9 +22,10 @@ class DovetailConfig(object):
 
         for extra_config_file in cls.dovetail_config['include_config']:
 
-            # The yardstick config file needs to be parsed later.
-            # Because it's related to the exact test case.
-            if extra_config_file.startswith("yardstick"):
+            # The yardstick and bottlenecks config files are with Jinja2.
+            # They need to be parsed later.
+            # All other config files should be transfer to like this gradually.
+            if extra_config_file.startswith(("yardstick", "bottlenecks")):
                 continue
             else:
                 file_path = os.path.join(conf_path, extra_config_file)
@@ -59,19 +60,3 @@ class DovetailConfig(object):
     def update_non_envs(cls, path, value):
         if value:
             cls.set_leaf_dict(cls.dovetail_config, path, value)
-
-    @classmethod
-    def get_opts(cls, valid_type):
-        project_config = cls.dovetail_config[valid_type]
-        if 'opts' in project_config.keys():
-            if project_config['opts']:
-                return project_config['opts']
-        return ""
-
-    @classmethod
-    def get_envs(cls, valid_type):
-        project_config = cls.dovetail_config[valid_type]
-        if 'envs' in project_config.keys():
-            if project_config['envs']:
-                return project_config['envs']
-        return ""
diff --git a/etc/conf/bottlenecks_config.yml b/etc/conf/bottlenecks_config.yml
index 005a04e3..3004e91c 100644
--- a/etc/conf/bottlenecks_config.yml
+++ b/etc/conf/bottlenecks_config.yml
@@ -1,8 +1,20 @@
 ---
+
+{% set validate_testcase = validate_testcase or '' %}
+{% set testcase = testcase or '' %}
+{% set os_insecure = os_insecure or 'False' %}
+{% set os_cacert = '' %}
+{% if os_insecure == 'True' %}
+    {% set os_cacert = ' -e OS_CACERT=False ' %}
+{% endif %}
+
 bottlenecks:
   image_name: opnfv/bottlenecks
   docker_tag: stable
   opts: '-id --privileged=true'
+  envs: '{{os_cacert}} -e Yardstick_TAG=stable -e OUTPUT_FILE={{testcase}}.out'
+  volumes:
+    - '-v /var/run/docker.sock:/var/run/docker.sock'
   config:
     dir: '/home/opnfv/userconfig'
   pre_condition: