From 27109727907de0a275946765b201fbc0bd71116f Mon Sep 17 00:00:00 2001 From: xudan Date: Wed, 25 Apr 2018 03:35:02 -0400 Subject: Decoupling Bottlenecks Docker config items from source code There are some special Docker setting items when creating Bottlenecks Docker Cntainers. Move the special settings to config files. Disable the function to push results to DB. JIRA: DOVETAIL-645 Change-Id: Icc033222edf74c0fe4853dede2794f4226fd4252 Signed-off-by: xudan --- dovetail/container.py | 61 +++++++++++--------------- dovetail/test_runner.py | 90 +++++++++++++++++++++------------------ dovetail/testcase.py | 8 ---- dovetail/utils/dovetail_config.py | 23 ++-------- etc/conf/bottlenecks_config.yml | 12 ++++++ 5 files changed, 88 insertions(+), 106 deletions(-) diff --git a/dovetail/container.py b/dovetail/container.py index 833f1ec9..1465cb5c 100644 --- a/dovetail/container.py +++ b/dovetail/container.py @@ -68,23 +68,6 @@ class Container(object): "File {} doesn't exist.".format(dovetail_config['openrc'])) return None - @classmethod - def set_bottlenecks_config(cls, testcase_name): - dovetail_config = dt_cfg.dovetail_config - yard_tag = dovetail_config['yardstick']['docker_tag'] - docker_vol = '-v /var/run/docker.sock:/var/run/docker.sock' - env = ('-e Yardstick_TAG={} -e OUTPUT_FILE={}.out' - .format(yard_tag, testcase_name)) - insecure = os.getenv("OS_INSECURE") - if insecure and insecure.lower() == 'true': - env = env + " -e OS_CACERT=False " - - report = "" - if dovetail_config['report_dest'].startswith("http"): - report = ("-e BOTTLENECKS_DB_TARGET={}" - .format(dovetail_config['report_dest'])) - return "{} {} {}".format(docker_vol, env, report) - @classmethod def set_vnftest_config(cls): dovetail_config = dt_cfg.dovetail_config @@ -103,17 +86,21 @@ class Container(object): return "%s %s" % (log_vol, key_vol) @classmethod - def create(cls, type, testcase_name, docker_image): + def create(cls, valid_type, testcase_name, docker_image): dovetail_config = dt_cfg.dovetail_config - opts = dovetail_config[type]['opts'] + project_cfg = dovetail_config[valid_type] # credentials file openrc.sh is neccessary - openrc = cls.openrc_volume(type) + openrc = cls.openrc_volume(valid_type) if not openrc: return None - opts = dt_cfg.get_opts(type) - envs = dt_cfg.get_envs(type) + opts = dt_utils.get_value_from_dict('opts', project_cfg) + envs = dt_utils.get_value_from_dict('envs', project_cfg) + volumes = dt_utils.get_value_from_dict('volumes', project_cfg) + opts = ' ' if not opts else opts + envs = ' ' if not envs else envs + volumes = ' ' if not volumes else ' '.join(volumes) # CI_DEBUG is used for showing the debug logs of the upstream projects # BUILD_TAG is the unique id for this test @@ -127,14 +114,13 @@ class Container(object): hosts_config = dt_utils.get_hosts_info(cls.logger) - # This part will be totally removed after remove the 3 functions + # This part will be totally removed after remove the 4 functions # set_functest_config has been removed - # set_yardstick_config - # set_bottlenecks_config + # set_yardstick_config has been removed + # set_bottlenecks_config has been removed + # set_vnftest_config config = " " - if type.lower() == "bottlenecks": - config = cls.set_bottlenecks_config(testcase_name) - if type.lower() == "vnftest": + if valid_type.lower() == "vnftest": config = cls.set_vnftest_config() if not config: return None @@ -142,7 +128,7 @@ class Container(object): # for refstack, support user self_defined configuration config_volume = \ ' -v %s:%s ' % (os.getenv("DOVETAIL_HOME"), - dovetail_config[type]['config']['dir']) + project_cfg['config']['dir']) cacert_volume = "" https_enabled = dt_utils.check_https_enabled(cls.logger) @@ -162,23 +148,24 @@ class Container(object): return None images_volume = '' - if dovetail_config[type]['config'].get('images', None): + if project_cfg['config'].get('images', None): images_volume = '-v {}:{}'.format( dovetail_config['images_dir'], - dovetail_config[type]['config']['images']) + project_cfg['config']['images']) result_volume = ' -v %s:%s ' % (dovetail_config['result_dir'], - dovetail_config[type]['result']['dir']) - cmd = 'sudo docker run {opts} {envs} {config} {hosts_config} ' \ - '{openrc} {cacert_volume} {config_volume} {result_volume} ' \ - '{images_volume} {docker_image} /bin/bash'.format(**locals()) + project_cfg['result']['dir']) + cmd = 'sudo docker run {opts} {envs} {volumes} {config} ' \ + '{hosts_config} {openrc} {cacert_volume} {config_volume} ' \ + '{result_volume} {images_volume} {docker_image} /bin/bash' \ + .format(**locals()) dt_utils.exec_cmd(cmd, cls.logger) ret, container_id = \ dt_utils.exec_cmd("sudo docker ps | grep " + docker_image + " | awk '{print $1}' | head -1", cls.logger) - cls.container_list[type] = container_id + cls.container_list[valid_type] = container_id - if type.lower() == 'vnftest': + if valid_type.lower() == 'vnftest': cls.set_vnftest_conf_file(container_id) return container_id diff --git a/dovetail/test_runner.py b/dovetail/test_runner.py index 28988de8..f43b4739 100644 --- a/dovetail/test_runner.py +++ b/dovetail/test_runner.py @@ -118,6 +118,50 @@ class DockerRunner(object): def save_logs(self): pass + @staticmethod + def _render(task_template, **kwargs): + return jinja2.Template(task_template).render(**kwargs) + + @staticmethod + def _add_testcase_info(testcase, config_item=None): + if not config_item: + config_item = {} + config_item['validate_testcase'] = testcase.validate_testcase() + config_item['testcase'] = testcase.name() + config_item['os_insecure'] = os.getenv("OS_INSECURE") + return config_item + + def _update_config(self, testcase): + config_item = None + pod_file = os.path.join(dt_cfg.dovetail_config['config_dir'], + dt_cfg.dovetail_config['pod_file']) + config_file = os.path.join(constants.CONF_PATH, self.config_file_name) + pod_info = dt_utils.read_yaml_file(pod_file, self.logger) + task_template = dt_utils.read_plain_file(config_file, self.logger) + if not task_template: + return None + if pod_info: + try: + process_info = pod_info['process_info'] + except KeyError as e: + process_info = None + else: + process_info = None + if process_info: + for item in process_info: + try: + if item['testcase_name'] == testcase.name(): + config_item = self._add_testcase_info(testcase, item) + break + except KeyError as e: + self.logger.error('Need key {} in {}'.format(e, item)) + if not config_item: + config_item = self._add_testcase_info(testcase) + full_task = self._render(task_template, **config_item) + full_task_yaml = yaml.load(full_task) + dt_cfg.dovetail_config.update(full_task_yaml) + return dt_cfg.dovetail_config + class FunctestRunner(DockerRunner): @@ -156,55 +200,17 @@ class YardstickRunner(DockerRunner): def __init__(self, testcase): self.type = 'yardstick' super(YardstickRunner, self).__init__(testcase) - self._update_yardstick_config(testcase) - - @staticmethod - def _render(task_template, **kwargs): - return jinja2.Template(task_template).render(**kwargs) - - @staticmethod - def _add_testcase_info(testcase, config_item=None): - if not config_item: - config_item = {} - config_item['validate_testcase'] = testcase.validate_testcase() - config_item['testcase'] = testcase.name() - config_item['os_insecure'] = os.getenv("OS_INSECURE") - return config_item - - def _update_yardstick_config(self, testcase): - config_item = None - pod_file = os.path.join(dt_cfg.dovetail_config['config_dir'], - dt_cfg.dovetail_config['pod_file']) - config_file = os.path.join(constants.CONF_PATH, self.config_file_name) - pod_info = dt_utils.read_yaml_file(pod_file, self.logger) - task_template = dt_utils.read_plain_file(config_file, self.logger) - if not (pod_info and task_template): - return None - try: - process_info = pod_info['process_info'] - except KeyError as e: - process_info = None - if process_info: - for item in process_info: - try: - if item['testcase_name'] == testcase.name(): - config_item = self._add_testcase_info(testcase, item) - break - except KeyError as e: - self.logger.error('Need key {} in {}'.format(e, item)) - if not config_item: - config_item = self._add_testcase_info(testcase) - full_task = self._render(task_template, **config_item) - full_task_yaml = yaml.load(full_task) - dt_cfg.dovetail_config.update(full_task_yaml) - return dt_cfg.dovetail_config + self._update_config(testcase) class BottlenecksRunner(DockerRunner): + config_file_name = 'bottlenecks_config.yml' + def __init__(self, testcase): self.type = 'bottlenecks' super(BottlenecksRunner, self).__init__(testcase) + self._update_config(testcase) class ShellRunner(object): diff --git a/dovetail/testcase.py b/dovetail/testcase.py index 866c33b8..221d07f4 100644 --- a/dovetail/testcase.py +++ b/dovetail/testcase.py @@ -325,14 +325,6 @@ class BottlenecksTestcase(Testcase): def __init__(self, testcase_yaml): super(BottlenecksTestcase, self).__init__(testcase_yaml) self.type = 'bottlenecks' - self._update_cmds() - - def _update_cmds(self): - if dt_cfg.dovetail_config['report_dest'].startswith("http"): - try: - self.testcase['validate']['cmds'][0] += ' --report' - except KeyError: - return class ShellTestcase(Testcase): diff --git a/dovetail/utils/dovetail_config.py b/dovetail/utils/dovetail_config.py index f582d4d7..394bcf73 100644 --- a/dovetail/utils/dovetail_config.py +++ b/dovetail/utils/dovetail_config.py @@ -22,9 +22,10 @@ class DovetailConfig(object): for extra_config_file in cls.dovetail_config['include_config']: - # The yardstick config file needs to be parsed later. - # Because it's related to the exact test case. - if extra_config_file.startswith("yardstick"): + # The yardstick and bottlenecks config files are with Jinja2. + # They need to be parsed later. + # All other config files should be transfer to like this gradually. + if extra_config_file.startswith(("yardstick", "bottlenecks")): continue else: file_path = os.path.join(conf_path, extra_config_file) @@ -59,19 +60,3 @@ class DovetailConfig(object): def update_non_envs(cls, path, value): if value: cls.set_leaf_dict(cls.dovetail_config, path, value) - - @classmethod - def get_opts(cls, valid_type): - project_config = cls.dovetail_config[valid_type] - if 'opts' in project_config.keys(): - if project_config['opts']: - return project_config['opts'] - return "" - - @classmethod - def get_envs(cls, valid_type): - project_config = cls.dovetail_config[valid_type] - if 'envs' in project_config.keys(): - if project_config['envs']: - return project_config['envs'] - return "" diff --git a/etc/conf/bottlenecks_config.yml b/etc/conf/bottlenecks_config.yml index 005a04e3..3004e91c 100644 --- a/etc/conf/bottlenecks_config.yml +++ b/etc/conf/bottlenecks_config.yml @@ -1,8 +1,20 @@ --- + +{% set validate_testcase = validate_testcase or '' %} +{% set testcase = testcase or '' %} +{% set os_insecure = os_insecure or 'False' %} +{% set os_cacert = '' %} +{% if os_insecure == 'True' %} + {% set os_cacert = ' -e OS_CACERT=False ' %} +{% endif %} + bottlenecks: image_name: opnfv/bottlenecks docker_tag: stable opts: '-id --privileged=true' + envs: '{{os_cacert}} -e Yardstick_TAG=stable -e OUTPUT_FILE={{testcase}}.out' + volumes: + - '-v /var/run/docker.sock:/var/run/docker.sock' config: dir: '/home/opnfv/userconfig' pre_condition: -- cgit 1.2.3-korg