diff options
Diffstat (limited to 'dovetail/container.py')
| -rw-r--r-- | dovetail/container.py | 360 |
1 files changed, 135 insertions, 225 deletions
diff --git a/dovetail/container.py b/dovetail/container.py index 89c68ccb..b2a9428f 100644 --- a/dovetail/container.py +++ b/dovetail/container.py @@ -9,21 +9,23 @@ # http://www.apache.org/licenses/LICENSE-2.0 # -import os +import docker +import sys -import utils.dovetail_logger as dt_logger -import utils.dovetail_utils as dt_utils -from utils.dovetail_config import DovetailConfig as dt_cfg +import dovetail.utils.dovetail_logger as dt_logger +import dovetail.utils.dovetail_utils as dt_utils +from dovetail.utils.dovetail_config import DovetailConfig as dt_cfg class Container(object): - container_list = {} - logger = None - def __init__(self): - pass + def __init__(self, testcase): + self.container = None + self.testcase = testcase + self.valid_type = self.testcase.validate_type() + self.client = docker.from_env(timeout=None) def __str__(self): pass @@ -32,260 +34,168 @@ class Container(object): def create_log(cls): cls.logger = dt_logger.Logger(__name__ + '.Container').getLogger() - @classmethod - def get(cls, type): - return cls.container_list[type] - - @classmethod - def _get_config(cls, field, project_cfg, testcase_cfg): + def _get_config(self, field, project_cfg, testcase_cfg): value = dt_utils.get_value_from_dict(field, testcase_cfg) if not value: value = dt_utils.get_value_from_dict(field, project_cfg) if not value: - cls.logger.error("Couldn't find key {}.".format(field)) + self.logger.error("Couldn't find key {}.".format(field)) return None return value - @classmethod - def get_docker_image(cls, testcase): - project_cfg = dt_cfg.dovetail_config[testcase.validate_type()] - testcase_cfg = testcase.testcase['validate'] + def get_docker_image(self): + project_cfg = dt_cfg.dovetail_config[self.valid_type] + testcase_cfg = self.testcase.testcase['validate'] - name = cls._get_config('image_name', project_cfg, testcase_cfg) - tag = cls._get_config('docker_tag', project_cfg, testcase_cfg) - return "{}:{}".format(name, tag) if name and tag else None + name = self._get_config('image_name', project_cfg, testcase_cfg) + tag = self._get_config('docker_tag', project_cfg, testcase_cfg) + return '{}:{}'.format(name, tag) if name and tag else None - # get the openrc_volume for creating the container - @classmethod - def openrc_volume(cls, type): + def create(self, docker_image): dovetail_config = dt_cfg.dovetail_config - dovetail_config['openrc'] = os.path.join(dovetail_config['config_dir'], - dovetail_config['env_file']) - if os.path.isfile(dovetail_config['openrc']): - openrc = ' -v %s:%s ' % (dovetail_config['openrc'], - dovetail_config[type]['openrc']) - return openrc - else: - cls.logger.error( - "File {} doesn't exist.".format(dovetail_config['openrc'])) - return None - - @classmethod - def set_vnftest_config(cls): - dovetail_config = dt_cfg.dovetail_config - - log_vol = '-v %s:%s ' % (dovetail_config['result_dir'], - dovetail_config["vnftest"]['result']['log']) - - key_file = os.path.join(dovetail_config['config_dir'], - dovetail_config['pri_key']) - key_container_path = dovetail_config["vnftest"]['result']['key_path'] - if not os.path.isfile(key_file): - cls.logger.debug("Key file {} is not found".format(key_file)) - key_vol = '' - else: - key_vol = '-v %s:%s ' % (key_file, key_container_path) - return "%s %s" % (log_vol, key_vol) - - @classmethod - def create(cls, valid_type, testcase_name, docker_image): - dovetail_config = dt_cfg.dovetail_config - project_cfg = dovetail_config[valid_type] - - # credentials file openrc.sh is neccessary - openrc = cls.openrc_volume(valid_type) - if not openrc: - return None - - opts = dt_utils.get_value_from_dict('opts', project_cfg) - envs = dt_utils.get_value_from_dict('envs', project_cfg) - volumes = dt_utils.get_value_from_dict('volumes', project_cfg) - opts = ' ' if not opts else opts - envs = ' ' if not envs else envs - volumes = ' ' if not volumes else ' '.join(volumes) - - # CI_DEBUG is used for showing the debug logs of the upstream projects - # BUILD_TAG is the unique id for this test - DEBUG = os.getenv('DEBUG') - if DEBUG is not None and DEBUG.lower() == "true": - envs = envs + ' -e CI_DEBUG=true' - else: - envs = envs + ' -e CI_DEBUG=false' - envs = envs + ' -e BUILD_TAG=%s-%s' % (dovetail_config['build_tag'], - testcase_name) - - hosts_config = dt_utils.get_hosts_info(cls.logger) - - # This part will be totally removed after remove the 4 functions - # set_functest_config has been removed - # set_yardstick_config has been removed - # set_bottlenecks_config has been removed - # set_vnftest_config - config = " " - if valid_type.lower() == "vnftest": - config = cls.set_vnftest_config() - if not config: - return None - - # for refstack, support user self_defined configuration - config_volume = \ - ' -v %s:%s ' % (os.getenv("DOVETAIL_HOME"), - project_cfg['config']['dir']) - - cacert_volume = "" - https_enabled = dt_utils.check_https_enabled(cls.logger) - cacert = os.getenv('OS_CACERT') - insecure = os.getenv('OS_INSECURE') - if cacert is not None: - if dt_utils.check_cacert_file(cacert, cls.logger): - cacert_volume = ' -v %s:%s ' % (cacert, cacert) - else: - return None - elif https_enabled: - if insecure and insecure.lower() == 'true': - cls.logger.debug("Use the insecure mode...") - else: - cls.logger.error("https enabled, please set OS_CACERT or " - "insecure mode...") - return None - - images_volume = '' - if project_cfg['config'].get('images', None): - images_volume = '-v {}:{}'.format( - dovetail_config['images_dir'], - project_cfg['config']['images']) - - result_volume = ' -v %s:%s ' % (dovetail_config['result_dir'], - project_cfg['result']['dir']) - cmd = 'sudo docker run {opts} {envs} {volumes} {config} ' \ - '{hosts_config} {openrc} {cacert_volume} {config_volume} ' \ - '{result_volume} {images_volume} {docker_image} /bin/bash' \ - .format(**locals()) - dt_utils.exec_cmd(cmd, cls.logger) - ret, container_id = \ - dt_utils.exec_cmd("sudo docker ps | grep " + docker_image + - " | awk '{print $1}' | head -1", cls.logger) - cls.container_list[valid_type] = container_id - - if valid_type.lower() == 'vnftest': - cls.set_vnftest_conf_file(container_id) - - return container_id - - @classmethod - def get_image_id(cls, image_name): - cmd = 'sudo docker images -q %s' % (image_name) - ret, image_id = dt_utils.exec_cmd(cmd, cls.logger) - if ret == 0: - return image_id - else: - return None + project_cfg = dovetail_config[self.valid_type] + + kwargs = dt_utils.get_value_from_dict('opts', project_cfg) + shell = dt_utils.get_value_from_dict('shell', project_cfg) + if not shell: + return None, "Lacking of key word 'shell' in config file." + env_list = dt_utils.get_value_from_dict('envs', project_cfg) + if env_list: + kwargs['environment'] = \ + [env for env in env_list if env is not None] + volume_list = dt_utils.get_value_from_dict('volumes', project_cfg) + kwargs['volumes'] = [vol for vol in volume_list if vol is not None] + + kwargs['mounts'], msg = dt_utils.get_mount_list(project_cfg) + if not kwargs['mounts']: + return None, msg + + kwargs['extra_hosts'] = dt_utils.get_hosts_info(self.logger) + + try: + self.container = self.client.containers.run( + docker_image, shell, **kwargs) + except (docker.errors.ContainerError, docker.errors.ImageNotFound, + docker.errors.APIError) as e: + return None, e + + return self.container.id, 'Successfully to create container.' + + def get_image_id(self, image_name): + try: + image_id = self.client.images.get(image_name).id + except (docker.errors.ImageNotFound, docker.errors.APIError): + image_id = None + return image_id # remove the image according to the image_id # if there exists containers using this image, then skip - @classmethod - def remove_image(cls, image_id): - cmd = "sudo docker ps -aq -f 'ancestor=%s'" % (image_id) - ret, msg = dt_utils.exec_cmd(cmd, cls.logger) - if msg and ret == 0: - cls.logger.debug('Image {} has containers, skip.'.format(image_id)) + def remove_image(self, image_id): + try: + containers = self.client.containers.list( + filters={'ancestor': image_id}) + except docker.errors.APIError: + containers = [] + if containers: + self.logger.debug('Image {} has containers, skip.' + .format(image_id)) return True - cmd = 'sudo docker rmi %s' % (image_id) - cls.logger.debug('Remove image {}.'.format(image_id)) - ret, msg = dt_utils.exec_cmd(cmd, cls.logger) - if ret == 0: - cls.logger.debug('Remove image {} successfully.'.format(image_id)) + self.logger.debug('Remove image {}.'.format(image_id)) + try: + self.client.images.remove(image_id) + self.logger.debug('Remove image {} successfully.'.format(image_id)) return True - cls.logger.error('Failed to remove image {}.'.format(image_id)) - return False + except (docker.errors.ImageNotFound, docker.errors.APIError): + self.logger.error('Failed to remove image {}.'.format(image_id)) + return False - @classmethod - def pull_image_only(cls, image_name): - cmd = 'sudo docker pull %s' % (image_name) - ret, msg = dt_utils.exec_cmd(cmd, cls.logger) - if ret != 0: - cls.logger.error( + def pull_image_only(self, image_name): + try: + self.client.images.pull(image_name) + self.logger.debug( + 'Success to pull docker image {}!'.format(image_name)) + return True + except docker.errors.APIError: + self.logger.error( 'Failed to pull docker image {}!'.format(image_name)) return False - cls.logger.debug('Success to pull docker image {}!'.format(image_name)) - return True - @classmethod - def pull_image(cls, docker_image): + def pull_image(self, docker_image): if not docker_image: return None - old_image_id = cls.get_image_id(docker_image) - if not cls.pull_image_only(docker_image): + old_image_id = self.get_image_id(docker_image) + if not self.pull_image_only(docker_image): return None - new_image_id = cls.get_image_id(docker_image) + new_image_id = self.get_image_id(docker_image) if not new_image_id: - cls.logger.error( - "Failed to get the id of image {}.".format(docker_image)) + self.logger.error( + 'Failed to get the id of image {}.'.format(docker_image)) return None if not old_image_id: return docker_image if new_image_id == old_image_id: - cls.logger.debug('Image {} has no changes, no need to remove.' - .format(docker_image)) + self.logger.debug('Image {} has no changes, no need to remove.' + .format(docker_image)) else: - cls.remove_image(old_image_id) + self.remove_image(old_image_id) return docker_image - @classmethod - def check_container_exist(cls, container_name): - cmd = ('sudo docker ps -aq -f name={}'.format(container_name)) - ret, msg = dt_utils.exec_cmd(cmd, cls.logger) - if ret == 0 and msg: - return True - return False - - @classmethod - def clean(cls, container_id, valid_type): - cmd = ('sudo docker rm -f {}'.format(container_id)) - dt_utils.exec_cmd(cmd, cls.logger) - if valid_type.lower() == 'bottlenecks': - containers = dt_cfg.dovetail_config[valid_type]['extra_container'] - for container in containers: - if cls.check_container_exist(container): - cmd = ('sudo docker rm -f {}'.format(container)) - dt_utils.exec_cmd(cmd, cls.logger) - - @classmethod - def exec_cmd(cls, container_id, sub_cmd, exit_on_error=False): - if sub_cmd == "": + def get_container(self, container_name): + try: + container = self.client.containers.get(container_name) + except (docker.errors.NotFound, docker.errors.APIError): + container = None + return container + + def clean(self): + try: + self.container.remove(force=True) + self.logger.debug( + 'container: {} was removed'.format(self.container.name)) + except docker.errors.APIError as e: + self.logger.error(e) + extra_containers = dt_utils.get_value_from_dict( + 'extra_container', dt_cfg.dovetail_config[self.valid_type]) + if extra_containers: + for container_name in extra_containers: + container = self.get_container(container_name) + if container: + try: + container.remove(force=True) + self.logger.debug( + 'container: {} was removed'.format(container_name)) + except docker.errors.APIError as e: + self.logger.error(e) + + def exec_cmd(self, sub_cmd, exit_on_error=False): + if not sub_cmd: return (1, 'sub_cmd is empty') - cmd = 'sudo docker exec %s /bin/bash -c "%s"' % (container_id, sub_cmd) - return dt_utils.exec_cmd(cmd, cls.logger, exit_on_error) - - @classmethod - def copy_file(cls, container_id, src_path, dest_path, - exit_on_error=False): + shell = dt_utils.get_value_from_dict( + 'shell', dt_cfg.dovetail_config[self.valid_type]) + if not shell: + return (1, 'shell is empty') + cmd = '{} -c "{}"'.format(shell, sub_cmd) + try: + result = self.container.exec_run(cmd) + except docker.errors.APIError as e: + result = (e.response.status_code, str(e)) + self.logger.error(e) + if exit_on_error: + sys.exit(1) + + return result + + def copy_file(self, src_path, dest_path, exit_on_error=False): if not src_path or not dest_path: return (1, 'src_path or dest_path is empty') cmd = 'cp %s %s' % (src_path, dest_path) - return cls.exec_cmd(container_id, cmd, exit_on_error) + return self.exec_cmd(cmd, exit_on_error) - @classmethod - def docker_copy(cls, container_id, src_path, dest_path): - if not src_path or not dest_path: - return (1, 'src_path or dest_path is empty') - cmd = 'docker cp %s %s:%s' % (src_path, container_id, dest_path) - return dt_utils.exec_cmd(cmd, cls.logger) - - @classmethod - def copy_files_in_container(cls, valid_type, container_id): - project_config = dt_cfg.dovetail_config[valid_type] + def copy_files_in_container(self): + project_config = dt_cfg.dovetail_config[self.valid_type] if 'copy_file_in_container' not in project_config.keys(): return if not project_config['copy_file_in_container']: return for item in project_config['copy_file_in_container']: - cls.copy_file(container_id, item['src_file'], item['dest_file']) - - @classmethod - def set_vnftest_conf_file(cls, container_id): - valid_type = 'vnftest' - for conf_file in dt_cfg.dovetail_config[valid_type]['vnftest_conf']: - src = conf_file['src_file'] - dest = conf_file['dest_file'] - cls.docker_copy(container_id, src, dest) + self.copy_file(item['src_file'], item['dest_file']) |