From 3cc7480d6af4c40fe9c14f50e365337619768cf5 Mon Sep 17 00:00:00 2001 From: chenjiankun Date: Tue, 27 Jun 2017 03:20:08 +0000 Subject: Kubernetes (k8s) support JIRA: YARDSTICK-682 We decide to support k8s in E release. We need to discuss with openretriver team and then rewrite the ping test case under k8s as the first step. Change-Id: I3f81ebca8de5c1f3a8b7d42581cd7342dc320239 Signed-off-by: chenjiankun --- yardstick/benchmark/contexts/kubernetes.py | 137 +++++++++++++++++++++++++++++ yardstick/common/constants.py | 1 + yardstick/common/kubernetes_utils.py | 137 +++++++++++++++++++++++++++++ yardstick/common/utils.py | 21 +++++ yardstick/orchestrator/kubernetes.py | 130 +++++++++++++++++++++++++++ 5 files changed, 426 insertions(+) create mode 100644 yardstick/benchmark/contexts/kubernetes.py create mode 100644 yardstick/common/kubernetes_utils.py create mode 100644 yardstick/orchestrator/kubernetes.py (limited to 'yardstick') diff --git a/yardstick/benchmark/contexts/kubernetes.py b/yardstick/benchmark/contexts/kubernetes.py new file mode 100644 index 000000000..cc3e326c6 --- /dev/null +++ b/yardstick/benchmark/contexts/kubernetes.py @@ -0,0 +1,137 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from __future__ import absolute_import +import logging +import time +import pkg_resources + +import paramiko + +from yardstick.benchmark.contexts.base import Context +from yardstick.orchestrator.kubernetes import KubernetesTemplate +from yardstick.common import kubernetes_utils as k8s_utils +from yardstick.common import utils + +LOG = logging.getLogger(__name__) +BITS_LENGTH = 2048 + + +class KubernetesContext(Context): + """Class that handle nodes info""" + + __context_type__ = "Kubernetes" + + def __init__(self): + self.name = '' + self.ssh_key = '' + self.key_path = '' + self.public_key_path = '' + self.template = None + + super(KubernetesContext, self).__init__() + + def init(self, attrs): + self.name = attrs.get('name', '') + + template_cfg = attrs.get('servers', {}) + self.template = KubernetesTemplate(self.name, template_cfg) + + self.ssh_key = '{}-key'.format(self.name) + + self.key_path = self._get_key_path() + self.public_key_path = '{}.pub'.format(self.key_path) + + def deploy(self): + LOG.info('Creating ssh key') + self._set_ssh_key() + + LOG.info('Launch containers') + self._create_rcs() + time.sleep(1) + self.template.get_rc_pods() + + self._wait_until_running() + + def undeploy(self): + self._delete_ssh_key() + self._delete_rcs() + self._delete_pods() + + super(KubernetesContext, self).undeploy() + + def _wait_until_running(self): + while not all(self._check_pod_status(p) for p in self.template.pods): + time.sleep(1) + + def _check_pod_status(self, pod): + status = k8s_utils.read_pod_status(pod) + LOG.debug('%s:%s', pod, status) + if status == 'Failed': + LOG.error('Pod %s status is failed', pod) + raise RuntimeError + if status != 'Running': + return False + return True + + def _create_rcs(self): + for obj in self.template.k8s_objs: + self._create_rc(obj.get_template()) + + def _create_rc(self, template): + k8s_utils.create_replication_controller(template) + + def _delete_rcs(self): + for rc in self.template.rcs: + self._delete_rc(rc) + + def _delete_rc(self, rc): + k8s_utils.delete_replication_controller(rc) + + def _delete_pods(self): + for pod in self.template.pods: + self._delete_pod(pod) + + def _delete_pod(self, pod): + k8s_utils.delete_pod(pod) + + def _get_key_path(self): + task_id = self.name.split('-')[-1] + k = 'files/yardstick_key-{}'.format(task_id) + return pkg_resources.resource_filename('yardstick.resources', k) + + def _set_ssh_key(self): + rsa_key = paramiko.RSAKey.generate(bits=BITS_LENGTH) + + LOG.info('Writing private key') + rsa_key.write_private_key_file(self.key_path) + + LOG.info('Writing public key') + key = '{} {}\n'.format(rsa_key.get_name(), rsa_key.get_base64()) + with open(self.public_key_path, 'w') as f: + f.write(key) + + LOG.info('Create configmap for ssh key') + k8s_utils.create_config_map(self.ssh_key, {'authorized_keys': key}) + + def _delete_ssh_key(self): + k8s_utils.delete_config_map(self.ssh_key) + utils.remove_file(self.key_path) + utils.remove_file(self.public_key_path) + + def _get_server(self, name): + resp = k8s_utils.get_pod_list() + hosts = ({'name': n.metadata.name, + 'ip': n.status.pod_ip, + 'user': 'root', + 'key_filename': self.key_path, + 'private_ip': n.status.pod_ip} + for n in resp.items if n.metadata.name.startswith(name)) + + return next(hosts, None) diff --git a/yardstick/common/constants.py b/yardstick/common/constants.py index 5cd34ba53..69485a4e4 100644 --- a/yardstick/common/constants.py +++ b/yardstick/common/constants.py @@ -56,6 +56,7 @@ ETC_HOSTS = get_param('file.etc_hosts', '/etc/hosts') CONF_FILE = join(CONF_DIR, 'yardstick.conf') POD_FILE = join(CONF_DIR, 'pod.yaml') CLOUDS_CONF = join(OPENSTACK_CONF_DIR, 'clouds.yml') +K8S_CONF_FILE = join(CONF_DIR, 'admin.conf') CONF_SAMPLE_FILE = join(CONF_SAMPLE_DIR, 'yardstick.conf.sample') FETCH_SCRIPT = get_param('file.fetch_script', 'utils/fetch_os_creds.sh') FETCH_SCRIPT = join(RELENG_DIR, FETCH_SCRIPT) diff --git a/yardstick/common/kubernetes_utils.py b/yardstick/common/kubernetes_utils.py new file mode 100644 index 000000000..e4c232830 --- /dev/null +++ b/yardstick/common/kubernetes_utils.py @@ -0,0 +1,137 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import logging + +from kubernetes import client +from kubernetes import config +from kubernetes.client.rest import ApiException + +from yardstick.common import constants as consts + +LOG = logging.getLogger(__name__) +LOG.setLevel(logging.DEBUG) + + +def get_core_api(): # pragma: no cover + try: + config.load_kube_config(config_file=consts.K8S_CONF_FILE) + except IOError: + LOG.exception('config file not found') + raise + + return client.CoreV1Api() + + +def create_replication_controller(template, + namespace='default', + wait=False, + **kwargs): # pragma: no cover + + core_v1_api = get_core_api() + try: + core_v1_api.create_namespaced_replication_controller(namespace, + template, + **kwargs) + except ApiException: + LOG.exception('Create replication controller failed') + raise + + +def delete_replication_controller(name, + namespace='default', + wait=False, + **kwargs): # pragma: no cover + + core_v1_api = get_core_api() + body = kwargs.get('body', client.V1DeleteOptions()) + kwargs.pop('body', None) + try: + core_v1_api.delete_namespaced_replication_controller(name, + namespace, + body, + **kwargs) + except ApiException: + LOG.exception('Delete replication controller failed') + raise + + +def delete_pod(name, + namespace='default', + wait=False, + **kwargs): # pragma: no cover + + core_v1_api = get_core_api() + body = kwargs.get('body', client.V1DeleteOptions()) + kwargs.pop('body', None) + try: + core_v1_api.delete_namespaced_pod(name, + namespace, + body, + **kwargs) + except ApiException: + LOG.exception('Delete pod failed') + raise + + +def read_pod(name, + namespace='default', + **kwargs): # pragma: no cover + core_v1_api = get_core_api() + try: + resp = core_v1_api.read_namespaced_pod(name, namespace, **kwargs) + except ApiException: + LOG.exception('Read pod failed') + raise + else: + return resp + + +def read_pod_status(name, namespace='default', **kwargs): # pragma: no cover + return read_pod(name).status.phase + + +def create_config_map(name, + data, + namespace='default', + wait=False, + **kwargs): # pragma: no cover + core_v1_api = get_core_api() + metadata = client.V1ObjectMeta(name=name) + body = client.V1ConfigMap(data=data, metadata=metadata) + try: + core_v1_api.create_namespaced_config_map(namespace, body, **kwargs) + except ApiException: + LOG.exception('Create config map failed') + raise + + +def delete_config_map(name, + namespace='default', + wait=False, + **kwargs): # pragma: no cover + core_v1_api = get_core_api() + body = kwargs.get('body', client.V1DeleteOptions()) + kwargs.pop('body', None) + try: + core_v1_api.delete_namespaced_config_map(name, + namespace, + body, + **kwargs) + except ApiException: + LOG.exception('Delete config map failed') + raise + + +def get_pod_list(namespace='default'): # pragma: no cover + core_v1_api = get_core_api() + try: + return core_v1_api.list_namespaced_pod(namespace=namespace) + except ApiException: + LOG.exception('Get pod list failed') + raise diff --git a/yardstick/common/utils.py b/yardstick/common/utils.py index 0c0bac934..a4f7b30dc 100644 --- a/yardstick/common/utils.py +++ b/yardstick/common/utils.py @@ -124,6 +124,14 @@ def makedirs(d): raise +def remove_file(path): + try: + os.remove(path) + except OSError as e: + if e.errno != errno.ENOENT: + raise + + def execute_command(cmd): exec_msg = "Executing command: '%s'" % cmd logger.debug(exec_msg) @@ -242,3 +250,16 @@ def change_obj_to_dict(obj): except TypeError: dic.update({k: v}) return dic + + +def set_dict_value(dic, keys, value): + return_dic = dic + + for key in keys.split('.'): + + return_dic.setdefault(key, {}) + if key == keys.split('.')[-1]: + return_dic[key] = value + else: + return_dic = return_dic[key] + return dic diff --git a/yardstick/orchestrator/kubernetes.py b/yardstick/orchestrator/kubernetes.py new file mode 100644 index 000000000..6d7045f58 --- /dev/null +++ b/yardstick/orchestrator/kubernetes.py @@ -0,0 +1,130 @@ +############################################################################## +# Copyright (c) 2017 Huawei Technologies Co.,Ltd. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +from __future__ import absolute_import +from __future__ import print_function + +from yardstick.common import utils +from yardstick.common import kubernetes_utils as k8s_utils + + +class KubernetesObject(object): + + def __init__(self, name, **kwargs): + super(KubernetesObject, self).__init__() + self.name = name + self.image = kwargs.get('image', 'openretriever/yardstick') + self.command = [kwargs.get('command', '/bin/bash')] + self.args = kwargs.get('args', []) + self.ssh_key = kwargs.get('ssh_key', 'yardstick_key') + + self.volumes = [] + + self.template = { + "apiVersion": "v1", + "kind": "ReplicationController", + "metadata": { + "name": "" + }, + "spec": { + "replicas": 1, + "template": { + "metadata": { + "labels": { + "app": "" + } + }, + "spec": { + "containers": [], + "volumes": [] + } + } + } + } + + self._change_value_according_name(name) + self._add_containers() + self._add_ssh_key_volume() + self._add_volumes() + + def get_template(self): + return self.template + + def _change_value_according_name(self, name): + utils.set_dict_value(self.template, 'metadata.name', name) + + utils.set_dict_value(self.template, + 'spec.template.metadata.labels.app', + name) + + def _add_containers(self): + containers = [self._add_container()] + utils.set_dict_value(self.template, + 'spec.template.spec.containers', + containers) + + def _add_container(self): + container_name = '{}-container'.format(self.name) + ssh_key_mount_path = "/root/.ssh/" + + container = { + "args": self.args, + "command": self.command, + "image": self.image, + "name": container_name, + "volumeMounts": [ + { + "mountPath": ssh_key_mount_path, + "name": self.ssh_key + } + ] + } + + return container + + def _add_volumes(self): + utils.set_dict_value(self.template, + 'spec.template.spec.volumes', + self.volumes) + + def _add_volume(self, volume): + self.volumes.append(volume) + + def _add_ssh_key_volume(self): + key_volume = { + "configMap": { + "name": self.ssh_key + }, + "name": self.ssh_key + } + self._add_volume(key_volume) + + +class KubernetesTemplate(object): + + def __init__(self, name, template_cfg): + self.name = name + self.ssh_key = '{}-key'.format(name) + + self.rcs = [self._get_rc_name(rc) for rc in template_cfg] + self.k8s_objs = [KubernetesObject(self._get_rc_name(rc), + ssh_key=self.ssh_key, + **cfg) + for rc, cfg in template_cfg.items()] + self.pods = [] + + def _get_rc_name(self, rc_name): + return '{}-{}'.format(rc_name, self.name) + + def get_rc_pods(self): + resp = k8s_utils.get_pod_list() + self.pods = [p.metadata.name for p in resp.items for s in self.rcs + if p.metadata.name.startswith(s)] + + return self.pods -- cgit 1.2.3-korg