From 10c4d35315d7ffd909520a1c7bc6a3b5b9b871ab Mon Sep 17 00:00:00 2001 From: Zenghui Shi Date: Wed, 4 Apr 2018 11:24:40 +0800 Subject: Add support for kubernetes deployment This patch adds capability to deploy kubernetes cluster instead of openstack. Kubernetes will be deployed using kubespray and is run after TripleO bookstraps overcloud nodes. JIRA: APEX-574 Change-Id: If9c171620c933a052b719e7112a50e22bbab667f Signed-off-by: Feng Pan Signed-off-by: Zenghui Shi --- apex/common/constants.py | 1 + apex/common/utils.py | 7 +- apex/deploy.py | 108 +++++++++++++++++++++++++++++-- apex/overcloud/deploy.py | 5 ++ apex/settings/deploy_settings.py | 5 +- apex/tests/test_apex_common_utils.py | 3 + apex/tests/test_apex_deploy.py | 62 +++++++++++++++++- apex/tests/test_apex_overcloud_deploy.py | 6 +- 8 files changed, 185 insertions(+), 12 deletions(-) (limited to 'apex') diff --git a/apex/common/constants.py b/apex/common/constants.py index 89c3e6e1..ee260b4f 100644 --- a/apex/common/constants.py +++ b/apex/common/constants.py @@ -70,3 +70,4 @@ VALID_DOCKER_SERVICES = { } DOCKERHUB_OOO = 'https://registry.hub.docker.com/v2/repositories' \ '/tripleomaster/' +KUBESPRAY_URL = 'https://github.com/kubernetes-incubator/kubespray.git' diff --git a/apex/common/utils.py b/apex/common/utils.py index 2ac900a3..013c7ac8 100644 --- a/apex/common/utils.py +++ b/apex/common/utils.py @@ -75,12 +75,17 @@ def run_ansible(ansible_vars, playbook, host='localhost', user='root', Executes ansible playbook and checks for errors :param ansible_vars: dictionary of variables to inject into ansible run :param playbook: playbook to execute + :param host: inventory file or string of target hosts + :param user: remote user to run ansible tasks :param tmp_dir: temp directory to store ansible command :param dry_run: Do not actually apply changes :return: None """ logging.info("Executing ansible playbook: {}".format(playbook)) - inv_host = "{},".format(host) + if not os.path.isfile(host): + inv_host = "{},".format(host) + else: + inv_host = host if host == 'localhost': conn_type = 'local' else: diff --git a/apex/deploy.py b/apex/deploy.py index 635a5d07..1e477ee0 100644 --- a/apex/deploy.py +++ b/apex/deploy.py @@ -10,6 +10,7 @@ ############################################################################## import argparse +import git import json import logging import os @@ -18,6 +19,7 @@ import pprint import shutil import sys import tempfile +import yaml import apex.virtual.configure_vm as vm_lib import apex.virtual.utils as virt_utils @@ -244,10 +246,10 @@ def main(): # Parse all settings deploy_settings = DeploySettings(args.deploy_settings_file) logging.info("Deploy settings are:\n {}".format(pprint.pformat( - deploy_settings))) + deploy_settings))) net_settings = NetworkSettings(args.network_settings_file) logging.info("Network settings are:\n {}".format(pprint.pformat( - net_settings))) + net_settings))) os_version = deploy_settings['deploy_options']['os_version'] net_env_file = os.path.join(args.deploy_dir, constants.NET_ENV_FILE) net_env = NetworkEnvironment(net_settings, net_env_file, @@ -468,7 +470,8 @@ def main(): 'deploy_overcloud.yml') virt_env = 'virtual-environment.yaml' bm_env = 'baremetal-environment.yaml' - for p_env in virt_env, bm_env: + k8s_env = 'kubernetes-environment.yaml' + for p_env in virt_env, bm_env, k8s_env: shutil.copyfile(os.path.join(args.deploy_dir, p_env), os.path.join(APEX_TEMP_DIR, p_env)) @@ -491,6 +494,7 @@ def main(): deploy_vars['os_version'] = os_version deploy_vars['http_proxy'] = net_settings.get('http_proxy', '') deploy_vars['https_proxy'] = net_settings.get('https_proxy', '') + deploy_vars['vim'] = ds_opts['vim'] for dns_server in net_settings['dns_servers']: deploy_vars['dns_server_args'] += " --dns-nameserver {}".format( dns_server) @@ -544,17 +548,107 @@ def main(): else: deploy_vars['overcloudrc_files'] = ['overcloudrc'] - post_undercloud = os.path.join(args.lib_dir, constants.ANSIBLE_PATH, + post_undercloud = os.path.join(args.lib_dir, + constants.ANSIBLE_PATH, 'post_deploy_undercloud.yml') - logging.info("Executing post deploy configuration undercloud playbook") + logging.info("Executing post deploy configuration undercloud " + "playbook") try: - utils.run_ansible(deploy_vars, post_undercloud, host=undercloud.ip, - user='stack', tmp_dir=APEX_TEMP_DIR) + utils.run_ansible(deploy_vars, post_undercloud, + host=undercloud.ip, user='stack', + tmp_dir=APEX_TEMP_DIR) logging.info("Post Deploy Undercloud Configuration Complete") except Exception: logging.error("Post Deploy Undercloud Configuration failed. " "Please check log") raise + + # Deploy kubernetes if enabled + # (TODO)zshi move handling of kubernetes deployment + # to its own deployment class + if deploy_vars['vim'] == 'k8s': + # clone kubespray repo + git.Repo.clone_from(constants.KUBESPRAY_URL, + os.path.join(APEX_TEMP_DIR, 'kubespray')) + shutil.copytree( + os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory', + 'sample'), + os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory', + 'apex')) + k8s_node_inventory = { + 'all': + {'hosts': {}, + 'children': { + 'k8s-cluster': { + 'children': { + 'kube-master': { + 'hosts': {} + }, + 'kube-node': { + 'hosts': {} + } + } + }, + 'etcd': { + 'hosts': {} + } + } + } + } + for node, ip in deploy_vars['overcloud_nodes'].items(): + k8s_node_inventory['all']['hosts'][node] = { + 'ansible_become': True, + 'ansible_ssh_host': ip, + 'ansible_become_user': 'root', + 'ip': ip + } + if 'controller' in node: + k8s_node_inventory['all']['children']['k8s-cluster'][ + 'children']['kube-master']['hosts'][node] = None + k8s_node_inventory['all']['children']['etcd'][ + 'hosts'][node] = None + elif 'compute' in node: + k8s_node_inventory['all']['children']['k8s-cluster'][ + 'children']['kube-node']['hosts'][node] = None + + kubespray_dir = os.path.join(APEX_TEMP_DIR, 'kubespray') + with open(os.path.join(kubespray_dir, 'inventory', 'apex', + 'apex.yaml'), 'w') as invfile: + yaml.dump(k8s_node_inventory, invfile, + default_flow_style=False) + k8s_deploy_vars = {} + # Add kubespray ansible control variables in k8s_deploy_vars, + # example: 'kube_network_plugin': 'flannel' + k8s_deploy = os.path.join(kubespray_dir, 'cluster.yml') + k8s_deploy_inv_file = os.path.join(kubespray_dir, 'inventory', + 'apex', 'apex.yaml') + + k8s_remove_pkgs = os.path.join(args.lib_dir, + constants.ANSIBLE_PATH, + 'k8s_remove_pkgs.yml') + try: + logging.debug("Removing any existing overcloud docker " + "packages") + utils.run_ansible(k8s_deploy_vars, k8s_remove_pkgs, + host=k8s_deploy_inv_file, + user='heat-admin', tmp_dir=APEX_TEMP_DIR) + logging.info("k8s Deploy Remove Existing Docker Related " + "Packages Complete") + except Exception: + logging.error("k8s Deploy Remove Existing Docker Related " + "Packages failed. Please check log") + raise + + try: + utils.run_ansible(k8s_deploy_vars, k8s_deploy, + host=k8s_deploy_inv_file, + user='heat-admin', tmp_dir=APEX_TEMP_DIR) + logging.info("k8s Deploy Overcloud Configuration Complete") + except Exception: + logging.error("k8s Deploy Overcloud Configuration failed." + "Please check log") + raise + # Post deploy overcloud node configuration # TODO(trozet): just parse all ds_opts as deploy vars one time deploy_vars['sfc'] = ds_opts['sfc'] diff --git a/apex/overcloud/deploy.py b/apex/overcloud/deploy.py index 2f33183c..92b42fff 100644 --- a/apex/overcloud/deploy.py +++ b/apex/overcloud/deploy.py @@ -204,6 +204,11 @@ def create_deploy_cmd(ds, ns, inv, tmp_dir, if ds_opts['sriov']: prep_sriov_env(ds, tmp_dir) + # Check for 'k8s' here intentionally, as we may support other values + # such as openstack/openshift for 'vim' option. + if ds_opts['vim'] == 'k8s': + deploy_options.append('kubernetes-environment.yaml') + if virtual: deploy_options.append('virtual-environment.yaml') else: diff --git a/apex/settings/deploy_settings.py b/apex/settings/deploy_settings.py index 29fe64fb..00e6d6c0 100644 --- a/apex/settings/deploy_settings.py +++ b/apex/settings/deploy_settings.py @@ -27,7 +27,8 @@ REQ_DEPLOY_SETTINGS = ['sdn_controller', 'l2gw', 'sriov', 'containers', - 'ceph_device'] + 'ceph_device', + 'vim'] OPT_DEPLOY_SETTINGS = ['performance', 'vsperf', @@ -113,6 +114,8 @@ class DeploySettings(dict): elif req_set == 'os_version': self['deploy_options'][req_set] = \ constants.DEFAULT_OS_VERSION + elif req_set == 'vim': + self['deploy_options'][req_set] = 'openstack' else: self['deploy_options'][req_set] = False elif req_set == 'odl_version' and self['deploy_options'][ diff --git a/apex/tests/test_apex_common_utils.py b/apex/tests/test_apex_common_utils.py index 0e4041ca..b6aa4c7f 100644 --- a/apex/tests/test_apex_common_utils.py +++ b/apex/tests/test_apex_common_utils.py @@ -64,8 +64,11 @@ class TestCommonUtils: def test_run_ansible(self): playbook = 'apex/tests/playbooks/test_playbook.yaml' + extra_vars = [{'testvar1': 'value1', 'testvar2': 'value2'}] assert_equal(utils.run_ansible(None, os.path.join(playbook), dry_run=True), None) + assert_equal(utils.run_ansible(extra_vars, os.path.join(playbook), + dry_run=True, host='1.1.1.1'), None) def test_failed_run_ansible(self): playbook = 'apex/tests/playbooks/test_failed_playbook.yaml' diff --git a/apex/tests/test_apex_deploy.py b/apex/tests/test_apex_deploy.py index b7941f6f..8e9756eb 100644 --- a/apex/tests/test_apex_deploy.py +++ b/apex/tests/test_apex_deploy.py @@ -144,6 +144,7 @@ class TestDeploy(unittest.TestCase): 'dataplane': 'ovs', 'sfc': False, 'vpn': False, + 'vim': 'openstack', 'yardstick': 'test', 'os_version': DEFAULT_OS_VERSION, 'containers': False}} @@ -220,6 +221,7 @@ class TestDeploy(unittest.TestCase): 'dataplane': 'ovs', 'sfc': False, 'vpn': False, + 'vim': 'openstack', 'yardstick': 'test', 'os_version': DEFAULT_OS_VERSION, 'containers': False}} @@ -281,6 +283,7 @@ class TestDeploy(unittest.TestCase): 'dataplane': 'ovs', 'sfc': False, 'vpn': False, + 'vim': 'openstack', 'yardstick': 'test', 'os_version': DEFAULT_OS_VERSION, 'containers': True}} @@ -303,6 +306,63 @@ class TestDeploy(unittest.TestCase): args.virt_compute_ram = 16 args.virt_default_ram = 10 main() - mock_oc_deploy.prep_image.assert_called + mock_oc_deploy.prep_image.assert_called() # TODO(trozet) add assertions here with arguments for functions in # deploy main + + @patch('apex.deploy.uc_builder') + @patch('apex.deploy.network_data.create_network_data') + @patch('apex.deploy.shutil') + @patch('apex.deploy.git') + @patch('apex.deploy.oc_deploy') + @patch('apex.deploy.uc_lib') + @patch('apex.deploy.build_vms') + @patch('apex.deploy.Inventory') + @patch('apex.deploy.virt_utils') + @patch('apex.deploy.oc_cfg') + @patch('apex.deploy.parsers') + @patch('apex.deploy.utils') + @patch('apex.deploy.NetworkEnvironment') + @patch('apex.deploy.NetworkSettings') + @patch('apex.deploy.DeploySettings') + @patch('apex.deploy.os') + @patch('apex.deploy.json') + @patch('apex.deploy.jumphost') + @patch('apex.deploy.validate_cross_settings') + @patch('apex.deploy.validate_deploy_args') + @patch('apex.deploy.create_deploy_parser') + @patch('builtins.open', a_mock_open, create=True) + def test_main_k8s(self, mock_parser, mock_val_args, mock_cross_sets, + mock_jumphost, mock_json, mock_os, + mock_deploy_sets, mock_net_sets, mock_net_env, + mock_utils, mock_parsers, mock_oc_cfg, + mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib, + mock_oc_deploy, mock_git, mock_shutil, + mock_network_data, mock_uc_builder): + net_sets_dict = {'networks': MagicMock(), + 'dns_servers': 'test'} + ds_opts_dict = {'global_params': MagicMock(), + 'deploy_options': {'gluon': False, + 'congress': True, + 'sdn_controller': False, + 'dataplane': 'ovs', + 'sfc': False, + 'vpn': False, + 'vim': 'k8s', + 'yardstick': 'test', + 'os_version': DEFAULT_OS_VERSION, + 'containers': False}} + args = mock_parser.return_value.parse_args.return_value + args.virtual = False + args.quickstart = False + args.debug = False + args.upstream = False + net_sets = mock_net_sets.return_value + net_sets.enabled_network_list = ['external'] + net_sets.__getitem__.side_effect = net_sets_dict.__getitem__ + net_sets.__contains__.side_effect = net_sets_dict.__contains__ + deploy_sets = mock_deploy_sets.return_value + deploy_sets.__getitem__.side_effect = ds_opts_dict.__getitem__ + deploy_sets.__contains__.side_effect = ds_opts_dict.__contains__ + mock_parsers.parse_nova_output.return_value = {'testnode1': 'test'} + main() diff --git a/apex/tests/test_apex_overcloud_deploy.py b/apex/tests/test_apex_overcloud_deploy.py index 83e2b02d..57d74bdd 100644 --- a/apex/tests/test_apex_overcloud_deploy.py +++ b/apex/tests/test_apex_overcloud_deploy.py @@ -104,7 +104,8 @@ class TestOvercloudDeploy(unittest.TestCase): 'containers': False, 'barometer': True, 'ceph': False, - 'sriov': False + 'sriov': False, + 'vim': 'openstack' }, 'global_params': MagicMock()} @@ -135,7 +136,8 @@ class TestOvercloudDeploy(unittest.TestCase): 'ceph': True, 'sdn_controller': 'opendaylight', 'sriov': False, - 'os_version': 'queens' + 'os_version': 'queens', + 'vim': 'openstack' }, 'global_params': MagicMock()} -- cgit 1.2.3-korg