Add support for kubernetes deployment

This patch adds capability to deploy kubernetes cluster instead of openstack.
Kubernetes will be deployed using kubespray and is run after TripleO bookstraps
overcloud nodes.

JIRA: APEX-574

Change-Id: If9c171620c933a052b719e7112a50e22bbab667f
Signed-off-by: Feng Pan <fpan@redhat.com>
Signed-off-by: Zenghui Shi <zshi@redhat.com>

1 files changed, 101 insertions, 7 deletions

diff --git a/apex/deploy.py b/apex/deploy.py
index 635a5d07..1e477ee0 100644
--- a/apex/deploy.py
+++ b/apex/deploy.py
@@ -10,6 +10,7 @@
 ##############################################################################
 
 import argparse
+import git
 import json
 import logging
 import os
@@ -18,6 +19,7 @@ import pprint
 import shutil
 import sys
 import tempfile
+import yaml
 
 import apex.virtual.configure_vm as vm_lib
 import apex.virtual.utils as virt_utils
@@ -244,10 +246,10 @@ def main():
     # Parse all settings
     deploy_settings = DeploySettings(args.deploy_settings_file)
     logging.info("Deploy settings are:\n {}".format(pprint.pformat(
-                 deploy_settings)))
+        deploy_settings)))
     net_settings = NetworkSettings(args.network_settings_file)
     logging.info("Network settings are:\n {}".format(pprint.pformat(
-                 net_settings)))
+        net_settings)))
     os_version = deploy_settings['deploy_options']['os_version']
     net_env_file = os.path.join(args.deploy_dir, constants.NET_ENV_FILE)
     net_env = NetworkEnvironment(net_settings, net_env_file,
@@ -468,7 +470,8 @@ def main():
                                        'deploy_overcloud.yml')
         virt_env = 'virtual-environment.yaml'
         bm_env = 'baremetal-environment.yaml'
-        for p_env in virt_env, bm_env:
+        k8s_env = 'kubernetes-environment.yaml'
+        for p_env in virt_env, bm_env, k8s_env:
             shutil.copyfile(os.path.join(args.deploy_dir, p_env),
                             os.path.join(APEX_TEMP_DIR, p_env))
 
@@ -491,6 +494,7 @@ def main():
         deploy_vars['os_version'] = os_version
         deploy_vars['http_proxy'] = net_settings.get('http_proxy', '')
         deploy_vars['https_proxy'] = net_settings.get('https_proxy', '')
+        deploy_vars['vim'] = ds_opts['vim']
         for dns_server in net_settings['dns_servers']:
             deploy_vars['dns_server_args'] += " --dns-nameserver {}".format(
                 dns_server)
@@ -544,17 +548,107 @@ def main():
         else:
             deploy_vars['overcloudrc_files'] = ['overcloudrc']
 
-        post_undercloud = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
+        post_undercloud = os.path.join(args.lib_dir,
+                                       constants.ANSIBLE_PATH,
                                        'post_deploy_undercloud.yml')
-        logging.info("Executing post deploy configuration undercloud playbook")
+        logging.info("Executing post deploy configuration undercloud "
+                     "playbook")
         try:
-            utils.run_ansible(deploy_vars, post_undercloud, host=undercloud.ip,
-                              user='stack', tmp_dir=APEX_TEMP_DIR)
+            utils.run_ansible(deploy_vars, post_undercloud,
+                              host=undercloud.ip, user='stack',
+                              tmp_dir=APEX_TEMP_DIR)
             logging.info("Post Deploy Undercloud Configuration Complete")
         except Exception:
             logging.error("Post Deploy Undercloud Configuration failed.  "
                           "Please check log")
             raise
+
+        # Deploy kubernetes if enabled
+        # (TODO)zshi move handling of kubernetes deployment
+        # to its own deployment class
+        if deploy_vars['vim'] == 'k8s':
+            # clone kubespray repo
+            git.Repo.clone_from(constants.KUBESPRAY_URL,
+                                os.path.join(APEX_TEMP_DIR, 'kubespray'))
+            shutil.copytree(
+                os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory',
+                             'sample'),
+                os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory',
+                             'apex'))
+            k8s_node_inventory = {
+                'all':
+                    {'hosts': {},
+                     'children': {
+                         'k8s-cluster': {
+                             'children': {
+                                 'kube-master': {
+                                     'hosts': {}
+                                 },
+                                 'kube-node': {
+                                     'hosts': {}
+                                 }
+                             }
+                         },
+                         'etcd': {
+                             'hosts': {}
+                         }
+                    }
+                    }
+            }
+            for node, ip in deploy_vars['overcloud_nodes'].items():
+                k8s_node_inventory['all']['hosts'][node] = {
+                    'ansible_become': True,
+                    'ansible_ssh_host': ip,
+                    'ansible_become_user': 'root',
+                    'ip': ip
+                }
+                if 'controller' in node:
+                    k8s_node_inventory['all']['children']['k8s-cluster'][
+                        'children']['kube-master']['hosts'][node] = None
+                    k8s_node_inventory['all']['children']['etcd'][
+                        'hosts'][node] = None
+                elif 'compute' in node:
+                    k8s_node_inventory['all']['children']['k8s-cluster'][
+                        'children']['kube-node']['hosts'][node] = None
+
+            kubespray_dir = os.path.join(APEX_TEMP_DIR, 'kubespray')
+            with open(os.path.join(kubespray_dir, 'inventory', 'apex',
+                                   'apex.yaml'), 'w') as invfile:
+                yaml.dump(k8s_node_inventory, invfile,
+                          default_flow_style=False)
+            k8s_deploy_vars = {}
+            # Add kubespray ansible control variables in k8s_deploy_vars,
+            # example: 'kube_network_plugin': 'flannel'
+            k8s_deploy = os.path.join(kubespray_dir, 'cluster.yml')
+            k8s_deploy_inv_file = os.path.join(kubespray_dir, 'inventory',
+                                               'apex', 'apex.yaml')
+
+            k8s_remove_pkgs = os.path.join(args.lib_dir,
+                                           constants.ANSIBLE_PATH,
+                                           'k8s_remove_pkgs.yml')
+            try:
+                logging.debug("Removing any existing overcloud docker "
+                              "packages")
+                utils.run_ansible(k8s_deploy_vars, k8s_remove_pkgs,
+                                  host=k8s_deploy_inv_file,
+                                  user='heat-admin', tmp_dir=APEX_TEMP_DIR)
+                logging.info("k8s Deploy Remove Existing Docker Related "
+                             "Packages Complete")
+            except Exception:
+                logging.error("k8s Deploy Remove Existing Docker Related "
+                              "Packages failed. Please check log")
+                raise
+
+            try:
+                utils.run_ansible(k8s_deploy_vars, k8s_deploy,
+                                  host=k8s_deploy_inv_file,
+                                  user='heat-admin', tmp_dir=APEX_TEMP_DIR)
+                logging.info("k8s Deploy Overcloud Configuration Complete")
+            except Exception:
+                logging.error("k8s Deploy Overcloud Configuration failed."
+                              "Please check log")
+                raise
+
         # Post deploy overcloud node configuration
         # TODO(trozet): just parse all ds_opts as deploy vars one time
         deploy_vars['sfc'] = ds_opts['sfc']