Add support for kubernetes deployment

This patch adds capability to deploy kubernetes cluster instead of openstack.
Kubernetes will be deployed using kubespray and is run after TripleO bookstraps
overcloud nodes.

JIRA: APEX-574

Change-Id: If9c171620c933a052b719e7112a50e22bbab667f
Signed-off-by: Feng Pan <fpan@redhat.com>
Signed-off-by: Zenghui Shi <zshi@redhat.com>

8 files changed, 185 insertions, 12 deletions

diff --git a/apex/common/constants.py b/apex/common/constants.py
index 89c3e6e1..ee260b4f 100644
--- a/apex/common/constants.py
+++ b/apex/common/constants.py
@@ -70,3 +70,4 @@ VALID_DOCKER_SERVICES = {
 }
 DOCKERHUB_OOO = 'https://registry.hub.docker.com/v2/repositories' \
                 '/tripleomaster/'
+KUBESPRAY_URL = 'https://github.com/kubernetes-incubator/kubespray.git'
diff --git a/apex/common/utils.py b/apex/common/utils.py
index 2ac900a3..013c7ac8 100644
--- a/apex/common/utils.py
+++ b/apex/common/utils.py
@@ -75,12 +75,17 @@ def run_ansible(ansible_vars, playbook, host='localhost', user='root',
     Executes ansible playbook and checks for errors
     :param ansible_vars: dictionary of variables to inject into ansible run
     :param playbook: playbook to execute
+    :param host: inventory file or string of target hosts
+    :param user: remote user to run ansible tasks
     :param tmp_dir: temp directory to store ansible command
     :param dry_run: Do not actually apply changes
     :return: None
     """
     logging.info("Executing ansible playbook: {}".format(playbook))
-    inv_host = "{},".format(host)
+    if not os.path.isfile(host):
+        inv_host = "{},".format(host)
+    else:
+        inv_host = host
     if host == 'localhost':
         conn_type = 'local'
     else:
diff --git a/apex/deploy.py b/apex/deploy.py
index 635a5d07..1e477ee0 100644
--- a/apex/deploy.py
+++ b/apex/deploy.py
@@ -10,6 +10,7 @@
 ##############################################################################
 
 import argparse
+import git
 import json
 import logging
 import os
@@ -18,6 +19,7 @@ import pprint
 import shutil
 import sys
 import tempfile
+import yaml
 
 import apex.virtual.configure_vm as vm_lib
 import apex.virtual.utils as virt_utils
@@ -244,10 +246,10 @@ def main():
     # Parse all settings
     deploy_settings = DeploySettings(args.deploy_settings_file)
     logging.info("Deploy settings are:\n {}".format(pprint.pformat(
-                 deploy_settings)))
+        deploy_settings)))
     net_settings = NetworkSettings(args.network_settings_file)
     logging.info("Network settings are:\n {}".format(pprint.pformat(
-                 net_settings)))
+        net_settings)))
     os_version = deploy_settings['deploy_options']['os_version']
     net_env_file = os.path.join(args.deploy_dir, constants.NET_ENV_FILE)
     net_env = NetworkEnvironment(net_settings, net_env_file,
@@ -468,7 +470,8 @@ def main():
                                        'deploy_overcloud.yml')
         virt_env = 'virtual-environment.yaml'
         bm_env = 'baremetal-environment.yaml'
-        for p_env in virt_env, bm_env:
+        k8s_env = 'kubernetes-environment.yaml'
+        for p_env in virt_env, bm_env, k8s_env:
             shutil.copyfile(os.path.join(args.deploy_dir, p_env),
                             os.path.join(APEX_TEMP_DIR, p_env))
 
@@ -491,6 +494,7 @@ def main():
         deploy_vars['os_version'] = os_version
         deploy_vars['http_proxy'] = net_settings.get('http_proxy', '')
         deploy_vars['https_proxy'] = net_settings.get('https_proxy', '')
+        deploy_vars['vim'] = ds_opts['vim']
         for dns_server in net_settings['dns_servers']:
             deploy_vars['dns_server_args'] += " --dns-nameserver {}".format(
                 dns_server)
@@ -544,17 +548,107 @@ def main():
         else:
             deploy_vars['overcloudrc_files'] = ['overcloudrc']
 
-        post_undercloud = os.path.join(args.lib_dir, constants.ANSIBLE_PATH,
+        post_undercloud = os.path.join(args.lib_dir,
+                                       constants.ANSIBLE_PATH,
                                        'post_deploy_undercloud.yml')
-        logging.info("Executing post deploy configuration undercloud playbook")
+        logging.info("Executing post deploy configuration undercloud "
+                     "playbook")
         try:
-            utils.run_ansible(deploy_vars, post_undercloud, host=undercloud.ip,
-                              user='stack', tmp_dir=APEX_TEMP_DIR)
+            utils.run_ansible(deploy_vars, post_undercloud,
+                              host=undercloud.ip, user='stack',
+                              tmp_dir=APEX_TEMP_DIR)
             logging.info("Post Deploy Undercloud Configuration Complete")
         except Exception:
             logging.error("Post Deploy Undercloud Configuration failed.  "
                           "Please check log")
             raise
+
+        # Deploy kubernetes if enabled
+        # (TODO)zshi move handling of kubernetes deployment
+        # to its own deployment class
+        if deploy_vars['vim'] == 'k8s':
+            # clone kubespray repo
+            git.Repo.clone_from(constants.KUBESPRAY_URL,
+                                os.path.join(APEX_TEMP_DIR, 'kubespray'))
+            shutil.copytree(
+                os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory',
+                             'sample'),
+                os.path.join(APEX_TEMP_DIR, 'kubespray', 'inventory',
+                             'apex'))
+            k8s_node_inventory = {
+                'all':
+                    {'hosts': {},
+                     'children': {
+                         'k8s-cluster': {
+                             'children': {
+                                 'kube-master': {
+                                     'hosts': {}
+                                 },
+                                 'kube-node': {
+                                     'hosts': {}
+                                 }
+                             }
+                         },
+                         'etcd': {
+                             'hosts': {}
+                         }
+                    }
+                    }
+            }
+            for node, ip in deploy_vars['overcloud_nodes'].items():
+                k8s_node_inventory['all']['hosts'][node] = {
+                    'ansible_become': True,
+                    'ansible_ssh_host': ip,
+                    'ansible_become_user': 'root',
+                    'ip': ip
+                }
+                if 'controller' in node:
+                    k8s_node_inventory['all']['children']['k8s-cluster'][
+                        'children']['kube-master']['hosts'][node] = None
+                    k8s_node_inventory['all']['children']['etcd'][
+                        'hosts'][node] = None
+                elif 'compute' in node:
+                    k8s_node_inventory['all']['children']['k8s-cluster'][
+                        'children']['kube-node']['hosts'][node] = None
+
+            kubespray_dir = os.path.join(APEX_TEMP_DIR, 'kubespray')
+            with open(os.path.join(kubespray_dir, 'inventory', 'apex',
+                                   'apex.yaml'), 'w') as invfile:
+                yaml.dump(k8s_node_inventory, invfile,
+                          default_flow_style=False)
+            k8s_deploy_vars = {}
+            # Add kubespray ansible control variables in k8s_deploy_vars,
+            # example: 'kube_network_plugin': 'flannel'
+            k8s_deploy = os.path.join(kubespray_dir, 'cluster.yml')
+            k8s_deploy_inv_file = os.path.join(kubespray_dir, 'inventory',
+                                               'apex', 'apex.yaml')
+
+            k8s_remove_pkgs = os.path.join(args.lib_dir,
+                                           constants.ANSIBLE_PATH,
+                                           'k8s_remove_pkgs.yml')
+            try:
+                logging.debug("Removing any existing overcloud docker "
+                              "packages")
+                utils.run_ansible(k8s_deploy_vars, k8s_remove_pkgs,
+                                  host=k8s_deploy_inv_file,
+                                  user='heat-admin', tmp_dir=APEX_TEMP_DIR)
+                logging.info("k8s Deploy Remove Existing Docker Related "
+                             "Packages Complete")
+            except Exception:
+                logging.error("k8s Deploy Remove Existing Docker Related "
+                              "Packages failed. Please check log")
+                raise
+
+            try:
+                utils.run_ansible(k8s_deploy_vars, k8s_deploy,
+                                  host=k8s_deploy_inv_file,
+                                  user='heat-admin', tmp_dir=APEX_TEMP_DIR)
+                logging.info("k8s Deploy Overcloud Configuration Complete")
+            except Exception:
+                logging.error("k8s Deploy Overcloud Configuration failed."
+                              "Please check log")
+                raise
+
         # Post deploy overcloud node configuration
         # TODO(trozet): just parse all ds_opts as deploy vars one time
         deploy_vars['sfc'] = ds_opts['sfc']
diff --git a/apex/overcloud/deploy.py b/apex/overcloud/deploy.py
index 2f33183c..92b42fff 100644
--- a/apex/overcloud/deploy.py
+++ b/apex/overcloud/deploy.py
@@ -204,6 +204,11 @@ def create_deploy_cmd(ds, ns, inv, tmp_dir,
     if ds_opts['sriov']:
         prep_sriov_env(ds, tmp_dir)
 
+    # Check for 'k8s' here intentionally, as we may support other values
+    # such as openstack/openshift for 'vim' option.
+    if ds_opts['vim'] == 'k8s':
+        deploy_options.append('kubernetes-environment.yaml')
+
     if virtual:
         deploy_options.append('virtual-environment.yaml')
     else:
diff --git a/apex/settings/deploy_settings.py b/apex/settings/deploy_settings.py
index 29fe64fb..00e6d6c0 100644
--- a/apex/settings/deploy_settings.py
+++ b/apex/settings/deploy_settings.py
@@ -27,7 +27,8 @@ REQ_DEPLOY_SETTINGS = ['sdn_controller',
                        'l2gw',
                        'sriov',
                        'containers',
-                       'ceph_device']
+                       'ceph_device',
+                       'vim']
 
 OPT_DEPLOY_SETTINGS = ['performance',
                        'vsperf',
@@ -113,6 +114,8 @@ class DeploySettings(dict):
                 elif req_set == 'os_version':
                     self['deploy_options'][req_set] = \
                         constants.DEFAULT_OS_VERSION
+                elif req_set == 'vim':
+                    self['deploy_options'][req_set] = 'openstack'
                 else:
                     self['deploy_options'][req_set] = False
             elif req_set == 'odl_version' and self['deploy_options'][
diff --git a/apex/tests/test_apex_common_utils.py b/apex/tests/test_apex_common_utils.py
index 0e4041ca..b6aa4c7f 100644
--- a/apex/tests/test_apex_common_utils.py
+++ b/apex/tests/test_apex_common_utils.py
@@ -64,8 +64,11 @@ class TestCommonUtils:
 
     def test_run_ansible(self):
         playbook = 'apex/tests/playbooks/test_playbook.yaml'
+        extra_vars = [{'testvar1': 'value1', 'testvar2': 'value2'}]
         assert_equal(utils.run_ansible(None, os.path.join(playbook),
                                        dry_run=True), None)
+        assert_equal(utils.run_ansible(extra_vars, os.path.join(playbook),
+                                       dry_run=True, host='1.1.1.1'), None)
 
     def test_failed_run_ansible(self):
         playbook = 'apex/tests/playbooks/test_failed_playbook.yaml'
diff --git a/apex/tests/test_apex_deploy.py b/apex/tests/test_apex_deploy.py
index b7941f6f..8e9756eb 100644
--- a/apex/tests/test_apex_deploy.py
+++ b/apex/tests/test_apex_deploy.py
@@ -144,6 +144,7 @@ class TestDeploy(unittest.TestCase):
                                            'dataplane': 'ovs',
                                            'sfc': False,
                                            'vpn': False,
+                                           'vim': 'openstack',
                                            'yardstick': 'test',
                                            'os_version': DEFAULT_OS_VERSION,
                                            'containers': False}}
@@ -220,6 +221,7 @@ class TestDeploy(unittest.TestCase):
                                            'dataplane': 'ovs',
                                            'sfc': False,
                                            'vpn': False,
+                                           'vim': 'openstack',
                                            'yardstick': 'test',
                                            'os_version': DEFAULT_OS_VERSION,
                                            'containers': False}}
@@ -281,6 +283,7 @@ class TestDeploy(unittest.TestCase):
                                            'dataplane': 'ovs',
                                            'sfc': False,
                                            'vpn': False,
+                                           'vim': 'openstack',
                                            'yardstick': 'test',
                                            'os_version': DEFAULT_OS_VERSION,
                                            'containers': True}}
@@ -303,6 +306,63 @@ class TestDeploy(unittest.TestCase):
         args.virt_compute_ram = 16
         args.virt_default_ram = 10
         main()
-        mock_oc_deploy.prep_image.assert_called
+        mock_oc_deploy.prep_image.assert_called()
         # TODO(trozet) add assertions here with arguments for functions in
         # deploy main
+
+    @patch('apex.deploy.uc_builder')
+    @patch('apex.deploy.network_data.create_network_data')
+    @patch('apex.deploy.shutil')
+    @patch('apex.deploy.git')
+    @patch('apex.deploy.oc_deploy')
+    @patch('apex.deploy.uc_lib')
+    @patch('apex.deploy.build_vms')
+    @patch('apex.deploy.Inventory')
+    @patch('apex.deploy.virt_utils')
+    @patch('apex.deploy.oc_cfg')
+    @patch('apex.deploy.parsers')
+    @patch('apex.deploy.utils')
+    @patch('apex.deploy.NetworkEnvironment')
+    @patch('apex.deploy.NetworkSettings')
+    @patch('apex.deploy.DeploySettings')
+    @patch('apex.deploy.os')
+    @patch('apex.deploy.json')
+    @patch('apex.deploy.jumphost')
+    @patch('apex.deploy.validate_cross_settings')
+    @patch('apex.deploy.validate_deploy_args')
+    @patch('apex.deploy.create_deploy_parser')
+    @patch('builtins.open', a_mock_open, create=True)
+    def test_main_k8s(self, mock_parser, mock_val_args, mock_cross_sets,
+                      mock_jumphost, mock_json, mock_os,
+                      mock_deploy_sets, mock_net_sets, mock_net_env,
+                      mock_utils, mock_parsers, mock_oc_cfg,
+                      mock_virt_utils, mock_inv, mock_build_vms, mock_uc_lib,
+                      mock_oc_deploy, mock_git, mock_shutil,
+                      mock_network_data, mock_uc_builder):
+        net_sets_dict = {'networks': MagicMock(),
+                         'dns_servers': 'test'}
+        ds_opts_dict = {'global_params': MagicMock(),
+                        'deploy_options': {'gluon': False,
+                                           'congress': True,
+                                           'sdn_controller': False,
+                                           'dataplane': 'ovs',
+                                           'sfc': False,
+                                           'vpn': False,
+                                           'vim': 'k8s',
+                                           'yardstick': 'test',
+                                           'os_version': DEFAULT_OS_VERSION,
+                                           'containers': False}}
+        args = mock_parser.return_value.parse_args.return_value
+        args.virtual = False
+        args.quickstart = False
+        args.debug = False
+        args.upstream = False
+        net_sets = mock_net_sets.return_value
+        net_sets.enabled_network_list = ['external']
+        net_sets.__getitem__.side_effect = net_sets_dict.__getitem__
+        net_sets.__contains__.side_effect = net_sets_dict.__contains__
+        deploy_sets = mock_deploy_sets.return_value
+        deploy_sets.__getitem__.side_effect = ds_opts_dict.__getitem__
+        deploy_sets.__contains__.side_effect = ds_opts_dict.__contains__
+        mock_parsers.parse_nova_output.return_value = {'testnode1': 'test'}
+        main()
diff --git a/apex/tests/test_apex_overcloud_deploy.py b/apex/tests/test_apex_overcloud_deploy.py
index 83e2b02d..57d74bdd 100644
--- a/apex/tests/test_apex_overcloud_deploy.py
+++ b/apex/tests/test_apex_overcloud_deploy.py
@@ -104,7 +104,8 @@ class TestOvercloudDeploy(unittest.TestCase):
                'containers': False,
                'barometer': True,
                'ceph': False,
-               'sriov': False
+               'sriov': False,
+               'vim': 'openstack'
                },
               'global_params': MagicMock()}
 
@@ -135,7 +136,8 @@ class TestOvercloudDeploy(unittest.TestCase):
                'ceph': True,
                'sdn_controller': 'opendaylight',
                'sriov': False,
-               'os_version': 'queens'
+               'os_version': 'queens',
+               'vim': 'openstack'
                },
               'global_params': MagicMock()}