Merge "deploy kubernetes in XCI"

11 files changed, 709 insertions, 0 deletions

diff --git a/xci/installer/kubespray/README b/xci/installer/kubespray/README
new file mode 100644
index 00000000..171091e8
--- /dev/null
+++ b/xci/installer/kubespray/README
@@ -0,0 +1,64 @@
+The xci/installer/kubespray/files/k8s-cluster.yml is obtained from kubespray.
+You can change the parameters according to your needs.
+When  starting the deploy, it will be copied to the right directory and will be used by kubespray
+
+For example:
+  kube_network_plugin: calico
+  docker_storage_options: -s overlay2
+  kube_service_addresses: 10.233.0.0/18
+
+Requirements:
+  1. Performance of hosts
+      The performance settings are not required officially. I recommend the following:
+       - VM_CPU=6
+       - VM_DISK=80GB
+       - VM_MEMORY_SIZE=16GB
+
+  2. Distributions
+       - Ubuntu 16.04
+
+  3. Packages:
+       - Ansible v2.4 (or newer) and python-netaddr is installed on the machine that will run Ansible commands
+       - Jinja 2.9 (or newer) is required to run the Ansible Playbooks
+
+  4. Others:
+       - The target servers must have access to the Internet in order to pull docker images.
+       - The target servers are configured to allow IPv4 forwarding.
+       - Your ssh key must be copied to all the servers part of your inventory.
+       - The firewalls are not managed, you'll need to implement your own rules the way you used to. In order to avoid any issue during the deployment you should disable your firewall.
+
+Flavors:
+  1. aio: Single host which acts as the deployment host, master and node.
+  2. mini: One deployment host, 1 master host and 1 node host.
+  3. noha: One deployment host, 1 master host and 2 node hosts.
+  4. ha: One deployment host, 3 master hosts and 2 node hosts.
+
+Components Installed:
+  1. etcd
+  2. network plugins:(one of following, which you can choose. Default is calico)
+       - calico
+       - flannel
+       - contive
+       - weave
+  3. kubernetes
+  4. docker
+
+How to use:
+
+Clone the OPNFV Releng repository
+
+    git clone https://gerrit.opnfv.org/gerrit/releng-xci.git
+
+Change into the directory where the sandbox script is located
+
+    cd releng-xci/xci
+
+Set the variable to run kubespray
+
+    export XCI_INSTALLER=kubespray
+    export DEPLOY_SCENARIO=k8-nosdn-nofeature
+    export XCI_FLAVOR=mini
+
+Execute sandbox script
+
+    ./xci-deploy.sh
diff --git a/xci/installer/kubespray/deploy.sh b/xci/installer/kubespray/deploy.sh
new file mode 100755
index 00000000..b04ed781
--- /dev/null
+++ b/xci/installer/kubespray/deploy.sh
@@ -0,0 +1,85 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2017 Huawei
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+K8_XCI_PLAYBOOKS="$(dirname $(realpath ${BASH_SOURCE[0]}))/playbooks"
+export ANSIBLE_ROLES_PATH=$HOME/.ansible/roles:/etc/ansible/roles:${XCI_PATH}/xci/playbooks/roles
+
+
+#-------------------------------------------------------------------------------
+# Configure localhost
+#-------------------------------------------------------------------------------
+# This playbook
+# - removes directories that were created by the previous xci run
+# - clones opnfv/releng-xci repository
+# - clones kubernetes-incubator/kubespray repository
+# - creates log directory
+#-------------------------------------------------------------------------------
+
+echo "Info: Configuring localhost for kubespray"
+echo "-----------------------------------------------------------------------"
+cd $XCI_PLAYBOOKS
+ansible-playbook ${XCI_ANSIBLE_VERBOSITY} -e XCI_PATH="${XCI_PATH}" \
+        -i ${XCI_FLAVOR_ANSIBLE_FILE_PATH}/inventory/inventory.cfg \
+        configure-localhost.yml
+echo "-----------------------------------------------------------------------"
+echo "Info: Configured localhost for kubespray"
+
+#-------------------------------------------------------------------------------
+# Configure deployment host, opnfv
+#-------------------------------------------------------------------------------
+# This playbook
+# - removes directories that were created by the previous xci run
+# - synchronize opnfv/releng-xci and kubernetes-incubator/kubespray repositories
+# - generates/prepares ssh keys
+# - copies flavor files to be used by kubespray
+# - install packages required by kubespray
+#-------------------------------------------------------------------------------
+echo "Info: Configuring opnfv deployment host for kubespray"
+echo "-----------------------------------------------------------------------"
+cd $K8_XCI_PLAYBOOKS
+ansible-playbook ${XCI_ANSIBLE_VERBOSITY} -e XCI_PATH="${XCI_PATH}" \
+        -i ${XCI_FLAVOR_ANSIBLE_FILE_PATH}/inventory/inventory.cfg \
+        configure-opnfvhost.yml
+echo "-----------------------------------------------------------------------"
+echo "Info: Configured opnfv deployment host for kubespray"
+
+#-------------------------------------------------------------------------------
+# Configure target hosts for kubespray
+#-------------------------------------------------------------------------------
+# This playbook is only run for the all flavors except aio since aio is configured by the configure-opnfvhost.yml
+# This playbook
+# - adds public keys to target hosts
+# - install packages required by kubespray
+# - configures haproxy service
+#-------------------------------------------------------------------------------
+if [ $XCI_FLAVOR != "aio" ]; then
+    echo "Info: Configuring target hosts for kubespray"
+    echo "-----------------------------------------------------------------------"
+    cd $K8_XCI_PLAYBOOKS
+    ansible-playbook ${XCI_ANSIBLE_VERBOSITY} -e XCI_PATH="${XCI_PATH}" \
+            -i ${XCI_FLAVOR_ANSIBLE_FILE_PATH}/inventory/inventory.cfg \
+            configure-targethosts.yml
+    echo "-----------------------------------------------------------------------"
+    echo "Info: Configured target hosts for kubespray"
+fi
+
+echo "Info: Using kubespray to deploy the kubernetes cluster"
+echo "-----------------------------------------------------------------------"
+ssh root@$OPNFV_HOST_IP "cd releng-xci/.cache/repos/kubespray;\
+         ansible-playbook ${XCI_ANSIBLE_VERBOSITY} \
+         -i opnfv_inventory/inventory.cfg cluster.yml -b | tee setup-kubernetes.log"
+scp root@$OPNFV_HOST_IP:~/releng-xci/.cache/repos/kubespray/setup-kubernetes.log \
+         $LOG_PATH/setup-kubernetes.log
+# check the log to see if we have any error
+if grep -q 'failed=1\|unreachable=1' $LOG_PATH/setup-kubernetes.log; then
+    echo "Error: Kubernetes cluster setup failed!"
+    exit 1
+fi
+echo "Info: Kubernetes installation is successfully completed!"
diff --git a/xci/installer/kubespray/files/aio/inventory/inventory.cfg b/xci/installer/kubespray/files/aio/inventory/inventory.cfg
new file mode 100644
index 00000000..a72d0fec
--- /dev/null
+++ b/xci/installer/kubespray/files/aio/inventory/inventory.cfg
@@ -0,0 +1,20 @@
+[all]
+opnfv ansible_host=192.168.122.2 ip=192.168.122.2
+
+[kube-master]
+opnfv
+
+[kube-node]
+opnfv
+
+[etcd]
+opnfv
+
+[k8s-cluster:children]
+kube-node
+kube-master
+
+[calico-rr]
+
+[vault]
+opnfv
diff --git a/xci/installer/kubespray/files/ha/inventory/group_vars/all.yml b/xci/installer/kubespray/files/ha/inventory/group_vars/all.yml
new file mode 100644
index 00000000..d1b946a7
--- /dev/null
+++ b/xci/installer/kubespray/files/ha/inventory/group_vars/all.yml
@@ -0,0 +1,8 @@
+---
+loadbalancer_apiserver:
+  address: 192.168.122.222
+  port: 8383
+
+apiserver_loadbalancer_domain_name: 192.168.122.222
+supplementary_addresses_in_ssl_keys:
+  - 192.168.122.222
diff --git a/xci/installer/kubespray/files/ha/inventory/inventory.cfg b/xci/installer/kubespray/files/ha/inventory/inventory.cfg
new file mode 100644
index 00000000..aae36329
--- /dev/null
+++ b/xci/installer/kubespray/files/ha/inventory/inventory.cfg
@@ -0,0 +1,32 @@
+[all]
+opnfv ansible_host=192.168.122.2 ip=192.168.122.2
+master1 ansible_host=192.168.122.3 ip=192.168.122.3
+master2 ansible_host=192.168.122.4 ip=192.168.122.4
+master3 ansible_host=192.168.122.5 ip=192.168.122.5
+node1 ansible_host=192.168.122.6 ip=192.168.122.6
+node2 ansible_host=192.168.122.7 ip=192.168.122.7
+
+[kube-master]
+master1
+master2
+master3
+
+[kube-node]
+node1
+node2
+
+[etcd]
+master1
+master2
+master3
+
+[k8s-cluster:children]
+kube-node
+kube-master
+
+[calico-rr]
+
+[vault]
+master1
+master2
+master3
diff --git a/xci/installer/kubespray/files/k8s-cluster.yml b/xci/installer/kubespray/files/k8s-cluster.yml
new file mode 100644
index 00000000..aeee573a
--- /dev/null
+++ b/xci/installer/kubespray/files/k8s-cluster.yml
@@ -0,0 +1,291 @@
+# Valid bootstrap options (required): ubuntu, coreos, centos, none
+bootstrap_os: none
+
+#Directory where etcd data stored
+etcd_data_dir: /var/lib/etcd
+
+# Directory where the binaries will be installed
+bin_dir: /usr/local/bin
+
+## The access_ip variable is used to define how other nodes should access
+## the node.  This is used in flannel to allow other flannel nodes to see
+## this node for example.  The access_ip is really useful AWS and Google
+## environments where the nodes are accessed remotely by the "public" ip,
+## but don't know about that address themselves.
+#access_ip: 1.1.1.1
+
+### LOADBALANCING AND ACCESS MODES
+## Enable multiaccess to configure etcd clients to access all of the etcd members directly
+## as the "http://hostX:port, http://hostY:port, ..." and ignore the proxy loadbalancers.
+## This may be the case if clients support and loadbalance multiple etcd servers  natively.
+#etcd_multiaccess: true
+
+## Internal loadbalancers for apiservers
+#loadbalancer_apiserver_localhost: true
+
+## Local loadbalancer should use this port instead, if defined.
+## Defaults to kube_apiserver_port (6443)
+#nginx_kube_apiserver_port: 8443
+
+### OTHER OPTIONAL VARIABLES
+## For some things, kubelet needs to load kernel modules.  For example, dynamic kernel services are needed
+## for mounting persistent volumes into containers.  These may not be loaded by preinstall kubernetes
+## processes.  For example, ceph and rbd backed volumes.  Set to true to allow kubelet to load kernel
+## modules.
+# kubelet_load_modules: false
+
+## Internal network total size. This is the prefix of the
+## entire network. Must be unused in your environment.
+#kube_network_prefix: 18
+
+## With calico it is possible to distributed routes with border routers of the datacenter.
+## Warning : enabling router peering will disable calico's default behavior ('node mesh').
+## The subnets of each nodes will be distributed by the datacenter router
+#peer_with_router: false
+
+## Upstream dns servers used by dnsmasq
+#upstream_dns_servers:
+#  - 8.8.8.8
+#  - 8.8.4.4
+
+## There are some changes specific to the cloud providers
+## for instance we need to encapsulate packets with some network plugins
+## If set the possible values are either 'gce', 'aws', 'azure', 'openstack', 'vsphere', or 'external'
+## When openstack is used make sure to source in the openstack credentials
+## like you would do when using nova-client before starting the playbook.
+#cloud_provider:
+
+## When OpenStack is used, Cinder version can be explicitly specified if autodetection fails (https://github.com/kubernetes/kubernetes/issues/50461)
+#openstack_blockstorage_version: "v1/v2/auto (default)"
+## When OpenStack is used, if LBaaSv2 is available you can enable it with the following variables.
+#openstack_lbaas_enabled: True
+#openstack_lbaas_subnet_id: "Neutron subnet ID (not network ID) to create LBaaS VIP"
+#openstack_lbaas_floating_network_id: "Neutron network ID (not subnet ID) to get floating IP from, disabled by default"
+#openstack_lbaas_create_monitor: "yes"
+#openstack_lbaas_monitor_delay: "1m"
+#openstack_lbaas_monitor_timeout: "30s"
+#openstack_lbaas_monitor_max_retries: "3"
+
+## Uncomment to enable experimental kubeadm deployment mode
+#kubeadm_enabled: false
+#kubeadm_token_first: "{{ lookup('password', 'credentials/kubeadm_token_first length=6  chars=ascii_lowercase,digits') }}"
+#kubeadm_token_second: "{{ lookup('password', 'credentials/kubeadm_token_second length=16 chars=ascii_lowercase,digits') }}"
+#kubeadm_token: "{{ kubeadm_token_first }}.{{ kubeadm_token_second }}"
+#
+## Set these proxy values in order to update package manager and docker daemon to use proxies
+#http_proxy: ""
+#https_proxy: ""
+## Refer to roles/kubespray-defaults/defaults/main.yml before modifying no_proxy
+#no_proxy: ""
+
+## Uncomment this if you want to force overlay/overlay2 as docker storage driver
+## Please note that overlay2 is only supported on newer kernels
+#docker_storage_options: -s overlay2
+
+# Uncomment this if you have more than 3 nameservers, then we'll only use the first 3.
+#docker_dns_servers_strict: false
+
+## Default packages to install within the cluster, f.e:
+#kpm_packages:
+#  - name: kube-system/grafana
+
+## Certificate Management
+## This setting determines whether certs are generated via scripts or whether a
+## cluster of Hashicorp's Vault is started to issue certificates (using etcd
+## as a backend). Options are "script" or "vault"
+#cert_management: script
+
+# Set to true to allow pre-checks to fail and continue deployment
+#ignore_assert_errors: false
+
+## Etcd auto compaction retention for mvcc key value store in hour
+#etcd_compaction_retention: 0
+
+## Set level of detail for etcd exported metrics, specify 'extensive' to include histogram metrics.
+#etcd_metrics: basic
+
+
+# Kubernetes configuration dirs and system namespace.
+# Those are where all the additional config stuff goes
+# kubernetes normally puts in /srv/kubernetes.
+# This puts them in a sane location and namespace.
+# Editing those values will almost surely break something.
+kube_config_dir: /etc/kubernetes
+kube_script_dir: "{{ bin_dir }}/kubernetes-scripts"
+kube_manifest_dir: "{{ kube_config_dir }}/manifests"
+system_namespace: kube-system
+
+# Logging directory (sysvinit systems)
+kube_log_dir: "/var/log/kubernetes"
+
+# This is where all the cert scripts and certs will be located
+kube_cert_dir: "{{ kube_config_dir }}/ssl"
+
+# This is where all of the bearer tokens will be stored
+kube_token_dir: "{{ kube_config_dir }}/tokens"
+
+# This is where to save basic auth file
+kube_users_dir: "{{ kube_config_dir }}/users"
+
+kube_api_anonymous_auth: false
+
+## Change this to use another Kubernetes version, e.g. a current beta release
+#kube_version: v1.9.0
+
+# Where the binaries will be downloaded.
+# Note: ensure that you've enough disk space (about 1G)
+local_release_dir: "/tmp/releases"
+# Random shifts for retrying failed ops like pushing/downloading
+retry_stagger: 5
+
+# This is the group that the cert creation scripts chgrp the
+# cert files to. Not really changable...
+kube_cert_group: kube-cert
+
+# Cluster Loglevel configuration
+kube_log_level: 2
+
+# Users to create for basic auth in Kubernetes API via HTTP
+# Optionally add groups for user
+kube_api_pwd: "{{ lookup('password', 'credentials/kube_user length=15 chars=ascii_letters,digits') }}"
+kube_users:
+  kube:
+    pass: "{{kube_api_pwd}}"
+    role: admin
+    groups:
+      - system:masters
+
+## It is possible to activate / deactivate selected authentication methods (basic auth, static token auth)
+#kube_oidc_auth: false
+#kube_basic_auth: false
+#kube_token_auth: false
+
+
+## Variables for OpenID Connect Configuration https://kubernetes.io/docs/admin/authentication/
+## To use OpenID you have to deploy additional an OpenID Provider (e.g Dex, Keycloak, ...)
+
+# kube_oidc_url: https:// ...
+# kube_oidc_client_id: kubernetes
+## Optional settings for OIDC
+# kube_oidc_ca_file: {{ kube_cert_dir }}/ca.pem
+# kube_oidc_username_claim: sub
+# kube_oidc_groups_claim: groups
+
+
+# Choose network plugin (calico, contiv, weave or flannel)
+# Can also be set to 'cloud', which lets the cloud provider setup appropriate routing
+kube_network_plugin: calico
+
+# weave's network password for encryption
+# if null then no network encryption
+# you can use --extra-vars to pass the password in command line
+weave_password: EnterPasswordHere
+
+# Weave uses consensus mode by default
+# Enabling seed mode allow to dynamically add or remove hosts
+# https://www.weave.works/docs/net/latest/ipam/
+weave_mode_seed: false
+
+# This two variable are automatically changed by the weave's role, do not manually change these values
+# To reset values :
+# weave_seed: uninitialized
+# weave_peers: uninitialized
+weave_seed: uninitialized
+weave_peers: uninitialized
+
+# Enable kubernetes network policies
+enable_network_policy: false
+
+# Kubernetes internal network for services, unused block of space.
+kube_service_addresses: 10.233.0.0/18
+
+# internal network. When used, it will assign IP
+# addresses from this range to individual pods.
+# This network must be unused in your network infrastructure!
+kube_pods_subnet: 10.233.64.0/18
+
+# internal network node size allocation (optional). This is the size allocated
+# to each node on your network.  With these defaults you should have
+# room for 4096 nodes with 254 pods per node.
+kube_network_node_prefix: 24
+
+# The port the API Server will be listening on.
+kube_apiserver_ip: "{{ kube_service_addresses|ipaddr('net')|ipaddr(1)|ipaddr('address') }}"
+kube_apiserver_port: 6443 # (https)
+kube_apiserver_insecure_port: 8080 # (http)
+
+# DNS configuration.
+# Kubernetes cluster name, also will be used as DNS domain
+cluster_name: cluster.local
+# Subdomains of DNS domain to be resolved via /etc/resolv.conf for hostnet pods
+ndots: 2
+# Can be dnsmasq_kubedns, kubedns or none
+dns_mode: kubedns
+# Can be docker_dns, host_resolvconf or none
+resolvconf_mode: docker_dns
+# Deploy netchecker app to verify DNS resolve as an HTTP service
+deploy_netchecker: false
+# Ip address of the kubernetes skydns service
+skydns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(3)|ipaddr('address') }}"
+dnsmasq_dns_server: "{{ kube_service_addresses|ipaddr('net')|ipaddr(2)|ipaddr('address') }}"
+dns_domain: "{{ cluster_name }}"
+
+# Path used to store Docker data
+docker_daemon_graph: "/var/lib/docker"
+
+## A string of extra options to pass to the docker daemon.
+## This string should be exactly as you wish it to appear.
+## An obvious use case is allowing insecure-registry access
+## to self hosted registries like so:
+
+docker_options: "--insecure-registry={{ kube_service_addresses }} --graph={{ docker_daemon_graph }}  {{ docker_log_opts }}"
+docker_bin_dir: "/usr/bin"
+
+# Settings for containerized control plane (etcd/kubelet/secrets)
+etcd_deployment_type: docker
+kubelet_deployment_type: host
+vault_deployment_type: docker
+helm_deployment_type: host
+
+# K8s image pull policy (imagePullPolicy)
+k8s_image_pull_policy: IfNotPresent
+
+# Kubernetes dashboard
+# RBAC required. see docs/getting-started.md for access details.
+dashboard_enabled: true
+
+# Monitoring apps for k8s
+efk_enabled: false
+
+# Helm deployment
+helm_enabled: false
+
+# Istio deployment
+istio_enabled: false
+
+# Local volume provisioner deployment
+local_volumes_enabled: false
+
+# Add Persistent Volumes Storage Class for corresponding cloud provider ( OpenStack is only supported now )
+persistent_volumes_enabled: false
+
+# Make a copy of kubeconfig on the host that runs Ansible in GITDIR/artifacts
+# kubeconfig_localhost: false
+# Download kubectl onto the host that runs Ansible in GITDIR/artifacts
+# kubectl_localhost: false
+
+# dnsmasq
+# dnsmasq_upstream_dns_servers:
+#  - /resolvethiszone.with/10.0.4.250
+#  - 8.8.8.8
+
+#  Enable creation of QoS cgroup hierarchy, if true top level QoS and pod cgroups are created. (default true)
+# kubelet_cgroups_per_qos: true
+
+# A comma separated list of levels of node allocatable enforcement to be enforced by kubelet.
+# Acceptible options are 'pods', 'system-reserved', 'kube-reserved' and ''. Default is "".
+# kubelet_enforce_node_allocatable: pods
+
+## Supplementary addresses that can be added in kubernetes ssl keys.
+## That can be usefull for example to setup a keepalived virtual IP
+# supplementary_addresses_in_ssl_keys: [10.0.0.1, 10.0.0.2, 10.0.0.3]
diff --git a/xci/installer/kubespray/files/mini/inventory/inventory.cfg b/xci/installer/kubespray/files/mini/inventory/inventory.cfg
new file mode 100644
index 00000000..bf8bf19b
--- /dev/null
+++ b/xci/installer/kubespray/files/mini/inventory/inventory.cfg
@@ -0,0 +1,22 @@
+[all]
+opnfv ansible_host=192.168.122.2 ip=192.168.122.2
+master1 ansible_host=192.168.122.3 ip=192.168.122.3
+node1 ansible_host=192.168.122.4 ip=192.168.122.4
+
+[kube-master]
+master1
+
+[kube-node]
+node1
+
+[etcd]
+master1
+
+[k8s-cluster:children]
+kube-node
+kube-master
+
+[calico-rr]
+
+[vault]
+master1
diff --git a/xci/installer/kubespray/files/noha/inventory/inventory.cfg b/xci/installer/kubespray/files/noha/inventory/inventory.cfg
new file mode 100644
index 00000000..73c1e0a1
--- /dev/null
+++ b/xci/installer/kubespray/files/noha/inventory/inventory.cfg
@@ -0,0 +1,24 @@
+[all]
+opnfv ansible_host=192.168.122.2 ip=192.168.122.2
+master1 ansible_host=192.168.122.3 ip=192.168.122.3
+node1 ansible_host=192.168.122.4 ip=192.168.122.4
+node2 ansible_host=192.168.122.5 ip=192.168.122.5
+
+[kube-master]
+master1
+
+[kube-node]
+node1
+node2
+
+[etcd]
+master1
+
+[k8s-cluster:children]
+kube-node
+kube-master
+
+[calico-rr]
+
+[vault]
+master1
diff --git a/xci/installer/kubespray/playbooks/configure-opnfvhost.yml b/xci/installer/kubespray/playbooks/configure-opnfvhost.yml
new file mode 100644
index 00000000..23f93852
--- /dev/null
+++ b/xci/installer/kubespray/playbooks/configure-opnfvhost.yml
@@ -0,0 +1,77 @@
+---
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+- hosts: opnfv
+  remote_user: root
+  vars_files:
+    - "{{ XCI_PATH }}/xci/var/opnfv.yml"
+
+  tasks:
+    - name: Load distribution variables
+      include_vars:
+        file: "{{ item }}"
+      with_items:
+        - "{{ XCI_PATH }}/xci/var/{{ ansible_os_family }}.yml"
+    - name: Set facts for remote deployment
+      set_fact:
+        remote_xci_path: "{{ ansible_env.HOME }}/releng-xci"
+        remote_xci_flavor_files: "{{ ansible_env.HOME }}/releng-xci/xci/installer/{{ XCI_INSTALLER }}/files/{{ XCI_FLAVOR }}"
+        remote_xci_playbooks: "{{ ansible_env.HOME }}/releng-xci/xci/playbooks"
+
+    - name: Copy releng-xci to remote host
+      synchronize:
+        src: "{{ XCI_PATH }}/"
+        dest: "{{ remote_xci_path }}"
+        recursive: yes
+        delete: yes
+
+    - name: generate SSH keys
+      shell: ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""
+      args:
+        creates: /root/.ssh/id_rsa
+    - name: add id_rsa.pub to authorized_keys
+      shell: cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys
+      when:  XCI_FLAVOR == 'aio'
+    - name: fetch public key
+      fetch:
+        src: "{{ ansible_env.HOME }}/.ssh/id_rsa.pub"
+        dest: "{{ XCI_PATH }}/xci/files/authorized_keys"
+        flat: yes
+    - name: delete the opnfv_inventory directory
+      file:
+        path: "{{ remote_xci_path }}/.cache/repos/kubespray/opnfv_inventory"
+        state: absent
+    - name: copy kubespray inventory directory
+      shell: "cp -rf {{ remote_xci_flavor_files }}/inventory \
+                   {{ remote_xci_path }}/.cache/repos/kubespray/opnfv_inventory"
+    - name: make sure kubespray/opnfv_inventory/group_vars/ exist
+      file:
+        path: "{{ remote_xci_path }}/.cache/repos/kubespray/opnfv_inventory/group_vars"
+        state: directory
+    - name: copy k8s_cluster.yml
+      shell: "cp -rf {{ remote_xci_path }}/xci/installer/kubespray/files/k8s-cluster.yml \
+                     {{ remote_xci_path }}/.cache/repos/kubespray/opnfv_inventory/group_vars"
+    - name: install dbus and ptyhon-netaddr
+      apt:
+        name: "{{item}}"
+        update_cache: yes
+      with_items: "{{ kube_require_packages }}"
+      when: XCI_FLAVOR == 'aio'
+    - name: pip install ansible
+      pip:
+        name: ansible
+        version: "{{ XCI_KUBE_ANSIBLE_PIP_VERSION }}"
+
+
+- hosts: localhost
+  remote_user: root
+  vars_files:
+    - "{{ XCI_PATH }}/xci/var/opnfv.yml"
+  tasks:
+    - name: Append public keys to authorized_keys
+      shell: "/bin/cat {{ ansible_env.HOME }}/.ssh/id_rsa.pub >> {{ XCI_PATH }}/xci/files/authorized_keys"
diff --git a/xci/installer/kubespray/playbooks/configure-targethosts.yml b/xci/installer/kubespray/playbooks/configure-targethosts.yml
new file mode 100644
index 00000000..7e29025b
--- /dev/null
+++ b/xci/installer/kubespray/playbooks/configure-targethosts.yml
@@ -0,0 +1,28 @@
+---
+- hosts: all
+  remote_user: root
+  tasks:
+    - name: add public key to host
+      copy:
+        src: "{{ XCI_PATH }}/xci/files/authorized_keys"
+        dest: /root/.ssh/authorized_keys
+    - name: install dbus and ptyhon-netaddr
+      apt:
+        name: "{{item}}"
+        update_cache: yes
+      with_items: "{{ kube_require_packages }}"
+
+- hosts: kube-master
+  remote_user: root
+  vars_files:
+    - "{{ XCI_PATH }}/xci/var/opnfv.yml"
+  pre_tasks:
+    - name: Load distribution variables
+      include_vars:
+        file: "{{ XCI_PATH }}/xci/var/{{ ansible_os_family }}.yml"
+  roles:
+    - role: "keepalived"
+      when:  XCI_FLAVOR == 'ha'
+    - role: "haproxy_server"
+      haproxy_service_configs: "{{ haproxy_default_services}}"
+      when:  XCI_FLAVOR == 'ha'
diff --git a/xci/installer/kubespray/playbooks/group_vars/all b/xci/installer/kubespray/playbooks/group_vars/all
new file mode 100644
index 00000000..06dccb68
--- /dev/null
+++ b/xci/installer/kubespray/playbooks/group_vars/all
@@ -0,0 +1,58 @@
+kube_require_packages:
+  - "python-netaddr"
+  - "dbus"
+
+keepalived_ubuntu_src: "uca"
+keepalived_uca_apt_repo_url: "{{ uca_apt_repo_url | default('http://ubuntu-cloud.archive.canonical.com/ubuntu') }}"
+
+keepalived_sync_groups:
+  haproxy:
+    instances:
+      - external
+
+haproxy_keepalived_external_interface: "{{ interface }}"
+haproxy_keepalived_authentication_password: 'keepalived'
+keepalived_instances:
+  external:
+    interface: "{{ haproxy_keepalived_external_interface }}"
+    state: "BACKUP"
+    virtual_router_id: "{{ haproxy_keepalived_external_virtual_router_id | default ('10') }}"
+    priority: "{{ ((ansible_play_hosts|length-ansible_play_hosts.index(inventory_hostname))*100)-((ansible_play_hosts|length-ansible_play_hosts.index(inventory_hostname))*50) }}"
+    authentication_password: "{{ haproxy_keepalived_authentication_password }}"
+    vips:
+      - "{{ haproxy_keepalived_external_vip_cidr | default('192.168.122.222/32')  }} dev {{ haproxy_keepalived_external_interface }}"
+
+haproxy_default_services:
+  - service:
+      haproxy_service_name: proxy-apiserver
+      haproxy_backend_nodes: "{{ groups['kube-master'] | default([]) }}"
+      haproxy_port: 8383
+      haproxy_backend_port: 6443
+      haproxy_balance_type: tcp
+
+haproxy_bind_on_non_local: "True"
+haproxy_use_keepalived: "True"
+keepalived_selinux_compile_rules:
+  - keepalived_ping
+  - keepalived_haproxy_pid_file
+
+# Ensure that the package state matches the global setting
+haproxy_package_state: "latest"
+
+haproxy_whitelist_networks:
+  - 192.168.0.0/16
+  - 172.16.0.0/12
+  - 10.0.0.0/8
+
+haproxy_galera_whitelist_networks: "{{ haproxy_whitelist_networks }}"
+haproxy_glance_registry_whitelist_networks: "{{ haproxy_whitelist_networks }}"
+haproxy_keystone_admin_whitelist_networks: "{{ haproxy_whitelist_networks }}"
+haproxy_nova_metadata_whitelist_networks: "{{ haproxy_whitelist_networks }}"
+haproxy_rabbitmq_management_whitelist_networks: "{{ haproxy_whitelist_networks }}"
+haproxy_repo_git_whitelist_networks: "{{ haproxy_whitelist_networks }}"
+haproxy_repo_cache_whitelist_networks: "{{ haproxy_whitelist_networks }}"
+haproxy_octavia_whitelist_networks: "{{ haproxy_whitelist_networks }}"
+haproxy_ssl: false
+
+internal_lb_vip_address: "192.168.122.222"
+external_lb_vip_address: "{{ internal_lb_vip_address }}"