spport k8s apiserver HA

    compass installer deploy k8s using kubespray for default,
but k8s apiserver HA  is not implemented by kubespray, This
patch aim  is to achieve the k8s apiserver HA
Change-Id: I805b5eb2f4efa7ca82fcef7bfd3f4cad35ed65b5
JIRA: -
Signed-off-by: hu xinhui <xinhui_hu@foxmail.com>

21 files changed, 345 insertions, 5 deletions

diff --git a/build.sh b/build.sh
index 7f67ee2a..590347b9 100755
--- a/build.sh
+++ b/build.sh
@@ -142,3 +142,4 @@ process_param $*
 prepare_env
 download_packages
 build_tar
+
diff --git a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml
index eb80066e..bfdc8958 100755
--- a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml
+++ b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml
@@ -25,6 +25,12 @@
   roles:
     - install-k8s-dependence
 
+- hosts: ha
+  remote_user: root
+  max_fail_percentage: 0
+  roles:
+    - ha
+
 - hosts: localhost
   remote_user: root
   max_fail_percentage: 0
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh b/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh
new file mode 100644
index 00000000..62e79b3b
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+count=`ss -tnl | grep 6443 | wc -l`
+
+if [ $count = 0 ]; then
+    exit 1
+else
+    exit 0
+fi
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml
new file mode 100644
index 00000000..03ed82ec
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml
@@ -0,0 +1,14 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+- name: restart haproxy
+  service: name=haproxy state=restarted enabled=yes
+
+- name: restart keepalived
+  service: name=keepalived state=restarted enabled=yes
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml
new file mode 100644
index 00000000..c7e58376
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml
@@ -0,0 +1,83 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+- include_vars: "{{ ansible_os_family }}.yml"
+
+- name: install keepalived  haproxy
+  action: "{{ ansible_pkg_mgr }} name={{ item }} state=present"
+  with_items: "{{ packages | union(packages_noarch) }}"
+
+- name: generate ha service list
+  lineinfile: dest=/opt/service create=yes line= '{{ item }}'
+  with_items: "{{ services | union(services_noarch) }}"
+
+- name: install pexpect
+  pip: name=pexpect state=present extra_args='--pre'
+
+- name: activate ip_nonlocal_bind
+  sysctl: name=net.ipv4.ip_nonlocal_bind value=1
+          state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_intvl
+  sysctl: name=net.ipv4.tcp_keepalive_intvl value=1
+          state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_probes
+  sysctl: name=net.ipv4.tcp_keepalive_probes value=5
+          state=present reload=yes
+
+- name: set net.ipv4.tcp_keepalive_time
+  sysctl: name=net.ipv4.tcp_keepalive_time value=5
+          state=present reload=yes
+
+- name: update haproxy cfg
+  template: src=haproxy.cfg dest=/etc/haproxy/haproxy.cfg
+  notify: restart haproxy
+
+- name: set haproxy enable flag
+  lineinfile: dest=/etc/default/haproxy state=present
+              regexp="ENABLED=*"
+              line="ENABLED=1"
+  notify: restart haproxy
+  when: ansible_os_family == "Debian"
+
+- name: set haproxy log
+  lineinfile: dest=/etc/rsyslog.conf state=present
+              regexp="local0.* /var/log/haproxy.log"
+              line="local0.* /var/log/haproxy.log"
+
+- name: set rsyslog udp module
+  lineinfile: dest=/etc/rsyslog.conf state=present
+              regexp="^#$ModLoad imudp"
+              line="$ModLoad imudp"
+
+- name: set rsyslog udp port
+  lineinfile: dest=/etc/rsyslog.conf state=present
+              regexp="^#$UDPServerRun 514"
+              line="$UDPServerRun 514"
+
+- name: set keepalived start param
+  lineinfile: dest=/etc/default/keepalived state=present
+              regexp="^DAEMON_ARGS=*"
+              line="DAEMON_ARGS=\"-D -d -S 1\""
+  when: ansible_os_family == "Debian"
+
+- name: set keepalived log
+  lineinfile: dest=/etc/rsyslog.conf state=present
+              regexp="local1.* /var/log/keepalived.log"
+              line="local1.* /var/log/keepalived.log"
+
+- name: update keepalived info
+  template: src=keepalived.conf dest=/etc/keepalived/keepalived.conf
+  notify: restart keepalived
+
+- name: restart rsyslog
+  shell: service rsyslog restart
+
+- meta: flush_handlers
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg b/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg
new file mode 100644
index 00000000..5cd240c0
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg
@@ -0,0 +1,48 @@
+
+global
+    #chroot /var/run/haproxy
+    daemon
+    user haproxy
+    group haproxy
+    maxconn 4000
+    pidfile /var/run/haproxy/haproxy.pid
+    #log 127.0.0.1 local0
+    tune.bufsize 1000000
+    stats socket /var/run/haproxy.sock
+    stats timeout 2m
+
+defaults
+    log global
+    maxconn 8000
+    option redispatch
+    option dontlognull
+    option splice-auto
+    timeout http-request 10s
+    timeout queue 1m
+    timeout connect 10s
+    timeout client 50s
+    timeout server 50s
+    timeout check 10s
+    retries 3
+
+listen  kubernetes-apiserver-https
+    bind {{ public_vip.ip }}:8383
+    option ssl-hello-chk
+    mode tcp
+    option tcpka
+    option tcplog
+    timeout client 3h
+    timeout server 3h
+    balance roundrobin
+{% for host,ip in haproxy_hosts.items() %}
+    server {{ host }} {{ ip }}:6443  weight 1 check inter 2000 rise 2 fall 5
+{% endfor %}
+
+listen stats
+    mode http
+    bind 0.0.0.0:9999
+    stats enable
+    stats refresh 30s
+    stats uri /
+    stats realm Global\ statistics
+    stats auth admin:admin
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf b/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf
new file mode 100644
index 00000000..c649bed5
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf
@@ -0,0 +1,49 @@
+global_defs {
+    router_id {{ inventory_hostname }}
+}
+
+vrrp_sync_group VG1 {
+    group {
+        internal_vip
+        public_vip
+    }
+}
+
+vrrp_instance internal_vip {
+    interface {{ sys_intf_mappings.mgmt.interface }}
+    virtual_router_id {{ vrouter_id_internal }}
+    state BACKUP
+    nopreempt
+    advert_int 1
+    priority {{ 50 + (host_index[inventory_hostname] * 50) }}
+
+    authentication {
+        auth_type PASS
+        auth_pass 1234
+    }
+
+
+    virtual_ipaddress {
+        {{ internal_vip.ip }}/{{ internal_vip.netmask }}  dev  {{ sys_intf_mappings.mgmt.interface }}
+    }
+}
+
+vrrp_instance public_vip {
+    interface {{ sys_intf_mappings.external.interface }}
+    virtual_router_id {{ vrouter_id_public }}
+    state BACKUP
+    nopreempt
+    advert_int 1
+    priority {{ 50 + (host_index[inventory_hostname] * 50) }}
+
+    authentication {
+        auth_type PASS
+        auth_pass 4321
+    }
+
+    virtual_ipaddress {
+        {{ network_cfg.public_vip.ip }}/{{ network_cfg.public_vip.netmask }}  dev {{ sys_intf_mappings.external.interface }}
+    }
+
+}
+
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml
new file mode 100644
index 00000000..b9f46bdf
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml
@@ -0,0 +1,11 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+services: []
+packages: []
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml
new file mode 100644
index 00000000..b9f46bdf
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml
@@ -0,0 +1,11 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+services: []
+packages: []
diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml
new file mode 100644
index 00000000..77735d1e
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml
@@ -0,0 +1,16 @@
+##############################################################################
+# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+packages_noarch:
+  - keepalived
+  - haproxy
+
+services_noarch:
+  - keepalived
+  - haproxy
diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml
index 6487e4ef..e683a3fe 100644
--- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml
@@ -9,6 +9,10 @@
 ---
 - include_vars: "{{ ansible_os_family }}.yml"
 
+- name: Install yum epel-release
+  command: yum -y install epel-release
+  when: ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7'
+
 - name: Install yum packages
   yum:
     pkg: "{{ item }}"
diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml
index e016b855..8ced18b4 100644
--- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml
@@ -2,6 +2,7 @@
 packages:
   - ubuntu-cloud-keyring
   - python-dev
+  - python-pip
   - openvswitch-switch
   - openvswitch-switch-dpdk
   - python-memcache
diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml
index 3ec18e7f..b7e1d3dc 100644
--- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml
@@ -1,6 +1,7 @@
 ---
 packages:
   - python-devel
+  - python-pip
   - gcc
   - redhat-lsb-core
   - python-crypto
diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml
index 713b6b5f..7158325a 100644
--- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml
@@ -8,7 +8,6 @@
 ##############################################################################
 ---
 packages_noarch:
-  - python-pip
   - ntp
 
 services_noarch: []
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 b/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2
new file mode 100644
index 00000000..d998d4cb
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2
@@ -0,0 +1,34 @@
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = kubernetes
+DNS.2 = kubernetes.default
+DNS.3 = kubernetes.default.svc
+DNS.4 = kubernetes.default.svc.{{ dns_domain }}
+DNS.5 = localhost
+{% for host in groups['kube-master'] %}
+DNS.{{ 5 + loop.index }} = {{ host }}
+{% endfor %}
+{% if loadbalancer_apiserver is defined  and apiserver_loadbalancer_domain_name is defined %}
+{% set idx =  groups['kube-master'] | length | int + 5 + 1 %}
+DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }}
+{% endif %}
+{% for host in groups['kube-master'] %}
+IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }}
+{% endfor %}
+{% set idx =  groups['kube-master'] | length | int * 2 + 1 %}
+IP.{{ idx }} = {{ kube_apiserver_ip }}
+IP.{{ idx + 1 }} = 127.0.0.1
+{% if supplementary_addresses_in_ssl_keys is defined %}
+{% set is = idx + 1 %}
+{% for addr in supplementary_addresses_in_ssl_keys %}
+IP.{{ is + loop.index }} = {{ addr }}
+{% endfor %}
+{% endif %}
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
index 2763e53e..8a78c68d 100644
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
@@ -96,6 +96,51 @@
     regexp: '^helm_enabled:'
     line: 'helm_enabled: {{ helm_flag }}'
 
+- name: enable external lb | set lb domain_nam
+  lineinfile:
+    dest: /opt/kargo_k8s/inventory/group_vars/all.yml
+    regexp: '^## apiserver_loadbalancer_domain_name:'
+    line: 'apiserver_loadbalancer_domain_name: {{ apiserver_loadbalancer_domain_name }}'
+
+- name: enable external lb |
+  lineinfile:
+    dest: /opt/kargo_k8s/inventory/group_vars/all.yml
+    regexp: '^#loadbalancer_apiserver:'
+    line: 'loadbalancer_apiserver:'
+
+- name: enable external lb | set vip address
+  lineinfile:
+    dest: /opt/kargo_k8s/inventory/group_vars/all.yml
+    regexp: '^#  address: 1.2.3.4'
+    line: '  address: {{ vipaddress }}'
+
+- name: enable external lb | set vip port
+  lineinfile:
+    dest: /opt/kargo_k8s/inventory/group_vars/all.yml
+    regexp: '^#  port: 1234'
+    line: '  port: {{ exlb_port }}'
+
+- name: enable internal lb
+  lineinfile:
+    dest: /opt/kargo_k8s/inventory/group_vars/all.yml
+    regexp: '^#loadbalancer_apiserver_localhost: true'
+    line: 'loadbalancer_apiserver_localhost: true'
+
+- name: add vip to ssl keys
+  lineinfile:
+    dest: /opt/kargo_k8s/inventory/group_vars/k8s-cluster.yml
+    line: 'supplementary_addresses_in_ssl_keys: [{{ vipaddress }}]'
+
+- name: rm openssl file
+  file:
+    path: /opt/kargo_k8s/roles/kubernetes/secrets/templates/openssl.conf.j2
+    state: absent
+
+- name: copy openssl.conf.j2
+  copy:
+    src: openssl.conf.j2
+    dest: /opt/kargo_k8s/roles/kubernetes/secrets/templates/openssl.conf.j2
+
 - name: copy overrided variables
   copy:
     src: "{{ item }}"
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml
index 2d396d06..b73056e5 100644
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml
@@ -1,2 +1,5 @@
 ---
 helm_flag: true
+apiserver_loadbalancer_domain_name: "{{ public_vip.ip  }}"
+vipaddress: "{{ public_vip.ip  }}"
+exlb_port: 8383
diff --git a/deploy/compass_conf/flavor/kubernetes.conf b/deploy/compass_conf/flavor/kubernetes.conf
index 35c43155..71acadff 100755
--- a/deploy/compass_conf/flavor/kubernetes.conf
+++ b/deploy/compass_conf/flavor/kubernetes.conf
@@ -4,7 +4,7 @@ FLAVORS = [{
     'display_name': 'ansible-kubernetes',
     'template': 'ansible-kubernetes.tmpl',
     'roles': [
-        'kube_master', 'etcd', 'kube_node'
+        'kube_master', 'etcd', 'kube_node', 'ha'
     ],
 }]
 
diff --git a/deploy/compass_conf/package_installer/ansible-kubernetes.conf b/deploy/compass_conf/package_installer/ansible-kubernetes.conf
index 32590c82..820691b7 100755
--- a/deploy/compass_conf/package_installer/ansible-kubernetes.conf
+++ b/deploy/compass_conf/package_installer/ansible-kubernetes.conf
@@ -7,7 +7,7 @@ SETTINGS = {
     'playbook_file': 'site.yml',
     'inventory_file': 'inventory.py',
     'inventory_json_file': 'inventory.json',
-    'inventory_group': ['kube_master', 'etcd', 'kube_node'],
+    'inventory_group': ['kube_master', 'etcd', 'kube_node', 'ha'],
     'group_variable': 'all',
     'etc_hosts_path': 'roles/pre-k8s/templates/hosts',
     'runner_dirs': ['roles','kubernetes/roles']
diff --git a/deploy/compass_conf/role/kubernetes_ansible.conf b/deploy/compass_conf/role/kubernetes_ansible.conf
index ae096f47..c27779ad 100755
--- a/deploy/compass_conf/role/kubernetes_ansible.conf
+++ b/deploy/compass_conf/role/kubernetes_ansible.conf
@@ -11,5 +11,10 @@ ROLES = [{
     'role': 'kube_node',
     'display_name': 'kube node',
     'description': 'kube Node'
-} 
+}, {
+    'role': 'ha',
+    'display_name': 'ha',
+    'description': 'ha'
+}
+ 
 ]
diff --git a/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl b/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl
index 440bf7d7..ff4d587c 100644
--- a/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl
+++ b/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl
@@ -82,7 +82,7 @@ dashboard_host: "{{ internal_ip }}"
 haproxy_hosts:
 #for $item in $has
 #set $hostname=$item["hostname"]
-  $hostname: $ip_settings[$hostname]["mgmt"]["ip"]
+  $hostname: $ip_settings[$hostname]["external"]["ip"]
 #end for
 
 host_index: