From c6b9a863cf92f824e8b8e3004f6e1f649170e4f1 Mon Sep 17 00:00:00 2001 From: hu xinhui Date: Wed, 10 Jan 2018 16:03:25 +0800 Subject: spport k8s apiserver HA compass installer deploy k8s using kubespray for default, but k8s apiserver HA is not implemented by kubespray, This patch aim is to achieve the k8s apiserver HA Change-Id: I805b5eb2f4efa7ca82fcef7bfd3f4cad35ed65b5 JIRA: - Signed-off-by: hu xinhui --- build.sh | 1 + .../ansible/kubernetes/ansible-kubernetes.yml | 6 ++ .../kubernetes/roles/ha/files/chk_k8s_master.sh | 9 +++ .../ansible/kubernetes/roles/ha/handlers/main.yml | 14 ++++ .../ansible/kubernetes/roles/ha/tasks/main.yml | 83 ++++++++++++++++++++++ .../kubernetes/roles/ha/templates/haproxy.cfg | 48 +++++++++++++ .../kubernetes/roles/ha/templates/keepalived.conf | 49 +++++++++++++ .../ansible/kubernetes/roles/ha/vars/Debian.yml | 11 +++ .../ansible/kubernetes/roles/ha/vars/RedHat.yml | 11 +++ .../ansible/kubernetes/roles/ha/vars/main.yml | 16 +++++ .../roles/install-k8s-dependence/tasks/main.yml | 4 ++ .../roles/install-k8s-dependence/vars/Debian.yml | 1 + .../roles/install-k8s-dependence/vars/RedHat.yml | 1 + .../roles/install-k8s-dependence/vars/main.yml | 1 - .../kubernetes/roles/kargo/files/openssl.conf.j2 | 34 +++++++++ .../ansible/kubernetes/roles/kargo/tasks/main.yml | 45 ++++++++++++ .../ansible/kubernetes/roles/kargo/vars/main.yml | 3 + deploy/compass_conf/flavor/kubernetes.conf | 2 +- .../package_installer/ansible-kubernetes.conf | 2 +- deploy/compass_conf/role/kubernetes_ansible.conf | 7 +- .../kubernetes/vars/ansible-kubernetes.tmpl | 2 +- 21 files changed, 345 insertions(+), 5 deletions(-) create mode 100644 deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh create mode 100644 deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml create mode 100644 deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml create mode 100644 deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg create mode 100644 deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf create mode 100644 deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml create mode 100644 deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml create mode 100644 deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml create mode 100644 deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 diff --git a/build.sh b/build.sh index 7f67ee2a..590347b9 100755 --- a/build.sh +++ b/build.sh @@ -142,3 +142,4 @@ process_param $* prepare_env download_packages build_tar + diff --git a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml index eb80066e..bfdc8958 100755 --- a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml +++ b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml @@ -25,6 +25,12 @@ roles: - install-k8s-dependence +- hosts: ha + remote_user: root + max_fail_percentage: 0 + roles: + - ha + - hosts: localhost remote_user: root max_fail_percentage: 0 diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh b/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh new file mode 100644 index 00000000..62e79b3b --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/files/chk_k8s_master.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +count=`ss -tnl | grep 6443 | wc -l` + +if [ $count = 0 ]; then + exit 1 +else + exit 0 +fi diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml new file mode 100644 index 00000000..03ed82ec --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/handlers/main.yml @@ -0,0 +1,14 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +- name: restart haproxy + service: name=haproxy state=restarted enabled=yes + +- name: restart keepalived + service: name=keepalived state=restarted enabled=yes diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml new file mode 100644 index 00000000..c7e58376 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/tasks/main.yml @@ -0,0 +1,83 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +- include_vars: "{{ ansible_os_family }}.yml" + +- name: install keepalived haproxy + action: "{{ ansible_pkg_mgr }} name={{ item }} state=present" + with_items: "{{ packages | union(packages_noarch) }}" + +- name: generate ha service list + lineinfile: dest=/opt/service create=yes line= '{{ item }}' + with_items: "{{ services | union(services_noarch) }}" + +- name: install pexpect + pip: name=pexpect state=present extra_args='--pre' + +- name: activate ip_nonlocal_bind + sysctl: name=net.ipv4.ip_nonlocal_bind value=1 + state=present reload=yes + +- name: set net.ipv4.tcp_keepalive_intvl + sysctl: name=net.ipv4.tcp_keepalive_intvl value=1 + state=present reload=yes + +- name: set net.ipv4.tcp_keepalive_probes + sysctl: name=net.ipv4.tcp_keepalive_probes value=5 + state=present reload=yes + +- name: set net.ipv4.tcp_keepalive_time + sysctl: name=net.ipv4.tcp_keepalive_time value=5 + state=present reload=yes + +- name: update haproxy cfg + template: src=haproxy.cfg dest=/etc/haproxy/haproxy.cfg + notify: restart haproxy + +- name: set haproxy enable flag + lineinfile: dest=/etc/default/haproxy state=present + regexp="ENABLED=*" + line="ENABLED=1" + notify: restart haproxy + when: ansible_os_family == "Debian" + +- name: set haproxy log + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="local0.* /var/log/haproxy.log" + line="local0.* /var/log/haproxy.log" + +- name: set rsyslog udp module + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="^#$ModLoad imudp" + line="$ModLoad imudp" + +- name: set rsyslog udp port + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="^#$UDPServerRun 514" + line="$UDPServerRun 514" + +- name: set keepalived start param + lineinfile: dest=/etc/default/keepalived state=present + regexp="^DAEMON_ARGS=*" + line="DAEMON_ARGS=\"-D -d -S 1\"" + when: ansible_os_family == "Debian" + +- name: set keepalived log + lineinfile: dest=/etc/rsyslog.conf state=present + regexp="local1.* /var/log/keepalived.log" + line="local1.* /var/log/keepalived.log" + +- name: update keepalived info + template: src=keepalived.conf dest=/etc/keepalived/keepalived.conf + notify: restart keepalived + +- name: restart rsyslog + shell: service rsyslog restart + +- meta: flush_handlers diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg b/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg new file mode 100644 index 00000000..5cd240c0 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/templates/haproxy.cfg @@ -0,0 +1,48 @@ + +global + #chroot /var/run/haproxy + daemon + user haproxy + group haproxy + maxconn 4000 + pidfile /var/run/haproxy/haproxy.pid + #log 127.0.0.1 local0 + tune.bufsize 1000000 + stats socket /var/run/haproxy.sock + stats timeout 2m + +defaults + log global + maxconn 8000 + option redispatch + option dontlognull + option splice-auto + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 50s + timeout server 50s + timeout check 10s + retries 3 + +listen kubernetes-apiserver-https + bind {{ public_vip.ip }}:8383 + option ssl-hello-chk + mode tcp + option tcpka + option tcplog + timeout client 3h + timeout server 3h + balance roundrobin +{% for host,ip in haproxy_hosts.items() %} + server {{ host }} {{ ip }}:6443 weight 1 check inter 2000 rise 2 fall 5 +{% endfor %} + +listen stats + mode http + bind 0.0.0.0:9999 + stats enable + stats refresh 30s + stats uri / + stats realm Global\ statistics + stats auth admin:admin diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf b/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf new file mode 100644 index 00000000..c649bed5 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/templates/keepalived.conf @@ -0,0 +1,49 @@ +global_defs { + router_id {{ inventory_hostname }} +} + +vrrp_sync_group VG1 { + group { + internal_vip + public_vip + } +} + +vrrp_instance internal_vip { + interface {{ sys_intf_mappings.mgmt.interface }} + virtual_router_id {{ vrouter_id_internal }} + state BACKUP + nopreempt + advert_int 1 + priority {{ 50 + (host_index[inventory_hostname] * 50) }} + + authentication { + auth_type PASS + auth_pass 1234 + } + + + virtual_ipaddress { + {{ internal_vip.ip }}/{{ internal_vip.netmask }} dev {{ sys_intf_mappings.mgmt.interface }} + } +} + +vrrp_instance public_vip { + interface {{ sys_intf_mappings.external.interface }} + virtual_router_id {{ vrouter_id_public }} + state BACKUP + nopreempt + advert_int 1 + priority {{ 50 + (host_index[inventory_hostname] * 50) }} + + authentication { + auth_type PASS + auth_pass 4321 + } + + virtual_ipaddress { + {{ network_cfg.public_vip.ip }}/{{ network_cfg.public_vip.netmask }} dev {{ sys_intf_mappings.external.interface }} + } + +} + diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml new file mode 100644 index 00000000..b9f46bdf --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/Debian.yml @@ -0,0 +1,11 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +services: [] +packages: [] diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml new file mode 100644 index 00000000..b9f46bdf --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/RedHat.yml @@ -0,0 +1,11 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +services: [] +packages: [] diff --git a/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml new file mode 100644 index 00000000..77735d1e --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/ha/vars/main.yml @@ -0,0 +1,16 @@ +############################################################################## +# Copyright (c) 2016 HUAWEI TECHNOLOGIES CO.,LTD and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +packages_noarch: + - keepalived + - haproxy + +services_noarch: + - keepalived + - haproxy diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml index 6487e4ef..e683a3fe 100644 --- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/tasks/main.yml @@ -9,6 +9,10 @@ --- - include_vars: "{{ ansible_os_family }}.yml" +- name: Install yum epel-release + command: yum -y install epel-release + when: ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7' + - name: Install yum packages yum: pkg: "{{ item }}" diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml index e016b855..8ced18b4 100644 --- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml +++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/Debian.yml @@ -2,6 +2,7 @@ packages: - ubuntu-cloud-keyring - python-dev + - python-pip - openvswitch-switch - openvswitch-switch-dpdk - python-memcache diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml index 3ec18e7f..b7e1d3dc 100644 --- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml +++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/RedHat.yml @@ -1,6 +1,7 @@ --- packages: - python-devel + - python-pip - gcc - redhat-lsb-core - python-crypto diff --git a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml index 713b6b5f..7158325a 100644 --- a/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/install-k8s-dependence/vars/main.yml @@ -8,7 +8,6 @@ ############################################################################## --- packages_noarch: - - python-pip - ntp services_noarch: [] diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 b/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 new file mode 100644 index 00000000..d998d4cb --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 @@ -0,0 +1,34 @@ +[req] +req_extensions = v3_req +distinguished_name = req_distinguished_name +[req_distinguished_name] +[ v3_req ] +basicConstraints = CA:FALSE +keyUsage = nonRepudiation, digitalSignature, keyEncipherment +subjectAltName = @alt_names +[alt_names] +DNS.1 = kubernetes +DNS.2 = kubernetes.default +DNS.3 = kubernetes.default.svc +DNS.4 = kubernetes.default.svc.{{ dns_domain }} +DNS.5 = localhost +{% for host in groups['kube-master'] %} +DNS.{{ 5 + loop.index }} = {{ host }} +{% endfor %} +{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} +{% set idx = groups['kube-master'] | length | int + 5 + 1 %} +DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }} +{% endif %} +{% for host in groups['kube-master'] %} +IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} +IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} +{% endfor %} +{% set idx = groups['kube-master'] | length | int * 2 + 1 %} +IP.{{ idx }} = {{ kube_apiserver_ip }} +IP.{{ idx + 1 }} = 127.0.0.1 +{% if supplementary_addresses_in_ssl_keys is defined %} +{% set is = idx + 1 %} +{% for addr in supplementary_addresses_in_ssl_keys %} +IP.{{ is + loop.index }} = {{ addr }} +{% endfor %} +{% endif %} diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml index 2763e53e..8a78c68d 100644 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml @@ -96,6 +96,51 @@ regexp: '^helm_enabled:' line: 'helm_enabled: {{ helm_flag }}' +- name: enable external lb | set lb domain_nam + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/all.yml + regexp: '^## apiserver_loadbalancer_domain_name:' + line: 'apiserver_loadbalancer_domain_name: {{ apiserver_loadbalancer_domain_name }}' + +- name: enable external lb | + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/all.yml + regexp: '^#loadbalancer_apiserver:' + line: 'loadbalancer_apiserver:' + +- name: enable external lb | set vip address + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/all.yml + regexp: '^# address: 1.2.3.4' + line: ' address: {{ vipaddress }}' + +- name: enable external lb | set vip port + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/all.yml + regexp: '^# port: 1234' + line: ' port: {{ exlb_port }}' + +- name: enable internal lb + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/all.yml + regexp: '^#loadbalancer_apiserver_localhost: true' + line: 'loadbalancer_apiserver_localhost: true' + +- name: add vip to ssl keys + lineinfile: + dest: /opt/kargo_k8s/inventory/group_vars/k8s-cluster.yml + line: 'supplementary_addresses_in_ssl_keys: [{{ vipaddress }}]' + +- name: rm openssl file + file: + path: /opt/kargo_k8s/roles/kubernetes/secrets/templates/openssl.conf.j2 + state: absent + +- name: copy openssl.conf.j2 + copy: + src: openssl.conf.j2 + dest: /opt/kargo_k8s/roles/kubernetes/secrets/templates/openssl.conf.j2 + - name: copy overrided variables copy: src: "{{ item }}" diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml index 2d396d06..b73056e5 100644 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml @@ -1,2 +1,5 @@ --- helm_flag: true +apiserver_loadbalancer_domain_name: "{{ public_vip.ip }}" +vipaddress: "{{ public_vip.ip }}" +exlb_port: 8383 diff --git a/deploy/compass_conf/flavor/kubernetes.conf b/deploy/compass_conf/flavor/kubernetes.conf index 35c43155..71acadff 100755 --- a/deploy/compass_conf/flavor/kubernetes.conf +++ b/deploy/compass_conf/flavor/kubernetes.conf @@ -4,7 +4,7 @@ FLAVORS = [{ 'display_name': 'ansible-kubernetes', 'template': 'ansible-kubernetes.tmpl', 'roles': [ - 'kube_master', 'etcd', 'kube_node' + 'kube_master', 'etcd', 'kube_node', 'ha' ], }] diff --git a/deploy/compass_conf/package_installer/ansible-kubernetes.conf b/deploy/compass_conf/package_installer/ansible-kubernetes.conf index 32590c82..820691b7 100755 --- a/deploy/compass_conf/package_installer/ansible-kubernetes.conf +++ b/deploy/compass_conf/package_installer/ansible-kubernetes.conf @@ -7,7 +7,7 @@ SETTINGS = { 'playbook_file': 'site.yml', 'inventory_file': 'inventory.py', 'inventory_json_file': 'inventory.json', - 'inventory_group': ['kube_master', 'etcd', 'kube_node'], + 'inventory_group': ['kube_master', 'etcd', 'kube_node', 'ha'], 'group_variable': 'all', 'etc_hosts_path': 'roles/pre-k8s/templates/hosts', 'runner_dirs': ['roles','kubernetes/roles'] diff --git a/deploy/compass_conf/role/kubernetes_ansible.conf b/deploy/compass_conf/role/kubernetes_ansible.conf index ae096f47..c27779ad 100755 --- a/deploy/compass_conf/role/kubernetes_ansible.conf +++ b/deploy/compass_conf/role/kubernetes_ansible.conf @@ -11,5 +11,10 @@ ROLES = [{ 'role': 'kube_node', 'display_name': 'kube node', 'description': 'kube Node' -} +}, { + 'role': 'ha', + 'display_name': 'ha', + 'description': 'ha' +} + ] diff --git a/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl b/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl index 440bf7d7..ff4d587c 100644 --- a/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl +++ b/deploy/compass_conf/templates/ansible_installer/kubernetes/vars/ansible-kubernetes.tmpl @@ -82,7 +82,7 @@ dashboard_host: "{{ internal_ip }}" haproxy_hosts: #for $item in $has #set $hostname=$item["hostname"] - $hostname: $ip_settings[$hostname]["mgmt"]["ip"] + $hostname: $ip_settings[$hostname]["external"]["ip"] #end for host_index: -- cgit 1.2.3-korg