From 41bdc69d9c6103d766889bc088c98791a7caf70b Mon Sep 17 00:00:00 2001 From: hu xinhui Date: Thu, 18 Oct 2018 14:31:14 +0800 Subject: 1.Optimization code for deploying k8s 2.Fix bugs for upgrade k8s version to v1.10.4 Change-Id: I6b17162574f4c4098eb6514cc067096e71f27f97 Signed-off-by: hu xinhui --- .../roles/kargo/files/extra-vars-aarch64.yml | 0 .../kubernetes/roles/kargo/files/extra-vars.yml | 7 - .../roles/kargo/files/generate_inventories.py | 95 ----------- .../kubernetes/roles/kargo/files/mirrors.repo | 32 ---- .../roles/kargo/files/mirrors_aarch64.repo | 0 .../kubernetes/roles/kargo/files/openssl.conf.j2 | 34 ---- .../ansible/kubernetes/roles/kargo/tasks/main.yml | 178 +++------------------ .../roles/kargo/templates/extra-vars.yml.j2 | 40 +++++ .../kubernetes/roles/kargo/templates/inventory.j2 | 26 +++ .../ansible/kubernetes/roles/kargo/vars/main.yml | 6 - 10 files changed, 92 insertions(+), 326 deletions(-) mode change 100644 => 100755 deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars-aarch64.yml delete mode 100644 deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars.yml delete mode 100755 deploy/adapters/ansible/kubernetes/roles/kargo/files/generate_inventories.py delete mode 100644 deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors.repo mode change 100644 => 100755 deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors_aarch64.repo delete mode 100644 deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 mode change 100644 => 100755 deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml create mode 100755 deploy/adapters/ansible/kubernetes/roles/kargo/templates/extra-vars.yml.j2 create mode 100644 deploy/adapters/ansible/kubernetes/roles/kargo/templates/inventory.j2 mode change 100644 => 100755 deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml (limited to 'deploy/adapters/ansible') diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars-aarch64.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars-aarch64.yml old mode 100644 new mode 100755 diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars.yml deleted file mode 100644 index e13e33ca..00000000 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/files/extra-vars.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- -# Override default kubespray variables - -# Just a placeholder to satisfy ansible -dummy_var: 0 - -# helm_enabled: true diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/generate_inventories.py b/deploy/adapters/ansible/kubernetes/roles/kargo/files/generate_inventories.py deleted file mode 100755 index 8f836011..00000000 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/files/generate_inventories.py +++ /dev/null @@ -1,95 +0,0 @@ -############################################################################## -# Copyright (c) 2016-2018 compass4nfv and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import yaml -import sys -import os -from jinja2 import Environment -try: - import json -except ImportError: - import simplejson as json - -INVENTORY_TEMPLATE = """ -[all] -{% for host, vales in hostvars.iteritems() %} -{{ host }} ansible_ssh_host={{ vales['ansible_ssh_host'] }} \ -ansible_ssh_pass=root ansible_user=root -{% endfor %} -[kube-master] -{% for host in kube_master %} -{{ host }} -{% endfor %} - -[etcd] -{% for host in etcd %} -{{ host }} -{% endfor %} - -[kube-node] -{% for host in kube_node %} -{{ host }} -{% endfor %} - -[k8s-cluster:children] -kube-node -kube-master - -[calico-rr] -[vault] -""" - - -def _byteify(data, ignore_dicts=False): - - if isinstance(data, unicode): - return data.encode('utf-8') - if isinstance(data, list): - return [_byteify(item, ignore_dicts=True) for item in data] - if isinstance(data, dict) and not ignore_dicts: - return { - _byteify(key, ignore_dicts=True): - _byteify(value, ignore_dicts=True) - for key, value in data.iteritems() - } - return data - - -def load_inventory(inventory): - if not os.path.exists(inventory): - raise RuntimeError('file: %s not exist' % inventory) - with open(inventory, 'r') as fd: - return json.load(fd, object_hook=_byteify) - - -def create_inventory_file(inventories_path, - hostvars, kube_master, etcd, kube_node): - content = Environment().from_string(INVENTORY_TEMPLATE).render( - hostvars=hostvars, kube_master=kube_master, - etcd=etcd, kube_node=kube_node) - with open(inventories_path, 'w+') as f: - f.write(content) - - -def main(inventories_path, local_inventory): - inventory_data = load_inventory(local_inventory) - hostvars = inventory_data['_meta']['hostvars'] - kube_node = inventory_data['kube_node']['hosts'] - kube_master = inventory_data['kube_master']['hosts'] - etcd = inventory_data['etcd']['hosts'] - - create_inventory_file(inventories_path, - hostvars, kube_master, etcd, kube_node) - - -if __name__ == "__main__": - path = yaml.load(sys.argv[1]) - local_inventory = yaml.load(sys.argv[2]) - - main(path, local_inventory) diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors.repo b/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors.repo deleted file mode 100644 index 4900db69..00000000 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors.repo +++ /dev/null @@ -1,32 +0,0 @@ -[base] -name=CentOS-$releasever - Base -mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os&infra=$infra -#baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/ -gpgcheck=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 - -#released updates -[updates] -name=CentOS-$releasever - Updates -mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates&infra=$infra -#baseurl=http://mirror.centos.org/centos/$releasever/updates/$basearch/ -gpgcheck=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 - -#additional packages that may be useful -[extras] -name=CentOS-$releasever - Extras -mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras&infra=$infra -#baseurl=http://mirror.centos.org/centos/$releasever/extras/$basearch/ -gpgcheck=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 - -#additional packages that extend functionality of existing packages -[centosplus] -name=CentOS-$releasever - Plus -mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus&infra=$infra -#baseurl=http://mirror.centos.org/centos/$releasever/centosplus/$basearch/ -gpgcheck=1 -enabled=0 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 - diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors_aarch64.repo b/deploy/adapters/ansible/kubernetes/roles/kargo/files/mirrors_aarch64.repo old mode 100644 new mode 100755 diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 b/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 deleted file mode 100644 index d998d4cb..00000000 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/files/openssl.conf.j2 +++ /dev/null @@ -1,34 +0,0 @@ -[req] -req_extensions = v3_req -distinguished_name = req_distinguished_name -[req_distinguished_name] -[ v3_req ] -basicConstraints = CA:FALSE -keyUsage = nonRepudiation, digitalSignature, keyEncipherment -subjectAltName = @alt_names -[alt_names] -DNS.1 = kubernetes -DNS.2 = kubernetes.default -DNS.3 = kubernetes.default.svc -DNS.4 = kubernetes.default.svc.{{ dns_domain }} -DNS.5 = localhost -{% for host in groups['kube-master'] %} -DNS.{{ 5 + loop.index }} = {{ host }} -{% endfor %} -{% if loadbalancer_apiserver is defined and apiserver_loadbalancer_domain_name is defined %} -{% set idx = groups['kube-master'] | length | int + 5 + 1 %} -DNS.{{ idx | string }} = {{ apiserver_loadbalancer_domain_name }} -{% endif %} -{% for host in groups['kube-master'] %} -IP.{{ 2 * loop.index - 1 }} = {{ hostvars[host]['access_ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} -IP.{{ 2 * loop.index }} = {{ hostvars[host]['ip'] | default(hostvars[host]['ansible_default_ipv4']['address']) }} -{% endfor %} -{% set idx = groups['kube-master'] | length | int * 2 + 1 %} -IP.{{ idx }} = {{ kube_apiserver_ip }} -IP.{{ idx + 1 }} = 127.0.0.1 -{% if supplementary_addresses_in_ssl_keys is defined %} -{% set is = idx + 1 %} -{% for addr in supplementary_addresses_in_ssl_keys %} -IP.{{ is + loop.index }} = {{ addr }} -{% endfor %} -{% endif %} diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml old mode 100644 new mode 100755 index 89d8db87..512121e2 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml @@ -7,166 +7,39 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## --- -- name: clean local repo conf - file: - path: /etc/yum.repos.d - state: absent - run_once: "True" - when: ansible_os_family == 'RedHat' - -- name: create local repo conf dir - file: - path: /etc/yum.repos.d - state: directory - run_once: "True" - when: ansible_os_family == 'RedHat' - -- name: configure local mirror repo - copy: - src: "{{ item }}" - dest: /etc/yum.repos.d/mirrors.repo - with_first_found: - - mirrors_{{ ansible_architecture }}.repo - - mirrors.repo - run_once: "True" - when: ansible_os_family == 'RedHat' - -- name: clean local pip conf to use official pip repo - file: - path: /root/.pip/pip.conf - state: absent - run_once: "True" - -- name: install dependency for ansible update - yum: - name: "{{ item }}" - state: latest - with_items: - - git - - libffi-devel - - openssl-devel - - python-devel - run_once: "True" - when: ansible_os_family == 'RedHat' - -- name: update python packages - pip: - name: "{{ item }}" - state: latest - with_items: - - netaddr - - jinja2 - -- name: copy inventories generate script - copy: - src: generate_inventories.py - dest: /tmp/generate_inventories.py +- name: check the kubespray sample path + stat: path=/opt/kargo_k8s/inventory/sample + register: sample_stat + +- name: Move kubespray group_vars folder + command: mv /opt/kargo_k8s/inventory/sample/group_vars /opt/kargo_k8s/inventory/ + when: sample_stat.stat.exists + +- name: generate kubespray inventory configure file + template: + src: "inventory.j2" + dest: "/opt/kargo_k8s/inventory/inventory.cfg" tags: - ansible -- name: copy inventoriy.json file - copy: - src: "{{ run_dir }}/inventories/inventory.json" - dest: /tmp/inventory.json - tags: - - ansible - -- name: generate kargo inventories - shell: > - python /tmp/generate_inventories.py \ - "/opt/kargo_k8s/inventory/inventory.cfg" \ - "/tmp/inventory.json" - tags: - - ansible - -- name: configure target hosts - shell: | - cd /opt/kargo_k8s - ansible -i inventory/inventory.cfg -m ping all - ansible -i inventory/inventory.cfg all -m shell -a "rm /etc/yum.repos.d/*" - ansible -i inventory/inventory.cfg all -m copy -a \ - "src=/etc/yum.repos.d/mirrors.repo dest=/etc/yum.repos.d" - tags: - - ansible - -- name: enable helm - lineinfile: - dest: /opt/kargo_k8s/inventory/sample/group_vars/k8s-cluster.yml - regexp: '^helm_enabled:' - line: 'helm_enabled: {{ helm_flag }}' - -- name: enable external lb | set lb domain_nam - lineinfile: - dest: /opt/kargo_k8s/inventory/sample/group_vars/all.yml - regexp: '^## apiserver_loadbalancer_domain_name:' - line: 'apiserver_loadbalancer_domain_name: {{ apiserver_loadbalancer_domain_name }}' - -- name: enable external lb | - lineinfile: - dest: /opt/kargo_k8s/inventory/sample/group_vars/all.yml - regexp: '^#loadbalancer_apiserver:' - line: 'loadbalancer_apiserver:' - -- name: enable external lb | set vip address - lineinfile: - dest: /opt/kargo_k8s/inventory/sample/group_vars/all.yml - regexp: '^# address: 1.2.3.4' - line: ' address: {{ vipaddress }}' - -- name: enable external lb | set vip port - lineinfile: - dest: /opt/kargo_k8s/inventory/sample/group_vars/all.yml - regexp: '^# port: 1234' - line: ' port: {{ exlb_port }}' - -- name: enable internal lb - lineinfile: - dest: /opt/kargo_k8s/inventory/sample/group_vars/all.yml - regexp: '^#loadbalancer_apiserver_localhost: true' - line: 'loadbalancer_apiserver_localhost: true' - -- name: enable http proxy - lineinfile: - dest: /opt/kargo_k8s/inventory/sample/group_vars/all.yml - regexp: '^#http_proxy:' - line: 'http_proxy: {{ http_proxy }}' - when: http_proxy != '' - -- name: enable https proxy - lineinfile: - dest: /opt/kargo_k8s/inventory/sample/group_vars/all.yml - regexp: '^#https_proxy:' - line: 'https_proxy: {{ https_proxy }}' - when: https_proxy !='' - -- name: use the user name and password login the dashboard - lineinfile: - dest: /opt/kargo_k8s/inventory/sample/group_vars/k8s-cluster.yml - regexp: '^#kube_basic_auth: false' - line: 'kube_basic_auth: true' - -- name: add vip to ssl keys - lineinfile: - dest: /opt/kargo_k8s/inventory/sample/group_vars/k8s-cluster.yml - line: 'supplementary_addresses_in_ssl_keys: [{{ vipaddress }}]' - -- name: rm openssl file - file: - path: /opt/kargo_k8s/roles/kubernetes/secrets/templates/openssl.conf.j2 - state: absent - -- name: copy openssl.conf.j2 - copy: - src: openssl.conf.j2 - dest: /opt/kargo_k8s/roles/kubernetes/secrets/templates/openssl.conf.j2 - -- name: copy overrided variables +- name: copy overrided variables for arm architecture copy: src: "{{ item }}" dest: /opt/kargo_k8s/extra-vars.yml with_first_found: - extra-vars-{{ ansible_architecture }}.yml - extra-vars.yml + - skip: true + +- name: copy overrided variables for kubespray + template: + src: "{{ item }}" + dest: "/opt/kargo_k8s/extra-vars.yml" + with_first_found: + - extra-vars-{{ ansible_architecture }}.yml.j2 + - extra-vars.yml.j2 + tags: + - ansible - name: copy 2flannel playbook to kargo copy: @@ -277,9 +150,10 @@ kube-controller-manager.manifest.j2", - name: run kargo playbook shell: | - cd /opt/kargo_k8s ansible-playbook -i inventory/inventory.cfg cluster.yml \ -e "@extra-vars.yml" -b -v 2>&1 | tee kargo.log + args: + chdir: "/opt/kargo_k8s" tags: - ansible diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/templates/extra-vars.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/kargo/templates/extra-vars.yml.j2 new file mode 100755 index 00000000..1d7a2fa2 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/templates/extra-vars.yml.j2 @@ -0,0 +1,40 @@ +--- +# Override default kubespray variables + +#dashboard_port: "{{dashboard_port|default('31746')}}" + +# kubespray configure +apiserver_loadbalancer_domain_name: "{{ public_vip.ip }}" +loadbalancer_apiserver: + address: "{{ public_vip.ip }}" + port: {{ loadbalancer_apiserver_port|default(8383) }} +loadbalancer_apiserver_localhost: {{ loadbalancer_apiserver_localhost|default(true) }} + +kube_basic_auth: {{ kube_basic_auth |default(true) }} +kube_network_plugin: {{ kube_network_plugin|default('calico') }} +# Monitoring apps for k8s +efk_enabled: {{ efk_enabled |default(true)}} +# Helm deployment +helm_enabled: {{ helm_enabled |default(true)}} +# Istio deployment +istio_enabled: {{ istio_enabled |default(false)}} +supplementary_addresses_in_ssl_keys: ["{{ public_vip.ip }}"] +#storage +local_volume_provisioner_enabled: {{local_volume_provisioner_enabled |default(false) }} +# local_volume_provisioner_namespace: "system_namespace" +# local_volume_provisioner_base_dir: /mnt/disks +# local_volume_provisioner_mount_dir: /mnt/disks +# local_volume_provisioner_storage_class: local-storage + +# CephFS provisioner deployment +cephfs_provisioner_enabled: {{ cephfs_provisioner_enabled |default(false)}} +# cephfs_provisioner_namespace: "cephfs-provisioner" +# cephfs_provisioner_cluster: ceph +# cephfs_provisioner_monitors: "172.24.0.1:6789,172.24.0.2:6789,172.24.0.3:6789" +# cephfs_provisioner_admin_id: admin +# cephfs_provisioner_secret: secret +# cephfs_provisioner_storage_class: cephfs +# cephfs_provisioner_reclaim_policy: Delete +# cephfs_provisioner_claim_root: /volumes +# cephfs_provisioner_deterministic_names: true + diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/templates/inventory.j2 b/deploy/adapters/ansible/kubernetes/roles/kargo/templates/inventory.j2 new file mode 100644 index 00000000..0120ae18 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/templates/inventory.j2 @@ -0,0 +1,26 @@ +[all] +{% for host, vales in hostvars.iteritems() %} +{{ host }} ansible_ssh_host={{ vales['ansible_ssh_host'] }} ansible_ssh_pass=root ansible_user=root +{% endfor %} + +[kube-master] +{% for host in hostvars[inventory_hostname]['groups']['kube_master'] %} +{{ host }} +{% endfor %} + +[etcd] +{% for host in hostvars[inventory_hostname]['groups']['etcd'] %} +{{ host }} +{% endfor %} + +[kube-node] +{% for host in hostvars[inventory_hostname]['groups']['kube_node'] %} +{{ host }} +{% endfor %} + +[k8s-cluster:children] +kube-node +kube-master + +[calico-rr] +[vault] diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml old mode 100644 new mode 100755 index 80490955..af9c9675 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/vars/main.yml @@ -1,9 +1,3 @@ --- -helm_flag: true -apiserver_loadbalancer_domain_name: "{{ public_vip.ip }}" -vipaddress: "{{ public_vip.ip }}" -exlb_port: 8383 -kubelet_fail_swap_on: false - http_proxy: "{{ proxy }}" https_proxy: "{{ proxy }}" -- cgit 1.2.3-korg