blob: fb0d43a874b296537d2f33dcef1909dc812f8dfc (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
|
# SPDX-FileCopyrightText: 2021 Intel Corporation.
#
# SPDX-License-Identifier: Apache-2.0
---
- hosts: 127.0.0.1
connection: local
tasks: []
roles:
- { role: kubespray_install }
environment: "{{ proxy_env | d({}) }}"
any_errors_fatal: true
- hosts: k8s-cluster
tasks: []
roles:
- role: cluster_defaults
- role: kubespray_target_setup
environment: "{{ proxy_env | d({}) }}"
any_errors_fatal: true
- hosts: all
gather_facts: false
tasks:
- name: prepare additional kubespray facts
set_fact:
kubelet_node_custom_flags_prepare: >-
{%- if native_cpu_manager_enabled | default(false) and native_cpu_manager_reserved_cpus is defined -%}
--reserved-cpus={{ native_cpu_manager_reserved_cpus }}
{%- endif -%}
enable_admission_plugins_prepare: >-
[EventRateLimit,{% if always_pull_enabled %} AlwaysPullImages,{% endif %} NodeRestriction{% if psp_enabled %}, PodSecurityPolicy{% endif %}]
bmra_docker_version: >-
{% if ansible_distribution_version >= '21.04' %}latest{% else %}19.03{%endif %}
flannel_backend_type: >-
{% if ansible_distribution_version >= '21.04' %}host-gw{% else %}vxlan{%endif %}
kube_config_dir: /etc/kubernetes
- name: set kube_cert_dir
set_fact:
kube_cert_dir: "{{ kube_config_dir }}/ssl"
kube_csr_dir: "{{ kube_config_dir }}/csr"
environment: "{{ proxy_env | d({}) }}"
any_errors_fatal: true
- hosts: all
tasks:
- name: add docker runtime vars
set_fact:
container_manager: docker
docker_iptables_enabled: true
docker_dns_servers_strict: false
docker_version: "{{ bmra_docker_version }}"
when: container_runtime == "docker"
- name: add containerd runtime vars
set_fact:
container_manager: containerd
etcd_deployment_type: host
containerd_extra_args: |2
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."{{ registry_local_address }}"]
endpoint = ["https://{{ registry_local_address }}"]
[plugins."io.containerd.grpc.v1.cri".registry.configs."{{ registry_local_address }}".tls]
ca_file = "/etc/containers/certs.d/{{ registry_local_address }}/ca.crt"
when: container_runtime == "containerd"
- name: run kubespray
import_playbook: kubespray/cluster.yml
vars:
kubeadm_enabled: true
multus_conf_file: /host/etc/cni/net.d/templates/00-multus.conf
nginx_image_tag: 1.21.1
override_system_hostname: false
kube_proxy_mode: iptables
enable_nodelocaldns: false
system_reserved: true
dashboard_enabled: true
system_cpu_reserved: "{{ native_cpu_manager_system_reserved_cpus | default('1000m') }}"
kube_cpu_reserved: "{{ native_cpu_manager_kube_reserved_cpus | default('1000m') }}"
kubelet_node_custom_flags: "{{ kubelet_node_custom_flags_prepare | from_yaml }}"
kube_api_anonymous_auth: true
kube_feature_gates:
- CPUManager=true # feature gate can be enabled by default, default policy is none in Kubernetes
- TopologyManager={{ topology_manager_enabled | default(true) }}
- RotateKubeletServerCertificate=true
# Kubernetes cluster hardening
kubernetes_audit: true
audit_log_maxbackups: 10
kube_controller_manager_bind_address: 127.0.0.1
kube_scheduler_bind_address: 127.0.0.1
kube_proxy_healthz_bind_address: 127.0.0.1
kube_proxy_metrics_bind_address: 127.0.0.1
kube_read_only_port: 0
kube_override_hostname: ""
kube_kubeadm_apiserver_extra_args:
service-account-lookup: true
service-account-key-file: "{{ kube_cert_dir }}/sa.key"
admission-control-config-file: "{{ kube_config_dir }}/admission-control/config.yaml"
kube_kubeadm_scheduler_extra_args:
address: 127.0.0.1
profiling: false
kube_kubeadm_controller_extra_args:
address: 127.0.0.1
service-account-private-key-file: "{{ kube_cert_dir }}/sa.key"
kubelet_config_extra_args:
protectKernelDefaults: true
cpuManagerPolicy: "{% if native_cpu_manager_enabled | default(false) %}static{% else %}none{% endif %}"
topologyManagerPolicy: "{{ topology_manager_policy | default('none') }}"
eventRecordQPS: 0
kube_apiserver_request_timeout: 60s
kube_apiserver_enable_admission_plugins: "{{ enable_admission_plugins_prepare | from_yaml }}"
podsecuritypolicy_enabled: "{{ psp_enabled }}"
kube_encrypt_secret_data: true
apiserver_extra_volumes:
- name: admission-control-config
hostPath: /etc/kubernetes/admission-control/
mountPath: /etc/kubernetes/admission-control/
readOnly: true
preinstall_selinux_state: "{{ selinux_mode | default('disabled') }}"
tls_cipher_suites:
- TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
- TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
- TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384
etcd_extra_vars:
ETCD_CIPHER_SUITES: "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384"
- hosts: k8s-cluster
tasks:
- name: restart docker daemon to recreate iptables rules
systemd: name=docker state=restarted
become: yes
when: container_runtime == "docker"
- name: restart kubelet to trigger static pods recreation
systemd: name=kubelet state=restarted
become: yes
# note: fix for the issue mentioned here:
# https://github.com/kubernetes-sigs/kubespray/blob/58f48500b1adac3f18466fa1c5cf8aa9d9838150/docs/flannel.md#flannel
- name: check if flannel.1 interface exists
stat:
path: /sys/class/net/flannel.1
when: kube_network_plugin == "flannel"
register: flannel_endpoint
- name: disable offloading features on flannel.1
command: ethtool --offload flannel.1 rx off tx off
become: yes
when:
- kube_network_plugin == "flannel"
- flannel_endpoint.stat.exists
- hosts: etcd
tasks:
- name: change /var/lib/etcd owner
file:
path: "{{ etcd_data_dir | default('/var/lib/etcd') }}"
owner: etcd
group: etcd
recurse: true
state: directory
mode: 0700
- name: change /var/lib/etcd permissions
file:
path: "{{ etcd_data_dir | default('/var/lib/etcd') }}"
owner: etcd
group: etcd
mode: '0700'
state: directory
- hosts: k8s-cluster
roles:
- role: cluster_defaults
tags: defaults
- role: container_registry
tags: registry
- role: dockerhub_credentials
when: "'/bmra/roles/dockerhub_credentials/vars/main.yml' is file"
environment: "{{ proxy_env | d({}) }}"
any_errors_fatal: true
- name: run certificate generation for mTLS in kubelet
import_playbook: kubelet-certificates.yml
|
