diff options
Diffstat (limited to 'deploy/adapters/ansible/kubernetes')
9 files changed, 502 insertions, 1 deletions
diff --git a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml index bfdc8958..1eae8a97 100755 --- a/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml +++ b/deploy/adapters/ansible/kubernetes/ansible-kubernetes.yml @@ -42,3 +42,15 @@ max_fail_percentage: 0 roles: - post-k8s + +- hosts: kube_node + remote_user: root + max_fail_percentage: 0 + roles: + - storage + +- hosts: storage_master + remote_user: root + max_fail_percentage: 0 + roles: + - stor4nfv diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel-apps/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/2flannel-apps/tasks/main.yml new file mode 100644 index 00000000..a7ff10e2 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/2flannel-apps/tasks/main.yml @@ -0,0 +1,22 @@ +--- +- name: "2Flannel | Create ServiceAccount ClusterRole and ClusterRoleBinding" + command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/cni-2flannel-rbac.yml" + run_once: true + when: rbac_enabled and two_flannel_rbac_manifest.changed + +- name: 2Flannel | Start Resources + kube: + name: "kube-2flannel" + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/cni-2flannel.yml" + resource: "ds" + namespace: "{{system_namespace}}" + state: "{{ item | ternary('latest','present') }}" + with_items: "{{ two_flannel_manifest.changed }}" + when: inventory_hostname == groups['kube-master'][0] + +- name: 2Flannel | Wait for flannel subnet.env file presence + wait_for: + path: /run/2flannel/networks/subnet1.env + delay: 5 + timeout: 600 diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/defaults/main.yml b/deploy/adapters/ansible/kubernetes/roles/2flannel/defaults/main.yml new file mode 100644 index 00000000..f9e61eda --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/defaults/main.yml @@ -0,0 +1,21 @@ +--- +# Flannel public IP +# The address that flannel should advertise as how to access the system +# Disabled until https://github.com/coreos/flannel/issues/712 is fixed +# flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}" + +## interface that should be used for flannel operations +## This is actually an inventory node-level item +# flannel_interface: + +# You can choose what type of flannel backend to use +# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md + +# Limits for apps +flannel_memory_limit: 500M +flannel_cpu_limit: 300m +flannel_memory_requests: 64M +flannel_cpu_requests: 150m + +two_flannel_network1: "{{ kube_pods_subnet }}" +two_flannel_network2: "10.235.64.0/18" diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/handlers/main.yml b/deploy/adapters/ansible/kubernetes/roles/2flannel/handlers/main.yml new file mode 100644 index 00000000..44ead00e --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/handlers/main.yml @@ -0,0 +1,61 @@ +--- +- name: 2Flannel | delete default docker bridge + command: ip link delete docker0 + failed_when: false + notify: 2Flannel | restart docker + +- name: 2Flannel | delete 2flannel interface + command: ip link delete flannel0 && ip link delete flannel1 + failed_when: false + +# special cases for atomic because it defaults to live-restore: true +# So we disable live-restore to pickup the new flannel IP. After +# we enable it, we have to restart docker again to pickup the new +# setting and restore the original behavior +- name: 2Flannel | restart docker + command: /bin/true + notify: + - 2Flannel | reload systemd + - 2Flannel | reload docker.socket + - 2Flannel | configure docker live-restore true (atomic) + - 2Flannel | reload docker + - 2Flannel | pause while Docker restarts + - 2Flannel | wait for docker + +- name: 2Flannel | reload systemd + shell: systemctl daemon-reload + +- name: 2Flannel | reload docker.socket + service: + name: docker.socket + state: restarted + when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] + +- name: 2Flannel | configure docker live-restore true (atomic) + replace: + name: /etc/docker/daemon.json + regexp: '"live-restore":.*true' + replace: '"live-restore": false' + when: is_atomic + +- name: 2Flannel | reload docker + service: + name: docker + state: restarted + +- name: 2Flannel | pause while Docker restarts + pause: + seconds: 10 + prompt: "Waiting for docker restart" + +- name: 2Flannel | wait for docker + command: "{{ docker_bin_dir }}/docker images" + register: docker_ready + retries: 10 + delay: 5 + until: docker_ready.rc == 0 + +- name: 2Flannel | reload kubelet + service: + name: kubelet + state: restarted diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/main.yml new file mode 100644 index 00000000..e7adeefe --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/main.yml @@ -0,0 +1,73 @@ +--- +- include: pre-upgrade.yml + +- name: 2Flannel | Verify if br_netfilter module exists + shell: "modinfo br_netfilter" + register: modinfo_br_netfilter + failed_when: modinfo_br_netfilter.rc not in [0, 1] + changed_when: false + +- name: 2Flannel | Enable br_netfilter module + modprobe: + name: br_netfilter + state: present + when: modinfo_br_netfilter.rc == 0 + +# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found +# if br_netfilter is not a module +- name: 2Flannel | Check if bridge-nf-call-iptables key exists + command: "sysctl net.bridge.bridge-nf-call-iptables" + failed_when: false + changed_when: false + register: sysctl_bridge_nf_call_iptables + +- name: 2Flannel | Enable bridge-nf-call tables + sysctl: + name: "{{ item }}" + state: present + value: 1 + reload: "yes" + when: modinfo_br_netfilter.rc == 1 and sysctl_bridge_nf_call_iptables.rc == 0 + with_items: + - net.bridge.bridge-nf-call-iptables + - net.bridge.bridge-nf-call-arptables + - net.bridge.bridge-nf-call-ip6tables + +- name: 2Flannel | Install Multus CNI + shell: |- + /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \ + bash -c \ + "git clone https://github.com/Intel-Corp/multus-cni && \ + cd multus-cni && ./build && cp bin/multus /opt/cni/bin" + +- name: 2Flannel | Create cni-flannel-rbac manifest + template: + src: cni-2flannel-rbac.yml.j2 + dest: "{{ kube_config_dir }}/cni-2flannel-rbac.yml" + register: two_flannel_rbac_manifest + when: inventory_hostname == groups['kube-master'][0] and rbac_enabled + +- name: 2Flannel | Create cni-flannel manifest + template: + src: cni-2flannel.yml.j2 + dest: "{{ kube_config_dir }}/cni-2flannel.yml" + register: two_flannel_manifest + when: inventory_hostname == groups['kube-master'][0] + +- name: 2Flannel | Set 2Flannel etcd configuration + shell: |- + ETCDCTL_CA_FILE=/etc/ssl/etcd/ssl/ca.pem \ + ETCDCTL_CERT_FILE=/etc/ssl/etcd/ssl/node-{{ ansible_hostname }}.pem \ + ETCDCTL_KEY_FILE=/etc/ssl/etcd/ssl/node-{{ ansible_hostname }}-key.pem \ + {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \ + set /{{ cluster_name }}/{{ item.config }}/network/config \ + '{ "Network": "{{ item.network }}", "Backend": {"Type":"udp", "Port":{{ item.port}} }}' + with_items: + - {config: "2flannel.1", + network: "{{ two_flannel_network1 }}", + port: 8285} + - {config: "2flannel.2", + network: "{{ two_flannel_network2 }}", + port: 8286} + delegate_to: "{{groups['etcd'][0]}}" + run_once: true diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/pre-upgrade.yml b/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/pre-upgrade.yml new file mode 100644 index 00000000..a49002fe --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/pre-upgrade.yml @@ -0,0 +1,14 @@ +--- +- name: 2Flannel pre-upgrade | Purge legacy flannel systemd unit file + file: + path: "/etc/systemd/system/docker.service.d/flannel-options.conf" + state: absent + notify: + - 2Flannel | delete default docker bridge + +- name: 2Flannel pre-upgrade | Purge legacy Flannel static pod manifest + file: + path: "{{ kube_manifest_dir }}/flannel-pod.manifest" + state: absent + notify: + - 2Flannel | delete flannel interface diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel-rbac.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel-rbac.yml.j2 new file mode 100644 index 00000000..b4d1be11 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel-rbac.yml.j2 @@ -0,0 +1,44 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: 2flannel + namespace: "{{system_namespace}}" +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: 2flannel +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: 2flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 2flannel +subjects: +- kind: ServiceAccount + name: 2flannel + namespace: "{{system_namespace}}" diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel.yml.j2 new file mode 100644 index 00000000..9c159d37 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel.yml.j2 @@ -0,0 +1,211 @@ +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-2flannel-cfg1 + namespace: "{{system_namespace}}" + labels: + tier: node + app: 2flannel +data: + cni-conf.json: | + { + "name": "2flannel-networks", + "type": "multus", + "delegates": [ + { + "type": "flannel", + "name": "flannel1", + "subnetFile": "/run/2flannel/networks/subnet2.env", + "dataDir": "/var/lib/cni/flannel/2", + "delegate": { + "bridge": "kbr1", + "isDefaultGateway": false + } + }, + { + "type": "flannel", + "name": "flannel0", + "subnetFile": "/run/2flannel/networks/subnet1.env", + "dataDir": "/var/lib/cni/flannel/1", + "masterplugin": true, + "delegate": { + "bridge": "kbr0", + "isDefaultGateway": true + } + } + ] + } + net-conf.json: | + { + "Network": {{ two_flannel_network1 }}, + "Backend": { + "Type": "udp", + "Port": 8285 + } + } +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-2flannel-cfg2 + namespace: "{{system_namespace}}" + labels: + tier: node + app: 2flannel +data: + net-conf.json: | + { + "Network": {{ two_flannel_network2 }}, + "Backend": { + "Type": "udp", + "Port": 8286 + } + } +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-2flannel + namespace: "{{system_namespace}}" + labels: + tier: node + k8s-app: 2flannel +spec: + template: + metadata: + labels: + tier: node + k8s-app: 2flannel + spec: +{% if rbac_enabled %} + serviceAccountName: 2flannel +{% endif %} + containers: + - name: kube-2flannel-1 + image: {{ flannel_image_repo }}:{{ flannel_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + limits: + cpu: {{ flannel_cpu_limit }} + memory: {{ flannel_memory_limit }} + requests: + cpu: {{ flannel_cpu_requests }} + memory: {{ flannel_memory_requests }} + command: [ "/opt/bin/flanneld", "--ip-masq", + "-etcd-endpoints={{ etcd_access_addresses }}", + "-etcd-prefix=/{{ cluster_name }}/2flannel.1/network", + "-etcd-cafile=/etc/ssl/etcd/ssl/ca.pem", + "-etcd-certfile=/etc/ssl/etcd/ssl/node-$(NODE_NAME).pem", + "-etcd-keyfile=/etc/ssl/etcd/ssl/node-$(NODE_NAME)-key.pem", + "-subnet-file=/run/2flannel/networks/subnet1.env" ] + securityContext: + privileged: true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run + - name: cni + mountPath: /etc/cni/net.d + - name: ssl + mountPath: /etc/ssl/etcd/ssl/ + - name: 2flannel-cfg1 + mountPath: /etc/kube-flannel/ + - name: kube-2flannel-2 + image: {{ flannel_image_repo }}:{{ flannel_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + resources: + limits: + cpu: {{ flannel_cpu_limit }} + memory: {{ flannel_memory_limit }} + requests: + cpu: {{ flannel_cpu_requests }} + memory: {{ flannel_memory_requests }} + command: [ "/opt/bin/flanneld", "--ip-masq", + "-etcd-endpoints={{ etcd_access_addresses }}", + "-etcd-prefix=/{{ cluster_name }}/2flannel.2/network", + "-etcd-cafile=/etc/ssl/etcd/ssl/ca.pem", + "-etcd-certfile=/etc/ssl/etcd/ssl/node-$(NODE_NAME).pem", + "-etcd-keyfile=/etc/ssl/etcd/ssl/node-$(NODE_NAME)-key.pem", + "-subnet-file=/run/2flannel/networks/subnet2.env" ] + securityContext: + privileged: true + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run + - name: cni + mountPath: /etc/cni/net.d + - name: ssl + mountPath: /etc/ssl/etcd/ssl/ + - name: 2flannel-cfg2 + mountPath: /etc/kube-flannel/ + - name: install-cni + image: {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }} + command: ["/install-cni.sh"] + env: + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: kube-2flannel-cfg1 + key: cni-conf.json + - name: CNI_CONF_NAME + value: "10-multus-2flannel.conf" + volumeMounts: + - name: cni + mountPath: /host/etc/cni/net.d + - name: host-cni-bin + mountPath: /host/opt/cni/bin/ + hostNetwork: true + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + volumes: + - name: run + hostPath: + path: /run + - name: cni + hostPath: + path: /etc/cni/net.d + - name: ssl + hostPath: + path: /etc/ssl/etcd/ssl/ + - name: 2flannel-cfg1 + configMap: + name: kube-2flannel-cfg1 + - name: 2flannel-cfg2 + configMap: + name: kube-2flannel-cfg2 + - name: host-cni-bin + hostPath: + path: /opt/cni/bin + updateStrategy: + rollingUpdate: + maxUnavailable: {{ serial | default('20%') }} + type: RollingUpdate diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml index f68eec43..187e2a52 100644 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml @@ -57,7 +57,6 @@ - netaddr - jinja2 - - name: copy inventories generate script copy: src: generate_inventories.py @@ -156,6 +155,50 @@ - extra-vars-{{ ansible_architecture }}.yml - extra-vars.yml +- name: copy 2flannel playbook to kargo + copy: + src: "{{ run_dir }}/roles/2flannel" + dest: /opt/kargo_k8s/roles/network_plugin + +- name: copy 2flannel-apps playbook to kargo + copy: + src: "{{ run_dir }}/roles/2flannel-apps/" + dest: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/2flannel + +- name: append 2flannel to network plugin + blockinfile: + path: /opt/kargo_k8s/roles/network_plugin/meta/main.yml + block: " - role: network_plugin/2flannel\n \ + when: kube_network_plugin == '2flannel'\n tags: 2flannel\n" + +- name: append 2flannel apps to network plugin + blockinfile: + path: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/meta/main.yml + block: " - role: kubernetes-apps/network_plugin/2flannel\n \ + when: kube_network_plugin == '2flannel'\n tags: 2flannel\n" + +- name: append 2flannel to valid kube_network_plugin list + replace: + path: "/opt/kargo_k8s/roles/kubernetes/{{ item.path }}" + regexp: "{{ item.regexp }}" + replace: "{{ item.replace }}" + with_items: + - {path: 'master/templates/manifests/kube-controller-manager.manifest.j2', + regexp: '"cloud", "flannel"', + replace: '"cloud", "flannel", "2flannel"'} + - {path: 'node/templates/kubelet.kubeadm.env.j2', + regexp: '"calico", "canal", "flannel", "weave"', + replace: '"calico", "canal", "flannel", "weave", "2flannel"'} + - {path: 'node/templates/kubelet.standard.env.j2', + regexp: '"calico", "canal", "flannel", "weave"', + replace: '"calico", "canal", "flannel", "weave", "2flannel"'} + - {path: 'node/templates/kubelet.rkt.service.j2', + regexp: '"calico", "weave", "canal", "flannel"', + replace: '"calico", "weave", "canal", "flannel", "2flannel"'} + - {path: 'preinstall/tasks/main.yml', + regexp: '"calico", "weave", "canal", "flannel"', + replace: '"calico", "weave", "canal", "flannel", "2flannel"'} + - name: run kargo playbook shell: | cd /opt/kargo_k8s |