add a multus with 2 fannel interfaces installation

Support deploying multus multiple flannel CNI plugins by setting
environment "kube_network_plugin" to "2flannel".

Change-Id: I23a3d42452b4a5e0858a18934f508649f5961e7a
Signed-off-by: Di Xu <di.xu@arm.com>

8 files changed, 490 insertions, 1 deletions

diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel-apps/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/2flannel-apps/tasks/main.yml
new file mode 100644
index 00000000..a7ff10e2
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/2flannel-apps/tasks/main.yml
@@ -0,0 +1,22 @@
+---
+- name: "2Flannel | Create ServiceAccount ClusterRole and ClusterRoleBinding"
+  command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/cni-2flannel-rbac.yml"
+  run_once: true
+  when: rbac_enabled and two_flannel_rbac_manifest.changed
+
+- name: 2Flannel | Start Resources
+  kube:
+    name: "kube-2flannel"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/cni-2flannel.yml"
+    resource: "ds"
+    namespace: "{{system_namespace}}"
+    state: "{{ item | ternary('latest','present') }}"
+  with_items: "{{ two_flannel_manifest.changed }}"
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: 2Flannel | Wait for flannel subnet.env file presence
+  wait_for:
+    path: /run/2flannel/networks/subnet1.env
+    delay: 5
+    timeout: 600
diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/defaults/main.yml b/deploy/adapters/ansible/kubernetes/roles/2flannel/defaults/main.yml
new file mode 100644
index 00000000..f9e61eda
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/defaults/main.yml
@@ -0,0 +1,21 @@
+---
+# Flannel public IP
+# The address that flannel should advertise as how to access the system
+# Disabled until https://github.com/coreos/flannel/issues/712 is fixed
+# flannel_public_ip: "{{ access_ip|default(ip|default(ansible_default_ipv4.address)) }}"
+
+## interface that should be used for flannel operations
+## This is actually an inventory node-level item
+# flannel_interface:
+
+# You can choose what type of flannel backend to use
+# please refer to flannel's docs : https://github.com/coreos/flannel/blob/master/README.md
+
+# Limits for apps
+flannel_memory_limit: 500M
+flannel_cpu_limit: 300m
+flannel_memory_requests: 64M
+flannel_cpu_requests: 150m
+
+two_flannel_network1: "{{ kube_pods_subnet }}"
+two_flannel_network2: "10.235.64.0/18"
diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/handlers/main.yml b/deploy/adapters/ansible/kubernetes/roles/2flannel/handlers/main.yml
new file mode 100644
index 00000000..44ead00e
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/handlers/main.yml
@@ -0,0 +1,61 @@
+---
+- name: 2Flannel | delete default docker bridge
+  command: ip link delete docker0
+  failed_when: false
+  notify: 2Flannel | restart docker
+
+- name: 2Flannel | delete 2flannel interface
+  command: ip link delete flannel0 && ip link delete flannel1
+  failed_when: false
+
+# special cases for atomic because it defaults to live-restore: true
+# So we disable live-restore to pickup the new flannel IP.  After
+# we enable it, we have to restart docker again to pickup the new
+# setting and restore the original behavior
+- name: 2Flannel | restart docker
+  command: /bin/true
+  notify:
+    - 2Flannel | reload systemd
+    - 2Flannel | reload docker.socket
+    - 2Flannel | configure docker live-restore true (atomic)
+    - 2Flannel | reload docker
+    - 2Flannel | pause while Docker restarts
+    - 2Flannel | wait for docker
+
+- name: 2Flannel | reload systemd
+  shell: systemctl daemon-reload
+
+- name: 2Flannel | reload docker.socket
+  service:
+    name: docker.socket
+    state: restarted
+  when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+
+- name: 2Flannel | configure docker live-restore true (atomic)
+  replace:
+    name: /etc/docker/daemon.json
+    regexp: '"live-restore":.*true'
+    replace: '"live-restore": false'
+  when: is_atomic
+
+- name: 2Flannel | reload docker
+  service:
+    name: docker
+    state: restarted
+
+- name: 2Flannel | pause while Docker restarts
+  pause:
+    seconds: 10
+    prompt: "Waiting for docker restart"
+
+- name: 2Flannel | wait for docker
+  command: "{{ docker_bin_dir }}/docker images"
+  register: docker_ready
+  retries: 10
+  delay: 5
+  until: docker_ready.rc == 0
+
+- name: 2Flannel | reload kubelet
+  service:
+    name: kubelet
+    state: restarted
diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/main.yml
new file mode 100644
index 00000000..e7adeefe
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/main.yml
@@ -0,0 +1,73 @@
+---
+- include: pre-upgrade.yml
+
+- name: 2Flannel | Verify if br_netfilter module exists
+  shell: "modinfo br_netfilter"
+  register: modinfo_br_netfilter
+  failed_when: modinfo_br_netfilter.rc not in [0, 1]
+  changed_when: false
+
+- name: 2Flannel | Enable br_netfilter module
+  modprobe:
+    name: br_netfilter
+    state: present
+  when: modinfo_br_netfilter.rc == 0
+
+# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled when found
+# if br_netfilter is not a module
+- name: 2Flannel | Check if bridge-nf-call-iptables key exists
+  command: "sysctl net.bridge.bridge-nf-call-iptables"
+  failed_when: false
+  changed_when: false
+  register: sysctl_bridge_nf_call_iptables
+
+- name: 2Flannel | Enable bridge-nf-call tables
+  sysctl:
+    name: "{{ item }}"
+    state: present
+    value: 1
+    reload: "yes"
+  when: modinfo_br_netfilter.rc == 1 and sysctl_bridge_nf_call_iptables.rc == 0
+  with_items:
+    - net.bridge.bridge-nf-call-iptables
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+
+- name: 2Flannel | Install Multus CNI
+  shell: |-
+    /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \
+    bash -c \
+    "git clone https://github.com/Intel-Corp/multus-cni && \
+    cd multus-cni && ./build && cp bin/multus /opt/cni/bin"
+
+- name: 2Flannel | Create cni-flannel-rbac manifest
+  template:
+    src: cni-2flannel-rbac.yml.j2
+    dest: "{{ kube_config_dir }}/cni-2flannel-rbac.yml"
+  register: two_flannel_rbac_manifest
+  when: inventory_hostname == groups['kube-master'][0] and rbac_enabled
+
+- name: 2Flannel | Create cni-flannel manifest
+  template:
+    src: cni-2flannel.yml.j2
+    dest: "{{ kube_config_dir }}/cni-2flannel.yml"
+  register: two_flannel_manifest
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: 2Flannel | Set 2Flannel etcd configuration
+  shell: |-
+    ETCDCTL_CA_FILE=/etc/ssl/etcd/ssl/ca.pem \
+    ETCDCTL_CERT_FILE=/etc/ssl/etcd/ssl/node-{{ ansible_hostname }}.pem \
+    ETCDCTL_KEY_FILE=/etc/ssl/etcd/ssl/node-{{ ansible_hostname }}-key.pem \
+    {{ bin_dir }}/etcdctl --peers={{ etcd_access_addresses }} \
+    set /{{ cluster_name }}/{{ item.config }}/network/config \
+    '{ "Network": "{{ item.network }}", "Backend": {"Type":"udp", "Port":{{ item.port}} }}'
+  with_items:
+    - {config: "2flannel.1",
+       network: "{{ two_flannel_network1 }}",
+       port: 8285}
+    - {config: "2flannel.2",
+       network: "{{ two_flannel_network2 }}",
+       port: 8286}
+  delegate_to: "{{groups['etcd'][0]}}"
+  run_once: true
diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/pre-upgrade.yml b/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/pre-upgrade.yml
new file mode 100644
index 00000000..a49002fe
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/tasks/pre-upgrade.yml
@@ -0,0 +1,14 @@
+---
+- name: 2Flannel pre-upgrade | Purge legacy flannel systemd unit file
+  file:
+    path: "/etc/systemd/system/docker.service.d/flannel-options.conf"
+    state: absent
+  notify:
+    - 2Flannel | delete default docker bridge
+
+- name: 2Flannel pre-upgrade | Purge legacy Flannel static pod manifest
+  file:
+    path: "{{ kube_manifest_dir }}/flannel-pod.manifest"
+    state: absent
+  notify:
+    - 2Flannel | delete flannel interface
diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel-rbac.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel-rbac.yml.j2
new file mode 100644
index 00000000..b4d1be11
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel-rbac.yml.j2
@@ -0,0 +1,44 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: 2flannel
+  namespace: "{{system_namespace}}"
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: 2flannel
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: 2flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: 2flannel
+subjects:
+- kind: ServiceAccount
+  name: 2flannel
+  namespace: "{{system_namespace}}"
diff --git a/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel.yml.j2
new file mode 100644
index 00000000..9c159d37
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/2flannel/templates/cni-2flannel.yml.j2
@@ -0,0 +1,211 @@
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-2flannel-cfg1
+  namespace: "{{system_namespace}}"
+  labels:
+    tier: node
+    app: 2flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "2flannel-networks",
+      "type": "multus",
+      "delegates": [
+          {
+            "type": "flannel",
+            "name": "flannel1",
+            "subnetFile": "/run/2flannel/networks/subnet2.env",
+            "dataDir": "/var/lib/cni/flannel/2",
+            "delegate": {
+              "bridge": "kbr1",
+              "isDefaultGateway": false
+            }
+          },
+          {
+            "type": "flannel",
+            "name": "flannel0",
+            "subnetFile": "/run/2flannel/networks/subnet1.env",
+            "dataDir": "/var/lib/cni/flannel/1",
+            "masterplugin": true,
+            "delegate": {
+              "bridge": "kbr0",
+              "isDefaultGateway": true
+            }
+          }
+      ]
+    }
+  net-conf.json: |
+    {
+      "Network": {{ two_flannel_network1 }},
+      "Backend": {
+        "Type": "udp",
+        "Port": 8285
+      }
+    }
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-2flannel-cfg2
+  namespace: "{{system_namespace}}"
+  labels:
+    tier: node
+    app: 2flannel
+data:
+  net-conf.json: |
+    {
+      "Network": {{ two_flannel_network2 }},
+      "Backend": {
+        "Type": "udp",
+        "Port": 8286
+      }
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-2flannel
+  namespace: "{{system_namespace}}"
+  labels:
+    tier: node
+    k8s-app: 2flannel
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        k8s-app: 2flannel
+    spec:
+{% if rbac_enabled %}
+      serviceAccountName: 2flannel
+{% endif %}
+      containers:
+      - name: kube-2flannel-1
+        image: {{ flannel_image_repo }}:{{ flannel_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        resources:
+          limits:
+            cpu: {{ flannel_cpu_limit }}
+            memory: {{ flannel_memory_limit }}
+          requests:
+            cpu: {{ flannel_cpu_requests }}
+            memory: {{ flannel_memory_requests }}
+        command: [ "/opt/bin/flanneld", "--ip-masq",
+            "-etcd-endpoints={{ etcd_access_addresses }}",
+            "-etcd-prefix=/{{ cluster_name }}/2flannel.1/network",
+            "-etcd-cafile=/etc/ssl/etcd/ssl/ca.pem",
+            "-etcd-certfile=/etc/ssl/etcd/ssl/node-$(NODE_NAME).pem",
+            "-etcd-keyfile=/etc/ssl/etcd/ssl/node-$(NODE_NAME)-key.pem",
+            "-subnet-file=/run/2flannel/networks/subnet1.env" ]
+        securityContext:
+          privileged: true
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: run
+          mountPath: /run
+        - name: cni
+          mountPath: /etc/cni/net.d
+        - name: ssl
+          mountPath: /etc/ssl/etcd/ssl/
+        - name: 2flannel-cfg1
+          mountPath: /etc/kube-flannel/
+      - name: kube-2flannel-2
+        image: {{ flannel_image_repo }}:{{ flannel_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        resources:
+          limits:
+            cpu: {{ flannel_cpu_limit }}
+            memory: {{ flannel_memory_limit }}
+          requests:
+            cpu: {{ flannel_cpu_requests }}
+            memory: {{ flannel_memory_requests }}
+        command: [ "/opt/bin/flanneld", "--ip-masq",
+            "-etcd-endpoints={{ etcd_access_addresses }}",
+            "-etcd-prefix=/{{ cluster_name }}/2flannel.2/network",
+            "-etcd-cafile=/etc/ssl/etcd/ssl/ca.pem",
+            "-etcd-certfile=/etc/ssl/etcd/ssl/node-$(NODE_NAME).pem",
+            "-etcd-keyfile=/etc/ssl/etcd/ssl/node-$(NODE_NAME)-key.pem",
+            "-subnet-file=/run/2flannel/networks/subnet2.env" ]
+        securityContext:
+          privileged: true
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: run
+          mountPath: /run
+        - name: cni
+          mountPath: /etc/cni/net.d
+        - name: ssl
+          mountPath: /etc/ssl/etcd/ssl/
+        - name: 2flannel-cfg2
+          mountPath: /etc/kube-flannel/
+      - name: install-cni
+        image: {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }}
+        command: ["/install-cni.sh"]
+        env:
+        # The CNI network config to install on each node.
+        - name: CNI_NETWORK_CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: kube-2flannel-cfg1
+              key: cni-conf.json
+        - name: CNI_CONF_NAME
+          value: "10-multus-2flannel.conf"
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: host-cni-bin
+          mountPath: /host/opt/cni/bin/
+      hostNetwork: true
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      volumes:
+        - name: run
+          hostPath:
+            path: /run
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: ssl
+          hostPath:
+            path: /etc/ssl/etcd/ssl/
+        - name: 2flannel-cfg1
+          configMap:
+            name: kube-2flannel-cfg1
+        - name: 2flannel-cfg2
+          configMap:
+            name: kube-2flannel-cfg2
+        - name: host-cni-bin
+          hostPath:
+            path: /opt/cni/bin
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: {{ serial | default('20%') }}
+    type: RollingUpdate
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
index f68eec43..187e2a52 100644
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
@@ -57,7 +57,6 @@
     - netaddr
     - jinja2
 
-
 - name: copy inventories generate script
   copy:
     src: generate_inventories.py
@@ -156,6 +155,50 @@
     - extra-vars-{{ ansible_architecture }}.yml
     - extra-vars.yml
 
+- name: copy 2flannel playbook to kargo
+  copy:
+    src: "{{ run_dir }}/roles/2flannel"
+    dest: /opt/kargo_k8s/roles/network_plugin
+
+- name: copy 2flannel-apps playbook to kargo
+  copy:
+    src: "{{ run_dir }}/roles/2flannel-apps/"
+    dest: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/2flannel
+
+- name: append 2flannel to network plugin
+  blockinfile:
+    path: /opt/kargo_k8s/roles/network_plugin/meta/main.yml
+    block: "  - role: network_plugin/2flannel\n    \
+      when: kube_network_plugin == '2flannel'\n    tags: 2flannel\n"
+
+- name: append 2flannel apps to network plugin
+  blockinfile:
+    path: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/meta/main.yml
+    block: "  - role: kubernetes-apps/network_plugin/2flannel\n    \
+      when: kube_network_plugin == '2flannel'\n    tags: 2flannel\n"
+
+- name: append 2flannel to valid kube_network_plugin list
+  replace:
+    path: "/opt/kargo_k8s/roles/kubernetes/{{ item.path }}"
+    regexp: "{{ item.regexp }}"
+    replace: "{{ item.replace }}"
+  with_items:
+    - {path: 'master/templates/manifests/kube-controller-manager.manifest.j2',
+       regexp: '"cloud", "flannel"',
+       replace: '"cloud", "flannel", "2flannel"'}
+    - {path: 'node/templates/kubelet.kubeadm.env.j2',
+       regexp: '"calico", "canal", "flannel", "weave"',
+       replace: '"calico", "canal", "flannel", "weave", "2flannel"'}
+    - {path: 'node/templates/kubelet.standard.env.j2',
+       regexp: '"calico", "canal", "flannel", "weave"',
+       replace: '"calico", "canal", "flannel", "weave", "2flannel"'}
+    - {path: 'node/templates/kubelet.rkt.service.j2',
+       regexp: '"calico", "weave", "canal", "flannel"',
+       replace: '"calico", "weave", "canal", "flannel", "2flannel"'}
+    - {path: 'preinstall/tasks/main.yml',
+       regexp: '"calico", "weave", "canal", "flannel"',
+       replace: '"calico", "weave", "canal", "flannel", "2flannel"'}
+
 - name: run kargo playbook
   shell: |
     cd /opt/kargo_k8s