From 0ab7ff90c3fec546275fd35ffc13cb787c03721e Mon Sep 17 00:00:00 2001
From: Pawan Verma <pawanjbs5@gmail.com>
Date: Mon, 4 Oct 2021 18:44:20 +0530
Subject: Add support for Calico, Cilium, Contiv-VPP and Danm in k8scluster
 deployment scripts.

This patch adds support for installing Calico, Cilium, Contiv-VPP and
Danm in Kubernetes cluster deployment ansible scripts.

Signed-off-by: Pawan Verma <pawanjbs5@gmail.com>
Change-Id: Ib76620fa0f63dd58e8496bbf31baf515f697bcde
---
 .../roles/clustermanager/defaults/main.yml         |   4 +-
 .../clustermanager/files/danm-cni-plugins.yaml     |  36 ++
 .../files/danm-netwatcher-daemonset.yaml           |  94 ++++
 .../files/danm-webhook-create-signed-cert.sh       | 121 +++++
 .../files/kube-flannel-daemonset.yml               | 486 +++------------------
 .../roles/clustermanager/tasks/clear-calico.yaml   |  10 +
 .../roles/clustermanager/tasks/clear-cilium.yaml   |   7 +
 .../roles/clustermanager/tasks/clear-contiv.yaml   |  11 +
 .../roles/clustermanager/tasks/clear-danm.yaml     |  63 +++
 .../roles/clustermanager/tasks/cni-pre-deploy.yml  |  12 +
 .../roles/clustermanager/tasks/deploy-calico.yaml  |  12 +
 .../roles/clustermanager/tasks/deploy-cilium.yaml  |  19 +
 .../roles/clustermanager/tasks/deploy-contiv.yaml  |  12 +
 .../roles/clustermanager/tasks/deploy-danm.yaml    | 125 ++++++
 .../k8scluster/roles/clustermanager/tasks/main.yml |  42 +-
 .../clustermanager/templates/danm-webhook.yaml     | 128 ++++++
 16 files changed, 747 insertions(+), 435 deletions(-)
 create mode 100644 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-cni-plugins.yaml
 create mode 100644 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-netwatcher-daemonset.yaml
 create mode 100755 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-webhook-create-signed-cert.sh
 create mode 100644 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-calico.yaml
 create mode 100644 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-cilium.yaml
 create mode 100644 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-contiv.yaml
 create mode 100644 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-danm.yaml
 create mode 100644 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-calico.yaml
 create mode 100644 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-cilium.yaml
 create mode 100644 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-contiv.yaml
 create mode 100644 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-danm.yaml
 create mode 100644 tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/templates/danm-webhook.yaml

(limited to 'tools/k8s/cluster-deployment/k8scluster/roles/clustermanager')

diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/defaults/main.yml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/defaults/main.yml
index 15f1f186..ab2ffc36 100644
--- a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/defaults/main.yml
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/defaults/main.yml
@@ -25,4 +25,6 @@ token_file: $HOME/log_init.txt
 
 
 
-PIP_executable_version: pip3.6
\ No newline at end of file
+PIP_executable_version: pip3.6
+helm_version: v3.7.0
+openshift_version: 0.11.1
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-cni-plugins.yaml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-cni-plugins.yaml
new file mode 100644
index 00000000..1fe77cd6
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-cni-plugins.yaml
@@ -0,0 +1,36 @@
+#
+# cloned from https://github.com/nokia/danm/blob/v4.3.0/integration/manifests/cni_plugins/cni_plugins_ds.yaml
+#
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: danm-cni
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      danm.k8s.io: danm-cni
+  template:
+    metadata:
+      labels:
+        danm.k8s.io: danm-cni
+    spec:
+      containers:
+        - name: danm-cni
+          image: danm-cni-plugins
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: host-cni
+              mountPath: /host/cni
+            - name: host-net-d
+              mountPath: /host/net.d
+      hostNetwork: true
+      terminationGracePeriodSeconds: 0
+      volumes:
+        - name: host-cni
+          hostPath:
+            path: /opt/cni/bin
+        - name: host-net-d
+          hostPath:
+            path: /etc/cni/net.d
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-netwatcher-daemonset.yaml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-netwatcher-daemonset.yaml
new file mode 100644
index 00000000..1b61a04a
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-netwatcher-daemonset.yaml
@@ -0,0 +1,94 @@
+#
+# cloned from https://github.com/nokia/danm/tree/v4.3.0/integration/manifests/netwatcher
+#
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: netwatcher
+  namespace: kube-system
+  labels:
+      kubernetes.io/cluster-service: "true"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:netwatcher
+rules:
+rules:
+- apiGroups:
+  - danm.k8s.io
+  resources:
+  - danmnets
+  - clusternetworks
+  - tenantnetworks
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - k8s.cni.cncf.io
+  resources:
+  - network-attachment-definitions
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    kubernetes.io/bootstrapping: rbac-defaults
+  name: system:netwatcher
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:netwatcher
+subjects:
+- kind: ServiceAccount
+  namespace: kube-system
+  name: netwatcher
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: netwatcher
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      danm.k8s.io: netwatcher
+  template:
+    metadata:
+      labels:
+        danm.k8s.io: netwatcher
+    spec:
+      serviceAccountName: netwatcher
+      hostNetwork: true
+      dnsPolicy: ClusterFirst
+      hostIPC: true
+      hostPID: true
+      containers:
+        - name: netwatcher
+          image: netwatcher
+          imagePullPolicy: IfNotPresent
+          securityContext:
+            capabilities:
+              add:
+                - SYS_PTRACE
+                - SYS_ADMIN
+                - NET_ADMIN
+                - NET_RAW
+      tolerations:
+       - effect: NoSchedule
+         operator: Exists
+       - effect: NoExecute
+         operator: Exists
+      terminationGracePeriodSeconds: 0
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-webhook-create-signed-cert.sh b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-webhook-create-signed-cert.sh
new file mode 100755
index 00000000..d1486f62
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/danm-webhook-create-signed-cert.sh
@@ -0,0 +1,121 @@
+#!/bin/sh
+
+set -e
+
+usage() {
+    cat <<EOF
+Generate certificate suitable for use with an sidecar-injector webhook service.
+This script uses k8s' CertificateSigningRequest API to a generate a
+certificate signed by k8s CA suitable for use with sidecar-injector webhook
+services. This requires permissions to create and approve CSR. See
+https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster for
+detailed explantion and additional instructions.
+The server key/cert k8s CA cert are stored in a k8s secret.
+usage: ${0} [OPTIONS]
+The following flags are required.
+       --service          Service name of webhook.
+       --namespace        Namespace where webhook service and secret reside.
+       --secret           Secret name for CA certificate and server certificate/key pair.
+EOF
+    exit 1
+}
+
+while [ $# -gt 0 ]; do
+    case ${1} in
+        --service)
+            service="$2"
+            shift
+            ;;
+        --secret)
+            secret="$2"
+            shift
+            ;;
+        --namespace)
+            namespace="$2"
+            shift
+            ;;
+        *)
+            usage
+            ;;
+    esac
+    shift
+done
+
+[ -z ${service} ] && service=danm-webhook-svc
+[ -z ${secret} ] && secret=danm-webhook-certs
+[ -z ${namespace} ] && namespace=kube-system
+
+if [ ! -x "$(command -v openssl)" ]; then
+    echo "openssl not found"
+    exit 1
+fi
+
+csrName=${service}.${namespace}
+tmpdir=$(mktemp -d)
+echo "creating certs in tmpdir ${tmpdir} "
+
+cat <<EOF >> ${tmpdir}/csr.conf
+[req]
+req_extensions = v3_req
+distinguished_name = req_distinguished_name
+[req_distinguished_name]
+[ v3_req ]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage = serverAuth
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = ${service}
+DNS.2 = ${service}.${namespace}
+DNS.3 = ${service}.${namespace}.svc
+EOF
+
+openssl genrsa -out ${tmpdir}/server-key.pem 2048
+openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf
+
+# clean-up any previously created CSR for our service. Ignore errors if not present.
+kubectl delete csr ${csrName} 2>/dev/null || true
+
+# create  server cert/key CSR and  send to k8s API
+cat <<EOF | kubectl create -f -
+apiVersion: certificates.k8s.io/v1beta1
+kind: CertificateSigningRequest
+metadata:
+  name: ${csrName}
+spec:
+  groups:
+  - system:authenticated
+  request: $(cat ${tmpdir}/server.csr | base64 | tr -d '\n')
+  usages:
+  - digital signature
+  - key encipherment
+  - server auth
+EOF
+
+# verify CSR has been created
+while true; do
+    kubectl get csr ${csrName}
+    if [ "$?" -eq 0 ]; then
+        break
+    fi
+done
+
+# approve and fetch the signed certificate
+kubectl certificate approve ${csrName}
+# verify certificate has been signed
+for x in $(seq 10); do
+    serverCert=$(kubectl get csr ${csrName} -o jsonpath='{.status.certificate}')
+    if [ -n ${serverCert} ]; then
+        break
+    fi
+    sleep 1
+done
+echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem
+
+
+# create the secret with CA cert and server cert/key
+kubectl create secret generic ${secret} \
+        --from-file=key.pem=${tmpdir}/server-key.pem \
+        --from-file=cert.pem=${tmpdir}/server-cert.pem \
+        --dry-run -o yaml |
+    kubectl -n ${namespace} apply -f -
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/kube-flannel-daemonset.yml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/kube-flannel-daemonset.yml
index 00110ad6..1233ead4 100644
--- a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/kube-flannel-daemonset.yml
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/files/kube-flannel-daemonset.yml
@@ -1,5 +1,5 @@
 #
-# cloned from https://github.com/coreos/flannel/blob/v0.12.0/Documentation/kube-flannel.yml
+# cloned from https://github.com/flannel-io/flannel/blob/v0.14.0/Documentation/kube-flannel.yml
 #
 ---
 apiVersion: policy/v1beta1
@@ -14,14 +14,14 @@ metadata:
 spec:
   privileged: false
   volumes:
-    - configMap
-    - secret
-    - emptyDir
-    - hostPath
+  - configMap
+  - secret
+  - emptyDir
+  - hostPath
   allowedHostPaths:
-    - pathPrefix: "/etc/cni/net.d"
-    - pathPrefix: "/etc/kube-flannel"
-    - pathPrefix: "/run/flannel"
+  - pathPrefix: "/etc/cni/net.d"
+  - pathPrefix: "/etc/kube-flannel"
+  - pathPrefix: "/run/flannel"
   readOnlyRootFilesystem: false
   # Users and groups
   runAsUser:
@@ -34,7 +34,7 @@ spec:
   allowPrivilegeEscalation: false
   defaultAllowPrivilegeEscalation: false
   # Capabilities
-  allowedCapabilities: ['NET_ADMIN']
+  allowedCapabilities: ['NET_ADMIN', 'NET_RAW']
   defaultAddCapabilities: []
   requiredDropCapabilities: []
   # Host namespaces
@@ -50,36 +50,36 @@ spec:
     rule: 'RunAsAny'
 ---
 kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: flannel
 rules:
-  - apiGroups: ['extensions']
-    resources: ['podsecuritypolicies']
-    verbs: ['use']
-    resourceNames: ['psp.flannel.unprivileged']
-  - apiGroups:
-      - ""
-    resources:
-      - pods
-    verbs:
-      - get
-  - apiGroups:
-      - ""
-    resources:
-      - nodes
-    verbs:
-      - list
-      - watch
-  - apiGroups:
-      - ""
-    resources:
-      - nodes/status
-    verbs:
-      - patch
+- apiGroups: ['extensions']
+  resources: ['podsecuritypolicies']
+  verbs: ['use']
+  resourceNames: ['psp.flannel.unprivileged']
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes/status
+  verbs:
+  - patch
 ---
 kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: flannel
 roleRef:
@@ -137,7 +137,7 @@ data:
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: kube-flannel-ds-amd64
+  name: kube-flannel-ds
   namespace: kube-system
   labels:
     tier: node
@@ -156,23 +156,20 @@ spec:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
-              - matchExpressions:
-                  - key: beta.kubernetes.io/os
-                    operator: In
-                    values:
-                      - linux
-                  - key: beta.kubernetes.io/arch
-                    operator: In
-                    values:
-                      - amd64
+            - matchExpressions:
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
       hostNetwork: true
+      priorityClassName: system-node-critical
       tolerations:
       - operator: Exists
         effect: NoSchedule
       serviceAccountName: flannel
       initContainers:
       - name: install-cni
-        image: quay.io/coreos/flannel:v0.12.0-amd64
+        image: quay.io/coreos/flannel:v0.14.0
         command:
         - cp
         args:
@@ -186,7 +183,7 @@ spec:
           mountPath: /etc/kube-flannel/
       containers:
       - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.12.0-amd64
+        image: quay.io/coreos/flannel:v0.14.0
         command:
         - /opt/bin/flanneld
         args:
@@ -202,7 +199,7 @@ spec:
         securityContext:
           privileged: false
           capabilities:
-            add: ["NET_ADMIN"]
+            add: ["NET_ADMIN", "NET_RAW"]
         env:
         - name: POD_NAME
           valueFrom:
@@ -218,389 +215,12 @@ spec:
         - name: flannel-cfg
           mountPath: /etc/kube-flannel/
       volumes:
-        - name: run
-          hostPath:
-            path: /run/flannel
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-flannel-ds-arm64
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel
-spec:
-  selector:
-    matchLabels:
-      app: flannel
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: beta.kubernetes.io/os
-                    operator: In
-                    values:
-                      - linux
-                  - key: beta.kubernetes.io/arch
-                    operator: In
-                    values:
-                      - arm64
-      hostNetwork: true
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.12.0-arm64
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/cni-conf.json
-        - /etc/cni/net.d/10-flannel.conflist
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.12.0-arm64
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --kube-subnet-mgr
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: false
-          capabilities:
-             add: ["NET_ADMIN"]
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run/flannel
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run/flannel
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-flannel-ds-arm
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel
-spec:
-  selector:
-    matchLabels:
-      app: flannel
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: beta.kubernetes.io/os
-                    operator: In
-                    values:
-                      - linux
-                  - key: beta.kubernetes.io/arch
-                    operator: In
-                    values:
-                      - arm
-      hostNetwork: true
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.12.0-arm
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/cni-conf.json
-        - /etc/cni/net.d/10-flannel.conflist
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.12.0-arm
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --kube-subnet-mgr
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: false
-          capabilities:
-             add: ["NET_ADMIN"]
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run/flannel
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run/flannel
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-flannel-ds-ppc64le
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel
-spec:
-  selector:
-    matchLabels:
-      app: flannel
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: beta.kubernetes.io/os
-                    operator: In
-                    values:
-                      - linux
-                  - key: beta.kubernetes.io/arch
-                    operator: In
-                    values:
-                      - ppc64le
-      hostNetwork: true
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.12.0-ppc64le
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/cni-conf.json
-        - /etc/cni/net.d/10-flannel.conflist
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.12.0-ppc64le
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --kube-subnet-mgr
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: false
-          capabilities:
-             add: ["NET_ADMIN"]
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run/flannel
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run/flannel
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: kube-flannel-ds-s390x
-  namespace: kube-system
-  labels:
-    tier: node
-    app: flannel
-spec:
-  selector:
-    matchLabels:
-      app: flannel
-  template:
-    metadata:
-      labels:
-        tier: node
-        app: flannel
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: beta.kubernetes.io/os
-                    operator: In
-                    values:
-                      - linux
-                  - key: beta.kubernetes.io/arch
-                    operator: In
-                    values:
-                      - s390x
-      hostNetwork: true
-      tolerations:
-      - operator: Exists
-        effect: NoSchedule
-      serviceAccountName: flannel
-      initContainers:
-      - name: install-cni
-        image: quay.io/coreos/flannel:v0.12.0-s390x
-        command:
-        - cp
-        args:
-        - -f
-        - /etc/kube-flannel/cni-conf.json
-        - /etc/cni/net.d/10-flannel.conflist
-        volumeMounts:
-        - name: cni
-          mountPath: /etc/cni/net.d
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      containers:
-      - name: kube-flannel
-        image: quay.io/coreos/flannel:v0.12.0-s390x
-        command:
-        - /opt/bin/flanneld
-        args:
-        - --ip-masq
-        - --kube-subnet-mgr
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "50Mi"
-          limits:
-            cpu: "100m"
-            memory: "50Mi"
-        securityContext:
-          privileged: false
-          capabilities:
-             add: ["NET_ADMIN"]
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-        volumeMounts:
-        - name: run
-          mountPath: /run/flannel
-        - name: flannel-cfg
-          mountPath: /etc/kube-flannel/
-      volumes:
-        - name: run
-          hostPath:
-            path: /run/flannel
-        - name: cni
-          hostPath:
-            path: /etc/cni/net.d
-        - name: flannel-cfg
-          configMap:
-            name: kube-flannel-cfg
-
+      - name: run
+        hostPath:
+          path: /run/flannel
+      - name: cni
+        hostPath:
+          path: /etc/cni/net.d
+      - name: flannel-cfg
+        configMap:
+          name: kube-flannel-cfg
\ No newline at end of file
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-calico.yaml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-calico.yaml
new file mode 100644
index 00000000..9e6f3fa6
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-calico.yaml
@@ -0,0 +1,10 @@
+---
+
+- name: Delete Calico
+  k8s:
+    state: absent
+    definition: '{{ item }}'
+  with_items: '{{ lookup("url", "https://docs.projectcalico.org/manifests/calico.yaml", split_lines=False) | from_yaml_all | list }}'
+  when: item is not none
+
+
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-cilium.yaml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-cilium.yaml
new file mode 100644
index 00000000..16b6e8c9
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-cilium.yaml
@@ -0,0 +1,7 @@
+---
+
+- name: Delete cilium
+  kubernetes.core.helm:
+   name: cilium
+   namespace: kube-system
+   state: absent
\ No newline at end of file
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-contiv.yaml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-contiv.yaml
new file mode 100644
index 00000000..553ac937
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-contiv.yaml
@@ -0,0 +1,11 @@
+---
+
+- name: Delete Contiv
+  k8s:
+    state: absent
+    definition: '{{ item }}'
+  with_items: '{{ lookup("url", "https://raw.githubusercontent.com/contiv/vpp/v3.4.2/k8s/contiv-vpp.yaml", split_lines=False) | from_yaml_all | list }}'
+  when: item is not none
+
+
+
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-danm.yaml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-danm.yaml
new file mode 100644
index 00000000..1f989805
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/clear-danm.yaml
@@ -0,0 +1,63 @@
+---
+
+- name: Delete DanmNet CRD's
+  k8s:
+    state: absent
+    definition: '{{ item }}'
+  with_items: '{{ lookup("url", "https://raw.githubusercontent.com/nokia/danm/v4.3.0/integration/crds/lightweight/DanmNet.yaml", split_lines=False) | from_yaml_all | list }}'
+  when: item is not none
+
+- name: Delete DanmEp CRD's
+  k8s:
+    state: absent
+    definition: '{{ item }}'
+  with_items: '{{ lookup("url", "https://raw.githubusercontent.com/nokia/danm/v4.3.0/integration/crds/lightweight/DanmEp.yaml", split_lines=False) | from_yaml_all | list }}'
+  when: item is not none
+
+
+- name: Delete Danm service account
+  k8s:
+    state: absent
+    kind: ServiceAccount
+    name: danm
+    namespace: kube-system
+
+- name: Delete Danm cni conf
+  command: sudo rm -f  /etc/cni/net.d/00-danm.conf
+
+- name: Delete Danm cni kubeconfig
+  command: sudo rm -f /etc/cni/net.d/danm-kubeconfig
+
+- name: Delete Danm rbac
+  k8s:
+    state: absent
+    definition: '{{ item }}'
+  with_items: '{{ lookup("url", "https://raw.githubusercontent.com/nokia/danm/v4.3.0/integration/cni_config/danm_rbac.yaml", split_lines=False) | from_yaml_all | list }}'
+  when: item is not none
+
+- name: Delete Danm cni plugins
+  k8s:
+    state: absent
+    definition: "{{ lookup('file', 'danm-cni-plugins.yaml') }}"
+
+- name: Delete Danm netwatcher
+  k8s:
+    state: absent
+    definition: "{{ lookup('file', 'danm-netwatcher-daemonset.yaml') }}"
+
+- name: Get CA Bundle
+  shell: kubectl config view --raw -o json | jq -r '.clusters[0].cluster."certificate-authority-data"' | tr -d '"'
+  register: danm_ca_bundle
+
+- name: Generate webhook deployment
+  template:
+    src: danm-webhook.yaml
+    dest: /tmp/danm-webhook.yaml
+    mode: 0644
+  vars:
+    ca_bundle: "{{ danm_ca_bundle.stdout }}"
+
+- name: Delete Danm webhook
+  k8s:
+    state: absent
+    src: /tmp/danm-webhook.yaml
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/cni-pre-deploy.yml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/cni-pre-deploy.yml
index b2f280ef..8e30c6e5 100644
--- a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/cni-pre-deploy.yml
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/cni-pre-deploy.yml
@@ -3,9 +3,21 @@
   pip:
     name: openshift
     executable: "{{ PIP_executable_version }}"
+    version: "{{ openshift_version }}"
   when: inventory_hostname in groups['master']
   become: yes
 
+- name: Install Helm
+  unarchive:
+    src: "https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz"
+    dest: "/tmp"
+    remote_src: yes
+  become: yes
+
+- name: Move helm to PATH
+  command: mv /tmp/linux-amd64/helm /usr/local/bin/helm
+  become: yes
+
 - name: Check whether /etc/cni/net.d/ exists
   stat:
     path: /etc/cni/net.d
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-calico.yaml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-calico.yaml
new file mode 100644
index 00000000..afee929a
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-calico.yaml
@@ -0,0 +1,12 @@
+---
+
+- name: Clean Calico
+  import_tasks: clear-calico.yaml
+
+- name: Deploy Calico
+  k8s:
+    state: present
+    apply: yes
+    definition: '{{ item }}'
+  with_items: '{{ lookup("url", "https://docs.projectcalico.org/manifests/calico.yaml", split_lines=False) | from_yaml_all | list }}'
+  when: item is not none
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-cilium.yaml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-cilium.yaml
new file mode 100644
index 00000000..fb48591e
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-cilium.yaml
@@ -0,0 +1,19 @@
+---
+
+- name: Add cilium helm repo
+  kubernetes.core.helm_repository:
+    name: cilium
+    repo_url: "https://helm.cilium.io/"
+
+- name: Clean Cilium
+  import_tasks: clear-cilium.yaml
+
+- name: Deploy cilium Chart
+  kubernetes.core.helm:
+   name: cilium
+   namespace: kube-system
+   chart_ref: cilium/cilium
+   wait: yes
+   values:
+    cni:
+      exclusive: false
\ No newline at end of file
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-contiv.yaml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-contiv.yaml
new file mode 100644
index 00000000..f08ad702
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-contiv.yaml
@@ -0,0 +1,12 @@
+---
+
+- name: Clean Contiv
+  import_tasks: clear-contiv.yaml
+
+- name: Deploy Contiv
+  k8s:
+    state: present
+    apply: yes
+    definition: '{{ item }}'
+  with_items: '{{ lookup("url", "https://raw.githubusercontent.com/contiv/vpp/v3.4.2/k8s/contiv-vpp.yaml", split_lines=False) | from_yaml_all | list }}'
+  when: item is not none
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-danm.yaml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-danm.yaml
new file mode 100644
index 00000000..04852e55
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/deploy-danm.yaml
@@ -0,0 +1,125 @@
+---
+
+- name: Clean Danm
+  import_tasks: clear-danm.yaml
+
+- name: Deploy DanmNet CRD
+  k8s:
+    state: present
+    apply: yes
+    definition: '{{ item }}'
+  with_items: '{{ lookup("url", "https://raw.githubusercontent.com/nokia/danm/v4.3.0/integration/crds/lightweight/DanmNet.yaml", split_lines=False) | from_yaml_all | list }}'
+  when: item is not none
+
+- name: Deploy DanmEp CRD
+  k8s:
+    state: present
+    apply: yes
+    definition: '{{ item }}'
+  with_items: '{{ lookup("url", "https://raw.githubusercontent.com/nokia/danm/v4.3.0/integration/crds/lightweight/DanmEp.yaml", split_lines=False) | from_yaml_all | list }}'
+  when: item is not none
+
+- name: Create Danm service account
+  command: kubectl create --namespace kube-system serviceaccount danm
+
+- name: Create Danm cni conf
+  copy:
+    dest: /etc/cni/net.d/00-danm.conf
+    mode: 0644
+    content: |
+      {
+        "cniVersion": "0.3.1",
+        "name": "meta_cni",
+        "type": "danm",
+        "kubeconfig": "/etc/cni/net.d/danm-kubeconfig",
+        "cniDir": "/etc/cni/net.d",
+        "namingScheme": "awesome",
+      }
+  become: yes
+
+- name: Get Cluster name
+  command: kubectl config view -o jsonpath='{.clusters[0].name}'
+  register: cluster_name
+
+- name: Get Cluster Server
+  command: kubectl config view -o jsonpath='{.clusters[0].cluster.server}'
+  register: cluster_server
+
+- name: Get Cluster CA certification
+  command: kubectl config view --flatten -o jsonpath='{.clusters[0].cluster.certificate-authority-data}'
+  register: cluster_ca_certificate
+
+- name: Get Danm Secret Name
+  command: kubectl get --namespace kube-system -o jsonpath='{.secrets[0].name}' serviceaccounts danm
+  register: danm_secret_name
+
+- name: Get Danm Service Account Token
+  shell: kubectl get --namespace kube-system secrets {{ danm_secret_name.stdout }} -o jsonpath='{.data.token}' | base64 -d
+  register: danm_service_account_token
+
+- name: Create Danm kubeconfig
+  copy:
+    dest: /etc/cni/net.d/danm-kubeconfig
+    mode: 0644
+    content: |
+      apiVersion: v1
+      kind: Config
+      current-context: default
+      clusters:
+      - cluster:
+          certificate-authority-data: {{ cluster_ca_certificate.stdout }}
+          server: {{ cluster_server.stdout }}
+        name: {{ cluster_name.stdout }}
+      contexts:
+      - context:
+          cluster: {{ cluster_name.stdout }}
+          user: danm
+        name: default
+      users:
+      - name: danm
+        user:
+          token: {{ danm_service_account_token.stdout }}
+      preferences: {}
+  become: yes
+
+- name: Deploy Danm rbac
+  k8s:
+    state: present
+    apply: yes
+    definition: '{{ item }}'
+  with_items: '{{ lookup("url", "https://raw.githubusercontent.com/nokia/danm/v4.3.0/integration/cni_config/danm_rbac.yaml", split_lines=False) | from_yaml_all | list }}'
+  when: item is not none
+  
+- name: Deploy Danm cni plugins
+  k8s:
+    state: present
+    apply: yes
+    wait: yes
+    definition: "{{ lookup('file', 'danm-cni-plugins.yaml') }}"
+
+- name: Deploy Danm netwatcher
+  k8s:
+    state: present
+    apply: yes
+    definition: "{{ lookup('file', 'danm-netwatcher-daemonset.yaml') }}"
+
+- name: Create Danm webhook signed cert
+  script: danm-webhook-create-signed-cert.sh
+
+- name: Get CA Bundle
+  shell: kubectl config view --raw -o json | jq -r '.clusters[0].cluster."certificate-authority-data"' | tr -d '"'
+  register: danm_ca_bundle
+
+- name: Generate webhook deployment
+  template:
+    src: danm-webhook.yaml
+    dest: /tmp/danm-webhook.yaml
+    mode: 0644
+  vars:
+    ca_bundle: "{{ danm_ca_bundle.stdout }}"
+
+- name: Deploy Danm webhook
+  k8s:
+    state: present
+    apply: yes
+    src: /tmp/danm-webhook.yaml
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/main.yml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/main.yml
index 28c3f501..519cd36a 100644
--- a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/main.yml
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/tasks/main.yml
@@ -29,7 +29,7 @@
 
 - name: deploy multus
   import_tasks: deploy-multus.yml
-  when: inventory_hostname in groups['master']
+  when: inventory_hostname in groups['master'] and 'danm' not in ansible_run_tags
   tags: deploy, cni
 
 - name: clear multus
@@ -67,6 +67,46 @@
   when: inventory_hostname in groups['master']
   tags: clear
 
+- name: deploy calico
+  import_tasks: deploy-calico.yaml
+  when: inventory_hostname in groups['master']
+  tags: deploy, cni
+
+- name: clear calico
+  import_tasks: clear-calico.yaml
+  when: inventory_hostname in groups['master']
+  tags: clear
+
+- name: deploy cilium
+  import_tasks: deploy-cilium.yaml
+  when: inventory_hostname in groups['master']
+  tags: deploy, cni
+
+- name: clear cilium
+  import_tasks: clear-cilium.yaml
+  when: inventory_hostname in groups['master']
+  tags: clear
+
+- name: deploy contiv
+  import_tasks: deploy-contiv.yaml
+  when: inventory_hostname in groups['master']
+  tags: deploy, cni
+
+- name: clear contiv
+  import_tasks: clear-contiv.yaml
+  when: inventory_hostname in groups['master']
+  tags: clear
+
+- name: deploy danm
+  import_tasks: deploy-danm.yaml
+  when: inventory_hostname in groups['master'] and 'danm' in ansible_run_tags
+  tags: deploy, cni, danm
+
+- name: clear danm
+  import_tasks: clear-danm.yaml
+  when: inventory_hostname in groups['master'] and 'danm' in ansible_run_tags
+  tags: clear
+
 - name: drain and delete workers from master
   import_tasks: clear-k8s-workers-drain.yml
   when: inventory_hostname in groups['workers']
diff --git a/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/templates/danm-webhook.yaml b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/templates/danm-webhook.yaml
new file mode 100644
index 00000000..1e5d66ee
--- /dev/null
+++ b/tools/k8s/cluster-deployment/k8scluster/roles/clustermanager/templates/danm-webhook.yaml
@@ -0,0 +1,128 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: danm-webhook
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: caas:danm-webhook
+rules:
+- apiGroups:
+  - danm.k8s.io
+  resources:
+  - tenantconfigs
+  - danmeps
+  verbs: [ "*" ]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: caas:danm-webhook
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: caas:danm-webhook
+subjects:
+- kind: ServiceAccount
+  name: danm-webhook
+  namespace: kube-system
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: danm-webhook-config
+  namespace: kube-system
+webhooks:
+  - name: danm-netvalidation.nokia.k8s.io
+    clientConfig:
+      service:
+        name: danm-webhook-svc
+        namespace: kube-system
+        path: "/netvalidation"
+      caBundle: {{ ca_bundle  }}
+    rules:
+      # UPDATE IS TEMPORARILY REMOVED DUE TO:https://github.com/nokia/danm/issues/144
+      - operations: ["CREATE"]
+        apiGroups: ["danm.k8s.io"]
+        apiVersions: ["v1"]
+        resources: ["danmnets","clusternetworks","tenantnetworks"]
+    failurePolicy: Fail
+    timeoutSeconds: 25
+  - name: danm-configvalidation.nokia.k8s.io
+    clientConfig:
+      service:
+        name: danm-webhook-svc
+        namespace: kube-system
+        path: "/confvalidation"
+      caBundle: {{ ca_bundle }}
+    rules:
+      - operations: ["CREATE","UPDATE"]
+        apiGroups: ["danm.k8s.io"]
+        apiVersions: ["v1"]
+        resources: ["tenantconfigs"]
+    failurePolicy: Fail
+    timeoutSeconds: 25
+  - name: danm-netdeletion.nokia.k8s.io
+    clientConfig:
+      service:
+        name: danm-webhook-svc
+        namespace: kube-system
+        path: "/netdeletion"
+      caBundle: {{ ca_bundle }}
+    rules:
+      - operations: ["DELETE"]
+        apiGroups: ["danm.k8s.io"]
+        apiVersions: ["v1"]
+        resources: ["danmnets","clusternetworks","tenantnetworks"]
+    failurePolicy: Fail
+    timeoutSeconds: 25
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: danm-webhook-svc
+  namespace: kube-system
+  labels:
+    danm: webhook
+spec:
+  ports:
+  - name: webhook
+    port: 443
+    targetPort: 8443
+  selector:
+    danm: webhook
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: danm-webhook-deployment
+  namespace: kube-system
+  labels:
+    danm: webhook
+spec:
+  selector:
+    matchLabels:
+     danm: webhook
+  template:
+    metadata:
+      name: danm-webhook
+      labels:
+        danm: webhook
+    spec:
+      serviceAccountName: danm-webhook
+      containers:
+        - name: danm-webhook
+          image: webhook
+          command: [ "/usr/local/bin/webhook", "-tls-cert-bundle=/etc/webhook/certs/cert.pem", "-tls-private-key-file=/etc/webhook/certs/key.pem", "bind-port=8443" ]
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: webhook-certs
+              mountPath: /etc/webhook/certs
+              readOnly: true
+      # Configure the directory holding the Webhook's server certificates
+      volumes:
+        - name: webhook-certs
+          secret:
+            secretName: danm-webhook-certs
\ No newline at end of file
-- 
cgit 1.2.3-korg