10 files changed, 376 insertions, 0 deletions
diff --git a/ansible/roles/init_kubeadm/defaults/main.yml b/ansible/roles/init_kubeadm/defaults/main.yml
new file mode 100644
index 000000000..3d868398d
--- /dev/null
+++ b/ansible/roles/init_kubeadm/defaults/main.yml
@@ -0,0 +1,27 @@
+# Copyright (c) 2018-2019 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+cmk_rbc_rules_url: https://raw.githubusercontent.com/intel/CPU-Manager-for-Kubernetes/master/resources/authorization/cmk-rbac-rules.yaml
+cmk_serviceaccount: https://raw.githubusercontent.com/intel/CPU-Manager-for-Kubernetes/master/resources/authorization/cmk-serviceaccount.yaml
+dpdk_devbind_path: "{{ INSTALL_BIN_PATH|default('/opt') }}"
+multus_config:
+  name: node-cni-network
+  type: multus
+  kubeconfig: /etc/kubernetes/kubelet.conf
+  delegates:
+    - type: flannel
+      delegate:
+        isDefaultGateway: true
+      hairpinMode: true
+      masterplugin: true
diff --git a/ansible/roles/init_kubeadm/tasks/kubeadm.yml b/ansible/roles/init_kubeadm/tasks/kubeadm.yml
new file mode 100644
index 000000000..7c808a01c
--- /dev/null
+++ b/ansible/roles/init_kubeadm/tasks/kubeadm.yml
@@ -0,0 +1,50 @@
+# Copyright (c) 2018-2019 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- set_fact:
+    service_cidr_param: --service-cidr={{ kubeadm_service_cidr }}
+  when: 'kubeadm_service_cidr is defined'
+
+- set_fact:
+    pod_network_cidr_param: --pod-network-cidr={{ kubeadm_pod_network_cidr }}
+  when: 'kubeadm_pod_network_cidr is defined'
+
+- set_fact:
+    kubernetes_version: --kubernetes-version {{ kubeadm_kubernetes_version }}
+  when: 'kubeadm_kubernetes_version is defined'
+
+- name: Initialize Kubernetes cluster
+  command: >
+           kubeadm init {{ pod_network_cidr_param|default('') }}
+           {{ service_cidr_param|default('') }} {{ kubernetes_version|default('') }}
+           --ignore-preflight-errors=all
+
+- name: Create Kubernetes configuration dir
+  file: path={{ ansible_env.HOME }}/.kube state=directory
+
+- name: Setup Kubernetes environment
+  copy:
+    src: /etc/kubernetes/admin.conf
+    dest: "{{ ansible_env.HOME }}/.kube/config"
+    remote_src: yes
+
+- name: Allow to schedule pods on the master
+  command: kubectl taint nodes --all node-role.kubernetes.io/master-
+
+- name: Wait for kube-dns pod to be in running state
+  command: kubectl get pods --namespace=kube-system -l k8s-app=kube-dns -o json
+  register: result
+  until: result.stdout|from_json|json_query('items[].status.phase|[0]') == "Running"
+  retries: 30
+  delay: 2
diff --git a/ansible/roles/init_kubeadm/tasks/kubectl.yml b/ansible/roles/init_kubeadm/tasks/kubectl.yml
new file mode 100644
index 000000000..3e9f2d71a
--- /dev/null
+++ b/ansible/roles/init_kubeadm/tasks/kubectl.yml
@@ -0,0 +1,26 @@
+# Copyright (c) 2018-2019 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Generate temporary resource file
+  tempfile: state=file prefix=kubectl.{{ item }}.
+  register: config_file
+
+- name: Generate {{ item }} resource file
+  template: src={{ item }}.j2 dest={{ config_file.path }}
+
+- name: Create {{ item }} resource
+  command: kubectl create -f {{ config_file.path }}
+
+- name: Create Kubernetes configuration dir
+  file: path={{ config_file.path  }} state=absent
diff --git a/ansible/roles/init_kubeadm/tasks/main.yml b/ansible/roles/init_kubeadm/tasks/main.yml
new file mode 100644
index 000000000..df7334ce1
--- /dev/null
+++ b/ansible/roles/init_kubeadm/tasks/main.yml
@@ -0,0 +1,70 @@
+# Copyright (c) 2018-2019 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+- name: Disable swap
+  command: swapoff -a
+  ignore_errors: true
+
+- name: Reset Kubernetes cluster
+  command: kubeadm reset
+
+- name: Clean Kubernetes directories (w/o removing the folder itself)
+  shell: rm -fr {{ item }}/*
+  with_items:
+    - /etc/kubernetes
+    - /var/lib/cni
+    - /etc/cmk
+
+- name: Create Multus CNI plugin dir
+  file: path=/etc/cni/net.d state=directory
+
+- name: Configure Multus CNI plugin
+  template: src=10-multus-cni.conf.j2 dest=/etc/cni/net.d/10-multus-cni.conf owner=root mode=0644
+
+- name: Change default kubelet cluster dns IP
+  lineinfile:
+    path: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
+    regexp: '^(.*)--cluster-dns=([0-9\.]*)( +.*)$'
+    line: '\1--cluster-dns={{ kubelet_cluster_dns_ip }}\3'
+    backrefs: yes
+    backup: yes
+
+- name: Systemd daemon reload
+  command: systemctl daemon-reload
+
+- name: Restart kubelet
+  service: name=kubelet state=restarted
+
+- name: Initialize kubeadm
+  include: kubeadm.yml
+
+- name: Create CMK Kubernetes resources
+  command: kubectl create -f {{ item }}
+  with_items:
+    - "{{ cmk_rbc_rules_url }}"
+    - "{{ cmk_serviceaccount }}"
+
+- name: Create Kubernetes resources
+  include: kubectl.yml
+  with_items:
+    - crd-network.yaml
+    - net-flannel.yaml
+    - roles.yaml
+    - kube-flannel.yaml
+    - cmk-init-pod.yaml
+
+- name: Create a ClusterRoleBinding for a particular ClusterRole
+  command: >
+           kubectl create clusterrolebinding multus-node-{{ ansible_hostname }}
+           --clusterrole=multus-crd-overpowered --user=system:node:{{ ansible_hostname }}
diff --git a/ansible/roles/init_kubeadm/templates/10-multus-cni.conf.j2 b/ansible/roles/init_kubeadm/templates/10-multus-cni.conf.j2
new file mode 100644
index 000000000..a68afaf26
--- /dev/null
+++ b/ansible/roles/init_kubeadm/templates/10-multus-cni.conf.j2
@@ -0,0 +1 @@
+{{ multus_config | to_nice_json }}
diff --git a/ansible/roles/init_kubeadm/templates/cmk-init-pod.yaml.j2 b/ansible/roles/init_kubeadm/templates/cmk-init-pod.yaml.j2
new file mode 100644
index 000000000..a4c735394
--- /dev/null
+++ b/ansible/roles/init_kubeadm/templates/cmk-init-pod.yaml.j2
@@ -0,0 +1,18 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: cmk-cluster-init-pod
+  name: cmk-cluster-init-pod
+spec:
+  serviceAccountName: cmk-serviceaccount
+  containers:
+  - args:
+      # Change this value to pass different options to cluster-init.
+      - "/cmk/cmk.py cluster-init --host-list={{ ansible_hostname }} --saname=cmk-serviceaccount --cmk-img=si-docker.ir.intel.com/vcmts-ubuntu/cmk --num-dp-cores=4 --dp-mode=spread --num-cp-cores=1 --cp-mode=spread"
+    command:
+    - "/bin/bash"
+    - "-c"
+    image: si-docker.ir.intel.com/vcmts-ubuntu/cmk
+    name: cmk-cluster-init-pod
+  restartPolicy: Never
diff --git a/ansible/roles/init_kubeadm/templates/crd-network.yaml.j2 b/ansible/roles/init_kubeadm/templates/crd-network.yaml.j2
new file mode 100644
index 000000000..180038b07
--- /dev/null
+++ b/ansible/roles/init_kubeadm/templates/crd-network.yaml.j2
@@ -0,0 +1,13 @@
+---
+apiVersion: "apiextensions.k8s.io/v1beta1"
+kind: CustomResourceDefinition
+metadata:
+  name: networks.kubernetes.com
+spec:
+  group: kubernetes.com
+  version: v1
+  scope: Namespaced
+  names:
+    plural: networks
+    singular: network
+    kind: Network
diff --git a/ansible/roles/init_kubeadm/templates/kube-flannel.yaml.j2 b/ansible/roles/init_kubeadm/templates/kube-flannel.yaml.j2
new file mode 100644
index 000000000..4cf63d6ca
--- /dev/null
+++ b/ansible/roles/init_kubeadm/templates/kube-flannel.yaml.j2
@@ -0,0 +1,145 @@
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: flannel
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: flannel
+subjects:
+- kind: ServiceAccount
+  name: flannel
+  namespace: kube-system
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: flannel
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "plugins": [
+        {
+          "type": "flannel",
+          "delegate": {
+            "hairpinMode": true,
+            "isDefaultGateway": true
+          }
+        },
+        {
+          "type": "portmap",
+          "capabilities": {
+            "portMappings": true
+          }
+        }
+      ]
+    }
+  net-conf.json: |
+    {
+      "Network": "{{ kubeadm_pod_network_cidr }}",
+      "Backend": {
+        "Type": "vxlan"
+      }
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-flannel-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: flannel
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      serviceAccountName: flannel
+      containers:
+      - name: kube-flannel
+        image: quay.io/coreos/flannel:v0.10.0-amd64
+        command:
+        - /opt/bin/flanneld
+        args:
+        - --ip-masq
+        - --kube-subnet-mgr
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: run
+          mountPath: /run
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      volumes:
+        - name: run
+          hostPath:
+            path: /run
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: flannel-cfg
+          configMap:
+            name: kube-flannel-cfg
diff --git a/ansible/roles/init_kubeadm/templates/net-flannel.yaml.j2 b/ansible/roles/init_kubeadm/templates/net-flannel.yaml.j2
new file mode 100644
index 000000000..b872cd613
--- /dev/null
+++ b/ansible/roles/init_kubeadm/templates/net-flannel.yaml.j2
@@ -0,0 +1,10 @@
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: flannel
+plugin: flannel
+args: '[{
+                "delegate": {
+                        "isDefaultGateway": true
+                }
+        }]'
diff --git a/ansible/roles/init_kubeadm/templates/roles.yaml.j2 b/ansible/roles/init_kubeadm/templates/roles.yaml.j2
new file mode 100644
index 000000000..635ba0c1e
--- /dev/null
+++ b/ansible/roles/init_kubeadm/templates/roles.yaml.j2
@@ -0,0 +1,16 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: multus-crd-overpowered
+rules:
+- apiGroups:
+  - '*'
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- nonResourceURLs:
+  - '*'
+  verbs:
+  - '*'