Merge "add a multus with sriov interfaces installation"

6 files changed, 434 insertions, 0 deletions

diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml
new file mode 100644
index 00000000..44263956
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml
@@ -0,0 +1,7 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+# Limits for apps
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml
new file mode 100644
index 00000000..221279b1
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml
@@ -0,0 +1,62 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+- name: Sriov | delete default docker bridge
+  command: ip link delete docker0
+  failed_when: false
+  notify: Sriov | restart docker
+
+# special cases for atomic because it defaults to live-restore: true
+# So we disable live-restore to pickup the new flannel IP.  After
+# we enable it, we have to restart docker again to pickup the new
+# setting and restore the original behavior
+- name: Sriov | restart docker
+  command: /bin/true
+  notify:
+    - Sriov | reload systemd
+    - Sriov | reload docker.socket
+    - Sriov | configure docker live-restore true (atomic)
+    - Sriov | reload docker
+    - Sriov | pause while Docker restarts
+    - Sriov | wait for docker
+
+- name: Sriov | reload systemd
+  shell: systemctl daemon-reload
+
+- name: Sriov | reload docker.socket
+  service:
+    name: docker.socket
+    state: restarted
+  when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+
+- name: Sriov | configure docker live-restore true (atomic)
+  replace:
+    name: /etc/docker/daemon.json
+    regexp: '"live-restore":.*true'
+    replace: '"live-restore": false'
+  when: is_atomic
+
+- name: Sriov | reload docker
+  service:
+    name: docker
+    state: restarted
+
+- name: Sriov | pause while Docker restarts
+  pause:
+    seconds: 10
+    prompt: "Waiting for docker restart"
+
+- name: Sriov | wait for docker
+  command: "{{ docker_bin_dir }}/docker images"
+  register: docker_ready
+  retries: 10
+  delay: 5
+  until: docker_ready.rc == 0
+
+- name: Sriov | reload kubelet
+  service:
+    name: kubelet
+    state: restarted
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml
new file mode 100644
index 00000000..0e3e2f6d
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml
@@ -0,0 +1,106 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+- name: Sriov | Verify if br_netfilter module exists
+  shell: "modinfo br_netfilter"
+  register: modinfo_br_netfilter
+  failed_when: modinfo_br_netfilter.rc not in [0, 1]
+  changed_when: false
+
+- name: Sriov | Enable br_netfilter module
+  modprobe:
+    name: br_netfilter
+    state: present
+  when: modinfo_br_netfilter.rc == 0
+
+# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled
+# when found if br_netfilter is not a module
+- name: Sriov | Check if bridge-nf-call-iptables key exists
+  command: "sysctl net.bridge.bridge-nf-call-iptables"
+  failed_when: false
+  changed_when: false
+  register: sysctl_bridge_nf_call_iptables
+
+- name: Sriov | Enable bridge-nf-call tables
+  sysctl:
+    name: "{{ item }}"
+    state: present
+    value: 1
+    reload: "yes"
+  when: modinfo_br_netfilter.rc == 1 and sysctl_bridge_nf_call_iptables.rc == 0
+  with_items:
+    - net.bridge.bridge-nf-call-iptables
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+
+- name: Sriov | Install Multus CNI
+  shell: |-
+    /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \
+    bash -c "git clone https://github.com/Intel-Corp/multus-cni && cd multus-cni \
+    && ./build && cp bin/multus /opt/cni/bin"
+
+- name: Sriov | Install Sriov CNI
+  shell: |-
+    /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \
+    bash -c "git clone https://github.com/hustcat/sriov-cni && cd sriov-cni \
+    && ./build && cp bin/sriov /opt/cni/bin"
+
+- name: Sriov | Install Flannel CNI
+  shell: |-
+    /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/host/opt/cni/bin/ \
+    {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }} \
+    sh -c "cp /opt/cni/bin/* /host/opt/cni/bin/"
+
+- name: Sriov | Remove all file in /etc/cni/net.d
+  shell: |-
+    rm -rf /etc/cni/net.d/
+    mkdir -p /etc/cni/net.d/
+
+- name: Sriov | Generate Sriov CNI Conf
+  copy:
+    content: |
+      {
+        "name": "minion-cni-network",
+        "type": "multus",
+        "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+        "delegates": [
+          {
+            "type": "flannel",
+            "masterplugin": true,
+            "delegate": {
+              "isDefaultGateway": true
+            }
+          }
+        ]
+      }
+    dest: "/etc/cni/net.d/multus-cni.conf"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Sriov | Enable DHCP CNI
+  shell: /opt/cni/bin/dhcp daemon &
+
+
+- name: Sriov | Create cni-sriov-rbac manifest
+  template:
+    src: cni-sriov-rbac.yml.j2
+    dest: "{{ kube_config_dir }}/cni-sriov-rbac.yml"
+  register: sriov_rbac_manifest
+  when: inventory_hostname == groups['kube-master'][0] and rbac_enabled
+
+- name: Sriov | Create cni-sriov manifest
+  template:
+    src: cni-sriov.yml.j2
+    dest: "{{ kube_config_dir }}/cni-sriov.yml"
+  register: sriov_manifest
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: Sriov | Sriov tests manifest
+  template:
+    src: sriov-test-pod.yml
+    dest: "{{ kube_config_dir }}/sriov-test-pod.yml"
+  when: inventory_hostname == groups['kube-master'][0]
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2
new file mode 100644
index 00000000..1298aeaa
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2
@@ -0,0 +1,49 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sriov
+  namespace: "{{system_namespace}}"
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: sriov
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: sriov
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: sriov
+subjects:
+- kind: ServiceAccount
+  name: sriov
+  namespace: "{{system_namespace}}"
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2
new file mode 100644
index 00000000..90c7f28c
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2
@@ -0,0 +1,159 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: extensions/v1beta1
+kind: ThirdPartyResource
+metadata:
+  name: network.kubernetes.com
+description: "A specification of a Network obj in the kubernetes"
+versions:
+- name: v1
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: flannel
+  namespace: {{system_namespace}}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-cfg
+  namespace: {{system_namespace}}
+  labels:
+    tier: node
+    app: flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "type": "flannel",
+      "delegate": {
+        "isDefaultGateway": true
+      }
+    }
+  net-conf.json: |
+    {
+      "Network": "10.244.0.0/16",
+      "Backend": {
+        "Type": "udp"
+      }
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-flannel-ds
+  namespace: {{system_namespace}}
+  labels:
+    tier: node
+    app: flannel
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      hostNetwork: true
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      serviceAccountName: flannel
+      containers:
+      - name: kube-flannel
+        image: {{ flannel_image_repo }}:{{ flannel_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
+        securityContext:
+          privileged: true
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: run
+          mountPath: /run
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      volumes:
+        - name: run
+          hostPath:
+            path: /run
+        - name: flannel-cfg
+          configMap:
+            name: kube-flannel-cfg
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: flannel-conf
+  namespace: default
+plugin: flannel
+args: '[
+  {
+    "masterplugin": true,
+    "delegate": {
+      "isDefaultGateway": true
+    }
+  }
+]'
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: sriov-conf1
+  namespace: default
+plugin: sriov
+args: '[
+  {
+    "master": "eth1.101",
+    "pfOnly": true,
+    "ipam": {
+      "type": "host-local",
+      "subnet": "192.168.123.0/24",
+      "rangeStart": "192.168.123.11",
+      "rangeEnd": "192.168.123.21",
+      "routes": [
+        {
+          "dst": "0.0.0.0/0"
+        }
+      ],
+      "gateway": "192.168.123.1"
+    }
+  }
+]'
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: sriov-conf2
+  namespace: default
+plugin: sriov
+args: '[
+  {
+    "master": "eth1.101",
+    "pfOnly": true,
+    "ipam": {
+      "type": "host-local",
+      "subnet": "192.168.123.0/24",
+      "rangeStart": "192.168.123.31",
+      "rangeEnd": "192.168.123.41",
+      "routes": [
+        {
+          "dst": "0.0.0.0/0"
+        }
+      ],
+      "gateway": "192.168.123.1"
+    }
+  }
+]'
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml
new file mode 100644
index 00000000..849aca85
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml
@@ -0,0 +1,51 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: multus-test1
+  annotations:
+    networks: '[
+      { "name": "flannel-conf" },
+      { "name": "sriov-conf1" }
+    ]'
+spec:
+  containers:
+    - name: multus-test
+      image: "busybox"
+      command: ["top"]
+      stdin: true
+      tty: true
+  nodeSelector:
+    kubernetes.io/hostname: "host1"
+  tolerations:
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: multus-test2
+  annotations:
+    networks: '[
+      { "name": "flannel-conf" },
+      { "name": "sriov-conf2" }
+    ]'
+spec:
+  containers:
+    - name: multus-test
+      image: "busybox"
+      command: ["top"]
+      stdin: true
+      tty: true
+  nodeSelector:
+    kubernetes.io/hostname: "host2"
+  tolerations:
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"