add a multus with sriov interfaces installation

Support deploying multus sriov CNI plugins by setting
environment "kube_network_plugin" to "sriov".

Change-Id: I3672fd7b6036063bdee57450c2100f39aa5ef68b
Signed-off-by: Di Xu <di.xu@arm.com>

12 files changed, 546 insertions, 17 deletions

diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
index 6d947623..b9d9c234 100644
--- a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml
@@ -217,6 +217,51 @@
   when:
     - stor4nfv is defined and stor4nfv == "Enable"
 
+- name: copy sriov playbook to kargo
+  copy:
+    src: "{{ run_dir }}/roles/sriov"
+    dest: /opt/kargo_k8s/roles/network_plugin
+
+- name: copy sriov-apps playbook to kargo
+  copy:
+    src: "{{ run_dir }}/roles/sriov-apps/"
+    dest: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/sriov
+
+- name: append sriov to network plugin
+  blockinfile:
+    path: /opt/kargo_k8s/roles/network_plugin/meta/main.yml
+    block: "  - role: network_plugin/sriov\n    when: kube_network_plugin == 'sriov'\n    \
+      tags: sriov\n"
+
+- name: append sriov apps to network plugin
+  blockinfile:
+    path: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/meta/main.yml
+    block: "  - role: kubernetes-apps/network_plugin/sriov\n    \
+      when: kube_network_plugin == 'sriov'\n    tags: sriov\n"
+
+- name: append sriov to valid kube_network_plugin list
+  replace:
+    path: "{{ item.path }}"
+    regexp: "{{ item.regexp }}"
+    replace: "{{ item.replace }}"
+  with_items:
+    - {path: "/opt/kargo_k8s/roles/kubernetes/master/templates/manifests/\
+kube-controller-manager.manifest.j2",
+       regexp: '"cloud", "flannel"',
+       replace: '"cloud", "flannel", "sriov"'}
+    - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2',
+       regexp: '"calico", "canal", "flannel", "weave"',
+       replace: '"calico", "canal", "flannel", "weave", "sriov"'}
+    - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.standard.env.j2',
+       regexp: '"calico", "canal", "flannel", "weave"',
+       replace: '"calico", "canal", "flannel", "weave", "sriov"'}
+    - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.rkt.service.j2',
+       regexp: '"calico", "weave", "canal", "flannel"',
+       replace: '"calico", "weave", "canal", "flannel", "sriov"'}
+    - {path: '/opt/kargo_k8s/roles/kubernetes/preinstall/tasks/main.yml',
+       regexp: '"calico", "weave", "canal", "flannel"',
+       replace: '"calico", "weave", "canal", "flannel", "sriov"'}
+
 - name: run kargo playbook
   shell: |
     cd /opt/kargo_k8s
diff --git a/deploy/adapters/ansible/kubernetes/roles/pre-k8s/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/pre-k8s/vars/main.yml
index b196bd25..6d6ecf44 100644
--- a/deploy/adapters/ansible/kubernetes/roles/pre-k8s/vars/main.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/pre-k8s/vars/main.yml
@@ -1,21 +1,21 @@
 ---
 aptpackages:
-- bridge-utils
-- debootstrap
-- ifenslave
-- ifenslave-2.6
-- lsof
-- lvm2
-- ntp
-- ntpdate
-- sudo
-- vlan
-- tcpdump
+  - bridge-utils
+  - debootstrap
+  - ifenslave
+  - ifenslave-2.6
+  - lsof
+  - lvm2
+  - ntp
+  - ntpdate
+  - sudo
+  - vlan
+  - tcpdump
 
 yumpackages:
-- bridge-utils
-- iputils
-- lvm2
-- ntp
-- tcpdump
-- vim
+  - bridge-utils
+  - iputils
+  - lvm2
+  - ntp
+  - tcpdump
+  - vim
diff --git a/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/tasks/RedHat.yml b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/tasks/RedHat.yml
index c59fdfc5..5b434dbe 100644
--- a/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/tasks/RedHat.yml
+++ b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/tasks/RedHat.yml
@@ -36,6 +36,17 @@
     src: ifcfg-eth.j2
     dest: /etc/sysconfig/network-scripts/ifcfg-{{sys_intf_mappings["external"]["interface"]}}
 
+- name: generate ifcfg-sriov
+  template:
+    src: ifcfg-sriov.j2
+    dest: /etc/sysconfig/network-scripts/ifcfg-{{ intf_sriov }}
+  when: intf_sriov|length > 0
+
+- name: remove ifcfg-br-sriov script
+  file:
+    path: /etc/sysconfig/network-scripts/ifcfg-br-sriov
+    state: absent
+
 - name: remove defualt gw
   lineinfile:
     dest: /etc/sysconfig/network
diff --git a/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/templates/ifcfg-sriov.j2 b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/templates/ifcfg-sriov.j2
new file mode 100644
index 00000000..c4005854
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/templates/ifcfg-sriov.j2
@@ -0,0 +1,12 @@
+DEVICE={{ intf_sriov }}
+BOOTPROTO=none
+ONBOOT=yes
+IPADDR={{ ip_settings[inventory_hostname]["tenant"]["ip"] }}
+NETMASK=255.255.255.0
+DEFROUTE="no"
+{% if sys_intf_mappings["tenant"]["vlan_tag"] | int %}
+{% set intf_vlan = "yes" %}
+{% else %}
+{% set intf_vlan = "no" %}
+{% endif %}
+VLAN={{ intf_vlan }}
diff --git a/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/vars/main.yml
new file mode 100644
index 00000000..e525bf5c
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/vars/main.yml
@@ -0,0 +1,7 @@
+---
+intf_sriov: |-
+  {%- set intf_sriov = sys_intf_mappings["tenant"]["interface"] %}
+  {%- if sys_intf_mappings["tenant"]["vlan_tag"] | int %}
+  {%- set intf_sriov = intf_sriov + '.' + sys_intf_mappings["tenant"]["vlan_tag"]|string %}
+  {%- endif %}
+  {{- intf_sriov }}
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov-apps/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov-apps/tasks/main.yml
new file mode 100644
index 00000000..662fa7bf
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov-apps/tasks/main.yml
@@ -0,0 +1,20 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+- name: "Sriov | Create ServiceAccount ClusterRole and ClusterRoleBinding"
+  command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/cni-sriov-rbac.yml"
+  run_once: true
+  when: rbac_enabled and sriov_rbac_manifest.changed
+
+- name: Sriov | Create Network Resources
+  kube:
+    name: "kube-sriov"
+    kubectl: "{{ bin_dir }}/kubectl"
+    filename: "{{ kube_config_dir }}/cni-sriov.yml"
+    namespace: "{{system_namespace}}"
+    state: "{{ item | ternary('latest','present') }}"
+  with_items: "{{ sriov_manifest.changed }}"
+  when: inventory_hostname == groups['kube-master'][0]
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml
new file mode 100644
index 00000000..44263956
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml
@@ -0,0 +1,7 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+# Limits for apps
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml
new file mode 100644
index 00000000..221279b1
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml
@@ -0,0 +1,62 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+- name: Sriov | delete default docker bridge
+  command: ip link delete docker0
+  failed_when: false
+  notify: Sriov | restart docker
+
+# special cases for atomic because it defaults to live-restore: true
+# So we disable live-restore to pickup the new flannel IP.  After
+# we enable it, we have to restart docker again to pickup the new
+# setting and restore the original behavior
+- name: Sriov | restart docker
+  command: /bin/true
+  notify:
+    - Sriov | reload systemd
+    - Sriov | reload docker.socket
+    - Sriov | configure docker live-restore true (atomic)
+    - Sriov | reload docker
+    - Sriov | pause while Docker restarts
+    - Sriov | wait for docker
+
+- name: Sriov | reload systemd
+  shell: systemctl daemon-reload
+
+- name: Sriov | reload docker.socket
+  service:
+    name: docker.socket
+    state: restarted
+  when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+
+- name: Sriov | configure docker live-restore true (atomic)
+  replace:
+    name: /etc/docker/daemon.json
+    regexp: '"live-restore":.*true'
+    replace: '"live-restore": false'
+  when: is_atomic
+
+- name: Sriov | reload docker
+  service:
+    name: docker
+    state: restarted
+
+- name: Sriov | pause while Docker restarts
+  pause:
+    seconds: 10
+    prompt: "Waiting for docker restart"
+
+- name: Sriov | wait for docker
+  command: "{{ docker_bin_dir }}/docker images"
+  register: docker_ready
+  retries: 10
+  delay: 5
+  until: docker_ready.rc == 0
+
+- name: Sriov | reload kubelet
+  service:
+    name: kubelet
+    state: restarted
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml
new file mode 100644
index 00000000..0e3e2f6d
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml
@@ -0,0 +1,106 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+- name: Sriov | Verify if br_netfilter module exists
+  shell: "modinfo br_netfilter"
+  register: modinfo_br_netfilter
+  failed_when: modinfo_br_netfilter.rc not in [0, 1]
+  changed_when: false
+
+- name: Sriov | Enable br_netfilter module
+  modprobe:
+    name: br_netfilter
+    state: present
+  when: modinfo_br_netfilter.rc == 0
+
+# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled
+# when found if br_netfilter is not a module
+- name: Sriov | Check if bridge-nf-call-iptables key exists
+  command: "sysctl net.bridge.bridge-nf-call-iptables"
+  failed_when: false
+  changed_when: false
+  register: sysctl_bridge_nf_call_iptables
+
+- name: Sriov | Enable bridge-nf-call tables
+  sysctl:
+    name: "{{ item }}"
+    state: present
+    value: 1
+    reload: "yes"
+  when: modinfo_br_netfilter.rc == 1 and sysctl_bridge_nf_call_iptables.rc == 0
+  with_items:
+    - net.bridge.bridge-nf-call-iptables
+    - net.bridge.bridge-nf-call-arptables
+    - net.bridge.bridge-nf-call-ip6tables
+
+- name: Sriov | Install Multus CNI
+  shell: |-
+    /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \
+    bash -c "git clone https://github.com/Intel-Corp/multus-cni && cd multus-cni \
+    && ./build && cp bin/multus /opt/cni/bin"
+
+- name: Sriov | Install Sriov CNI
+  shell: |-
+    /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \
+    bash -c "git clone https://github.com/hustcat/sriov-cni && cd sriov-cni \
+    && ./build && cp bin/sriov /opt/cni/bin"
+
+- name: Sriov | Install Flannel CNI
+  shell: |-
+    /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/host/opt/cni/bin/ \
+    {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }} \
+    sh -c "cp /opt/cni/bin/* /host/opt/cni/bin/"
+
+- name: Sriov | Remove all file in /etc/cni/net.d
+  shell: |-
+    rm -rf /etc/cni/net.d/
+    mkdir -p /etc/cni/net.d/
+
+- name: Sriov | Generate Sriov CNI Conf
+  copy:
+    content: |
+      {
+        "name": "minion-cni-network",
+        "type": "multus",
+        "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+        "delegates": [
+          {
+            "type": "flannel",
+            "masterplugin": true,
+            "delegate": {
+              "isDefaultGateway": true
+            }
+          }
+        ]
+      }
+    dest: "/etc/cni/net.d/multus-cni.conf"
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Sriov | Enable DHCP CNI
+  shell: /opt/cni/bin/dhcp daemon &
+
+
+- name: Sriov | Create cni-sriov-rbac manifest
+  template:
+    src: cni-sriov-rbac.yml.j2
+    dest: "{{ kube_config_dir }}/cni-sriov-rbac.yml"
+  register: sriov_rbac_manifest
+  when: inventory_hostname == groups['kube-master'][0] and rbac_enabled
+
+- name: Sriov | Create cni-sriov manifest
+  template:
+    src: cni-sriov.yml.j2
+    dest: "{{ kube_config_dir }}/cni-sriov.yml"
+  register: sriov_manifest
+  when: inventory_hostname == groups['kube-master'][0]
+
+- name: Sriov | Sriov tests manifest
+  template:
+    src: sriov-test-pod.yml
+    dest: "{{ kube_config_dir }}/sriov-test-pod.yml"
+  when: inventory_hostname == groups['kube-master'][0]
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2
new file mode 100644
index 00000000..1298aeaa
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2
@@ -0,0 +1,49 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sriov
+  namespace: "{{system_namespace}}"
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: sriov
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: sriov
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: sriov
+subjects:
+- kind: ServiceAccount
+  name: sriov
+  namespace: "{{system_namespace}}"
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2
new file mode 100644
index 00000000..90c7f28c
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2
@@ -0,0 +1,159 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: extensions/v1beta1
+kind: ThirdPartyResource
+metadata:
+  name: network.kubernetes.com
+description: "A specification of a Network obj in the kubernetes"
+versions:
+- name: v1
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: flannel
+  namespace: {{system_namespace}}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-flannel-cfg
+  namespace: {{system_namespace}}
+  labels:
+    tier: node
+    app: flannel
+data:
+  cni-conf.json: |
+    {
+      "name": "cbr0",
+      "type": "flannel",
+      "delegate": {
+        "isDefaultGateway": true
+      }
+    }
+  net-conf.json: |
+    {
+      "Network": "10.244.0.0/16",
+      "Backend": {
+        "Type": "udp"
+      }
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-flannel-ds
+  namespace: {{system_namespace}}
+  labels:
+    tier: node
+    app: flannel
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: flannel
+    spec:
+      hostNetwork: true
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      serviceAccountName: flannel
+      containers:
+      - name: kube-flannel
+        image: {{ flannel_image_repo }}:{{ flannel_image_tag }}
+        imagePullPolicy: {{ k8s_image_pull_policy }}
+        command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
+        securityContext:
+          privileged: true
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        volumeMounts:
+        - name: run
+          mountPath: /run
+        - name: flannel-cfg
+          mountPath: /etc/kube-flannel/
+      volumes:
+        - name: run
+          hostPath:
+            path: /run
+        - name: flannel-cfg
+          configMap:
+            name: kube-flannel-cfg
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: flannel-conf
+  namespace: default
+plugin: flannel
+args: '[
+  {
+    "masterplugin": true,
+    "delegate": {
+      "isDefaultGateway": true
+    }
+  }
+]'
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: sriov-conf1
+  namespace: default
+plugin: sriov
+args: '[
+  {
+    "master": "eth1.101",
+    "pfOnly": true,
+    "ipam": {
+      "type": "host-local",
+      "subnet": "192.168.123.0/24",
+      "rangeStart": "192.168.123.11",
+      "rangeEnd": "192.168.123.21",
+      "routes": [
+        {
+          "dst": "0.0.0.0/0"
+        }
+      ],
+      "gateway": "192.168.123.1"
+    }
+  }
+]'
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: sriov-conf2
+  namespace: default
+plugin: sriov
+args: '[
+  {
+    "master": "eth1.101",
+    "pfOnly": true,
+    "ipam": {
+      "type": "host-local",
+      "subnet": "192.168.123.0/24",
+      "rangeStart": "192.168.123.31",
+      "rangeEnd": "192.168.123.41",
+      "routes": [
+        {
+          "dst": "0.0.0.0/0"
+        }
+      ],
+      "gateway": "192.168.123.1"
+    }
+  }
+]'
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml
new file mode 100644
index 00000000..849aca85
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml
@@ -0,0 +1,51 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: multus-test1
+  annotations:
+    networks: '[
+      { "name": "flannel-conf" },
+      { "name": "sriov-conf1" }
+    ]'
+spec:
+  containers:
+    - name: multus-test
+      image: "busybox"
+      command: ["top"]
+      stdin: true
+      tty: true
+  nodeSelector:
+    kubernetes.io/hostname: "host1"
+  tolerations:
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: multus-test2
+  annotations:
+    networks: '[
+      { "name": "flannel-conf" },
+      { "name": "sriov-conf2" }
+    ]'
+spec:
+  containers:
+    - name: multus-test
+      image: "busybox"
+      command: ["top"]
+      stdin: true
+      tty: true
+  nodeSelector:
+    kubernetes.io/hostname: "host2"
+  tolerations:
+    - key: "node-role.kubernetes.io/master"
+      operator: "Exists"
+      effect: "NoSchedule"