diff options
Diffstat (limited to 'deploy/adapters/ansible/kubernetes/roles/sriov')
6 files changed, 434 insertions, 0 deletions
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml new file mode 100644 index 00000000..44263956 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml @@ -0,0 +1,7 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +# Limits for apps diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml new file mode 100644 index 00000000..221279b1 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml @@ -0,0 +1,62 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +- name: Sriov | delete default docker bridge + command: ip link delete docker0 + failed_when: false + notify: Sriov | restart docker + +# special cases for atomic because it defaults to live-restore: true +# So we disable live-restore to pickup the new flannel IP. After +# we enable it, we have to restart docker again to pickup the new +# setting and restore the original behavior +- name: Sriov | restart docker + command: /bin/true + notify: + - Sriov | reload systemd + - Sriov | reload docker.socket + - Sriov | configure docker live-restore true (atomic) + - Sriov | reload docker + - Sriov | pause while Docker restarts + - Sriov | wait for docker + +- name: Sriov | reload systemd + shell: systemctl daemon-reload + +- name: Sriov | reload docker.socket + service: + name: docker.socket + state: restarted + when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] + +- name: Sriov | configure docker live-restore true (atomic) + replace: + name: /etc/docker/daemon.json + regexp: '"live-restore":.*true' + replace: '"live-restore": false' + when: is_atomic + +- name: Sriov | reload docker + service: + name: docker + state: restarted + +- name: Sriov | pause while Docker restarts + pause: + seconds: 10 + prompt: "Waiting for docker restart" + +- name: Sriov | wait for docker + command: "{{ docker_bin_dir }}/docker images" + register: docker_ready + retries: 10 + delay: 5 + until: docker_ready.rc == 0 + +- name: Sriov | reload kubelet + service: + name: kubelet + state: restarted diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml new file mode 100644 index 00000000..0e3e2f6d --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml @@ -0,0 +1,106 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +- name: Sriov | Verify if br_netfilter module exists + shell: "modinfo br_netfilter" + register: modinfo_br_netfilter + failed_when: modinfo_br_netfilter.rc not in [0, 1] + changed_when: false + +- name: Sriov | Enable br_netfilter module + modprobe: + name: br_netfilter + state: present + when: modinfo_br_netfilter.rc == 0 + +# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled +# when found if br_netfilter is not a module +- name: Sriov | Check if bridge-nf-call-iptables key exists + command: "sysctl net.bridge.bridge-nf-call-iptables" + failed_when: false + changed_when: false + register: sysctl_bridge_nf_call_iptables + +- name: Sriov | Enable bridge-nf-call tables + sysctl: + name: "{{ item }}" + state: present + value: 1 + reload: "yes" + when: modinfo_br_netfilter.rc == 1 and sysctl_bridge_nf_call_iptables.rc == 0 + with_items: + - net.bridge.bridge-nf-call-iptables + - net.bridge.bridge-nf-call-arptables + - net.bridge.bridge-nf-call-ip6tables + +- name: Sriov | Install Multus CNI + shell: |- + /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \ + bash -c "git clone https://github.com/Intel-Corp/multus-cni && cd multus-cni \ + && ./build && cp bin/multus /opt/cni/bin" + +- name: Sriov | Install Sriov CNI + shell: |- + /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \ + bash -c "git clone https://github.com/hustcat/sriov-cni && cd sriov-cni \ + && ./build && cp bin/sriov /opt/cni/bin" + +- name: Sriov | Install Flannel CNI + shell: |- + /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/host/opt/cni/bin/ \ + {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }} \ + sh -c "cp /opt/cni/bin/* /host/opt/cni/bin/" + +- name: Sriov | Remove all file in /etc/cni/net.d + shell: |- + rm -rf /etc/cni/net.d/ + mkdir -p /etc/cni/net.d/ + +- name: Sriov | Generate Sriov CNI Conf + copy: + content: | + { + "name": "minion-cni-network", + "type": "multus", + "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml", + "delegates": [ + { + "type": "flannel", + "masterplugin": true, + "delegate": { + "isDefaultGateway": true + } + } + ] + } + dest: "/etc/cni/net.d/multus-cni.conf" + owner: root + group: root + mode: 0644 + +- name: Sriov | Enable DHCP CNI + shell: /opt/cni/bin/dhcp daemon & + + +- name: Sriov | Create cni-sriov-rbac manifest + template: + src: cni-sriov-rbac.yml.j2 + dest: "{{ kube_config_dir }}/cni-sriov-rbac.yml" + register: sriov_rbac_manifest + when: inventory_hostname == groups['kube-master'][0] and rbac_enabled + +- name: Sriov | Create cni-sriov manifest + template: + src: cni-sriov.yml.j2 + dest: "{{ kube_config_dir }}/cni-sriov.yml" + register: sriov_manifest + when: inventory_hostname == groups['kube-master'][0] + +- name: Sriov | Sriov tests manifest + template: + src: sriov-test-pod.yml + dest: "{{ kube_config_dir }}/sriov-test-pod.yml" + when: inventory_hostname == groups['kube-master'][0] diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2 new file mode 100644 index 00000000..1298aeaa --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2 @@ -0,0 +1,49 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sriov + namespace: "{{system_namespace}}" +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: sriov +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: sriov +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sriov +subjects: +- kind: ServiceAccount + name: sriov + namespace: "{{system_namespace}}" diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2 new file mode 100644 index 00000000..90c7f28c --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2 @@ -0,0 +1,159 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +apiVersion: extensions/v1beta1 +kind: ThirdPartyResource +metadata: + name: network.kubernetes.com +description: "A specification of a Network obj in the kubernetes" +versions: +- name: v1 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flannel + namespace: {{system_namespace}} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-cfg + namespace: {{system_namespace}} + labels: + tier: node + app: flannel +data: + cni-conf.json: | + { + "name": "cbr0", + "type": "flannel", + "delegate": { + "isDefaultGateway": true + } + } + net-conf.json: | + { + "Network": "10.244.0.0/16", + "Backend": { + "Type": "udp" + } + } +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-flannel-ds + namespace: {{system_namespace}} + labels: + tier: node + app: flannel +spec: + template: + metadata: + labels: + tier: node + app: flannel + spec: + hostNetwork: true + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: flannel + containers: + - name: kube-flannel + image: {{ flannel_image_repo }}:{{ flannel_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] + securityContext: + privileged: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + - name: run + hostPath: + path: /run + - name: flannel-cfg + configMap: + name: kube-flannel-cfg +--- +apiVersion: "kubernetes.com/v1" +kind: Network +metadata: + name: flannel-conf + namespace: default +plugin: flannel +args: '[ + { + "masterplugin": true, + "delegate": { + "isDefaultGateway": true + } + } +]' +--- +apiVersion: "kubernetes.com/v1" +kind: Network +metadata: + name: sriov-conf1 + namespace: default +plugin: sriov +args: '[ + { + "master": "eth1.101", + "pfOnly": true, + "ipam": { + "type": "host-local", + "subnet": "192.168.123.0/24", + "rangeStart": "192.168.123.11", + "rangeEnd": "192.168.123.21", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "gateway": "192.168.123.1" + } + } +]' +--- +apiVersion: "kubernetes.com/v1" +kind: Network +metadata: + name: sriov-conf2 + namespace: default +plugin: sriov +args: '[ + { + "master": "eth1.101", + "pfOnly": true, + "ipam": { + "type": "host-local", + "subnet": "192.168.123.0/24", + "rangeStart": "192.168.123.31", + "rangeEnd": "192.168.123.41", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "gateway": "192.168.123.1" + } + } +]' diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml new file mode 100644 index 00000000..849aca85 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml @@ -0,0 +1,51 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: multus-test1 + annotations: + networks: '[ + { "name": "flannel-conf" }, + { "name": "sriov-conf1" } + ]' +spec: + containers: + - name: multus-test + image: "busybox" + command: ["top"] + stdin: true + tty: true + nodeSelector: + kubernetes.io/hostname: "host1" + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" +--- +apiVersion: v1 +kind: Pod +metadata: + name: multus-test2 + annotations: + networks: '[ + { "name": "flannel-conf" }, + { "name": "sriov-conf2" } + ]' +spec: + containers: + - name: multus-test + image: "busybox" + command: ["top"] + stdin: true + tty: true + nodeSelector: + kubernetes.io/hostname: "host2" + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" |