summaryrefslogtreecommitdiffstats
path: root/deploy/adapters/ansible/kubernetes/roles/sriov
diff options
context:
space:
mode:
authorJustin chi <chigang@huawei.com>2018-03-09 03:51:29 +0000
committerGerrit Code Review <gerrit@opnfv.org>2018-03-09 03:51:29 +0000
commit6bc387014a355483900cd5542961096e8b307cf4 (patch)
treeafacb1701655cc058316984e46eb52a138c23d83 /deploy/adapters/ansible/kubernetes/roles/sriov
parentba134f4d524bd885cd40c346f32fcbfb5e99d4c5 (diff)
parent3ad4238fbf8a8043cfbe6623b22b8d16e82a408f (diff)
Merge "add a multus with sriov interfaces installation"
Diffstat (limited to 'deploy/adapters/ansible/kubernetes/roles/sriov')
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml7
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml62
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml106
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j249
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2159
-rw-r--r--deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml51
6 files changed, 434 insertions, 0 deletions
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml
new file mode 100644
index 00000000..44263956
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml
@@ -0,0 +1,7 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+# Limits for apps
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml
new file mode 100644
index 00000000..221279b1
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml
@@ -0,0 +1,62 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+- name: Sriov | delete default docker bridge
+ command: ip link delete docker0
+ failed_when: false
+ notify: Sriov | restart docker
+
+# special cases for atomic because it defaults to live-restore: true
+# So we disable live-restore to pickup the new flannel IP. After
+# we enable it, we have to restart docker again to pickup the new
+# setting and restore the original behavior
+- name: Sriov | restart docker
+ command: /bin/true
+ notify:
+ - Sriov | reload systemd
+ - Sriov | reload docker.socket
+ - Sriov | configure docker live-restore true (atomic)
+ - Sriov | reload docker
+ - Sriov | pause while Docker restarts
+ - Sriov | wait for docker
+
+- name: Sriov | reload systemd
+ shell: systemctl daemon-reload
+
+- name: Sriov | reload docker.socket
+ service:
+ name: docker.socket
+ state: restarted
+ when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS']
+
+- name: Sriov | configure docker live-restore true (atomic)
+ replace:
+ name: /etc/docker/daemon.json
+ regexp: '"live-restore":.*true'
+ replace: '"live-restore": false'
+ when: is_atomic
+
+- name: Sriov | reload docker
+ service:
+ name: docker
+ state: restarted
+
+- name: Sriov | pause while Docker restarts
+ pause:
+ seconds: 10
+ prompt: "Waiting for docker restart"
+
+- name: Sriov | wait for docker
+ command: "{{ docker_bin_dir }}/docker images"
+ register: docker_ready
+ retries: 10
+ delay: 5
+ until: docker_ready.rc == 0
+
+- name: Sriov | reload kubelet
+ service:
+ name: kubelet
+ state: restarted
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml
new file mode 100644
index 00000000..0e3e2f6d
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml
@@ -0,0 +1,106 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+- name: Sriov | Verify if br_netfilter module exists
+ shell: "modinfo br_netfilter"
+ register: modinfo_br_netfilter
+ failed_when: modinfo_br_netfilter.rc not in [0, 1]
+ changed_when: false
+
+- name: Sriov | Enable br_netfilter module
+ modprobe:
+ name: br_netfilter
+ state: present
+ when: modinfo_br_netfilter.rc == 0
+
+# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled
+# when found if br_netfilter is not a module
+- name: Sriov | Check if bridge-nf-call-iptables key exists
+ command: "sysctl net.bridge.bridge-nf-call-iptables"
+ failed_when: false
+ changed_when: false
+ register: sysctl_bridge_nf_call_iptables
+
+- name: Sriov | Enable bridge-nf-call tables
+ sysctl:
+ name: "{{ item }}"
+ state: present
+ value: 1
+ reload: "yes"
+ when: modinfo_br_netfilter.rc == 1 and sysctl_bridge_nf_call_iptables.rc == 0
+ with_items:
+ - net.bridge.bridge-nf-call-iptables
+ - net.bridge.bridge-nf-call-arptables
+ - net.bridge.bridge-nf-call-ip6tables
+
+- name: Sriov | Install Multus CNI
+ shell: |-
+ /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \
+ bash -c "git clone https://github.com/Intel-Corp/multus-cni && cd multus-cni \
+ && ./build && cp bin/multus /opt/cni/bin"
+
+- name: Sriov | Install Sriov CNI
+ shell: |-
+ /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \
+ bash -c "git clone https://github.com/hustcat/sriov-cni && cd sriov-cni \
+ && ./build && cp bin/sriov /opt/cni/bin"
+
+- name: Sriov | Install Flannel CNI
+ shell: |-
+ /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/host/opt/cni/bin/ \
+ {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }} \
+ sh -c "cp /opt/cni/bin/* /host/opt/cni/bin/"
+
+- name: Sriov | Remove all file in /etc/cni/net.d
+ shell: |-
+ rm -rf /etc/cni/net.d/
+ mkdir -p /etc/cni/net.d/
+
+- name: Sriov | Generate Sriov CNI Conf
+ copy:
+ content: |
+ {
+ "name": "minion-cni-network",
+ "type": "multus",
+ "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+ "delegates": [
+ {
+ "type": "flannel",
+ "masterplugin": true,
+ "delegate": {
+ "isDefaultGateway": true
+ }
+ }
+ ]
+ }
+ dest: "/etc/cni/net.d/multus-cni.conf"
+ owner: root
+ group: root
+ mode: 0644
+
+- name: Sriov | Enable DHCP CNI
+ shell: /opt/cni/bin/dhcp daemon &
+
+
+- name: Sriov | Create cni-sriov-rbac manifest
+ template:
+ src: cni-sriov-rbac.yml.j2
+ dest: "{{ kube_config_dir }}/cni-sriov-rbac.yml"
+ register: sriov_rbac_manifest
+ when: inventory_hostname == groups['kube-master'][0] and rbac_enabled
+
+- name: Sriov | Create cni-sriov manifest
+ template:
+ src: cni-sriov.yml.j2
+ dest: "{{ kube_config_dir }}/cni-sriov.yml"
+ register: sriov_manifest
+ when: inventory_hostname == groups['kube-master'][0]
+
+- name: Sriov | Sriov tests manifest
+ template:
+ src: sriov-test-pod.yml
+ dest: "{{ kube_config_dir }}/sriov-test-pod.yml"
+ when: inventory_hostname == groups['kube-master'][0]
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2
new file mode 100644
index 00000000..1298aeaa
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2
@@ -0,0 +1,49 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: sriov
+ namespace: "{{system_namespace}}"
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: sriov
+rules:
+ - apiGroups:
+ - ""
+ resources:
+ - pods
+ verbs:
+ - get
+ - apiGroups:
+ - ""
+ resources:
+ - nodes
+ verbs:
+ - list
+ - watch
+ - apiGroups:
+ - ""
+ resources:
+ - nodes/status
+ verbs:
+ - patch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ name: sriov
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: ClusterRole
+ name: sriov
+subjects:
+- kind: ServiceAccount
+ name: sriov
+ namespace: "{{system_namespace}}"
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2
new file mode 100644
index 00000000..90c7f28c
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2
@@ -0,0 +1,159 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: extensions/v1beta1
+kind: ThirdPartyResource
+metadata:
+ name: network.kubernetes.com
+description: "A specification of a Network obj in the kubernetes"
+versions:
+- name: v1
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: flannel
+ namespace: {{system_namespace}}
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+ name: kube-flannel-cfg
+ namespace: {{system_namespace}}
+ labels:
+ tier: node
+ app: flannel
+data:
+ cni-conf.json: |
+ {
+ "name": "cbr0",
+ "type": "flannel",
+ "delegate": {
+ "isDefaultGateway": true
+ }
+ }
+ net-conf.json: |
+ {
+ "Network": "10.244.0.0/16",
+ "Backend": {
+ "Type": "udp"
+ }
+ }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: kube-flannel-ds
+ namespace: {{system_namespace}}
+ labels:
+ tier: node
+ app: flannel
+spec:
+ template:
+ metadata:
+ labels:
+ tier: node
+ app: flannel
+ spec:
+ hostNetwork: true
+ tolerations:
+ - key: node-role.kubernetes.io/master
+ operator: Exists
+ effect: NoSchedule
+ serviceAccountName: flannel
+ containers:
+ - name: kube-flannel
+ image: {{ flannel_image_repo }}:{{ flannel_image_tag }}
+ imagePullPolicy: {{ k8s_image_pull_policy }}
+ command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ]
+ securityContext:
+ privileged: true
+ env:
+ - name: POD_NAME
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.name
+ - name: POD_NAMESPACE
+ valueFrom:
+ fieldRef:
+ fieldPath: metadata.namespace
+ volumeMounts:
+ - name: run
+ mountPath: /run
+ - name: flannel-cfg
+ mountPath: /etc/kube-flannel/
+ volumes:
+ - name: run
+ hostPath:
+ path: /run
+ - name: flannel-cfg
+ configMap:
+ name: kube-flannel-cfg
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+ name: flannel-conf
+ namespace: default
+plugin: flannel
+args: '[
+ {
+ "masterplugin": true,
+ "delegate": {
+ "isDefaultGateway": true
+ }
+ }
+]'
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+ name: sriov-conf1
+ namespace: default
+plugin: sriov
+args: '[
+ {
+ "master": "eth1.101",
+ "pfOnly": true,
+ "ipam": {
+ "type": "host-local",
+ "subnet": "192.168.123.0/24",
+ "rangeStart": "192.168.123.11",
+ "rangeEnd": "192.168.123.21",
+ "routes": [
+ {
+ "dst": "0.0.0.0/0"
+ }
+ ],
+ "gateway": "192.168.123.1"
+ }
+ }
+]'
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+ name: sriov-conf2
+ namespace: default
+plugin: sriov
+args: '[
+ {
+ "master": "eth1.101",
+ "pfOnly": true,
+ "ipam": {
+ "type": "host-local",
+ "subnet": "192.168.123.0/24",
+ "rangeStart": "192.168.123.31",
+ "rangeEnd": "192.168.123.41",
+ "routes": [
+ {
+ "dst": "0.0.0.0/0"
+ }
+ ],
+ "gateway": "192.168.123.1"
+ }
+ }
+]'
diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml
new file mode 100644
index 00000000..849aca85
--- /dev/null
+++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml
@@ -0,0 +1,51 @@
+# Copyright (C) 2018, ARM Limited and contributors.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: multus-test1
+ annotations:
+ networks: '[
+ { "name": "flannel-conf" },
+ { "name": "sriov-conf1" }
+ ]'
+spec:
+ containers:
+ - name: multus-test
+ image: "busybox"
+ command: ["top"]
+ stdin: true
+ tty: true
+ nodeSelector:
+ kubernetes.io/hostname: "host1"
+ tolerations:
+ - key: "node-role.kubernetes.io/master"
+ operator: "Exists"
+ effect: "NoSchedule"
+---
+apiVersion: v1
+kind: Pod
+metadata:
+ name: multus-test2
+ annotations:
+ networks: '[
+ { "name": "flannel-conf" },
+ { "name": "sriov-conf2" }
+ ]'
+spec:
+ containers:
+ - name: multus-test
+ image: "busybox"
+ command: ["top"]
+ stdin: true
+ tty: true
+ nodeSelector:
+ kubernetes.io/hostname: "host2"
+ tolerations:
+ - key: "node-role.kubernetes.io/master"
+ operator: "Exists"
+ effect: "NoSchedule"