diff options
Diffstat (limited to 'deploy/adapters')
12 files changed, 546 insertions, 17 deletions
diff --git a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml index 6d947623..b9d9c234 100644 --- a/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/kargo/tasks/main.yml @@ -217,6 +217,51 @@ when: - stor4nfv is defined and stor4nfv == "Enable" +- name: copy sriov playbook to kargo + copy: + src: "{{ run_dir }}/roles/sriov" + dest: /opt/kargo_k8s/roles/network_plugin + +- name: copy sriov-apps playbook to kargo + copy: + src: "{{ run_dir }}/roles/sriov-apps/" + dest: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/sriov + +- name: append sriov to network plugin + blockinfile: + path: /opt/kargo_k8s/roles/network_plugin/meta/main.yml + block: " - role: network_plugin/sriov\n when: kube_network_plugin == 'sriov'\n \ + tags: sriov\n" + +- name: append sriov apps to network plugin + blockinfile: + path: /opt/kargo_k8s/roles/kubernetes-apps/network_plugin/meta/main.yml + block: " - role: kubernetes-apps/network_plugin/sriov\n \ + when: kube_network_plugin == 'sriov'\n tags: sriov\n" + +- name: append sriov to valid kube_network_plugin list + replace: + path: "{{ item.path }}" + regexp: "{{ item.regexp }}" + replace: "{{ item.replace }}" + with_items: + - {path: "/opt/kargo_k8s/roles/kubernetes/master/templates/manifests/\ +kube-controller-manager.manifest.j2", + regexp: '"cloud", "flannel"', + replace: '"cloud", "flannel", "sriov"'} + - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.kubeadm.env.j2', + regexp: '"calico", "canal", "flannel", "weave"', + replace: '"calico", "canal", "flannel", "weave", "sriov"'} + - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.standard.env.j2', + regexp: '"calico", "canal", "flannel", "weave"', + replace: '"calico", "canal", "flannel", "weave", "sriov"'} + - {path: '/opt/kargo_k8s/roles/kubernetes/node/templates/kubelet.rkt.service.j2', + regexp: '"calico", "weave", "canal", "flannel"', + replace: '"calico", "weave", "canal", "flannel", "sriov"'} + - {path: '/opt/kargo_k8s/roles/kubernetes/preinstall/tasks/main.yml', + regexp: '"calico", "weave", "canal", "flannel"', + replace: '"calico", "weave", "canal", "flannel", "sriov"'} + - name: run kargo playbook shell: | cd /opt/kargo_k8s diff --git a/deploy/adapters/ansible/kubernetes/roles/pre-k8s/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/pre-k8s/vars/main.yml index b196bd25..6d6ecf44 100644 --- a/deploy/adapters/ansible/kubernetes/roles/pre-k8s/vars/main.yml +++ b/deploy/adapters/ansible/kubernetes/roles/pre-k8s/vars/main.yml @@ -1,21 +1,21 @@ --- aptpackages: -- bridge-utils -- debootstrap -- ifenslave -- ifenslave-2.6 -- lsof -- lvm2 -- ntp -- ntpdate -- sudo -- vlan -- tcpdump + - bridge-utils + - debootstrap + - ifenslave + - ifenslave-2.6 + - lsof + - lvm2 + - ntp + - ntpdate + - sudo + - vlan + - tcpdump yumpackages: -- bridge-utils -- iputils -- lvm2 -- ntp -- tcpdump -- vim + - bridge-utils + - iputils + - lvm2 + - ntp + - tcpdump + - vim diff --git a/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/tasks/RedHat.yml b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/tasks/RedHat.yml index c59fdfc5..5b434dbe 100644 --- a/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/tasks/RedHat.yml +++ b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/tasks/RedHat.yml @@ -36,6 +36,17 @@ src: ifcfg-eth.j2 dest: /etc/sysconfig/network-scripts/ifcfg-{{sys_intf_mappings["external"]["interface"]}} +- name: generate ifcfg-sriov + template: + src: ifcfg-sriov.j2 + dest: /etc/sysconfig/network-scripts/ifcfg-{{ intf_sriov }} + when: intf_sriov|length > 0 + +- name: remove ifcfg-br-sriov script + file: + path: /etc/sysconfig/network-scripts/ifcfg-br-sriov + state: absent + - name: remove defualt gw lineinfile: dest: /etc/sysconfig/network diff --git a/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/templates/ifcfg-sriov.j2 b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/templates/ifcfg-sriov.j2 new file mode 100644 index 00000000..c4005854 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/templates/ifcfg-sriov.j2 @@ -0,0 +1,12 @@ +DEVICE={{ intf_sriov }} +BOOTPROTO=none +ONBOOT=yes +IPADDR={{ ip_settings[inventory_hostname]["tenant"]["ip"] }} +NETMASK=255.255.255.0 +DEFROUTE="no" +{% if sys_intf_mappings["tenant"]["vlan_tag"] | int %} +{% set intf_vlan = "yes" %} +{% else %} +{% set intf_vlan = "no" %} +{% endif %} +VLAN={{ intf_vlan }} diff --git a/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/vars/main.yml b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/vars/main.yml new file mode 100644 index 00000000..e525bf5c --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/setup-k8s-network/vars/main.yml @@ -0,0 +1,7 @@ +--- +intf_sriov: |- + {%- set intf_sriov = sys_intf_mappings["tenant"]["interface"] %} + {%- if sys_intf_mappings["tenant"]["vlan_tag"] | int %} + {%- set intf_sriov = intf_sriov + '.' + sys_intf_mappings["tenant"]["vlan_tag"]|string %} + {%- endif %} + {{- intf_sriov }} diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov-apps/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov-apps/tasks/main.yml new file mode 100644 index 00000000..662fa7bf --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov-apps/tasks/main.yml @@ -0,0 +1,20 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +- name: "Sriov | Create ServiceAccount ClusterRole and ClusterRoleBinding" + command: "{{ bin_dir }}/kubectl apply -f {{ kube_config_dir }}/cni-sriov-rbac.yml" + run_once: true + when: rbac_enabled and sriov_rbac_manifest.changed + +- name: Sriov | Create Network Resources + kube: + name: "kube-sriov" + kubectl: "{{ bin_dir }}/kubectl" + filename: "{{ kube_config_dir }}/cni-sriov.yml" + namespace: "{{system_namespace}}" + state: "{{ item | ternary('latest','present') }}" + with_items: "{{ sriov_manifest.changed }}" + when: inventory_hostname == groups['kube-master'][0] diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml new file mode 100644 index 00000000..44263956 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/defaults/main.yml @@ -0,0 +1,7 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +# Limits for apps diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml new file mode 100644 index 00000000..221279b1 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/handlers/main.yml @@ -0,0 +1,62 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +- name: Sriov | delete default docker bridge + command: ip link delete docker0 + failed_when: false + notify: Sriov | restart docker + +# special cases for atomic because it defaults to live-restore: true +# So we disable live-restore to pickup the new flannel IP. After +# we enable it, we have to restart docker again to pickup the new +# setting and restore the original behavior +- name: Sriov | restart docker + command: /bin/true + notify: + - Sriov | reload systemd + - Sriov | reload docker.socket + - Sriov | configure docker live-restore true (atomic) + - Sriov | reload docker + - Sriov | pause while Docker restarts + - Sriov | wait for docker + +- name: Sriov | reload systemd + shell: systemctl daemon-reload + +- name: Sriov | reload docker.socket + service: + name: docker.socket + state: restarted + when: ansible_os_family in ['CoreOS', 'Container Linux by CoreOS'] + +- name: Sriov | configure docker live-restore true (atomic) + replace: + name: /etc/docker/daemon.json + regexp: '"live-restore":.*true' + replace: '"live-restore": false' + when: is_atomic + +- name: Sriov | reload docker + service: + name: docker + state: restarted + +- name: Sriov | pause while Docker restarts + pause: + seconds: 10 + prompt: "Waiting for docker restart" + +- name: Sriov | wait for docker + command: "{{ docker_bin_dir }}/docker images" + register: docker_ready + retries: 10 + delay: 5 + until: docker_ready.rc == 0 + +- name: Sriov | reload kubelet + service: + name: kubelet + state: restarted diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml new file mode 100644 index 00000000..0e3e2f6d --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/tasks/main.yml @@ -0,0 +1,106 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +- name: Sriov | Verify if br_netfilter module exists + shell: "modinfo br_netfilter" + register: modinfo_br_netfilter + failed_when: modinfo_br_netfilter.rc not in [0, 1] + changed_when: false + +- name: Sriov | Enable br_netfilter module + modprobe: + name: br_netfilter + state: present + when: modinfo_br_netfilter.rc == 0 + +# kube-proxy needs net.bridge.bridge-nf-call-iptables enabled +# when found if br_netfilter is not a module +- name: Sriov | Check if bridge-nf-call-iptables key exists + command: "sysctl net.bridge.bridge-nf-call-iptables" + failed_when: false + changed_when: false + register: sysctl_bridge_nf_call_iptables + +- name: Sriov | Enable bridge-nf-call tables + sysctl: + name: "{{ item }}" + state: present + value: 1 + reload: "yes" + when: modinfo_br_netfilter.rc == 1 and sysctl_bridge_nf_call_iptables.rc == 0 + with_items: + - net.bridge.bridge-nf-call-iptables + - net.bridge.bridge-nf-call-arptables + - net.bridge.bridge-nf-call-ip6tables + +- name: Sriov | Install Multus CNI + shell: |- + /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \ + bash -c "git clone https://github.com/Intel-Corp/multus-cni && cd multus-cni \ + && ./build && cp bin/multus /opt/cni/bin" + +- name: Sriov | Install Sriov CNI + shell: |- + /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/opt/cni/bin/ golang:1.9 \ + bash -c "git clone https://github.com/hustcat/sriov-cni && cd sriov-cni \ + && ./build && cp bin/sriov /opt/cni/bin" + +- name: Sriov | Install Flannel CNI + shell: |- + /usr/bin/docker run --rm --network=host -v /opt/cni/bin/:/host/opt/cni/bin/ \ + {{ flannel_cni_image_repo }}:{{ flannel_cni_image_tag }} \ + sh -c "cp /opt/cni/bin/* /host/opt/cni/bin/" + +- name: Sriov | Remove all file in /etc/cni/net.d + shell: |- + rm -rf /etc/cni/net.d/ + mkdir -p /etc/cni/net.d/ + +- name: Sriov | Generate Sriov CNI Conf + copy: + content: | + { + "name": "minion-cni-network", + "type": "multus", + "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml", + "delegates": [ + { + "type": "flannel", + "masterplugin": true, + "delegate": { + "isDefaultGateway": true + } + } + ] + } + dest: "/etc/cni/net.d/multus-cni.conf" + owner: root + group: root + mode: 0644 + +- name: Sriov | Enable DHCP CNI + shell: /opt/cni/bin/dhcp daemon & + + +- name: Sriov | Create cni-sriov-rbac manifest + template: + src: cni-sriov-rbac.yml.j2 + dest: "{{ kube_config_dir }}/cni-sriov-rbac.yml" + register: sriov_rbac_manifest + when: inventory_hostname == groups['kube-master'][0] and rbac_enabled + +- name: Sriov | Create cni-sriov manifest + template: + src: cni-sriov.yml.j2 + dest: "{{ kube_config_dir }}/cni-sriov.yml" + register: sriov_manifest + when: inventory_hostname == groups['kube-master'][0] + +- name: Sriov | Sriov tests manifest + template: + src: sriov-test-pod.yml + dest: "{{ kube_config_dir }}/sriov-test-pod.yml" + when: inventory_hostname == groups['kube-master'][0] diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2 new file mode 100644 index 00000000..1298aeaa --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov-rbac.yml.j2 @@ -0,0 +1,49 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sriov + namespace: "{{system_namespace}}" +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: sriov +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: sriov +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: sriov +subjects: +- kind: ServiceAccount + name: sriov + namespace: "{{system_namespace}}" diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2 b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2 new file mode 100644 index 00000000..90c7f28c --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/cni-sriov.yml.j2 @@ -0,0 +1,159 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +apiVersion: extensions/v1beta1 +kind: ThirdPartyResource +metadata: + name: network.kubernetes.com +description: "A specification of a Network obj in the kubernetes" +versions: +- name: v1 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flannel + namespace: {{system_namespace}} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-cfg + namespace: {{system_namespace}} + labels: + tier: node + app: flannel +data: + cni-conf.json: | + { + "name": "cbr0", + "type": "flannel", + "delegate": { + "isDefaultGateway": true + } + } + net-conf.json: | + { + "Network": "10.244.0.0/16", + "Backend": { + "Type": "udp" + } + } +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-flannel-ds + namespace: {{system_namespace}} + labels: + tier: node + app: flannel +spec: + template: + metadata: + labels: + tier: node + app: flannel + spec: + hostNetwork: true + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: flannel + containers: + - name: kube-flannel + image: {{ flannel_image_repo }}:{{ flannel_image_tag }} + imagePullPolicy: {{ k8s_image_pull_policy }} + command: [ "/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr" ] + securityContext: + privileged: true + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: run + mountPath: /run + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + - name: run + hostPath: + path: /run + - name: flannel-cfg + configMap: + name: kube-flannel-cfg +--- +apiVersion: "kubernetes.com/v1" +kind: Network +metadata: + name: flannel-conf + namespace: default +plugin: flannel +args: '[ + { + "masterplugin": true, + "delegate": { + "isDefaultGateway": true + } + } +]' +--- +apiVersion: "kubernetes.com/v1" +kind: Network +metadata: + name: sriov-conf1 + namespace: default +plugin: sriov +args: '[ + { + "master": "eth1.101", + "pfOnly": true, + "ipam": { + "type": "host-local", + "subnet": "192.168.123.0/24", + "rangeStart": "192.168.123.11", + "rangeEnd": "192.168.123.21", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "gateway": "192.168.123.1" + } + } +]' +--- +apiVersion: "kubernetes.com/v1" +kind: Network +metadata: + name: sriov-conf2 + namespace: default +plugin: sriov +args: '[ + { + "master": "eth1.101", + "pfOnly": true, + "ipam": { + "type": "host-local", + "subnet": "192.168.123.0/24", + "rangeStart": "192.168.123.31", + "rangeEnd": "192.168.123.41", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "gateway": "192.168.123.1" + } + } +]' diff --git a/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml new file mode 100644 index 00000000..849aca85 --- /dev/null +++ b/deploy/adapters/ansible/kubernetes/roles/sriov/templates/sriov-test-pod.yml @@ -0,0 +1,51 @@ +# Copyright (C) 2018, ARM Limited and contributors. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +--- +apiVersion: v1 +kind: Pod +metadata: + name: multus-test1 + annotations: + networks: '[ + { "name": "flannel-conf" }, + { "name": "sriov-conf1" } + ]' +spec: + containers: + - name: multus-test + image: "busybox" + command: ["top"] + stdin: true + tty: true + nodeSelector: + kubernetes.io/hostname: "host1" + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" +--- +apiVersion: v1 +kind: Pod +metadata: + name: multus-test2 + annotations: + networks: '[ + { "name": "flannel-conf" }, + { "name": "sriov-conf2" } + ]' +spec: + containers: + - name: multus-test + image: "busybox" + command: ["top"] + stdin: true + tty: true + nodeSelector: + kubernetes.io/hostname: "host2" + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" |