diff options
Diffstat (limited to 'src/arm/cni-deploy/roles/vhost-vpp')
9 files changed, 265 insertions, 0 deletions
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/0001-net-virtio-ethdev.patch b/src/arm/cni-deploy/roles/vhost-vpp/files/0001-net-virtio-ethdev.patch new file mode 100644 index 0000000..171ff4d --- /dev/null +++ b/src/arm/cni-deploy/roles/vhost-vpp/files/0001-net-virtio-ethdev.patch @@ -0,0 +1,16 @@ +diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c +index e320811..c1b1640 100644 +--- a/drivers/net/virtio/virtio_ethdev.c ++++ b/drivers/net/virtio/virtio_ethdev.c +@@ -1754,6 +1754,11 @@ virtio_dev_start(struct rte_eth_dev *dev) + virtqueue_notify(rxvq->vq); + } + ++ for (i = 0; i < dev->data->nb_tx_queues; i++) { ++ txvq = dev->data->tx_queues[i]; ++ virtqueue_notify(txvq->vq); ++ } ++ + PMD_INIT_LOG(DEBUG, "Notified backend at initialization"); + + for (i = 0; i < dev->data->nb_rx_queues; i++) { diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/Dockerfile.vpp1710-dpdk1708 b/src/arm/cni-deploy/roles/vhost-vpp/files/Dockerfile.vpp1710-dpdk1708 new file mode 100644 index 0000000..2f83534 --- /dev/null +++ b/src/arm/cni-deploy/roles/vhost-vpp/files/Dockerfile.vpp1710-dpdk1708 @@ -0,0 +1,24 @@ +FROM ubuntu:xenial + +RUN apt-get update && \ + apt-get install -y git make openssl libcrypto++-dev libnuma-dev && \ + apt-get autoclean + +RUN git clone https://gerrit.fd.io/r/vpp -b stable/1710 /root/vpp-1710 + +WORKDIR /root/vpp-1710 +COPY ./0001-net-virtio-ethdev.patch dpdk/dpdk-17.08_patches/0001-net-virtio-ethdev.patch +RUN sed -i "s/sudo -E //g" Makefile +RUN make UNATTENDED=yes install-dep + +WORKDIR /root/vpp-1710/build-root +RUN ./bootstrap.sh +RUN make PLATFORM=vpp TAG=vpp_debug vpp-install +RUN mkdir -p /etc/vpp && \ + cp /root/vpp-1710/src/vpp/conf/startup.conf /etc/vpp/startup.conf && \ + cp /root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin/* /usr/bin && \ + ln -s /root/vpp-1710/build-root/install-vpp_debug-native/vpp/lib64/vpp_plugins /usr/lib/vpp_plugins +RUN groupadd vpp + +ENV PATH "$PATH:/root/vpp-1710/build-root/install-vpp_debug-native/dpdk/bin" +ENV PATH "$PATH:/root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin" diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/setvpp.sh b/src/arm/cni-deploy/roles/vhost-vpp/files/setvpp.sh new file mode 100755 index 0000000..15b0d27 --- /dev/null +++ b/src/arm/cni-deploy/roles/vhost-vpp/files/setvpp.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -x + +cid=`sed -ne '/hostname/p' /proc/1/task/1/mountinfo | awk -F '/' '{print $6}'` +cid_s=${cid:0:12} +filename=${cid_s}-net1.json +ifstring=`cat /vhost-user-net-plugin/${cid}/${cid_s}-net1.json | awk -F ',' '{print $4}'` +ifmac=`echo ${ifstring} | awk -F '\"' '{print $4}'` + +ipstr=$(cat /vhost-user-net-plugin/${cid}/${cid_s}-net1-ip4.conf |grep "ipAddr") +ipaddr=$(echo $ipstr | awk -F '\"' '{print $4}') +ipaddr1=$(echo $ipaddr | cut -d / -f 1) + +vdev_str="vdev virtio_user0,path=/vhost-user-net-plugin/$cid/$cid_s-net1,mac=$ifmac" + +sed -i.bak '/# dpdk/a\dpdk \{' /etc/vpp/startup.conf +sed -i.bak "/# vdev eth_bond1,mode=1/a\\$vdev_str" /etc/vpp/startup.conf +sed -i.bak '/# socket-mem/a\\}' /etc/vpp/startup.conf + +vpp -c /etc/vpp/startup.conf & + +sleep 40 + +vppctl set int state VirtioUser0/0/0 up +vppctl set int ip address VirtioUser0/0/0 ${ipaddr1}/24 +vppctl show int +vppctl show int address + +echo ${ipaddr1} > /vhost-user-net-plugin/$(hostname) diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/startup.conf b/src/arm/cni-deploy/roles/vhost-vpp/files/startup.conf new file mode 100644 index 0000000..ae86e38 --- /dev/null +++ b/src/arm/cni-deploy/roles/vhost-vpp/files/startup.conf @@ -0,0 +1,21 @@ +unix { + nodaemon + log /tmp/vpp.log + full-coredump + cli-listen /run/vpp/cli.sock + gid vpp +} +api-trace { + on +} +api-segment { + gid vpp +} +cpu { + main-core 1 + corelist-workers 2-3 + workers 2 +} +dpdk { + uio-driver vfio-pci +} diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/vhostuser-obj.yml b/src/arm/cni-deploy/roles/vhost-vpp/files/vhostuser-obj.yml new file mode 100644 index 0000000..1e9bc66 --- /dev/null +++ b/src/arm/cni-deploy/roles/vhost-vpp/files/vhostuser-obj.yml @@ -0,0 +1,28 @@ +--- +apiVersion: "kubernetes.com/v1" +kind: Network +metadata: + name: vhostuser-networkobj +plugin: vhostuser +args: '[ + { + "type": "vhostuser", + "name": "vhostuser-network", + "if0name": "net1", + "vhost": { + "vhost_tool": "/opt/cni/bin/vpp-config.py" + }, + "ipam": { + "type": "host-local", + "subnet": "10.56.217.0/24", + "rangeStart": "10.56.217.131", + "rangeEnd": "10.56.217.190", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "gateway": "10.56.217.1" + } + } +]' diff --git a/src/arm/cni-deploy/roles/vhost-vpp/tasks/crd.yml b/src/arm/cni-deploy/roles/vhost-vpp/tasks/crd.yml new file mode 100644 index 0000000..ad36c90 --- /dev/null +++ b/src/arm/cni-deploy/roles/vhost-vpp/tasks/crd.yml @@ -0,0 +1,13 @@ +--- +- name: Copy Vhostuser yaml + copy: + src: vhostuser-obj.yml + dest: /tmp/vhostuser-obj.yml + +- name: Copy VPP testpod template + template: + src: vpp-testpod.yml.j2 + dest: /root/vpp-testpod.yml + +- name: Create Vhostuser network object + shell: kubectl apply -f /tmp/vhostuser-obj.yml diff --git a/src/arm/cni-deploy/roles/vhost-vpp/tasks/main.yml b/src/arm/cni-deploy/roles/vhost-vpp/tasks/main.yml new file mode 100644 index 0000000..df890ea --- /dev/null +++ b/src/arm/cni-deploy/roles/vhost-vpp/tasks/main.yml @@ -0,0 +1,18 @@ +--- +- name: Build Vhostuser CNI + shell: > + docker run --rm --network host -v /opt/cni/bin:/opt/cni/bin golang:1.9 + bash -c "git clone {{ vhostuser_repo }} vhostuser_cni && cd vhostuser_cni + && git checkout {{ vhostuser_commit }} && ./build + && cp bin/vhostuser /opt/cni/bin/ + && cp tests/vpp-config-debug.py /opt/cni/bin/vpp-config.py" + args: + creates: /opt/cni/bin/vhostuser + +- name: Import CRD task + import_tasks: crd.yml + when: inventory_hostname == groups["kube-master"][0] + +- name: Import VPP task + import_tasks: vpp.yml + when: inventory_hostname in groups["kube-node"] diff --git a/src/arm/cni-deploy/roles/vhost-vpp/tasks/vpp.yml b/src/arm/cni-deploy/roles/vhost-vpp/tasks/vpp.yml new file mode 100644 index 0000000..7f5be05 --- /dev/null +++ b/src/arm/cni-deploy/roles/vhost-vpp/tasks/vpp.yml @@ -0,0 +1,47 @@ +--- +- name: Create dest directories + file: + path: "{{ item }}" + state: directory + with_items: + - /tmp/vpp1710/ + - /var/lib/cni/vhostuser/ + - /etc/vpp/ + +- name: Copy VPP files + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + with_items: + - {src: "Dockerfile.vpp1710-dpdk1708", dest: "/tmp/vpp1710/Dockerfile"} + - {src: "0001-net-virtio-ethdev.patch", dest: "/tmp/vpp1710/0001-net-virtio-ethdev.patch"} + - {src: "setvpp.sh", dest: "/var/lib/cni/vhostuser/setvpp.sh"} + - {src: "startup.conf", dest: "/etc/vpp/startup.conf"} + +- name: Check if VPP image exists + shell: docker inspect --type=image vpp-1710:virtio-patched > /dev/null 2>&1 + ignore_errors: "yes" + register: check_vpp + +- name: Building VPP container. Be patient... + shell: docker build -t vpp-1710:virtio-patched --network host . + args: + chdir: /tmp/vpp1710/ + when: check_vpp is failed + +- name: Copy VPP binaries to host + shell: > + docker run --rm -v /root/vpp-1710/build-root:/root/vpp-host vpp-1710:virtio-patched + /bin/cp -a /root/vpp-1710/build-root/install-vpp_debug-native /root/vpp-host + && /bin/cp /root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin/* /usr/bin + && /bin/rm -rf /usr/lib/vpp_plugins + && ln -s /root/vpp-1710/build-root/install-vpp_debug-native/vpp/lib64/vpp_plugins /usr/lib/vpp_plugins + && (groupadd vpp || true) + +- name: Copy libcrypto.so.1.0.0 for CentOS + shell: > + docker run --rm -v /usr/lib64:/root/lib64-centos vpp-1710:virtio-patched + /bin/cp /lib/aarch64-linux-gnu/libcrypto.so.1.0.0 /root/lib64-centos/ + args: + creates: /usr/lib64/libcrypto.so.1.0.0 + when: ansible_os_family == "RedHat" diff --git a/src/arm/cni-deploy/roles/vhost-vpp/templates/vpp-testpod.yml.j2 b/src/arm/cni-deploy/roles/vhost-vpp/templates/vpp-testpod.yml.j2 new file mode 100644 index 0000000..2efd4e0 --- /dev/null +++ b/src/arm/cni-deploy/roles/vhost-vpp/templates/vpp-testpod.yml.j2 @@ -0,0 +1,68 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: vpp-test1 + annotations: + networks: '[ + { "name": "flannel-networkobj" }, + { "name": "vhostuser-networkobj" } + ]' +spec: + containers: + - name: vpp-test1 + image: vpp-1710:virtio-patched + imagePullPolicy: "Never" + stdin: true + terminationMessagePath: /dev/termination-log + tty: true + securityContext: + privileged: true + volumeMounts: + - mountPath: /vhost-user-net-plugin + name: vhost-user-net-plugin + - mountPath: /mnt/huge + name: huge + nodeSelector: + kubernetes.io/hostname: "{{ groups['kube-node'][0] }}" + volumes: + - name: vhost-user-net-plugin + hostPath: + path: /var/lib/cni/vhostuser + - name: huge + hostPath: + path: /mnt/huge +--- +apiVersion: v1 +kind: Pod +metadata: + name: vpp-test2 + annotations: + networks: '[ + { "name": "flannel-networkobj" }, + { "name": "vhostuser-networkobj" } + ]' +spec: + containers: + - name: vpp-test2 + image: vpp-1710:virtio-patched + imagePullPolicy: "Never" + stdin: true + terminationMessagePath: /dev/termination-log + tty: true + securityContext: + privileged: true + volumeMounts: + - mountPath: /vhost-user-net-plugin + name: vhost-user-net-plugin + - mountPath: /mnt/huge + name: huge + nodeSelector: + kubernetes.io/hostname: "{{ groups['kube-node'][0] }}" + volumes: + - name: vhost-user-net-plugin + hostPath: + path: /var/lib/cni/vhostuser + - name: huge + hostPath: + path: /mnt/huge |