From 2e2a9c8f10c0b320dd621a86237707acb3441a21 Mon Sep 17 00:00:00 2001
From: Yibo Cai <yibo.cai@arm.com>
Date: Tue, 31 Jul 2018 13:08:38 +0800
Subject: src/arm: add k8s cni deployment tasks

Deploy Multus, SRIOV, Vhostuser(VPP+DPDK) with CRD.

Change-Id: I787d5fd61c75f17d50fbaf8f86c08bdc44c557b8
Signed-off-by: Yibo Cai <yibo.cai@arm.com>
---
 .../vhost-vpp/files/0001-net-virtio-ethdev.patch   | 16 +++++
 .../vhost-vpp/files/Dockerfile.vpp1710-dpdk1708    | 24 ++++++++
 src/arm/cni-deploy/roles/vhost-vpp/files/setvpp.sh | 30 ++++++++++
 .../cni-deploy/roles/vhost-vpp/files/startup.conf  | 21 +++++++
 .../roles/vhost-vpp/files/vhostuser-obj.yml        | 28 +++++++++
 src/arm/cni-deploy/roles/vhost-vpp/tasks/crd.yml   | 13 +++++
 src/arm/cni-deploy/roles/vhost-vpp/tasks/main.yml  | 18 ++++++
 src/arm/cni-deploy/roles/vhost-vpp/tasks/vpp.yml   | 47 +++++++++++++++
 .../roles/vhost-vpp/templates/vpp-testpod.yml.j2   | 68 ++++++++++++++++++++++
 9 files changed, 265 insertions(+)
 create mode 100644 src/arm/cni-deploy/roles/vhost-vpp/files/0001-net-virtio-ethdev.patch
 create mode 100644 src/arm/cni-deploy/roles/vhost-vpp/files/Dockerfile.vpp1710-dpdk1708
 create mode 100755 src/arm/cni-deploy/roles/vhost-vpp/files/setvpp.sh
 create mode 100644 src/arm/cni-deploy/roles/vhost-vpp/files/startup.conf
 create mode 100644 src/arm/cni-deploy/roles/vhost-vpp/files/vhostuser-obj.yml
 create mode 100644 src/arm/cni-deploy/roles/vhost-vpp/tasks/crd.yml
 create mode 100644 src/arm/cni-deploy/roles/vhost-vpp/tasks/main.yml
 create mode 100644 src/arm/cni-deploy/roles/vhost-vpp/tasks/vpp.yml
 create mode 100644 src/arm/cni-deploy/roles/vhost-vpp/templates/vpp-testpod.yml.j2

(limited to 'src/arm/cni-deploy/roles/vhost-vpp')

diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/0001-net-virtio-ethdev.patch b/src/arm/cni-deploy/roles/vhost-vpp/files/0001-net-virtio-ethdev.patch
new file mode 100644
index 0000000..171ff4d
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/files/0001-net-virtio-ethdev.patch
@@ -0,0 +1,16 @@
+diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
+index e320811..c1b1640 100644
+--- a/drivers/net/virtio/virtio_ethdev.c
++++ b/drivers/net/virtio/virtio_ethdev.c
+@@ -1754,6 +1754,11 @@ virtio_dev_start(struct rte_eth_dev *dev)
+ 		virtqueue_notify(rxvq->vq);
+ 	}
+ 
++	for (i = 0; i < dev->data->nb_tx_queues; i++) {
++		txvq = dev->data->tx_queues[i];
++		virtqueue_notify(txvq->vq);
++	}
++
+ 	PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
+ 
+ 	for (i = 0; i < dev->data->nb_rx_queues; i++) {
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/Dockerfile.vpp1710-dpdk1708 b/src/arm/cni-deploy/roles/vhost-vpp/files/Dockerfile.vpp1710-dpdk1708
new file mode 100644
index 0000000..2f83534
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/files/Dockerfile.vpp1710-dpdk1708
@@ -0,0 +1,24 @@
+FROM ubuntu:xenial
+
+RUN apt-get update && \
+    apt-get install -y git make openssl libcrypto++-dev libnuma-dev && \
+    apt-get autoclean
+
+RUN git clone https://gerrit.fd.io/r/vpp -b stable/1710 /root/vpp-1710
+
+WORKDIR /root/vpp-1710
+COPY ./0001-net-virtio-ethdev.patch dpdk/dpdk-17.08_patches/0001-net-virtio-ethdev.patch
+RUN sed -i "s/sudo -E //g" Makefile
+RUN make UNATTENDED=yes install-dep
+
+WORKDIR /root/vpp-1710/build-root
+RUN ./bootstrap.sh
+RUN make PLATFORM=vpp TAG=vpp_debug vpp-install
+RUN mkdir -p /etc/vpp && \
+    cp /root/vpp-1710/src/vpp/conf/startup.conf /etc/vpp/startup.conf && \
+    cp /root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin/* /usr/bin && \
+    ln -s /root/vpp-1710/build-root/install-vpp_debug-native/vpp/lib64/vpp_plugins /usr/lib/vpp_plugins
+RUN groupadd vpp
+
+ENV PATH "$PATH:/root/vpp-1710/build-root/install-vpp_debug-native/dpdk/bin"
+ENV PATH "$PATH:/root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin"
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/setvpp.sh b/src/arm/cni-deploy/roles/vhost-vpp/files/setvpp.sh
new file mode 100755
index 0000000..15b0d27
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/files/setvpp.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -x
+
+cid=`sed -ne '/hostname/p' /proc/1/task/1/mountinfo | awk -F '/' '{print $6}'`
+cid_s=${cid:0:12}
+filename=${cid_s}-net1.json
+ifstring=`cat /vhost-user-net-plugin/${cid}/${cid_s}-net1.json | awk -F ',' '{print $4}'`
+ifmac=`echo ${ifstring} | awk -F '\"' '{print $4}'`
+
+ipstr=$(cat /vhost-user-net-plugin/${cid}/${cid_s}-net1-ip4.conf  |grep "ipAddr")
+ipaddr=$(echo $ipstr | awk -F '\"' '{print $4}')
+ipaddr1=$(echo $ipaddr | cut -d / -f 1)
+
+vdev_str="vdev virtio_user0,path=/vhost-user-net-plugin/$cid/$cid_s-net1,mac=$ifmac"
+
+sed -i.bak '/# dpdk/a\dpdk \{' /etc/vpp/startup.conf
+sed -i.bak "/# vdev eth_bond1,mode=1/a\\$vdev_str" /etc/vpp/startup.conf
+sed -i.bak '/# socket-mem/a\\}' /etc/vpp/startup.conf
+
+vpp -c /etc/vpp/startup.conf &
+
+sleep 40
+
+vppctl set int state VirtioUser0/0/0 up
+vppctl set int ip address VirtioUser0/0/0 ${ipaddr1}/24
+vppctl show int
+vppctl show int address
+
+echo ${ipaddr1} > /vhost-user-net-plugin/$(hostname)
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/startup.conf b/src/arm/cni-deploy/roles/vhost-vpp/files/startup.conf
new file mode 100644
index 0000000..ae86e38
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/files/startup.conf
@@ -0,0 +1,21 @@
+unix {
+  nodaemon
+  log /tmp/vpp.log
+  full-coredump
+  cli-listen /run/vpp/cli.sock
+  gid vpp
+}
+api-trace {
+  on
+}
+api-segment {
+  gid vpp
+}
+cpu {
+  main-core 1
+  corelist-workers 2-3
+  workers 2
+}
+dpdk {
+  uio-driver vfio-pci
+}
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/vhostuser-obj.yml b/src/arm/cni-deploy/roles/vhost-vpp/files/vhostuser-obj.yml
new file mode 100644
index 0000000..1e9bc66
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/files/vhostuser-obj.yml
@@ -0,0 +1,28 @@
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: vhostuser-networkobj
+plugin: vhostuser
+args: '[
+  {
+    "type": "vhostuser",
+    "name": "vhostuser-network",
+    "if0name": "net1",
+    "vhost": {
+      "vhost_tool": "/opt/cni/bin/vpp-config.py"
+    },
+    "ipam": {
+      "type": "host-local",
+      "subnet": "10.56.217.0/24",
+      "rangeStart": "10.56.217.131",
+      "rangeEnd": "10.56.217.190",
+      "routes": [
+        {
+          "dst": "0.0.0.0/0"
+        }
+      ],
+      "gateway": "10.56.217.1"
+    }
+  }
+]'
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/tasks/crd.yml b/src/arm/cni-deploy/roles/vhost-vpp/tasks/crd.yml
new file mode 100644
index 0000000..ad36c90
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/tasks/crd.yml
@@ -0,0 +1,13 @@
+---
+- name: Copy Vhostuser yaml
+  copy:
+    src: vhostuser-obj.yml
+    dest: /tmp/vhostuser-obj.yml
+
+- name: Copy VPP testpod template
+  template:
+    src: vpp-testpod.yml.j2
+    dest: /root/vpp-testpod.yml
+
+- name: Create Vhostuser network object
+  shell: kubectl apply -f /tmp/vhostuser-obj.yml
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/tasks/main.yml b/src/arm/cni-deploy/roles/vhost-vpp/tasks/main.yml
new file mode 100644
index 0000000..df890ea
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/tasks/main.yml
@@ -0,0 +1,18 @@
+---
+- name: Build Vhostuser CNI
+  shell: >
+    docker run --rm --network host -v /opt/cni/bin:/opt/cni/bin golang:1.9
+    bash -c "git clone {{ vhostuser_repo }} vhostuser_cni && cd vhostuser_cni
+    && git checkout {{ vhostuser_commit }} && ./build
+    && cp bin/vhostuser /opt/cni/bin/
+    && cp tests/vpp-config-debug.py /opt/cni/bin/vpp-config.py"
+  args:
+    creates: /opt/cni/bin/vhostuser
+
+- name: Import CRD task
+  import_tasks: crd.yml
+  when: inventory_hostname == groups["kube-master"][0]
+
+- name: Import VPP task
+  import_tasks: vpp.yml
+  when: inventory_hostname in groups["kube-node"]
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/tasks/vpp.yml b/src/arm/cni-deploy/roles/vhost-vpp/tasks/vpp.yml
new file mode 100644
index 0000000..7f5be05
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/tasks/vpp.yml
@@ -0,0 +1,47 @@
+---
+- name: Create dest directories
+  file:
+    path: "{{ item }}"
+    state: directory
+  with_items:
+    - /tmp/vpp1710/
+    - /var/lib/cni/vhostuser/
+    - /etc/vpp/
+
+- name: Copy VPP files
+  copy:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+  with_items:
+    - {src: "Dockerfile.vpp1710-dpdk1708", dest: "/tmp/vpp1710/Dockerfile"}
+    - {src: "0001-net-virtio-ethdev.patch", dest: "/tmp/vpp1710/0001-net-virtio-ethdev.patch"}
+    - {src: "setvpp.sh", dest: "/var/lib/cni/vhostuser/setvpp.sh"}
+    - {src: "startup.conf", dest: "/etc/vpp/startup.conf"}
+
+- name: Check if VPP image exists
+  shell: docker inspect --type=image vpp-1710:virtio-patched > /dev/null 2>&1
+  ignore_errors: "yes"
+  register: check_vpp
+
+- name: Building VPP container. Be patient...
+  shell: docker build -t vpp-1710:virtio-patched --network host .
+  args:
+    chdir: /tmp/vpp1710/
+  when: check_vpp is failed
+
+- name: Copy VPP binaries to host
+  shell: >
+    docker run --rm -v /root/vpp-1710/build-root:/root/vpp-host vpp-1710:virtio-patched
+    /bin/cp -a /root/vpp-1710/build-root/install-vpp_debug-native /root/vpp-host
+    && /bin/cp /root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin/* /usr/bin
+    && /bin/rm -rf /usr/lib/vpp_plugins
+    && ln -s /root/vpp-1710/build-root/install-vpp_debug-native/vpp/lib64/vpp_plugins /usr/lib/vpp_plugins
+    && (groupadd vpp || true)
+
+- name: Copy libcrypto.so.1.0.0 for CentOS
+  shell: >
+    docker run --rm -v /usr/lib64:/root/lib64-centos vpp-1710:virtio-patched
+    /bin/cp /lib/aarch64-linux-gnu/libcrypto.so.1.0.0 /root/lib64-centos/
+  args:
+    creates: /usr/lib64/libcrypto.so.1.0.0
+  when: ansible_os_family == "RedHat"
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/templates/vpp-testpod.yml.j2 b/src/arm/cni-deploy/roles/vhost-vpp/templates/vpp-testpod.yml.j2
new file mode 100644
index 0000000..2efd4e0
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/templates/vpp-testpod.yml.j2
@@ -0,0 +1,68 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vpp-test1
+  annotations:
+    networks: '[
+        { "name": "flannel-networkobj" },
+        { "name": "vhostuser-networkobj" }
+    ]'
+spec:
+  containers:
+    - name: vpp-test1
+      image: vpp-1710:virtio-patched
+      imagePullPolicy: "Never"
+      stdin: true
+      terminationMessagePath: /dev/termination-log
+      tty: true
+      securityContext:
+        privileged: true
+      volumeMounts:
+        - mountPath: /vhost-user-net-plugin
+          name: vhost-user-net-plugin
+        - mountPath: /mnt/huge
+          name: huge
+  nodeSelector:
+    kubernetes.io/hostname: "{{ groups['kube-node'][0] }}"
+  volumes:
+    - name: vhost-user-net-plugin
+      hostPath:
+        path: /var/lib/cni/vhostuser
+    - name: huge
+      hostPath:
+        path: /mnt/huge
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vpp-test2
+  annotations:
+    networks: '[
+        { "name": "flannel-networkobj" },
+        { "name": "vhostuser-networkobj" }
+    ]'
+spec:
+  containers:
+    - name: vpp-test2
+      image: vpp-1710:virtio-patched
+      imagePullPolicy: "Never"
+      stdin: true
+      terminationMessagePath: /dev/termination-log
+      tty: true
+      securityContext:
+        privileged: true
+      volumeMounts:
+        - mountPath: /vhost-user-net-plugin
+          name: vhost-user-net-plugin
+        - mountPath: /mnt/huge
+          name: huge
+  nodeSelector:
+    kubernetes.io/hostname: "{{ groups['kube-node'][0] }}"
+  volumes:
+    - name: vhost-user-net-plugin
+      hostPath:
+        path: /var/lib/cni/vhostuser
+    - name: huge
+      hostPath:
+        path: /mnt/huge
-- 
cgit 1.2.3-korg