src/arm: add k8s cni deployment tasks

Deploy Multus, SRIOV, Vhostuser(VPP+DPDK) with CRD.

Change-Id: I787d5fd61c75f17d50fbaf8f86c08bdc44c557b8
Signed-off-by: Yibo Cai <yibo.cai@arm.com>

28 files changed, 673 insertions, 0 deletions

diff --git a/src/arm/cni-deploy/.gitignore b/src/arm/cni-deploy/.gitignore
new file mode 100644
index 0000000..a8b42eb
--- /dev/null
+++ b/src/arm/cni-deploy/.gitignore
@@ -0,0 +1 @@
+*.retry
diff --git a/src/arm/cni-deploy/deploy.yml b/src/arm/cni-deploy/deploy.yml
new file mode 100644
index 0000000..c54353a
--- /dev/null
+++ b/src/arm/cni-deploy/deploy.yml
@@ -0,0 +1,32 @@
+---
+- name: Fixup default flannel
+  hosts: kube-master
+  gather_facts: "no"
+  vars_files:
+    - "vars/global"
+  roles:
+    - {role: flannel, tags: [flannel]}
+
+- name: Deploy Multus CNI
+  hosts: all
+  gather_facts: "no"
+  vars_files:
+    - "vars/global"
+  roles:
+    - {role: multus, tags: [multus]}
+
+- name: Deploy SRIOV CNI
+  hosts: all
+  gather_facts: "no"
+  vars_files:
+    - "vars/global"
+  roles:
+    - {role: sriov, tags: [sriov]}
+
+- name: Deploy Vhostuser CNI and VPP
+  hosts: all
+  gather_facts: "yes"
+  vars_files:
+    - "vars/global"
+  roles:
+    - {role: vhost-vpp, tags: [vhost-vpp]}
diff --git a/src/arm/cni-deploy/inventory/inventory.cfg b/src/arm/cni-deploy/inventory/inventory.cfg
new file mode 100644
index 0000000..cd8bb25
--- /dev/null
+++ b/src/arm/cni-deploy/inventory/inventory.cfg
@@ -0,0 +1,18 @@
+# compass-tasks: /opt/kargo_k8s/inventory/inventory.cfg
+
+[all]
+host2 ansible_ssh_host=10.1.0.51 ansible_ssh_pass=root ansible_user=root
+host1 ansible_ssh_host=10.1.0.50 ansible_ssh_pass=root ansible_user=root
+
+[kube-master]
+host1
+
+[etcd]
+host1
+
+[kube-node]
+host2
+
+[k8s-cluster:children]
+kube-node
+kube-master
diff --git a/src/arm/cni-deploy/roles/flannel/files/cni-flannel-ds.yml b/src/arm/cni-deploy/roles/flannel/files/cni-flannel-ds.yml
new file mode 100644
index 0000000..a99983b
--- /dev/null
+++ b/src/arm/cni-deploy/roles/flannel/files/cni-flannel-ds.yml
@@ -0,0 +1,86 @@
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-flannel
+  namespace: "kube-system"
+  labels:
+    tier: node
+    k8s-app: flannel
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        k8s-app: flannel
+    spec:
+      serviceAccountName: flannel
+      containers:
+        - name: kube-flannel
+          image: quay.io/coreos/flannel:v0.9.1-arm64
+          imagePullPolicy: IfNotPresent
+          resources:
+            limits:
+              cpu: 300m
+              memory: 500M
+            requests:
+              cpu: 150m
+              memory: 64M
+          command: ["/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"]
+          securityContext:
+            privileged: true
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          volumeMounts:
+            - name: run
+              mountPath: /run
+            - name: cni
+              mountPath: /etc/cni/net.d
+            - name: flannel-cfg
+              mountPath: /etc/kube-flannel/
+        # - name: install-cni
+        #   image: linaro/flannel-cni-arm64:v0.3.0
+        #   command: ["/install-cni.sh"]
+        #   env:
+        #     # The CNI network config to install on each node.
+        #     - name: CNI_NETWORK_CONFIG
+        #       valueFrom:
+        #         configMapKeyRef:
+        #           name: kube-flannel-cfg
+        #           key: cni-conf.json
+        #     - name: CNI_CONF_NAME
+        #       value: "10-flannel.conflist"
+        #   volumeMounts:
+        #     - name: cni
+        #       mountPath: /host/etc/cni/net.d
+        #     - name: host-cni-bin
+        #       mountPath: /host/opt/cni/bin/
+      hostNetwork: true
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          operator: Exists
+          effect: NoSchedule
+      volumes:
+        - name: run
+          hostPath:
+            path: /run
+        - name: cni
+          hostPath:
+            path: /etc/cni/net.d
+        - name: flannel-cfg
+          configMap:
+            name: kube-flannel-cfg
+        # - name: host-cni-bin
+        #   hostPath:
+        #     path: /opt/cni/bin
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 20%
+    type: RollingUpdate
diff --git a/src/arm/cni-deploy/roles/flannel/tasks/main.yml b/src/arm/cni-deploy/roles/flannel/tasks/main.yml
new file mode 100644
index 0000000..4f1a910
--- /dev/null
+++ b/src/arm/cni-deploy/roles/flannel/tasks/main.yml
@@ -0,0 +1,12 @@
+---
+- name: Copy flannel daemonset file
+  copy:
+    src: cni-flannel-ds.yml
+    dest: /tmp/cni-flannel-ds.yml
+
+- name: Apply flannel daemonset
+  shell: kubectl apply -f /tmp/cni-flannel-ds.yml
+  ignore_errors: "yes"
+
+- name: Sleep 10 seconds
+  wait_for: timeout=10
diff --git a/src/arm/cni-deploy/roles/multus/files/10-multus.conf b/src/arm/cni-deploy/roles/multus/files/10-multus.conf
new file mode 100644
index 0000000..3726413
--- /dev/null
+++ b/src/arm/cni-deploy/roles/multus/files/10-multus.conf
@@ -0,0 +1,13 @@
+{
+  "name": "multus-cni-network",
+  "type": "multus",
+  "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml",
+  "delegates": [{
+    "type": "flannel",
+    "masterplugin": true,
+    "delegate": {
+      "isDefaultGateway": true
+    }
+  }]
+}
+
diff --git a/src/arm/cni-deploy/roles/multus/files/clusterrole.yml b/src/arm/cni-deploy/roles/multus/files/clusterrole.yml
new file mode 100644
index 0000000..fb056d4
--- /dev/null
+++ b/src/arm/cni-deploy/roles/multus/files/clusterrole.yml
@@ -0,0 +1,16 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: multus-crd-overpowered
+rules:
+  - apiGroups:
+      - '*'
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - nonResourceURLs:
+      - '*'
+    verbs:
+      - '*'
diff --git a/src/arm/cni-deploy/roles/multus/files/crdnetwork.yml b/src/arm/cni-deploy/roles/multus/files/crdnetwork.yml
new file mode 100644
index 0000000..9aefdb8
--- /dev/null
+++ b/src/arm/cni-deploy/roles/multus/files/crdnetwork.yml
@@ -0,0 +1,15 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: networks.kubernetes.com
+spec:
+  group: kubernetes.com
+  version: v1
+  scope: Namespaced
+  names:
+    plural: networks
+    singular: network
+    kind: Network
+    shortNames:
+      - net
diff --git a/src/arm/cni-deploy/roles/multus/files/flannel-obj.yml b/src/arm/cni-deploy/roles/multus/files/flannel-obj.yml
new file mode 100644
index 0000000..bd7891d
--- /dev/null
+++ b/src/arm/cni-deploy/roles/multus/files/flannel-obj.yml
@@ -0,0 +1,13 @@
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: flannel-networkobj
+plugin: flannel
+args: '[
+  {
+    "delegate": {
+      "isDefaultGateway": true
+    }
+  }
+]'
diff --git a/src/arm/cni-deploy/roles/multus/handlers/main.yml b/src/arm/cni-deploy/roles/multus/handlers/main.yml
new file mode 100644
index 0000000..8474d34
--- /dev/null
+++ b/src/arm/cni-deploy/roles/multus/handlers/main.yml
@@ -0,0 +1,4 @@
+- name: Restart kubelet
+  service:
+    name: kubelet
+    state: restarted
diff --git a/src/arm/cni-deploy/roles/multus/tasks/crd.yml b/src/arm/cni-deploy/roles/multus/tasks/crd.yml
new file mode 100644
index 0000000..cacf98a
--- /dev/null
+++ b/src/arm/cni-deploy/roles/multus/tasks/crd.yml
@@ -0,0 +1,44 @@
+---
+- name: Copy yaml files
+  copy:
+    src: "{{ item }}"
+    dest: "/tmp/{{ item }}"
+  with_items:
+    - clusterrole.yml
+    - crdnetwork.yml
+    - flannel-obj.yml
+
+- name: Copy macvlan template
+  template:
+    src: macvlan-obj.yml.j2
+    dest: /tmp/macvlan-obj.yml
+
+- name: Copy Multus testpod template
+  template:
+    src: multus-testpod.yml.j2
+    dest: /root/multus-testpod.yml
+
+- name: Create cluster role
+  shell: kubectl apply -f /tmp/clusterrole.yml
+
+- name: Check if role binding is created
+  shell: kubectl get clusterrolebinding multus-node-{{ item }}
+  register: check_rb
+  ignore_errors: "yes"
+  with_items: "{{ groups['all'] }}"
+
+- name: Create role binding
+  shell: >
+    kubectl create clusterrolebinding multus-node-{{ item }}
+    --clusterrole=multus-crd-overpowered
+    --user=system:node:{{ item }}
+  when: check_rb is failed
+  with_items: "{{ groups['all'] }}"
+
+- name: Create network CRD
+  shell: kubectl apply -f /tmp/crdnetwork.yml
+
+- name: Create flannel and macvlan network objects
+  shell: >
+    kubectl apply -f /tmp/flannel-obj.yml &&
+    kubectl apply -f /tmp/macvlan-obj.yml
diff --git a/src/arm/cni-deploy/roles/multus/tasks/main.yml b/src/arm/cni-deploy/roles/multus/tasks/main.yml
new file mode 100644
index 0000000..a200215
--- /dev/null
+++ b/src/arm/cni-deploy/roles/multus/tasks/main.yml
@@ -0,0 +1,24 @@
+---
+- name: Build Multus CNI
+  shell: >
+    docker run --rm --network host -v /opt/cni/bin:/opt/cni/bin golang:1.9
+    bash -c "git clone {{ multus_repo }} multus_cni && cd multus_cni &&
+    git checkout {{ multus_commit }} && ./build && cp bin/multus /opt/cni/bin/"
+  args:
+    creates: /opt/cni/bin/multus
+
+- name: Remove default CNI configuration
+  shell: rm -f /etc/cni/net.d/*
+  args:
+    warn: "no"
+
+- name: Set Multus as default CNI
+  copy:
+    src: 10-multus.conf
+    dest: /etc/cni/net.d/
+  notify:
+    - Restart kubelet
+
+- name: Import CRD task
+  import_tasks: crd.yml
+  when: inventory_hostname == groups["kube-master"][0]
diff --git a/src/arm/cni-deploy/roles/multus/templates/macvlan-obj.yml.j2 b/src/arm/cni-deploy/roles/multus/templates/macvlan-obj.yml.j2
new file mode 100644
index 0000000..b5a549f
--- /dev/null
+++ b/src/arm/cni-deploy/roles/multus/templates/macvlan-obj.yml.j2
@@ -0,0 +1,22 @@
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: macvlan-networkobj
+plugin: macvlan
+args: '[
+  {
+    "master": "{{ macvlan_master }}",
+    "mode": "vepa",
+    "ipam": {
+      "type": "host-local",
+      "subnet": "{{ macvlan_subnet }}",
+      "rangeStart": "{{ macvlan_range_start }}",
+      "rangeEnd": "{{ macvlan_range_end }}",
+      "routes": [
+        { "dst": "0.0.0.0/0" }
+      ],
+      "gateway": "{{ macvlan_gateway }}"
+    }
+  }
+]'
diff --git a/src/arm/cni-deploy/roles/multus/templates/multus-testpod.yml.j2 b/src/arm/cni-deploy/roles/multus/templates/multus-testpod.yml.j2
new file mode 100644
index 0000000..4884846
--- /dev/null
+++ b/src/arm/cni-deploy/roles/multus/templates/multus-testpod.yml.j2
@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: multus-test
+  annotations:
+    networks: '[
+      { "name": "flannel-networkobj" },
+      { "name": "macvlan-networkobj" }
+    ]'
+spec:
+  containers:
+    - name: multus-test
+      image: "busybox"
+      command: ["sleep", "100d"]
+      stdin: true
+      tty: true
+  nodeSelector:
+    kubernetes.io/hostname: "{{ groups['kube-node'][0] }}"
diff --git a/src/arm/cni-deploy/roles/sriov/tasks/crd.yml b/src/arm/cni-deploy/roles/sriov/tasks/crd.yml
new file mode 100644
index 0000000..5cc7892
--- /dev/null
+++ b/src/arm/cni-deploy/roles/sriov/tasks/crd.yml
@@ -0,0 +1,13 @@
+---
+- name: Copy SRIOV template
+  template:
+    src: sriov-obj.yml.j2
+    dest: /tmp/sriov-obj.yml
+
+- name: Copy SRIOV testpod template
+  template:
+    src: sriov-testpod.yml.j2
+    dest: /root/sriov-testpod.yml
+
+- name: Create SRIOV network object
+  shell: kubectl apply -f /tmp/sriov-obj.yml
diff --git a/src/arm/cni-deploy/roles/sriov/tasks/main.yml b/src/arm/cni-deploy/roles/sriov/tasks/main.yml
new file mode 100644
index 0000000..9c190ad
--- /dev/null
+++ b/src/arm/cni-deploy/roles/sriov/tasks/main.yml
@@ -0,0 +1,12 @@
+---
+- name: Build SRIOV CNI
+  shell: >
+    docker run --rm --network host -v /opt/cni/bin:/opt/cni/bin golang:1.9
+    bash -c "git clone {{ sriov_repo }} sriov_cni && cd sriov_cni &&
+    git checkout {{ sriov_commit }} && ./build && cp bin/sriov /opt/cni/bin/"
+  args:
+    creates: /opt/cni/bin/sriov
+
+- name: Import CRD task
+  import_tasks: crd.yml
+  when: inventory_hostname == groups["kube-master"][0]
diff --git a/src/arm/cni-deploy/roles/sriov/templates/sriov-obj.yml.j2 b/src/arm/cni-deploy/roles/sriov/templates/sriov-obj.yml.j2
new file mode 100644
index 0000000..6c67968
--- /dev/null
+++ b/src/arm/cni-deploy/roles/sriov/templates/sriov-obj.yml.j2
@@ -0,0 +1,25 @@
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: sriov-networkobj
+plugin: sriov
+args: '[
+  {
+    "master": "{{ sriov_master }}",
+    "pfOnly": true,
+    "if0name": "net0",
+    "ipam": {
+      "type": "host-local",
+      "subnet": "{{ sriov_subnet }}",
+      "rangeStart": "{{ sriov_range_start }}",
+      "rangeEnd": "{{ sriov_range_end }}",
+      "routes": [
+        {
+          "dst": "0.0.0.0/0"
+        }
+      ],
+      "gateway": "{{ sriov_gateway }}"
+    }
+  }
+]'
diff --git a/src/arm/cni-deploy/roles/sriov/templates/sriov-testpod.yml.j2 b/src/arm/cni-deploy/roles/sriov/templates/sriov-testpod.yml.j2
new file mode 100644
index 0000000..c1d01bc
--- /dev/null
+++ b/src/arm/cni-deploy/roles/sriov/templates/sriov-testpod.yml.j2
@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: sriov-test
+  annotations:
+    networks: '[
+      { "name": "flannel-networkobj" },
+      { "name": "sriov-networkobj" }
+    ]'
+spec:
+  containers:
+    - name: sriov-test
+      image: "busybox"
+      command: ["sleep", "100d"]
+      stdin: true
+      tty: true
+  nodeSelector:
+    kubernetes.io/hostname: "{{ groups['kube-node'][0] }}"
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/0001-net-virtio-ethdev.patch b/src/arm/cni-deploy/roles/vhost-vpp/files/0001-net-virtio-ethdev.patch
new file mode 100644
index 0000000..171ff4d
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/files/0001-net-virtio-ethdev.patch
@@ -0,0 +1,16 @@
+diff --git a/drivers/net/virtio/virtio_ethdev.c b/drivers/net/virtio/virtio_ethdev.c
+index e320811..c1b1640 100644
+--- a/drivers/net/virtio/virtio_ethdev.c
++++ b/drivers/net/virtio/virtio_ethdev.c
+@@ -1754,6 +1754,11 @@ virtio_dev_start(struct rte_eth_dev *dev)
+ 		virtqueue_notify(rxvq->vq);
+ 	}
+ 
++	for (i = 0; i < dev->data->nb_tx_queues; i++) {
++		txvq = dev->data->tx_queues[i];
++		virtqueue_notify(txvq->vq);
++	}
++
+ 	PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
+ 
+ 	for (i = 0; i < dev->data->nb_rx_queues; i++) {
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/Dockerfile.vpp1710-dpdk1708 b/src/arm/cni-deploy/roles/vhost-vpp/files/Dockerfile.vpp1710-dpdk1708
new file mode 100644
index 0000000..2f83534
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/files/Dockerfile.vpp1710-dpdk1708
@@ -0,0 +1,24 @@
+FROM ubuntu:xenial
+
+RUN apt-get update && \
+    apt-get install -y git make openssl libcrypto++-dev libnuma-dev && \
+    apt-get autoclean
+
+RUN git clone https://gerrit.fd.io/r/vpp -b stable/1710 /root/vpp-1710
+
+WORKDIR /root/vpp-1710
+COPY ./0001-net-virtio-ethdev.patch dpdk/dpdk-17.08_patches/0001-net-virtio-ethdev.patch
+RUN sed -i "s/sudo -E //g" Makefile
+RUN make UNATTENDED=yes install-dep
+
+WORKDIR /root/vpp-1710/build-root
+RUN ./bootstrap.sh
+RUN make PLATFORM=vpp TAG=vpp_debug vpp-install
+RUN mkdir -p /etc/vpp && \
+    cp /root/vpp-1710/src/vpp/conf/startup.conf /etc/vpp/startup.conf && \
+    cp /root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin/* /usr/bin && \
+    ln -s /root/vpp-1710/build-root/install-vpp_debug-native/vpp/lib64/vpp_plugins /usr/lib/vpp_plugins
+RUN groupadd vpp
+
+ENV PATH "$PATH:/root/vpp-1710/build-root/install-vpp_debug-native/dpdk/bin"
+ENV PATH "$PATH:/root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin"
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/setvpp.sh b/src/arm/cni-deploy/roles/vhost-vpp/files/setvpp.sh
new file mode 100755
index 0000000..15b0d27
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/files/setvpp.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -x
+
+cid=`sed -ne '/hostname/p' /proc/1/task/1/mountinfo | awk -F '/' '{print $6}'`
+cid_s=${cid:0:12}
+filename=${cid_s}-net1.json
+ifstring=`cat /vhost-user-net-plugin/${cid}/${cid_s}-net1.json | awk -F ',' '{print $4}'`
+ifmac=`echo ${ifstring} | awk -F '\"' '{print $4}'`
+
+ipstr=$(cat /vhost-user-net-plugin/${cid}/${cid_s}-net1-ip4.conf  |grep "ipAddr")
+ipaddr=$(echo $ipstr | awk -F '\"' '{print $4}')
+ipaddr1=$(echo $ipaddr | cut -d / -f 1)
+
+vdev_str="vdev virtio_user0,path=/vhost-user-net-plugin/$cid/$cid_s-net1,mac=$ifmac"
+
+sed -i.bak '/# dpdk/a\dpdk \{' /etc/vpp/startup.conf
+sed -i.bak "/# vdev eth_bond1,mode=1/a\\$vdev_str" /etc/vpp/startup.conf
+sed -i.bak '/# socket-mem/a\\}' /etc/vpp/startup.conf
+
+vpp -c /etc/vpp/startup.conf &
+
+sleep 40
+
+vppctl set int state VirtioUser0/0/0 up
+vppctl set int ip address VirtioUser0/0/0 ${ipaddr1}/24
+vppctl show int
+vppctl show int address
+
+echo ${ipaddr1} > /vhost-user-net-plugin/$(hostname)
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/startup.conf b/src/arm/cni-deploy/roles/vhost-vpp/files/startup.conf
new file mode 100644
index 0000000..ae86e38
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/files/startup.conf
@@ -0,0 +1,21 @@
+unix {
+  nodaemon
+  log /tmp/vpp.log
+  full-coredump
+  cli-listen /run/vpp/cli.sock
+  gid vpp
+}
+api-trace {
+  on
+}
+api-segment {
+  gid vpp
+}
+cpu {
+  main-core 1
+  corelist-workers 2-3
+  workers 2
+}
+dpdk {
+  uio-driver vfio-pci
+}
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/files/vhostuser-obj.yml b/src/arm/cni-deploy/roles/vhost-vpp/files/vhostuser-obj.yml
new file mode 100644
index 0000000..1e9bc66
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/files/vhostuser-obj.yml
@@ -0,0 +1,28 @@
+---
+apiVersion: "kubernetes.com/v1"
+kind: Network
+metadata:
+  name: vhostuser-networkobj
+plugin: vhostuser
+args: '[
+  {
+    "type": "vhostuser",
+    "name": "vhostuser-network",
+    "if0name": "net1",
+    "vhost": {
+      "vhost_tool": "/opt/cni/bin/vpp-config.py"
+    },
+    "ipam": {
+      "type": "host-local",
+      "subnet": "10.56.217.0/24",
+      "rangeStart": "10.56.217.131",
+      "rangeEnd": "10.56.217.190",
+      "routes": [
+        {
+          "dst": "0.0.0.0/0"
+        }
+      ],
+      "gateway": "10.56.217.1"
+    }
+  }
+]'
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/tasks/crd.yml b/src/arm/cni-deploy/roles/vhost-vpp/tasks/crd.yml
new file mode 100644
index 0000000..ad36c90
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/tasks/crd.yml
@@ -0,0 +1,13 @@
+---
+- name: Copy Vhostuser yaml
+  copy:
+    src: vhostuser-obj.yml
+    dest: /tmp/vhostuser-obj.yml
+
+- name: Copy VPP testpod template
+  template:
+    src: vpp-testpod.yml.j2
+    dest: /root/vpp-testpod.yml
+
+- name: Create Vhostuser network object
+  shell: kubectl apply -f /tmp/vhostuser-obj.yml
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/tasks/main.yml b/src/arm/cni-deploy/roles/vhost-vpp/tasks/main.yml
new file mode 100644
index 0000000..df890ea
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/tasks/main.yml
@@ -0,0 +1,18 @@
+---
+- name: Build Vhostuser CNI
+  shell: >
+    docker run --rm --network host -v /opt/cni/bin:/opt/cni/bin golang:1.9
+    bash -c "git clone {{ vhostuser_repo }} vhostuser_cni && cd vhostuser_cni
+    && git checkout {{ vhostuser_commit }} && ./build
+    && cp bin/vhostuser /opt/cni/bin/
+    && cp tests/vpp-config-debug.py /opt/cni/bin/vpp-config.py"
+  args:
+    creates: /opt/cni/bin/vhostuser
+
+- name: Import CRD task
+  import_tasks: crd.yml
+  when: inventory_hostname == groups["kube-master"][0]
+
+- name: Import VPP task
+  import_tasks: vpp.yml
+  when: inventory_hostname in groups["kube-node"]
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/tasks/vpp.yml b/src/arm/cni-deploy/roles/vhost-vpp/tasks/vpp.yml
new file mode 100644
index 0000000..7f5be05
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/tasks/vpp.yml
@@ -0,0 +1,47 @@
+---
+- name: Create dest directories
+  file:
+    path: "{{ item }}"
+    state: directory
+  with_items:
+    - /tmp/vpp1710/
+    - /var/lib/cni/vhostuser/
+    - /etc/vpp/
+
+- name: Copy VPP files
+  copy:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+  with_items:
+    - {src: "Dockerfile.vpp1710-dpdk1708", dest: "/tmp/vpp1710/Dockerfile"}
+    - {src: "0001-net-virtio-ethdev.patch", dest: "/tmp/vpp1710/0001-net-virtio-ethdev.patch"}
+    - {src: "setvpp.sh", dest: "/var/lib/cni/vhostuser/setvpp.sh"}
+    - {src: "startup.conf", dest: "/etc/vpp/startup.conf"}
+
+- name: Check if VPP image exists
+  shell: docker inspect --type=image vpp-1710:virtio-patched > /dev/null 2>&1
+  ignore_errors: "yes"
+  register: check_vpp
+
+- name: Building VPP container. Be patient...
+  shell: docker build -t vpp-1710:virtio-patched --network host .
+  args:
+    chdir: /tmp/vpp1710/
+  when: check_vpp is failed
+
+- name: Copy VPP binaries to host
+  shell: >
+    docker run --rm -v /root/vpp-1710/build-root:/root/vpp-host vpp-1710:virtio-patched
+    /bin/cp -a /root/vpp-1710/build-root/install-vpp_debug-native /root/vpp-host
+    && /bin/cp /root/vpp-1710/build-root/install-vpp_debug-native/vpp/bin/* /usr/bin
+    && /bin/rm -rf /usr/lib/vpp_plugins
+    && ln -s /root/vpp-1710/build-root/install-vpp_debug-native/vpp/lib64/vpp_plugins /usr/lib/vpp_plugins
+    && (groupadd vpp || true)
+
+- name: Copy libcrypto.so.1.0.0 for CentOS
+  shell: >
+    docker run --rm -v /usr/lib64:/root/lib64-centos vpp-1710:virtio-patched
+    /bin/cp /lib/aarch64-linux-gnu/libcrypto.so.1.0.0 /root/lib64-centos/
+  args:
+    creates: /usr/lib64/libcrypto.so.1.0.0
+  when: ansible_os_family == "RedHat"
diff --git a/src/arm/cni-deploy/roles/vhost-vpp/templates/vpp-testpod.yml.j2 b/src/arm/cni-deploy/roles/vhost-vpp/templates/vpp-testpod.yml.j2
new file mode 100644
index 0000000..2efd4e0
--- /dev/null
+++ b/src/arm/cni-deploy/roles/vhost-vpp/templates/vpp-testpod.yml.j2
@@ -0,0 +1,68 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vpp-test1
+  annotations:
+    networks: '[
+        { "name": "flannel-networkobj" },
+        { "name": "vhostuser-networkobj" }
+    ]'
+spec:
+  containers:
+    - name: vpp-test1
+      image: vpp-1710:virtio-patched
+      imagePullPolicy: "Never"
+      stdin: true
+      terminationMessagePath: /dev/termination-log
+      tty: true
+      securityContext:
+        privileged: true
+      volumeMounts:
+        - mountPath: /vhost-user-net-plugin
+          name: vhost-user-net-plugin
+        - mountPath: /mnt/huge
+          name: huge
+  nodeSelector:
+    kubernetes.io/hostname: "{{ groups['kube-node'][0] }}"
+  volumes:
+    - name: vhost-user-net-plugin
+      hostPath:
+        path: /var/lib/cni/vhostuser
+    - name: huge
+      hostPath:
+        path: /mnt/huge
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vpp-test2
+  annotations:
+    networks: '[
+        { "name": "flannel-networkobj" },
+        { "name": "vhostuser-networkobj" }
+    ]'
+spec:
+  containers:
+    - name: vpp-test2
+      image: vpp-1710:virtio-patched
+      imagePullPolicy: "Never"
+      stdin: true
+      terminationMessagePath: /dev/termination-log
+      tty: true
+      securityContext:
+        privileged: true
+      volumeMounts:
+        - mountPath: /vhost-user-net-plugin
+          name: vhost-user-net-plugin
+        - mountPath: /mnt/huge
+          name: huge
+  nodeSelector:
+    kubernetes.io/hostname: "{{ groups['kube-node'][0] }}"
+  volumes:
+    - name: vhost-user-net-plugin
+      hostPath:
+        path: /var/lib/cni/vhostuser
+    - name: huge
+      hostPath:
+        path: /mnt/huge
diff --git a/src/arm/cni-deploy/vars/global b/src/arm/cni-deploy/vars/global
new file mode 100644
index 0000000..35d76b4
--- /dev/null
+++ b/src/arm/cni-deploy/vars/global
@@ -0,0 +1,20 @@
+multus_repo: https://github.com/Intel-Corp/multus-cni
+multus_commit: 61959e04
+
+sriov_repo: https://github.com/hustcat/sriov-cni
+sriov_commit: 8b7ed984
+
+vhostuser_repo: https://github.com/yibo-cai/vhost-user-net-plugin
+vhostuser_commit: e8dc9d8e
+
+macvlan_master: eth2
+macvlan_subnet: 192.168.166.0/24
+macvlan_range_start: 192.168.166.11
+macvlan_range_end: 192.168.166.30
+macvlan_gateway: 192.168.166.1
+
+sriov_master: eth2
+sriov_subnet: 192.168.166.0/24
+sriov_range_start: 192.168.166.31
+sriov_range_end: 192.168.166.50
+sriov_gateway: 192.168.166.1