17 files changed, 424 insertions, 39 deletions
diff --git a/src/cni/ovsdpdk/Dockerfile b/src/cni/ovsdpdk/Dockerfile
new file mode 100644
index 0000000..2a7208c
--- /dev/null
+++ b/src/cni/ovsdpdk/Dockerfile
@@ -0,0 +1,7 @@
+FROM ubuntu:16.04
+
+WORKDIR /cni
+ADD setup_ovsdpdk.sh .
+ADD teardown_ovsdpdk.sh .
+ADD ovsdpdk .
+ADD install_cni.sh .
diff --git a/src/cni/ovsdpdk/Vagrantfile b/src/cni/ovsdpdk/Vagrantfile
new file mode 100644
index 0000000..f170c69
--- /dev/null
+++ b/src/cni/ovsdpdk/Vagrantfile
@@ -0,0 +1,16 @@
+Vagrant.require_version ">= 1.8.6"
+Vagrant.configure("2") do |config|
+
+  config.vm.box = "bento/ubuntu-16.04"
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--memory", 4096]
+    vb.customize ["modifyvm", :id, "--cpus", 4]
+    vb.customize "post-boot",["controlvm", :id, "setlinkstate1", "on"]
+  end
+
+  config.vm.define "cni-ovsdpdk" do |config|
+    config.vm.hostname = "cni-ovsdpdk"
+    config.vm.provision "shell", path: "build_cni_ovsdpdk.sh", privileged: false
+  end
+
+end
diff --git a/src/cni/ovsdpdk/build_cni_ovsdpdk.sh b/src/cni/ovsdpdk/build_cni_ovsdpdk.sh
new file mode 100755
index 0000000..71318d8
--- /dev/null
+++ b/src/cni/ovsdpdk/build_cni_ovsdpdk.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+sudo apt-get update
+sudo apt-get install -y docker.io devscripts git
+wget -qO- https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz | sudo tar -C /usr/local -xz
+echo 'export GOPATH=/go; export PATH=/usr/local/go/bin:$GOPATH/bin:$PATH' >> ~/.bashrc
+export GOPATH=/go; export PATH=/usr/local/go/bin:$GOPATH/bin:$PATH
+git clone https://github.com/containernetworking/cni
+echo sudo CNI_PATH=$CNI_PATH ./priv-net-run.sh ifconfig
+git clone https://github.com/containernetworking/plugins
+cd plugins
+git checkout 5544d9ced0d6e908fe26e9dbe529c7feb87d21f5
+patch -p1 < /vagrant/ovsdpdk.patch
+./build.sh
+cd bin
+cp /vagrant/* .
+sudo docker build -t openretriever/cni-ovsdpdk .
diff --git a/src/cni/ovsdpdk/install_cni.sh b/src/cni/ovsdpdk/install_cni.sh
new file mode 100644
index 0000000..8f5b78a
--- /dev/null
+++ b/src/cni/ovsdpdk/install_cni.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -ex
+cp /cni/ovsdpdk /opt/cni/bin
+cp /cni/setup_ovsdpdk.sh /opt/cni/bin
+cp /cni/teardown_ovsdpdk.sh /opt/cni/bin
+cp /etc/kube-ovsdpdk/cni-conf.json  /etc/cni/net.d/10-ovsdpdk.conf
+while true; do sleep 3600; done
diff --git a/src/cni/ovsdpdk/kube_ovsdpdk.yml b/src/cni/ovsdpdk/kube_ovsdpdk.yml
new file mode 100644
index 0000000..2bcebdc
--- /dev/null
+++ b/src/cni/ovsdpdk/kube_ovsdpdk.yml
@@ -0,0 +1,71 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ovsdpdk
+  namespace: kube-system
+---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: kube-ovsdpdk-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    app: ovsdpdk
+data:
+  cni-conf.json: |
+    {
+      "name": "ovsdpdk",
+      "type": "ovsdpdk",
+      "bridge": "br-dpdk",
+      "ipam": {
+        "type": "host-local",
+        "subnet": "10.244.0.0/16"
+      }
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-ovsdpdk-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: ovsdpdk
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: ovsdpdk
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      serviceAccountName: ovsdpdk
+      containers:
+      - name: install-cni
+        image: openretriever/cni-ovsdpdk
+        command: [ "/bin/bash", "/cni/install_cni.sh" ]
+        volumeMounts:
+        - name: cni-bin
+          mountPath: /opt/cni/bin
+        - name: cni-cfg
+          mountPath: /etc/cni/net.d
+        - name: ovsdpdk-cfg
+          mountPath: /etc/kube-ovsdpdk
+      volumes:
+        - name: cni-bin
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-cfg
+          hostPath:
+            path: /etc/cni/net.d
+        - name: ovsdpdk-cfg
+          configMap:
+            name: kube-ovsdpdk-cfg
diff --git a/src/cni/ovsdpdk/ovsdpdk.patch b/src/cni/ovsdpdk/ovsdpdk.patch
new file mode 100644
index 0000000..67b3703
--- /dev/null
+++ b/src/cni/ovsdpdk/ovsdpdk.patch
@@ -0,0 +1,136 @@
+diff --git a/build.sh b/build.sh
+index cd21ba8..bc60d91 100755
+--- a/build.sh
++++ b/build.sh
+@@ -19,7 +19,7 @@ export GOPATH=${PWD}/gopath
+ mkdir -p "${PWD}/bin"
+ 
+ echo "Building plugins"
+-PLUGINS="plugins/meta/* plugins/main/* plugins/ipam/* plugins/sample"
++PLUGINS="plugins/main/ovsdpdk plugins/main/bridge plugins/ipam/host-local"
+ for d in $PLUGINS; do
+ 	if [ -d "$d" ]; then
+ 		plugin="$(basename "$d")"
+diff --git a/plugins/main/ovsdpdk/ovsdpdk.go b/plugins/main/ovsdpdk/ovsdpdk.go
+new file mode 100644
+index 0000000..1b931d4
+--- /dev/null
++++ b/plugins/main/ovsdpdk/ovsdpdk.go
+@@ -0,0 +1,117 @@
++// Copyright 2014 CNI authors
++//
++// Licensed under the Apache License, Version 2.0 (the "License");
++// you may not use this file except in compliance with the License.
++// You may obtain a copy of the License at
++//
++//     http://www.apache.org/licenses/LICENSE-2.0
++//
++// Unless required by applicable law or agreed to in writing, software
++// distributed under the License is distributed on an "AS IS" BASIS,
++// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++// See the License for the specific language governing permissions and
++// limitations under the License.
++
++package main
++
++import (
++	"encoding/json"
++	"errors"
++	"fmt"
++	//"net"
++	"runtime"
++	//"syscall"
++        "os/exec"
++	//"io/ioutil"
++
++	"github.com/containernetworking/cni/pkg/skel"
++	"github.com/containernetworking/cni/pkg/types"
++	"github.com/containernetworking/cni/pkg/types/current"
++	"github.com/containernetworking/cni/pkg/version"
++	//"github.com/containernetworking/plugins/pkg/ip"
++	"github.com/containernetworking/plugins/pkg/ipam"
++	//"github.com/containernetworking/plugins/pkg/ns"
++	//"github.com/containernetworking/plugins/pkg/utils"
++	//"github.com/vishvananda/netlink"
++)
++
++const defaultBrName = "cni0"
++
++type NetConf struct {
++	types.NetConf
++	BrName       string `json:"bridge"`
++}
++
++func init() {
++	// this ensures that main runs only on main thread (thread group leader).
++	// since namespace ops (unshare, setns) are done for a single thread, we
++	// must ensure that the goroutine does not jump from OS thread to thread
++	runtime.LockOSThread()
++}
++
++func loadNetConf(bytes []byte) (*NetConf, string, error) {
++	n := &NetConf{
++		BrName: defaultBrName,
++	}
++	if err := json.Unmarshal(bytes, n); err != nil {
++		return nil, "", fmt.Errorf("failed to load netconf: %v", err)
++	}
++	return n, n.CNIVersion, nil
++}
++
++func setupVhostUser(args *skel.CmdArgs, types string) error {
++    exec.Command("/bin/bash", "/opt/cni/bin/setup_ovsdpdk.sh", args.Netns, args.ContainerID, types).Output()
++    return nil
++}
++
++
++func cmdAdd(args *skel.CmdArgs) error {
++	n, cniVersion, err := loadNetConf(args.StdinData)
++	if err != nil {
++		return err
++	}
++
++	// run the IPAM plugin and get back the config to apply
++	r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData)
++	if err != nil {
++		return err
++	}
++
++	// Convert whatever the IPAM result was into the current Result type
++	result, err := current.NewResultFromResult(r)
++	if err != nil {
++		return err
++	}
++
++	if len(result.IPs) == 0 {
++		return errors.New("IPAM plugin returned missing IP config")
++	}
++
++        setupVhostUser(args, result.String())
++
++        return types.PrintResult(result, cniVersion)
++}
++
++func tearDownVhostUser(args *skel.CmdArgs) error {
++    exec.Command("/bin/bash", "/opt/cni/bin/teardown_ovsdpdk.sh", args.Netns, args.ContainerID).Output()
++    return nil
++}
++
++func cmdDel(args *skel.CmdArgs) error {
++        n, _, err := loadNetConf(args.StdinData)
++        if err != nil {
++                return err
++        }
++
++        if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil {
++                return err
++        }
++
++        tearDownVhostUser(args)
++        return err
++
++}
++
++func main() {
++	skel.PluginMain(cmdAdd, cmdDel, version.All)
++}
diff --git a/src/cni/ovsdpdk/setup_ovsdpdk.sh b/src/cni/ovsdpdk/setup_ovsdpdk.sh
new file mode 100755
index 0000000..a1813c9
--- /dev/null
+++ b/src/cni/ovsdpdk/setup_ovsdpdk.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+netns=$1
+containerid=$2
+ip=$3
+pid=$(echo $netns | cut -f3 -d"/")
+
+sudo ovs-vsctl --may-exist add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev
+sudo ovs-vsctl --may-exist add-port br-dpdk vhost-user-$pid -- set Interface vhost-user-$pid type=dpdkvhostuser
+sudo ln -sf $netns /var/run/netns/$pid
+sudo ip link add dummy-$pid type dummy
+sudo ip link set dummy-$pid netns $pid
+sudo mkdir -p /var/run/cni
+echo $ip | sudo tee /var/run/cni/netconf-$pid
diff --git a/src/cni/ovsdpdk/teardown_ovsdpdk.sh b/src/cni/ovsdpdk/teardown_ovsdpdk.sh
new file mode 100755
index 0000000..857738e
--- /dev/null
+++ b/src/cni/ovsdpdk/teardown_ovsdpdk.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+netns=$1
+containerid=$2
+pid=$(echo $netns | cut -f3 -d"/")
+
+sudo ovs-vsctl del-port br-dpdk vhost-user-$pid
+sudo ip netns exec $pid link delete dummy-$pid
+sudo rm -rf /var/run/cni/netconf-$pid
diff --git a/src/vagrant/k8s_kubeadm/Vagrantfile b/src/vagrant/k8s_kubeadm/Vagrantfile
index 3baf072..3890e57 100644
--- a/src/vagrant/k8s_kubeadm/Vagrantfile
+++ b/src/vagrant/k8s_kubeadm/Vagrantfile
@@ -4,9 +4,14 @@ Vagrant.require_version ">= 1.8.6"
 Vagrant.configure("2") do |config|
 
   config.vm.box = "bento/ubuntu-16.04"
+  config.vm.synced_folder "../..", "/src"
+  config.vm.provision "shell", path: "host_setup.sh", privileged: false
   config.vm.provider :virtualbox do |vb|
     vb.customize ["modifyvm", :id, "--memory", 4096]
     vb.customize ["modifyvm", :id, "--cpus", 4]
+    vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+    vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"]
+    vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"]
     vb.customize "post-boot",["controlvm", :id, "setlinkstate1", "on"]
   end
 
@@ -14,6 +19,7 @@ Vagrant.configure("2") do |config|
     config.vm.hostname = "master"
     config.vm.provision "shell", path: "master_setup.sh", privileged: false
     config.vm.network :private_network, ip: "192.168.1.10"
+    config.vm.network :private_network, ip: "192.168.2.10"
   end
 
   (1 .. $num_workers).each do |i|
@@ -21,6 +27,7 @@ Vagrant.configure("2") do |config|
       config.vm.hostname = vm_name
       config.vm.provision "shell", path: "worker_setup.sh", privileged: false
       config.vm.network :private_network, ip: "192.168.1.#{i+20}"
+      config.vm.network :private_network, ip: "192.168.2.#{i+20}"
     end
   end
 
diff --git a/src/vagrant/k8s_kubeadm/examples/virtio-user.yaml b/src/vagrant/k8s_kubeadm/examples/virtio-user.yaml
new file mode 100644
index 0000000..9ab1e06
--- /dev/null
+++ b/src/vagrant/k8s_kubeadm/examples/virtio-user.yaml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: virtiouser
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: virtiouser
+    spec:
+      containers:
+      - name: virtiouser
+        image: openretriever/virtio-user-ping
+        volumeMounts:
+        - mountPath: /dev/hugepages
+          name: hugepage-volume
+        - mountPath: /var/run
+          name: vhost-volume
+        command:
+          - /root/setup_virtio_user.sh
+      volumes:
+      - name: hugepage-volume
+        hostPath:
+          path: /dev/hugepages
+      - name: vhost-volume
+        hostPath:
+          path: /var/run
+      restartPolicy: Always
diff --git a/src/vagrant/k8s_kubeadm/host_setup.sh b/src/vagrant/k8s_kubeadm/host_setup.sh
new file mode 100644
index 0000000..990df7f
--- /dev/null
+++ b/src/vagrant/k8s_kubeadm/host_setup.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+cat << EOF | sudo tee /etc/hosts
+127.0.0.1    localhost
+192.168.1.10 master
+192.168.1.21 worker1
+192.168.1.22 worker2
+192.168.1.23 worker3
+EOF
+
+curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
+deb http://apt.kubernetes.io/ kubernetes-xenial main
+EOF
+sudo apt-get update
+sudo apt-get install -y docker.io
+sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00
diff --git a/src/vagrant/k8s_kubeadm/master_setup.sh b/src/vagrant/k8s_kubeadm/master_setup.sh
index 3bf4d4d..e98e2bb 100644
--- a/src/vagrant/k8s_kubeadm/master_setup.sh
+++ b/src/vagrant/k8s_kubeadm/master_setup.sh
@@ -1,23 +1,6 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
-set -e
-HOME=`pwd`
-
-cat << EOF | sudo tee /etc/hosts
-127.0.0.1    localhost
-192.168.1.10 master
-192.168.1.21 worker1
-192.168.1.22 worker2
-192.168.1.23 worker3
-EOF
-
-curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
-cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
-deb http://apt.kubernetes.io/ kubernetes-xenial main
-EOF
-sudo apt-get update
-sudo apt-get install -y docker.io
-sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00
+sudo ifconfig br-dpdk 10.244.0.1/16 up
 
 sudo kubeadm init --apiserver-advertise-address 192.168.1.10  --service-cidr=192.168.1.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0
 sudo cp /etc/kubernetes/admin.conf $HOME/
@@ -28,3 +11,5 @@ echo "export KUBECONFIG=$HOME/admin.conf" >> $HOME/.bash_profile
 kubectl apply -f http://git.io/weave-kube-1.6
 #kubectl apply -f http://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
 #kubectl apply -f http://docs.projectcalico.org/v2.1/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml
+#kubectl apply -f /vagrant/k8s_kubeadm/dpdk/kube_ovsdpdk.yml
+#kubectl apply -f /src/cni/ovsdpdk/kube_ovsdpdk.yml
diff --git a/src/vagrant/k8s_kubeadm/ovsdpdk/Dockerfile b/src/vagrant/k8s_kubeadm/ovsdpdk/Dockerfile
new file mode 100644
index 0000000..a63df8f
--- /dev/null
+++ b/src/vagrant/k8s_kubeadm/ovsdpdk/Dockerfile
@@ -0,0 +1,8 @@
+FROM ubuntu:16.04
+
+WORKDIR /ovsdpdk
+RUN apt-get update && apt-get install -y sudo
+ADD install.sh .
+RUN bash ./install.sh
+ADD start.sh .
+RUN chmod 755 start.sh
diff --git a/src/vagrant/k8s_kubeadm/ovsdpdk/install.sh b/src/vagrant/k8s_kubeadm/ovsdpdk/install.sh
new file mode 100644
index 0000000..66fb71d
--- /dev/null
+++ b/src/vagrant/k8s_kubeadm/ovsdpdk/install.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+sudo apt-get update
+sudo apt-get install -y openvswitch-switch-dpdk linux-image-extra-4.4.0-75-generic
+sudo update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk
+echo "DPDK_OPTS='--dpdk -c 0x1 -n 4 -m 1024'" | sudo tee -a /etc/default/openvswitch-switch
diff --git a/src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml b/src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml
new file mode 100644
index 0000000..d79da15
--- /dev/null
+++ b/src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml
@@ -0,0 +1,53 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ovsdpdk
+  namespace: kube-system
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-ovsdpdk-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: ovsdpdk
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: ovsdpdk
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      serviceAccountName: ovsdpdk
+      containers:
+      - name: install-ovsdpdk
+        image: openretriever/ubuntu1604-ovsdpdk
+        command: [ "/bin/bash", "/ovsdpdk/start.sh" ]
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: local-bin
+          mountPath: /usr/local/bin
+        - name: var-run
+          mountPath: /var/run/openvswitch
+        - name: dev-hugepage
+          mountPath: /dev
+      volumes:
+        - name: local-bin
+          hostPath:
+            path: /usr/local/bin
+        - name: var-run
+          hostPath:
+            path: /var/run/openvswitch
+        - name: dev-hugepage
+          hostPath:
+            path: /dev
diff --git a/src/vagrant/k8s_kubeadm/ovsdpdk/start.sh b/src/vagrant/k8s_kubeadm/ovsdpdk/start.sh
new file mode 100644
index 0000000..08d8143
--- /dev/null
+++ b/src/vagrant/k8s_kubeadm/ovsdpdk/start.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -ex
+sudo sysctl -w vm.nr_hugepages=2048
+sudo mount -t hugetlbfs -o pagesize=2M none /dev/hugepages
+cp /usr/bin/ovs-vsctl /usr/local/bin
+sudo service dpdk restart
+sudo service openvswitch-switch restart
+sudo ovs-vsctl add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev
+sudo modprobe uio_pci_generic
+#sudo ip address flush enp0s9
+#sudo /usr/share/dpdk/tools/dpdk_nic_bind.py --bind=uio_pci_generic enp0s9
+#sudo ovs-vsctl add-port br-dpdk dpdk0 -- set Interface dpdk0 type=dpdk
+while true; do sleep 3600; done
+echo sudo docker run -ti --privileged -v /dev:/dev -v /usr/local/bin:/usr/local/bin -v /var/run/openvswitch/:/var/run/openvswitch/ dpdk /ovsdpdk/start.sh
diff --git a/src/vagrant/k8s_kubeadm/worker_setup.sh b/src/vagrant/k8s_kubeadm/worker_setup.sh
index a9c8336..b68d800 100644
--- a/src/vagrant/k8s_kubeadm/worker_setup.sh
+++ b/src/vagrant/k8s_kubeadm/worker_setup.sh
@@ -1,22 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
-set -e
-HOME=`pwd`
-
-cat << EOF | sudo tee /etc/hosts
-127.0.0.1    localhost
-192.168.1.10 master
-192.168.1.21 worker1
-192.168.1.22 worker2
-192.168.1.23 worker3
-EOF
-
-curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
-cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
-deb http://apt.kubernetes.io/ kubernetes-xenial main
-EOF
-sudo apt-get update
-sudo apt-get install -y docker.io
-sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00
+set -ex
 sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true
-echo "vagrant ssh master -c '/vagrant/examples/nginx-app.sh'"