seperate k8s basic installation and ovsdpdk installation

Change-Id: I9ba3b52feebb7be6037d88f07f8c861ddbfe9bdf
Signed-off-by: Guo Ruijing <ruijing.guo@intel.com>

9 files changed, 196 insertions, 0 deletions

diff --git a/src/vagrant/kubeadm_ovsdpdk/Vagrantfile b/src/vagrant/kubeadm_ovsdpdk/Vagrantfile
new file mode 100644
index 0000000..f1e7aac
--- /dev/null
+++ b/src/vagrant/kubeadm_ovsdpdk/Vagrantfile
@@ -0,0 +1,34 @@
+$num_workers=3
+
+Vagrant.require_version ">= 1.8.6"
+Vagrant.configure("2") do |config|
+
+  config.vm.box = "bento/ubuntu-16.04"
+  config.vm.synced_folder "../..", "/src"
+  config.vm.provision "shell", path: "host_setup.sh", privileged: false
+  config.vm.provider :virtualbox do |vb|
+    vb.customize ["modifyvm", :id, "--memory", 4096]
+    vb.customize ["modifyvm", :id, "--cpus", 4]
+    vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"]
+    vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"]
+    vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"]
+    vb.customize "post-boot",["controlvm", :id, "setlinkstate1", "on"]
+  end
+
+  config.vm.define "master" do |config|
+    config.vm.hostname = "master"
+    config.vm.provision "shell", path: "master_setup.sh", privileged: false
+    config.vm.network :private_network, ip: "192.168.1.10"
+    config.vm.network :private_network, ip: "10.244.0.10"
+  end
+
+  (1 .. $num_workers).each do |i|
+    config.vm.define vm_name = "worker%d" % [i] do |config|
+      config.vm.hostname = vm_name
+      config.vm.provision "shell", path: "worker_setup.sh", privileged: false
+      config.vm.network :private_network, ip: "192.168.1.#{i+20}"
+      config.vm.network :private_network, ip: "10.244.0.#{i+20}"
+    end
+  end
+
+end
diff --git a/src/vagrant/kubeadm_ovsdpdk/examples/virtio-user.yaml b/src/vagrant/kubeadm_ovsdpdk/examples/virtio-user.yaml
new file mode 100644
index 0000000..9ab1e06
--- /dev/null
+++ b/src/vagrant/kubeadm_ovsdpdk/examples/virtio-user.yaml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: virtiouser
+spec:
+  replicas: 2
+  template:
+    metadata:
+      labels:
+        app: virtiouser
+    spec:
+      containers:
+      - name: virtiouser
+        image: openretriever/virtio-user-ping
+        volumeMounts:
+        - mountPath: /dev/hugepages
+          name: hugepage-volume
+        - mountPath: /var/run
+          name: vhost-volume
+        command:
+          - /root/setup_virtio_user.sh
+      volumes:
+      - name: hugepage-volume
+        hostPath:
+          path: /dev/hugepages
+      - name: vhost-volume
+        hostPath:
+          path: /var/run
+      restartPolicy: Always
diff --git a/src/vagrant/kubeadm_ovsdpdk/host_setup.sh b/src/vagrant/kubeadm_ovsdpdk/host_setup.sh
new file mode 100644
index 0000000..990df7f
--- /dev/null
+++ b/src/vagrant/kubeadm_ovsdpdk/host_setup.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+cat << EOF | sudo tee /etc/hosts
+127.0.0.1    localhost
+192.168.1.10 master
+192.168.1.21 worker1
+192.168.1.22 worker2
+192.168.1.23 worker3
+EOF
+
+curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
+cat <<EOF | sudo tee /etc/apt/sources.list.d/kubernetes.list
+deb http://apt.kubernetes.io/ kubernetes-xenial main
+EOF
+sudo apt-get update
+sudo apt-get install -y docker.io
+sudo apt-get install -y --allow-downgrades kubelet=1.7.0-00 kubeadm=1.7.0-00 kubectl=1.7.0-00 kubernetes-cni=0.5.1-00
diff --git a/src/vagrant/kubeadm_ovsdpdk/master_setup.sh b/src/vagrant/kubeadm_ovsdpdk/master_setup.sh
new file mode 100644
index 0000000..f5bb6fc
--- /dev/null
+++ b/src/vagrant/kubeadm_ovsdpdk/master_setup.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+sudo kubeadm init --apiserver-advertise-address 192.168.1.10  --service-cidr=192.168.1.0/24 --pod-network-cidr=10.244.0.0/16 --token 8c5adc.1cec8dbf339093f0
+sudo cp /etc/kubernetes/admin.conf $HOME/
+sudo chown $(id -u):$(id -g) $HOME/admin.conf
+export KUBECONFIG=$HOME/admin.conf
+echo "export KUBECONFIG=$HOME/admin.conf" >> $HOME/.bash_profile
+
+kubectl apply -f /vagrant/ovsdpdk/kube_ovsdpdk.yml
+kubectl apply -f /src/cni/ovsdpdk/kube_cniovsdpdk.yml
diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile
new file mode 100644
index 0000000..a63df8f
--- /dev/null
+++ b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile
@@ -0,0 +1,8 @@
+FROM ubuntu:16.04
+
+WORKDIR /ovsdpdk
+RUN apt-get update && apt-get install -y sudo
+ADD install.sh .
+RUN bash ./install.sh
+ADD start.sh .
+RUN chmod 755 start.sh
diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh
new file mode 100644
index 0000000..f565025
--- /dev/null
+++ b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -ex
+sudo apt-get update
+sudo apt-get install -y openvswitch-switch-dpdk pciutils vim
+sudo update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk
diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml
new file mode 100644
index 0000000..ed04ccd
--- /dev/null
+++ b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml
@@ -0,0 +1,58 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: ovsdpdk
+  namespace: kube-system
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: kube-ovsdpdk-ds
+  namespace: kube-system
+  labels:
+    tier: node
+    app: ovsdpdk
+spec:
+  template:
+    metadata:
+      labels:
+        tier: node
+        app: ovsdpdk
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/arch: amd64
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+      serviceAccountName: ovsdpdk
+      containers:
+      - name: install-ovsdpdk
+        image: openretriever/ubuntu1604-ovsdpdk
+        command: [ "/bin/bash", "/ovsdpdk/start.sh" ]
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: local-bin
+          mountPath: /usr/local/bin
+        - name: var-run
+          mountPath: /var/run/openvswitch
+        - name: lib-modules
+          mountPath: /lib/modules
+        - name: dev-hugepage
+          mountPath: /dev
+      volumes:
+        - name: local-bin
+          hostPath:
+            path: /usr/local/bin
+        - name: var-run
+          hostPath:
+            path: /var/run/openvswitch
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: dev-hugepage
+          hostPath:
+            path: /dev
diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh
new file mode 100644
index 0000000..fed7985
--- /dev/null
+++ b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -ex
+
+for i in {1..10}
+do
+    sudo sysctl -w vm.nr_hugepages=2048; sleep 1
+done
+sudo modprobe uio_pci_generic
+ip=$(ip a s enp0s9 | grep inet | grep -v inet6 | sed "s/.*inet//" | cut -f2 -d' ')
+sudo ip address flush enp0s9
+sudo /usr/share/dpdk/tools/dpdk_nic_bind.py --bind=uio_pci_generic enp0s9
+sudo sysctl -w vm.nr_hugepages=1024
+sudo mount -t hugetlbfs -o pagesize=2M none /dev/hugepages
+sudo cp /usr/bin/ovs-vsctl /usr/local/bin
+memory=$(grep HugePages_Total /proc/meminfo | cut -f2 -d:)
+echo "DPDK_OPTS='--dpdk -c 0x1 -n 2 -m $memory'" | sudo tee -a /etc/default/openvswitch-switch
+sudo service dpdk restart
+sudo service openvswitch-switch restart
+sudo pkill ovs-vswitchd
+sudo ovs-vswitchd --dpdk -c 0x1 -n 2 -m $memory -- unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach --monitor
+sudo ovs-vsctl add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev
+sudo ovs-vsctl add-port br-dpdk dpdk0 -- set Interface dpdk0 type=dpdk
+sudo ip a a $ip dev br-dpdk
+sudo ip link set dev br-dpdk up
+while true; do sleep 3600; done
+echo sudo docker build -t openretriever/ubuntu1604-ovsdpdk .
+echo sudo docker run -ti --privileged --net=host -v /dev:/dev -v /usr/local/bin:/usr/local/bin -v /var/run/openvswitch/:/var/run/openvswitch/ -v /lib/modules/:/lib/modules openretriever/ubuntu1604-ovsdpdk bash
diff --git a/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh b/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh
new file mode 100644
index 0000000..b68d800
--- /dev/null
+++ b/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+set -ex
+sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true