From eecc28839b7088f6bd6be9538db7ddc1986b369f Mon Sep 17 00:00:00 2001 From: Guo Ruijing Date: Wed, 18 Oct 2017 19:59:55 -0400 Subject: seperate k8s basic installation and ovsdpdk installation Change-Id: I9ba3b52feebb7be6037d88f07f8c861ddbfe9bdf Signed-off-by: Guo Ruijing --- src/vagrant/README.md | 23 +++++++++ src/vagrant/k8s_kubeadm/README.md | 23 --------- src/vagrant/k8s_kubeadm/Vagrantfile | 34 ------------- src/vagrant/k8s_kubeadm/examples/nginx-app.sh | 10 ---- src/vagrant/k8s_kubeadm/examples/nginx-app.yaml | 31 ------------ src/vagrant/k8s_kubeadm/examples/virtio-user.yaml | 29 ----------- src/vagrant/k8s_kubeadm/examples/yardstick.sh | 4 -- src/vagrant/k8s_kubeadm/host_setup.sh | 19 ------- src/vagrant/k8s_kubeadm/master_setup.sh | 13 ----- src/vagrant/k8s_kubeadm/ovsdpdk/Dockerfile | 8 --- src/vagrant/k8s_kubeadm/ovsdpdk/install.sh | 6 --- src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml | 58 ---------------------- src/vagrant/k8s_kubeadm/ovsdpdk/start.sh | 28 ----------- src/vagrant/k8s_kubeadm/worker_setup.sh | 4 -- src/vagrant/kubeadm_basic/Vagrantfile | 34 +++++++++++++ src/vagrant/kubeadm_basic/examples/nginx-app.sh | 10 ++++ src/vagrant/kubeadm_basic/examples/nginx-app.yaml | 31 ++++++++++++ src/vagrant/kubeadm_basic/host_setup.sh | 19 +++++++ src/vagrant/kubeadm_basic/master_setup.sh | 11 ++++ src/vagrant/kubeadm_basic/worker_setup.sh | 4 ++ src/vagrant/kubeadm_ovsdpdk/Vagrantfile | 34 +++++++++++++ .../kubeadm_ovsdpdk/examples/virtio-user.yaml | 29 +++++++++++ src/vagrant/kubeadm_ovsdpdk/host_setup.sh | 19 +++++++ src/vagrant/kubeadm_ovsdpdk/master_setup.sh | 10 ++++ src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile | 8 +++ src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh | 6 +++ .../kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml | 58 ++++++++++++++++++++++ src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh | 28 +++++++++++ src/vagrant/kubeadm_ovsdpdk/worker_setup.sh | 4 ++ 29 files changed, 328 insertions(+), 267 deletions(-) create mode 100644 src/vagrant/README.md delete mode 100644 src/vagrant/k8s_kubeadm/README.md delete mode 100644 src/vagrant/k8s_kubeadm/Vagrantfile delete mode 100755 src/vagrant/k8s_kubeadm/examples/nginx-app.sh delete mode 100644 src/vagrant/k8s_kubeadm/examples/nginx-app.yaml delete mode 100644 src/vagrant/k8s_kubeadm/examples/virtio-user.yaml delete mode 100755 src/vagrant/k8s_kubeadm/examples/yardstick.sh delete mode 100644 src/vagrant/k8s_kubeadm/host_setup.sh delete mode 100644 src/vagrant/k8s_kubeadm/master_setup.sh delete mode 100644 src/vagrant/k8s_kubeadm/ovsdpdk/Dockerfile delete mode 100644 src/vagrant/k8s_kubeadm/ovsdpdk/install.sh delete mode 100644 src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml delete mode 100644 src/vagrant/k8s_kubeadm/ovsdpdk/start.sh delete mode 100644 src/vagrant/k8s_kubeadm/worker_setup.sh create mode 100644 src/vagrant/kubeadm_basic/Vagrantfile create mode 100755 src/vagrant/kubeadm_basic/examples/nginx-app.sh create mode 100644 src/vagrant/kubeadm_basic/examples/nginx-app.yaml create mode 100644 src/vagrant/kubeadm_basic/host_setup.sh create mode 100644 src/vagrant/kubeadm_basic/master_setup.sh create mode 100644 src/vagrant/kubeadm_basic/worker_setup.sh create mode 100644 src/vagrant/kubeadm_ovsdpdk/Vagrantfile create mode 100644 src/vagrant/kubeadm_ovsdpdk/examples/virtio-user.yaml create mode 100644 src/vagrant/kubeadm_ovsdpdk/host_setup.sh create mode 100644 src/vagrant/kubeadm_ovsdpdk/master_setup.sh create mode 100644 src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile create mode 100644 src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh create mode 100644 src/vagrant/kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml create mode 100644 src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh create mode 100644 src/vagrant/kubeadm_ovsdpdk/worker_setup.sh (limited to 'src') diff --git a/src/vagrant/README.md b/src/vagrant/README.md new file mode 100644 index 0000000..ed5f6c6 --- /dev/null +++ b/src/vagrant/README.md @@ -0,0 +1,23 @@ +Vagrant +======= + +vagrant is to create kubernetes cluster using kubeadm. +kubernetes installation by kubeadm can be refered to +https://kubernetes.io/docs/getting-started-guides/kubeadm + +Vagrant Setup +------------- + +sudo apt-get install -y virtualbox +wget --no-check-certificate https://releases.hashicorp.com/vagrant/1.8.6/vagrant_1.8.6_x86_64.deb +sudo dpkg -i vagrant_1.8.6_x86_64.deb + +K8s Setup +--------- + +vagrant up + +K8s Cleanup +----------- + +vagrant destroy -f diff --git a/src/vagrant/k8s_kubeadm/README.md b/src/vagrant/k8s_kubeadm/README.md deleted file mode 100644 index ed5f6c6..0000000 --- a/src/vagrant/k8s_kubeadm/README.md +++ /dev/null @@ -1,23 +0,0 @@ -Vagrant -======= - -vagrant is to create kubernetes cluster using kubeadm. -kubernetes installation by kubeadm can be refered to -https://kubernetes.io/docs/getting-started-guides/kubeadm - -Vagrant Setup -------------- - -sudo apt-get install -y virtualbox -wget --no-check-certificate https://releases.hashicorp.com/vagrant/1.8.6/vagrant_1.8.6_x86_64.deb -sudo dpkg -i vagrant_1.8.6_x86_64.deb - -K8s Setup ---------- - -vagrant up - -K8s Cleanup ------------ - -vagrant destroy -f diff --git a/src/vagrant/k8s_kubeadm/Vagrantfile b/src/vagrant/k8s_kubeadm/Vagrantfile deleted file mode 100644 index f1e7aac..0000000 --- a/src/vagrant/k8s_kubeadm/Vagrantfile +++ /dev/null @@ -1,34 +0,0 @@ -$num_workers=3 - -Vagrant.require_version ">= 1.8.6" -Vagrant.configure("2") do |config| - - config.vm.box = "bento/ubuntu-16.04" - config.vm.synced_folder "../..", "/src" - config.vm.provision "shell", path: "host_setup.sh", privileged: false - config.vm.provider :virtualbox do |vb| - vb.customize ["modifyvm", :id, "--memory", 4096] - vb.customize ["modifyvm", :id, "--cpus", 4] - vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"] - vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"] - vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"] - vb.customize "post-boot",["controlvm", :id, "setlinkstate1", "on"] - end - - config.vm.define "master" do |config| - config.vm.hostname = "master" - config.vm.provision "shell", path: "master_setup.sh", privileged: false - config.vm.network :private_network, ip: "192.168.1.10" - config.vm.network :private_network, ip: "10.244.0.10" - end - - (1 .. $num_workers).each do |i| - config.vm.define vm_name = "worker%d" % [i] do |config| - config.vm.hostname = vm_name - config.vm.provision "shell", path: "worker_setup.sh", privileged: false - config.vm.network :private_network, ip: "192.168.1.#{i+20}" - config.vm.network :private_network, ip: "10.244.0.#{i+20}" - end - end - -end diff --git a/src/vagrant/k8s_kubeadm/examples/nginx-app.sh b/src/vagrant/k8s_kubeadm/examples/nginx-app.sh deleted file mode 100755 index bfd0613..0000000 --- a/src/vagrant/k8s_kubeadm/examples/nginx-app.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -kubectl create -f /vagrant/examples/nginx-app.yaml -kubectl get nodes -kubectl get services -kubectl get pods -kubectl get rc -sleep 120 -svcip=$(kubectl get services nginx -o json | grep clusterIP | cut -f4 -d'"') -wget http://$svcip diff --git a/src/vagrant/k8s_kubeadm/examples/nginx-app.yaml b/src/vagrant/k8s_kubeadm/examples/nginx-app.yaml deleted file mode 100644 index f80881a..0000000 --- a/src/vagrant/k8s_kubeadm/examples/nginx-app.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: nginx - labels: - app: nginx -spec: - type: NodePort - ports: - - port: 80 - protocol: TCP - name: http - selector: - app: nginx ---- -apiVersion: v1 -kind: ReplicationController -metadata: - name: nginx -spec: - replicas: 2 - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 diff --git a/src/vagrant/k8s_kubeadm/examples/virtio-user.yaml b/src/vagrant/k8s_kubeadm/examples/virtio-user.yaml deleted file mode 100644 index 9ab1e06..0000000 --- a/src/vagrant/k8s_kubeadm/examples/virtio-user.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: virtiouser -spec: - replicas: 2 - template: - metadata: - labels: - app: virtiouser - spec: - containers: - - name: virtiouser - image: openretriever/virtio-user-ping - volumeMounts: - - mountPath: /dev/hugepages - name: hugepage-volume - - mountPath: /var/run - name: vhost-volume - command: - - /root/setup_virtio_user.sh - volumes: - - name: hugepage-volume - hostPath: - path: /dev/hugepages - - name: vhost-volume - hostPath: - path: /var/run - restartPolicy: Always diff --git a/src/vagrant/k8s_kubeadm/examples/yardstick.sh b/src/vagrant/k8s_kubeadm/examples/yardstick.sh deleted file mode 100755 index bc1eecf..0000000 --- a/src/vagrant/k8s_kubeadm/examples/yardstick.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -sudo docker run -tid -v /etc/kubernetes/admin.conf:/etc/yardstick/admin.conf --name yardstick opnfv/yardstick:latest -sudo docker exec -ti yardstick yardstick task start yardstick/samples/ping_k8s.yaml diff --git a/src/vagrant/k8s_kubeadm/host_setup.sh b/src/vagrant/k8s_kubeadm/host_setup.sh deleted file mode 100644 index 990df7f..0000000 --- a/src/vagrant/k8s_kubeadm/host_setup.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -set -ex - -cat << EOF | sudo tee /etc/hosts -127.0.0.1 localhost -192.168.1.10 master -192.168.1.21 worker1 -192.168.1.22 worker2 -192.168.1.23 worker3 -EOF - -curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - -cat <> $HOME/.bash_profile - -#kubectl apply -f http://git.io/weave-kube-1.6 -#kubectl apply -f http://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml -#kubectl apply -f http://docs.projectcalico.org/v2.1/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml -kubectl apply -f /vagrant/ovsdpdk/kube_ovsdpdk.yml -kubectl apply -f /src/cni/ovsdpdk/kube_cniovsdpdk.yml diff --git a/src/vagrant/k8s_kubeadm/ovsdpdk/Dockerfile b/src/vagrant/k8s_kubeadm/ovsdpdk/Dockerfile deleted file mode 100644 index a63df8f..0000000 --- a/src/vagrant/k8s_kubeadm/ovsdpdk/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM ubuntu:16.04 - -WORKDIR /ovsdpdk -RUN apt-get update && apt-get install -y sudo -ADD install.sh . -RUN bash ./install.sh -ADD start.sh . -RUN chmod 755 start.sh diff --git a/src/vagrant/k8s_kubeadm/ovsdpdk/install.sh b/src/vagrant/k8s_kubeadm/ovsdpdk/install.sh deleted file mode 100644 index f565025..0000000 --- a/src/vagrant/k8s_kubeadm/ovsdpdk/install.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -ex -sudo apt-get update -sudo apt-get install -y openvswitch-switch-dpdk pciutils vim -sudo update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk diff --git a/src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml b/src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml deleted file mode 100644 index ed04ccd..0000000 --- a/src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ovsdpdk - namespace: kube-system ---- -apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - name: kube-ovsdpdk-ds - namespace: kube-system - labels: - tier: node - app: ovsdpdk -spec: - template: - metadata: - labels: - tier: node - app: ovsdpdk - spec: - hostNetwork: true - nodeSelector: - beta.kubernetes.io/arch: amd64 - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - serviceAccountName: ovsdpdk - containers: - - name: install-ovsdpdk - image: openretriever/ubuntu1604-ovsdpdk - command: [ "/bin/bash", "/ovsdpdk/start.sh" ] - securityContext: - privileged: true - volumeMounts: - - name: local-bin - mountPath: /usr/local/bin - - name: var-run - mountPath: /var/run/openvswitch - - name: lib-modules - mountPath: /lib/modules - - name: dev-hugepage - mountPath: /dev - volumes: - - name: local-bin - hostPath: - path: /usr/local/bin - - name: var-run - hostPath: - path: /var/run/openvswitch - - name: lib-modules - hostPath: - path: /lib/modules - - name: dev-hugepage - hostPath: - path: /dev diff --git a/src/vagrant/k8s_kubeadm/ovsdpdk/start.sh b/src/vagrant/k8s_kubeadm/ovsdpdk/start.sh deleted file mode 100644 index fed7985..0000000 --- a/src/vagrant/k8s_kubeadm/ovsdpdk/start.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -ex - -for i in {1..10} -do - sudo sysctl -w vm.nr_hugepages=2048; sleep 1 -done -sudo modprobe uio_pci_generic -ip=$(ip a s enp0s9 | grep inet | grep -v inet6 | sed "s/.*inet//" | cut -f2 -d' ') -sudo ip address flush enp0s9 -sudo /usr/share/dpdk/tools/dpdk_nic_bind.py --bind=uio_pci_generic enp0s9 -sudo sysctl -w vm.nr_hugepages=1024 -sudo mount -t hugetlbfs -o pagesize=2M none /dev/hugepages -sudo cp /usr/bin/ovs-vsctl /usr/local/bin -memory=$(grep HugePages_Total /proc/meminfo | cut -f2 -d:) -echo "DPDK_OPTS='--dpdk -c 0x1 -n 2 -m $memory'" | sudo tee -a /etc/default/openvswitch-switch -sudo service dpdk restart -sudo service openvswitch-switch restart -sudo pkill ovs-vswitchd -sudo ovs-vswitchd --dpdk -c 0x1 -n 2 -m $memory -- unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach --monitor -sudo ovs-vsctl add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev -sudo ovs-vsctl add-port br-dpdk dpdk0 -- set Interface dpdk0 type=dpdk -sudo ip a a $ip dev br-dpdk -sudo ip link set dev br-dpdk up -while true; do sleep 3600; done -echo sudo docker build -t openretriever/ubuntu1604-ovsdpdk . -echo sudo docker run -ti --privileged --net=host -v /dev:/dev -v /usr/local/bin:/usr/local/bin -v /var/run/openvswitch/:/var/run/openvswitch/ -v /lib/modules/:/lib/modules openretriever/ubuntu1604-ovsdpdk bash diff --git a/src/vagrant/k8s_kubeadm/worker_setup.sh b/src/vagrant/k8s_kubeadm/worker_setup.sh deleted file mode 100644 index b68d800..0000000 --- a/src/vagrant/k8s_kubeadm/worker_setup.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -set -ex -sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true diff --git a/src/vagrant/kubeadm_basic/Vagrantfile b/src/vagrant/kubeadm_basic/Vagrantfile new file mode 100644 index 0000000..f1e7aac --- /dev/null +++ b/src/vagrant/kubeadm_basic/Vagrantfile @@ -0,0 +1,34 @@ +$num_workers=3 + +Vagrant.require_version ">= 1.8.6" +Vagrant.configure("2") do |config| + + config.vm.box = "bento/ubuntu-16.04" + config.vm.synced_folder "../..", "/src" + config.vm.provision "shell", path: "host_setup.sh", privileged: false + config.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--memory", 4096] + vb.customize ["modifyvm", :id, "--cpus", 4] + vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"] + vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"] + vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"] + vb.customize "post-boot",["controlvm", :id, "setlinkstate1", "on"] + end + + config.vm.define "master" do |config| + config.vm.hostname = "master" + config.vm.provision "shell", path: "master_setup.sh", privileged: false + config.vm.network :private_network, ip: "192.168.1.10" + config.vm.network :private_network, ip: "10.244.0.10" + end + + (1 .. $num_workers).each do |i| + config.vm.define vm_name = "worker%d" % [i] do |config| + config.vm.hostname = vm_name + config.vm.provision "shell", path: "worker_setup.sh", privileged: false + config.vm.network :private_network, ip: "192.168.1.#{i+20}" + config.vm.network :private_network, ip: "10.244.0.#{i+20}" + end + end + +end diff --git a/src/vagrant/kubeadm_basic/examples/nginx-app.sh b/src/vagrant/kubeadm_basic/examples/nginx-app.sh new file mode 100755 index 0000000..bfd0613 --- /dev/null +++ b/src/vagrant/kubeadm_basic/examples/nginx-app.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +kubectl create -f /vagrant/examples/nginx-app.yaml +kubectl get nodes +kubectl get services +kubectl get pods +kubectl get rc +sleep 120 +svcip=$(kubectl get services nginx -o json | grep clusterIP | cut -f4 -d'"') +wget http://$svcip diff --git a/src/vagrant/kubeadm_basic/examples/nginx-app.yaml b/src/vagrant/kubeadm_basic/examples/nginx-app.yaml new file mode 100644 index 0000000..f80881a --- /dev/null +++ b/src/vagrant/kubeadm_basic/examples/nginx-app.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx + labels: + app: nginx +spec: + type: NodePort + ports: + - port: 80 + protocol: TCP + name: http + selector: + app: nginx +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: nginx +spec: + replicas: 2 + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 diff --git a/src/vagrant/kubeadm_basic/host_setup.sh b/src/vagrant/kubeadm_basic/host_setup.sh new file mode 100644 index 0000000..990df7f --- /dev/null +++ b/src/vagrant/kubeadm_basic/host_setup.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -ex + +cat << EOF | sudo tee /etc/hosts +127.0.0.1 localhost +192.168.1.10 master +192.168.1.21 worker1 +192.168.1.22 worker2 +192.168.1.23 worker3 +EOF + +curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +cat <> $HOME/.bash_profile + +kubectl apply -f http://git.io/weave-kube-1.6 +#kubectl apply -f http://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml +#kubectl apply -f http://docs.projectcalico.org/v2.1/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml diff --git a/src/vagrant/kubeadm_basic/worker_setup.sh b/src/vagrant/kubeadm_basic/worker_setup.sh new file mode 100644 index 0000000..b68d800 --- /dev/null +++ b/src/vagrant/kubeadm_basic/worker_setup.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +set -ex +sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true diff --git a/src/vagrant/kubeadm_ovsdpdk/Vagrantfile b/src/vagrant/kubeadm_ovsdpdk/Vagrantfile new file mode 100644 index 0000000..f1e7aac --- /dev/null +++ b/src/vagrant/kubeadm_ovsdpdk/Vagrantfile @@ -0,0 +1,34 @@ +$num_workers=3 + +Vagrant.require_version ">= 1.8.6" +Vagrant.configure("2") do |config| + + config.vm.box = "bento/ubuntu-16.04" + config.vm.synced_folder "../..", "/src" + config.vm.provision "shell", path: "host_setup.sh", privileged: false + config.vm.provider :virtualbox do |vb| + vb.customize ["modifyvm", :id, "--memory", 4096] + vb.customize ["modifyvm", :id, "--cpus", 4] + vb.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"] + vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"] + vb.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"] + vb.customize "post-boot",["controlvm", :id, "setlinkstate1", "on"] + end + + config.vm.define "master" do |config| + config.vm.hostname = "master" + config.vm.provision "shell", path: "master_setup.sh", privileged: false + config.vm.network :private_network, ip: "192.168.1.10" + config.vm.network :private_network, ip: "10.244.0.10" + end + + (1 .. $num_workers).each do |i| + config.vm.define vm_name = "worker%d" % [i] do |config| + config.vm.hostname = vm_name + config.vm.provision "shell", path: "worker_setup.sh", privileged: false + config.vm.network :private_network, ip: "192.168.1.#{i+20}" + config.vm.network :private_network, ip: "10.244.0.#{i+20}" + end + end + +end diff --git a/src/vagrant/kubeadm_ovsdpdk/examples/virtio-user.yaml b/src/vagrant/kubeadm_ovsdpdk/examples/virtio-user.yaml new file mode 100644 index 0000000..9ab1e06 --- /dev/null +++ b/src/vagrant/kubeadm_ovsdpdk/examples/virtio-user.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ReplicationController +metadata: + name: virtiouser +spec: + replicas: 2 + template: + metadata: + labels: + app: virtiouser + spec: + containers: + - name: virtiouser + image: openretriever/virtio-user-ping + volumeMounts: + - mountPath: /dev/hugepages + name: hugepage-volume + - mountPath: /var/run + name: vhost-volume + command: + - /root/setup_virtio_user.sh + volumes: + - name: hugepage-volume + hostPath: + path: /dev/hugepages + - name: vhost-volume + hostPath: + path: /var/run + restartPolicy: Always diff --git a/src/vagrant/kubeadm_ovsdpdk/host_setup.sh b/src/vagrant/kubeadm_ovsdpdk/host_setup.sh new file mode 100644 index 0000000..990df7f --- /dev/null +++ b/src/vagrant/kubeadm_ovsdpdk/host_setup.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -ex + +cat << EOF | sudo tee /etc/hosts +127.0.0.1 localhost +192.168.1.10 master +192.168.1.21 worker1 +192.168.1.22 worker2 +192.168.1.23 worker3 +EOF + +curl -s http://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add - +cat <> $HOME/.bash_profile + +kubectl apply -f /vagrant/ovsdpdk/kube_ovsdpdk.yml +kubectl apply -f /src/cni/ovsdpdk/kube_cniovsdpdk.yml diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile new file mode 100644 index 0000000..a63df8f --- /dev/null +++ b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:16.04 + +WORKDIR /ovsdpdk +RUN apt-get update && apt-get install -y sudo +ADD install.sh . +RUN bash ./install.sh +ADD start.sh . +RUN chmod 755 start.sh diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh new file mode 100644 index 0000000..f565025 --- /dev/null +++ b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +set -ex +sudo apt-get update +sudo apt-get install -y openvswitch-switch-dpdk pciutils vim +sudo update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml new file mode 100644 index 0000000..ed04ccd --- /dev/null +++ b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml @@ -0,0 +1,58 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ovsdpdk + namespace: kube-system +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-ovsdpdk-ds + namespace: kube-system + labels: + tier: node + app: ovsdpdk +spec: + template: + metadata: + labels: + tier: node + app: ovsdpdk + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: ovsdpdk + containers: + - name: install-ovsdpdk + image: openretriever/ubuntu1604-ovsdpdk + command: [ "/bin/bash", "/ovsdpdk/start.sh" ] + securityContext: + privileged: true + volumeMounts: + - name: local-bin + mountPath: /usr/local/bin + - name: var-run + mountPath: /var/run/openvswitch + - name: lib-modules + mountPath: /lib/modules + - name: dev-hugepage + mountPath: /dev + volumes: + - name: local-bin + hostPath: + path: /usr/local/bin + - name: var-run + hostPath: + path: /var/run/openvswitch + - name: lib-modules + hostPath: + path: /lib/modules + - name: dev-hugepage + hostPath: + path: /dev diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh new file mode 100644 index 0000000..fed7985 --- /dev/null +++ b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -ex + +for i in {1..10} +do + sudo sysctl -w vm.nr_hugepages=2048; sleep 1 +done +sudo modprobe uio_pci_generic +ip=$(ip a s enp0s9 | grep inet | grep -v inet6 | sed "s/.*inet//" | cut -f2 -d' ') +sudo ip address flush enp0s9 +sudo /usr/share/dpdk/tools/dpdk_nic_bind.py --bind=uio_pci_generic enp0s9 +sudo sysctl -w vm.nr_hugepages=1024 +sudo mount -t hugetlbfs -o pagesize=2M none /dev/hugepages +sudo cp /usr/bin/ovs-vsctl /usr/local/bin +memory=$(grep HugePages_Total /proc/meminfo | cut -f2 -d:) +echo "DPDK_OPTS='--dpdk -c 0x1 -n 2 -m $memory'" | sudo tee -a /etc/default/openvswitch-switch +sudo service dpdk restart +sudo service openvswitch-switch restart +sudo pkill ovs-vswitchd +sudo ovs-vswitchd --dpdk -c 0x1 -n 2 -m $memory -- unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach --monitor +sudo ovs-vsctl add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev +sudo ovs-vsctl add-port br-dpdk dpdk0 -- set Interface dpdk0 type=dpdk +sudo ip a a $ip dev br-dpdk +sudo ip link set dev br-dpdk up +while true; do sleep 3600; done +echo sudo docker build -t openretriever/ubuntu1604-ovsdpdk . +echo sudo docker run -ti --privileged --net=host -v /dev:/dev -v /usr/local/bin:/usr/local/bin -v /var/run/openvswitch/:/var/run/openvswitch/ -v /lib/modules/:/lib/modules openretriever/ubuntu1604-ovsdpdk bash diff --git a/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh b/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh new file mode 100644 index 0000000..b68d800 --- /dev/null +++ b/src/vagrant/kubeadm_ovsdpdk/worker_setup.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +set -ex +sudo kubeadm join --token 8c5adc.1cec8dbf339093f0 192.168.1.10:6443 || true -- cgit 1.2.3-korg