From da51aabb3171afb8107f837fd0ccfe1add403c4d Mon Sep 17 00:00:00 2001 From: Guo Ruijing Date: Thu, 3 Aug 2017 17:54:29 -0700 Subject: Fix some issues in ovsdpdk, cni for ovsdpdk Change-Id: Ib919a14a74c6d33c74c7d4f9baa5fd81b9054cbd Signed-off-by: Guo Ruijing --- src/cni/ovsdpdk/install_cni.sh | 2 +- src/cni/ovsdpdk/kube_cniovsdpdk.yml | 71 ++++++++++++++++++++++++ src/cni/ovsdpdk/kube_ovsdpdk.yml | 71 ------------------------ src/vagrant/k8s_kubeadm/master_setup.sh | 6 +- src/vagrant/k8s_kubeadm/ovsdpdk/install.sh | 3 +- src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml | 5 ++ src/vagrant/k8s_kubeadm/ovsdpdk/start.sh | 24 +++++--- 7 files changed, 98 insertions(+), 84 deletions(-) create mode 100644 src/cni/ovsdpdk/kube_cniovsdpdk.yml delete mode 100644 src/cni/ovsdpdk/kube_ovsdpdk.yml diff --git a/src/cni/ovsdpdk/install_cni.sh b/src/cni/ovsdpdk/install_cni.sh index 8f5b78a..1996700 100644 --- a/src/cni/ovsdpdk/install_cni.sh +++ b/src/cni/ovsdpdk/install_cni.sh @@ -4,5 +4,5 @@ set -ex cp /cni/ovsdpdk /opt/cni/bin cp /cni/setup_ovsdpdk.sh /opt/cni/bin cp /cni/teardown_ovsdpdk.sh /opt/cni/bin -cp /etc/kube-ovsdpdk/cni-conf.json /etc/cni/net.d/10-ovsdpdk.conf +cp /etc/kube-cniovsdpdk/cni-conf.json /etc/cni/net.d/10-ovsdpdk.conf while true; do sleep 3600; done diff --git a/src/cni/ovsdpdk/kube_cniovsdpdk.yml b/src/cni/ovsdpdk/kube_cniovsdpdk.yml new file mode 100644 index 0000000..8e8ed0e --- /dev/null +++ b/src/cni/ovsdpdk/kube_cniovsdpdk.yml @@ -0,0 +1,71 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cniovsdpdk + namespace: kube-system +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-cniovsdpdk-cfg + namespace: kube-system + labels: + tier: node + app: cniovsdpdk +data: + cni-conf.json: | + { + "name": "ovsdpdk", + "type": "ovsdpdk", + "bridge": "br-dpdk", + "ipam": { + "type": "host-local", + "subnet": "10.244.0.0/16" + } + } +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-cniovsdpdk-ds + namespace: kube-system + labels: + tier: node + app: cniovsdpdk +spec: + template: + metadata: + labels: + tier: node + app: cniovsdpdk + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: cniovsdpdk + containers: + - name: install-cni + image: openretriever/cni-ovsdpdk + command: [ "/bin/bash", "/cni/install_cni.sh" ] + volumeMounts: + - name: cni-bin + mountPath: /opt/cni/bin + - name: cni-cfg + mountPath: /etc/cni/net.d + - name: cniovsdpdk-cfg + mountPath: /etc/kube-cniovsdpdk + volumes: + - name: cni-bin + hostPath: + path: /opt/cni/bin + - name: cni-cfg + hostPath: + path: /etc/cni/net.d + - name: cniovsdpdk-cfg + configMap: + name: kube-cniovsdpdk-cfg diff --git a/src/cni/ovsdpdk/kube_ovsdpdk.yml b/src/cni/ovsdpdk/kube_ovsdpdk.yml deleted file mode 100644 index 2bcebdc..0000000 --- a/src/cni/ovsdpdk/kube_ovsdpdk.yml +++ /dev/null @@ -1,71 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ovsdpdk - namespace: kube-system ---- -kind: ConfigMap -apiVersion: v1 -metadata: - name: kube-ovsdpdk-cfg - namespace: kube-system - labels: - tier: node - app: ovsdpdk -data: - cni-conf.json: | - { - "name": "ovsdpdk", - "type": "ovsdpdk", - "bridge": "br-dpdk", - "ipam": { - "type": "host-local", - "subnet": "10.244.0.0/16" - } - } ---- -apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - name: kube-ovsdpdk-ds - namespace: kube-system - labels: - tier: node - app: ovsdpdk -spec: - template: - metadata: - labels: - tier: node - app: ovsdpdk - spec: - hostNetwork: true - nodeSelector: - beta.kubernetes.io/arch: amd64 - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - serviceAccountName: ovsdpdk - containers: - - name: install-cni - image: openretriever/cni-ovsdpdk - command: [ "/bin/bash", "/cni/install_cni.sh" ] - volumeMounts: - - name: cni-bin - mountPath: /opt/cni/bin - - name: cni-cfg - mountPath: /etc/cni/net.d - - name: ovsdpdk-cfg - mountPath: /etc/kube-ovsdpdk - volumes: - - name: cni-bin - hostPath: - path: /opt/cni/bin - - name: cni-cfg - hostPath: - path: /etc/cni/net.d - - name: ovsdpdk-cfg - configMap: - name: kube-ovsdpdk-cfg diff --git a/src/vagrant/k8s_kubeadm/master_setup.sh b/src/vagrant/k8s_kubeadm/master_setup.sh index e98e2bb..06785b6 100644 --- a/src/vagrant/k8s_kubeadm/master_setup.sh +++ b/src/vagrant/k8s_kubeadm/master_setup.sh @@ -8,8 +8,8 @@ sudo chown $(id -u):$(id -g) $HOME/admin.conf export KUBECONFIG=$HOME/admin.conf echo "export KUBECONFIG=$HOME/admin.conf" >> $HOME/.bash_profile -kubectl apply -f http://git.io/weave-kube-1.6 +#kubectl apply -f http://git.io/weave-kube-1.6 #kubectl apply -f http://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml #kubectl apply -f http://docs.projectcalico.org/v2.1/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml -#kubectl apply -f /vagrant/k8s_kubeadm/dpdk/kube_ovsdpdk.yml -#kubectl apply -f /src/cni/ovsdpdk/kube_ovsdpdk.yml +kubectl apply -f /vagrant/ovsdpdk/kube_ovsdpdk.yml +kubectl apply -f /src/cni/ovsdpdk/kube_cniovsdpdk.yml diff --git a/src/vagrant/k8s_kubeadm/ovsdpdk/install.sh b/src/vagrant/k8s_kubeadm/ovsdpdk/install.sh index 66fb71d..f565025 100644 --- a/src/vagrant/k8s_kubeadm/ovsdpdk/install.sh +++ b/src/vagrant/k8s_kubeadm/ovsdpdk/install.sh @@ -2,6 +2,5 @@ set -ex sudo apt-get update -sudo apt-get install -y openvswitch-switch-dpdk linux-image-extra-4.4.0-75-generic +sudo apt-get install -y openvswitch-switch-dpdk pciutils vim sudo update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk -echo "DPDK_OPTS='--dpdk -c 0x1 -n 4 -m 1024'" | sudo tee -a /etc/default/openvswitch-switch diff --git a/src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml b/src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml index d79da15..ed04ccd 100644 --- a/src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml +++ b/src/vagrant/k8s_kubeadm/ovsdpdk/kube_ovsdpdk.yml @@ -39,6 +39,8 @@ spec: mountPath: /usr/local/bin - name: var-run mountPath: /var/run/openvswitch + - name: lib-modules + mountPath: /lib/modules - name: dev-hugepage mountPath: /dev volumes: @@ -48,6 +50,9 @@ spec: - name: var-run hostPath: path: /var/run/openvswitch + - name: lib-modules + hostPath: + path: /lib/modules - name: dev-hugepage hostPath: path: /dev diff --git a/src/vagrant/k8s_kubeadm/ovsdpdk/start.sh b/src/vagrant/k8s_kubeadm/ovsdpdk/start.sh index 08d8143..d31a242 100644 --- a/src/vagrant/k8s_kubeadm/ovsdpdk/start.sh +++ b/src/vagrant/k8s_kubeadm/ovsdpdk/start.sh @@ -1,15 +1,25 @@ #!/bin/bash set -ex -sudo sysctl -w vm.nr_hugepages=2048 + +for i in {1..10} +do + sudo sysctl -w vm.nr_hugepages=2048; sleep 1 +done +sudo modprobe uio_pci_generic +sudo ip address flush enp0s9 +sudo /usr/share/dpdk/tools/dpdk_nic_bind.py --bind=uio_pci_generic enp0s9 +sudo sysctl -w vm.nr_hugepages=1024 sudo mount -t hugetlbfs -o pagesize=2M none /dev/hugepages -cp /usr/bin/ovs-vsctl /usr/local/bin +sudo cp /usr/bin/ovs-vsctl /usr/local/bin +memory=$(grep HugePages_Total /proc/meminfo | cut -f2 -d:) +echo "DPDK_OPTS='--dpdk -c 0x1 -n 2 -m $memory'" | sudo tee -a /etc/default/openvswitch-switch sudo service dpdk restart sudo service openvswitch-switch restart +sudo pkill ovs-vswitchd +sudo ovs-vswitchd --dpdk -c 0x1 -n 2 -m $memory -- unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach --monitor sudo ovs-vsctl add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev -sudo modprobe uio_pci_generic -#sudo ip address flush enp0s9 -#sudo /usr/share/dpdk/tools/dpdk_nic_bind.py --bind=uio_pci_generic enp0s9 -#sudo ovs-vsctl add-port br-dpdk dpdk0 -- set Interface dpdk0 type=dpdk +sudo ovs-vsctl add-port br-dpdk dpdk0 -- set Interface dpdk0 type=dpdk while true; do sleep 3600; done -echo sudo docker run -ti --privileged -v /dev:/dev -v /usr/local/bin:/usr/local/bin -v /var/run/openvswitch/:/var/run/openvswitch/ dpdk /ovsdpdk/start.sh +echo sudo docker build -t openretriever/ubuntu1604-ovsdpdk . +echo sudo docker run -ti --privileged --net=host -v /dev:/dev -v /usr/local/bin:/usr/local/bin -v /var/run/openvswitch/:/var/run/openvswitch/ -v /lib/modules/:/lib/modules openretriever/ubuntu1604-ovsdpdk bash -- cgit 1.2.3-korg