From a8388222d5d1f11be506e9e79f5e2476c447177e Mon Sep 17 00:00:00 2001 From: Guo Ruijing Date: Thu, 23 Nov 2017 12:58:05 -0500 Subject: build ovsdpdk cni during deployment Change-Id: I1c3d4b44bab0d75fae93f7fabce7bcfdcc3b9f35 Signed-off-by: Guo Ruijing --- src/cni/ovsdpdk/Dockerfile.build | 6 - src/cni/ovsdpdk/Dockerfile.cniovsdpdk | 3 - src/cni/ovsdpdk/build.sh | 20 +-- src/cni/ovsdpdk/build_cni.sh | 13 -- src/cni/ovsdpdk/cni/install_cni.sh | 8 -- src/cni/ovsdpdk/cni/setup_ovsdpdk.sh | 16 --- src/cni/ovsdpdk/cni/teardown_ovsdpdk.sh | 9 -- src/cni/ovsdpdk/install_cni.sh | 22 ++++ src/cni/ovsdpdk/kube_cniovsdpdk.yml | 14 ++- src/cni/ovsdpdk/kube_ovsdpdk.yml | 61 +++++++++ src/cni/ovsdpdk/ovsdpdk.go | 107 ++++++++++++++++ src/cni/ovsdpdk/ovsdpdk.patch | 136 --------------------- src/cni/ovsdpdk/setup_ovsdpdk.sh | 16 +++ src/cni/ovsdpdk/start.sh | 32 +++++ src/cni/ovsdpdk/teardown_ovsdpdk.sh | 9 ++ .../kubeadm_ovsdpdk/examples/virtio-user.sh | 2 +- src/vagrant/kubeadm_ovsdpdk/master_setup.sh | 2 +- src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile | 8 -- src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh | 6 - .../kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml | 58 --------- src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh | 28 ----- 21 files changed, 261 insertions(+), 315 deletions(-) delete mode 100644 src/cni/ovsdpdk/Dockerfile.build delete mode 100644 src/cni/ovsdpdk/Dockerfile.cniovsdpdk mode change 100644 => 100755 src/cni/ovsdpdk/build.sh delete mode 100644 src/cni/ovsdpdk/build_cni.sh delete mode 100644 src/cni/ovsdpdk/cni/install_cni.sh delete mode 100755 src/cni/ovsdpdk/cni/setup_ovsdpdk.sh delete mode 100755 src/cni/ovsdpdk/cni/teardown_ovsdpdk.sh create mode 100755 src/cni/ovsdpdk/install_cni.sh create mode 100644 src/cni/ovsdpdk/kube_ovsdpdk.yml create mode 100644 src/cni/ovsdpdk/ovsdpdk.go delete mode 100644 src/cni/ovsdpdk/ovsdpdk.patch create mode 100755 src/cni/ovsdpdk/setup_ovsdpdk.sh create mode 100644 src/cni/ovsdpdk/start.sh create mode 100755 src/cni/ovsdpdk/teardown_ovsdpdk.sh delete mode 100644 src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile delete mode 100644 src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh delete mode 100644 src/vagrant/kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml delete mode 100644 src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh (limited to 'src') diff --git a/src/cni/ovsdpdk/Dockerfile.build b/src/cni/ovsdpdk/Dockerfile.build deleted file mode 100644 index ed5731e..0000000 --- a/src/cni/ovsdpdk/Dockerfile.build +++ /dev/null @@ -1,6 +0,0 @@ -FROM ubuntu:16.04 - -RUN apt-get update && apt-get install -y sudo git wget -ADD ovsdpdk.patch / -ADD build_cni.sh / -RUN bash /build_cni.sh diff --git a/src/cni/ovsdpdk/Dockerfile.cniovsdpdk b/src/cni/ovsdpdk/Dockerfile.cniovsdpdk deleted file mode 100644 index 2dd4f3c..0000000 --- a/src/cni/ovsdpdk/Dockerfile.cniovsdpdk +++ /dev/null @@ -1,3 +0,0 @@ -FROM ubuntu:16.04 - -ADD cni /cni diff --git a/src/cni/ovsdpdk/build.sh b/src/cni/ovsdpdk/build.sh old mode 100644 new mode 100755 index 625c7cc..5ddea99 --- a/src/cni/ovsdpdk/build.sh +++ b/src/cni/ovsdpdk/build.sh @@ -2,24 +2,6 @@ set -ex DIR="$(dirname `readlink -f $0`)" -PWD="$(pwd)" cd $DIR - -sudo apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D -sudo apt-key adv -k 58118E89F3A912897C070ADBF76221572C52609D -cat << EOF | sudo tee /etc/apt/sources.list.d/docker.list -deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main -EOF - -sudo apt-get install -y --allow-downgrades docker-engine=1.12.6-0~ubuntu-xenial -sudo docker rmi -f cniovsdpdk.build || true -sudo docker build -t cniovsdpdk.build . -f Dockerfile.build -sudo docker run -v $DIR/cni:/build -t cniovsdpdk.build cp /plugins/bin/ovsdpdk /build -sudo docker build -t openretriever/cnicniovsdpdk . -f Dockerfile.cniovsdpdk -sudo rm -rf $DIR/cni/ovsdpdk -echo git clone https://github.com/containernetworking/cni -echo sudo CNI_PATH=$CNI_PATH ./priv-net-run.sh ifconfig -echo sudo docker login openretriever/cnicniovsdpdk -u user -p password -echo sudo docker push openretriever/cnicniovsdpdk -cd $PWD +sudo docker run -v $DIR:/build ubuntu:16.04 /build/install_cni.sh diff --git a/src/cni/ovsdpdk/build_cni.sh b/src/cni/ovsdpdk/build_cni.sh deleted file mode 100644 index dbb3dd5..0000000 --- a/src/cni/ovsdpdk/build_cni.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -ex - -sudo apt-get update -sudo apt-get install -y docker.io devscripts git jq -wget -qO- https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz | sudo tar -C /usr/local -xz -echo 'export GOPATH=/go; export PATH=/usr/local/go/bin:$GOPATH/bin:$PATH' >> ~/.bashrc -export GOPATH=/go; export PATH=/usr/local/go/bin:$GOPATH/bin:$PATH -git clone https://github.com/containernetworking/plugins -cd plugins -git checkout 5544d9ced0d6e908fe26e9dbe529c7feb87d21f5 -patch -p1 < ../ovsdpdk.patch -bash ./build.sh diff --git a/src/cni/ovsdpdk/cni/install_cni.sh b/src/cni/ovsdpdk/cni/install_cni.sh deleted file mode 100644 index 1996700..0000000 --- a/src/cni/ovsdpdk/cni/install_cni.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -ex -cp /cni/ovsdpdk /opt/cni/bin -cp /cni/setup_ovsdpdk.sh /opt/cni/bin -cp /cni/teardown_ovsdpdk.sh /opt/cni/bin -cp /etc/kube-cniovsdpdk/cni-conf.json /etc/cni/net.d/10-ovsdpdk.conf -while true; do sleep 3600; done diff --git a/src/cni/ovsdpdk/cni/setup_ovsdpdk.sh b/src/cni/ovsdpdk/cni/setup_ovsdpdk.sh deleted file mode 100755 index 6009b7a..0000000 --- a/src/cni/ovsdpdk/cni/setup_ovsdpdk.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -netns=$1 -containerid=$2 -ip=$3 -pid=$(echo $netns | cut -f3 -d"/") - -sudo ovs-vsctl --may-exist add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev -sudo ovs-vsctl --may-exist add-port br-dpdk vhost-user-$pid -- set Interface vhost-user-$pid type=dpdkvhostuser - -sudo ln -sf $netns /var/run/netns/$pid -sudo ip link add dummy-$pid type dummy -sudo ip link set dummy-$pid netns $pid - -sudo mkdir -p /var/run/cni -echo $ip | sudo tee /var/run/cni/netconf-$pid diff --git a/src/cni/ovsdpdk/cni/teardown_ovsdpdk.sh b/src/cni/ovsdpdk/cni/teardown_ovsdpdk.sh deleted file mode 100755 index 857738e..0000000 --- a/src/cni/ovsdpdk/cni/teardown_ovsdpdk.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -netns=$1 -containerid=$2 -pid=$(echo $netns | cut -f3 -d"/") - -sudo ovs-vsctl del-port br-dpdk vhost-user-$pid -sudo ip netns exec $pid link delete dummy-$pid -sudo rm -rf /var/run/cni/netconf-$pid diff --git a/src/cni/ovsdpdk/install_cni.sh b/src/cni/ovsdpdk/install_cni.sh new file mode 100755 index 0000000..946d04c --- /dev/null +++ b/src/cni/ovsdpdk/install_cni.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -ex + + +export PATH=/usr/local/go/bin:$PATH + +DIR="$(dirname `readlink -f $0`)" +cd $DIR +apt-get update && apt-get install -y git wget gcc +wget -qO- https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz | tar -C /usr/local/ -xz +rm -rf plugins +git clone https://github.com/containernetworking/plugins +cd plugins +git checkout 5544d9ced0d6e908fe26e9dbe529c7feb87d21f5 +mkdir plugins/main/ovsdpdk +cp ../ovsdpdk.go plugins/main/ovsdpdk +sed -i "s,PLUGINS=.*,PLUGINS=plugins/main/ovsdpdk," build.sh +./build.sh +mkdir -p /opt/cni/bin +cp bin/ovsdpdk /opt/cni/bin +cp ../setup_ovsdpdk.sh ../teardown_ovsdpdk.sh /opt/cni/bin +rm -rf plugins diff --git a/src/cni/ovsdpdk/kube_cniovsdpdk.yml b/src/cni/ovsdpdk/kube_cniovsdpdk.yml index 3684719..89491da 100644 --- a/src/cni/ovsdpdk/kube_cniovsdpdk.yml +++ b/src/cni/ovsdpdk/kube_cniovsdpdk.yml @@ -51,13 +51,21 @@ spec: operator: Exists effect: NoSchedule serviceAccountName: cniovsdpdk - containers: + initContainers: - name: install-cni - image: openretriever/cni-ovsdpdk - command: [ "/bin/bash", "/cni/install_cni.sh" ] + image: ubuntu:16.04 + command: + - bash + - "-c" + - "apt-get update && apt-get install -y git && git clone http://github.com/opnfv/container4nfv && cd container4nfv && git fetch https://gerrit.opnfv.org/gerrit/container4nfv refs/changes/43/47743/5 && git checkout FETCH_HEAD && bash ./src/cni/ovsdpdk/install_cni.sh" volumeMounts: - name: cni-bin mountPath: /opt/cni/bin + containers: + - name: run-cni + image: busybox:1.27.2 + command: [ 'sh', '-c', 'cp /etc/kube-cniovsdpdk/cni-conf.json /etc/cni/net.d/10-ovsdpdk.conf; while true; do sleep 10000; done' ] + volumeMounts: - name: cni-cfg mountPath: /etc/cni/net.d - name: cniovsdpdk-cfg diff --git a/src/cni/ovsdpdk/kube_ovsdpdk.yml b/src/cni/ovsdpdk/kube_ovsdpdk.yml new file mode 100644 index 0000000..76ea51f --- /dev/null +++ b/src/cni/ovsdpdk/kube_ovsdpdk.yml @@ -0,0 +1,61 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ovsdpdk + namespace: kube-system +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-ovsdpdk-ds + namespace: kube-system + labels: + tier: node + app: ovsdpdk +spec: + template: + metadata: + labels: + tier: node + app: ovsdpdk + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: ovsdpdk + containers: + - name: install-ovsdpdk + image: ubuntu:16.04 + command: + - bash + - "-c" + - "apt-get update && apt-get install -y git && git clone http://github.com/opnfv/container4nfv && cd container4nfv && git fetch https://gerrit.opnfv.org/gerrit/container4nfv refs/changes/43/47743/5 && git checkout FETCH_HEAD && bash ./src/cni/ovsdpdk/start.sh" + securityContext: + privileged: true + volumeMounts: + - name: local-bin + mountPath: /usr/local/bin + - name: var-run + mountPath: /var/run/openvswitch + - name: lib-modules + mountPath: /lib/modules + - name: dev-hugepage + mountPath: /dev + volumes: + - name: local-bin + hostPath: + path: /usr/local/bin + - name: var-run + hostPath: + path: /var/run/openvswitch + - name: lib-modules + hostPath: + path: /lib/modules + - name: dev-hugepage + hostPath: + path: /dev diff --git a/src/cni/ovsdpdk/ovsdpdk.go b/src/cni/ovsdpdk/ovsdpdk.go new file mode 100644 index 0000000..4c388b5 --- /dev/null +++ b/src/cni/ovsdpdk/ovsdpdk.go @@ -0,0 +1,107 @@ +// Copyright 2017 4 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "runtime" + "os/exec" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/cni/pkg/version" + "github.com/containernetworking/plugins/pkg/ipam" +) + +const defaultBrName = "br-dpdk" + +type NetConf struct { + types.NetConf + BrName string `json:"bridge"` +} + +func init() { + // this ensures that main runs only on main thread (thread group leader). + // since namespace ops (unshare, setns) are done for a single thread, we + // must ensure that the goroutine does not jump from OS thread to thread + runtime.LockOSThread() +} + +func loadNetConf(bytes []byte) (*NetConf, string, error) { + n := &NetConf{ + BrName: defaultBrName, + } + if err := json.Unmarshal(bytes, n); err != nil { + return nil, "", fmt.Errorf("failed to load netconf: %v", err) + } + return n, n.CNIVersion, nil +} + +func setupVhostUser(args *skel.CmdArgs, types string) error { + exec.Command("/bin/bash", "/opt/cni/bin/setup_ovsdpdk.sh", args.Netns, args.ContainerID, types).Output() + return nil +} + +func cmdAdd(args *skel.CmdArgs) error { + n, cniVersion, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + + // run the IPAM plugin and get back the config to apply + r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) + if err != nil { + return err + } + + // Convert whatever the IPAM result was into the current Result type + result, err := current.NewResultFromResult(r) + if err != nil { + return err + } + + if len(result.IPs) == 0 { + return errors.New("IPAM plugin returned missing IP config") + } + + setupVhostUser(args, result.String()) + + return types.PrintResult(result, cniVersion) +} + +func tearDownVhostUser(args *skel.CmdArgs) error { + exec.Command("/bin/bash", "/opt/cni/bin/teardown_ovsdpdk.sh", args.Netns, args.ContainerID).Output() + return nil +} + +func cmdDel(args *skel.CmdArgs) error { + n, _, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + + if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil { + return err + } + + tearDownVhostUser(args) + return err +} + +func main() { + skel.PluginMain(cmdAdd, cmdDel, version.All) +} diff --git a/src/cni/ovsdpdk/ovsdpdk.patch b/src/cni/ovsdpdk/ovsdpdk.patch deleted file mode 100644 index 67b3703..0000000 --- a/src/cni/ovsdpdk/ovsdpdk.patch +++ /dev/null @@ -1,136 +0,0 @@ -diff --git a/build.sh b/build.sh -index cd21ba8..bc60d91 100755 ---- a/build.sh -+++ b/build.sh -@@ -19,7 +19,7 @@ export GOPATH=${PWD}/gopath - mkdir -p "${PWD}/bin" - - echo "Building plugins" --PLUGINS="plugins/meta/* plugins/main/* plugins/ipam/* plugins/sample" -+PLUGINS="plugins/main/ovsdpdk plugins/main/bridge plugins/ipam/host-local" - for d in $PLUGINS; do - if [ -d "$d" ]; then - plugin="$(basename "$d")" -diff --git a/plugins/main/ovsdpdk/ovsdpdk.go b/plugins/main/ovsdpdk/ovsdpdk.go -new file mode 100644 -index 0000000..1b931d4 ---- /dev/null -+++ b/plugins/main/ovsdpdk/ovsdpdk.go -@@ -0,0 +1,117 @@ -+// Copyright 2014 CNI authors -+// -+// Licensed under the Apache License, Version 2.0 (the "License"); -+// you may not use this file except in compliance with the License. -+// You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, software -+// distributed under the License is distributed on an "AS IS" BASIS, -+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+// See the License for the specific language governing permissions and -+// limitations under the License. -+ -+package main -+ -+import ( -+ "encoding/json" -+ "errors" -+ "fmt" -+ //"net" -+ "runtime" -+ //"syscall" -+ "os/exec" -+ //"io/ioutil" -+ -+ "github.com/containernetworking/cni/pkg/skel" -+ "github.com/containernetworking/cni/pkg/types" -+ "github.com/containernetworking/cni/pkg/types/current" -+ "github.com/containernetworking/cni/pkg/version" -+ //"github.com/containernetworking/plugins/pkg/ip" -+ "github.com/containernetworking/plugins/pkg/ipam" -+ //"github.com/containernetworking/plugins/pkg/ns" -+ //"github.com/containernetworking/plugins/pkg/utils" -+ //"github.com/vishvananda/netlink" -+) -+ -+const defaultBrName = "cni0" -+ -+type NetConf struct { -+ types.NetConf -+ BrName string `json:"bridge"` -+} -+ -+func init() { -+ // this ensures that main runs only on main thread (thread group leader). -+ // since namespace ops (unshare, setns) are done for a single thread, we -+ // must ensure that the goroutine does not jump from OS thread to thread -+ runtime.LockOSThread() -+} -+ -+func loadNetConf(bytes []byte) (*NetConf, string, error) { -+ n := &NetConf{ -+ BrName: defaultBrName, -+ } -+ if err := json.Unmarshal(bytes, n); err != nil { -+ return nil, "", fmt.Errorf("failed to load netconf: %v", err) -+ } -+ return n, n.CNIVersion, nil -+} -+ -+func setupVhostUser(args *skel.CmdArgs, types string) error { -+ exec.Command("/bin/bash", "/opt/cni/bin/setup_ovsdpdk.sh", args.Netns, args.ContainerID, types).Output() -+ return nil -+} -+ -+ -+func cmdAdd(args *skel.CmdArgs) error { -+ n, cniVersion, err := loadNetConf(args.StdinData) -+ if err != nil { -+ return err -+ } -+ -+ // run the IPAM plugin and get back the config to apply -+ r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) -+ if err != nil { -+ return err -+ } -+ -+ // Convert whatever the IPAM result was into the current Result type -+ result, err := current.NewResultFromResult(r) -+ if err != nil { -+ return err -+ } -+ -+ if len(result.IPs) == 0 { -+ return errors.New("IPAM plugin returned missing IP config") -+ } -+ -+ setupVhostUser(args, result.String()) -+ -+ return types.PrintResult(result, cniVersion) -+} -+ -+func tearDownVhostUser(args *skel.CmdArgs) error { -+ exec.Command("/bin/bash", "/opt/cni/bin/teardown_ovsdpdk.sh", args.Netns, args.ContainerID).Output() -+ return nil -+} -+ -+func cmdDel(args *skel.CmdArgs) error { -+ n, _, err := loadNetConf(args.StdinData) -+ if err != nil { -+ return err -+ } -+ -+ if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil { -+ return err -+ } -+ -+ tearDownVhostUser(args) -+ return err -+ -+} -+ -+func main() { -+ skel.PluginMain(cmdAdd, cmdDel, version.All) -+} diff --git a/src/cni/ovsdpdk/setup_ovsdpdk.sh b/src/cni/ovsdpdk/setup_ovsdpdk.sh new file mode 100755 index 0000000..6009b7a --- /dev/null +++ b/src/cni/ovsdpdk/setup_ovsdpdk.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +netns=$1 +containerid=$2 +ip=$3 +pid=$(echo $netns | cut -f3 -d"/") + +sudo ovs-vsctl --may-exist add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev +sudo ovs-vsctl --may-exist add-port br-dpdk vhost-user-$pid -- set Interface vhost-user-$pid type=dpdkvhostuser + +sudo ln -sf $netns /var/run/netns/$pid +sudo ip link add dummy-$pid type dummy +sudo ip link set dummy-$pid netns $pid + +sudo mkdir -p /var/run/cni +echo $ip | sudo tee /var/run/cni/netconf-$pid diff --git a/src/cni/ovsdpdk/start.sh b/src/cni/ovsdpdk/start.sh new file mode 100644 index 0000000..53b5fe1 --- /dev/null +++ b/src/cni/ovsdpdk/start.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -ex + +for i in {1..10} +do + sysctl -w vm.nr_hugepages=2048; sleep 1 +done + +apt-get update +apt-get install -y openvswitch-switch-dpdk pciutils vim +update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk +modprobe uio_pci_generic +ip=$(ip a s eth2 | grep inet | grep -v inet6 | sed "s/.*inet//" | cut -f2 -d' ') +ip address flush eth2 +/usr/share/dpdk/tools/dpdk_nic_bind.py --bind=uio_pci_generic eth2 +sysctl -w vm.nr_hugepages=1024 +mount -t hugetlbfs -o pagesize=2M none /dev/hugepages +cp /usr/bin/ovs-vsctl /usr/local/bin +memory=$(grep HugePages_Total /proc/meminfo | cut -f2 -d:) +echo "DPDK_OPTS='--dpdk -c 0x1 -n 2 -m $memory'" | tee -a /etc/default/openvswitch-switch +service dpdk restart +service openvswitch-switch restart +pkill ovs-vswitchd +ovs-vswitchd --dpdk -c 0x1 -n 2 -m $memory -- unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach --monitor +ovs-vsctl add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev +ovs-vsctl add-port br-dpdk dpdk0 -- set Interface dpdk0 type=dpdk +ip a a $ip dev br-dpdk +ip link set dev br-dpdk up +while true; do sleep 3600; done +echo docker build -t openretriever/ubuntu1604-ovsdpdk . +echo docker run -ti --privileged --net=host -v /dev:/dev -v /usr/local/bin:/usr/local/bin -v /var/run/openvswitch/:/var/run/openvswitch/ -v /lib/modules/:/lib/modules openretriever/ubuntu1604-ovsdpdk bash diff --git a/src/cni/ovsdpdk/teardown_ovsdpdk.sh b/src/cni/ovsdpdk/teardown_ovsdpdk.sh new file mode 100755 index 0000000..857738e --- /dev/null +++ b/src/cni/ovsdpdk/teardown_ovsdpdk.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +netns=$1 +containerid=$2 +pid=$(echo $netns | cut -f3 -d"/") + +sudo ovs-vsctl del-port br-dpdk vhost-user-$pid +sudo ip netns exec $pid link delete dummy-$pid +sudo rm -rf /var/run/cni/netconf-$pid diff --git a/src/vagrant/kubeadm_ovsdpdk/examples/virtio-user.sh b/src/vagrant/kubeadm_ovsdpdk/examples/virtio-user.sh index dd918de..0cd9d0e 100755 --- a/src/vagrant/kubeadm_ovsdpdk/examples/virtio-user.sh +++ b/src/vagrant/kubeadm_ovsdpdk/examples/virtio-user.sh @@ -20,7 +20,7 @@ set -ex kubectl delete rc --all kubectl apply -f /vagrant/examples/virtio-user.yaml r="0" -while [ $r -ne "4" ] +while [ $r -ne "6" ] do r=$(kubectl get pods --all-namespaces | grep ovsdpdk | grep Run | wc -l) sleep 20 diff --git a/src/vagrant/kubeadm_ovsdpdk/master_setup.sh b/src/vagrant/kubeadm_ovsdpdk/master_setup.sh index b9cd5db..ad8538a 100644 --- a/src/vagrant/kubeadm_ovsdpdk/master_setup.sh +++ b/src/vagrant/kubeadm_ovsdpdk/master_setup.sh @@ -8,5 +8,5 @@ sudo chown $(id -u):$(id -g) $HOME/admin.conf export KUBECONFIG=$HOME/admin.conf echo "export KUBECONFIG=$HOME/admin.conf" >> $HOME/.bash_profile -kubectl apply -f /vagrant/ovsdpdk/kube_ovsdpdk.yml +kubectl apply -f /src/cni/ovsdpdk/kube_ovsdpdk.yml kubectl apply -f /src/cni/ovsdpdk/kube_cniovsdpdk.yml diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile deleted file mode 100644 index a63df8f..0000000 --- a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM ubuntu:16.04 - -WORKDIR /ovsdpdk -RUN apt-get update && apt-get install -y sudo -ADD install.sh . -RUN bash ./install.sh -ADD start.sh . -RUN chmod 755 start.sh diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh deleted file mode 100644 index f565025..0000000 --- a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/install.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -ex -sudo apt-get update -sudo apt-get install -y openvswitch-switch-dpdk pciutils vim -sudo update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml deleted file mode 100644 index ed04ccd..0000000 --- a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/kube_ovsdpdk.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: ovsdpdk - namespace: kube-system ---- -apiVersion: extensions/v1beta1 -kind: DaemonSet -metadata: - name: kube-ovsdpdk-ds - namespace: kube-system - labels: - tier: node - app: ovsdpdk -spec: - template: - metadata: - labels: - tier: node - app: ovsdpdk - spec: - hostNetwork: true - nodeSelector: - beta.kubernetes.io/arch: amd64 - tolerations: - - key: node-role.kubernetes.io/master - operator: Exists - effect: NoSchedule - serviceAccountName: ovsdpdk - containers: - - name: install-ovsdpdk - image: openretriever/ubuntu1604-ovsdpdk - command: [ "/bin/bash", "/ovsdpdk/start.sh" ] - securityContext: - privileged: true - volumeMounts: - - name: local-bin - mountPath: /usr/local/bin - - name: var-run - mountPath: /var/run/openvswitch - - name: lib-modules - mountPath: /lib/modules - - name: dev-hugepage - mountPath: /dev - volumes: - - name: local-bin - hostPath: - path: /usr/local/bin - - name: var-run - hostPath: - path: /var/run/openvswitch - - name: lib-modules - hostPath: - path: /lib/modules - - name: dev-hugepage - hostPath: - path: /dev diff --git a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh b/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh deleted file mode 100644 index 4fc3d17..0000000 --- a/src/vagrant/kubeadm_ovsdpdk/ovsdpdk/start.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -ex - -for i in {1..10} -do - sudo sysctl -w vm.nr_hugepages=2048; sleep 1 -done -sudo modprobe uio_pci_generic -ip=$(ip a s eth2 | grep inet | grep -v inet6 | sed "s/.*inet//" | cut -f2 -d' ') -sudo ip address flush eth2 -sudo /usr/share/dpdk/tools/dpdk_nic_bind.py --bind=uio_pci_generic eth2 -sudo sysctl -w vm.nr_hugepages=1024 -sudo mount -t hugetlbfs -o pagesize=2M none /dev/hugepages -sudo cp /usr/bin/ovs-vsctl /usr/local/bin -memory=$(grep HugePages_Total /proc/meminfo | cut -f2 -d:) -echo "DPDK_OPTS='--dpdk -c 0x1 -n 2 -m $memory'" | sudo tee -a /etc/default/openvswitch-switch -sudo service dpdk restart -sudo service openvswitch-switch restart -sudo pkill ovs-vswitchd -sudo ovs-vswitchd --dpdk -c 0x1 -n 2 -m $memory -- unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach --monitor -sudo ovs-vsctl add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev -sudo ovs-vsctl add-port br-dpdk dpdk0 -- set Interface dpdk0 type=dpdk -sudo ip a a $ip dev br-dpdk -sudo ip link set dev br-dpdk up -while true; do sleep 3600; done -echo sudo docker build -t openretriever/ubuntu1604-ovsdpdk . -echo sudo docker run -ti --privileged --net=host -v /dev:/dev -v /usr/local/bin:/usr/local/bin -v /var/run/openvswitch/:/var/run/openvswitch/ -v /lib/modules/:/lib/modules openretriever/ubuntu1604-ovsdpdk bash -- cgit 1.2.3-korg