diff options
Diffstat (limited to 'src/cni/ovsdpdk')
-rw-r--r-- | src/cni/ovsdpdk/Dockerfile.build | 6 | ||||
-rw-r--r-- | src/cni/ovsdpdk/Dockerfile.cniovsdpdk | 3 | ||||
-rwxr-xr-x[-rw-r--r--] | src/cni/ovsdpdk/build.sh | 20 | ||||
-rw-r--r-- | src/cni/ovsdpdk/build_cni.sh | 13 | ||||
-rw-r--r-- | src/cni/ovsdpdk/cni/install_cni.sh | 8 | ||||
-rwxr-xr-x | src/cni/ovsdpdk/install_cni.sh | 22 | ||||
-rw-r--r-- | src/cni/ovsdpdk/kube_cniovsdpdk.yml | 14 | ||||
-rw-r--r-- | src/cni/ovsdpdk/kube_ovsdpdk.yml | 61 | ||||
-rw-r--r-- | src/cni/ovsdpdk/ovsdpdk.go | 107 | ||||
-rw-r--r-- | src/cni/ovsdpdk/ovsdpdk.patch | 136 | ||||
-rwxr-xr-x | src/cni/ovsdpdk/setup_ovsdpdk.sh (renamed from src/cni/ovsdpdk/cni/setup_ovsdpdk.sh) | 0 | ||||
-rw-r--r-- | src/cni/ovsdpdk/start.sh | 32 | ||||
-rwxr-xr-x | src/cni/ovsdpdk/teardown_ovsdpdk.sh (renamed from src/cni/ovsdpdk/cni/teardown_ovsdpdk.sh) | 0 |
13 files changed, 234 insertions, 188 deletions
diff --git a/src/cni/ovsdpdk/Dockerfile.build b/src/cni/ovsdpdk/Dockerfile.build deleted file mode 100644 index ed5731e..0000000 --- a/src/cni/ovsdpdk/Dockerfile.build +++ /dev/null @@ -1,6 +0,0 @@ -FROM ubuntu:16.04 - -RUN apt-get update && apt-get install -y sudo git wget -ADD ovsdpdk.patch / -ADD build_cni.sh / -RUN bash /build_cni.sh diff --git a/src/cni/ovsdpdk/Dockerfile.cniovsdpdk b/src/cni/ovsdpdk/Dockerfile.cniovsdpdk deleted file mode 100644 index 2dd4f3c..0000000 --- a/src/cni/ovsdpdk/Dockerfile.cniovsdpdk +++ /dev/null @@ -1,3 +0,0 @@ -FROM ubuntu:16.04 - -ADD cni /cni diff --git a/src/cni/ovsdpdk/build.sh b/src/cni/ovsdpdk/build.sh index 625c7cc..5ddea99 100644..100755 --- a/src/cni/ovsdpdk/build.sh +++ b/src/cni/ovsdpdk/build.sh @@ -2,24 +2,6 @@ set -ex DIR="$(dirname `readlink -f $0`)" -PWD="$(pwd)" cd $DIR - -sudo apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D -sudo apt-key adv -k 58118E89F3A912897C070ADBF76221572C52609D -cat << EOF | sudo tee /etc/apt/sources.list.d/docker.list -deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main -EOF - -sudo apt-get install -y --allow-downgrades docker-engine=1.12.6-0~ubuntu-xenial -sudo docker rmi -f cniovsdpdk.build || true -sudo docker build -t cniovsdpdk.build . -f Dockerfile.build -sudo docker run -v $DIR/cni:/build -t cniovsdpdk.build cp /plugins/bin/ovsdpdk /build -sudo docker build -t openretriever/cnicniovsdpdk . -f Dockerfile.cniovsdpdk -sudo rm -rf $DIR/cni/ovsdpdk -echo git clone https://github.com/containernetworking/cni -echo sudo CNI_PATH=$CNI_PATH ./priv-net-run.sh ifconfig -echo sudo docker login openretriever/cnicniovsdpdk -u user -p password -echo sudo docker push openretriever/cnicniovsdpdk -cd $PWD +sudo docker run -v $DIR:/build ubuntu:16.04 /build/install_cni.sh diff --git a/src/cni/ovsdpdk/build_cni.sh b/src/cni/ovsdpdk/build_cni.sh deleted file mode 100644 index dbb3dd5..0000000 --- a/src/cni/ovsdpdk/build_cni.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -ex - -sudo apt-get update -sudo apt-get install -y docker.io devscripts git jq -wget -qO- https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz | sudo tar -C /usr/local -xz -echo 'export GOPATH=/go; export PATH=/usr/local/go/bin:$GOPATH/bin:$PATH' >> ~/.bashrc -export GOPATH=/go; export PATH=/usr/local/go/bin:$GOPATH/bin:$PATH -git clone https://github.com/containernetworking/plugins -cd plugins -git checkout 5544d9ced0d6e908fe26e9dbe529c7feb87d21f5 -patch -p1 < ../ovsdpdk.patch -bash ./build.sh diff --git a/src/cni/ovsdpdk/cni/install_cni.sh b/src/cni/ovsdpdk/cni/install_cni.sh deleted file mode 100644 index 1996700..0000000 --- a/src/cni/ovsdpdk/cni/install_cni.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -ex -cp /cni/ovsdpdk /opt/cni/bin -cp /cni/setup_ovsdpdk.sh /opt/cni/bin -cp /cni/teardown_ovsdpdk.sh /opt/cni/bin -cp /etc/kube-cniovsdpdk/cni-conf.json /etc/cni/net.d/10-ovsdpdk.conf -while true; do sleep 3600; done diff --git a/src/cni/ovsdpdk/install_cni.sh b/src/cni/ovsdpdk/install_cni.sh new file mode 100755 index 0000000..946d04c --- /dev/null +++ b/src/cni/ovsdpdk/install_cni.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -ex + + +export PATH=/usr/local/go/bin:$PATH + +DIR="$(dirname `readlink -f $0`)" +cd $DIR +apt-get update && apt-get install -y git wget gcc +wget -qO- https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz | tar -C /usr/local/ -xz +rm -rf plugins +git clone https://github.com/containernetworking/plugins +cd plugins +git checkout 5544d9ced0d6e908fe26e9dbe529c7feb87d21f5 +mkdir plugins/main/ovsdpdk +cp ../ovsdpdk.go plugins/main/ovsdpdk +sed -i "s,PLUGINS=.*,PLUGINS=plugins/main/ovsdpdk," build.sh +./build.sh +mkdir -p /opt/cni/bin +cp bin/ovsdpdk /opt/cni/bin +cp ../setup_ovsdpdk.sh ../teardown_ovsdpdk.sh /opt/cni/bin +rm -rf plugins diff --git a/src/cni/ovsdpdk/kube_cniovsdpdk.yml b/src/cni/ovsdpdk/kube_cniovsdpdk.yml index 3684719..89491da 100644 --- a/src/cni/ovsdpdk/kube_cniovsdpdk.yml +++ b/src/cni/ovsdpdk/kube_cniovsdpdk.yml @@ -51,13 +51,21 @@ spec: operator: Exists effect: NoSchedule serviceAccountName: cniovsdpdk - containers: + initContainers: - name: install-cni - image: openretriever/cni-ovsdpdk - command: [ "/bin/bash", "/cni/install_cni.sh" ] + image: ubuntu:16.04 + command: + - bash + - "-c" + - "apt-get update && apt-get install -y git && git clone http://github.com/opnfv/container4nfv && cd container4nfv && git fetch https://gerrit.opnfv.org/gerrit/container4nfv refs/changes/43/47743/5 && git checkout FETCH_HEAD && bash ./src/cni/ovsdpdk/install_cni.sh" volumeMounts: - name: cni-bin mountPath: /opt/cni/bin + containers: + - name: run-cni + image: busybox:1.27.2 + command: [ 'sh', '-c', 'cp /etc/kube-cniovsdpdk/cni-conf.json /etc/cni/net.d/10-ovsdpdk.conf; while true; do sleep 10000; done' ] + volumeMounts: - name: cni-cfg mountPath: /etc/cni/net.d - name: cniovsdpdk-cfg diff --git a/src/cni/ovsdpdk/kube_ovsdpdk.yml b/src/cni/ovsdpdk/kube_ovsdpdk.yml new file mode 100644 index 0000000..76ea51f --- /dev/null +++ b/src/cni/ovsdpdk/kube_ovsdpdk.yml @@ -0,0 +1,61 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ovsdpdk + namespace: kube-system +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: kube-ovsdpdk-ds + namespace: kube-system + labels: + tier: node + app: ovsdpdk +spec: + template: + metadata: + labels: + tier: node + app: ovsdpdk + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + serviceAccountName: ovsdpdk + containers: + - name: install-ovsdpdk + image: ubuntu:16.04 + command: + - bash + - "-c" + - "apt-get update && apt-get install -y git && git clone http://github.com/opnfv/container4nfv && cd container4nfv && git fetch https://gerrit.opnfv.org/gerrit/container4nfv refs/changes/43/47743/5 && git checkout FETCH_HEAD && bash ./src/cni/ovsdpdk/start.sh" + securityContext: + privileged: true + volumeMounts: + - name: local-bin + mountPath: /usr/local/bin + - name: var-run + mountPath: /var/run/openvswitch + - name: lib-modules + mountPath: /lib/modules + - name: dev-hugepage + mountPath: /dev + volumes: + - name: local-bin + hostPath: + path: /usr/local/bin + - name: var-run + hostPath: + path: /var/run/openvswitch + - name: lib-modules + hostPath: + path: /lib/modules + - name: dev-hugepage + hostPath: + path: /dev diff --git a/src/cni/ovsdpdk/ovsdpdk.go b/src/cni/ovsdpdk/ovsdpdk.go new file mode 100644 index 0000000..4c388b5 --- /dev/null +++ b/src/cni/ovsdpdk/ovsdpdk.go @@ -0,0 +1,107 @@ +// Copyright 2017 4 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "runtime" + "os/exec" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/cni/pkg/version" + "github.com/containernetworking/plugins/pkg/ipam" +) + +const defaultBrName = "br-dpdk" + +type NetConf struct { + types.NetConf + BrName string `json:"bridge"` +} + +func init() { + // this ensures that main runs only on main thread (thread group leader). + // since namespace ops (unshare, setns) are done for a single thread, we + // must ensure that the goroutine does not jump from OS thread to thread + runtime.LockOSThread() +} + +func loadNetConf(bytes []byte) (*NetConf, string, error) { + n := &NetConf{ + BrName: defaultBrName, + } + if err := json.Unmarshal(bytes, n); err != nil { + return nil, "", fmt.Errorf("failed to load netconf: %v", err) + } + return n, n.CNIVersion, nil +} + +func setupVhostUser(args *skel.CmdArgs, types string) error { + exec.Command("/bin/bash", "/opt/cni/bin/setup_ovsdpdk.sh", args.Netns, args.ContainerID, types).Output() + return nil +} + +func cmdAdd(args *skel.CmdArgs) error { + n, cniVersion, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + + // run the IPAM plugin and get back the config to apply + r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) + if err != nil { + return err + } + + // Convert whatever the IPAM result was into the current Result type + result, err := current.NewResultFromResult(r) + if err != nil { + return err + } + + if len(result.IPs) == 0 { + return errors.New("IPAM plugin returned missing IP config") + } + + setupVhostUser(args, result.String()) + + return types.PrintResult(result, cniVersion) +} + +func tearDownVhostUser(args *skel.CmdArgs) error { + exec.Command("/bin/bash", "/opt/cni/bin/teardown_ovsdpdk.sh", args.Netns, args.ContainerID).Output() + return nil +} + +func cmdDel(args *skel.CmdArgs) error { + n, _, err := loadNetConf(args.StdinData) + if err != nil { + return err + } + + if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil { + return err + } + + tearDownVhostUser(args) + return err +} + +func main() { + skel.PluginMain(cmdAdd, cmdDel, version.All) +} diff --git a/src/cni/ovsdpdk/ovsdpdk.patch b/src/cni/ovsdpdk/ovsdpdk.patch deleted file mode 100644 index 67b3703..0000000 --- a/src/cni/ovsdpdk/ovsdpdk.patch +++ /dev/null @@ -1,136 +0,0 @@ -diff --git a/build.sh b/build.sh -index cd21ba8..bc60d91 100755 ---- a/build.sh -+++ b/build.sh -@@ -19,7 +19,7 @@ export GOPATH=${PWD}/gopath - mkdir -p "${PWD}/bin" - - echo "Building plugins" --PLUGINS="plugins/meta/* plugins/main/* plugins/ipam/* plugins/sample" -+PLUGINS="plugins/main/ovsdpdk plugins/main/bridge plugins/ipam/host-local" - for d in $PLUGINS; do - if [ -d "$d" ]; then - plugin="$(basename "$d")" -diff --git a/plugins/main/ovsdpdk/ovsdpdk.go b/plugins/main/ovsdpdk/ovsdpdk.go -new file mode 100644 -index 0000000..1b931d4 ---- /dev/null -+++ b/plugins/main/ovsdpdk/ovsdpdk.go -@@ -0,0 +1,117 @@ -+// Copyright 2014 CNI authors -+// -+// Licensed under the Apache License, Version 2.0 (the "License"); -+// you may not use this file except in compliance with the License. -+// You may obtain a copy of the License at -+// -+// http://www.apache.org/licenses/LICENSE-2.0 -+// -+// Unless required by applicable law or agreed to in writing, software -+// distributed under the License is distributed on an "AS IS" BASIS, -+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -+// See the License for the specific language governing permissions and -+// limitations under the License. -+ -+package main -+ -+import ( -+ "encoding/json" -+ "errors" -+ "fmt" -+ //"net" -+ "runtime" -+ //"syscall" -+ "os/exec" -+ //"io/ioutil" -+ -+ "github.com/containernetworking/cni/pkg/skel" -+ "github.com/containernetworking/cni/pkg/types" -+ "github.com/containernetworking/cni/pkg/types/current" -+ "github.com/containernetworking/cni/pkg/version" -+ //"github.com/containernetworking/plugins/pkg/ip" -+ "github.com/containernetworking/plugins/pkg/ipam" -+ //"github.com/containernetworking/plugins/pkg/ns" -+ //"github.com/containernetworking/plugins/pkg/utils" -+ //"github.com/vishvananda/netlink" -+) -+ -+const defaultBrName = "cni0" -+ -+type NetConf struct { -+ types.NetConf -+ BrName string `json:"bridge"` -+} -+ -+func init() { -+ // this ensures that main runs only on main thread (thread group leader). -+ // since namespace ops (unshare, setns) are done for a single thread, we -+ // must ensure that the goroutine does not jump from OS thread to thread -+ runtime.LockOSThread() -+} -+ -+func loadNetConf(bytes []byte) (*NetConf, string, error) { -+ n := &NetConf{ -+ BrName: defaultBrName, -+ } -+ if err := json.Unmarshal(bytes, n); err != nil { -+ return nil, "", fmt.Errorf("failed to load netconf: %v", err) -+ } -+ return n, n.CNIVersion, nil -+} -+ -+func setupVhostUser(args *skel.CmdArgs, types string) error { -+ exec.Command("/bin/bash", "/opt/cni/bin/setup_ovsdpdk.sh", args.Netns, args.ContainerID, types).Output() -+ return nil -+} -+ -+ -+func cmdAdd(args *skel.CmdArgs) error { -+ n, cniVersion, err := loadNetConf(args.StdinData) -+ if err != nil { -+ return err -+ } -+ -+ // run the IPAM plugin and get back the config to apply -+ r, err := ipam.ExecAdd(n.IPAM.Type, args.StdinData) -+ if err != nil { -+ return err -+ } -+ -+ // Convert whatever the IPAM result was into the current Result type -+ result, err := current.NewResultFromResult(r) -+ if err != nil { -+ return err -+ } -+ -+ if len(result.IPs) == 0 { -+ return errors.New("IPAM plugin returned missing IP config") -+ } -+ -+ setupVhostUser(args, result.String()) -+ -+ return types.PrintResult(result, cniVersion) -+} -+ -+func tearDownVhostUser(args *skel.CmdArgs) error { -+ exec.Command("/bin/bash", "/opt/cni/bin/teardown_ovsdpdk.sh", args.Netns, args.ContainerID).Output() -+ return nil -+} -+ -+func cmdDel(args *skel.CmdArgs) error { -+ n, _, err := loadNetConf(args.StdinData) -+ if err != nil { -+ return err -+ } -+ -+ if err := ipam.ExecDel(n.IPAM.Type, args.StdinData); err != nil { -+ return err -+ } -+ -+ tearDownVhostUser(args) -+ return err -+ -+} -+ -+func main() { -+ skel.PluginMain(cmdAdd, cmdDel, version.All) -+} diff --git a/src/cni/ovsdpdk/cni/setup_ovsdpdk.sh b/src/cni/ovsdpdk/setup_ovsdpdk.sh index 6009b7a..6009b7a 100755 --- a/src/cni/ovsdpdk/cni/setup_ovsdpdk.sh +++ b/src/cni/ovsdpdk/setup_ovsdpdk.sh diff --git a/src/cni/ovsdpdk/start.sh b/src/cni/ovsdpdk/start.sh new file mode 100644 index 0000000..53b5fe1 --- /dev/null +++ b/src/cni/ovsdpdk/start.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -ex + +for i in {1..10} +do + sysctl -w vm.nr_hugepages=2048; sleep 1 +done + +apt-get update +apt-get install -y openvswitch-switch-dpdk pciutils vim +update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk +modprobe uio_pci_generic +ip=$(ip a s eth2 | grep inet | grep -v inet6 | sed "s/.*inet//" | cut -f2 -d' ') +ip address flush eth2 +/usr/share/dpdk/tools/dpdk_nic_bind.py --bind=uio_pci_generic eth2 +sysctl -w vm.nr_hugepages=1024 +mount -t hugetlbfs -o pagesize=2M none /dev/hugepages +cp /usr/bin/ovs-vsctl /usr/local/bin +memory=$(grep HugePages_Total /proc/meminfo | cut -f2 -d:) +echo "DPDK_OPTS='--dpdk -c 0x1 -n 2 -m $memory'" | tee -a /etc/default/openvswitch-switch +service dpdk restart +service openvswitch-switch restart +pkill ovs-vswitchd +ovs-vswitchd --dpdk -c 0x1 -n 2 -m $memory -- unix:/var/run/openvswitch/db.sock -vconsole:emer -vsyslog:err -vfile:info --mlockall --no-chdir --log-file=/var/log/openvswitch/ovs-vswitchd.log --pidfile=/var/run/openvswitch/ovs-vswitchd.pid --detach --monitor +ovs-vsctl add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev +ovs-vsctl add-port br-dpdk dpdk0 -- set Interface dpdk0 type=dpdk +ip a a $ip dev br-dpdk +ip link set dev br-dpdk up +while true; do sleep 3600; done +echo docker build -t openretriever/ubuntu1604-ovsdpdk . +echo docker run -ti --privileged --net=host -v /dev:/dev -v /usr/local/bin:/usr/local/bin -v /var/run/openvswitch/:/var/run/openvswitch/ -v /lib/modules/:/lib/modules openretriever/ubuntu1604-ovsdpdk bash diff --git a/src/cni/ovsdpdk/cni/teardown_ovsdpdk.sh b/src/cni/ovsdpdk/teardown_ovsdpdk.sh index 857738e..857738e 100755 --- a/src/cni/ovsdpdk/cni/teardown_ovsdpdk.sh +++ b/src/cni/ovsdpdk/teardown_ovsdpdk.sh |