aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--build/Dockerfile17
-rw-r--r--cmd/nfn-agent/nfn-agent.go23
-rw-r--r--cmd/ovn4nfvk8s-cni/app/helper_linux.go20
-rw-r--r--cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go321
-rw-r--r--deploy/ovn4nfv-k8s-plugin.yaml (renamed from deploy/ovn4nfv-k8s-plugin-daemonset.yml)63
-rw-r--r--go.mod2
-rw-r--r--go.sum2
-rw-r--r--internal/pkg/cniserver/cni.go295
-rw-r--r--internal/pkg/cniserver/cniserver.go235
-rw-r--r--internal/pkg/cnishim/cnishim.go111
-rw-r--r--internal/pkg/config/config.go17
-rw-r--r--internal/pkg/ovn/utils.go2
-rw-r--r--pkg/controller/pod/pod_controller.go2
-rw-r--r--utilities/docker/debian/Dockerfile21
-rwxr-xr-xutilities/kernel/debian/install_kernel_modules.sh32
15 files changed, 802 insertions, 361 deletions
diff --git a/build/Dockerfile b/build/Dockerfile
index 9530772..af88d13 100644
--- a/build/Dockerfile
+++ b/build/Dockerfile
@@ -8,13 +8,21 @@ ENV https_proxy $HTTPS_PROXY
ENV no_proxy $NO_PROXY
RUN apt-get update && apt-get install -y -qq apt-transport-https make curl net-tools iproute2 \
- && echo "deb https://packages.wand.net.nz bionic ovs-2.10" > /etc/apt/sources.list.d/ovs.list \
- && curl https://packages.wand.net.nz/keyring.gpg -o /etc/apt/trusted.gpg.d/wand.gpg \
- && apt-get update && apt install -y -qq ovn-common openvswitch-common openvswitch-switch
+ wget software-properties-common setpriv dpkg-dev netcat jq
+
+RUN mkdir -p /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb
+RUN bash -xc "\
+pushd /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb; \
+wget -q -nv -O- https://api.github.com/repos/akraino-icn/ovs/releases/tags/v2.12.0 2>/dev/null | jq -r '.assets[] | select(.browser_download_url | contains("\""deb"\"")) | .browser_download_url' | wget -i -; \
+dpkg-scanpackages . | gzip -c9 > Packages.gz; \
+popd; \
+"
+RUN echo "deb [trusted=yes] file:///opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb ./" | tee -a /etc/apt/sources.list > /dev/null
+RUN apt-get update && apt-get install -y -qq ovn-common=2.12.0-1 openvswitch-common=2.12.0-1 openvswitch-switch=2.12.0-1
ENV GOLANG_VERSION 1.14.1
RUN curl -sSL https://storage.googleapis.com/golang/go$GOLANG_VERSION.linux-amd64.tar.gz \
- | tar -v -C /usr/local -xz
+ | tar -v -C /usr/local -xz
ENV PATH /usr/local/go/bin:$PATH
RUN mkdir -p /go/src /go/bin && chmod -R 777 /go
@@ -32,5 +40,4 @@ ENV OPERATOR=/usr/local/bin/nfn-operator \
USER_NAME=nfn-operator
RUN cp -r build/bin/* /usr/local/bin/
-
ENTRYPOINT ["entrypoint"]
diff --git a/cmd/nfn-agent/nfn-agent.go b/cmd/nfn-agent/nfn-agent.go
index 3c85dbd..8d33778 100644
--- a/cmd/nfn-agent/nfn-agent.go
+++ b/cmd/nfn-agent/nfn-agent.go
@@ -9,8 +9,11 @@ import (
"os"
"os/signal"
pb "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify/proto"
+ cs "ovn4nfv-k8s-plugin/internal/pkg/cniserver"
"ovn4nfv-k8s-plugin/internal/pkg/ovn"
logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+ "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/rest"
"strings"
"syscall"
"time"
@@ -298,6 +301,26 @@ func main() {
client := pb.NewNfnNotifyClient(conn)
errorChannel = make(chan string)
+ // creates the in-cluster config
+ config, err := rest.InClusterConfig()
+ if err != nil {
+ log.Error(err, "Unable to create in-cluster config")
+ return
+ }
+
+ // creates the clientset
+ clientset, err := kubernetes.NewForConfig(config)
+ if err != nil {
+ log.Error(err, "Unable to create clientset for in-cluster config")
+ return
+ }
+
+ cniserver := cs.NewCNIServer("",clientset)
+ err = cniserver.Start(cs.HandleCNIcommandRequest)
+ if err != nil {
+ log.Error(err, "Unable to start cni server")
+ return
+ }
// Run client in background
go subscribeNotif(client)
shutdownHandler(errorChannel)
diff --git a/cmd/ovn4nfvk8s-cni/app/helper_linux.go b/cmd/ovn4nfvk8s-cni/app/helper_linux.go
index 2dba628..1702597 100644
--- a/cmd/ovn4nfvk8s-cni/app/helper_linux.go
+++ b/cmd/ovn4nfvk8s-cni/app/helper_linux.go
@@ -10,8 +10,6 @@ import (
"strings"
"github.com/sirupsen/logrus"
-
- "github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types/current"
"github.com/containernetworking/plugins/pkg/ip"
"github.com/containernetworking/plugins/pkg/ns"
@@ -106,10 +104,10 @@ func setupInterface(netns ns.NetNS, containerID, ifName, macAddress, ipAddress,
}
// ConfigureInterface sets up the container interface
-var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway string, idx, mtu int) ([]*current.Interface, error) {
- netns, err := ns.GetNS(args.Netns)
+var ConfigureInterface = func(containerNetns, containerID, ifName, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway string, idx, mtu int) ([]*current.Interface, error) {
+ netns, err := ns.GetNS(containerNetns)
if err != nil {
- return nil, fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
+ return nil, fmt.Errorf("failed to open netns %q: %v", containerNetns, err)
}
defer netns.Close()
@@ -118,10 +116,10 @@ var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress
ifaceID = fmt.Sprintf("%s_%s_%s", namespace, podName, interfaceName)
} else {
ifaceID = fmt.Sprintf("%s_%s", namespace, podName)
- interfaceName = args.IfName
+ interfaceName = ifName
defaultGateway = "true"
}
- hostIface, contIface, err := setupInterface(netns, args.ContainerID, interfaceName, macAddress, ipAddress, gatewayIP, defaultGateway, idx, mtu)
+ hostIface, contIface, err := setupInterface(netns, containerID, interfaceName, macAddress, ipAddress, gatewayIP, defaultGateway, idx, mtu)
if err != nil {
return nil, err
}
@@ -132,7 +130,7 @@ var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress
fmt.Sprintf("external_ids:attached_mac=%s", macAddress),
fmt.Sprintf("external_ids:iface-id=%s", ifaceID),
fmt.Sprintf("external_ids:ip_address=%s", ipAddress),
- fmt.Sprintf("external_ids:sandbox=%s", args.ContainerID),
+ fmt.Sprintf("external_ids:sandbox=%s", containerID),
}
var out []byte
@@ -160,10 +158,10 @@ func setupRoute(netns ns.NetNS, dst, gw, dev string) error {
}
// ConfigureRoute sets up the container routes
-var ConfigureRoute = func(args *skel.CmdArgs, dst, gw, dev string) error {
- netns, err := ns.GetNS(args.Netns)
+var ConfigureRoute = func(containerNetns, dst, gw, dev string) error {
+ netns, err := ns.GetNS(containerNetns)
if err != nil {
- return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
+ return fmt.Errorf("failed to open netns %q: %v", containerNetns, err)
}
defer netns.Close()
err = setupRoute(netns, dst, gw, dev)
diff --git a/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go b/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go
index c176700..cb75ecd 100644
--- a/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go
+++ b/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go
@@ -1,335 +1,40 @@
-// +build linux
-
package main
import (
- "encoding/json"
- "fmt"
- "net"
"os"
- "strconv"
- "strings"
- "time"
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
+ cni "ovn4nfv-k8s-plugin/internal/pkg/cnishim"
+ "ovn4nfv-k8s-plugin/internal/pkg/config"
+
"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
- "github.com/containernetworking/cni/pkg/types/current"
"github.com/containernetworking/cni/pkg/version"
- "k8s.io/apimachinery/pkg/util/wait"
-
- "ovn4nfv-k8s-plugin/internal/pkg/kube"
-
- "ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app"
- "ovn4nfv-k8s-plugin/internal/pkg/config"
+ "github.com/containernetworking/plugins/pkg/utils/buildversion"
)
-const (
- ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces"
-)
-
-func argString2Map(args string) (map[string]string, error) {
- argsMap := make(map[string]string)
-
- pairs := strings.Split(args, ";")
- for _, pair := range pairs {
- kv := strings.Split(pair, "=")
- if len(kv) != 2 {
- return nil, fmt.Errorf("ARGS: invalid pair %q", pair)
- }
- keyString := kv[0]
- valueString := kv[1]
- argsMap[keyString] = valueString
- }
-
- return argsMap, nil
-}
-
-func parseOvnNetworkObject(ovnnetwork string) ([]map[string]string, error) {
- var ovnNet []map[string]string
-
- if ovnnetwork == "" {
- return nil, fmt.Errorf("parseOvnNetworkObject:error")
- }
-
- if err := json.Unmarshal([]byte(ovnnetwork), &ovnNet); err != nil {
- return nil, fmt.Errorf("parseOvnNetworkObject: failed to load ovn network err: %v | ovn network: %v", err, ovnnetwork)
- }
-
- return ovnNet, nil
-}
-
-func mergeWithResult(srcObj, dstObj types.Result) (types.Result, error) {
-
- if dstObj == nil {
- return srcObj, nil
- }
- src, err := current.NewResultFromResult(srcObj)
- if err != nil {
- return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err)
- }
- dst, err := current.NewResultFromResult(dstObj)
- if err != nil {
- return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err)
- }
-
- ifacesLength := len(dst.Interfaces)
-
- for _, iface := range src.Interfaces {
- dst.Interfaces = append(dst.Interfaces, iface)
- }
- for _, ip := range src.IPs {
- if ip.Interface != nil && *(ip.Interface) != -1 {
- ip.Interface = current.Int(*(ip.Interface) + ifacesLength)
- }
- dst.IPs = append(dst.IPs, ip)
- }
- for _, route := range src.Routes {
- dst.Routes = append(dst.Routes, route)
- }
-
- for _, ns := range src.DNS.Nameservers {
- dst.DNS.Nameservers = append(dst.DNS.Nameservers, ns)
- }
- for _, s := range src.DNS.Search {
- dst.DNS.Search = append(dst.DNS.Search, s)
- }
- for _, opt := range src.DNS.Options {
- dst.DNS.Options = append(dst.DNS.Options, opt)
- }
- // TODO: what about DNS.domain?
- return dst, nil
-}
-
-func prettyPrint(i interface{}) string {
- s, _ := json.MarshalIndent(i, "", "\t")
- return string(s)
-}
-
-func addMultipleInterfaces(args *skel.CmdArgs, ovnAnnotation, namespace, podName string) types.Result {
- logrus.Infof("ovn4nfvk8s-cni: addMultipleInterfaces ")
-
- var ovnAnnotatedMap []map[string]string
- ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation)
- if err != nil {
- logrus.Errorf("addLogicalPort : Error Parsing Ovn Network List %v %v", ovnAnnotatedMap, err)
- return nil
- }
- if namespace == "" || podName == "" {
- logrus.Errorf("required CNI variable missing")
- return nil
- }
- var interfacesArray []*current.Interface
- var index int
- var result *current.Result
- var dstResult types.Result
- for _, ovnNet := range ovnAnnotatedMap {
- ipAddress := ovnNet["ip_address"]
- macAddress := ovnNet["mac_address"]
- gatewayIP := ovnNet["gateway_ip"]
- defaultGateway := ovnNet["defaultGateway"]
-
- if ipAddress == "" || macAddress == "" {
- logrus.Errorf("failed in pod annotation key extract")
- return nil
- }
-
- index++
- interfaceName := ovnNet["interface"]
- if interfaceName == "" {
- logrus.Errorf("addMultipleInterfaces: interface can't be null")
- return nil
- }
- interfacesArray, err = app.ConfigureInterface(args, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway, index, config.Default.MTU)
- if err != nil {
- logrus.Errorf("Failed to configure interface in pod: %v", err)
- return nil
- }
- addr, addrNet, err := net.ParseCIDR(ipAddress)
- if err != nil {
- logrus.Errorf("failed to parse IP address %q: %v", ipAddress, err)
- return nil
- }
- ipVersion := "6"
- if addr.To4() != nil {
- ipVersion = "4"
- }
- var routes types.Route
- if defaultGateway == "true" {
- defaultAddr, defaultAddrNet, _ := net.ParseCIDR("0.0.0.0/0")
- routes = types.Route{Dst: net.IPNet{IP: defaultAddr, Mask: defaultAddrNet.Mask}, GW: net.ParseIP(gatewayIP)}
-
- result = &current.Result{
- Interfaces: interfacesArray,
- IPs: []*current.IPConfig{
- {
- Version: ipVersion,
- Interface: current.Int(1),
- Address: net.IPNet{IP: addr, Mask: addrNet.Mask},
- Gateway: net.ParseIP(gatewayIP),
- },
- },
- Routes: []*types.Route{&routes},
- }
- } else {
- result = &current.Result{
- Interfaces: interfacesArray,
- IPs: []*current.IPConfig{
- {
- Version: ipVersion,
- Interface: current.Int(1),
- Address: net.IPNet{IP: addr, Mask: addrNet.Mask},
- Gateway: net.ParseIP(gatewayIP),
- },
- },
- }
-
- }
- // Build the result structure to pass back to the runtime
- dstResult, err = mergeWithResult(types.Result(result), dstResult)
- if err != nil {
- logrus.Errorf("Failed to merge results: %v", err)
- return nil
- }
- }
- logrus.Infof("addMultipleInterfaces: %s", prettyPrint(dstResult))
- return dstResult
-}
-
-func addRoutes(args *skel.CmdArgs, ovnAnnotation string, dstResult types.Result) types.Result {
- logrus.Infof("ovn4nfvk8s-cni: addRoutes ")
-
- var ovnAnnotatedMap []map[string]string
- ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation)
- if err != nil {
- logrus.Errorf("addLogicalPort : Error Parsing Ovn Route List %v", err)
- return nil
- }
-
- var result types.Result
- var routes []*types.Route
- for _, ovnNet := range ovnAnnotatedMap {
- dst := ovnNet["dst"]
- gw := ovnNet["gw"]
- dev := ovnNet["dev"]
- if dst == "" || gw == "" || dev == "" {
- logrus.Errorf("failed in pod annotation key extract")
- return nil
- }
- err = app.ConfigureRoute(args, dst, gw, dev)
- if err != nil {
- logrus.Errorf("Failed to configure interface in pod: %v", err)
- return nil
- }
- dstAddr, dstAddrNet, _ := net.ParseCIDR(dst)
- routes = append(routes, &types.Route{
- Dst: net.IPNet{IP: dstAddr, Mask: dstAddrNet.Mask},
- GW: net.ParseIP(gw),
- })
- }
-
- result = &current.Result{
- Routes: routes,
- }
- // Build the result structure to pass back to the runtime
- dstResult, err = mergeWithResult(result, dstResult)
- if err != nil {
- logrus.Errorf("Failed to merge results: %v", err)
- return nil
- }
- logrus.Infof("addRoutes: %s", prettyPrint(dstResult))
- return dstResult
-
-}
-
-func cmdAdd(args *skel.CmdArgs) error {
- logrus.Infof("ovn4nfvk8s-cni: cmdAdd ")
- conf := &types.NetConf{}
- if err := json.Unmarshal(args.StdinData, conf); err != nil {
- return fmt.Errorf("failed to load netconf: %v", err)
- }
-
- argsMap, err := argString2Map(args.Args)
- if err != nil {
- return err
- }
-
- namespace := argsMap["K8S_POD_NAMESPACE"]
- podName := argsMap["K8S_POD_NAME"]
- if namespace == "" || podName == "" {
- return fmt.Errorf("required CNI variable missing")
- }
-
- clientset, err := config.NewClientset(&config.Kubernetes)
- if err != nil {
- return fmt.Errorf("Could not create clientset for kubernetes: %v", err)
- }
- kubecli := &kube.Kube{KClient: clientset}
-
- // Get the IP address and MAC address from the API server.
- var annotationBackoff = wait.Backoff{Duration: 1 * time.Second, Steps: 14, Factor: 1.5, Jitter: 0.1}
- var annotation map[string]string
- if err := wait.ExponentialBackoff(annotationBackoff, func() (bool, error) {
- annotation, err = kubecli.GetAnnotationsOnPod(namespace, podName)
- if err != nil {
- // TODO: check if err is non recoverable
- logrus.Warningf("Error while obtaining pod annotations - %v", err)
- return false, nil
- }
- if _, ok := annotation[ovn4nfvAnnotationTag]; ok {
- return true, nil
- }
- return false, nil
- }); err != nil {
- return fmt.Errorf("failed to get pod annotation - %v", err)
- }
- logrus.Infof("ovn4nfvk8s-cni: Annotation Found ")
- ovnAnnotation, ok := annotation[ovn4nfvAnnotationTag]
- if !ok {
- return fmt.Errorf("Error while obtaining pod annotations")
- }
- result := addMultipleInterfaces(args, ovnAnnotation, namespace, podName)
- // Add Routes to the pod if annotation found for routes
- ovnRouteAnnotation, ok := annotation["ovnNetworkRoutes"]
- if ok {
- logrus.Infof("ovn4nfvk8s-cni: ovnNetworkRoutes Annotation Found %+v", ovnRouteAnnotation)
- result = addRoutes(args, ovnRouteAnnotation, result)
- }
-
- return result.Print()
-}
-
-func cmdDel(args *skel.CmdArgs) error {
- logrus.Infof("ovn4nfvk8s-cni: cmdDel ")
- for i := 0; i < 10; i++ {
- ifaceName := args.ContainerID[:14] + strconv.Itoa(i)
- done, err := app.PlatformSpecificCleanup(ifaceName)
- if err != nil {
- logrus.Errorf("Teardown error: %v", err)
- }
- if done {
- break
- }
- }
- return nil
-}
-
func main() {
- logrus.Infof("ovn4nfvk8s-cni invoked")
+ logrus.Infof("ovn4nfvk8s-cni shim cni")
c := cli.NewApp()
c.Name = "ovn4nfvk8s-cni"
c.Usage = "a CNI plugin to set up or tear down a additional interfaces with OVN"
- c.Version = "0.0.2"
+ c.Version = "0.1.0"
c.Flags = config.Flags
+ ep := cni.CNIEndpoint("")
c.Action = func(ctx *cli.Context) error {
if _, err := config.InitConfig(ctx); err != nil {
return err
}
+ skel.PluginMain(
+ ep.CmdAdd,
+ ep.CmdCheck,
+ ep.CmdDel,
+ version.All,
+ buildversion.BuildString("ovn4nfv-k8s shim cni"))
- skel.PluginMain(cmdAdd, nil, cmdDel, version.All, "")
return nil
}
diff --git a/deploy/ovn4nfv-k8s-plugin-daemonset.yml b/deploy/ovn4nfv-k8s-plugin.yaml
index 13e749f..5d5017e 100644
--- a/deploy/ovn4nfv-k8s-plugin-daemonset.yml
+++ b/deploy/ovn4nfv-k8s-plugin.yaml
@@ -284,10 +284,9 @@ apiVersion: v1
kind: ServiceAccount
metadata:
name: k8s-nfn-sa
- namespace: operator
+ namespace: kube-system
---
-
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
@@ -298,6 +297,7 @@ rules:
- ""
resources:
- pods
+ - pods/status
- services
- endpoints
- persistentvolumeclaims
@@ -361,7 +361,7 @@ apiVersion: v1
kind: Service
metadata:
name: nfn-operator
- namespace: operator
+ namespace: kube-system
spec:
type: NodePort
ports:
@@ -378,7 +378,7 @@ apiVersion: v1
kind: ConfigMap
metadata:
name: ovn-controller-network
- namespace: operator
+ namespace: kube-system
data:
OVN_SUBNET: "10.244.64.0/18"
OVN_GATEWAYIP: "10.244.64.20/18"
@@ -391,7 +391,7 @@ apiVersion: apps/v1
kind: Deployment
metadata:
name: nfn-operator
- namespace: operator
+ namespace: kube-system
spec:
replicas: 1
selector:
@@ -408,10 +408,10 @@ spec:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- - key: nfnType
+ - key: ovn4nfv-k8s-plugin
operator: In
values:
- - operator
+ - ovn-control-plane
tolerations:
- key: "node-role.kubernetes.io/master"
effect: "NoSchedule"
@@ -429,10 +429,6 @@ spec:
- containerPort: 50000
protocol: TCP
env:
- - name: HOST_IP
- valueFrom:
- fieldRef:
- fieldPath: status.hostIP
- name: POD_NAME
valueFrom:
fieldRef:
@@ -445,7 +441,7 @@ kind: ConfigMap
apiVersion: v1
metadata:
name: ovn4nfv-cni-config
- namespace: operator
+ namespace: kube-system
labels:
app: ovn4nfv
data:
@@ -459,14 +455,20 @@ data:
plugin=ovn4nfvk8s-cni
[kubernetes]
- kubeconfig=/etc/kubernetes/admin.conf
+ kubeconfig=/etc/cni/net.d/ovn4nfv-k8s.d/ovn4nfv-k8s.kubeconfig
+ 00-network.conf: |
+ {
+ "name": "ovn4nfv-k8s-plugin",
+ "type": "ovn4nfvk8s-cni",
+ "cniVersion": "0.3.1"
+ }
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: ovn4nfv-cni
- namespace: operator
+ namespace: kube-system
labels:
app: ovn4nfv
spec:
@@ -483,6 +485,7 @@ spec:
tolerations:
- operator: Exists
effect: NoSchedule
+ serviceAccountName: k8s-nfn-sa
containers:
- name: ovn4nfv
image: integratedcloudnative/ovn4nfv-k8s-plugin:master
@@ -497,13 +500,20 @@ spec:
securityContext:
privileged: true
volumeMounts:
+ - name: cni
+ mountPath: /host/etc/cni/net.d
- name: cnibin
mountPath: /host/opt/cni/bin
- name: cniconf
mountPath: /host/etc/openvswitch
- name: ovn4nfv-cfg
mountPath: /tmp/ovn4nfv-conf
+ - name: ovn4nfv-cni-net-conf
+ mountPath: /tmp/ovn4nfv-cni
volumes:
+ - name: cni
+ hostPath:
+ path: /etc/cni/net.d
- name: cnibin
hostPath:
path: /opt/cni/bin
@@ -516,13 +526,18 @@ spec:
items:
- key: ovn4nfv_k8s.conf
path: ovn4nfv_k8s.conf
-
+ - name: ovn4nfv-cni-net-conf
+ configMap:
+ name: ovn4nfv-cni-config
+ items:
+ - key: 00-network.conf
+ path: 00-network.conf
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: nfn-agent
- namespace: operator
+ namespace: kube-system
labels:
app: nfn-agent
spec:
@@ -534,11 +549,13 @@ spec:
app: nfn-agent
spec:
hostNetwork: true
+ hostPID: true
nodeSelector:
beta.kubernetes.io/arch: amd64
tolerations:
- operator: Exists
effect: NoSchedule
+ serviceAccountName: k8s-nfn-sa
containers:
- name: nfn-agent
image: integratedcloudnative/ovn4nfv-k8s-plugin:master
@@ -556,12 +573,20 @@ spec:
fieldRef:
fieldPath: spec.nodeName
securityContext:
+ runAsUser: 0
+ capabilities:
+ add: ["NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"]
privileged: true
volumeMounts:
+ - mountPath: /var/run/dbus/
+ name: host-var-run-dbus
+ readOnly: true
- mountPath: /run/openvswitch
name: host-run-ovs
- mountPath: /var/run/openvswitch
name: host-var-run-ovs
+ - mountPath: /var/run/ovn4nfv-k8s-plugin
+ name: host-var-cniserver-socket-dir
volumes:
- name: host-run-ovs
hostPath:
@@ -569,3 +594,9 @@ spec:
- name: host-var-run-ovs
hostPath:
path: /var/run/openvswitch
+ - name: host-var-run-dbus
+ hostPath:
+ path: /var/run/dbus
+ - name: host-var-cniserver-socket-dir
+ hostPath:
+ path: /var/run/ovn4nfv-k8s-plugin
diff --git a/go.mod b/go.mod
index d98f1e5..985c693 100644
--- a/go.mod
+++ b/go.mod
@@ -22,6 +22,7 @@ require (
github.com/google/btree v1.0.0 // indirect
github.com/google/go-cmp v0.3.1 // indirect
github.com/gophercloud/gophercloud v0.2.0 // indirect
+ github.com/gorilla/mux v1.7.4
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.8.5 // indirect
github.com/hashicorp/golang-lru v0.5.3 // indirect
@@ -67,6 +68,7 @@ require (
k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
k8s.io/code-generator v0.17.0 // indirect
k8s.io/gengo v0.0.0-20191120174120-e74f70b9b27e // indirect
+ k8s.io/klog v1.0.0
k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a
k8s.io/utils v0.0.0-20190801114015-581e00157fb1
sigs.k8s.io/controller-runtime v0.2.0-beta.4
diff --git a/go.sum b/go.sum
index fd474b6..4963c29 100644
--- a/go.sum
+++ b/go.sum
@@ -319,6 +319,8 @@ github.com/gophercloud/gophercloud v0.2.0 h1:lD2Bce2xBAMNNcFZ0dObTpXkGLlVIb33RPV
github.com/gophercloud/gophercloud v0.2.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc=
+github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY=
github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
diff --git a/internal/pkg/cniserver/cni.go b/internal/pkg/cniserver/cni.go
new file mode 100644
index 0000000..2c91f04
--- /dev/null
+++ b/internal/pkg/cniserver/cni.go
@@ -0,0 +1,295 @@
+package cniserver
+
+import (
+ "encoding/json"
+ "k8s.io/apimachinery/pkg/util/wait"
+ "fmt"
+ "net"
+ "strconv"
+ "net/http"
+ "time"
+ "k8s.io/klog"
+
+ "k8s.io/client-go/kubernetes"
+ "github.com/containernetworking/cni/pkg/types"
+ "github.com/containernetworking/cni/pkg/types/current"
+ "ovn4nfv-k8s-plugin/internal/pkg/kube"
+ "k8s.io/apimachinery/pkg/api/errors"
+ "ovn4nfv-k8s-plugin/internal/pkg/config"
+ "ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app"
+)
+
+const (
+ ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces"
+)
+
+func parseOvnNetworkObject(ovnnetwork string) ([]map[string]string, error) {
+ var ovnNet []map[string]string
+
+ if ovnnetwork == "" {
+ return nil, fmt.Errorf("parseOvnNetworkObject:error")
+ }
+
+ if err := json.Unmarshal([]byte(ovnnetwork), &ovnNet); err != nil {
+ return nil, fmt.Errorf("parseOvnNetworkObject: failed to load ovn network err: %v | ovn network: %v", err, ovnnetwork)
+ }
+
+ return ovnNet, nil
+}
+
+func mergeWithResult(srcObj, dstObj types.Result) (types.Result, error) {
+
+ if dstObj == nil {
+ return srcObj, nil
+ }
+ src, err := current.NewResultFromResult(srcObj)
+ if err != nil {
+ return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err)
+ }
+ dst, err := current.NewResultFromResult(dstObj)
+ if err != nil {
+ return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err)
+ }
+
+ ifacesLength := len(dst.Interfaces)
+
+ for _, iface := range src.Interfaces {
+ dst.Interfaces = append(dst.Interfaces, iface)
+ }
+ for _, ip := range src.IPs {
+ if ip.Interface != nil && *(ip.Interface) != -1 {
+ ip.Interface = current.Int(*(ip.Interface) + ifacesLength)
+ }
+ dst.IPs = append(dst.IPs, ip)
+ }
+ for _, route := range src.Routes {
+ dst.Routes = append(dst.Routes, route)
+ }
+
+ for _, ns := range src.DNS.Nameservers {
+ dst.DNS.Nameservers = append(dst.DNS.Nameservers, ns)
+ }
+ for _, s := range src.DNS.Search {
+ dst.DNS.Search = append(dst.DNS.Search, s)
+ }
+ for _, opt := range src.DNS.Options {
+ dst.DNS.Options = append(dst.DNS.Options, opt)
+ }
+ // TODO: what about DNS.domain?
+ return dst, nil
+}
+
+func prettyPrint(i interface{}) string {
+ s, _ := json.MarshalIndent(i, "", "\t")
+ return string(s)
+}
+
+func isNotFoundError(err error) bool {
+ statusErr, ok := err.(*errors.StatusError)
+ return ok && statusErr.Status().Code == http.StatusNotFound
+}
+
+func (cr *CNIServerRequest) addMultipleInterfaces(ovnAnnotation, namespace, podName string) types.Result {
+ klog.Infof("ovn4nfvk8s-cni: addMultipleInterfaces ")
+ var ovnAnnotatedMap []map[string]string
+ ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation)
+ if err != nil {
+ klog.Errorf("addLogicalPort : Error Parsing Ovn Network List %v %v", ovnAnnotatedMap, err)
+ return nil
+ }
+ if namespace == "" || podName == "" {
+ klog.Errorf("required CNI variable missing")
+ return nil
+ }
+ var interfacesArray []*current.Interface
+ var index int
+ var result *current.Result
+ var dstResult types.Result
+ for _, ovnNet := range ovnAnnotatedMap {
+ ipAddress := ovnNet["ip_address"]
+ macAddress := ovnNet["mac_address"]
+ gatewayIP := ovnNet["gateway_ip"]
+ defaultGateway := ovnNet["defaultGateway"]
+
+ if ipAddress == "" || macAddress == "" {
+ klog.Errorf("failed in pod annotation key extract")
+ return nil
+ }
+
+ index++
+ interfaceName := ovnNet["interface"]
+ if interfaceName == "" {
+ klog.Errorf("addMultipleInterfaces: interface can't be null")
+ return nil
+ }
+ klog.Infof("addMultipleInterfaces: ipAddress %v %v", ipAddress, interfaceName)
+ interfacesArray, err = app.ConfigureInterface(cr.Netns, cr.SandboxID, cr.IfName, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway, index, config.Default.MTU)
+ if err != nil {
+ klog.Errorf("Failed to configure interface in pod: %v", err)
+ return nil
+ }
+ addr, addrNet, err := net.ParseCIDR(ipAddress)
+ if err != nil {
+ klog.Errorf("failed to parse IP address %q: %v", ipAddress, err)
+ return nil
+ }
+ ipVersion := "6"
+ if addr.To4() != nil {
+ ipVersion = "4"
+ }
+ var routes types.Route
+ if defaultGateway == "true" {
+ defaultAddr, defaultAddrNet, _ := net.ParseCIDR("0.0.0.0/0")
+ routes = types.Route{Dst: net.IPNet{IP: defaultAddr, Mask: defaultAddrNet.Mask}, GW: net.ParseIP(gatewayIP)}
+
+ result = &current.Result{
+ Interfaces: interfacesArray,
+ IPs: []*current.IPConfig{
+ {
+ Version: ipVersion,
+ Interface: current.Int(1),
+ Address: net.IPNet{IP: addr, Mask: addrNet.Mask},
+ Gateway: net.ParseIP(gatewayIP),
+ },
+ },
+ Routes: []*types.Route{&routes},
+ }
+ } else {
+ result = &current.Result{
+ Interfaces: interfacesArray,
+ IPs: []*current.IPConfig{
+ {
+ Version: ipVersion,
+ Interface: current.Int(1),
+ Address: net.IPNet{IP: addr, Mask: addrNet.Mask},
+ Gateway: net.ParseIP(gatewayIP),
+ },
+ },
+ }
+
+ }
+ // Build the result structure to pass back to the runtime
+ dstResult, err = mergeWithResult(types.Result(result), dstResult)
+ if err != nil {
+ klog.Errorf("Failed to merge results: %v", err)
+ return nil
+ }
+ }
+ klog.Infof("addMultipleInterfaces: results %s", prettyPrint(dstResult))
+ return dstResult
+}
+
+func (cr *CNIServerRequest) addRoutes(ovnAnnotation string, dstResult types.Result) types.Result {
+ klog.Infof("ovn4nfvk8s-cni: addRoutes ")
+ var ovnAnnotatedMap []map[string]string
+ ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation)
+ if err != nil {
+ klog.Errorf("addLogicalPort : Error Parsing Ovn Route List %v", err)
+ return nil
+ }
+
+ var result types.Result
+ var routes []*types.Route
+ for _, ovnNet := range ovnAnnotatedMap {
+ dst := ovnNet["dst"]
+ gw := ovnNet["gw"]
+ dev := ovnNet["dev"]
+ if dst == "" || gw == "" || dev == "" {
+ klog.Errorf("failed in pod annotation key extract")
+ return nil
+ }
+ err = app.ConfigureRoute(cr.Netns, dst, gw, dev)
+ if err != nil {
+ klog.Errorf("Failed to configure interface in pod: %v", err)
+ return nil
+ }
+ dstAddr, dstAddrNet, _ := net.ParseCIDR(dst)
+ routes = append(routes, &types.Route{
+ Dst: net.IPNet{IP: dstAddr, Mask: dstAddrNet.Mask},
+ GW: net.ParseIP(gw),
+ })
+ }
+
+ result = &current.Result{
+ Routes: routes,
+ }
+ // Build the result structure to pass back to the runtime
+ dstResult, err = mergeWithResult(result, dstResult)
+ if err != nil {
+ klog.Errorf("Failed to merge results: %v", err)
+ return nil
+ }
+ klog.Infof("addRoutes: results %s", prettyPrint(dstResult))
+ return dstResult
+}
+
+func (cr *CNIServerRequest) cmdAdd(kclient kubernetes.Interface) ([]byte, error) {
+ klog.Infof("ovn4nfvk8s-cni: cmdAdd")
+ namespace := cr.PodNamespace
+ podname := cr.PodName
+ if namespace == "" || podname == "" {
+ return nil, fmt.Errorf("required CNI variable missing")
+ }
+ klog.Infof("ovn4nfvk8s-cni: cmdAdd for pod podname:%s and namespace:%s", podname, namespace)
+ kubecli := &kube.Kube{KClient: kclient}
+ // Get the IP address and MAC address from the API server.
+ var annotationBackoff = wait.Backoff{Duration: 1 * time.Second, Steps: 14, Factor: 1.5, Jitter: 0.1}
+ var annotation map[string]string
+ var err error
+ if err = wait.ExponentialBackoff(annotationBackoff, func() (bool, error) {
+ annotation, err = kubecli.GetAnnotationsOnPod(namespace, podname)
+ if err != nil {
+ if isNotFoundError(err) {
+ return false, fmt.Errorf("Error - pod not found - %v", err)
+ }
+ klog.Infof("ovn4nfvk8s-cni: cmdAdd Warning - Error while obtaining pod annotations - %v", err)
+ return false,nil
+ }
+ if _, ok := annotation[ovn4nfvAnnotationTag]; ok {
+ return true, nil
+ }
+ return false, nil
+ }); err != nil {
+ return nil, fmt.Errorf("failed to get pod annotation - %v", err)
+ }
+
+ klog.Infof("ovn4nfvk8s-cni: cmdAdd Annotation Found ")
+ ovnAnnotation, ok := annotation[ovn4nfvAnnotationTag]
+ if !ok {
+ return nil, fmt.Errorf("Error while obtaining pod annotations")
+ }
+ result := cr.addMultipleInterfaces(ovnAnnotation, namespace, podname)
+ //Add Routes to the pod if annotation found for routes
+ ovnRouteAnnotation, ok := annotation["ovnNetworkRoutes"]
+ if ok {
+ klog.Infof("ovn4nfvk8s-cni: ovnNetworkRoutes Annotation Found %+v", ovnRouteAnnotation)
+ result = cr.addRoutes(ovnRouteAnnotation, result)
+ }
+
+ if result == nil {
+ klog.Errorf("result struct the ovn4nfv-k8s-plugin cniserver")
+ return nil, fmt.Errorf("result is nil from cni server response")
+ }
+
+ responseBytes, err := json.Marshal(result)
+ if err != nil {
+ return nil, fmt.Errorf("failed to marshal pod request response: %v", err)
+ }
+
+ return responseBytes, nil
+}
+
+func (cr *CNIServerRequest) cmdDel() ([]byte, error) {
+ klog.Infof("cmdDel ")
+ for i := 0; i < 10; i++ {
+ ifaceName := cr.SandboxID[:14] + strconv.Itoa(i)
+ done, err := app.PlatformSpecificCleanup(ifaceName)
+ if err != nil {
+ klog.Errorf("Teardown error: %v", err)
+ }
+ if done {
+ break
+ }
+ }
+ return []byte{}, nil
+}
diff --git a/internal/pkg/cniserver/cniserver.go b/internal/pkg/cniserver/cniserver.go
new file mode 100644
index 0000000..eaa7105
--- /dev/null
+++ b/internal/pkg/cniserver/cniserver.go
@@ -0,0 +1,235 @@
+package cniserver
+
+import (
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "strings"
+ "os"
+ "net"
+ "path/filepath"
+ "syscall"
+ "k8s.io/klog"
+
+ "github.com/containernetworking/cni/pkg/types"
+ "github.com/gorilla/mux"
+ "k8s.io/client-go/kubernetes"
+ "ovn4nfv-k8s-plugin/internal/pkg/config"
+ utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+ utilwait "k8s.io/apimachinery/pkg/util/wait"
+)
+
+const CNIServerRunDir string = "/var/run/ovn4nfv-k8s-plugin/cniserver"
+const CNIServerSocketName string = "ovn4nfv-k8s-plugin-cni-server.sock"
+const CNIServerSocketPath string = CNIServerRunDir + "/" + CNIServerSocketName
+
+
+type CNIcommand string
+
+const CNIAdd CNIcommand = "ADD"
+const CNIUpdate CNIcommand = "UPDATE"
+const CNIDel CNIcommand = "DEL"
+
+type CNIServerRequest struct {
+ Command CNIcommand
+ PodNamespace string
+ PodName string
+ SandboxID string
+ Netns string
+ IfName string
+ CNIConf *types.NetConf
+}
+
+type cniServerRequestFunc func(request *CNIServerRequest, k8sclient kubernetes.Interface) ([]byte, error)
+
+type CNIEndpointRequest struct {
+ ArgEnv map[string]string `json:"env,omitempty"`
+ NetConfig []byte `json:"config,omitempty"`
+}
+type CNIServer struct {
+ http.Server
+ requestFunc cniServerRequestFunc
+ serverrundir string
+ k8sclient kubernetes.Interface
+}
+
+func NewCNIServer(serverRunSir string, k8sclient kubernetes.Interface) *CNIServer {
+ klog.Infof("Setting up CNI server in nfn-agent")
+ if len(serverRunSir) == 0 {
+ serverRunSir = CNIServerRunDir
+ }
+
+ router := mux.NewRouter()
+ cs := &CNIServer{
+ Server: http.Server{
+ Handler: router,
+ },
+ serverrundir: serverRunSir,
+ k8sclient: k8sclient,
+ }
+ router.NotFoundHandler = http.HandlerFunc(http.NotFound)
+ router.HandleFunc("/", cs.handleCNIShimRequest).Methods("POST")
+ return cs
+}
+
+func loadCNIShimArgs(env map[string]string) (map[string]string, error) {
+ cnishimArgs, ok := env["CNI_ARGS"]
+ if !ok {
+ return nil, fmt.Errorf("cnishim req missing CNI_ARGS: '%s'", env)
+ }
+
+ mapArgs := make(map[string]string)
+ for _, arg := range strings.Split(cnishimArgs, ";") {
+ parts := strings.Split(arg, "=")
+ if len(parts) != 2 {
+ return nil, fmt.Errorf("invalid CNI_ARG from cnishim '%s'", arg)
+ }
+ mapArgs[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
+ }
+ return mapArgs, nil
+}
+
+func loadCNIRequestToCNIServer(r *CNIEndpointRequest) (*CNIServerRequest, error) {
+ cmd, ok := r.ArgEnv["CNI_COMMAND"]
+ if !ok {
+ return nil, fmt.Errorf("cnishim req missing CNI_COMMAND")
+ }
+
+ cnishimreq := &CNIServerRequest{
+ Command: CNIcommand(cmd),
+ }
+
+ cnishimreq.SandboxID, ok = r.ArgEnv["CNI_CONTAINERID"]
+ if !ok {
+ return nil, fmt.Errorf("cnishim req missing CNI_CONTAINERID")
+ }
+
+ cnishimreq.Netns, ok = r.ArgEnv["CNI_NETNS"]
+ if !ok {
+ return nil, fmt.Errorf("cnishim req missing CNI_NETNS")
+ }
+
+ cnishimreq.IfName, ok = r.ArgEnv["CNI_IFNAME"]
+ if !ok {
+ return nil, fmt.Errorf("cnishim req missing CNI_IFNAME")
+ }
+
+ cnishimArgs, err := loadCNIShimArgs(r.ArgEnv)
+ if err != nil {
+ return nil, err
+ }
+
+ cnishimreq.PodNamespace, ok = cnishimArgs["K8S_POD_NAMESPACE"]
+ if !ok {
+ return nil, fmt.Errorf("cnishim req missing K8S_POD_NAMESPACE")
+ }
+
+ cnishimreq.PodName, ok = cnishimArgs["K8S_POD_NAME"]
+ if !ok {
+ return nil, fmt.Errorf("cnishim req missing K8S_POD_NAME")
+ }
+
+ netconf, err := config.ConfigureNetConf(r.NetConfig)
+ if err != nil {
+ return nil, fmt.Errorf("cnishim req CNI arg configuration failed:%v",err)
+ }
+
+ cnishimreq.CNIConf = netconf
+ return cnishimreq, nil
+}
+
+func (cs *CNIServer) handleCNIShimRequest(w http.ResponseWriter, r *http.Request) {
+ var cr CNIEndpointRequest
+ b, _ := ioutil.ReadAll(r.Body)
+ if err := json.Unmarshal(b, &cr); err != nil {
+ http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+ return
+ }
+
+ req, err := loadCNIRequestToCNIServer(&cr)
+ if err != nil {
+ http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+ return
+ }
+
+ klog.Infof("Waiting for %s result for CNI server pod %s/%s", req.Command, req.PodNamespace, req.PodName)
+ result, err := cs.requestFunc(req, cs.k8sclient)
+ if err != nil {
+ http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
+ } else {
+ w.Header().Set("Content-Type", "application/json")
+ if _, err := w.Write(result); err != nil {
+ klog.Warningf("Error writing %s HTTP response: %v", req.Command, err)
+ }
+ }
+}
+
+func HandleCNIcommandRequest(request *CNIServerRequest, k8sclient kubernetes.Interface) ([]byte, error) {
+ var result []byte
+ var err error
+ klog.Infof("[PodNamespace:%s/PodName:%s] dispatching pod network request %v", request.PodNamespace, request.PodName, request)
+ klog.Infof("k8sclient %s", fmt.Sprintf("%v",k8sclient))
+ switch request.Command {
+ case CNIAdd:
+ result, err = request.cmdAdd(k8sclient)
+ case CNIDel:
+ result, err = request.cmdDel()
+ default:
+ }
+ klog.Infof("[PodNamespace:%s/PodName:%s] CNI request %v, result %q, err %v", request.PodNamespace, request.PodName, request, string(result), err)
+ if err != nil {
+ return nil, fmt.Errorf("[PodNamespace:%s/PodName:%s] CNI request %v %v", request.PodNamespace, request.PodName, request, err)
+ }
+ return result, nil
+}
+
+func (cs *CNIServer) Start(requestFunc cniServerRequestFunc) error {
+ if requestFunc == nil {
+ return fmt.Errorf("no CNI request handler")
+ }
+ cs.requestFunc = requestFunc
+ socketPath := filepath.Join(cs.serverrundir, CNIServerSocketName)
+ if err := os.RemoveAll(cs.serverrundir); err != nil && !os.IsNotExist(err) {
+ info, err := os.Stat(cs.serverrundir)
+ if err != nil {
+ return fmt.Errorf("failed to stat old cni server info socket directory %s: %v", cs.serverrundir, err)
+ }
+ tmp := info.Sys()
+ statt, ok := tmp.(*syscall.Stat_t)
+ if !ok {
+ return fmt.Errorf("failed to read CNI Server info socket directory stat info: %T", tmp)
+ }
+ if statt.Uid != 0 {
+ return fmt.Errorf("insecure owner of CNI Server info socket directory %s: %v", cs.serverrundir, statt.Uid)
+ }
+
+ if info.Mode()&0777 != 0700 {
+ return fmt.Errorf("insecure permissions on CNI Server info socket directory %s: %v", cs.serverrundir, info.Mode())
+ }
+
+ if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) {
+ return fmt.Errorf("failed to remove old CNI Server info socket %s: %v", socketPath, err)
+ }
+ }
+ if err := os.MkdirAll(cs.serverrundir, 0700); err != nil {
+ return fmt.Errorf("failed to create CNI Server info socket directory %s: %v", cs.serverrundir, err)
+ }
+
+ unixListener, err := net.Listen("unix", socketPath)
+ if err != nil {
+ return fmt.Errorf("failed to listen on CNI Server info socket: %v", err)
+ }
+ if err := os.Chmod(socketPath, 0600); err != nil {
+ unixListener.Close()
+ return fmt.Errorf("failed to set CNI Server info socket mode: %v", err)
+ }
+
+ cs.SetKeepAlivesEnabled(false)
+ go utilwait.Forever(func() {
+ if err := cs.Serve(unixListener); err != nil {
+ utilruntime.HandleError(fmt.Errorf("CNI server Serve() failed: %v", err))
+ }
+ }, 0)
+ return nil
+}
diff --git a/internal/pkg/cnishim/cnishim.go b/internal/pkg/cnishim/cnishim.go
new file mode 100644
index 0000000..ffedbfb
--- /dev/null
+++ b/internal/pkg/cnishim/cnishim.go
@@ -0,0 +1,111 @@
+package cni
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io/ioutil"
+ "net"
+ "net/http"
+ "os"
+ "ovn4nfv-k8s-plugin/internal/pkg/cniserver"
+ "ovn4nfv-k8s-plugin/internal/pkg/config"
+ "strings"
+
+ "github.com/containernetworking/cni/pkg/skel"
+ "github.com/containernetworking/cni/pkg/types"
+ "github.com/containernetworking/cni/pkg/types/current"
+ "github.com/sirupsen/logrus"
+)
+
+const CNIEndpointURLReq string = "http://dummy/"
+
+type Endpoint struct {
+ cniServerSocketPath string
+}
+
+func CNIEndpoint(cniServerSocketPath string) *Endpoint {
+ if len(cniServerSocketPath) == 0 {
+ cniServerSocketPath = cniserver.CNIServerSocketPath
+ }
+ return &Endpoint{cniServerSocketPath: cniServerSocketPath}
+}
+
+func cniEndpointRequest(args *skel.CmdArgs) *cniserver.CNIEndpointRequest {
+ osEnvMap := make(map[string]string)
+ for _, item := range os.Environ() {
+ idx := strings.Index(item, "=")
+ if idx > 0 {
+ osEnvMap[strings.TrimSpace(item[:idx])] = item[idx+1:]
+ }
+ }
+
+ return &cniserver.CNIEndpointRequest{
+ ArgEnv: osEnvMap,
+ NetConfig: args.StdinData,
+ }
+}
+
+func (ep *Endpoint) sendCNIServerReq(req *cniserver.CNIEndpointRequest) ([]byte, error) {
+ cnireqdata, err := json.Marshal(req)
+ if err != nil {
+ return nil, fmt.Errorf("sendCNIServerReq: failed to Marshal CNIShim Req %v:%v", req, err)
+ }
+
+ httpc := http.Client{
+ Transport: &http.Transport{
+ Dial: func(proto, addr string) (net.Conn, error) {
+ return net.Dial("unix", ep.cniServerSocketPath)
+ },
+ },
+ }
+
+ reponse, err := httpc.Post(CNIEndpointURLReq, "application/json", bytes.NewReader(cnireqdata))
+ if err != nil {
+ return nil, fmt.Errorf("Failed to send CNIServer request: %v", err)
+ }
+ defer reponse.Body.Close()
+
+ rbody, err := ioutil.ReadAll(reponse.Body)
+ if err != nil {
+ return nil, fmt.Errorf("Failed to read the CNI Server reponse:%v", err)
+ }
+
+ if reponse.StatusCode != 200 {
+ return nil, fmt.Errorf("CNI Server request is failed with reponse status %v and reponse body %s", reponse.StatusCode, string(rbody))
+ }
+
+ return rbody, nil
+}
+
+func (ep *Endpoint) CmdAdd(args *skel.CmdArgs) error {
+ logrus.Infof("ovn4nfvk8s-cni: cmdAdd ")
+ conf, err := config.ConfigureNetConf(args.StdinData)
+ if err != nil {
+ return fmt.Errorf("invalid stdin args")
+ }
+ logrus.Infof("ovn4nfvk8s-cni: cmdAdd configure net conf details -%+v", conf)
+ req := cniEndpointRequest(args)
+ logrus.Infof("ovn4nfvk8s-cni: cmdAdd CNIEndpoint Request:%+v",req)
+ reponsebody, err := ep.sendCNIServerReq(req)
+ if err != nil {
+ return err
+ }
+ result, err := current.NewResult(reponsebody)
+ if err != nil {
+ return fmt.Errorf("failed to unmarshall CNIServer Result reponse %v - err:%v", string(reponsebody), err)
+ }
+
+ return types.PrintResult(result, conf.CNIVersion)
+}
+
+func (ep *Endpoint) CmdCheck(args *skel.CmdArgs) error {
+ return nil
+}
+
+func (ep *Endpoint) CmdDel(args *skel.CmdArgs) error {
+ logrus.Infof("ovn4nfvk8s-cni: cmdDel ")
+ req := cniEndpointRequest(args)
+ _, err := ep.sendCNIServerReq(req)
+ return err
+}
diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go
index 002ad80..e9ad3e1 100644
--- a/internal/pkg/config/config.go
+++ b/internal/pkg/config/config.go
@@ -1,6 +1,7 @@
package config
import (
+ "encoding/json"
"fmt"
"os"
"path/filepath"
@@ -8,6 +9,8 @@ import (
"github.com/sirupsen/logrus"
"github.com/urfave/cli"
+ "github.com/containernetworking/cni/pkg/types"
+ "github.com/containernetworking/cni/pkg/version"
gcfg "gopkg.in/gcfg.v1"
"k8s.io/client-go/kubernetes"
@@ -283,3 +286,17 @@ func NewClientset(conf *KubernetesConfig) (*kubernetes.Clientset, error) {
return kubernetes.NewForConfig(kconfig)
}
+
+func ConfigureNetConf(bytes []byte) (*types.NetConf, error) {
+ conf := &types.NetConf{}
+ if err := json.Unmarshal(bytes, conf); err != nil {
+ return nil, fmt.Errorf("failed to load netconf: %v", err)
+ }
+
+ if conf.RawPrevResult != nil {
+ if err := version.ParsePrevResult(conf); err != nil {
+ return nil, err
+ }
+ }
+ return conf, nil
+}
diff --git a/internal/pkg/ovn/utils.go b/internal/pkg/ovn/utils.go
index 9b388b7..c2e9142 100644
--- a/internal/pkg/ovn/utils.go
+++ b/internal/pkg/ovn/utils.go
@@ -65,7 +65,7 @@ func SetExec(exec kexec.Interface) error {
if err != nil {
return err
}
- runner.hostIP = os.Getenv("HOST_IP")
+ runner.hostIP = os.Getenv("OVN_NB_TCP_SERVICE_HOST")
// OVN Host Port
runner.hostPort = "6641"
log.Info("Host Port", "IP", runner.hostIP, "Port", runner.hostPort)
diff --git a/pkg/controller/pod/pod_controller.go b/pkg/controller/pod/pod_controller.go
index 75ed731..d195782 100644
--- a/pkg/controller/pod/pod_controller.go
+++ b/pkg/controller/pod/pod_controller.go
@@ -170,7 +170,7 @@ func (r *ReconcilePod) addLogicalPorts(pod *corev1.Pod) error {
return err
}
}
-
+
switch {
case nfn.Type == "ovn4nfv":
ovnCtl, err := ovn.GetOvnController()
diff --git a/utilities/docker/debian/Dockerfile b/utilities/docker/debian/Dockerfile
index 674ee7e..5c91136 100644
--- a/utilities/docker/debian/Dockerfile
+++ b/utilities/docker/debian/Dockerfile
@@ -2,29 +2,12 @@ FROM ubuntu:18.04 as base
USER root
-RUN apt-get update && apt-get install -y iproute2 curl software-properties-common setpriv dpkg-dev netcat
+RUN apt-get update && apt-get install -y iproute2 curl software-properties-common setpriv dpkg-dev netcat jq wget
RUN mkdir -p /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb
RUN bash -xc "\
pushd /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch-dev_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-common_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-dkms_2.12.0-1_all.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-source_2.12.0-1_all.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-dbg_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-ipsec_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-pki_2.12.0-1_all.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-switch_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-testcontroller_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-test_2.12.0-1_all.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-vtep_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-central_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-common_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-controller-vtep_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-docker_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-host_2.12.0-1_amd64.deb; \
-curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/python-openvswitch_2.12.0-1_all.deb; \
+wget -q -nv -O- https://api.github.com/repos/akraino-icn/ovs/releases/tags/v2.12.0 2>/dev/null | jq -r '.assets[] | select(.browser_download_url | contains("\""deb"\"")) | .browser_download_url' | wget -i -; \
dpkg-scanpackages . | gzip -c9 > Packages.gz; \
popd; \
"
diff --git a/utilities/kernel/debian/install_kernel_modules.sh b/utilities/kernel/debian/install_kernel_modules.sh
new file mode 100755
index 0000000..20228e8
--- /dev/null
+++ b/utilities/kernel/debian/install_kernel_modules.sh
@@ -0,0 +1,32 @@
+!#/bin/bash
+
+apt-get install apt-transport-https dpkg-dev -y
+
+mkdir -p /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb
+pushd /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch-dev_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-common_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-dkms_2.12.0-1_all.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-source_2.12.0-1_all.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-dbg_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-ipsec_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-pki_2.12.0-1_all.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-switch_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-testcontroller_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-test_2.12.0-1_all.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-vtep_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-central_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-common_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-controller-vtep_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-docker_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-host_2.12.0-1_amd64.deb
+curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/python-openvswitch_2.12.0-1_all.deb
+dpkg-scanpackages . | gzip -c9 > Packages.gz
+popd
+
+sudo apt-get install apt-transport-https
+echo "deb [trusted=yes] file:///opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb ./" | tee -a /etc/apt/sources.list > /dev/null
+cp /etc/apt/sources.list /etc/apt/sources.list~
+sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list
+apt-get update && apt-get build-dep dkms -y && apt-get install openvswitch-datapath-dkms=2.12.0-1 -y