diff options
author | Kuralamudhan Ramakrishnan <kuralamudhan.ramakrishnan@intel.com> | 2020-04-21 17:19:34 +0000 |
---|---|---|
committer | Kuralamudhan Ramakrishnan <kuralamudhan.ramakrishnan@intel.com> | 2020-09-17 16:25:10 -0700 |
commit | 3de63ee756f9d7c0a4524b40a89e92b918a9249f (patch) | |
tree | 3579bc50aee0a38d7c63c5787fca6205ced7a2df | |
parent | 7f01772cdf3916026a93e9e9ac5ce54d57401476 (diff) |
Adding cnishim and cniserver
- inspired from ovn-kubernetes and sdn openshift - cniserver & cnishim concepts
- removed cni binary to depend on the host ovs binary installation
- encapsulated all the binaries within the ovn and ovs containers
- ovn4nfv-k8s cni server run along with nfn-agent
- cnishim act as the httpclient and cniserver as httpservers
- cnishim wrap all the cni commands to cniserver
- cniserver do the actual network pumping work and send result back to cnishim
- cnishim print the results as per the cni spec requirement
- currently supports only debian installation for ovn daemon
- support for debian kernel installation
- Consolidated all yaml into single ovn4nfv-k8s-plugin
Signed-off-by: Kuralamudhan Ramakrishnan <kuralamudhan.ramakrishnan@intel.com>
Change-Id: I1e2b114d90f717baa2ee94ff379c849d73b2754e
-rw-r--r-- | build/Dockerfile | 17 | ||||
-rw-r--r-- | cmd/nfn-agent/nfn-agent.go | 23 | ||||
-rw-r--r-- | cmd/ovn4nfvk8s-cni/app/helper_linux.go | 20 | ||||
-rw-r--r-- | cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go | 321 | ||||
-rw-r--r-- | deploy/ovn4nfv-k8s-plugin.yaml (renamed from deploy/ovn4nfv-k8s-plugin-daemonset.yml) | 63 | ||||
-rw-r--r-- | go.mod | 2 | ||||
-rw-r--r-- | go.sum | 2 | ||||
-rw-r--r-- | internal/pkg/cniserver/cni.go | 295 | ||||
-rw-r--r-- | internal/pkg/cniserver/cniserver.go | 235 | ||||
-rw-r--r-- | internal/pkg/cnishim/cnishim.go | 111 | ||||
-rw-r--r-- | internal/pkg/config/config.go | 17 | ||||
-rw-r--r-- | internal/pkg/ovn/utils.go | 2 | ||||
-rw-r--r-- | pkg/controller/pod/pod_controller.go | 2 | ||||
-rw-r--r-- | utilities/docker/debian/Dockerfile | 21 | ||||
-rwxr-xr-x | utilities/kernel/debian/install_kernel_modules.sh | 32 |
15 files changed, 802 insertions, 361 deletions
diff --git a/build/Dockerfile b/build/Dockerfile index 9530772..af88d13 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -8,13 +8,21 @@ ENV https_proxy $HTTPS_PROXY ENV no_proxy $NO_PROXY RUN apt-get update && apt-get install -y -qq apt-transport-https make curl net-tools iproute2 \ - && echo "deb https://packages.wand.net.nz bionic ovs-2.10" > /etc/apt/sources.list.d/ovs.list \ - && curl https://packages.wand.net.nz/keyring.gpg -o /etc/apt/trusted.gpg.d/wand.gpg \ - && apt-get update && apt install -y -qq ovn-common openvswitch-common openvswitch-switch + wget software-properties-common setpriv dpkg-dev netcat jq + +RUN mkdir -p /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb +RUN bash -xc "\ +pushd /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb; \ +wget -q -nv -O- https://api.github.com/repos/akraino-icn/ovs/releases/tags/v2.12.0 2>/dev/null | jq -r '.assets[] | select(.browser_download_url | contains("\""deb"\"")) | .browser_download_url' | wget -i -; \ +dpkg-scanpackages . | gzip -c9 > Packages.gz; \ +popd; \ +" +RUN echo "deb [trusted=yes] file:///opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb ./" | tee -a /etc/apt/sources.list > /dev/null +RUN apt-get update && apt-get install -y -qq ovn-common=2.12.0-1 openvswitch-common=2.12.0-1 openvswitch-switch=2.12.0-1 ENV GOLANG_VERSION 1.14.1 RUN curl -sSL https://storage.googleapis.com/golang/go$GOLANG_VERSION.linux-amd64.tar.gz \ - | tar -v -C /usr/local -xz + | tar -v -C /usr/local -xz ENV PATH /usr/local/go/bin:$PATH RUN mkdir -p /go/src /go/bin && chmod -R 777 /go @@ -32,5 +40,4 @@ ENV OPERATOR=/usr/local/bin/nfn-operator \ USER_NAME=nfn-operator RUN cp -r build/bin/* /usr/local/bin/ - ENTRYPOINT ["entrypoint"] diff --git a/cmd/nfn-agent/nfn-agent.go b/cmd/nfn-agent/nfn-agent.go index 3c85dbd..8d33778 100644 --- a/cmd/nfn-agent/nfn-agent.go +++ b/cmd/nfn-agent/nfn-agent.go @@ -9,8 +9,11 @@ import ( "os" "os/signal" pb "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify/proto" + cs "ovn4nfv-k8s-plugin/internal/pkg/cniserver" "ovn4nfv-k8s-plugin/internal/pkg/ovn" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "strings" "syscall" "time" @@ -298,6 +301,26 @@ func main() { client := pb.NewNfnNotifyClient(conn) errorChannel = make(chan string) + // creates the in-cluster config + config, err := rest.InClusterConfig() + if err != nil { + log.Error(err, "Unable to create in-cluster config") + return + } + + // creates the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Error(err, "Unable to create clientset for in-cluster config") + return + } + + cniserver := cs.NewCNIServer("",clientset) + err = cniserver.Start(cs.HandleCNIcommandRequest) + if err != nil { + log.Error(err, "Unable to start cni server") + return + } // Run client in background go subscribeNotif(client) shutdownHandler(errorChannel) diff --git a/cmd/ovn4nfvk8s-cni/app/helper_linux.go b/cmd/ovn4nfvk8s-cni/app/helper_linux.go index 2dba628..1702597 100644 --- a/cmd/ovn4nfvk8s-cni/app/helper_linux.go +++ b/cmd/ovn4nfvk8s-cni/app/helper_linux.go @@ -10,8 +10,6 @@ import ( "strings" "github.com/sirupsen/logrus" - - "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/plugins/pkg/ip" "github.com/containernetworking/plugins/pkg/ns" @@ -106,10 +104,10 @@ func setupInterface(netns ns.NetNS, containerID, ifName, macAddress, ipAddress, } // ConfigureInterface sets up the container interface -var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway string, idx, mtu int) ([]*current.Interface, error) { - netns, err := ns.GetNS(args.Netns) +var ConfigureInterface = func(containerNetns, containerID, ifName, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway string, idx, mtu int) ([]*current.Interface, error) { + netns, err := ns.GetNS(containerNetns) if err != nil { - return nil, fmt.Errorf("failed to open netns %q: %v", args.Netns, err) + return nil, fmt.Errorf("failed to open netns %q: %v", containerNetns, err) } defer netns.Close() @@ -118,10 +116,10 @@ var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress ifaceID = fmt.Sprintf("%s_%s_%s", namespace, podName, interfaceName) } else { ifaceID = fmt.Sprintf("%s_%s", namespace, podName) - interfaceName = args.IfName + interfaceName = ifName defaultGateway = "true" } - hostIface, contIface, err := setupInterface(netns, args.ContainerID, interfaceName, macAddress, ipAddress, gatewayIP, defaultGateway, idx, mtu) + hostIface, contIface, err := setupInterface(netns, containerID, interfaceName, macAddress, ipAddress, gatewayIP, defaultGateway, idx, mtu) if err != nil { return nil, err } @@ -132,7 +130,7 @@ var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress fmt.Sprintf("external_ids:attached_mac=%s", macAddress), fmt.Sprintf("external_ids:iface-id=%s", ifaceID), fmt.Sprintf("external_ids:ip_address=%s", ipAddress), - fmt.Sprintf("external_ids:sandbox=%s", args.ContainerID), + fmt.Sprintf("external_ids:sandbox=%s", containerID), } var out []byte @@ -160,10 +158,10 @@ func setupRoute(netns ns.NetNS, dst, gw, dev string) error { } // ConfigureRoute sets up the container routes -var ConfigureRoute = func(args *skel.CmdArgs, dst, gw, dev string) error { - netns, err := ns.GetNS(args.Netns) +var ConfigureRoute = func(containerNetns, dst, gw, dev string) error { + netns, err := ns.GetNS(containerNetns) if err != nil { - return fmt.Errorf("failed to open netns %q: %v", args.Netns, err) + return fmt.Errorf("failed to open netns %q: %v", containerNetns, err) } defer netns.Close() err = setupRoute(netns, dst, gw, dev) diff --git a/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go b/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go index c176700..cb75ecd 100644 --- a/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go +++ b/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go @@ -1,335 +1,40 @@ -// +build linux - package main import ( - "encoding/json" - "fmt" - "net" "os" - "strconv" - "strings" - "time" "github.com/sirupsen/logrus" "github.com/urfave/cli" + cni "ovn4nfv-k8s-plugin/internal/pkg/cnishim" + "ovn4nfv-k8s-plugin/internal/pkg/config" + "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" - "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/cni/pkg/version" - "k8s.io/apimachinery/pkg/util/wait" - - "ovn4nfv-k8s-plugin/internal/pkg/kube" - - "ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app" - "ovn4nfv-k8s-plugin/internal/pkg/config" + "github.com/containernetworking/plugins/pkg/utils/buildversion" ) -const ( - ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces" -) - -func argString2Map(args string) (map[string]string, error) { - argsMap := make(map[string]string) - - pairs := strings.Split(args, ";") - for _, pair := range pairs { - kv := strings.Split(pair, "=") - if len(kv) != 2 { - return nil, fmt.Errorf("ARGS: invalid pair %q", pair) - } - keyString := kv[0] - valueString := kv[1] - argsMap[keyString] = valueString - } - - return argsMap, nil -} - -func parseOvnNetworkObject(ovnnetwork string) ([]map[string]string, error) { - var ovnNet []map[string]string - - if ovnnetwork == "" { - return nil, fmt.Errorf("parseOvnNetworkObject:error") - } - - if err := json.Unmarshal([]byte(ovnnetwork), &ovnNet); err != nil { - return nil, fmt.Errorf("parseOvnNetworkObject: failed to load ovn network err: %v | ovn network: %v", err, ovnnetwork) - } - - return ovnNet, nil -} - -func mergeWithResult(srcObj, dstObj types.Result) (types.Result, error) { - - if dstObj == nil { - return srcObj, nil - } - src, err := current.NewResultFromResult(srcObj) - if err != nil { - return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err) - } - dst, err := current.NewResultFromResult(dstObj) - if err != nil { - return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err) - } - - ifacesLength := len(dst.Interfaces) - - for _, iface := range src.Interfaces { - dst.Interfaces = append(dst.Interfaces, iface) - } - for _, ip := range src.IPs { - if ip.Interface != nil && *(ip.Interface) != -1 { - ip.Interface = current.Int(*(ip.Interface) + ifacesLength) - } - dst.IPs = append(dst.IPs, ip) - } - for _, route := range src.Routes { - dst.Routes = append(dst.Routes, route) - } - - for _, ns := range src.DNS.Nameservers { - dst.DNS.Nameservers = append(dst.DNS.Nameservers, ns) - } - for _, s := range src.DNS.Search { - dst.DNS.Search = append(dst.DNS.Search, s) - } - for _, opt := range src.DNS.Options { - dst.DNS.Options = append(dst.DNS.Options, opt) - } - // TODO: what about DNS.domain? - return dst, nil -} - -func prettyPrint(i interface{}) string { - s, _ := json.MarshalIndent(i, "", "\t") - return string(s) -} - -func addMultipleInterfaces(args *skel.CmdArgs, ovnAnnotation, namespace, podName string) types.Result { - logrus.Infof("ovn4nfvk8s-cni: addMultipleInterfaces ") - - var ovnAnnotatedMap []map[string]string - ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation) - if err != nil { - logrus.Errorf("addLogicalPort : Error Parsing Ovn Network List %v %v", ovnAnnotatedMap, err) - return nil - } - if namespace == "" || podName == "" { - logrus.Errorf("required CNI variable missing") - return nil - } - var interfacesArray []*current.Interface - var index int - var result *current.Result - var dstResult types.Result - for _, ovnNet := range ovnAnnotatedMap { - ipAddress := ovnNet["ip_address"] - macAddress := ovnNet["mac_address"] - gatewayIP := ovnNet["gateway_ip"] - defaultGateway := ovnNet["defaultGateway"] - - if ipAddress == "" || macAddress == "" { - logrus.Errorf("failed in pod annotation key extract") - return nil - } - - index++ - interfaceName := ovnNet["interface"] - if interfaceName == "" { - logrus.Errorf("addMultipleInterfaces: interface can't be null") - return nil - } - interfacesArray, err = app.ConfigureInterface(args, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway, index, config.Default.MTU) - if err != nil { - logrus.Errorf("Failed to configure interface in pod: %v", err) - return nil - } - addr, addrNet, err := net.ParseCIDR(ipAddress) - if err != nil { - logrus.Errorf("failed to parse IP address %q: %v", ipAddress, err) - return nil - } - ipVersion := "6" - if addr.To4() != nil { - ipVersion = "4" - } - var routes types.Route - if defaultGateway == "true" { - defaultAddr, defaultAddrNet, _ := net.ParseCIDR("0.0.0.0/0") - routes = types.Route{Dst: net.IPNet{IP: defaultAddr, Mask: defaultAddrNet.Mask}, GW: net.ParseIP(gatewayIP)} - - result = ¤t.Result{ - Interfaces: interfacesArray, - IPs: []*current.IPConfig{ - { - Version: ipVersion, - Interface: current.Int(1), - Address: net.IPNet{IP: addr, Mask: addrNet.Mask}, - Gateway: net.ParseIP(gatewayIP), - }, - }, - Routes: []*types.Route{&routes}, - } - } else { - result = ¤t.Result{ - Interfaces: interfacesArray, - IPs: []*current.IPConfig{ - { - Version: ipVersion, - Interface: current.Int(1), - Address: net.IPNet{IP: addr, Mask: addrNet.Mask}, - Gateway: net.ParseIP(gatewayIP), - }, - }, - } - - } - // Build the result structure to pass back to the runtime - dstResult, err = mergeWithResult(types.Result(result), dstResult) - if err != nil { - logrus.Errorf("Failed to merge results: %v", err) - return nil - } - } - logrus.Infof("addMultipleInterfaces: %s", prettyPrint(dstResult)) - return dstResult -} - -func addRoutes(args *skel.CmdArgs, ovnAnnotation string, dstResult types.Result) types.Result { - logrus.Infof("ovn4nfvk8s-cni: addRoutes ") - - var ovnAnnotatedMap []map[string]string - ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation) - if err != nil { - logrus.Errorf("addLogicalPort : Error Parsing Ovn Route List %v", err) - return nil - } - - var result types.Result - var routes []*types.Route - for _, ovnNet := range ovnAnnotatedMap { - dst := ovnNet["dst"] - gw := ovnNet["gw"] - dev := ovnNet["dev"] - if dst == "" || gw == "" || dev == "" { - logrus.Errorf("failed in pod annotation key extract") - return nil - } - err = app.ConfigureRoute(args, dst, gw, dev) - if err != nil { - logrus.Errorf("Failed to configure interface in pod: %v", err) - return nil - } - dstAddr, dstAddrNet, _ := net.ParseCIDR(dst) - routes = append(routes, &types.Route{ - Dst: net.IPNet{IP: dstAddr, Mask: dstAddrNet.Mask}, - GW: net.ParseIP(gw), - }) - } - - result = ¤t.Result{ - Routes: routes, - } - // Build the result structure to pass back to the runtime - dstResult, err = mergeWithResult(result, dstResult) - if err != nil { - logrus.Errorf("Failed to merge results: %v", err) - return nil - } - logrus.Infof("addRoutes: %s", prettyPrint(dstResult)) - return dstResult - -} - -func cmdAdd(args *skel.CmdArgs) error { - logrus.Infof("ovn4nfvk8s-cni: cmdAdd ") - conf := &types.NetConf{} - if err := json.Unmarshal(args.StdinData, conf); err != nil { - return fmt.Errorf("failed to load netconf: %v", err) - } - - argsMap, err := argString2Map(args.Args) - if err != nil { - return err - } - - namespace := argsMap["K8S_POD_NAMESPACE"] - podName := argsMap["K8S_POD_NAME"] - if namespace == "" || podName == "" { - return fmt.Errorf("required CNI variable missing") - } - - clientset, err := config.NewClientset(&config.Kubernetes) - if err != nil { - return fmt.Errorf("Could not create clientset for kubernetes: %v", err) - } - kubecli := &kube.Kube{KClient: clientset} - - // Get the IP address and MAC address from the API server. - var annotationBackoff = wait.Backoff{Duration: 1 * time.Second, Steps: 14, Factor: 1.5, Jitter: 0.1} - var annotation map[string]string - if err := wait.ExponentialBackoff(annotationBackoff, func() (bool, error) { - annotation, err = kubecli.GetAnnotationsOnPod(namespace, podName) - if err != nil { - // TODO: check if err is non recoverable - logrus.Warningf("Error while obtaining pod annotations - %v", err) - return false, nil - } - if _, ok := annotation[ovn4nfvAnnotationTag]; ok { - return true, nil - } - return false, nil - }); err != nil { - return fmt.Errorf("failed to get pod annotation - %v", err) - } - logrus.Infof("ovn4nfvk8s-cni: Annotation Found ") - ovnAnnotation, ok := annotation[ovn4nfvAnnotationTag] - if !ok { - return fmt.Errorf("Error while obtaining pod annotations") - } - result := addMultipleInterfaces(args, ovnAnnotation, namespace, podName) - // Add Routes to the pod if annotation found for routes - ovnRouteAnnotation, ok := annotation["ovnNetworkRoutes"] - if ok { - logrus.Infof("ovn4nfvk8s-cni: ovnNetworkRoutes Annotation Found %+v", ovnRouteAnnotation) - result = addRoutes(args, ovnRouteAnnotation, result) - } - - return result.Print() -} - -func cmdDel(args *skel.CmdArgs) error { - logrus.Infof("ovn4nfvk8s-cni: cmdDel ") - for i := 0; i < 10; i++ { - ifaceName := args.ContainerID[:14] + strconv.Itoa(i) - done, err := app.PlatformSpecificCleanup(ifaceName) - if err != nil { - logrus.Errorf("Teardown error: %v", err) - } - if done { - break - } - } - return nil -} - func main() { - logrus.Infof("ovn4nfvk8s-cni invoked") + logrus.Infof("ovn4nfvk8s-cni shim cni") c := cli.NewApp() c.Name = "ovn4nfvk8s-cni" c.Usage = "a CNI plugin to set up or tear down a additional interfaces with OVN" - c.Version = "0.0.2" + c.Version = "0.1.0" c.Flags = config.Flags + ep := cni.CNIEndpoint("") c.Action = func(ctx *cli.Context) error { if _, err := config.InitConfig(ctx); err != nil { return err } + skel.PluginMain( + ep.CmdAdd, + ep.CmdCheck, + ep.CmdDel, + version.All, + buildversion.BuildString("ovn4nfv-k8s shim cni")) - skel.PluginMain(cmdAdd, nil, cmdDel, version.All, "") return nil } diff --git a/deploy/ovn4nfv-k8s-plugin-daemonset.yml b/deploy/ovn4nfv-k8s-plugin.yaml index 13e749f..5d5017e 100644 --- a/deploy/ovn4nfv-k8s-plugin-daemonset.yml +++ b/deploy/ovn4nfv-k8s-plugin.yaml @@ -284,10 +284,9 @@ apiVersion: v1 kind: ServiceAccount metadata: name: k8s-nfn-sa - namespace: operator + namespace: kube-system --- - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -298,6 +297,7 @@ rules: - "" resources: - pods + - pods/status - services - endpoints - persistentvolumeclaims @@ -361,7 +361,7 @@ apiVersion: v1 kind: Service metadata: name: nfn-operator - namespace: operator + namespace: kube-system spec: type: NodePort ports: @@ -378,7 +378,7 @@ apiVersion: v1 kind: ConfigMap metadata: name: ovn-controller-network - namespace: operator + namespace: kube-system data: OVN_SUBNET: "10.244.64.0/18" OVN_GATEWAYIP: "10.244.64.20/18" @@ -391,7 +391,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: nfn-operator - namespace: operator + namespace: kube-system spec: replicas: 1 selector: @@ -408,10 +408,10 @@ spec: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: nfnType + - key: ovn4nfv-k8s-plugin operator: In values: - - operator + - ovn-control-plane tolerations: - key: "node-role.kubernetes.io/master" effect: "NoSchedule" @@ -429,10 +429,6 @@ spec: - containerPort: 50000 protocol: TCP env: - - name: HOST_IP - valueFrom: - fieldRef: - fieldPath: status.hostIP - name: POD_NAME valueFrom: fieldRef: @@ -445,7 +441,7 @@ kind: ConfigMap apiVersion: v1 metadata: name: ovn4nfv-cni-config - namespace: operator + namespace: kube-system labels: app: ovn4nfv data: @@ -459,14 +455,20 @@ data: plugin=ovn4nfvk8s-cni [kubernetes] - kubeconfig=/etc/kubernetes/admin.conf + kubeconfig=/etc/cni/net.d/ovn4nfv-k8s.d/ovn4nfv-k8s.kubeconfig + 00-network.conf: | + { + "name": "ovn4nfv-k8s-plugin", + "type": "ovn4nfvk8s-cni", + "cniVersion": "0.3.1" + } --- apiVersion: extensions/v1beta1 kind: DaemonSet metadata: name: ovn4nfv-cni - namespace: operator + namespace: kube-system labels: app: ovn4nfv spec: @@ -483,6 +485,7 @@ spec: tolerations: - operator: Exists effect: NoSchedule + serviceAccountName: k8s-nfn-sa containers: - name: ovn4nfv image: integratedcloudnative/ovn4nfv-k8s-plugin:master @@ -497,13 +500,20 @@ spec: securityContext: privileged: true volumeMounts: + - name: cni + mountPath: /host/etc/cni/net.d - name: cnibin mountPath: /host/opt/cni/bin - name: cniconf mountPath: /host/etc/openvswitch - name: ovn4nfv-cfg mountPath: /tmp/ovn4nfv-conf + - name: ovn4nfv-cni-net-conf + mountPath: /tmp/ovn4nfv-cni volumes: + - name: cni + hostPath: + path: /etc/cni/net.d - name: cnibin hostPath: path: /opt/cni/bin @@ -516,13 +526,18 @@ spec: items: - key: ovn4nfv_k8s.conf path: ovn4nfv_k8s.conf - + - name: ovn4nfv-cni-net-conf + configMap: + name: ovn4nfv-cni-config + items: + - key: 00-network.conf + path: 00-network.conf --- apiVersion: extensions/v1beta1 kind: DaemonSet metadata: name: nfn-agent - namespace: operator + namespace: kube-system labels: app: nfn-agent spec: @@ -534,11 +549,13 @@ spec: app: nfn-agent spec: hostNetwork: true + hostPID: true nodeSelector: beta.kubernetes.io/arch: amd64 tolerations: - operator: Exists effect: NoSchedule + serviceAccountName: k8s-nfn-sa containers: - name: nfn-agent image: integratedcloudnative/ovn4nfv-k8s-plugin:master @@ -556,12 +573,20 @@ spec: fieldRef: fieldPath: spec.nodeName securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN", "SYS_ADMIN", "SYS_PTRACE"] privileged: true volumeMounts: + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true - mountPath: /run/openvswitch name: host-run-ovs - mountPath: /var/run/openvswitch name: host-var-run-ovs + - mountPath: /var/run/ovn4nfv-k8s-plugin + name: host-var-cniserver-socket-dir volumes: - name: host-run-ovs hostPath: @@ -569,3 +594,9 @@ spec: - name: host-var-run-ovs hostPath: path: /var/run/openvswitch + - name: host-var-run-dbus + hostPath: + path: /var/run/dbus + - name: host-var-cniserver-socket-dir + hostPath: + path: /var/run/ovn4nfv-k8s-plugin @@ -22,6 +22,7 @@ require ( github.com/google/btree v1.0.0 // indirect github.com/google/go-cmp v0.3.1 // indirect github.com/gophercloud/gophercloud v0.2.0 // indirect + github.com/gorilla/mux v1.7.4 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/grpc-ecosystem/grpc-gateway v1.8.5 // indirect github.com/hashicorp/golang-lru v0.5.3 // indirect @@ -67,6 +68,7 @@ require ( k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible k8s.io/code-generator v0.17.0 // indirect k8s.io/gengo v0.0.0-20191120174120-e74f70b9b27e // indirect + k8s.io/klog v1.0.0 k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a k8s.io/utils v0.0.0-20190801114015-581e00157fb1 sigs.k8s.io/controller-runtime v0.2.0-beta.4 @@ -319,6 +319,8 @@ github.com/gophercloud/gophercloud v0.2.0 h1:lD2Bce2xBAMNNcFZ0dObTpXkGLlVIb33RPV github.com/gophercloud/gophercloud v0.2.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= diff --git a/internal/pkg/cniserver/cni.go b/internal/pkg/cniserver/cni.go new file mode 100644 index 0000000..2c91f04 --- /dev/null +++ b/internal/pkg/cniserver/cni.go @@ -0,0 +1,295 @@ +package cniserver + +import ( + "encoding/json" + "k8s.io/apimachinery/pkg/util/wait" + "fmt" + "net" + "strconv" + "net/http" + "time" + "k8s.io/klog" + + "k8s.io/client-go/kubernetes" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "ovn4nfv-k8s-plugin/internal/pkg/kube" + "k8s.io/apimachinery/pkg/api/errors" + "ovn4nfv-k8s-plugin/internal/pkg/config" + "ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app" +) + +const ( + ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces" +) + +func parseOvnNetworkObject(ovnnetwork string) ([]map[string]string, error) { + var ovnNet []map[string]string + + if ovnnetwork == "" { + return nil, fmt.Errorf("parseOvnNetworkObject:error") + } + + if err := json.Unmarshal([]byte(ovnnetwork), &ovnNet); err != nil { + return nil, fmt.Errorf("parseOvnNetworkObject: failed to load ovn network err: %v | ovn network: %v", err, ovnnetwork) + } + + return ovnNet, nil +} + +func mergeWithResult(srcObj, dstObj types.Result) (types.Result, error) { + + if dstObj == nil { + return srcObj, nil + } + src, err := current.NewResultFromResult(srcObj) + if err != nil { + return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err) + } + dst, err := current.NewResultFromResult(dstObj) + if err != nil { + return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err) + } + + ifacesLength := len(dst.Interfaces) + + for _, iface := range src.Interfaces { + dst.Interfaces = append(dst.Interfaces, iface) + } + for _, ip := range src.IPs { + if ip.Interface != nil && *(ip.Interface) != -1 { + ip.Interface = current.Int(*(ip.Interface) + ifacesLength) + } + dst.IPs = append(dst.IPs, ip) + } + for _, route := range src.Routes { + dst.Routes = append(dst.Routes, route) + } + + for _, ns := range src.DNS.Nameservers { + dst.DNS.Nameservers = append(dst.DNS.Nameservers, ns) + } + for _, s := range src.DNS.Search { + dst.DNS.Search = append(dst.DNS.Search, s) + } + for _, opt := range src.DNS.Options { + dst.DNS.Options = append(dst.DNS.Options, opt) + } + // TODO: what about DNS.domain? + return dst, nil +} + +func prettyPrint(i interface{}) string { + s, _ := json.MarshalIndent(i, "", "\t") + return string(s) +} + +func isNotFoundError(err error) bool { + statusErr, ok := err.(*errors.StatusError) + return ok && statusErr.Status().Code == http.StatusNotFound +} + +func (cr *CNIServerRequest) addMultipleInterfaces(ovnAnnotation, namespace, podName string) types.Result { + klog.Infof("ovn4nfvk8s-cni: addMultipleInterfaces ") + var ovnAnnotatedMap []map[string]string + ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation) + if err != nil { + klog.Errorf("addLogicalPort : Error Parsing Ovn Network List %v %v", ovnAnnotatedMap, err) + return nil + } + if namespace == "" || podName == "" { + klog.Errorf("required CNI variable missing") + return nil + } + var interfacesArray []*current.Interface + var index int + var result *current.Result + var dstResult types.Result + for _, ovnNet := range ovnAnnotatedMap { + ipAddress := ovnNet["ip_address"] + macAddress := ovnNet["mac_address"] + gatewayIP := ovnNet["gateway_ip"] + defaultGateway := ovnNet["defaultGateway"] + + if ipAddress == "" || macAddress == "" { + klog.Errorf("failed in pod annotation key extract") + return nil + } + + index++ + interfaceName := ovnNet["interface"] + if interfaceName == "" { + klog.Errorf("addMultipleInterfaces: interface can't be null") + return nil + } + klog.Infof("addMultipleInterfaces: ipAddress %v %v", ipAddress, interfaceName) + interfacesArray, err = app.ConfigureInterface(cr.Netns, cr.SandboxID, cr.IfName, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway, index, config.Default.MTU) + if err != nil { + klog.Errorf("Failed to configure interface in pod: %v", err) + return nil + } + addr, addrNet, err := net.ParseCIDR(ipAddress) + if err != nil { + klog.Errorf("failed to parse IP address %q: %v", ipAddress, err) + return nil + } + ipVersion := "6" + if addr.To4() != nil { + ipVersion = "4" + } + var routes types.Route + if defaultGateway == "true" { + defaultAddr, defaultAddrNet, _ := net.ParseCIDR("0.0.0.0/0") + routes = types.Route{Dst: net.IPNet{IP: defaultAddr, Mask: defaultAddrNet.Mask}, GW: net.ParseIP(gatewayIP)} + + result = ¤t.Result{ + Interfaces: interfacesArray, + IPs: []*current.IPConfig{ + { + Version: ipVersion, + Interface: current.Int(1), + Address: net.IPNet{IP: addr, Mask: addrNet.Mask}, + Gateway: net.ParseIP(gatewayIP), + }, + }, + Routes: []*types.Route{&routes}, + } + } else { + result = ¤t.Result{ + Interfaces: interfacesArray, + IPs: []*current.IPConfig{ + { + Version: ipVersion, + Interface: current.Int(1), + Address: net.IPNet{IP: addr, Mask: addrNet.Mask}, + Gateway: net.ParseIP(gatewayIP), + }, + }, + } + + } + // Build the result structure to pass back to the runtime + dstResult, err = mergeWithResult(types.Result(result), dstResult) + if err != nil { + klog.Errorf("Failed to merge results: %v", err) + return nil + } + } + klog.Infof("addMultipleInterfaces: results %s", prettyPrint(dstResult)) + return dstResult +} + +func (cr *CNIServerRequest) addRoutes(ovnAnnotation string, dstResult types.Result) types.Result { + klog.Infof("ovn4nfvk8s-cni: addRoutes ") + var ovnAnnotatedMap []map[string]string + ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation) + if err != nil { + klog.Errorf("addLogicalPort : Error Parsing Ovn Route List %v", err) + return nil + } + + var result types.Result + var routes []*types.Route + for _, ovnNet := range ovnAnnotatedMap { + dst := ovnNet["dst"] + gw := ovnNet["gw"] + dev := ovnNet["dev"] + if dst == "" || gw == "" || dev == "" { + klog.Errorf("failed in pod annotation key extract") + return nil + } + err = app.ConfigureRoute(cr.Netns, dst, gw, dev) + if err != nil { + klog.Errorf("Failed to configure interface in pod: %v", err) + return nil + } + dstAddr, dstAddrNet, _ := net.ParseCIDR(dst) + routes = append(routes, &types.Route{ + Dst: net.IPNet{IP: dstAddr, Mask: dstAddrNet.Mask}, + GW: net.ParseIP(gw), + }) + } + + result = ¤t.Result{ + Routes: routes, + } + // Build the result structure to pass back to the runtime + dstResult, err = mergeWithResult(result, dstResult) + if err != nil { + klog.Errorf("Failed to merge results: %v", err) + return nil + } + klog.Infof("addRoutes: results %s", prettyPrint(dstResult)) + return dstResult +} + +func (cr *CNIServerRequest) cmdAdd(kclient kubernetes.Interface) ([]byte, error) { + klog.Infof("ovn4nfvk8s-cni: cmdAdd") + namespace := cr.PodNamespace + podname := cr.PodName + if namespace == "" || podname == "" { + return nil, fmt.Errorf("required CNI variable missing") + } + klog.Infof("ovn4nfvk8s-cni: cmdAdd for pod podname:%s and namespace:%s", podname, namespace) + kubecli := &kube.Kube{KClient: kclient} + // Get the IP address and MAC address from the API server. + var annotationBackoff = wait.Backoff{Duration: 1 * time.Second, Steps: 14, Factor: 1.5, Jitter: 0.1} + var annotation map[string]string + var err error + if err = wait.ExponentialBackoff(annotationBackoff, func() (bool, error) { + annotation, err = kubecli.GetAnnotationsOnPod(namespace, podname) + if err != nil { + if isNotFoundError(err) { + return false, fmt.Errorf("Error - pod not found - %v", err) + } + klog.Infof("ovn4nfvk8s-cni: cmdAdd Warning - Error while obtaining pod annotations - %v", err) + return false,nil + } + if _, ok := annotation[ovn4nfvAnnotationTag]; ok { + return true, nil + } + return false, nil + }); err != nil { + return nil, fmt.Errorf("failed to get pod annotation - %v", err) + } + + klog.Infof("ovn4nfvk8s-cni: cmdAdd Annotation Found ") + ovnAnnotation, ok := annotation[ovn4nfvAnnotationTag] + if !ok { + return nil, fmt.Errorf("Error while obtaining pod annotations") + } + result := cr.addMultipleInterfaces(ovnAnnotation, namespace, podname) + //Add Routes to the pod if annotation found for routes + ovnRouteAnnotation, ok := annotation["ovnNetworkRoutes"] + if ok { + klog.Infof("ovn4nfvk8s-cni: ovnNetworkRoutes Annotation Found %+v", ovnRouteAnnotation) + result = cr.addRoutes(ovnRouteAnnotation, result) + } + + if result == nil { + klog.Errorf("result struct the ovn4nfv-k8s-plugin cniserver") + return nil, fmt.Errorf("result is nil from cni server response") + } + + responseBytes, err := json.Marshal(result) + if err != nil { + return nil, fmt.Errorf("failed to marshal pod request response: %v", err) + } + + return responseBytes, nil +} + +func (cr *CNIServerRequest) cmdDel() ([]byte, error) { + klog.Infof("cmdDel ") + for i := 0; i < 10; i++ { + ifaceName := cr.SandboxID[:14] + strconv.Itoa(i) + done, err := app.PlatformSpecificCleanup(ifaceName) + if err != nil { + klog.Errorf("Teardown error: %v", err) + } + if done { + break + } + } + return []byte{}, nil +} diff --git a/internal/pkg/cniserver/cniserver.go b/internal/pkg/cniserver/cniserver.go new file mode 100644 index 0000000..eaa7105 --- /dev/null +++ b/internal/pkg/cniserver/cniserver.go @@ -0,0 +1,235 @@ +package cniserver + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "os" + "net" + "path/filepath" + "syscall" + "k8s.io/klog" + + "github.com/containernetworking/cni/pkg/types" + "github.com/gorilla/mux" + "k8s.io/client-go/kubernetes" + "ovn4nfv-k8s-plugin/internal/pkg/config" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + utilwait "k8s.io/apimachinery/pkg/util/wait" +) + +const CNIServerRunDir string = "/var/run/ovn4nfv-k8s-plugin/cniserver" +const CNIServerSocketName string = "ovn4nfv-k8s-plugin-cni-server.sock" +const CNIServerSocketPath string = CNIServerRunDir + "/" + CNIServerSocketName + + +type CNIcommand string + +const CNIAdd CNIcommand = "ADD" +const CNIUpdate CNIcommand = "UPDATE" +const CNIDel CNIcommand = "DEL" + +type CNIServerRequest struct { + Command CNIcommand + PodNamespace string + PodName string + SandboxID string + Netns string + IfName string + CNIConf *types.NetConf +} + +type cniServerRequestFunc func(request *CNIServerRequest, k8sclient kubernetes.Interface) ([]byte, error) + +type CNIEndpointRequest struct { + ArgEnv map[string]string `json:"env,omitempty"` + NetConfig []byte `json:"config,omitempty"` +} +type CNIServer struct { + http.Server + requestFunc cniServerRequestFunc + serverrundir string + k8sclient kubernetes.Interface +} + +func NewCNIServer(serverRunSir string, k8sclient kubernetes.Interface) *CNIServer { + klog.Infof("Setting up CNI server in nfn-agent") + if len(serverRunSir) == 0 { + serverRunSir = CNIServerRunDir + } + + router := mux.NewRouter() + cs := &CNIServer{ + Server: http.Server{ + Handler: router, + }, + serverrundir: serverRunSir, + k8sclient: k8sclient, + } + router.NotFoundHandler = http.HandlerFunc(http.NotFound) + router.HandleFunc("/", cs.handleCNIShimRequest).Methods("POST") + return cs +} + +func loadCNIShimArgs(env map[string]string) (map[string]string, error) { + cnishimArgs, ok := env["CNI_ARGS"] + if !ok { + return nil, fmt.Errorf("cnishim req missing CNI_ARGS: '%s'", env) + } + + mapArgs := make(map[string]string) + for _, arg := range strings.Split(cnishimArgs, ";") { + parts := strings.Split(arg, "=") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid CNI_ARG from cnishim '%s'", arg) + } + mapArgs[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1]) + } + return mapArgs, nil +} + +func loadCNIRequestToCNIServer(r *CNIEndpointRequest) (*CNIServerRequest, error) { + cmd, ok := r.ArgEnv["CNI_COMMAND"] + if !ok { + return nil, fmt.Errorf("cnishim req missing CNI_COMMAND") + } + + cnishimreq := &CNIServerRequest{ + Command: CNIcommand(cmd), + } + + cnishimreq.SandboxID, ok = r.ArgEnv["CNI_CONTAINERID"] + if !ok { + return nil, fmt.Errorf("cnishim req missing CNI_CONTAINERID") + } + + cnishimreq.Netns, ok = r.ArgEnv["CNI_NETNS"] + if !ok { + return nil, fmt.Errorf("cnishim req missing CNI_NETNS") + } + + cnishimreq.IfName, ok = r.ArgEnv["CNI_IFNAME"] + if !ok { + return nil, fmt.Errorf("cnishim req missing CNI_IFNAME") + } + + cnishimArgs, err := loadCNIShimArgs(r.ArgEnv) + if err != nil { + return nil, err + } + + cnishimreq.PodNamespace, ok = cnishimArgs["K8S_POD_NAMESPACE"] + if !ok { + return nil, fmt.Errorf("cnishim req missing K8S_POD_NAMESPACE") + } + + cnishimreq.PodName, ok = cnishimArgs["K8S_POD_NAME"] + if !ok { + return nil, fmt.Errorf("cnishim req missing K8S_POD_NAME") + } + + netconf, err := config.ConfigureNetConf(r.NetConfig) + if err != nil { + return nil, fmt.Errorf("cnishim req CNI arg configuration failed:%v",err) + } + + cnishimreq.CNIConf = netconf + return cnishimreq, nil +} + +func (cs *CNIServer) handleCNIShimRequest(w http.ResponseWriter, r *http.Request) { + var cr CNIEndpointRequest + b, _ := ioutil.ReadAll(r.Body) + if err := json.Unmarshal(b, &cr); err != nil { + http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest) + return + } + + req, err := loadCNIRequestToCNIServer(&cr) + if err != nil { + http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest) + return + } + + klog.Infof("Waiting for %s result for CNI server pod %s/%s", req.Command, req.PodNamespace, req.PodName) + result, err := cs.requestFunc(req, cs.k8sclient) + if err != nil { + http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest) + } else { + w.Header().Set("Content-Type", "application/json") + if _, err := w.Write(result); err != nil { + klog.Warningf("Error writing %s HTTP response: %v", req.Command, err) + } + } +} + +func HandleCNIcommandRequest(request *CNIServerRequest, k8sclient kubernetes.Interface) ([]byte, error) { + var result []byte + var err error + klog.Infof("[PodNamespace:%s/PodName:%s] dispatching pod network request %v", request.PodNamespace, request.PodName, request) + klog.Infof("k8sclient %s", fmt.Sprintf("%v",k8sclient)) + switch request.Command { + case CNIAdd: + result, err = request.cmdAdd(k8sclient) + case CNIDel: + result, err = request.cmdDel() + default: + } + klog.Infof("[PodNamespace:%s/PodName:%s] CNI request %v, result %q, err %v", request.PodNamespace, request.PodName, request, string(result), err) + if err != nil { + return nil, fmt.Errorf("[PodNamespace:%s/PodName:%s] CNI request %v %v", request.PodNamespace, request.PodName, request, err) + } + return result, nil +} + +func (cs *CNIServer) Start(requestFunc cniServerRequestFunc) error { + if requestFunc == nil { + return fmt.Errorf("no CNI request handler") + } + cs.requestFunc = requestFunc + socketPath := filepath.Join(cs.serverrundir, CNIServerSocketName) + if err := os.RemoveAll(cs.serverrundir); err != nil && !os.IsNotExist(err) { + info, err := os.Stat(cs.serverrundir) + if err != nil { + return fmt.Errorf("failed to stat old cni server info socket directory %s: %v", cs.serverrundir, err) + } + tmp := info.Sys() + statt, ok := tmp.(*syscall.Stat_t) + if !ok { + return fmt.Errorf("failed to read CNI Server info socket directory stat info: %T", tmp) + } + if statt.Uid != 0 { + return fmt.Errorf("insecure owner of CNI Server info socket directory %s: %v", cs.serverrundir, statt.Uid) + } + + if info.Mode()&0777 != 0700 { + return fmt.Errorf("insecure permissions on CNI Server info socket directory %s: %v", cs.serverrundir, info.Mode()) + } + + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove old CNI Server info socket %s: %v", socketPath, err) + } + } + if err := os.MkdirAll(cs.serverrundir, 0700); err != nil { + return fmt.Errorf("failed to create CNI Server info socket directory %s: %v", cs.serverrundir, err) + } + + unixListener, err := net.Listen("unix", socketPath) + if err != nil { + return fmt.Errorf("failed to listen on CNI Server info socket: %v", err) + } + if err := os.Chmod(socketPath, 0600); err != nil { + unixListener.Close() + return fmt.Errorf("failed to set CNI Server info socket mode: %v", err) + } + + cs.SetKeepAlivesEnabled(false) + go utilwait.Forever(func() { + if err := cs.Serve(unixListener); err != nil { + utilruntime.HandleError(fmt.Errorf("CNI server Serve() failed: %v", err)) + } + }, 0) + return nil +} diff --git a/internal/pkg/cnishim/cnishim.go b/internal/pkg/cnishim/cnishim.go new file mode 100644 index 0000000..ffedbfb --- /dev/null +++ b/internal/pkg/cnishim/cnishim.go @@ -0,0 +1,111 @@ +package cni + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net" + "net/http" + "os" + "ovn4nfv-k8s-plugin/internal/pkg/cniserver" + "ovn4nfv-k8s-plugin/internal/pkg/config" + "strings" + + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/sirupsen/logrus" +) + +const CNIEndpointURLReq string = "http://dummy/" + +type Endpoint struct { + cniServerSocketPath string +} + +func CNIEndpoint(cniServerSocketPath string) *Endpoint { + if len(cniServerSocketPath) == 0 { + cniServerSocketPath = cniserver.CNIServerSocketPath + } + return &Endpoint{cniServerSocketPath: cniServerSocketPath} +} + +func cniEndpointRequest(args *skel.CmdArgs) *cniserver.CNIEndpointRequest { + osEnvMap := make(map[string]string) + for _, item := range os.Environ() { + idx := strings.Index(item, "=") + if idx > 0 { + osEnvMap[strings.TrimSpace(item[:idx])] = item[idx+1:] + } + } + + return &cniserver.CNIEndpointRequest{ + ArgEnv: osEnvMap, + NetConfig: args.StdinData, + } +} + +func (ep *Endpoint) sendCNIServerReq(req *cniserver.CNIEndpointRequest) ([]byte, error) { + cnireqdata, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("sendCNIServerReq: failed to Marshal CNIShim Req %v:%v", req, err) + } + + httpc := http.Client{ + Transport: &http.Transport{ + Dial: func(proto, addr string) (net.Conn, error) { + return net.Dial("unix", ep.cniServerSocketPath) + }, + }, + } + + reponse, err := httpc.Post(CNIEndpointURLReq, "application/json", bytes.NewReader(cnireqdata)) + if err != nil { + return nil, fmt.Errorf("Failed to send CNIServer request: %v", err) + } + defer reponse.Body.Close() + + rbody, err := ioutil.ReadAll(reponse.Body) + if err != nil { + return nil, fmt.Errorf("Failed to read the CNI Server reponse:%v", err) + } + + if reponse.StatusCode != 200 { + return nil, fmt.Errorf("CNI Server request is failed with reponse status %v and reponse body %s", reponse.StatusCode, string(rbody)) + } + + return rbody, nil +} + +func (ep *Endpoint) CmdAdd(args *skel.CmdArgs) error { + logrus.Infof("ovn4nfvk8s-cni: cmdAdd ") + conf, err := config.ConfigureNetConf(args.StdinData) + if err != nil { + return fmt.Errorf("invalid stdin args") + } + logrus.Infof("ovn4nfvk8s-cni: cmdAdd configure net conf details -%+v", conf) + req := cniEndpointRequest(args) + logrus.Infof("ovn4nfvk8s-cni: cmdAdd CNIEndpoint Request:%+v",req) + reponsebody, err := ep.sendCNIServerReq(req) + if err != nil { + return err + } + result, err := current.NewResult(reponsebody) + if err != nil { + return fmt.Errorf("failed to unmarshall CNIServer Result reponse %v - err:%v", string(reponsebody), err) + } + + return types.PrintResult(result, conf.CNIVersion) +} + +func (ep *Endpoint) CmdCheck(args *skel.CmdArgs) error { + return nil +} + +func (ep *Endpoint) CmdDel(args *skel.CmdArgs) error { + logrus.Infof("ovn4nfvk8s-cni: cmdDel ") + req := cniEndpointRequest(args) + _, err := ep.sendCNIServerReq(req) + return err +} diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 002ad80..e9ad3e1 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -8,6 +9,8 @@ import ( "github.com/sirupsen/logrus" "github.com/urfave/cli" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/version" gcfg "gopkg.in/gcfg.v1" "k8s.io/client-go/kubernetes" @@ -283,3 +286,17 @@ func NewClientset(conf *KubernetesConfig) (*kubernetes.Clientset, error) { return kubernetes.NewForConfig(kconfig) } + +func ConfigureNetConf(bytes []byte) (*types.NetConf, error) { + conf := &types.NetConf{} + if err := json.Unmarshal(bytes, conf); err != nil { + return nil, fmt.Errorf("failed to load netconf: %v", err) + } + + if conf.RawPrevResult != nil { + if err := version.ParsePrevResult(conf); err != nil { + return nil, err + } + } + return conf, nil +} diff --git a/internal/pkg/ovn/utils.go b/internal/pkg/ovn/utils.go index 9b388b7..c2e9142 100644 --- a/internal/pkg/ovn/utils.go +++ b/internal/pkg/ovn/utils.go @@ -65,7 +65,7 @@ func SetExec(exec kexec.Interface) error { if err != nil { return err } - runner.hostIP = os.Getenv("HOST_IP") + runner.hostIP = os.Getenv("OVN_NB_TCP_SERVICE_HOST") // OVN Host Port runner.hostPort = "6641" log.Info("Host Port", "IP", runner.hostIP, "Port", runner.hostPort) diff --git a/pkg/controller/pod/pod_controller.go b/pkg/controller/pod/pod_controller.go index 75ed731..d195782 100644 --- a/pkg/controller/pod/pod_controller.go +++ b/pkg/controller/pod/pod_controller.go @@ -170,7 +170,7 @@ func (r *ReconcilePod) addLogicalPorts(pod *corev1.Pod) error { return err } } - + switch { case nfn.Type == "ovn4nfv": ovnCtl, err := ovn.GetOvnController() diff --git a/utilities/docker/debian/Dockerfile b/utilities/docker/debian/Dockerfile index 674ee7e..5c91136 100644 --- a/utilities/docker/debian/Dockerfile +++ b/utilities/docker/debian/Dockerfile @@ -2,29 +2,12 @@ FROM ubuntu:18.04 as base USER root -RUN apt-get update && apt-get install -y iproute2 curl software-properties-common setpriv dpkg-dev netcat +RUN apt-get update && apt-get install -y iproute2 curl software-properties-common setpriv dpkg-dev netcat jq wget RUN mkdir -p /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb RUN bash -xc "\ pushd /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch-dev_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-common_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-dkms_2.12.0-1_all.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-source_2.12.0-1_all.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-dbg_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-ipsec_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-pki_2.12.0-1_all.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-switch_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-testcontroller_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-test_2.12.0-1_all.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-vtep_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-central_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-common_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-controller-vtep_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-docker_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-host_2.12.0-1_amd64.deb; \ -curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/python-openvswitch_2.12.0-1_all.deb; \ +wget -q -nv -O- https://api.github.com/repos/akraino-icn/ovs/releases/tags/v2.12.0 2>/dev/null | jq -r '.assets[] | select(.browser_download_url | contains("\""deb"\"")) | .browser_download_url' | wget -i -; \ dpkg-scanpackages . | gzip -c9 > Packages.gz; \ popd; \ " diff --git a/utilities/kernel/debian/install_kernel_modules.sh b/utilities/kernel/debian/install_kernel_modules.sh new file mode 100755 index 0000000..20228e8 --- /dev/null +++ b/utilities/kernel/debian/install_kernel_modules.sh @@ -0,0 +1,32 @@ +!#/bin/bash + +apt-get install apt-transport-https dpkg-dev -y + +mkdir -p /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb +pushd /opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch-dev_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/libopenvswitch_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-common_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-dkms_2.12.0-1_all.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-datapath-source_2.12.0-1_all.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-dbg_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-ipsec_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-pki_2.12.0-1_all.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-switch_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-testcontroller_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-test_2.12.0-1_all.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/openvswitch-vtep_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-central_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-common_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-controller-vtep_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-docker_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/ovn-host_2.12.0-1_amd64.deb +curl --insecure --compressed -O -L https://github.com/akraino-icn/ovs/releases/download/v2.12.0/python-openvswitch_2.12.0-1_all.deb +dpkg-scanpackages . | gzip -c9 > Packages.gz +popd + +sudo apt-get install apt-transport-https +echo "deb [trusted=yes] file:///opt/ovn4nfv-k8s-plugin/dist/ubuntu/deb ./" | tee -a /etc/apt/sources.list > /dev/null +cp /etc/apt/sources.list /etc/apt/sources.list~ +sed -Ei 's/^# deb-src /deb-src /' /etc/apt/sources.list +apt-get update && apt-get build-dep dkms -y && apt-get install openvswitch-datapath-dkms=2.12.0-1 -y |