From 3de63ee756f9d7c0a4524b40a89e92b918a9249f Mon Sep 17 00:00:00 2001 From: Kuralamudhan Ramakrishnan Date: Tue, 21 Apr 2020 17:19:34 +0000 Subject: Adding cnishim and cniserver - inspired from ovn-kubernetes and sdn openshift - cniserver & cnishim concepts - removed cni binary to depend on the host ovs binary installation - encapsulated all the binaries within the ovn and ovs containers - ovn4nfv-k8s cni server run along with nfn-agent - cnishim act as the httpclient and cniserver as httpservers - cnishim wrap all the cni commands to cniserver - cniserver do the actual network pumping work and send result back to cnishim - cnishim print the results as per the cni spec requirement - currently supports only debian installation for ovn daemon - support for debian kernel installation - Consolidated all yaml into single ovn4nfv-k8s-plugin Signed-off-by: Kuralamudhan Ramakrishnan Change-Id: I1e2b114d90f717baa2ee94ff379c849d73b2754e --- cmd/nfn-agent/nfn-agent.go | 23 +++ cmd/ovn4nfvk8s-cni/app/helper_linux.go | 20 +- cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go | 321 ++------------------------------- 3 files changed, 45 insertions(+), 319 deletions(-) (limited to 'cmd') diff --git a/cmd/nfn-agent/nfn-agent.go b/cmd/nfn-agent/nfn-agent.go index 3c85dbd..8d33778 100644 --- a/cmd/nfn-agent/nfn-agent.go +++ b/cmd/nfn-agent/nfn-agent.go @@ -9,8 +9,11 @@ import ( "os" "os/signal" pb "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify/proto" + cs "ovn4nfv-k8s-plugin/internal/pkg/cniserver" "ovn4nfv-k8s-plugin/internal/pkg/ovn" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "strings" "syscall" "time" @@ -298,6 +301,26 @@ func main() { client := pb.NewNfnNotifyClient(conn) errorChannel = make(chan string) + // creates the in-cluster config + config, err := rest.InClusterConfig() + if err != nil { + log.Error(err, "Unable to create in-cluster config") + return + } + + // creates the clientset + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + log.Error(err, "Unable to create clientset for in-cluster config") + return + } + + cniserver := cs.NewCNIServer("",clientset) + err = cniserver.Start(cs.HandleCNIcommandRequest) + if err != nil { + log.Error(err, "Unable to start cni server") + return + } // Run client in background go subscribeNotif(client) shutdownHandler(errorChannel) diff --git a/cmd/ovn4nfvk8s-cni/app/helper_linux.go b/cmd/ovn4nfvk8s-cni/app/helper_linux.go index 2dba628..1702597 100644 --- a/cmd/ovn4nfvk8s-cni/app/helper_linux.go +++ b/cmd/ovn4nfvk8s-cni/app/helper_linux.go @@ -10,8 +10,6 @@ import ( "strings" "github.com/sirupsen/logrus" - - "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/plugins/pkg/ip" "github.com/containernetworking/plugins/pkg/ns" @@ -106,10 +104,10 @@ func setupInterface(netns ns.NetNS, containerID, ifName, macAddress, ipAddress, } // ConfigureInterface sets up the container interface -var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway string, idx, mtu int) ([]*current.Interface, error) { - netns, err := ns.GetNS(args.Netns) +var ConfigureInterface = func(containerNetns, containerID, ifName, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway string, idx, mtu int) ([]*current.Interface, error) { + netns, err := ns.GetNS(containerNetns) if err != nil { - return nil, fmt.Errorf("failed to open netns %q: %v", args.Netns, err) + return nil, fmt.Errorf("failed to open netns %q: %v", containerNetns, err) } defer netns.Close() @@ -118,10 +116,10 @@ var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress ifaceID = fmt.Sprintf("%s_%s_%s", namespace, podName, interfaceName) } else { ifaceID = fmt.Sprintf("%s_%s", namespace, podName) - interfaceName = args.IfName + interfaceName = ifName defaultGateway = "true" } - hostIface, contIface, err := setupInterface(netns, args.ContainerID, interfaceName, macAddress, ipAddress, gatewayIP, defaultGateway, idx, mtu) + hostIface, contIface, err := setupInterface(netns, containerID, interfaceName, macAddress, ipAddress, gatewayIP, defaultGateway, idx, mtu) if err != nil { return nil, err } @@ -132,7 +130,7 @@ var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress fmt.Sprintf("external_ids:attached_mac=%s", macAddress), fmt.Sprintf("external_ids:iface-id=%s", ifaceID), fmt.Sprintf("external_ids:ip_address=%s", ipAddress), - fmt.Sprintf("external_ids:sandbox=%s", args.ContainerID), + fmt.Sprintf("external_ids:sandbox=%s", containerID), } var out []byte @@ -160,10 +158,10 @@ func setupRoute(netns ns.NetNS, dst, gw, dev string) error { } // ConfigureRoute sets up the container routes -var ConfigureRoute = func(args *skel.CmdArgs, dst, gw, dev string) error { - netns, err := ns.GetNS(args.Netns) +var ConfigureRoute = func(containerNetns, dst, gw, dev string) error { + netns, err := ns.GetNS(containerNetns) if err != nil { - return fmt.Errorf("failed to open netns %q: %v", args.Netns, err) + return fmt.Errorf("failed to open netns %q: %v", containerNetns, err) } defer netns.Close() err = setupRoute(netns, dst, gw, dev) diff --git a/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go b/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go index c176700..cb75ecd 100644 --- a/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go +++ b/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go @@ -1,335 +1,40 @@ -// +build linux - package main import ( - "encoding/json" - "fmt" - "net" "os" - "strconv" - "strings" - "time" "github.com/sirupsen/logrus" "github.com/urfave/cli" + cni "ovn4nfv-k8s-plugin/internal/pkg/cnishim" + "ovn4nfv-k8s-plugin/internal/pkg/config" + "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" - "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/cni/pkg/version" - "k8s.io/apimachinery/pkg/util/wait" - - "ovn4nfv-k8s-plugin/internal/pkg/kube" - - "ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app" - "ovn4nfv-k8s-plugin/internal/pkg/config" + "github.com/containernetworking/plugins/pkg/utils/buildversion" ) -const ( - ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces" -) - -func argString2Map(args string) (map[string]string, error) { - argsMap := make(map[string]string) - - pairs := strings.Split(args, ";") - for _, pair := range pairs { - kv := strings.Split(pair, "=") - if len(kv) != 2 { - return nil, fmt.Errorf("ARGS: invalid pair %q", pair) - } - keyString := kv[0] - valueString := kv[1] - argsMap[keyString] = valueString - } - - return argsMap, nil -} - -func parseOvnNetworkObject(ovnnetwork string) ([]map[string]string, error) { - var ovnNet []map[string]string - - if ovnnetwork == "" { - return nil, fmt.Errorf("parseOvnNetworkObject:error") - } - - if err := json.Unmarshal([]byte(ovnnetwork), &ovnNet); err != nil { - return nil, fmt.Errorf("parseOvnNetworkObject: failed to load ovn network err: %v | ovn network: %v", err, ovnnetwork) - } - - return ovnNet, nil -} - -func mergeWithResult(srcObj, dstObj types.Result) (types.Result, error) { - - if dstObj == nil { - return srcObj, nil - } - src, err := current.NewResultFromResult(srcObj) - if err != nil { - return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err) - } - dst, err := current.NewResultFromResult(dstObj) - if err != nil { - return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err) - } - - ifacesLength := len(dst.Interfaces) - - for _, iface := range src.Interfaces { - dst.Interfaces = append(dst.Interfaces, iface) - } - for _, ip := range src.IPs { - if ip.Interface != nil && *(ip.Interface) != -1 { - ip.Interface = current.Int(*(ip.Interface) + ifacesLength) - } - dst.IPs = append(dst.IPs, ip) - } - for _, route := range src.Routes { - dst.Routes = append(dst.Routes, route) - } - - for _, ns := range src.DNS.Nameservers { - dst.DNS.Nameservers = append(dst.DNS.Nameservers, ns) - } - for _, s := range src.DNS.Search { - dst.DNS.Search = append(dst.DNS.Search, s) - } - for _, opt := range src.DNS.Options { - dst.DNS.Options = append(dst.DNS.Options, opt) - } - // TODO: what about DNS.domain? - return dst, nil -} - -func prettyPrint(i interface{}) string { - s, _ := json.MarshalIndent(i, "", "\t") - return string(s) -} - -func addMultipleInterfaces(args *skel.CmdArgs, ovnAnnotation, namespace, podName string) types.Result { - logrus.Infof("ovn4nfvk8s-cni: addMultipleInterfaces ") - - var ovnAnnotatedMap []map[string]string - ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation) - if err != nil { - logrus.Errorf("addLogicalPort : Error Parsing Ovn Network List %v %v", ovnAnnotatedMap, err) - return nil - } - if namespace == "" || podName == "" { - logrus.Errorf("required CNI variable missing") - return nil - } - var interfacesArray []*current.Interface - var index int - var result *current.Result - var dstResult types.Result - for _, ovnNet := range ovnAnnotatedMap { - ipAddress := ovnNet["ip_address"] - macAddress := ovnNet["mac_address"] - gatewayIP := ovnNet["gateway_ip"] - defaultGateway := ovnNet["defaultGateway"] - - if ipAddress == "" || macAddress == "" { - logrus.Errorf("failed in pod annotation key extract") - return nil - } - - index++ - interfaceName := ovnNet["interface"] - if interfaceName == "" { - logrus.Errorf("addMultipleInterfaces: interface can't be null") - return nil - } - interfacesArray, err = app.ConfigureInterface(args, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway, index, config.Default.MTU) - if err != nil { - logrus.Errorf("Failed to configure interface in pod: %v", err) - return nil - } - addr, addrNet, err := net.ParseCIDR(ipAddress) - if err != nil { - logrus.Errorf("failed to parse IP address %q: %v", ipAddress, err) - return nil - } - ipVersion := "6" - if addr.To4() != nil { - ipVersion = "4" - } - var routes types.Route - if defaultGateway == "true" { - defaultAddr, defaultAddrNet, _ := net.ParseCIDR("0.0.0.0/0") - routes = types.Route{Dst: net.IPNet{IP: defaultAddr, Mask: defaultAddrNet.Mask}, GW: net.ParseIP(gatewayIP)} - - result = ¤t.Result{ - Interfaces: interfacesArray, - IPs: []*current.IPConfig{ - { - Version: ipVersion, - Interface: current.Int(1), - Address: net.IPNet{IP: addr, Mask: addrNet.Mask}, - Gateway: net.ParseIP(gatewayIP), - }, - }, - Routes: []*types.Route{&routes}, - } - } else { - result = ¤t.Result{ - Interfaces: interfacesArray, - IPs: []*current.IPConfig{ - { - Version: ipVersion, - Interface: current.Int(1), - Address: net.IPNet{IP: addr, Mask: addrNet.Mask}, - Gateway: net.ParseIP(gatewayIP), - }, - }, - } - - } - // Build the result structure to pass back to the runtime - dstResult, err = mergeWithResult(types.Result(result), dstResult) - if err != nil { - logrus.Errorf("Failed to merge results: %v", err) - return nil - } - } - logrus.Infof("addMultipleInterfaces: %s", prettyPrint(dstResult)) - return dstResult -} - -func addRoutes(args *skel.CmdArgs, ovnAnnotation string, dstResult types.Result) types.Result { - logrus.Infof("ovn4nfvk8s-cni: addRoutes ") - - var ovnAnnotatedMap []map[string]string - ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation) - if err != nil { - logrus.Errorf("addLogicalPort : Error Parsing Ovn Route List %v", err) - return nil - } - - var result types.Result - var routes []*types.Route - for _, ovnNet := range ovnAnnotatedMap { - dst := ovnNet["dst"] - gw := ovnNet["gw"] - dev := ovnNet["dev"] - if dst == "" || gw == "" || dev == "" { - logrus.Errorf("failed in pod annotation key extract") - return nil - } - err = app.ConfigureRoute(args, dst, gw, dev) - if err != nil { - logrus.Errorf("Failed to configure interface in pod: %v", err) - return nil - } - dstAddr, dstAddrNet, _ := net.ParseCIDR(dst) - routes = append(routes, &types.Route{ - Dst: net.IPNet{IP: dstAddr, Mask: dstAddrNet.Mask}, - GW: net.ParseIP(gw), - }) - } - - result = ¤t.Result{ - Routes: routes, - } - // Build the result structure to pass back to the runtime - dstResult, err = mergeWithResult(result, dstResult) - if err != nil { - logrus.Errorf("Failed to merge results: %v", err) - return nil - } - logrus.Infof("addRoutes: %s", prettyPrint(dstResult)) - return dstResult - -} - -func cmdAdd(args *skel.CmdArgs) error { - logrus.Infof("ovn4nfvk8s-cni: cmdAdd ") - conf := &types.NetConf{} - if err := json.Unmarshal(args.StdinData, conf); err != nil { - return fmt.Errorf("failed to load netconf: %v", err) - } - - argsMap, err := argString2Map(args.Args) - if err != nil { - return err - } - - namespace := argsMap["K8S_POD_NAMESPACE"] - podName := argsMap["K8S_POD_NAME"] - if namespace == "" || podName == "" { - return fmt.Errorf("required CNI variable missing") - } - - clientset, err := config.NewClientset(&config.Kubernetes) - if err != nil { - return fmt.Errorf("Could not create clientset for kubernetes: %v", err) - } - kubecli := &kube.Kube{KClient: clientset} - - // Get the IP address and MAC address from the API server. - var annotationBackoff = wait.Backoff{Duration: 1 * time.Second, Steps: 14, Factor: 1.5, Jitter: 0.1} - var annotation map[string]string - if err := wait.ExponentialBackoff(annotationBackoff, func() (bool, error) { - annotation, err = kubecli.GetAnnotationsOnPod(namespace, podName) - if err != nil { - // TODO: check if err is non recoverable - logrus.Warningf("Error while obtaining pod annotations - %v", err) - return false, nil - } - if _, ok := annotation[ovn4nfvAnnotationTag]; ok { - return true, nil - } - return false, nil - }); err != nil { - return fmt.Errorf("failed to get pod annotation - %v", err) - } - logrus.Infof("ovn4nfvk8s-cni: Annotation Found ") - ovnAnnotation, ok := annotation[ovn4nfvAnnotationTag] - if !ok { - return fmt.Errorf("Error while obtaining pod annotations") - } - result := addMultipleInterfaces(args, ovnAnnotation, namespace, podName) - // Add Routes to the pod if annotation found for routes - ovnRouteAnnotation, ok := annotation["ovnNetworkRoutes"] - if ok { - logrus.Infof("ovn4nfvk8s-cni: ovnNetworkRoutes Annotation Found %+v", ovnRouteAnnotation) - result = addRoutes(args, ovnRouteAnnotation, result) - } - - return result.Print() -} - -func cmdDel(args *skel.CmdArgs) error { - logrus.Infof("ovn4nfvk8s-cni: cmdDel ") - for i := 0; i < 10; i++ { - ifaceName := args.ContainerID[:14] + strconv.Itoa(i) - done, err := app.PlatformSpecificCleanup(ifaceName) - if err != nil { - logrus.Errorf("Teardown error: %v", err) - } - if done { - break - } - } - return nil -} - func main() { - logrus.Infof("ovn4nfvk8s-cni invoked") + logrus.Infof("ovn4nfvk8s-cni shim cni") c := cli.NewApp() c.Name = "ovn4nfvk8s-cni" c.Usage = "a CNI plugin to set up or tear down a additional interfaces with OVN" - c.Version = "0.0.2" + c.Version = "0.1.0" c.Flags = config.Flags + ep := cni.CNIEndpoint("") c.Action = func(ctx *cli.Context) error { if _, err := config.InitConfig(ctx); err != nil { return err } + skel.PluginMain( + ep.CmdAdd, + ep.CmdCheck, + ep.CmdDel, + version.All, + buildversion.BuildString("ovn4nfv-k8s shim cni")) - skel.PluginMain(cmdAdd, nil, cmdDel, version.All, "") return nil } -- cgit 1.2.3-korg