Adding cnishim and cniserver

- inspired from ovn-kubernetes and sdn openshift - cniserver & cnishim concepts
- removed cni binary to depend on the host ovs binary installation
- encapsulated all the binaries within the ovn and ovs containers
- ovn4nfv-k8s cni server run along with nfn-agent
- cnishim act as the httpclient and cniserver as httpservers
- cnishim wrap all the cni commands to cniserver
- cniserver do the actual network pumping work and send result back to cnishim
- cnishim print the results as per the cni spec requirement
- currently supports only debian installation for ovn daemon
- support for debian kernel installation
- Consolidated all yaml into single ovn4nfv-k8s-plugin

Signed-off-by: Kuralamudhan Ramakrishnan <kuralamudhan.ramakrishnan@intel.com>
Change-Id: I1e2b114d90f717baa2ee94ff379c849d73b2754e

3 files changed, 45 insertions, 319 deletions

diff --git a/cmd/nfn-agent/nfn-agent.go b/cmd/nfn-agent/nfn-agent.go
index 3c85dbd..8d33778 100644
--- a/cmd/nfn-agent/nfn-agent.go
+++ b/cmd/nfn-agent/nfn-agent.go
@@ -9,8 +9,11 @@ import (
 	"os"
 	"os/signal"
 	pb "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify/proto"
+        cs "ovn4nfv-k8s-plugin/internal/pkg/cniserver"
 	"ovn4nfv-k8s-plugin/internal/pkg/ovn"
 	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+        "k8s.io/client-go/kubernetes"
+        "k8s.io/client-go/rest"
 	"strings"
 	"syscall"
 	"time"
@@ -298,6 +301,26 @@ func main() {
 	client := pb.NewNfnNotifyClient(conn)
 	errorChannel = make(chan string)
 
+        // creates the in-cluster config
+	config, err := rest.InClusterConfig()
+	if err != nil {
+                log.Error(err, "Unable to create in-cluster config")
+		return
+	}
+
+	// creates the clientset
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		log.Error(err, "Unable to create clientset for in-cluster config")
+                return
+	}
+
+        cniserver := cs.NewCNIServer("",clientset)
+        err = cniserver.Start(cs.HandleCNIcommandRequest)
+        if err != nil {
+                log.Error(err, "Unable to start cni server")
+                return
+        }
 	// Run client in background
 	go subscribeNotif(client)
 	shutdownHandler(errorChannel)
diff --git a/cmd/ovn4nfvk8s-cni/app/helper_linux.go b/cmd/ovn4nfvk8s-cni/app/helper_linux.go
index 2dba628..1702597 100644
--- a/cmd/ovn4nfvk8s-cni/app/helper_linux.go
+++ b/cmd/ovn4nfvk8s-cni/app/helper_linux.go
@@ -10,8 +10,6 @@ import (
 	"strings"
 
 	"github.com/sirupsen/logrus"
-
-	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types/current"
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/pkg/ns"
@@ -106,10 +104,10 @@ func setupInterface(netns ns.NetNS, containerID, ifName, macAddress, ipAddress,
 }
 
 // ConfigureInterface sets up the container interface
-var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway string, idx, mtu int) ([]*current.Interface, error) {
-	netns, err := ns.GetNS(args.Netns)
+var ConfigureInterface = func(containerNetns, containerID, ifName, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway string, idx, mtu int) ([]*current.Interface, error) {
+	netns, err := ns.GetNS(containerNetns)
 	if err != nil {
-		return nil, fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
+		return nil, fmt.Errorf("failed to open netns %q: %v", containerNetns, err)
 	}
 	defer netns.Close()
 
@@ -118,10 +116,10 @@ var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress
 		ifaceID = fmt.Sprintf("%s_%s_%s", namespace, podName, interfaceName)
 	} else {
 		ifaceID = fmt.Sprintf("%s_%s", namespace, podName)
-		interfaceName = args.IfName
+		interfaceName = ifName
 		defaultGateway = "true"
 	}
-	hostIface, contIface, err := setupInterface(netns, args.ContainerID, interfaceName, macAddress, ipAddress, gatewayIP, defaultGateway, idx, mtu)
+	hostIface, contIface, err := setupInterface(netns, containerID, interfaceName, macAddress, ipAddress, gatewayIP, defaultGateway, idx, mtu)
 	if err != nil {
 		return nil, err
 	}
@@ -132,7 +130,7 @@ var ConfigureInterface = func(args *skel.CmdArgs, namespace, podName, macAddress
 		fmt.Sprintf("external_ids:attached_mac=%s", macAddress),
 		fmt.Sprintf("external_ids:iface-id=%s", ifaceID),
 		fmt.Sprintf("external_ids:ip_address=%s", ipAddress),
-		fmt.Sprintf("external_ids:sandbox=%s", args.ContainerID),
+		fmt.Sprintf("external_ids:sandbox=%s", containerID),
 	}
 
 	var out []byte
@@ -160,10 +158,10 @@ func setupRoute(netns ns.NetNS, dst, gw, dev string) error {
 }
 
 // ConfigureRoute sets up the container routes
-var ConfigureRoute = func(args *skel.CmdArgs, dst, gw, dev string) error {
-	netns, err := ns.GetNS(args.Netns)
+var ConfigureRoute = func(containerNetns, dst, gw, dev string) error {
+	netns, err := ns.GetNS(containerNetns)
 	if err != nil {
-		return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
+		return fmt.Errorf("failed to open netns %q: %v", containerNetns, err)
 	}
 	defer netns.Close()
 	err = setupRoute(netns, dst, gw, dev)
diff --git a/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go b/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go
index c176700..cb75ecd 100644
--- a/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go
+++ b/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go
@@ -1,335 +1,40 @@
-// +build linux
-
 package main
 
 import (
-	"encoding/json"
-	"fmt"
-	"net"
 	"os"
-	"strconv"
-	"strings"
-	"time"
 
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 
+	cni "ovn4nfv-k8s-plugin/internal/pkg/cnishim"
+	"ovn4nfv-k8s-plugin/internal/pkg/config"
+
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
-	"github.com/containernetworking/cni/pkg/types/current"
 	"github.com/containernetworking/cni/pkg/version"
-	"k8s.io/apimachinery/pkg/util/wait"
-
-	"ovn4nfv-k8s-plugin/internal/pkg/kube"
-
-	"ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app"
-	"ovn4nfv-k8s-plugin/internal/pkg/config"
+	"github.com/containernetworking/plugins/pkg/utils/buildversion"
 )
 
-const (
-	ovn4nfvAnnotationTag = "k8s.plugin.opnfv.org/ovnInterfaces"
-)
-
-func argString2Map(args string) (map[string]string, error) {
-	argsMap := make(map[string]string)
-
-	pairs := strings.Split(args, ";")
-	for _, pair := range pairs {
-		kv := strings.Split(pair, "=")
-		if len(kv) != 2 {
-			return nil, fmt.Errorf("ARGS: invalid pair %q", pair)
-		}
-		keyString := kv[0]
-		valueString := kv[1]
-		argsMap[keyString] = valueString
-	}
-
-	return argsMap, nil
-}
-
-func parseOvnNetworkObject(ovnnetwork string) ([]map[string]string, error) {
-	var ovnNet []map[string]string
-
-	if ovnnetwork == "" {
-		return nil, fmt.Errorf("parseOvnNetworkObject:error")
-	}
-
-	if err := json.Unmarshal([]byte(ovnnetwork), &ovnNet); err != nil {
-		return nil, fmt.Errorf("parseOvnNetworkObject: failed to load ovn network err: %v | ovn network: %v", err, ovnnetwork)
-	}
-
-	return ovnNet, nil
-}
-
-func mergeWithResult(srcObj, dstObj types.Result) (types.Result, error) {
-
-	if dstObj == nil {
-		return srcObj, nil
-	}
-	src, err := current.NewResultFromResult(srcObj)
-	if err != nil {
-		return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err)
-	}
-	dst, err := current.NewResultFromResult(dstObj)
-	if err != nil {
-		return nil, fmt.Errorf("Couldn't convert old result to current version: %v", err)
-	}
-
-	ifacesLength := len(dst.Interfaces)
-
-	for _, iface := range src.Interfaces {
-		dst.Interfaces = append(dst.Interfaces, iface)
-	}
-	for _, ip := range src.IPs {
-		if ip.Interface != nil && *(ip.Interface) != -1 {
-			ip.Interface = current.Int(*(ip.Interface) + ifacesLength)
-		}
-		dst.IPs = append(dst.IPs, ip)
-	}
-	for _, route := range src.Routes {
-		dst.Routes = append(dst.Routes, route)
-	}
-
-	for _, ns := range src.DNS.Nameservers {
-		dst.DNS.Nameservers = append(dst.DNS.Nameservers, ns)
-	}
-	for _, s := range src.DNS.Search {
-		dst.DNS.Search = append(dst.DNS.Search, s)
-	}
-	for _, opt := range src.DNS.Options {
-		dst.DNS.Options = append(dst.DNS.Options, opt)
-	}
-	// TODO: what about DNS.domain?
-	return dst, nil
-}
-
-func prettyPrint(i interface{}) string {
-	s, _ := json.MarshalIndent(i, "", "\t")
-	return string(s)
-}
-
-func addMultipleInterfaces(args *skel.CmdArgs, ovnAnnotation, namespace, podName string) types.Result {
-	logrus.Infof("ovn4nfvk8s-cni: addMultipleInterfaces ")
-
-	var ovnAnnotatedMap []map[string]string
-	ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation)
-	if err != nil {
-		logrus.Errorf("addLogicalPort : Error Parsing Ovn Network List %v %v", ovnAnnotatedMap, err)
-		return nil
-	}
-	if namespace == "" || podName == "" {
-		logrus.Errorf("required CNI variable missing")
-		return nil
-	}
-	var interfacesArray []*current.Interface
-	var index int
-	var result *current.Result
-	var dstResult types.Result
-	for _, ovnNet := range ovnAnnotatedMap {
-		ipAddress := ovnNet["ip_address"]
-		macAddress := ovnNet["mac_address"]
-		gatewayIP := ovnNet["gateway_ip"]
-		defaultGateway := ovnNet["defaultGateway"]
-
-		if ipAddress == "" || macAddress == "" {
-			logrus.Errorf("failed in pod annotation key extract")
-			return nil
-		}
-
-		index++
-		interfaceName := ovnNet["interface"]
-		if interfaceName == "" {
-			logrus.Errorf("addMultipleInterfaces: interface can't be null")
-			return nil
-		}
-		interfacesArray, err = app.ConfigureInterface(args, namespace, podName, macAddress, ipAddress, gatewayIP, interfaceName, defaultGateway, index, config.Default.MTU)
-		if err != nil {
-			logrus.Errorf("Failed to configure interface in pod: %v", err)
-			return nil
-		}
-		addr, addrNet, err := net.ParseCIDR(ipAddress)
-		if err != nil {
-			logrus.Errorf("failed to parse IP address %q: %v", ipAddress, err)
-			return nil
-		}
-		ipVersion := "6"
-		if addr.To4() != nil {
-			ipVersion = "4"
-		}
-		var routes types.Route
-		if defaultGateway == "true" {
-			defaultAddr, defaultAddrNet, _ := net.ParseCIDR("0.0.0.0/0")
-			routes = types.Route{Dst: net.IPNet{IP: defaultAddr, Mask: defaultAddrNet.Mask}, GW: net.ParseIP(gatewayIP)}
-
-			result = &current.Result{
-				Interfaces: interfacesArray,
-				IPs: []*current.IPConfig{
-					{
-						Version:   ipVersion,
-						Interface: current.Int(1),
-						Address:   net.IPNet{IP: addr, Mask: addrNet.Mask},
-						Gateway:   net.ParseIP(gatewayIP),
-					},
-				},
-				Routes: []*types.Route{&routes},
-			}
-		} else {
-			result = &current.Result{
-				Interfaces: interfacesArray,
-				IPs: []*current.IPConfig{
-					{
-						Version:   ipVersion,
-						Interface: current.Int(1),
-						Address:   net.IPNet{IP: addr, Mask: addrNet.Mask},
-						Gateway:   net.ParseIP(gatewayIP),
-					},
-				},
-			}
-
-		}
-		// Build the result structure to pass back to the runtime
-		dstResult, err = mergeWithResult(types.Result(result), dstResult)
-		if err != nil {
-			logrus.Errorf("Failed to merge results: %v", err)
-			return nil
-		}
-	}
-	logrus.Infof("addMultipleInterfaces:  %s", prettyPrint(dstResult))
-	return dstResult
-}
-
-func addRoutes(args *skel.CmdArgs, ovnAnnotation string, dstResult types.Result) types.Result {
-	logrus.Infof("ovn4nfvk8s-cni: addRoutes ")
-
-	var ovnAnnotatedMap []map[string]string
-	ovnAnnotatedMap, err := parseOvnNetworkObject(ovnAnnotation)
-	if err != nil {
-		logrus.Errorf("addLogicalPort : Error Parsing Ovn Route List %v", err)
-		return nil
-	}
-
-	var result types.Result
-	var routes []*types.Route
-	for _, ovnNet := range ovnAnnotatedMap {
-		dst := ovnNet["dst"]
-		gw := ovnNet["gw"]
-		dev := ovnNet["dev"]
-		if dst == "" || gw == "" || dev == "" {
-			logrus.Errorf("failed in pod annotation key extract")
-			return nil
-		}
-		err = app.ConfigureRoute(args, dst, gw, dev)
-		if err != nil {
-			logrus.Errorf("Failed to configure interface in pod: %v", err)
-			return nil
-		}
-		dstAddr, dstAddrNet, _ := net.ParseCIDR(dst)
-		routes = append(routes, &types.Route{
-			Dst: net.IPNet{IP: dstAddr, Mask: dstAddrNet.Mask},
-			GW:  net.ParseIP(gw),
-		})
-	}
-
-	result = &current.Result{
-		Routes: routes,
-	}
-	// Build the result structure to pass back to the runtime
-	dstResult, err = mergeWithResult(result, dstResult)
-	if err != nil {
-		logrus.Errorf("Failed to merge results: %v", err)
-		return nil
-	}
-	logrus.Infof("addRoutes:  %s", prettyPrint(dstResult))
-	return dstResult
-
-}
-
-func cmdAdd(args *skel.CmdArgs) error {
-	logrus.Infof("ovn4nfvk8s-cni: cmdAdd ")
-	conf := &types.NetConf{}
-	if err := json.Unmarshal(args.StdinData, conf); err != nil {
-		return fmt.Errorf("failed to load netconf: %v", err)
-	}
-
-	argsMap, err := argString2Map(args.Args)
-	if err != nil {
-		return err
-	}
-
-	namespace := argsMap["K8S_POD_NAMESPACE"]
-	podName := argsMap["K8S_POD_NAME"]
-	if namespace == "" || podName == "" {
-		return fmt.Errorf("required CNI variable missing")
-	}
-
-	clientset, err := config.NewClientset(&config.Kubernetes)
-	if err != nil {
-		return fmt.Errorf("Could not create clientset for kubernetes: %v", err)
-	}
-	kubecli := &kube.Kube{KClient: clientset}
-
-	// Get the IP address and MAC address from the API server.
-	var annotationBackoff = wait.Backoff{Duration: 1 * time.Second, Steps: 14, Factor: 1.5, Jitter: 0.1}
-	var annotation map[string]string
-	if err := wait.ExponentialBackoff(annotationBackoff, func() (bool, error) {
-		annotation, err = kubecli.GetAnnotationsOnPod(namespace, podName)
-		if err != nil {
-			// TODO: check if err is non recoverable
-			logrus.Warningf("Error while obtaining pod annotations - %v", err)
-			return false, nil
-		}
-		if _, ok := annotation[ovn4nfvAnnotationTag]; ok {
-			return true, nil
-		}
-		return false, nil
-	}); err != nil {
-		return fmt.Errorf("failed to get pod annotation - %v", err)
-	}
-	logrus.Infof("ovn4nfvk8s-cni: Annotation Found ")
-	ovnAnnotation, ok := annotation[ovn4nfvAnnotationTag]
-	if !ok {
-		return fmt.Errorf("Error while obtaining pod annotations")
-	}
-	result := addMultipleInterfaces(args, ovnAnnotation, namespace, podName)
-	// Add Routes to the pod if annotation found for routes
-	ovnRouteAnnotation, ok := annotation["ovnNetworkRoutes"]
-	if ok {
-		logrus.Infof("ovn4nfvk8s-cni: ovnNetworkRoutes Annotation Found %+v", ovnRouteAnnotation)
-		result = addRoutes(args, ovnRouteAnnotation, result)
-	}
-
-	return result.Print()
-}
-
-func cmdDel(args *skel.CmdArgs) error {
-	logrus.Infof("ovn4nfvk8s-cni: cmdDel ")
-	for i := 0; i < 10; i++ {
-		ifaceName := args.ContainerID[:14] + strconv.Itoa(i)
-		done, err := app.PlatformSpecificCleanup(ifaceName)
-		if err != nil {
-			logrus.Errorf("Teardown error: %v", err)
-		}
-		if done {
-			break
-		}
-	}
-	return nil
-}
-
 func main() {
-	logrus.Infof("ovn4nfvk8s-cni invoked")
+	logrus.Infof("ovn4nfvk8s-cni shim cni")
 	c := cli.NewApp()
 	c.Name = "ovn4nfvk8s-cni"
 	c.Usage = "a CNI plugin to set up or tear down a additional interfaces with OVN"
-	c.Version = "0.0.2"
+	c.Version = "0.1.0"
 	c.Flags = config.Flags
 
+	ep := cni.CNIEndpoint("")
 	c.Action = func(ctx *cli.Context) error {
 		if _, err := config.InitConfig(ctx); err != nil {
 			return err
 		}
+		skel.PluginMain(
+			ep.CmdAdd,
+			ep.CmdCheck,
+			ep.CmdDel,
+			version.All,
+			buildversion.BuildString("ovn4nfv-k8s shim cni"))
 
-		skel.PluginMain(cmdAdd, nil, cmdDel, version.All, "")
 		return nil
 	}