Adding node interface, SNAT and OVN Node switch port

    - Adding nfn proto for node interface MAC and IP address
    - Adding node switch ports in NFN Operator
    - Adding grpc client in nfn agent to get mac and IP address for node interface
    - Adding feature to create Node interface with OVS internal port
    - Make sure pod controller requeue the pod request for empty node field in pod spec
    - Unique node interface name using SHA and maintain 15 charactor for OVS switch port
    - Adding SNAT for default interface in each node
    - Adding iptables modules for SNAT rules

Signed-off-by: Kuralamudhan Ramakrishnan <kuralamudhan.ramakrishnan@intel.com>
Change-Id: I6cfa36e45007e796eb651345f9f0751329defcf7

2 files changed, 101 insertions, 16 deletions

diff --git a/cmd/nfn-agent/nfn-agent.go b/cmd/nfn-agent/nfn-agent.go
index 8d33778..b2ee630 100644
--- a/cmd/nfn-agent/nfn-agent.go
+++ b/cmd/nfn-agent/nfn-agent.go
@@ -3,21 +3,26 @@ package main
 import (
 	"context"
 	"fmt"
-	"google.golang.org/grpc"
 	"io"
-	kexec "k8s.io/utils/exec"
 	"os"
 	"os/signal"
+	cs "ovn4nfv-k8s-plugin/internal/pkg/cniserver"
 	pb "ovn4nfv-k8s-plugin/internal/pkg/nfnNotify/proto"
-        cs "ovn4nfv-k8s-plugin/internal/pkg/cniserver"
 	"ovn4nfv-k8s-plugin/internal/pkg/ovn"
-	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
-        "k8s.io/client-go/kubernetes"
-        "k8s.io/client-go/rest"
 	"strings"
 	"syscall"
 	"time"
+
+	"google.golang.org/grpc"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	kexec "k8s.io/utils/exec"
+	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+
 	//"google.golang.org/grpc/keepalive"
+
+	"ovn4nfv-k8s-plugin/cmd/ovn4nfvk8s-cni/app"
+
 	"google.golang.org/grpc/status"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 )
@@ -201,6 +206,19 @@ DIRECTPRNETWORK:
 	}
 }
 
+func createNodeOVSInternalPort(payload *pb.Notification_InSync) error {
+	nodeIntfIPAddr := strings.Trim(strings.TrimSpace(payload.InSync.GetNodeIntfIpAddress()), "\"")
+	nodeIntfMacAddr := strings.Trim(strings.TrimSpace(payload.InSync.GetNodeIntfMacAddress()), "\"")
+	nodeName := os.Getenv("NFN_NODE_NAME")
+
+	err := app.CreateNodeOVSInternalPort(nodeIntfIPAddr, nodeIntfMacAddr, nodeName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func handleNotif(msg *pb.Notification) {
 	switch msg.GetCniType() {
 	case "ovn4nfv":
@@ -239,6 +257,12 @@ func handleNotif(msg *pb.Notification) {
 			}
 
 		case *pb.Notification_InSync:
+			if payload.InSync.GetNodeIntfIpAddress() != "" && payload.InSync.GetNodeIntfMacAddress() != "" {
+				err := createNodeOVSInternalPort(payload)
+				if err != nil {
+					return
+				}
+			}
 			inSyncVlanProvidernetwork()
 			inSyncDirectProvidernetwork()
 			pnCreateStore = nil
@@ -301,10 +325,10 @@ func main() {
 	client := pb.NewNfnNotifyClient(conn)
 	errorChannel = make(chan string)
 
-        // creates the in-cluster config
+	// creates the in-cluster config
 	config, err := rest.InClusterConfig()
 	if err != nil {
-                log.Error(err, "Unable to create in-cluster config")
+		log.Error(err, "Unable to create in-cluster config")
 		return
 	}
 
@@ -312,15 +336,15 @@ func main() {
 	clientset, err := kubernetes.NewForConfig(config)
 	if err != nil {
 		log.Error(err, "Unable to create clientset for in-cluster config")
-                return
+		return
 	}
 
-        cniserver := cs.NewCNIServer("",clientset)
-        err = cniserver.Start(cs.HandleCNIcommandRequest)
-        if err != nil {
-                log.Error(err, "Unable to start cni server")
-                return
-        }
+	cniserver := cs.NewCNIServer("", clientset)
+	err = cniserver.Start(cs.HandleCNIcommandRequest)
+	if err != nil {
+		log.Error(err, "Unable to start cni server")
+		return
+	}
 	// Run client in background
 	go subscribeNotif(client)
 	shutdownHandler(errorChannel)
diff --git a/cmd/ovn4nfvk8s-cni/app/helper_linux.go b/cmd/ovn4nfvk8s-cni/app/helper_linux.go
index 1702597..cfcd4e9 100644
--- a/cmd/ovn4nfvk8s-cni/app/helper_linux.go
+++ b/cmd/ovn4nfvk8s-cni/app/helper_linux.go
@@ -6,13 +6,15 @@ import (
 	"fmt"
 	"net"
 	"os/exec"
+	"ovn4nfv-k8s-plugin/internal/pkg/config"
+	"ovn4nfv-k8s-plugin/internal/pkg/network"
 	"strconv"
 	"strings"
 
-	"github.com/sirupsen/logrus"
 	"github.com/containernetworking/cni/pkg/types/current"
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 )
 
@@ -35,6 +37,65 @@ func renameLink(curName, newName string) error {
 	return nil
 }
 
+//Todo Comments
+func CreateNodeOVSInternalPort(nodeintfipaddr, nodeintfmacaddr, node string) error {
+	nodeName := strings.ToLower(node)
+	nodeOVSInternalIntfName := config.GetNodeIntfName(nodeName)
+
+	hwAddr, err := net.ParseMAC(nodeintfmacaddr)
+	if err != nil {
+		logrus.Errorf("Error is converting %q to net hwaddr: %v", nodeOVSInternalIntfName, err)
+		return fmt.Errorf("Error is converting %q to net hwaddr: %v", nodeOVSInternalIntfName, err)
+	}
+
+	ovsArgs := []string{
+		"add-port", "br-int", nodeOVSInternalIntfName, "--", "set",
+		"interface", nodeOVSInternalIntfName, "type=internal",
+		fmt.Sprintf("mac_in_use=%s", strings.ReplaceAll(hwAddr.String(), ":", "\\:")),
+		fmt.Sprintf("mac=%s", strings.ReplaceAll(hwAddr.String(), ":", "\\:")),
+		fmt.Sprintf("external_ids:iface-id=%s", nodeOVSInternalIntfName),
+	}
+	logrus.Infof("ovs-vsctl args - %v", ovsArgs)
+
+	//var out []byte
+	out, err := exec.Command("ovs-vsctl", ovsArgs...).CombinedOutput()
+	if err != nil {
+		logrus.Errorf("failure in creating Node OVS internal port - %s: %v - %q", nodeOVSInternalIntfName, err, string(out))
+		return fmt.Errorf("failure in creating Node OVS internal port - %s: %v - %q", nodeOVSInternalIntfName, err, string(out))
+	}
+	logrus.Infof("ovs-vsctl args - %v output:%v", ovsArgs, string(out))
+
+	link, err := netlink.LinkByName(nodeOVSInternalIntfName)
+	if err != nil {
+		logrus.Errorf("failed to get netlink for Node OVS internal port %s: %v", nodeOVSInternalIntfName, err)
+		return fmt.Errorf("failed to get netlink for Node OVS internal port %s: %v", nodeOVSInternalIntfName, err)
+	}
+
+	if err := netlink.LinkSetUp(link); err != nil {
+		logrus.Errorf("failed to set up netlink for Node OVS internal port %s: %v", nodeOVSInternalIntfName, err)
+		return fmt.Errorf("failed to set up netlink for Node OVS internal port %s: %v", nodeOVSInternalIntfName, err)
+	}
+
+	addr, err := netlink.ParseAddr(nodeintfipaddr)
+	if err != nil {
+		logrus.Errorf("failed to parse IP addr %s: %v", nodeintfipaddr, err)
+		return fmt.Errorf("failed to parse IP addr %s: %v", nodeintfipaddr, err)
+	}
+	err = netlink.AddrAdd(link, addr)
+	if err != nil {
+		logrus.Errorf("failed to parse IP addr %s: %v", nodeintfipaddr, err)
+		return fmt.Errorf("failed to add IP addr %s to %s: %v", nodeintfipaddr, nodeOVSInternalIntfName, err)
+	}
+
+	err = network.SetupAndEnsureIPTables(network.MasqRules(nodeOVSInternalIntfName))
+	if err != nil {
+		logrus.Errorf("failed to apply snat rule for %s: %v", nodeOVSInternalIntfName, err)
+		return fmt.Errorf("failed to apply snat rule for %s: %v", nodeOVSInternalIntfName, err)
+	}
+
+	return nil
+}
+
 func setupInterface(netns ns.NetNS, containerID, ifName, macAddress, ipAddress, gatewayIP, defaultGateway string, idx, mtu int) (*current.Interface, *current.Interface, error) {
 	hostIface := &current.Interface{}
 	contIface := &current.Interface{}