Update OVN utils to add provider network support

Currently provider networks support only VLAN
based networks. Added functions for VLAN
and ovs bridge.

Change-Id: I63ac7266aac92021ee3a44f49644a8a79ce4fed6
Signed-off-by: Ritu Sood <ritu.sood@intel.com>

4 files changed, 284 insertions, 41 deletions

diff --git a/internal/pkg/ovn/common.go b/internal/pkg/ovn/common.go
index 09d770b..d3e477a 100644
--- a/internal/pkg/ovn/common.go
+++ b/internal/pkg/ovn/common.go
@@ -3,15 +3,192 @@ package ovn
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/vishvananda/netlink"
 	"math/big"
 	"math/rand"
 	"net"
 	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
+	"strings"
 	"time"
 )
 
 var log = logf.Log.WithName("ovn")
 
+// CreateVlan creates VLAN with vlanID
+func CreateVlan(vlanID, interfaceName, logicalInterfaceName string) error {
+	if interfaceName == "" || vlanID == "" || logicalInterfaceName == "" {
+		return fmt.Errorf("CreateVlan invalid parameters: %v %v %v", interfaceName, vlanID, logicalInterfaceName)
+	}
+	_, err := netlink.LinkByName(logicalInterfaceName)
+	if err == nil {
+		return err
+	}
+	stdout, stderr, err := RunIP("link", "add", "link", interfaceName, "name", logicalInterfaceName, "type", "vlan", "id", vlanID)
+	if err != nil {
+		log.Error(err, "Failed to create Vlan", "stdout", stdout, "stderr", stderr)
+		return err
+	}
+	stdout, stderr, err = RunIP("link", "set", logicalInterfaceName, "alias", "nfn-"+logicalInterfaceName)
+	if err != nil {
+		log.Error(err, "Failed to create Vlan", "stdout", stdout, "stderr", stderr)
+		return err
+	}
+	return nil
+}
+
+// DeleteVlan deletes VLAN with logicalInterface Name
+func DeleteVlan(logicalInterfaceName string) error {
+	if logicalInterfaceName == "" {
+		return fmt.Errorf("DeleteVlan invalid parameters")
+	}
+	stdout, stderr, err := RunIP("link", "del", "dev", logicalInterfaceName)
+	if err != nil {
+		log.Error(err, "Failed to create Vlan", "stdout", stdout, "stderr", stderr)
+		return err
+	}
+	return nil
+}
+
+// GetVlan returns a list of VLAN configured on the node
+func GetVlan() []string {
+	var intfList []string
+	links, err := netlink.LinkList()
+	if err != nil {
+
+	}
+	for _, l := range links {
+		if strings.Contains(l.Attrs().Alias, "nfn-") {
+			intfList = append(intfList, l.Attrs().Name)
+		}
+	}
+	return intfList
+}
+
+// CreatePnBridge creates Provider network bridge and mappings
+func CreatePnBridge(nwName, brName, intfName string) error {
+	if nwName == "" || brName == "" || intfName == "" {
+		return fmt.Errorf("CreatePnBridge invalid parameters")
+	}
+	// Create Bridge
+	stdout, stderr, err := RunOVSVsctl("--may-exist", "add-br", brName)
+	if err != nil {
+		log.Error(err, "Failed to create Bridge", "stdout", stdout, "stderr", stderr)
+		return err
+	}
+	stdout, stderr, err = RunOVSVsctl("--may-exist", "add-port", brName, intfName)
+	if err != nil {
+		log.Error(err, "Failed to add port to Bridge", "stdout", stdout, "stderr", stderr)
+		return err
+	}
+	stdout, stderr, err = RunOVSVsctl("set", "bridge", brName, "external_ids:nfn="+nwName)
+	if err != nil {
+		log.Error(err, "Failed to set nfn-alias", "stdout", stdout, "stderr", stderr)
+		return err
+	}
+	// Update ovn-bridge-mappings
+	updateOvnBridgeMapping(brName, nwName, "add")
+	return nil
+}
+
+// DeletePnBridge creates Provider network bridge and mappings
+func DeletePnBridge(nwName, brName string) error {
+	if nwName == "" || brName == "" {
+		return fmt.Errorf("DeletePnBridge invalid parameters")
+	}
+	// Delete Bridge
+	stdout, stderr, err := RunOVSVsctl("--if-exist", "del-br", brName)
+	if err != nil {
+		log.Error(err, "Failed to delete Bridge", "stdout", stdout, "stderr", stderr)
+		return err
+	}
+	updateOvnBridgeMapping(brName, nwName, "delete")
+
+	return nil
+}
+
+// GetPnBridge returns Provider networks with external ids
+func GetPnBridge(externalID string) []string {
+	if externalID == "" {
+		log.Error(fmt.Errorf("GetBridge invalid parameters"), "Invalid")
+	}
+	stdout, stderr, err := RunOVSVsctl("list-br")
+	if err != nil {
+		log.Error(err, "No bridges found", "stdout", stdout, "stderr", stderr)
+		return nil
+	}
+	brNames := strings.Split(stdout, "\n")
+	var brList []string
+	for _, name := range brNames {
+		stdout, stderr, err = RunOVSVsctl("get", "bridge", name, "external_ids:"+externalID)
+		if err != nil {
+			if !strings.Contains(stderr, "no key") {
+				log.Error(err, "Unknown error reading external_ids", "stdout", stdout, "stderr", stderr)
+			}
+			continue
+		}
+		if stdout == "" {
+			continue
+		} else {
+			brList = append(brList, name)
+		}
+	}
+	return brList
+}
+
+// Update ovn-bridge-mappings
+func updateOvnBridgeMapping(brName, nwName, action string) error {
+	stdout, stderr, err := RunOVSVsctl("get", "open", ".", "external-ids:ovn-bridge-mappings")
+	if err != nil {
+		if !strings.Contains(stderr, "no key") {
+			log.Error(err, "Failed to get ovn-bridge-mappings", "stdout", stdout, "stderr", stderr)
+			return err
+		}
+	}
+	// Convert csv string to map
+	mm := make(map[string]string)
+	if len(stdout) > 0 {
+		am := strings.Split(stdout, ",")
+		for _, label := range am {
+			l := strings.Split(label, ":")
+			if len(l) == 0 {
+				log.Error(fmt.Errorf("Syntax error label: %v", label), "ovnBridgeMapping")
+				return nil
+			}
+			mm[strings.TrimSpace(l[0])] = strings.TrimSpace(l[1])
+		}
+	}
+	if action == "add" {
+		mm[nwName] = brName
+	} else if action == "delete" {
+		delete(mm, nwName)
+		if len(mm) == 0 {
+			// No mapping needed
+			stdout, stderr, err = RunOVSVsctl("remove", "open", ".", "external-ids", "ovn-bridge-mappings")
+			if err != nil {
+				log.Error(err, "Failed to remove ovn-bridge-mappings", "stdout", stdout, "stderr", stderr)
+				return err
+			}
+			return nil
+		}
+	} else {
+		return fmt.Errorf("Invalid action %s", action)
+	}
+	var mapping string
+	for key, value := range mm {
+		mapping = mapping + fmt.Sprintf("%s:%s,", key, value)
+	}
+	// Remove trailing ,
+	mapping = mapping[:len(mapping)-1]
+	extIDMap := "external-ids:ovn-bridge-mappings=" + mapping
+
+	stdout, stderr, err = RunOVSVsctl("set", "open", ".", extIDMap)
+	if err != nil {
+		log.Error(err, "Failed to set ovn-bridge-mappings", "stdout", stdout, "stderr", stderr)
+		return err
+	}
+	return nil
+}
+
 func parseOvnNetworkObject(ovnnetwork string) ([]map[string]interface{}, error) {
 	var ovnNet []map[string]interface{}
 
diff --git a/internal/pkg/ovn/ovn.go b/internal/pkg/ovn/ovn.go
index 12a4912..6e9cd79 100644
--- a/internal/pkg/ovn/ovn.go
+++ b/internal/pkg/ovn/ovn.go
@@ -72,6 +72,11 @@ func (oc *Controller) AddLogicalPorts(pod *kapi.Pod, ovnNetObjs []map[string]int
 		return
 	}
 
+	if _, ok := pod.Annotations[Ovn4nfvAnnotationTag]; ok {
+		log.V(1).Info("AddLogicalPorts : Pod annotation found")
+		return
+	}
+
 	var ovnString, outStr string
 	ovnString = "["
 	var ns netInterface
@@ -94,20 +99,6 @@ func (oc *Controller) AddLogicalPorts(pod *kapi.Pod, ovnNetObjs []map[string]int
 		if ns.DefaultGateway == "" {
 			ns.DefaultGateway = "false"
 		}
-		if ns.NetType == "" || ns.NetType != "provider" {
-			ns.NetType = "virtual"
-		}
-		if ns.NetType == "provider" {
-			if ns.IPAddress == "" {
-				log.Info("ipAddress must be provided for netType Provider")
-				return
-			}
-			if ns.DefaultGateway == "true" {
-				log.Info("defaultGateway not supported for provider network - Use ovnNetworkRoutes to add routes")
-				return
-			}
-
-		}
 		outStr = oc.addLogicalPortWithSwitch(pod, ns.Name, ns.IPAddress, ns.MacAddress, ns.Interface, ns.NetType)
 		if outStr == "" {
 			return
@@ -155,37 +146,30 @@ func (oc *Controller) DeleteLogicalPorts(name, namespace string) {
 }
 
 // CreateNetwork in OVN controller
-func (oc *Controller) CreateNetwork(cr *k8sv1alpha1.Network) error {
+func (oc *Controller) createOvnLS(name, subnet, gatewayIP, excludeIps string) (gatewayIPMask string, err error) {
 	var stdout, stderr string
 
-	// Currently only these fields are supported
-	name := cr.Name
-	subnet := cr.Spec.Ipv4Subnets[0].Subnet
-	gatewayIP := cr.Spec.Ipv4Subnets[0].Gateway
-	excludeIps := cr.Spec.Ipv4Subnets[0].ExcludeIps
-
 	output, stderr, err := RunOVNNbctl("--data=bare", "--no-heading",
 		"--columns=name", "find", "logical_switch", "name="+name)
 	if err != nil {
 		log.Error(err, "Error in reading logical switch", "stderr", stderr)
-		return nil
+		return
 	}
 
 	if strings.Compare(name, output) == 0 {
 		log.V(1).Info("Logical Switch already exists, delete first to update/recreate", "name", name)
-		return nil
+		return "", fmt.Errorf("LS exists")
 	}
 
 	_, cidr, err := net.ParseCIDR(subnet)
 	if err != nil {
 		log.Error(err, "ovnNetwork '%s' invalid subnet CIDR", "name", name)
-		return err
+		return
 
 	}
 	firstIP := NextIP(cidr.IP)
 	n, _ := cidr.Mask.Size()
 
-	var gatewayIPMask string
 	var gwIP net.IP
 	if gatewayIP != "" {
 		gwIP, _, err = net.ParseCIDR(gatewayIP)
@@ -209,6 +193,23 @@ func (oc *Controller) CreateNetwork(cr *k8sv1alpha1.Network) error {
 	}
 	if err != nil {
 		log.Error(err, "Failed to create a logical switch", "name", name, "stdout", stdout, "stderr", stderr)
+		return
+	}
+	return
+}
+
+// CreateNetwork in OVN controller
+func (oc *Controller) CreateNetwork(cr *k8sv1alpha1.Network) error {
+	var stdout, stderr string
+
+	// Currently only these fields are supported
+	name := cr.Name
+	subnet := cr.Spec.Ipv4Subnets[0].Subnet
+	gatewayIP := cr.Spec.Ipv4Subnets[0].Gateway
+	excludeIps := cr.Spec.Ipv4Subnets[0].ExcludeIps
+
+	gatewayIPMask, err := oc.createOvnLS(name, subnet, gatewayIP, excludeIps)
+	if err != nil {
 		return err
 	}
 
@@ -256,6 +257,45 @@ func (oc *Controller) DeleteNetwork(cr *k8sv1alpha1.Network) error {
 	return nil
 }
 
+// CreateProviderNetwork in OVN controller
+func (oc *Controller) CreateProviderNetwork(cr *k8sv1alpha1.ProviderNetwork) error {
+	var stdout, stderr string
+
+	// Currently only these fields are supported
+	name := cr.Name
+	subnet := cr.Spec.Ipv4Subnets[0].Subnet
+	gatewayIP := cr.Spec.Ipv4Subnets[0].Gateway
+	excludeIps := cr.Spec.Ipv4Subnets[0].ExcludeIps
+	_, err := oc.createOvnLS(name, subnet, gatewayIP, excludeIps)
+	if err != nil {
+		return err
+	}
+
+	// Add localnet port.
+	stdout, stderr, err = RunOVNNbctl("--wait=hv", "--", "--may-exist", "lsp-add", name, "server-localnet_"+name, "--",
+		"lsp-set-addresses", "server-localnet_"+name, "unknown", "--",
+		"lsp-set-type", "server-localnet_"+name, "localnet", "--",
+		"lsp-set-options", "server-localnet_"+name, "network_name=nw_"+name)
+	if err != nil {
+		log.Error(err, "Failed to add logical port to switch", "stderr", stderr, "stdout", stdout)
+		return err
+	}
+
+	return nil
+}
+
+// DeleteProviderNetwork in OVN controller
+func (oc *Controller) DeleteProviderNetwork(cr *k8sv1alpha1.ProviderNetwork) error {
+
+	name := cr.Name
+	stdout, stderr, err := RunOVNNbctl("--if-exist", "--wait=hv", "ls-del", name)
+	if err != nil {
+		log.Error(err, "Failed to delete switch", "name", name, "stdout", stdout, "stderr", stderr)
+		return err
+	}
+	return nil
+}
+
 // FindLogicalSwitch returns true if switch exists
 func (oc *Controller) FindLogicalSwitch(name string) bool {
 	// get logical switch from OVN
@@ -275,7 +315,6 @@ func (oc *Controller) getGatewayFromSwitch(logicalSwitch string) (string, string
 	var gatewayIPMaskStr, stderr string
 	var ok bool
 	var err error
-	log.V(1).Info("getGatewayFromSwitch", "logicalSwitch", logicalSwitch)
 	if gatewayIPMaskStr, ok = oc.gatewayCache[logicalSwitch]; !ok {
 		gatewayIPMaskStr, stderr, err = RunOVNNbctl("--if-exists",
 			"get", "logical_switch", logicalSwitch,
@@ -388,16 +427,12 @@ func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipA
 		return
 	}
 
-	if netType == "virtual" {
-		gatewayIP, mask, err := oc.getGatewayFromSwitch(logicalSwitch)
-		if err != nil {
-			log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch)
-			return
-		}
-		annotation = fmt.Sprintf(`{\"ip_address\":\"%s/%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], mask, addresses[0], gatewayIP)
-	} else {
-		annotation = fmt.Sprintf(`{\"ip_address\":\"%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], addresses[0], "")
+	gatewayIP, mask, err := oc.getGatewayFromSwitch(logicalSwitch)
+	if err != nil {
+		log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch)
+		return
 	}
+	annotation = fmt.Sprintf(`{\"ip_address\":\"%s/%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], mask, addresses[0], gatewayIP)
 
 	return annotation
 }
diff --git a/internal/pkg/ovn/ovn_test.go b/internal/pkg/ovn/ovn_test.go
index 6e38759..5ea01d1 100644
--- a/internal/pkg/ovn/ovn_test.go
+++ b/internal/pkg/ovn/ovn_test.go
@@ -128,6 +128,11 @@ var _ = Describe("Add logical Port", func() {
 				Output: macIPAddress,
 			})
 
+			fakeCmds = ovntest.AddFakeCmd(fakeCmds, &ovntest.ExpectedCmd{
+				Cmd:    "ovn-nbctl --timeout=15 --if-exists get logical_switch " + netName + " external_ids:gateway_ip",
+				Output: gwCIDR,
+			})
+
 			fexec := &fakeexec.FakeExec{
 				CommandScript: fakeCmds,
 				LookPathFunc: func(file string) (string, error) {
diff --git a/internal/pkg/ovn/utils.go b/internal/pkg/ovn/utils.go
index 615c2f9..3b3b53b 100644
--- a/internal/pkg/ovn/utils.go
+++ b/internal/pkg/ovn/utils.go
@@ -12,12 +12,16 @@ import (
 const (
 	ovsCommandTimeout = 15
 	ovnNbctlCommand   = "ovn-nbctl"
+	ovsVsctlCommand   = "ovs-vsctl"
+	ipCommand         = "ip"
 )
 
 // Exec runs various OVN and OVS utilities
 type execHelper struct {
 	exec      kexec.Interface
 	nbctlPath string
+	vsctlPath string
+	ipPath    string
 	hostIP    string
 	hostPort  string
 }
@@ -26,11 +30,6 @@ var runner *execHelper
 
 // SetupOvnUtils does internal OVN initialization
 var SetupOvnUtils = func() error {
-	runner.hostIP = os.Getenv("HOST_IP")
-	// OVN Host Port
-	runner.hostPort = "6641"
-	log.Info("Host Port", "IP", runner.hostIP, "Port", runner.hostPort)
-
 	// Setup Distributed Router
 	err := setupDistributedRouter(ovn4nfvRouterName)
 	if err != nil {
@@ -50,6 +49,19 @@ func SetExec(exec kexec.Interface) error {
 	if err != nil {
 		return err
 	}
+	runner.vsctlPath, err = exec.LookPath(ovsVsctlCommand)
+	if err != nil {
+		return err
+	}
+	runner.ipPath, err = exec.LookPath(ipCommand)
+	if err != nil {
+		return err
+	}
+	runner.hostIP = os.Getenv("HOST_IP")
+	// OVN Host Port
+	runner.hostPort = "6641"
+	log.Info("Host Port", "IP", runner.hostIP, "Port", runner.hostPort)
+
 	return nil
 }
 
@@ -87,7 +99,7 @@ func run(cmdPath string, args ...string) (*bytes.Buffer, *bytes.Buffer, error) {
 	log.V(1).Info("exec:", "cmdPath", cmdPath, "args", strings.Join(args, " "))
 	err := cmd.Run()
 	if err != nil {
-		log.Error(err, "Error:", "cmdPath", cmdPath, "args", strings.Join(args, " "), "stdout", stdout, "stderr", stderr)
+		log.Info("ovs", "Error:", err, "cmdPath", cmdPath, "args", strings.Join(args, " "), "stdout", stdout, "stderr", stderr)
 	} else {
 		log.V(1).Info("output:", "stdout", stdout)
 	}
@@ -112,3 +124,17 @@ func RunOVNNbctlWithTimeout(timeout int, args ...string) (string, string, error)
 func RunOVNNbctl(args ...string) (string, string, error) {
 	return RunOVNNbctlWithTimeout(ovsCommandTimeout, args...)
 }
+
+// RunIP runs a command via the iproute2 "ip" utility
+func RunIP(args ...string) (string, string, error) {
+	stdout, stderr, err := run(runner.ipPath, args...)
+	return strings.TrimSpace(stdout.String()), stderr.String(), err
+}
+
+// RunOVSVsctl runs a command via ovs-vsctl.
+func RunOVSVsctl(args ...string) (string, string, error) {
+	cmdArgs := []string{fmt.Sprintf("--timeout=%d", ovsCommandTimeout)}
+	cmdArgs = append(cmdArgs, args...)
+	stdout, stderr, err := run(runner.vsctlPath, cmdArgs...)
+	return strings.Trim(strings.TrimSpace(stdout.String()), "\""), stderr.String(), err
+}