From a502b61a7f946e2c20da7237e13c174f1b6e982a Mon Sep 17 00:00:00 2001 From: Ritu Sood Date: Fri, 27 Sep 2019 16:55:43 -0700 Subject: Update OVN utils to add provider network support Currently provider networks support only VLAN based networks. Added functions for VLAN and ovs bridge. Change-Id: I63ac7266aac92021ee3a44f49644a8a79ce4fed6 Signed-off-by: Ritu Sood --- internal/pkg/ovn/common.go | 177 +++++++++++++++++++++++++++++++++++++++++++ internal/pkg/ovn/ovn.go | 105 ++++++++++++++++--------- internal/pkg/ovn/ovn_test.go | 5 ++ internal/pkg/ovn/utils.go | 38 ++++++++-- 4 files changed, 284 insertions(+), 41 deletions(-) diff --git a/internal/pkg/ovn/common.go b/internal/pkg/ovn/common.go index 09d770b..d3e477a 100644 --- a/internal/pkg/ovn/common.go +++ b/internal/pkg/ovn/common.go @@ -3,15 +3,192 @@ package ovn import ( "encoding/json" "fmt" + "github.com/vishvananda/netlink" "math/big" "math/rand" "net" logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "strings" "time" ) var log = logf.Log.WithName("ovn") +// CreateVlan creates VLAN with vlanID +func CreateVlan(vlanID, interfaceName, logicalInterfaceName string) error { + if interfaceName == "" || vlanID == "" || logicalInterfaceName == "" { + return fmt.Errorf("CreateVlan invalid parameters: %v %v %v", interfaceName, vlanID, logicalInterfaceName) + } + _, err := netlink.LinkByName(logicalInterfaceName) + if err == nil { + return err + } + stdout, stderr, err := RunIP("link", "add", "link", interfaceName, "name", logicalInterfaceName, "type", "vlan", "id", vlanID) + if err != nil { + log.Error(err, "Failed to create Vlan", "stdout", stdout, "stderr", stderr) + return err + } + stdout, stderr, err = RunIP("link", "set", logicalInterfaceName, "alias", "nfn-"+logicalInterfaceName) + if err != nil { + log.Error(err, "Failed to create Vlan", "stdout", stdout, "stderr", stderr) + return err + } + return nil +} + +// DeleteVlan deletes VLAN with logicalInterface Name +func DeleteVlan(logicalInterfaceName string) error { + if logicalInterfaceName == "" { + return fmt.Errorf("DeleteVlan invalid parameters") + } + stdout, stderr, err := RunIP("link", "del", "dev", logicalInterfaceName) + if err != nil { + log.Error(err, "Failed to create Vlan", "stdout", stdout, "stderr", stderr) + return err + } + return nil +} + +// GetVlan returns a list of VLAN configured on the node +func GetVlan() []string { + var intfList []string + links, err := netlink.LinkList() + if err != nil { + + } + for _, l := range links { + if strings.Contains(l.Attrs().Alias, "nfn-") { + intfList = append(intfList, l.Attrs().Name) + } + } + return intfList +} + +// CreatePnBridge creates Provider network bridge and mappings +func CreatePnBridge(nwName, brName, intfName string) error { + if nwName == "" || brName == "" || intfName == "" { + return fmt.Errorf("CreatePnBridge invalid parameters") + } + // Create Bridge + stdout, stderr, err := RunOVSVsctl("--may-exist", "add-br", brName) + if err != nil { + log.Error(err, "Failed to create Bridge", "stdout", stdout, "stderr", stderr) + return err + } + stdout, stderr, err = RunOVSVsctl("--may-exist", "add-port", brName, intfName) + if err != nil { + log.Error(err, "Failed to add port to Bridge", "stdout", stdout, "stderr", stderr) + return err + } + stdout, stderr, err = RunOVSVsctl("set", "bridge", brName, "external_ids:nfn="+nwName) + if err != nil { + log.Error(err, "Failed to set nfn-alias", "stdout", stdout, "stderr", stderr) + return err + } + // Update ovn-bridge-mappings + updateOvnBridgeMapping(brName, nwName, "add") + return nil +} + +// DeletePnBridge creates Provider network bridge and mappings +func DeletePnBridge(nwName, brName string) error { + if nwName == "" || brName == "" { + return fmt.Errorf("DeletePnBridge invalid parameters") + } + // Delete Bridge + stdout, stderr, err := RunOVSVsctl("--if-exist", "del-br", brName) + if err != nil { + log.Error(err, "Failed to delete Bridge", "stdout", stdout, "stderr", stderr) + return err + } + updateOvnBridgeMapping(brName, nwName, "delete") + + return nil +} + +// GetPnBridge returns Provider networks with external ids +func GetPnBridge(externalID string) []string { + if externalID == "" { + log.Error(fmt.Errorf("GetBridge invalid parameters"), "Invalid") + } + stdout, stderr, err := RunOVSVsctl("list-br") + if err != nil { + log.Error(err, "No bridges found", "stdout", stdout, "stderr", stderr) + return nil + } + brNames := strings.Split(stdout, "\n") + var brList []string + for _, name := range brNames { + stdout, stderr, err = RunOVSVsctl("get", "bridge", name, "external_ids:"+externalID) + if err != nil { + if !strings.Contains(stderr, "no key") { + log.Error(err, "Unknown error reading external_ids", "stdout", stdout, "stderr", stderr) + } + continue + } + if stdout == "" { + continue + } else { + brList = append(brList, name) + } + } + return brList +} + +// Update ovn-bridge-mappings +func updateOvnBridgeMapping(brName, nwName, action string) error { + stdout, stderr, err := RunOVSVsctl("get", "open", ".", "external-ids:ovn-bridge-mappings") + if err != nil { + if !strings.Contains(stderr, "no key") { + log.Error(err, "Failed to get ovn-bridge-mappings", "stdout", stdout, "stderr", stderr) + return err + } + } + // Convert csv string to map + mm := make(map[string]string) + if len(stdout) > 0 { + am := strings.Split(stdout, ",") + for _, label := range am { + l := strings.Split(label, ":") + if len(l) == 0 { + log.Error(fmt.Errorf("Syntax error label: %v", label), "ovnBridgeMapping") + return nil + } + mm[strings.TrimSpace(l[0])] = strings.TrimSpace(l[1]) + } + } + if action == "add" { + mm[nwName] = brName + } else if action == "delete" { + delete(mm, nwName) + if len(mm) == 0 { + // No mapping needed + stdout, stderr, err = RunOVSVsctl("remove", "open", ".", "external-ids", "ovn-bridge-mappings") + if err != nil { + log.Error(err, "Failed to remove ovn-bridge-mappings", "stdout", stdout, "stderr", stderr) + return err + } + return nil + } + } else { + return fmt.Errorf("Invalid action %s", action) + } + var mapping string + for key, value := range mm { + mapping = mapping + fmt.Sprintf("%s:%s,", key, value) + } + // Remove trailing , + mapping = mapping[:len(mapping)-1] + extIDMap := "external-ids:ovn-bridge-mappings=" + mapping + + stdout, stderr, err = RunOVSVsctl("set", "open", ".", extIDMap) + if err != nil { + log.Error(err, "Failed to set ovn-bridge-mappings", "stdout", stdout, "stderr", stderr) + return err + } + return nil +} + func parseOvnNetworkObject(ovnnetwork string) ([]map[string]interface{}, error) { var ovnNet []map[string]interface{} diff --git a/internal/pkg/ovn/ovn.go b/internal/pkg/ovn/ovn.go index 12a4912..6e9cd79 100644 --- a/internal/pkg/ovn/ovn.go +++ b/internal/pkg/ovn/ovn.go @@ -72,6 +72,11 @@ func (oc *Controller) AddLogicalPorts(pod *kapi.Pod, ovnNetObjs []map[string]int return } + if _, ok := pod.Annotations[Ovn4nfvAnnotationTag]; ok { + log.V(1).Info("AddLogicalPorts : Pod annotation found") + return + } + var ovnString, outStr string ovnString = "[" var ns netInterface @@ -94,20 +99,6 @@ func (oc *Controller) AddLogicalPorts(pod *kapi.Pod, ovnNetObjs []map[string]int if ns.DefaultGateway == "" { ns.DefaultGateway = "false" } - if ns.NetType == "" || ns.NetType != "provider" { - ns.NetType = "virtual" - } - if ns.NetType == "provider" { - if ns.IPAddress == "" { - log.Info("ipAddress must be provided for netType Provider") - return - } - if ns.DefaultGateway == "true" { - log.Info("defaultGateway not supported for provider network - Use ovnNetworkRoutes to add routes") - return - } - - } outStr = oc.addLogicalPortWithSwitch(pod, ns.Name, ns.IPAddress, ns.MacAddress, ns.Interface, ns.NetType) if outStr == "" { return @@ -155,37 +146,30 @@ func (oc *Controller) DeleteLogicalPorts(name, namespace string) { } // CreateNetwork in OVN controller -func (oc *Controller) CreateNetwork(cr *k8sv1alpha1.Network) error { +func (oc *Controller) createOvnLS(name, subnet, gatewayIP, excludeIps string) (gatewayIPMask string, err error) { var stdout, stderr string - // Currently only these fields are supported - name := cr.Name - subnet := cr.Spec.Ipv4Subnets[0].Subnet - gatewayIP := cr.Spec.Ipv4Subnets[0].Gateway - excludeIps := cr.Spec.Ipv4Subnets[0].ExcludeIps - output, stderr, err := RunOVNNbctl("--data=bare", "--no-heading", "--columns=name", "find", "logical_switch", "name="+name) if err != nil { log.Error(err, "Error in reading logical switch", "stderr", stderr) - return nil + return } if strings.Compare(name, output) == 0 { log.V(1).Info("Logical Switch already exists, delete first to update/recreate", "name", name) - return nil + return "", fmt.Errorf("LS exists") } _, cidr, err := net.ParseCIDR(subnet) if err != nil { log.Error(err, "ovnNetwork '%s' invalid subnet CIDR", "name", name) - return err + return } firstIP := NextIP(cidr.IP) n, _ := cidr.Mask.Size() - var gatewayIPMask string var gwIP net.IP if gatewayIP != "" { gwIP, _, err = net.ParseCIDR(gatewayIP) @@ -209,6 +193,23 @@ func (oc *Controller) CreateNetwork(cr *k8sv1alpha1.Network) error { } if err != nil { log.Error(err, "Failed to create a logical switch", "name", name, "stdout", stdout, "stderr", stderr) + return + } + return +} + +// CreateNetwork in OVN controller +func (oc *Controller) CreateNetwork(cr *k8sv1alpha1.Network) error { + var stdout, stderr string + + // Currently only these fields are supported + name := cr.Name + subnet := cr.Spec.Ipv4Subnets[0].Subnet + gatewayIP := cr.Spec.Ipv4Subnets[0].Gateway + excludeIps := cr.Spec.Ipv4Subnets[0].ExcludeIps + + gatewayIPMask, err := oc.createOvnLS(name, subnet, gatewayIP, excludeIps) + if err != nil { return err } @@ -256,6 +257,45 @@ func (oc *Controller) DeleteNetwork(cr *k8sv1alpha1.Network) error { return nil } +// CreateProviderNetwork in OVN controller +func (oc *Controller) CreateProviderNetwork(cr *k8sv1alpha1.ProviderNetwork) error { + var stdout, stderr string + + // Currently only these fields are supported + name := cr.Name + subnet := cr.Spec.Ipv4Subnets[0].Subnet + gatewayIP := cr.Spec.Ipv4Subnets[0].Gateway + excludeIps := cr.Spec.Ipv4Subnets[0].ExcludeIps + _, err := oc.createOvnLS(name, subnet, gatewayIP, excludeIps) + if err != nil { + return err + } + + // Add localnet port. + stdout, stderr, err = RunOVNNbctl("--wait=hv", "--", "--may-exist", "lsp-add", name, "server-localnet_"+name, "--", + "lsp-set-addresses", "server-localnet_"+name, "unknown", "--", + "lsp-set-type", "server-localnet_"+name, "localnet", "--", + "lsp-set-options", "server-localnet_"+name, "network_name=nw_"+name) + if err != nil { + log.Error(err, "Failed to add logical port to switch", "stderr", stderr, "stdout", stdout) + return err + } + + return nil +} + +// DeleteProviderNetwork in OVN controller +func (oc *Controller) DeleteProviderNetwork(cr *k8sv1alpha1.ProviderNetwork) error { + + name := cr.Name + stdout, stderr, err := RunOVNNbctl("--if-exist", "--wait=hv", "ls-del", name) + if err != nil { + log.Error(err, "Failed to delete switch", "name", name, "stdout", stdout, "stderr", stderr) + return err + } + return nil +} + // FindLogicalSwitch returns true if switch exists func (oc *Controller) FindLogicalSwitch(name string) bool { // get logical switch from OVN @@ -275,7 +315,6 @@ func (oc *Controller) getGatewayFromSwitch(logicalSwitch string) (string, string var gatewayIPMaskStr, stderr string var ok bool var err error - log.V(1).Info("getGatewayFromSwitch", "logicalSwitch", logicalSwitch) if gatewayIPMaskStr, ok = oc.gatewayCache[logicalSwitch]; !ok { gatewayIPMaskStr, stderr, err = RunOVNNbctl("--if-exists", "get", "logical_switch", logicalSwitch, @@ -388,16 +427,12 @@ func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipA return } - if netType == "virtual" { - gatewayIP, mask, err := oc.getGatewayFromSwitch(logicalSwitch) - if err != nil { - log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch) - return - } - annotation = fmt.Sprintf(`{\"ip_address\":\"%s/%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], mask, addresses[0], gatewayIP) - } else { - annotation = fmt.Sprintf(`{\"ip_address\":\"%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], addresses[0], "") + gatewayIP, mask, err := oc.getGatewayFromSwitch(logicalSwitch) + if err != nil { + log.Error(err, "Error obtaining gateway address for switch", "logicalSwitch", logicalSwitch) + return } + annotation = fmt.Sprintf(`{\"ip_address\":\"%s/%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], mask, addresses[0], gatewayIP) return annotation } diff --git a/internal/pkg/ovn/ovn_test.go b/internal/pkg/ovn/ovn_test.go index 6e38759..5ea01d1 100644 --- a/internal/pkg/ovn/ovn_test.go +++ b/internal/pkg/ovn/ovn_test.go @@ -128,6 +128,11 @@ var _ = Describe("Add logical Port", func() { Output: macIPAddress, }) + fakeCmds = ovntest.AddFakeCmd(fakeCmds, &ovntest.ExpectedCmd{ + Cmd: "ovn-nbctl --timeout=15 --if-exists get logical_switch " + netName + " external_ids:gateway_ip", + Output: gwCIDR, + }) + fexec := &fakeexec.FakeExec{ CommandScript: fakeCmds, LookPathFunc: func(file string) (string, error) { diff --git a/internal/pkg/ovn/utils.go b/internal/pkg/ovn/utils.go index 615c2f9..3b3b53b 100644 --- a/internal/pkg/ovn/utils.go +++ b/internal/pkg/ovn/utils.go @@ -12,12 +12,16 @@ import ( const ( ovsCommandTimeout = 15 ovnNbctlCommand = "ovn-nbctl" + ovsVsctlCommand = "ovs-vsctl" + ipCommand = "ip" ) // Exec runs various OVN and OVS utilities type execHelper struct { exec kexec.Interface nbctlPath string + vsctlPath string + ipPath string hostIP string hostPort string } @@ -26,11 +30,6 @@ var runner *execHelper // SetupOvnUtils does internal OVN initialization var SetupOvnUtils = func() error { - runner.hostIP = os.Getenv("HOST_IP") - // OVN Host Port - runner.hostPort = "6641" - log.Info("Host Port", "IP", runner.hostIP, "Port", runner.hostPort) - // Setup Distributed Router err := setupDistributedRouter(ovn4nfvRouterName) if err != nil { @@ -50,6 +49,19 @@ func SetExec(exec kexec.Interface) error { if err != nil { return err } + runner.vsctlPath, err = exec.LookPath(ovsVsctlCommand) + if err != nil { + return err + } + runner.ipPath, err = exec.LookPath(ipCommand) + if err != nil { + return err + } + runner.hostIP = os.Getenv("HOST_IP") + // OVN Host Port + runner.hostPort = "6641" + log.Info("Host Port", "IP", runner.hostIP, "Port", runner.hostPort) + return nil } @@ -87,7 +99,7 @@ func run(cmdPath string, args ...string) (*bytes.Buffer, *bytes.Buffer, error) { log.V(1).Info("exec:", "cmdPath", cmdPath, "args", strings.Join(args, " ")) err := cmd.Run() if err != nil { - log.Error(err, "Error:", "cmdPath", cmdPath, "args", strings.Join(args, " "), "stdout", stdout, "stderr", stderr) + log.Info("ovs", "Error:", err, "cmdPath", cmdPath, "args", strings.Join(args, " "), "stdout", stdout, "stderr", stderr) } else { log.V(1).Info("output:", "stdout", stdout) } @@ -112,3 +124,17 @@ func RunOVNNbctlWithTimeout(timeout int, args ...string) (string, string, error) func RunOVNNbctl(args ...string) (string, string, error) { return RunOVNNbctlWithTimeout(ovsCommandTimeout, args...) } + +// RunIP runs a command via the iproute2 "ip" utility +func RunIP(args ...string) (string, string, error) { + stdout, stderr, err := run(runner.ipPath, args...) + return strings.TrimSpace(stdout.String()), stderr.String(), err +} + +// RunOVSVsctl runs a command via ovs-vsctl. +func RunOVSVsctl(args ...string) (string, string, error) { + cmdArgs := []string{fmt.Sprintf("--timeout=%d", ovsCommandTimeout)} + cmdArgs = append(cmdArgs, args...) + stdout, stderr, err := run(runner.vsctlPath, cmdArgs...) + return strings.Trim(strings.TrimSpace(stdout.String()), "\""), stderr.String(), err +} -- cgit 1.2.3-korg