diff options
Diffstat (limited to 'internal/pkg/ovn')
-rw-r--r-- | internal/pkg/ovn/common.go | 117 | ||||
-rw-r--r-- | internal/pkg/ovn/ovn.go | 286 | ||||
-rw-r--r-- | internal/pkg/ovn/ovn_test.go | 5 | ||||
-rw-r--r-- | internal/pkg/ovn/pods.go | 292 | ||||
-rw-r--r-- | internal/pkg/ovn/router.go | 50 | ||||
-rw-r--r-- | internal/pkg/ovn/utils.go | 120 |
6 files changed, 469 insertions, 401 deletions
diff --git a/internal/pkg/ovn/common.go b/internal/pkg/ovn/common.go index 16923ea..b504440 100644 --- a/internal/pkg/ovn/common.go +++ b/internal/pkg/ovn/common.go @@ -4,77 +4,57 @@ import ( "encoding/json" "fmt" "github.com/sirupsen/logrus" - "strings" + "math/big" + "math/rand" + "net" + "time" ) -func (oc *Controller) getIPFromOvnAnnotation(ovnAnnotation string) string { - if ovnAnnotation == "" { - return "" - } +func SetupDistributedRouter(name string) error { - var ovnAnnotationMap map[string]string - err := json.Unmarshal([]byte(ovnAnnotation), &ovnAnnotationMap) + // Make sure br-int is created. + stdout, stderr, err := RunOVSVsctlUnix("--", "--may-exist", "add-br", "br-int") if err != nil { - logrus.Errorf("Error in json unmarshaling ovn annotation "+ - "(%v)", err) - return "" + logrus.Errorf("Failed to create br-int, stdout: %q, stderr: %q, error: %v", stdout, stderr, err) + return err } - - ipAddressMask := strings.Split(ovnAnnotationMap["ip_address"], "/") - if len(ipAddressMask) != 2 { - logrus.Errorf("Error in splitting ip address") - return "" + // Create a single common distributed router for the cluster. + stdout, stderr, err = RunOVNNbctlUnix("--", "--may-exist", "lr-add", name, "--", "set", "logical_router", name, "external_ids:ovn4nfv-cluster-router=yes") + if err != nil { + logrus.Errorf("Failed to create a single common distributed router for the cluster, stdout: %q, stderr: %q, error: %v", stdout, stderr, err) + return err } - - return ipAddressMask[0] -} - -func (oc *Controller) getMacFromOvnAnnotation(ovnAnnotation string) string { - if ovnAnnotation == "" { - return "" + // Create a logical switch called "ovn4nfv-join" that will be used to connect gateway routers to the distributed router. + // The "ovn4nfv-join" will be allocated IP addresses in the range 100.64.1.0/24. + stdout, stderr, err = RunOVNNbctlUnix("--may-exist", "ls-add", "ovn4nfv-join") + if err != nil { + logrus.Errorf("Failed to create logical switch called \"ovn4nfv-join\", stdout: %q, stderr: %q, error: %v", stdout, stderr, err) + return err } - - var ovnAnnotationMap map[string]string - err := json.Unmarshal([]byte(ovnAnnotation), &ovnAnnotationMap) + // Connect the distributed router to "ovn4nfv-join". + routerMac, stderr, err := RunOVNNbctlUnix("--if-exist", "get", "logical_router_port", "rtoj-"+name, "mac") if err != nil { - logrus.Errorf("Error in json unmarshaling ovn annotation "+ - "(%v)", err) - return "" + logrus.Errorf("Failed to get logical router port rtoj-%v, stderr: %q, error: %v", name, stderr, err) + return err } - - return ovnAnnotationMap["mac_address"] -} - -func stringSliceMembership(slice []string, key string) bool { - for _, val := range slice { - if val == key { - return true + if routerMac == "" { + routerMac = generateMac() + stdout, stderr, err = RunOVNNbctlUnix("--", "--may-exist", "lrp-add", name, "rtoj-"+name, routerMac, "100.64.1.1/24", "--", "set", "logical_router_port", "rtoj-"+name, "external_ids:connect_to_ovn4nfvjoin=yes") + if err != nil { + logrus.Errorf("Failed to add logical router port rtoj-%v, stdout: %q, stderr: %q, error: %v", name, stdout, stderr, err) + return err } } - return false -} - -func (oc *Controller) getNetworkFromOvnAnnotation(ovnAnnotation string) string { - if ovnAnnotation == "" { - logrus.Errorf("getNetworkFromOvnAnnotation ovnAnnotation: %s", ovnAnnotation) - return "" - } - logrus.Infof("getNetworkFromOvnAnnotation ovnAnnotation: %s", ovnAnnotation) - - var ovnAnnotationMap map[string]string - err := json.Unmarshal([]byte(ovnAnnotation), &ovnAnnotationMap) + // Connect the switch "ovn4nfv-join" to the router. + stdout, stderr, err = RunOVNNbctlUnix("--", "--may-exist", "lsp-add", "ovn4nfv-join", "jtor-"+name, "--", "set", "logical_switch_port", "jtor-"+name, "type=router", "options:router-port=rtoj-"+name, "addresses="+"\""+routerMac+"\"") if err != nil { - logrus.Errorf("Error in json unmarshaling ovn annotation "+ - "(%v)", err) - return "" - } - for key, value := range ovnAnnotationMap { - logrus.Infof("getNetworkFromOvnAnnotation %s: %s", key, value) + logrus.Errorf("Failed to add logical switch port to logical router, stdout: %q, stderr: %q, error: %v", stdout, stderr, err) + return err } - return ovnAnnotationMap["name"] + return nil } -func (oc *Controller) parseOvnNetworkObject(ovnnetwork string) ([]map[string]interface{}, error) { +func parseOvnNetworkObject(ovnnetwork string) ([]map[string]interface{}, error) { var ovnNet []map[string]interface{} if ovnnetwork == "" { @@ -87,3 +67,28 @@ func (oc *Controller) parseOvnNetworkObject(ovnnetwork string) ([]map[string]int return ovnNet, nil } + +// generateMac generates mac address. +func generateMac() string { + prefix := "00:00:00" + newRand := rand.New(rand.NewSource(time.Now().UnixNano())) + mac := fmt.Sprintf("%s:%02x:%02x:%02x", prefix, newRand.Intn(255), newRand.Intn(255), newRand.Intn(255)) + return mac +} + +// NextIP returns IP incremented by 1 +func NextIP(ip net.IP) net.IP { + i := ipToInt(ip) + return intToIP(i.Add(i, big.NewInt(1))) +} + +func ipToInt(ip net.IP) *big.Int { + if v := ip.To4(); v != nil { + return big.NewInt(0).SetBytes(v) + } + return big.NewInt(0).SetBytes(ip.To16()) +} + +func intToIP(i *big.Int) net.IP { + return net.IP(i.Bytes()) +} diff --git a/internal/pkg/ovn/ovn.go b/internal/pkg/ovn/ovn.go index ec2ccbd..470416b 100644 --- a/internal/pkg/ovn/ovn.go +++ b/internal/pkg/ovn/ovn.go @@ -2,6 +2,10 @@ package ovn import ( "fmt" + "strings" + "time" + + "github.com/sirupsen/logrus" kapi "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" @@ -69,3 +73,285 @@ func (oc *Controller) WatchPods() error { }, oc.syncPods) return err } + +func (oc *Controller) syncPods(pods []interface{}) { +} + +func (oc *Controller) getGatewayFromSwitch(logicalSwitch string) (string, string, error) { + var gatewayIPMaskStr, stderr string + var ok bool + var err error + logrus.Infof("getGatewayFromSwitch: %s", logicalSwitch) + if gatewayIPMaskStr, ok = oc.gatewayCache[logicalSwitch]; !ok { + gatewayIPMaskStr, stderr, err = RunOVNNbctlUnix("--if-exists", + "get", "logical_switch", logicalSwitch, + "external_ids:gateway_ip") + if err != nil { + logrus.Errorf("Failed to get gateway IP: %s, stderr: %q, %v", + gatewayIPMaskStr, stderr, err) + return "", "", err + } + if gatewayIPMaskStr == "" { + return "", "", fmt.Errorf("Empty gateway IP in logical switch %s", + logicalSwitch) + } + oc.gatewayCache[logicalSwitch] = gatewayIPMaskStr + } + gatewayIPMask := strings.Split(gatewayIPMaskStr, "/") + if len(gatewayIPMask) != 2 { + return "", "", fmt.Errorf("Failed to get IP and Mask from gateway CIDR: %s", + gatewayIPMaskStr) + } + gatewayIP := gatewayIPMask[0] + mask := gatewayIPMask[1] + return gatewayIP, mask, nil +} + +func (oc *Controller) deleteLogicalPort(pod *kapi.Pod) { + + if pod.Spec.HostNetwork { + return + } + + logrus.Infof("Deleting pod: %s", pod.Name) + logicalPort := fmt.Sprintf("%s_%s", pod.Namespace, pod.Name) + + // get the list of logical ports from OVN + output, stderr, err := RunOVNNbctlUnix("--data=bare", "--no-heading", + "--columns=name", "find", "logical_switch_port", "external_ids:pod=true") + if err != nil { + logrus.Errorf("Error in obtaining list of logical ports, "+ + "stderr: %q, err: %v", + stderr, err) + return + } + logrus.Infof("Exising Ports : %s. ", output) + existingLogicalPorts := strings.Fields(output) + for _, existingPort := range existingLogicalPorts { + if strings.Contains(existingPort, logicalPort) { + // found, delete this logical port + logrus.Infof("Deleting: %s. ", existingPort) + out, stderr, err := RunOVNNbctlUnix("--if-exists", "lsp-del", + existingPort) + if err != nil { + logrus.Errorf("Error in deleting pod's logical port "+ + "stdout: %q, stderr: %q err: %v", + out, stderr, err) + } else { + delete(oc.logicalPortCache, existingPort) + } + } + } + return +} + +func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipAddress, macAddress, interfaceName, netType string) (annotation string) { + var out, stderr string + var err error + var isStaticIP bool + if pod.Spec.HostNetwork { + return + } + + if !oc.logicalSwitchCache[logicalSwitch] { + oc.logicalSwitchCache[logicalSwitch] = true + } + var portName string + if interfaceName != "" { + portName = fmt.Sprintf("%s_%s_%s", pod.Namespace, pod.Name, interfaceName) + } else { + return + } + + logrus.Infof("Creating logical port for %s on switch %s", portName, logicalSwitch) + + if ipAddress != "" && macAddress != "" { + isStaticIP = true + } + if ipAddress != "" && macAddress == "" { + macAddress = generateMac() + isStaticIP = true + } + + if isStaticIP { + out, stderr, err = RunOVNNbctlUnix("--may-exist", "lsp-add", + logicalSwitch, portName, "--", "lsp-set-addresses", portName, + fmt.Sprintf("%s %s", macAddress, ipAddress), "--", "--if-exists", + "clear", "logical_switch_port", portName, "dynamic_addresses", "--", "set", + "logical_switch_port", portName, + "external-ids:namespace="+pod.Namespace, + "external-ids:logical_switch="+logicalSwitch, + "external-ids:pod=true") + if err != nil { + logrus.Errorf("Failed to add logical port to switch "+ + "stdout: %q, stderr: %q (%v)", + out, stderr, err) + return + } + } else { + out, stderr, err = RunOVNNbctlUnix("--wait=sb", "--", + "--may-exist", "lsp-add", logicalSwitch, portName, + "--", "lsp-set-addresses", + portName, "dynamic", "--", "set", + "logical_switch_port", portName, + "external-ids:namespace="+pod.Namespace, + "external-ids:logical_switch="+logicalSwitch, + "external-ids:pod=true") + if err != nil { + logrus.Errorf("Error while creating logical port %s "+ + "stdout: %q, stderr: %q (%v)", + portName, out, stderr, err) + return + } + } + oc.logicalPortCache[portName] = logicalSwitch + + count := 30 + for count > 0 { + if isStaticIP { + out, stderr, err = RunOVNNbctlUnix("get", + "logical_switch_port", portName, "addresses") + } else { + out, stderr, err = RunOVNNbctlUnix("get", + "logical_switch_port", portName, "dynamic_addresses") + } + if err == nil && out != "[]" { + break + } + if err != nil { + logrus.Errorf("Error while obtaining addresses for %s - %v", portName, + err) + return + } + time.Sleep(time.Second) + count-- + } + if count == 0 { + logrus.Errorf("Error while obtaining addresses for %s "+ + "stdout: %q, stderr: %q, (%v)", portName, out, stderr, err) + return + } + + // static addresses have format ["0a:00:00:00:00:01 192.168.1.3"], while + // dynamic addresses have format "0a:00:00:00:00:01 192.168.1.3". + outStr := strings.TrimLeft(out, `[`) + outStr = strings.TrimRight(outStr, `]`) + outStr = strings.Trim(outStr, `"`) + addresses := strings.Split(outStr, " ") + if len(addresses) != 2 { + logrus.Errorf("Error while obtaining addresses for %s", portName) + return + } + + if netType == "virtual" { + gatewayIP, mask, err := oc.getGatewayFromSwitch(logicalSwitch) + if err != nil { + logrus.Errorf("Error obtaining gateway address for switch %s: %s", logicalSwitch, err) + return + } + annotation = fmt.Sprintf(`{\"ip_address\":\"%s/%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], mask, addresses[0], gatewayIP) + } else { + annotation = fmt.Sprintf(`{\"ip_address\":\"%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], addresses[0], "") + } + + return annotation +} + +func (oc *Controller) findLogicalSwitch(name string) bool { + // get logical switch from OVN + output, stderr, err := RunOVNNbctlUnix("--data=bare", "--no-heading", + "--columns=name", "find", "logical_switch", "name="+name) + if err != nil { + logrus.Errorf("Error in obtaining list of logical switch, "+ + "stderr: %q, err: %v", + stderr, err) + return false + } + + if strings.Compare(name, output) == 0 { + return true + } else { + logrus.Errorf("Error finding Switch %v", name) + return false + } +} + +func (oc *Controller) addLogicalPort(pod *kapi.Pod) { + var logicalSwitch string + var ipAddress, macAddress, interfaceName, defaultGateway, netType string + + annotation := pod.Annotations["ovnNetwork"] + + if annotation != "" { + ovnNetObjs, err := parseOvnNetworkObject(annotation) + if err != nil { + logrus.Errorf("addLogicalPort : Error Parsing OvnNetwork List") + return + } + var ovnString, outStr string + ovnString = "[" + for _, net := range ovnNetObjs { + logicalSwitch = net["name"].(string) + if !oc.findLogicalSwitch(logicalSwitch) { + logrus.Errorf("Logical Switch not found") + return + } + if _, ok := net["interface"]; ok { + interfaceName = net["interface"].(string) + } else { + logrus.Errorf("Interface name must be provided") + return + } + if _, ok := net["ipAddress"]; ok { + ipAddress = net["ipAddress"].(string) + } else { + ipAddress = "" + } + if _, ok := net["macAddress"]; ok { + macAddress = net["macAddress"].(string) + } else { + macAddress = "" + } + if _, ok := net["defaultGateway"]; ok { + defaultGateway = net["defaultGateway"].(string) + } else { + defaultGateway = "false" + } + if _, ok := net["netType"]; ok { + netType = net["netType"].(string) + } else { + netType = "virtual" + } + if netType != "provider" && netType != "virtual" { + logrus.Errorf("netType is not supported") + return + } + if netType == "provider" && ipAddress == "" { + logrus.Errorf("ipAddress must be provided for netType Provider") + return + } + if netType == "provider" && defaultGateway == "true" { + logrus.Errorf("defaultGateway not supported for provider network - Use ovnNetworkRoutes to add routes") + return + } + outStr = oc.addLogicalPortWithSwitch(pod, logicalSwitch, ipAddress, macAddress, interfaceName, netType) + if outStr == "" { + return + } + last := len(outStr) - 1 + tmpString := outStr[:last] + tmpString += "," + "\\\"defaultGateway\\\":" + "\\\"" + defaultGateway + "\\\"" + tmpString += "," + "\\\"interface\\\":" + "\\\"" + interfaceName + "\\\"}" + ovnString += tmpString + ovnString += "," + } + last := len(ovnString) - 1 + ovnString = ovnString[:last] + ovnString += "]" + logrus.Debugf("ovnIfaceList - %v", ovnString) + err = oc.kube.SetAnnotationOnPod(pod, "ovnIfaceList", ovnString) + if err != nil { + logrus.Errorf("Failed to set annotation on pod %s - %v", pod.Name, err) + } + } +} diff --git a/internal/pkg/ovn/ovn_test.go b/internal/pkg/ovn/ovn_test.go index f974c30..bc33d35 100644 --- a/internal/pkg/ovn/ovn_test.go +++ b/internal/pkg/ovn/ovn_test.go @@ -13,7 +13,6 @@ import ( "ovn4nfv-k8s-plugin/internal/pkg/config" "ovn4nfv-k8s-plugin/internal/pkg/factory" ovntest "ovn4nfv-k8s-plugin/internal/pkg/testing" - "ovn4nfv-k8s-plugin/internal/pkg/util" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -70,7 +69,7 @@ var _ = Describe("Add logical Port", func() { }, } - err := util.SetExec(fexec) + err := SetExec(fexec) Expect(err).NotTo(HaveOccurred()) fakeClient := &fake.Clientset{} @@ -138,7 +137,7 @@ var _ = Describe("Add logical Port", func() { return fmt.Sprintf("/fake-bin/%s", file), nil }, } - err := util.SetExec(fexec) + err := SetExec(fexec) Expect(err).NotTo(HaveOccurred()) fakeClient := &fake.Clientset{} diff --git a/internal/pkg/ovn/pods.go b/internal/pkg/ovn/pods.go deleted file mode 100644 index 6db6b43..0000000 --- a/internal/pkg/ovn/pods.go +++ /dev/null @@ -1,292 +0,0 @@ -package ovn - -import ( - "fmt" - "strings" - "time" - - "github.com/sirupsen/logrus" - kapi "k8s.io/api/core/v1" - "ovn4nfv-k8s-plugin/internal/pkg/util" -) - -func (oc *Controller) syncPods(pods []interface{}) { -} -func (oc *Controller) getGatewayFromSwitch(logicalSwitch string) (string, string, error) { - var gatewayIPMaskStr, stderr string - var ok bool - var err error - logrus.Infof("getGatewayFromSwitch: %s", logicalSwitch) - if gatewayIPMaskStr, ok = oc.gatewayCache[logicalSwitch]; !ok { - gatewayIPMaskStr, stderr, err = util.RunOVNNbctlUnix("--if-exists", - "get", "logical_switch", logicalSwitch, - "external_ids:gateway_ip") - if err != nil { - logrus.Errorf("Failed to get gateway IP: %s, stderr: %q, %v", - gatewayIPMaskStr, stderr, err) - return "", "", err - } - if gatewayIPMaskStr == "" { - return "", "", fmt.Errorf("Empty gateway IP in logical switch %s", - logicalSwitch) - } - oc.gatewayCache[logicalSwitch] = gatewayIPMaskStr - } - gatewayIPMask := strings.Split(gatewayIPMaskStr, "/") - if len(gatewayIPMask) != 2 { - return "", "", fmt.Errorf("Failed to get IP and Mask from gateway CIDR: %s", - gatewayIPMaskStr) - } - gatewayIP := gatewayIPMask[0] - mask := gatewayIPMask[1] - return gatewayIP, mask, nil -} - -func (oc *Controller) deleteLogicalPort(pod *kapi.Pod) { - - if pod.Spec.HostNetwork { - return - } - - logrus.Infof("Deleting pod: %s", pod.Name) - logicalPort := fmt.Sprintf("%s_%s", pod.Namespace, pod.Name) - - // get the list of logical ports from OVN - output, stderr, err := util.RunOVNNbctlUnix("--data=bare", "--no-heading", - "--columns=name", "find", "logical_switch_port", "external_ids:pod=true") - if err != nil { - logrus.Errorf("Error in obtaining list of logical ports, "+ - "stderr: %q, err: %v", - stderr, err) - return - } - logrus.Infof("Exising Ports : %s. ", output) - existingLogicalPorts := strings.Fields(output) - for _, existingPort := range existingLogicalPorts { - if strings.Contains(existingPort, logicalPort) { - // found, delete this logical port - logrus.Infof("Deleting: %s. ", existingPort) - out, stderr, err := util.RunOVNNbctlUnix("--if-exists", "lsp-del", - existingPort) - if err != nil { - logrus.Errorf("Error in deleting pod's logical port "+ - "stdout: %q, stderr: %q err: %v", - out, stderr, err) - } else { - delete(oc.logicalPortCache, existingPort) - } - } - } - return -} - -func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipAddress, macAddress, interfaceName, netType string) (annotation string) { - var out, stderr string - var err error - var isStaticIP bool - if pod.Spec.HostNetwork { - return - } - - if !oc.logicalSwitchCache[logicalSwitch] { - oc.logicalSwitchCache[logicalSwitch] = true - } - var portName string - if interfaceName != "" { - portName = fmt.Sprintf("%s_%s_%s", pod.Namespace, pod.Name, interfaceName) - } else { - return - } - - logrus.Infof("Creating logical port for %s on switch %s", portName, logicalSwitch) - - if ipAddress != "" && macAddress != "" { - isStaticIP = true - } - if ipAddress != "" && macAddress == "" { - macAddress = util.GenerateMac() - isStaticIP = true - } - - if isStaticIP { - out, stderr, err = util.RunOVNNbctlUnix("--may-exist", "lsp-add", - logicalSwitch, portName, "--", "lsp-set-addresses", portName, - fmt.Sprintf("%s %s", macAddress, ipAddress), "--", "--if-exists", - "clear", "logical_switch_port", portName, "dynamic_addresses", "--", "set", - "logical_switch_port", portName, - "external-ids:namespace="+pod.Namespace, - "external-ids:logical_switch="+logicalSwitch, - "external-ids:pod=true") - if err != nil { - logrus.Errorf("Failed to add logical port to switch "+ - "stdout: %q, stderr: %q (%v)", - out, stderr, err) - return - } - } else { - out, stderr, err = util.RunOVNNbctlUnix("--wait=sb", "--", - "--may-exist", "lsp-add", logicalSwitch, portName, - "--", "lsp-set-addresses", - portName, "dynamic", "--", "set", - "logical_switch_port", portName, - "external-ids:namespace="+pod.Namespace, - "external-ids:logical_switch="+logicalSwitch, - "external-ids:pod=true") - if err != nil { - logrus.Errorf("Error while creating logical port %s "+ - "stdout: %q, stderr: %q (%v)", - portName, out, stderr, err) - return - } - } - oc.logicalPortCache[portName] = logicalSwitch - - count := 30 - for count > 0 { - if isStaticIP { - out, stderr, err = util.RunOVNNbctlUnix("get", - "logical_switch_port", portName, "addresses") - } else { - out, stderr, err = util.RunOVNNbctlUnix("get", - "logical_switch_port", portName, "dynamic_addresses") - } - if err == nil && out != "[]" { - break - } - if err != nil { - logrus.Errorf("Error while obtaining addresses for %s - %v", portName, - err) - return - } - time.Sleep(time.Second) - count-- - } - if count == 0 { - logrus.Errorf("Error while obtaining addresses for %s "+ - "stdout: %q, stderr: %q, (%v)", portName, out, stderr, err) - return - } - - // static addresses have format ["0a:00:00:00:00:01 192.168.1.3"], while - // dynamic addresses have format "0a:00:00:00:00:01 192.168.1.3". - outStr := strings.TrimLeft(out, `[`) - outStr = strings.TrimRight(outStr, `]`) - outStr = strings.Trim(outStr, `"`) - addresses := strings.Split(outStr, " ") - if len(addresses) != 2 { - logrus.Errorf("Error while obtaining addresses for %s", portName) - return - } - - if netType == "virtual" { - gatewayIP, mask, err := oc.getGatewayFromSwitch(logicalSwitch) - if err != nil { - logrus.Errorf("Error obtaining gateway address for switch %s: %s", logicalSwitch, err) - return - } - annotation = fmt.Sprintf(`{\"ip_address\":\"%s/%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], mask, addresses[0], gatewayIP) - } else { - annotation = fmt.Sprintf(`{\"ip_address\":\"%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], addresses[0], "") - } - - return annotation -} - -func findLogicalSwitch(name string) bool { - // get logical switch from OVN - output, stderr, err := util.RunOVNNbctlUnix("--data=bare", "--no-heading", - "--columns=name", "find", "logical_switch", "name="+name) - if err != nil { - logrus.Errorf("Error in obtaining list of logical switch, "+ - "stderr: %q, err: %v", - stderr, err) - return false - } - - if strings.Compare(name, output) == 0 { - return true - } else { - logrus.Errorf("Error finding Switch %v", name) - return false - } -} - -func (oc *Controller) addLogicalPort(pod *kapi.Pod) { - var logicalSwitch string - var ipAddress, macAddress, interfaceName, defaultGateway, netType string - - annotation := pod.Annotations["ovnNetwork"] - - if annotation != "" { - ovnNetObjs, err := oc.parseOvnNetworkObject(annotation) - if err != nil { - logrus.Errorf("addLogicalPort : Error Parsing OvnNetwork List") - return - } - var ovnString, outStr string - ovnString = "[" - for _, net := range ovnNetObjs { - logicalSwitch = net["name"].(string) - if !findLogicalSwitch(logicalSwitch) { - logrus.Errorf("Logical Switch not found") - return - } - if _, ok := net["interface"]; ok { - interfaceName = net["interface"].(string) - } else { - logrus.Errorf("Interface name must be provided") - return - } - if _, ok := net["ipAddress"]; ok { - ipAddress = net["ipAddress"].(string) - } else { - ipAddress = "" - } - if _, ok := net["macAddress"]; ok { - macAddress = net["macAddress"].(string) - } else { - macAddress = "" - } - if _, ok := net["defaultGateway"]; ok { - defaultGateway = net["defaultGateway"].(string) - } else { - defaultGateway = "false" - } - if _, ok := net["netType"]; ok { - netType = net["netType"].(string) - } else { - netType = "virtual" - } - if netType != "provider" && netType != "virtual" { - logrus.Errorf("netType is not supported") - return - } - if netType == "provider" && ipAddress == "" { - logrus.Errorf("ipAddress must be provided for netType Provider") - return - } - if netType == "provider" && defaultGateway == "true" { - logrus.Errorf("defaultGateway not supported for provider network - Use ovnNetworkRoutes to add routes") - return - } - outStr = oc.addLogicalPortWithSwitch(pod, logicalSwitch, ipAddress, macAddress, interfaceName, netType) - if outStr == "" { - return - } - last := len(outStr) - 1 - tmpString := outStr[:last] - tmpString += "," + "\\\"defaultGateway\\\":" + "\\\"" + defaultGateway + "\\\"" - tmpString += "," + "\\\"interface\\\":" + "\\\"" + interfaceName + "\\\"}" - ovnString += tmpString - ovnString += "," - } - last := len(ovnString) - 1 - ovnString = ovnString[:last] - ovnString += "]" - logrus.Debugf("ovnIfaceList - %v", ovnString) - err = oc.kube.SetAnnotationOnPod(pod, "ovnIfaceList", ovnString) - if err != nil { - logrus.Errorf("Failed to set annotation on pod %s - %v", pod.Name, err) - } - } -} diff --git a/internal/pkg/ovn/router.go b/internal/pkg/ovn/router.go deleted file mode 100644 index d98c463..0000000 --- a/internal/pkg/ovn/router.go +++ /dev/null @@ -1,50 +0,0 @@ -package ovn - -import ( - "github.com/sirupsen/logrus" - "ovn4nfv-k8s-plugin/internal/pkg/util" -) - -func SetupMaster(name string) error { - - // Make sure br-int is created. - stdout, stderr, err := util.RunOVSVsctl("--", "--may-exist", "add-br", "br-int") - if err != nil { - logrus.Errorf("Failed to create br-int, stdout: %q, stderr: %q, error: %v", stdout, stderr, err) - return err - } - // Create a single common distributed router for the cluster. - stdout, stderr, err = util.RunOVNNbctlUnix("--", "--may-exist", "lr-add", name, "--", "set", "logical_router", name, "external_ids:ovn4nfv-cluster-router=yes") - if err != nil { - logrus.Errorf("Failed to create a single common distributed router for the cluster, stdout: %q, stderr: %q, error: %v", stdout, stderr, err) - return err - } - // Create a logical switch called "ovn4nfv-join" that will be used to connect gateway routers to the distributed router. - // The "ovn4nfv-join" will be allocated IP addresses in the range 100.64.1.0/24. - stdout, stderr, err = util.RunOVNNbctlUnix("--may-exist", "ls-add", "ovn4nfv-join") - if err != nil { - logrus.Errorf("Failed to create logical switch called \"ovn4nfv-join\", stdout: %q, stderr: %q, error: %v", stdout, stderr, err) - return err - } - // Connect the distributed router to "ovn4nfv-join". - routerMac, stderr, err := util.RunOVNNbctlUnix("--if-exist", "get", "logical_router_port", "rtoj-"+name, "mac") - if err != nil { - logrus.Errorf("Failed to get logical router port rtoj-%v, stderr: %q, error: %v", name, stderr, err) - return err - } - if routerMac == "" { - routerMac = util.GenerateMac() - stdout, stderr, err = util.RunOVNNbctlUnix("--", "--may-exist", "lrp-add", name, "rtoj-"+name, routerMac, "100.64.1.1/24", "--", "set", "logical_router_port", "rtoj-"+name, "external_ids:connect_to_ovn4nfvjoin=yes") - if err != nil { - logrus.Errorf("Failed to add logical router port rtoj-%v, stdout: %q, stderr: %q, error: %v", name, stdout, stderr, err) - return err - } - } - // Connect the switch "ovn4nfv-join" to the router. - stdout, stderr, err = util.RunOVNNbctlUnix("--", "--may-exist", "lsp-add", "ovn4nfv-join", "jtor-"+name, "--", "set", "logical_switch_port", "jtor-"+name, "type=router", "options:router-port=rtoj-"+name, "addresses="+"\""+routerMac+"\"") - if err != nil { - logrus.Errorf("Failed to add logical switch port to logical router, stdout: %q, stderr: %q, error: %v", stdout, stderr, err) - return err - } - return nil -} diff --git a/internal/pkg/ovn/utils.go b/internal/pkg/ovn/utils.go new file mode 100644 index 0000000..1700bf8 --- /dev/null +++ b/internal/pkg/ovn/utils.go @@ -0,0 +1,120 @@ +package ovn + +import ( + "bytes" + "fmt" + "strings" + "time" + "unicode" + + "github.com/sirupsen/logrus" + kexec "k8s.io/utils/exec" +) + +const ( + ovsCommandTimeout = 15 + ovsVsctlCommand = "ovs-vsctl" + ovsOfctlCommand = "ovs-ofctl" + ovnNbctlCommand = "ovn-nbctl" + ipCommand = "ip" +) + +// Exec runs various OVN and OVS utilities +type execHelper struct { + exec kexec.Interface + ofctlPath string + vsctlPath string + nbctlPath string + ipPath string +} + +var runner *execHelper + +// SetExec validates executable paths and saves the given exec interface +// to be used for running various OVS and OVN utilites +func SetExec(exec kexec.Interface) error { + var err error + + runner = &execHelper{exec: exec} + runner.ofctlPath, err = exec.LookPath(ovsOfctlCommand) + if err != nil { + return err + } + runner.vsctlPath, err = exec.LookPath(ovsVsctlCommand) + if err != nil { + return err + } + runner.nbctlPath, err = exec.LookPath(ovnNbctlCommand) + if err != nil { + return err + } + runner.ipPath, err = exec.LookPath(ipCommand) + if err != nil { + return err + } + return nil +} + +// Run the ovn-ctl command and retry if "Connection refused" +// poll waitng for service to become available +func runOVNretry(cmdPath string, args ...string) (*bytes.Buffer, *bytes.Buffer, error) { + + retriesLeft := 200 + for { + stdout, stderr, err := run(cmdPath, args...) + if err == nil { + return stdout, stderr, err + } + + // Connection refused + // Master may not be up so keep trying + if strings.Contains(stderr.String(), "Connection refused") { + if retriesLeft == 0 { + return stdout, stderr, err + } + retriesLeft-- + time.Sleep(2 * time.Second) + } else { + // Some other problem for caller to handle + return stdout, stderr, err + } + } +} + +func run(cmdPath string, args ...string) (*bytes.Buffer, *bytes.Buffer, error) { + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + cmd := runner.exec.Command(cmdPath, args...) + cmd.SetStdout(stdout) + cmd.SetStderr(stderr) + logrus.Debugf("exec: %s %s", cmdPath, strings.Join(args, " ")) + err := cmd.Run() + if err != nil { + logrus.Debugf("exec: %s %s => %v", cmdPath, strings.Join(args, " "), err) + } + return stdout, stderr, err +} + +// RunOVSVsctl runs a command via ovs-vsctl. +func RunOVSVsctlUnix(args ...string) (string, string, error) { + cmdArgs := []string{fmt.Sprintf("--timeout=%d", ovsCommandTimeout)} + cmdArgs = append(cmdArgs, args...) + stdout, stderr, err := run(runner.vsctlPath, cmdArgs...) + return strings.Trim(strings.TrimSpace(stdout.String()), "\""), stderr.String(), err +} + +// RunOVNNbctlUnix runs command via ovn-nbctl, with ovn-nbctl using the unix +// domain sockets to connect to the ovsdb-server backing the OVN NB database. +func RunOVNNbctlUnix(args ...string) (string, string, error) { + cmdArgs := []string{fmt.Sprintf("--timeout=%d", ovsCommandTimeout)} + cmdArgs = append(cmdArgs, args...) + stdout, stderr, err := runOVNretry(runner.nbctlPath, cmdArgs...) + return strings.Trim(strings.TrimFunc(stdout.String(), unicode.IsSpace), "\""), + stderr.String(), err +} + +// RunIP runs a command via the iproute2 "ip" utility +func RunIP(args ...string) (string, string, error) { + stdout, stderr, err := run(runner.ipPath, args...) + return strings.TrimSpace(stdout.String()), stderr.String(), err +} |