diff options
| -rw-r--r-- | README.rst | 164 | ||||
| -rw-r--r-- | cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go | 2 | ||||
| -rw-r--r-- | internal/pkg/ovn/ovn_test.go | 80 | ||||
| -rw-r--r-- | internal/pkg/ovn/pods.go | 53 |
4 files changed, 221 insertions, 78 deletions
@@ -17,7 +17,7 @@ Problem statement ----------------- Networking applications are of three types - Management applications, -Control plane applications and data plane applications. Management +Control plane applications and data plane applications. Management and control plane applications are similar to Enterprise applications, but data plane applications different in following aspects: @@ -66,9 +66,8 @@ NFV workloads can be, New Proposal ------------ -A new plugin addressing the below requirements, - -- For networking workloads as well typical application workloads +A new plugin addresses the below requirements, for networking +workloads as well typical application workloads - Multi-interface support - Multi-IP address support - Dynamic creation of virtual networks @@ -83,74 +82,122 @@ native support for virtual network abstractions, such as virtual L2 and L3 overlays and security groups. Services such as DHCP are also desirable features. Just like OVS, OVN’s design goal is to have a production quality implementation that can operate at significant -scale. +scale. + +**OVN4NFVK8s Plugin development** + +ovn-kubernetes_ plugin is part of OVN project which provides OVN +integration with Kubernetes but doesn't address the requirements +as given above. To meet those requirements like multiple interfaces, +IPs, dynamic creation of virtual networks, etc., OVN4NFVK8s plugin is +created. It assumes that it will be used in conjuction with Multus_ +or other similar CNI which allows for the co-existance of multiple +CNI plugins in runtime. This plugin assumes that the first interface +in a Pod is provided by some other Plugin/CNI like Flannel or even +OVN-Kubernetes. It is only responsible to add multiple interfaces +based on the Pod annotations. The code is based on ovn-kubernetes_. + + +.. note:: + + This plugin is currently tested to work with Multus and Flannel + providing the first network interface. + +To meet the requirement of multiple interfaces and IP's per pod, +a Pod annotation like below is required when working with Multus: + + +.. code-block:: yaml + + + annotations: + k8s.v1.cni.cncf.io/networks: '[{ "name": "ovn-networkobj"}]' + ovnNetwork '[ + { "name": <name of OVN Logical Switch>, "interfaceRequest": "eth1" }, + { "name": <name of OVN Logical Switch>, "interfaceRequest": "eth2" } + ]' + +Based on these annotations watcher service in OVN4NFVK8s plugin assumes +logical switch is already present. Dynamic IP addresses are assigned +(static IP's also supported) and annotations are updated. + +When the Pod is initialized on a node, OVN4NFVK8s CNI creates multiple +interfaces and assigns IP addresses for the pod based on the annotations. + +**Multus Configuration** +Multus CRD definition for OVN: + +.. code-block:: yaml + + apiVersion: "k8s.cni.cncf.io/v1" + kind: NetworkAttachmentDefinition + metadata: + name: ovn-networkobj + spec: + config: '{ + "cniVersion": "0.3.1", + "name": "ovn4nfv-k8s-plugin", + "type": "ovn4nfvk8s-cni" + }' + +Please refer to Multus_ for details about how this configuration is used + +CNI configuration file for Multus with Flannel: + +.. code-block:: yaml + + { + "type": "multus", + "name": "multus-cni", + "cniVersion": "0.3.1", + "kubeconfig": "/etc/kubernetes/admin.conf", + "delegates": [ + { + "type": "flannel", + "cniVersion": "0.3.1", + "masterplugin": true, + "delegate": { + "isDefaultGateway": false + } + } + ] + } -**K8S-OVN4NFV Plugin development** +Refer Kubernetes_ documentation for the order in which CNI configurations +are applied. -Some code and ideas are being taken from ovn-kubernetes_ plugin -work that was done as part of OVN project. Due to good number of -changes, it is a new plugin with its own code base. This plugin -assumes that the first interface in a Pod is provided by some other -Plugin/CNI like Flannel or even OVN-Kubernetes and this plugin is -only responsible to add multiple interfaces based on the Pod -annotations. This plugin is currently tested to work with Multus as -CNI and Flannel as first interface. -Its functionality is divided into to following: +**Build** -- Initialization: +For building the project: - - Register itself as watcher to K8S API Server to receive POD events - and service events. - - Creates a distributed router - - Creates gateway - - Creates a logical switch to connect distributed router with - Gateway. - - Creates a subnet between distributed router & Gateway. - - Assigns first two IP addresses of the subnet to router and - Gateway. - - Created router port and gateway port as part of assigning IP - address and MAC addresses. +.. code-block:: bash -- Watcher: + cd ovn4nfv-k8s-plugin + make - - Upon POD bring up event - - Checks the annotations specific to OVN. - - For each network on which POD is going to be brought up - - Validates whether the logical switch is already present. If not, - it is considered as error. - - If IP address and MAC addresses are not static, it asks OVN to - assign IP and MAC address. - - Collects all IP addresses/MAC addresses assigned. Puts them as - annotations (dynamic information) for that POD. +This will output two files ovn4nfvk8s and ovn4nfvk8s-cni which are the plugin + and CNI binaries respectively. - - Upon POD deletion event +ovn4nfvk8s plugin requires some configuration at start up. - - Returns the IP address and MAC address back to OVN pool. +Example configuration file (default location/etc/openvswitch/ovn4nfv_k8s.conf) -- OVN CNI +.. code-block:: yaml -This is present in every minion node. CNI is expected to be called -once for all OVN networks either Kubelet directly or via Multus. + [logging] + loglevel=5 + logfile=/var/log/openvswitch/ovn4k8s.log - - Add: + [cni] + conf-dir=/etc/cni/net.d + plugin=ovn4nfvk8s-cni - - Wait for annotations to be filled up by the watcher. From - annotations, it knows set of IP Address, MAC address and Routes - to be added. - - Using network APIs for each element in the set: - - Creates veth pair. - - Assigns the IP address and MAC address to one end of veth pair. - Other end veth pair is assigned to br-int. - - Creates routes based on the route list provided in annotations. + [kubernetes] + kubeconfig=/etc/kubernetes/admin.conf - - If isDefaultRoute is set in annotations, it creates default route - using this veth. - - Delete - - Removes veth pair. - - Removes routes. **Figure** @@ -185,12 +232,13 @@ once for all OVN networks either Kubelet directly or via Multus. +--------------------+ - Complete Architecture can be found in ovn-kubernetes documentation at github **References** -.. _ovn-kubernetes: https://wiki.opnfv.org/display/OV/K8S+OVN+NFV+Plugin +.. _ovn-kubernetes: https://github.com/openvswitch/ovn-kubernetes +.. _Multus: https://github.com/intel/multus-cni +.. _Kubernetes: https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/ **Authors/Contributors** diff --git a/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go b/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go index 73ef887..7c90d02 100644 --- a/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go +++ b/cmd/ovn4nfvk8s-cni/ovn4nfvk8s-cni.go @@ -127,7 +127,7 @@ func addMultipleInterfaces(args *skel.CmdArgs, ovnAnnotation, namespace, podName gatewayIP := ovnNet["gateway_ip"] defaultGateway := ovnNet["defaultGateway"] - if ipAddress == "" || macAddress == "" || gatewayIP == "" { + if ipAddress == "" || macAddress == "" { logrus.Errorf("failed in pod annotation key extract") return nil } diff --git a/internal/pkg/ovn/ovn_test.go b/internal/pkg/ovn/ovn_test.go index 99a96ae..312a26a 100644 --- a/internal/pkg/ovn/ovn_test.go +++ b/internal/pkg/ovn/ovn_test.go @@ -55,13 +55,13 @@ var _ = Describe("Add logical Port", func() { }) fakeCmds = ovntest.AddFakeCmd(fakeCmds, &ovntest.ExpectedCmd{ - Cmd: "ovn-nbctl --timeout=15 --if-exists get logical_switch " + netName + " external_ids:gateway_ip", - Output: gwCIDR, - }) - fakeCmds = ovntest.AddFakeCmd(fakeCmds, &ovntest.ExpectedCmd{ Cmd: "ovn-nbctl --timeout=15 get logical_switch_port " + portName + " dynamic_addresses", Output: macIPAddress, }) + fakeCmds = ovntest.AddFakeCmd(fakeCmds, &ovntest.ExpectedCmd{ + Cmd: "ovn-nbctl --timeout=15 --if-exists get logical_switch " + netName + " external_ids:gateway_ip", + Output: gwCIDR, + }) fexec := &fakeexec.FakeExec{ CommandScript: fakeCmds, @@ -100,7 +100,7 @@ var _ = Describe("Add logical Port", func() { ) ovnController.addLogicalPort(&okPod) - _, _ = ovnController.kube.GetAnnotationsOnPod("", "ok") + Expect(fexec.CommandCalls).To(Equal(len(fakeCmds))) return nil } @@ -108,4 +108,74 @@ var _ = Describe("Add logical Port", func() { err := app.Run([]string{app.Name}) Expect(err).NotTo(HaveOccurred()) }) + + It("tests Pod provider", func() { + app.Action = func(ctx *cli.Context) error { + const ( + gwIP string = "10.1.1.1" + gwCIDR string = gwIP + "/24" + netName string = "ovn-prot-net" + portName string = "_ok_net0" + macIPAddress string = "0a:00:00:00:00:01 192.168.1.3/24" + ) + fakeCmds := ovntest.AddFakeCmd(nil, &ovntest.ExpectedCmd{ + Cmd: "ovn-nbctl --timeout=15 --data=bare --no-heading --columns=name find logical_switch " + "name=" + netName, + Output: netName, + }) + + fakeCmds = ovntest.AddFakeCmdsNoOutputNoError(fakeCmds, []string{ + "ovn-nbctl --timeout=15 --may-exist lsp-add " + netName + " " + portName + " -- lsp-set-addresses " + portName + " " + macIPAddress + " -- --if-exists clear logical_switch_port " + portName + " dynamic_addresses", + }) + + fakeCmds = ovntest.AddFakeCmd(fakeCmds, &ovntest.ExpectedCmd{ + Cmd: "ovn-nbctl --timeout=15 get logical_switch_port " + portName + " addresses", + Output: macIPAddress, + }) + + fexec := &fakeexec.FakeExec{ + CommandScript: fakeCmds, + LookPathFunc: func(file string) (string, error) { + return fmt.Sprintf("/fake-bin/%s", file), nil + }, + } + err := util.SetExec(fexec) + Expect(err).NotTo(HaveOccurred()) + + _, err = config.InitConfig(ctx, fexec, nil) + Expect(err).NotTo(HaveOccurred()) + + fakeClient := &fake.Clientset{} + var fakeWatchFactory factory.WatchFactory + + ovnController := NewOvnController(fakeClient, &fakeWatchFactory) + var ( + okPod = v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "ok", + Annotations: map[string]string{"ovnNetwork": "[{ \"name\": \"ovn-prot-net\", \"interface\": \"net0\", \"netType\": \"provider\", \"ipAddress\": \"192.168.1.3/24\", \"macAddress\": \"0a:00:00:00:00:01\" }]"}, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "by-name", + }, + {}, + }, + }, + } + ) + ovnController.addLogicalPort(&okPod) + Expect(fexec.CommandCalls).To(Equal(len(fakeCmds))) + + return nil + } + + err := app.Run([]string{app.Name}) + Expect(err).NotTo(HaveOccurred()) + }) + }) diff --git a/internal/pkg/ovn/pods.go b/internal/pkg/ovn/pods.go index cc3d459..fcd258e 100644 --- a/internal/pkg/ovn/pods.go +++ b/internal/pkg/ovn/pods.go @@ -80,7 +80,7 @@ func (oc *Controller) deleteLogicalPort(pod *kapi.Pod) { return } -func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipAddress, macAddress, interfaceName string) (annotation string) { +func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipAddress, macAddress, interfaceName, netType string) (annotation string) { var out, stderr string var err error var isStaticIP bool @@ -112,7 +112,11 @@ func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipA out, stderr, err = util.RunOVNNbctlUnix("--may-exist", "lsp-add", logicalSwitch, portName, "--", "lsp-set-addresses", portName, fmt.Sprintf("%s %s", macAddress, ipAddress), "--", "--if-exists", - "clear", "logical_switch_port", portName, "dynamic_addresses") + "clear", "logical_switch_port", portName, "dynamic_addresses", "--", "set", + "logical_switch_port", portName, + "external-ids:namespace="+pod.Namespace, + "external-ids:logical_switch="+logicalSwitch, + "external-ids:pod=true") if err != nil { logrus.Errorf("Failed to add logical port to switch "+ "stdout: %q, stderr: %q (%v)", @@ -136,11 +140,6 @@ func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipA } } oc.logicalPortCache[portName] = logicalSwitch - gatewayIP, mask, err := oc.getGatewayFromSwitch(logicalSwitch) - if err != nil { - logrus.Errorf("Error obtaining gateway address for switch %s: %s", logicalSwitch, err) - return - } count := 30 for count > 0 { @@ -178,7 +177,18 @@ func (oc *Controller) addLogicalPortWithSwitch(pod *kapi.Pod, logicalSwitch, ipA logrus.Errorf("Error while obtaining addresses for %s", portName) return } - annotation = fmt.Sprintf(`{\"ip_address\":\"%s/%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], mask, addresses[0], gatewayIP) + + if netType == "virtual" { + gatewayIP, mask, err := oc.getGatewayFromSwitch(logicalSwitch) + if err != nil { + logrus.Errorf("Error obtaining gateway address for switch %s: %s", logicalSwitch, err) + return + } + annotation = fmt.Sprintf(`{\"ip_address\":\"%s/%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], mask, addresses[0], gatewayIP) + } else { + annotation = fmt.Sprintf(`{\"ip_address\":\"%s\", \"mac_address\":\"%s\", \"gateway_ip\": \"%s\"}`, addresses[1], addresses[0], "") + } + return annotation } @@ -203,7 +213,7 @@ func findLogicalSwitch(name string) bool { func (oc *Controller) addLogicalPort(pod *kapi.Pod) { var logicalSwitch string - var ipAddress, macAddress, interfaceName, defaultGateway string + var ipAddress, macAddress, interfaceName, defaultGateway, netType string annotation := pod.Annotations["ovnNetwork"] @@ -217,10 +227,15 @@ func (oc *Controller) addLogicalPort(pod *kapi.Pod) { ovnString = "[" for _, net := range ovnNetObjs { logicalSwitch = net["name"].(string) + if !findLogicalSwitch(logicalSwitch) { + logrus.Errorf("Logical Switch not found") + return + } if _, ok := net["interface"]; ok { interfaceName = net["interface"].(string) } else { - interfaceName = "" + logrus.Errorf("Interface name must be provided") + return } if _, ok := net["ipAddress"]; ok { ipAddress = net["ipAddress"].(string) @@ -237,14 +252,24 @@ func (oc *Controller) addLogicalPort(pod *kapi.Pod) { } else { defaultGateway = "false" } - if !findLogicalSwitch(logicalSwitch) { + if _, ok := net["netType"]; ok { + netType = net["netType"].(string) + } else { + netType = "virtual" + } + if netType != "provider" && netType != "virtual" { + logrus.Errorf("netType is not supported") return } - if interfaceName == "" { - logrus.Errorf("Interface name must be provided") + if netType == "provider" && ipAddress == "" { + logrus.Errorf("ipAddress must be provided for netType Provider") + return + } + if netType == "provider" && defaultGateway == "true" { + logrus.Errorf("defaultGateway not supported for provider network - Use ovnNetworkRoutes to add routes") return } - outStr = oc.addLogicalPortWithSwitch(pod, logicalSwitch, ipAddress, macAddress, interfaceName) + outStr = oc.addLogicalPortWithSwitch(pod, logicalSwitch, ipAddress, macAddress, interfaceName, netType) if outStr == "" { return } |