summaryrefslogtreecommitdiffstats
path: root/kernel/Documentation/driver-model/porting.txt
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/Documentation/driver-model/porting.txt')
-rw-r--r--kernel/Documentation/driver-model/porting.txt445
1 files changed, 445 insertions, 0 deletions
diff --git a/kernel/Documentation/driver-model/porting.txt b/kernel/Documentation/driver-model/porting.txt
new file mode 100644
index 000000000..92d86f727
--- /dev/null
+++ b/kernel/Documentation/driver-model/porting.txt
@@ -0,0 +1,445 @@
+
+Porting Drivers to the New Driver Model
+
+Patrick Mochel
+
+7 January 2003
+
+
+Overview
+
+Please refer to Documentation/driver-model/*.txt for definitions of
+various driver types and concepts.
+
+Most of the work of porting devices drivers to the new model happens
+at the bus driver layer. This was intentional, to minimize the
+negative effect on kernel drivers, and to allow a gradual transition
+of bus drivers.
+
+In a nutshell, the driver model consists of a set of objects that can
+be embedded in larger, bus-specific objects. Fields in these generic
+objects can replace fields in the bus-specific objects.
+
+The generic objects must be registered with the driver model core. By
+doing so, they will exported via the sysfs filesystem. sysfs can be
+mounted by doing
+
+ # mount -t sysfs sysfs /sys
+
+
+
+The Process
+
+Step 0: Read include/linux/device.h for object and function definitions.
+
+Step 1: Registering the bus driver.
+
+
+- Define a struct bus_type for the bus driver.
+
+struct bus_type pci_bus_type = {
+ .name = "pci",
+};
+
+
+- Register the bus type.
+ This should be done in the initialization function for the bus type,
+ which is usually the module_init(), or equivalent, function.
+
+static int __init pci_driver_init(void)
+{
+ return bus_register(&pci_bus_type);
+}
+
+subsys_initcall(pci_driver_init);
+
+
+ The bus type may be unregistered (if the bus driver may be compiled
+ as a module) by doing:
+
+ bus_unregister(&pci_bus_type);
+
+
+- Export the bus type for others to use.
+
+ Other code may wish to reference the bus type, so declare it in a
+ shared header file and export the symbol.
+
+From include/linux/pci.h:
+
+extern struct bus_type pci_bus_type;
+
+
+From file the above code appears in:
+
+EXPORT_SYMBOL(pci_bus_type);
+
+
+
+- This will cause the bus to show up in /sys/bus/pci/ with two
+ subdirectories: 'devices' and 'drivers'.
+
+# tree -d /sys/bus/pci/
+/sys/bus/pci/
+|-- devices
+`-- drivers
+
+
+
+Step 2: Registering Devices.
+
+struct device represents a single device. It mainly contains metadata
+describing the relationship the device has to other entities.
+
+
+- Embed a struct device in the bus-specific device type.
+
+
+struct pci_dev {
+ ...
+ struct device dev; /* Generic device interface */
+ ...
+};
+
+ It is recommended that the generic device not be the first item in
+ the struct to discourage programmers from doing mindless casts
+ between the object types. Instead macros, or inline functions,
+ should be created to convert from the generic object type.
+
+
+#define to_pci_dev(n) container_of(n, struct pci_dev, dev)
+
+or
+
+static inline struct pci_dev * to_pci_dev(struct kobject * kobj)
+{
+ return container_of(n, struct pci_dev, dev);
+}
+
+ This allows the compiler to verify type-safety of the operations
+ that are performed (which is Good).
+
+
+- Initialize the device on registration.
+
+ When devices are discovered or registered with the bus type, the
+ bus driver should initialize the generic device. The most important
+ things to initialize are the bus_id, parent, and bus fields.
+
+ The bus_id is an ASCII string that contains the device's address on
+ the bus. The format of this string is bus-specific. This is
+ necessary for representing devices in sysfs.
+
+ parent is the physical parent of the device. It is important that
+ the bus driver sets this field correctly.
+
+ The driver model maintains an ordered list of devices that it uses
+ for power management. This list must be in order to guarantee that
+ devices are shutdown before their physical parents, and vice versa.
+ The order of this list is determined by the parent of registered
+ devices.
+
+ Also, the location of the device's sysfs directory depends on a
+ device's parent. sysfs exports a directory structure that mirrors
+ the device hierarchy. Accurately setting the parent guarantees that
+ sysfs will accurately represent the hierarchy.
+
+ The device's bus field is a pointer to the bus type the device
+ belongs to. This should be set to the bus_type that was declared
+ and initialized before.
+
+ Optionally, the bus driver may set the device's name and release
+ fields.
+
+ The name field is an ASCII string describing the device, like
+
+ "ATI Technologies Inc Radeon QD"
+
+ The release field is a callback that the driver model core calls
+ when the device has been removed, and all references to it have
+ been released. More on this in a moment.
+
+
+- Register the device.
+
+ Once the generic device has been initialized, it can be registered
+ with the driver model core by doing:
+
+ device_register(&dev->dev);
+
+ It can later be unregistered by doing:
+
+ device_unregister(&dev->dev);
+
+ This should happen on buses that support hotpluggable devices.
+ If a bus driver unregisters a device, it should not immediately free
+ it. It should instead wait for the driver model core to call the
+ device's release method, then free the bus-specific object.
+ (There may be other code that is currently referencing the device
+ structure, and it would be rude to free the device while that is
+ happening).
+
+
+ When the device is registered, a directory in sysfs is created.
+ The PCI tree in sysfs looks like:
+
+/sys/devices/pci0/
+|-- 00:00.0
+|-- 00:01.0
+| `-- 01:00.0
+|-- 00:02.0
+| `-- 02:1f.0
+| `-- 03:00.0
+|-- 00:1e.0
+| `-- 04:04.0
+|-- 00:1f.0
+|-- 00:1f.1
+| |-- ide0
+| | |-- 0.0
+| | `-- 0.1
+| `-- ide1
+| `-- 1.0
+|-- 00:1f.2
+|-- 00:1f.3
+`-- 00:1f.5
+
+ Also, symlinks are created in the bus's 'devices' directory
+ that point to the device's directory in the physical hierarchy.
+
+/sys/bus/pci/devices/
+|-- 00:00.0 -> ../../../devices/pci0/00:00.0
+|-- 00:01.0 -> ../../../devices/pci0/00:01.0
+|-- 00:02.0 -> ../../../devices/pci0/00:02.0
+|-- 00:1e.0 -> ../../../devices/pci0/00:1e.0
+|-- 00:1f.0 -> ../../../devices/pci0/00:1f.0
+|-- 00:1f.1 -> ../../../devices/pci0/00:1f.1
+|-- 00:1f.2 -> ../../../devices/pci0/00:1f.2
+|-- 00:1f.3 -> ../../../devices/pci0/00:1f.3
+|-- 00:1f.5 -> ../../../devices/pci0/00:1f.5
+|-- 01:00.0 -> ../../../devices/pci0/00:01.0/01:00.0
+|-- 02:1f.0 -> ../../../devices/pci0/00:02.0/02:1f.0
+|-- 03:00.0 -> ../../../devices/pci0/00:02.0/02:1f.0/03:00.0
+`-- 04:04.0 -> ../../../devices/pci0/00:1e.0/04:04.0
+
+
+
+Step 3: Registering Drivers.
+
+struct device_driver is a simple driver structure that contains a set
+of operations that the driver model core may call.
+
+
+- Embed a struct device_driver in the bus-specific driver.
+
+ Just like with devices, do something like:
+
+struct pci_driver {
+ ...
+ struct device_driver driver;
+};
+
+
+- Initialize the generic driver structure.
+
+ When the driver registers with the bus (e.g. doing pci_register_driver()),
+ initialize the necessary fields of the driver: the name and bus
+ fields.
+
+
+- Register the driver.
+
+ After the generic driver has been initialized, call
+
+ driver_register(&drv->driver);
+
+ to register the driver with the core.
+
+ When the driver is unregistered from the bus, unregister it from the
+ core by doing:
+
+ driver_unregister(&drv->driver);
+
+ Note that this will block until all references to the driver have
+ gone away. Normally, there will not be any.
+
+
+- Sysfs representation.
+
+ Drivers are exported via sysfs in their bus's 'driver's directory.
+ For example:
+
+/sys/bus/pci/drivers/
+|-- 3c59x
+|-- Ensoniq AudioPCI
+|-- agpgart-amdk7
+|-- e100
+`-- serial
+
+
+Step 4: Define Generic Methods for Drivers.
+
+struct device_driver defines a set of operations that the driver model
+core calls. Most of these operations are probably similar to
+operations the bus already defines for drivers, but taking different
+parameters.
+
+It would be difficult and tedious to force every driver on a bus to
+simultaneously convert their drivers to generic format. Instead, the
+bus driver should define single instances of the generic methods that
+forward call to the bus-specific drivers. For instance:
+
+
+static int pci_device_remove(struct device * dev)
+{
+ struct pci_dev * pci_dev = to_pci_dev(dev);
+ struct pci_driver * drv = pci_dev->driver;
+
+ if (drv) {
+ if (drv->remove)
+ drv->remove(pci_dev);
+ pci_dev->driver = NULL;
+ }
+ return 0;
+}
+
+
+The generic driver should be initialized with these methods before it
+is registered.
+
+ /* initialize common driver fields */
+ drv->driver.name = drv->name;
+ drv->driver.bus = &pci_bus_type;
+ drv->driver.probe = pci_device_probe;
+ drv->driver.resume = pci_device_resume;
+ drv->driver.suspend = pci_device_suspend;
+ drv->driver.remove = pci_device_remove;
+
+ /* register with core */
+ driver_register(&drv->driver);
+
+
+Ideally, the bus should only initialize the fields if they are not
+already set. This allows the drivers to implement their own generic
+methods.
+
+
+Step 5: Support generic driver binding.
+
+The model assumes that a device or driver can be dynamically
+registered with the bus at any time. When registration happens,
+devices must be bound to a driver, or drivers must be bound to all
+devices that it supports.
+
+A driver typically contains a list of device IDs that it supports. The
+bus driver compares these IDs to the IDs of devices registered with it.
+The format of the device IDs, and the semantics for comparing them are
+bus-specific, so the generic model does attempt to generalize them.
+
+Instead, a bus may supply a method in struct bus_type that does the
+comparison:
+
+ int (*match)(struct device * dev, struct device_driver * drv);
+
+match should return '1' if the driver supports the device, and '0'
+otherwise.
+
+When a device is registered, the bus's list of drivers is iterated
+over. bus->match() is called for each one until a match is found.
+
+When a driver is registered, the bus's list of devices is iterated
+over. bus->match() is called for each device that is not already
+claimed by a driver.
+
+When a device is successfully bound to a driver, device->driver is
+set, the device is added to a per-driver list of devices, and a
+symlink is created in the driver's sysfs directory that points to the
+device's physical directory:
+
+/sys/bus/pci/drivers/
+|-- 3c59x
+| `-- 00:0b.0 -> ../../../../devices/pci0/00:0b.0
+|-- Ensoniq AudioPCI
+|-- agpgart-amdk7
+| `-- 00:00.0 -> ../../../../devices/pci0/00:00.0
+|-- e100
+| `-- 00:0c.0 -> ../../../../devices/pci0/00:0c.0
+`-- serial
+
+
+This driver binding should replace the existing driver binding
+mechanism the bus currently uses.
+
+
+Step 6: Supply a hotplug callback.
+
+Whenever a device is registered with the driver model core, the
+userspace program /sbin/hotplug is called to notify userspace.
+Users can define actions to perform when a device is inserted or
+removed.
+
+The driver model core passes several arguments to userspace via
+environment variables, including
+
+- ACTION: set to 'add' or 'remove'
+- DEVPATH: set to the device's physical path in sysfs.
+
+A bus driver may also supply additional parameters for userspace to
+consume. To do this, a bus must implement the 'hotplug' method in
+struct bus_type:
+
+ int (*hotplug) (struct device *dev, char **envp,
+ int num_envp, char *buffer, int buffer_size);
+
+This is called immediately before /sbin/hotplug is executed.
+
+
+Step 7: Cleaning up the bus driver.
+
+The generic bus, device, and driver structures provide several fields
+that can replace those defined privately to the bus driver.
+
+- Device list.
+
+struct bus_type contains a list of all devices registered with the bus
+type. This includes all devices on all instances of that bus type.
+An internal list that the bus uses may be removed, in favor of using
+this one.
+
+The core provides an iterator to access these devices.
+
+int bus_for_each_dev(struct bus_type * bus, struct device * start,
+ void * data, int (*fn)(struct device *, void *));
+
+
+- Driver list.
+
+struct bus_type also contains a list of all drivers registered with
+it. An internal list of drivers that the bus driver maintains may
+be removed in favor of using the generic one.
+
+The drivers may be iterated over, like devices:
+
+int bus_for_each_drv(struct bus_type * bus, struct device_driver * start,
+ void * data, int (*fn)(struct device_driver *, void *));
+
+
+Please see drivers/base/bus.c for more information.
+
+
+- rwsem
+
+struct bus_type contains an rwsem that protects all core accesses to
+the device and driver lists. This can be used by the bus driver
+internally, and should be used when accessing the device or driver
+lists the bus maintains.
+
+
+- Device and driver fields.
+
+Some of the fields in struct device and struct device_driver duplicate
+fields in the bus-specific representations of these objects. Feel free
+to remove the bus-specific ones and favor the generic ones. Note
+though, that this will likely mean fixing up all the drivers that
+reference the bus-specific fields (though those should all be 1-line
+changes).
+
7 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984
/*
// Copyright (c) 2010-2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
*/

#include <rte_mbuf.h>
#include <rte_hash.h>
#include <rte_hash_crc.h>
#include <rte_ether.h>
#include <rte_ip.h>
#include <rte_version.h>
#include <rte_byteorder.h>
#include <rte_lpm.h>

#include "prox_lua_types.h"
#include "prox_lua.h"
#include "prox_malloc.h"
#include "prox_cksum.h"
#include "prefetch.h"
#include "etypes.h"
#include "log.h"
#include "quit.h"
#include "task_init.h"
#include "task_base.h"
#include "lconf.h"
#include "log.h"
#include "prox_port_cfg.h"
#include "hash_entry_types.h"
#include "prox_shared.h"
#include "handle_cgnat.h"

#define ALL_32_BITS 0xffffffff
#define BIT_16_TO_31 0xffff0000
#define BIT_8_TO_15 0x0000ff00
#define BIT_0_TO_15 0x0000ffff

struct private_key {
		uint32_t ip_addr;
		uint16_t l4_port;
} __attribute__((packed));

struct private_flow_entry {
	uint64_t flow_time;
	uint32_t ip_addr;
	uint32_t private_ip_idx;
	uint16_t l4_port;
};

struct public_key {
	uint32_t ip_addr;
	uint16_t l4_port;
} __attribute__((packed));

struct public_entry {
	uint32_t ip_addr;
	uint16_t l4_port;
	uint32_t private_ip_idx;
	uint8_t dpdk_port;
};

struct public_ip_config_info {
	uint32_t public_ip;
	uint32_t max_port_count;
	uint32_t port_free_count;
	uint16_t *port_list;
};

struct private_ip_info {
	uint64_t mac_aging_time;
	uint32_t public_ip;
	uint32_t public_ip_idx;
	struct rte_ether *private_mac;
	uint8_t static_entry;
};

struct task_nat {
	struct task_base base;
	struct rte_hash  *private_ip_hash;
	struct rte_hash  *private_ip_port_hash;
	struct rte_hash  *public_ip_port_hash;
	struct private_flow_entry *private_flow_entries;
	struct public_entry *public_entries;
	struct next_hop *next_hops;
	struct lcore_cfg *lconf;
	struct rte_lpm *ipv4_lpm;
	uint32_t total_free_port_count;
	uint32_t number_free_rules;
	int    private;
	uint32_t public_ip_count;
	uint32_t last_ip;
	struct public_ip_config_info *public_ip_config_info;
	struct private_ip_info *private_ip_info;
	uint8_t runtime_flags;
	int offload_crc;
	uint64_t src_mac[PROX_MAX_PORTS];
	uint64_t src_mac_from_dpdk_port[PROX_MAX_PORTS];
	volatile int dump_public_hash;
	volatile int dump_private_hash;
};
static __m128i proto_ipsrc_portsrc_mask;
static __m128i proto_ipdst_portdst_mask;
struct pkt_eth_ipv4 {
	prox_rte_ether_hdr ether_hdr;
	prox_rte_ipv4_hdr  ipv4_hdr;
	prox_rte_udp_hdr  udp_hdr;
} __attribute__((packed));

void task_cgnat_dump_public_hash(struct task_nat *task)
{
	task->dump_public_hash = 1;
}

void task_cgnat_dump_private_hash(struct task_nat *task)
{
	task->dump_private_hash = 1;
}

static void set_l2(struct task_nat *task, struct rte_mbuf *mbuf, uint8_t nh_idx)
{
	prox_rte_ether_hdr *peth = rte_pktmbuf_mtod(mbuf, prox_rte_ether_hdr *);
	*((uint64_t *)(&peth->d_addr)) = task->next_hops[nh_idx].mac_port_8bytes;
	*((uint64_t *)(&peth->s_addr)) = task->src_mac[task->next_hops[nh_idx].mac_port.out_idx];
}

static uint8_t route_ipv4(struct task_nat *task, struct rte_mbuf *mbuf)
{
	struct pkt_eth_ipv4 *pkt = rte_pktmbuf_mtod(mbuf, struct pkt_eth_ipv4 *);
	prox_rte_ipv4_hdr *ip = &pkt->ipv4_hdr;
	prox_rte_ether_hdr *peth_out;
	uint8_t tx_port;
	uint32_t dst_ip;

	switch(ip->next_proto_id) {
	case IPPROTO_TCP:
	case IPPROTO_UDP:
		dst_ip = ip->dst_addr;
		break;
	default:
		/* Routing for other protocols is not implemented */
		plogx_info("Routing nit implemented for this protocol\n");
		return OUT_DISCARD;
	}

#if RTE_VERSION >= RTE_VERSION_NUM(16,4,0,1)
	uint32_t next_hop_index;
#else
	uint8_t next_hop_index;
#endif
	if (unlikely(rte_lpm_lookup(task->ipv4_lpm, rte_bswap32(dst_ip), &next_hop_index) != 0)) {
		uint8_t* dst_ipp = (uint8_t*)&dst_ip;
		plog_warn("lpm_lookup failed for ip %d.%d.%d.%d: rc = %d\n",
			dst_ipp[0], dst_ipp[1], dst_ipp[2], dst_ipp[3], -ENOENT);
		return OUT_DISCARD;
	}

	tx_port = task->next_hops[next_hop_index].mac_port.out_idx;
	set_l2(task, mbuf, next_hop_index);
	return tx_port;
}

static int release_ip(struct task_nat *task, uint32_t *ip_addr, int public_ip_idx)
{
	return 0;
}

static int release_port(struct task_nat *task, uint32_t public_ip_idx, uint16_t udp_src_port)
{
	struct public_ip_config_info *public_ip_config_info = &task->public_ip_config_info[public_ip_idx];
	if (public_ip_config_info->max_port_count > public_ip_config_info->port_free_count) {
		public_ip_config_info->port_list[public_ip_config_info->port_free_count] = udp_src_port;
		public_ip_config_info->port_free_count++;
		task->total_free_port_count ++;
		plogx_dbg("Now %d free ports for IP %d.%d.%d.%d\n", public_ip_config_info->port_free_count, IP4(public_ip_config_info->public_ip));
	} else {
		plogx_err("Unable to release port for ip index %d: max_port_count = %d, port_free_count = %d", public_ip_idx, public_ip_config_info->max_port_count, public_ip_config_info->port_free_count);
		return -1;
	}
	return 0;
}

static int get_new_ip(struct task_nat *task, uint32_t *ip_addr)
{
	struct public_ip_config_info *ip_info;
	if (++task->last_ip >= task->public_ip_count)
		task->last_ip = 0;
	for (uint32_t ip_idx = task->last_ip; ip_idx < task->public_ip_count; ip_idx++) {
		ip_info = &task->public_ip_config_info[ip_idx];
		plogx_dbg("Checking public IP index %d\n", ip_idx);
		if ((ip_info->port_free_count) > 0) {
			plogx_dbg("Public IP index %d (IP %d.%d.%d.%d) has %d free ports\n", ip_idx, IP4(ip_info->public_ip), ip_info->port_free_count);
			*ip_addr = ip_info->public_ip;
			task->last_ip = ip_idx;
			return ip_idx;
		}
	}
	for (uint32_t ip_idx = 0; ip_idx < task->last_ip; ip_idx++) {
		ip_info = &task->public_ip_config_info[ip_idx];
		if ((ip_info->port_free_count) > 0) {
			plogx_dbg("Public IP index %d (IP %d.%d.%d.%d) has %d free ports\n", ip_idx, IP4(ip_info->public_ip), ip_info->port_free_count);
			*ip_addr = ip_info->public_ip;
			task->last_ip = ip_idx;
			return ip_idx;
		}
	}
	return -1;
}

static int get_new_port(struct task_nat *task, uint32_t ip_idx, uint16_t *udp_src_port)
{
	int ret;
	struct public_ip_config_info *public_ip_config_info = &task->public_ip_config_info[ip_idx];
	if (public_ip_config_info->port_free_count > 0) {
		public_ip_config_info->port_free_count--;
		*udp_src_port = public_ip_config_info->port_list[public_ip_config_info->port_free_count];
		task->total_free_port_count --;
		plogx_info("Now %d free ports for IP %d.%d.%d.%d\n", public_ip_config_info->port_free_count, IP4(public_ip_config_info->public_ip));
	} else
		return -1;
	return 0;
}

static int delete_port_entry(struct task_nat *task, uint8_t proto, uint32_t private_ip, uint16_t private_port,  uint32_t public_ip, uint16_t public_port, int public_ip_idx)
{
	int ret;
	struct private_key private_key;
	struct public_key public_key;
//	private_key.proto = proto;
	private_key.ip_addr = private_ip;
	private_key.l4_port = private_port;
	ret = rte_hash_del_key(task->private_ip_port_hash, (const void *)&private_key);
	if (ret < 0) {
		plogx_info("Unable delete key ip %d.%d.%d.%d / port %x in private ip_port hash\n", IP4(private_ip), private_port);
		return -1;
	} else {
		plogx_dbg("Deleted ip %d.%d.%d.%d / port %x from private ip_port hash\n", IP4(private_ip), private_port);
	}
	public_key.ip_addr = public_ip;
	public_key.l4_port = public_port;
	ret = rte_hash_del_key(task->public_ip_port_hash, (const void *)&public_key);
	if (ret < 0) {
		plogx_info("Unable delete key ip %d.%d.%d.%d / port %x in public ip_port hash\n", IP4(public_ip), public_port);
		return -1;
	} else {
		plogx_dbg("Deleted ip %d.%d.%d.%d / port %x (hash index %d) from public ip_port hash\n", IP4(public_ip), public_port, ret);
		release_port(task, public_ip_idx, public_port);
	}
	return 0;
}

static int add_new_port_entry(struct task_nat *task, uint8_t proto, int public_ip_idx, int private_ip_idx, uint32_t private_src_ip, uint16_t private_udp_port, struct rte_mbuf *mbuf, uint64_t tsc, uint16_t *port)
{
	struct private_key private_key;
	struct public_key public_key;
	uint32_t ip = task->public_ip_config_info[public_ip_idx].public_ip;
	int ret;
	if (get_new_port(task, public_ip_idx, port) < 0) {
		plogx_info("Unable to find new port for IP %x\n", private_src_ip);
		return -1;
	}
//	private_key.proto = proto;
	private_key.ip_addr = private_src_ip;
	private_key.l4_port = private_udp_port;
	ret = rte_hash_add_key(task->private_ip_port_hash, (const void *)&private_key);
	if (ret < 0) {
		plogx_info("Unable add ip %d.%d.%d.%d / port %x in private ip_port hash\n", IP4(private_src_ip), private_udp_port);
		release_port(task, public_ip_idx, *port);
		return -1;
	} else if (task->private_flow_entries[ret].ip_addr) {
		plogx_dbg("Race condition properly handled: port alrerady added\n");
		release_port(task, public_ip_idx, *port);
		return ret;
	} else {
		plogx_dbg("Added ip %d.%d.%d.%d / port %x in private ip_port hash => %d.%d.%d.%d / %d - index = %d\n", IP4(private_src_ip), private_udp_port, IP4(ip), *port, ret);
	}
	task->private_flow_entries[ret].ip_addr = ip;
	task->private_flow_entries[ret].l4_port = *port;
	task->private_flow_entries[ret].flow_time = tsc;
       	task->private_flow_entries[ret].private_ip_idx = private_ip_idx;

	public_key.ip_addr = ip;
	public_key.l4_port = *port;
	plogx_dbg("Adding key ip %d.%d.%d.%d / port %x in public ip_port hash\n", IP4(ip), *port);
	ret = rte_hash_add_key(task->public_ip_port_hash, (const void *)&public_key);
	if (ret < 0) {
		plogx_info("Unable add ip %x / port %x in public ip_port hash\n", ip, *port);
		// TODO: remove from private_ip_port_hash
		release_port(task, public_ip_idx, *port);
		return -1;
	} else {
		plogx_dbg("Added ip %d.%d.%d.%d / port %x in public ip_port hash\n", IP4(ip), *port);
	}
	task->public_entries[ret].ip_addr = private_src_ip;
	task->public_entries[ret].l4_port = private_udp_port;
	task->public_entries[ret].dpdk_port = mbuf->port;
       	task->public_entries[ret].private_ip_idx = private_ip_idx;
	return ret;
}

static int handle_nat_bulk(struct task_base *tbase, struct rte_mbuf **mbufs, uint16_t n_pkts)
{
        struct task_nat *task = (struct task_nat *)tbase;
        uint8_t out[MAX_PKT_BURST];
        uint16_t j;
	uint32_t *ip_addr, public_ip, private_ip;
	uint16_t *udp_src_port, port, private_port, public_port;
	struct pkt_eth_ipv4 *pkt[MAX_PKT_BURST];
	int ret, private_ip_idx, public_ip_idx = -1, port_idx;
	int new_entry = 0;
	uint8_t proto;
	uint64_t tsc = rte_rdtsc();
	void *keys[MAX_PKT_BURST];
	int32_t positions[MAX_PKT_BURST];
	int map[MAX_PKT_BURST] = {0};

	if (unlikely(task->dump_public_hash)) {
		const struct public_key *next_key;
		void *next_data;
		uint32_t iter = 0;
		int i = 0;
		int ret;

		while ((ret = rte_hash_iterate(task->public_ip_port_hash, (const void **)&next_key, &next_data, &iter)) >= 0) {
			plogx_info("Public entry %d (index %d): ip = %d.%d.%d.%d, port = %d ===> private entry: ip = %d.%d.%d.%d, port = %d\n", i++, ret, IP4(next_key->ip_addr), next_key->l4_port, IP4(task->public_entries[ret].ip_addr),task->public_entries[ret].l4_port);
		}
		task->dump_public_hash = 0;
	}
	if (unlikely(task->dump_private_hash)) {
		const struct private_key *next_key;
		void *next_data;
		uint32_t iter = 0;
		int i = 0;
		int ret;

		while ((ret = rte_hash_iterate(task->private_ip_port_hash, (const void **)&next_key, &next_data, &iter)) >= 0) {
			plogx_info("Private entry %d (index %d): ip = %d.%d.%d.%d, port = %d ===> public entry: ip = %d.%d.%d.%d, port = %d\n", i++, ret, IP4(next_key->ip_addr), next_key->l4_port, IP4(task->private_flow_entries[ret].ip_addr),task->private_flow_entries[ret].l4_port);
		}
		task->dump_private_hash = 0;
	}

       	for (j = 0; j < n_pkts; ++j) {
               	PREFETCH0(mbufs[j]);
	}
       	for (j = 0; j < n_pkts; ++j) {
		pkt[j] = rte_pktmbuf_mtod(mbufs[j], struct pkt_eth_ipv4 *);
               	PREFETCH0(pkt[j]);
	}
	if (task->private) {
       		struct private_key key[MAX_PKT_BURST];
        	for (j = 0; j < n_pkts; ++j) {
			/* Currently, only support eth/ipv4 packets */
			if (pkt[j]->ether_hdr.ether_type != ETYPE_IPv4) {
				plogx_info("Currently, only support eth/ipv4 packets\n");
				out[j] = OUT_DISCARD;
				keys[j] = (void *)NULL;
				continue;
			}
       			key[j].ip_addr = pkt[j]->ipv4_hdr.src_addr;
			key[j].l4_port = pkt[j]->udp_hdr.src_port;
			keys[j] = &key[j];
		}
		ret = rte_hash_lookup_bulk(task->private_ip_port_hash, (const void **)&keys, n_pkts, positions);
		if (unlikely(ret < 0)) {
			plogx_info("lookup_bulk failed in private_ip_port_hash\n");
			return -1;
		}
		int n_new_mapping = 0;
        	for (j = 0; j < n_pkts; ++j) {
			port_idx = positions[j];
			if (unlikely(port_idx < 0)) {
				plogx_dbg("ip %d.%d.%d.%d / port %x not found in private ip/port hash\n", IP4(pkt[j]->ipv4_hdr.src_addr), pkt[j]->udp_hdr.src_port);
				map[n_new_mapping] = j;
				keys[n_new_mapping++] = (void *)&(pkt[j]->ipv4_hdr.src_addr);
			} else {
				ip_addr = &(pkt[j]->ipv4_hdr.src_addr);
				udp_src_port = &(pkt[j]->udp_hdr.src_port);
				plogx_dbg("ip/port %d.%d.%d.%d / %x found in private ip/port hash\n", IP4(pkt[j]->ipv4_hdr.src_addr), pkt[j]->udp_hdr.src_port);
       				*ip_addr = task->private_flow_entries[port_idx].ip_addr;
       				*udp_src_port = task->private_flow_entries[port_idx].l4_port;
				uint64_t flow_time = task->private_flow_entries[port_idx].flow_time;
				if (flow_time + tsc_hz < tsc) {
					task->private_flow_entries[port_idx].flow_time = tsc;
				}
				private_ip_idx = task->private_flow_entries[port_idx].private_ip_idx;
				if (task->private_ip_info[private_ip_idx].mac_aging_time + tsc_hz < tsc)
					task->private_ip_info[private_ip_idx].mac_aging_time = tsc;
				prox_ip_udp_cksum(mbufs[j], &pkt[j]->ipv4_hdr, sizeof(prox_rte_ether_hdr), sizeof(prox_rte_ipv4_hdr), task->offload_crc);
				out[j] =  route_ipv4(task, mbufs[j]);
			}
		}

		if (n_new_mapping) {
			// Find whether at least IP is already known...
			ret = rte_hash_lookup_bulk(task->private_ip_hash, (const void **)&keys, n_new_mapping, positions);
			if (unlikely(ret < 0)) {
				plogx_info("lookup_bulk failed for private_ip_hash\n");
				for (int k = 0; k < n_new_mapping; ++k) {
					j = map[k];
					out[j] = OUT_DISCARD;
				}
				n_new_mapping = 0;
			}
       			for (int k = 0; k < n_new_mapping; ++k) {
				private_ip_idx = positions[k];
				j = map[k];
				ip_addr = &(pkt[j]->ipv4_hdr.src_addr);
				proto = pkt[j]->ipv4_hdr.next_proto_id;
				udp_src_port = &(pkt[j]->udp_hdr.src_port);
				int new_ip_entry = 0;

				if (unlikely(private_ip_idx < 0)) {
					private_ip = *ip_addr;
					private_port = *udp_src_port;
					plogx_dbg("Did not find private ip %d.%d.%d.%d in ip hash table, looking for new public ip\n", IP4(*ip_addr));
					// IP not found, need to get a new IP/port mapping
					public_ip_idx = get_new_ip(task, &public_ip);
					if (public_ip_idx < 0) {
						plogx_info("Unable to find new ip/port\n");
						out[j] = OUT_DISCARD;
						continue;
					} else {
						plogx_dbg("found new public ip %d.%d.%d.%d at public IP index %d\n", IP4(public_ip), public_ip_idx);
					}
					private_ip_idx = rte_hash_add_key(task->private_ip_hash, (const void *)ip_addr);
					// The key might be added multiple time - in case the same key was present in the bulk_lookup multiple times
					// As such this is not an issue - the add_key will returns the index as for a new key
					// This scenario should not happen often in real time use case
					// as a for a new flow (flow renewal), probably only one packet will be sent (e.g. TCP SYN)
					if (private_ip_idx < 0) {
						release_ip(task, &public_ip, public_ip_idx);
						plogx_info("Unable add ip %d.%d.%d.%d in private ip hash\n", IP4(*ip_addr));
						out[j] = OUT_DISCARD;
						continue;
					} else if (task->private_ip_info[private_ip_idx].public_ip) {
						plogx_info("race condition properly handled : ip %d.%d.%d.%d already in private ip hash\n", IP4(*ip_addr));
						release_ip(task, &public_ip, public_ip_idx);
						public_ip = task->private_ip_info[private_ip_idx].public_ip;
						public_ip_idx = task->private_ip_info[private_ip_idx].public_ip_idx;
					} else {
						plogx_dbg("Added ip %d.%d.%d.%d in private ip hash\n", IP4(*ip_addr));
						rte_memcpy(&task->private_ip_info[private_ip_idx].private_mac, ((uint8_t *)pkt) + 6, 6);
						task->private_ip_info[private_ip_idx].public_ip = public_ip;
						task->private_ip_info[private_ip_idx].static_entry = 0;
						task->private_ip_info[private_ip_idx].public_ip_idx = public_ip_idx;
						new_ip_entry = 1;
					}
				} else {
					public_ip = task->private_ip_info[private_ip_idx].public_ip;
					public_ip_idx = task->private_ip_info[private_ip_idx].public_ip_idx;
				}
				port_idx = add_new_port_entry(task, proto, public_ip_idx, private_ip_idx, *ip_addr, *udp_src_port, mbufs[j], tsc, &public_port);
				if (port_idx < 0) {
					// TODO: delete IP in ip_hash
					if ((new_ip_entry) && (task->last_ip != 0)) {
						release_ip(task, &public_ip, public_ip_idx);
						task->last_ip--;
					} else if (new_ip_entry) {
						release_ip(task, &public_ip, public_ip_idx);
						task->last_ip = task->public_ip_count-1;
					}
					plogx_info("Failed to add new port entry\n");
					out[j] = OUT_DISCARD;
					continue;
				} else {
					private_ip = *ip_addr;
					private_port = *udp_src_port;
					plogx_info("Added new ip/port: private ip/port = %d.%d.%d.%d/%x public ip/port = %d.%d.%d.%d/%x, index = %d\n", IP4(private_ip), private_port, IP4(public_ip), public_port, port_idx);
				}
       				// task->private_flow_entries[port_idx].ip_addr = task->private_ip_info[private_ip_idx].public_ip;
				plogx_info("Added new port: private ip/port = %d.%d.%d.%d/%x, public ip/port = %d.%d.%d.%d/%x\n", IP4(private_ip), private_port, IP4(task->private_ip_info[private_ip_idx].public_ip), public_port);
       				*ip_addr = public_ip ;
       				*udp_src_port = public_port;
				uint64_t flow_time = task->private_flow_entries[port_idx].flow_time;
				if (flow_time + tsc_hz < tsc) {
					task->private_flow_entries[port_idx].flow_time = tsc;
				}
				if (task->private_ip_info[private_ip_idx].mac_aging_time + tsc_hz < tsc)
					task->private_ip_info[private_ip_idx].mac_aging_time = tsc;
				prox_ip_udp_cksum(mbufs[j], &pkt[j]->ipv4_hdr, sizeof(prox_rte_ether_hdr), sizeof(prox_rte_ipv4_hdr), task->offload_crc);
				// TODO: if route fails while just added new key in table, should we delete the key from the table?
				out[j] =  route_ipv4(task, mbufs[j]);
				if (out[j] && new_entry) {
					delete_port_entry(task, proto, private_ip, private_port, *ip_addr, *udp_src_port, public_ip_idx);
					plogx_info("Deleted port: private ip/port = %d.%d.%d.%d/%x, public ip/port = %d.%d.%d.%d/%x\n", IP4(private_ip), private_port, IP4(*ip_addr), *udp_src_port);
				}
			}
		}
        	return task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
	} else {
		struct public_key public_key[MAX_PKT_BURST];
        	for (j = 0; j < n_pkts; ++j) {
			/* Currently, only support eth/ipv4 packets */
			if (pkt[j]->ether_hdr.ether_type != ETYPE_IPv4) {
				plogx_info("Currently, only support eth/ipv4 packets\n");
				out[j] = OUT_DISCARD;
				keys[j] = (void *)NULL;
				continue;
			}
       			public_key[j].ip_addr = pkt[j]->ipv4_hdr.dst_addr;
			public_key[j].l4_port = pkt[j]->udp_hdr.dst_port;
			keys[j] = &public_key[j];
		}
		ret = rte_hash_lookup_bulk(task->public_ip_port_hash, (const void **)&keys, n_pkts, positions);
		if (ret < 0) {
			plogx_err("Failed lookup bulk public_ip_port_hash\n");
			return -1;
		}
        	for (j = 0; j < n_pkts; ++j) {
			port_idx = positions[j];
       			ip_addr = &(pkt[j]->ipv4_hdr.dst_addr);
			udp_src_port = &(pkt[j]->udp_hdr.dst_port);
			if (port_idx < 0) {
				plogx_err("Failed to find ip/port %d.%d.%d.%d/%x in public_ip_port_hash\n", IP4(*ip_addr), *udp_src_port);
				out[j] = OUT_DISCARD;
			} else {
				plogx_dbg("Found ip/port %d.%d.%d.%d/%x in public_ip_port_hash\n", IP4(*ip_addr), *udp_src_port);
        			*ip_addr = task->public_entries[port_idx].ip_addr;
				*udp_src_port = task->public_entries[port_idx].l4_port;
				private_ip_idx = task->public_entries[port_idx].private_ip_idx;
				plogx_dbg("Found private IP info for ip %d.%d.%d.%d\n", IP4(*ip_addr));
				rte_memcpy(((uint8_t *)(pkt[j])) + 0, &task->private_ip_info[private_ip_idx].private_mac, 6);
				rte_memcpy(((uint8_t *)(pkt[j])) + 6, &task->src_mac_from_dpdk_port[task->public_entries[port_idx].dpdk_port], 6);
				out[j] = task->public_entries[port_idx].dpdk_port;
			}
			prox_ip_udp_cksum(mbufs[j], &pkt[j]->ipv4_hdr, sizeof(prox_rte_ether_hdr), sizeof(prox_rte_ipv4_hdr), task->offload_crc);
		}
        	return task->base.tx_pkt(&task->base, mbufs, n_pkts, out);
	}

}

static int lua_to_hash_nat(struct task_args *targ, struct lua_State *L, enum lua_place from, const char *name, uint8_t socket)
{
	struct rte_hash *tmp_priv_ip_hash, *tmp_priv_hash, *tmp_pub_hash;
	struct private_flow_entry *tmp_priv_flow_entries;
	struct public_entry *tmp_pub_entries;
	uint32_t n_entries = 0;;
	uint32_t ip_from, ip_to;
	uint16_t port_from, port_to;
	int ret, idx, pop, pop2, pop3, n_static_entries = 0;
	uint32_t dst_ip1, dst_ip2;
	struct val_range dst_port;
	struct public_ip_config_info *ip_info;
	struct public_ip_config_info *tmp_public_ip_config_info;

	if ((pop = lua_getfrom(L, from, name)) < 0)
		return -1;

        if (!lua_istable(L, -1)) {
                plogx_err("Can't read cgnat since data is not a table\n");
                return -1;
        }

	struct tmp_public_ip {
        	uint32_t ip_beg;
		uint32_t ip_end;
        	uint16_t port_beg;
		uint16_t port_end;
	};
	struct tmp_static_ip {
		uint32_t private_ip;
		uint32_t public_ip;
	};
	struct tmp_static_ip_port {
		uint32_t private_ip;
		uint32_t public_ip;
		uint32_t n_ports;
		uint16_t private_port;
		uint16_t public_port;
		int ip_found;
		uint8_t port_found;
	};
	uint32_t n_public_groups = 0;
	uint32_t n_public_ip = 0;
	uint32_t n_static_ip = 0;
	uint32_t n_static_ip_port = 0;
	unsigned int i = 0;
	struct tmp_public_ip *tmp_public_ip = NULL;
	struct tmp_static_ip *tmp_static_ip = NULL;
	struct tmp_static_ip_port *tmp_static_ip_port = NULL;

	// Look for Dynamic entries configuration
	plogx_info("Reading dynamic NAT table\n");
	if ((pop2 = lua_getfrom(L, TABLE, "dynamic")) < 0) {
		plogx_info("No dynamic table found\n");
	} else {
		uint64_t n_ip, n_port;
        	if (!lua_istable(L, -1)) {
                	plogx_err("Can't read cgnat since data is not a table\n");
                	return -1;
        	}
		lua_len(L, -1);
		n_public_groups = lua_tointeger(L, -1);
		plogx_info("%d groups of public IP\n", n_public_groups);
		tmp_public_ip = (struct tmp_public_ip *)malloc(n_public_groups * sizeof(struct tmp_public_ip));
		PROX_PANIC(tmp_public_ip == NULL, "Failed to allocated tmp_public_ip\n");
		lua_pop(L, 1);
		lua_pushnil(L);

		while (lua_next(L, -2)) {
			if (lua_to_ip(L, TABLE, "public_ip_range_start", &dst_ip1) ||
		    		lua_to_ip(L, TABLE, "public_ip_range_stop", &dst_ip2) ||
		    		lua_to_val_range(L, TABLE, "public_port", &dst_port))
					return -1;
			PROX_PANIC(dst_ip2 < dst_ip1, "public_ip_range error: %d.%d.%d.%d < %d.%d.%d.%d\n", (dst_ip2 >> 24), (dst_ip2 >> 16) & 0xFF, (dst_ip2 >> 8) & 0xFF, dst_ip2 & 0xFF, dst_ip1 >> 24, (dst_ip1 >> 16) & 0xFF, (dst_ip1 >> 8) & 0xFF, dst_ip1 & 0xFF);
			PROX_PANIC(dst_port.end < dst_port.beg, "public_port error: %d < %d\n", dst_port.end, dst_port.beg);
			n_ip = dst_ip2 - dst_ip1 + 1;
			n_port =  dst_port.end - dst_port.beg + 1;
			n_public_ip += n_ip;
			plogx_info("Found IP from %d.%d.%d.%d to %d.%d.%d.%d and port from %d to %d\n", dst_ip1 >> 24, (dst_ip1 >> 16) & 0xFF, (dst_ip1 >> 8) & 0xFF, dst_ip1 & 0xFF, (dst_ip2 >> 24), (dst_ip2 >> 16) & 0xFF, (dst_ip2 >> 8) & 0xFF, dst_ip2 & 0xFF, dst_port.beg, dst_port.end);
			tmp_public_ip[i].ip_beg = dst_ip1;
			tmp_public_ip[i].ip_end = dst_ip2;
			tmp_public_ip[i].port_beg = dst_port.beg;
			tmp_public_ip[i++].port_end = dst_port.end;
			n_entries += n_ip * n_port;
			lua_pop(L, 1);
		}
		lua_pop(L, pop2);

	}
	i = 0;
	if ((pop2 = lua_getfrom(L, TABLE, "static_ip")) < 0) {
		plogx_info("No static ip table found\n");
	} else {
        	if (!lua_istable(L, -1)) {
                	plogx_err("Can't read cgnat since data is not a table\n");
                	return -1;
		}

		lua_len(L, -1);
		n_static_ip = lua_tointeger(L, -1);
		plogx_info("%d entries in static ip table\n", n_static_ip);
		lua_pop(L, 1);
		tmp_static_ip = (struct tmp_static_ip *)malloc(n_static_ip * sizeof(struct tmp_static_ip));
		PROX_PANIC(tmp_static_ip == NULL, "Failed to allocated tmp_static_ip\n");
		lua_pushnil(L);
		while (lua_next(L, -2)) {
			if (lua_to_ip(L, TABLE, "src_ip", &ip_from) ||
		    		lua_to_ip(L, TABLE, "dst_ip", &ip_to))
					return -1;
			ip_from = rte_bswap32(ip_from);
			ip_to = rte_bswap32(ip_to);
			tmp_static_ip[i].private_ip = ip_from;
			tmp_static_ip[i++].public_ip = ip_to;
			for (unsigned int j = 0; j < n_public_groups; j++) {
				if ((tmp_public_ip[j].ip_beg <= ip_to) && (ip_to <= tmp_public_ip[j].ip_end)) {
					PROX_PANIC(1, "list of static ip mapping overlap with list of dynamic IP => not supported yet\n");
				}
			}
			n_public_ip++;
			lua_pop(L, 1);
		}
		lua_pop(L, pop2);
	}

	i = 0;
	if ((pop2 = lua_getfrom(L, TABLE, "static_ip_port")) < 0) {
		plogx_info("No static table found\n");
	} else {
        	if (!lua_istable(L, -1)) {
                	plogx_err("Can't read cgnat since data is not a table\n");
                	return -1;
		}

		lua_len(L, -1);
		n_static_ip_port = lua_tointeger(L, -1);
		plogx_info("%d entries in static table\n", n_static_ip_port);
		lua_pop(L, 1);
		tmp_static_ip_port = (struct tmp_static_ip_port *)malloc(n_static_ip_port * sizeof(struct tmp_static_ip_port));
		PROX_PANIC(tmp_static_ip_port == NULL, "Failed to allocated tmp_static_ip_port\n");
		lua_pushnil(L);

		while (lua_next(L, -2)) {
			if (lua_to_ip(L, TABLE, "src_ip", &ip_from) ||
		    		lua_to_ip(L, TABLE, "dst_ip", &ip_to) ||
		    		lua_to_port(L, TABLE, "src_port", &port_from) ||
		    		lua_to_port(L, TABLE, "dst_port", &port_to))
					return -1;

			ip_from = rte_bswap32(ip_from);
			ip_to = rte_bswap32(ip_to);
			port_from = rte_bswap16(port_from);
			port_to = rte_bswap16(port_to);
			tmp_static_ip_port[i].private_ip = ip_from;
			tmp_static_ip_port[i].public_ip = ip_to;
			tmp_static_ip_port[i].private_port = port_from;
			tmp_static_ip_port[i].public_port = port_to;
			tmp_static_ip_port[i].n_ports = 1;
			for (unsigned int j = 0; j < n_public_groups; j++) {
				if ((tmp_public_ip[j].ip_beg <= rte_bswap32(ip_to)) && (rte_bswap32(ip_to) <= tmp_public_ip[j].ip_end)) {
					tmp_static_ip_port[i].ip_found = j + 11;
					PROX_PANIC(1, "list of static ip/port mapping overlap with list of dynamic IP => not supported yet\n");
				}
			}
			for (unsigned int j = 0; j < n_static_ip; j++) {
				if ((tmp_static_ip[j].public_ip == ip_to) ) {
					tmp_static_ip_port[i].ip_found = j + 1;
					PROX_PANIC(1, "list of static ip/port mapping overlap with list of static ip => not supported yet\n");
				}
			}
			for (unsigned int j = 0; j <= i; j++) {
				if (ip_to == tmp_static_ip_port[j].public_ip) {
					tmp_static_ip_port[i].ip_found = j + 1;
					tmp_static_ip_port[j].n_ports++;
					tmp_static_ip_port[i].n_ports = 0;
				}
			}
			i++;
			if (!tmp_static_ip_port[i].ip_found) {
				n_public_ip++;
				n_entries++;
			}
			lua_pop(L, 1);
		}
		lua_pop(L, pop2);
	}
	lua_pop(L, pop);

	tmp_public_ip_config_info = (struct public_ip_config_info *)prox_zmalloc(n_public_ip * sizeof(struct public_ip_config_info), socket);
	PROX_PANIC(tmp_public_ip_config_info == NULL, "Failed to allocate PUBLIC IP INFO\n");
	plogx_info("%d PUBLIC IP INFO allocated\n", n_public_ip);

	struct private_ip_info *tmp_priv_ip_info = (struct private_ip_info *)prox_zmalloc(4 * n_public_ip * sizeof(struct public_ip_config_info), socket);
	PROX_PANIC(tmp_priv_ip_info == NULL, "Failed to allocate PRIVATE IP INFO\n");
	plogx_info("%d PRIVATE IP INFO allocated\n", 4 * n_public_ip);

	uint32_t ip_free_count = 0;
	for (i = 0; i < n_public_groups; i++) {
		for (uint32_t ip = tmp_public_ip[i].ip_beg; ip <= tmp_public_ip[i].ip_end; ip++) {
			ip_info = &tmp_public_ip_config_info[ip_free_count];
			ip_info->public_ip = rte_bswap32(ip);
			ip_info->port_list = (uint16_t *)prox_zmalloc((dst_port.end - dst_port.beg) * sizeof(uint16_t), socket);
                       	PROX_PANIC(ip_info->port_list == NULL, "Failed to allocate list of ports for ip %x\n", ip);
			for (uint32_t port = tmp_public_ip[i].port_beg; port <= tmp_public_ip[i].port_end; port++) {
				ip_info->port_list[ip_info->port_free_count] = rte_bswap16(port);
				ip_info->port_free_count++;
			}
			ip_info->max_port_count = ip_info->port_free_count;
			plogx_dbg("Added IP %d.%d.%d.%d with ports from %x to %x at index %x\n", IP4(ip_info->public_ip), tmp_public_ip[i].port_beg, tmp_public_ip[i].port_end, ip_free_count);
			ip_free_count++;
		}
	}
	uint32_t public_ip_count = ip_free_count;
	for (i = 0; i < n_static_ip; i++) {
		ip_info = &tmp_public_ip_config_info[ip_free_count];
		ip_info->public_ip = tmp_static_ip[i].public_ip;
		ip_info->port_list = NULL;
		ip_info->max_port_count = 0;
		ip_free_count++;
	}
	for (i = 0; i < n_static_ip_port; i++) {
		if (!tmp_static_ip_port[i].ip_found) {
			ip_info = &tmp_public_ip_config_info[ip_free_count];
			ip_info->public_ip = tmp_static_ip_port[i].public_ip;
			ip_info->port_list = (uint16_t *)prox_zmalloc(tmp_static_ip_port[i].n_ports * sizeof(uint16_t), socket);
                	PROX_PANIC(ip_info->port_list == NULL, "Failed to allocate list of ports for ip %x\n", tmp_static_ip_port[i].public_ip);
			ip_info->port_list[ip_info->port_free_count] = tmp_static_ip_port[i].public_port;
			ip_info->port_free_count++;
			ip_info->max_port_count = ip_info->port_free_count;
			ip_free_count++;
		} else {
			for (unsigned j = 0; j < ip_free_count; j++) {
				ip_info = &tmp_public_ip_config_info[j];
				if (ip_info->public_ip == tmp_static_ip_port[i].public_ip) {
					ip_info = &tmp_public_ip_config_info[j];
					ip_info->port_list[ip_info->port_free_count] = tmp_static_ip_port[i].public_port;
					ip_info->port_free_count++;
					ip_info->max_port_count = ip_info->port_free_count;
					break;
				}
			}
		}
	}
	plogx_info("%d entries in dynamic table\n", n_entries);

	n_entries = n_entries * 4;
	static char hash_name[30];
	sprintf(hash_name, "A%03d_hash_nat_table", targ->lconf->id);
	struct rte_hash_parameters hash_params = {
		.name = hash_name,
		.entries = n_entries,
		.key_len = sizeof(struct private_key),
		.hash_func = rte_hash_crc,
		.hash_func_init_val = 0,
	};
	plogx_info("hash table name = %s\n", hash_params.name);
	struct private_key private_key;
	struct public_key public_key;
	tmp_priv_hash = rte_hash_create(&hash_params);
	PROX_PANIC(tmp_priv_hash == NULL, "Failed to set up private hash table for NAT\n");
	plogx_info("private hash table allocated, with %d entries of size %d\n", hash_params.entries, hash_params.key_len);

	tmp_priv_flow_entries = (struct private_flow_entry *)prox_zmalloc(n_entries * sizeof(struct private_flow_entry), socket);
	PROX_PANIC(tmp_priv_flow_entries == NULL, "Failed to allocate memory for private NAT %u entries\n", n_entries);
	plogx_info("private data allocated, with %d entries of size %ld\n", n_entries, sizeof(struct private_flow_entry));

	hash_name[0]++;
	//hash_params.name[0]++;
	plogx_info("hash table name = %s\n", hash_params.name);
	hash_params.key_len = sizeof(uint32_t);
	hash_params.entries = 4 * ip_free_count;
	tmp_priv_ip_hash = rte_hash_create(&hash_params);
	PROX_PANIC(tmp_priv_ip_hash == NULL, "Failed to set up private ip hash table for NAT\n");
	plogx_info("private ip hash table allocated, with %d entries of size %d\n", hash_params.entries, hash_params.key_len);

	hash_name[0]++;
	//hash_params.name[0]++;
	plogx_info("hash table name = %s\n", hash_params.name);
	hash_params.entries = n_entries;
	hash_params.key_len = sizeof(struct public_key),
	tmp_pub_hash = rte_hash_create(&hash_params);
	PROX_PANIC(tmp_pub_hash == NULL, "Failed to set up public hash table for NAT\n");
	plogx_info("public hash table allocated, with %d entries of size %d\n", hash_params.entries, hash_params.key_len);

	hash_name[0]++;
	//hash_params.name[0]++;
	tmp_pub_entries = (struct public_entry *)prox_zmalloc(n_entries * sizeof(struct public_entry), socket);
	PROX_PANIC(tmp_pub_entries == NULL, "Failed to allocate memory for public NAT %u entries\n", n_entries);
	plogx_info("public data allocated, with %d entries of size %ld\n", n_entries, sizeof(struct private_flow_entry));

	for (i = 0; i < n_static_ip_port; i++) {
		ip_to = tmp_static_ip_port[i].public_ip;
		ip_from = tmp_static_ip_port[i].private_ip;
		port_to = tmp_static_ip_port[i].public_port;
		port_from = tmp_static_ip_port[i].private_port;
		private_key.ip_addr = ip_from;
		private_key.l4_port = port_from;
		ret = rte_hash_lookup(tmp_priv_hash, (const void *)&private_key);
		PROX_PANIC(ret >= 0, "Key %x %x already exists in NAT private hash table\n", ip_from, port_from);

		idx = rte_hash_add_key(tmp_priv_ip_hash, (const void *)&ip_from);
		PROX_PANIC(idx < 0, "Failed to add ip %x to NAT private hash table\n", ip_from);
		ret = rte_hash_add_key(tmp_priv_hash, (const void *)&private_key);
		PROX_PANIC(ret < 0, "Failed to add Key %x %x to NAT private hash table\n", ip_from, port_from);
		tmp_priv_flow_entries[ret].ip_addr = ip_to;
		tmp_priv_flow_entries[ret].flow_time = -1;
		tmp_priv_flow_entries[ret].private_ip_idx = idx;
		tmp_priv_flow_entries[ret].l4_port = port_to;

		public_key.ip_addr = ip_to;
		public_key.l4_port = port_to;
		ret = rte_hash_lookup(tmp_pub_hash, (const void *)&public_key);
		PROX_PANIC(ret >= 0, "Key %d.%d.%d.%d port %x (for private IP %d.%d.%d.%d port %x) already exists in NAT public hash table fir IP %d.%d.%d.%d port %x\n", IP4(ip_to), port_to, IP4(ip_from), port_from, IP4(tmp_pub_entries[ret].ip_addr), tmp_pub_entries[ret].l4_port);

		ret = rte_hash_add_key(tmp_pub_hash, (const void *)&public_key);
		PROX_PANIC(ret < 0, "Failed to add Key %x %x to NAT public hash table\n", ip_to, port_to);
		tmp_pub_entries[ret].ip_addr = ip_from;
		tmp_pub_entries[ret].l4_port = port_from;
		tmp_pub_entries[ret].private_ip_idx = idx;
	}

	for (uint8_t task_id = 0; task_id < targ->lconf->n_tasks_all; ++task_id) {
		struct task_args *target_targ = (struct task_args *)&(targ->lconf->targs[task_id]);
		enum task_mode smode = target_targ->mode;
		if (CGNAT == smode) {
			target_targ->public_ip_count = public_ip_count;
			target_targ->private_ip_hash = tmp_priv_ip_hash;
			target_targ->private_ip_port_hash = tmp_priv_hash;
			target_targ->private_ip_info = tmp_priv_ip_info;
			target_targ->private_flow_entries = tmp_priv_flow_entries;
			target_targ->public_ip_port_hash = tmp_pub_hash;
			target_targ->public_entries = tmp_pub_entries;
			target_targ->public_ip_config_info = tmp_public_ip_config_info;
		}
	}
	return 0;
}

static void early_init_task_nat(struct task_args *targ)
{
	int ret;
	const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);
	if (!targ->private_ip_hash) {
		ret = lua_to_hash_nat(targ, prox_lua(), GLOBAL, targ->nat_table, socket_id);
		PROX_PANIC(ret != 0, "Failed to load NAT table from lua:\n%s\n", get_lua_to_errors());
	}
}

static void init_task_nat(struct task_base *tbase, struct task_args *targ)
{
	struct task_nat *task = (struct task_nat *)tbase;
	const int socket_id = rte_lcore_to_socket_id(targ->lconf->id);

	/* Use destination IP by default. */
	task->private = targ->use_src;

	PROX_PANIC(!strcmp(targ->nat_table, ""), "No nat table specified\n");
	task->lconf = targ->lconf;
	task->runtime_flags = targ->runtime_flags;

	task->public_ip_count = targ->public_ip_count;
	task->last_ip = targ->public_ip_count;
	task->private_ip_hash = targ->private_ip_hash;
	task->private_ip_port_hash = targ->private_ip_port_hash;
	task->private_ip_info = targ->private_ip_info;
	task->private_flow_entries = targ->private_flow_entries;
	task->public_ip_port_hash = targ->public_ip_port_hash;
	task->public_entries = targ->public_entries;
	task->public_ip_config_info = targ->public_ip_config_info;

	proto_ipsrc_portsrc_mask = _mm_set_epi32(BIT_0_TO_15, 0, ALL_32_BITS, BIT_8_TO_15);
	proto_ipdst_portdst_mask = _mm_set_epi32(BIT_16_TO_31, ALL_32_BITS, 0, BIT_8_TO_15);

	struct lpm4 *lpm;

	PROX_PANIC(!strcmp(targ->route_table, ""), "route table not specified\n");
	if (targ->flags & TASK_ARG_LOCAL_LPM) {
		int ret = lua_to_lpm4(prox_lua(), GLOBAL, targ->route_table, socket_id, &lpm);
		PROX_PANIC(ret, "Failed to load IPv4 LPM:\n%s\n", get_lua_to_errors());
		prox_sh_add_socket(socket_id, targ->route_table, lpm);
		task->number_free_rules = lpm->n_free_rules;
	} else {
		lpm = prox_sh_find_socket(socket_id, targ->route_table);
		if (!lpm) {
			int ret = lua_to_lpm4(prox_lua(), GLOBAL, targ->route_table, socket_id, &lpm);
			PROX_PANIC(ret, "Failed to load IPv4 LPM:\n%s\n", get_lua_to_errors());
			prox_sh_add_socket(socket_id, targ->route_table, lpm);
		}
	}
	task->ipv4_lpm = lpm->rte_lpm;
	task->next_hops = lpm->next_hops;
	task->number_free_rules = lpm->n_free_rules;

	for (uint32_t i = 0; i < MAX_HOP_INDEX; i++) {
		int tx_port = task->next_hops[i].mac_port.out_idx;
		if ((tx_port > targ->nb_txports - 1) && (tx_port > targ->nb_txrings - 1)) {
			PROX_PANIC(1, "Routing Table contains port %d but only %d tx port/ %d ring:\n", tx_port, targ->nb_txports, targ->nb_txrings);
		}
	}

	if (targ->nb_txrings) {
		struct task_args *dtarg;
		struct core_task ct;
		for (uint32_t i = 0; i < targ->nb_txrings; ++i) {
			ct = targ->core_task_set[0].core_task[i];
			dtarg = core_targ_get(ct.core, ct.task);
			dtarg = find_reachable_task_sending_to_port(dtarg);
			task->src_mac[i] = (0x0000ffffffffffff & ((*(uint64_t*)&prox_port_cfg[dtarg->tx_port_queue[0].port].eth_addr))) | ((uint64_t)ETYPE_IPv4 << (64 - 16));
			task->src_mac_from_dpdk_port[dtarg->tx_port_queue[0].port] = task->src_mac[i];
			plogx_dbg("src_mac = %lx for port %d %d\n", task->src_mac[i], i, dtarg->tx_port_queue[0].port);
		}
	} else {
		for (uint32_t i = 0; i < targ->nb_txports; ++i) {
			task->src_mac[i] = (0x0000ffffffffffff & ((*(uint64_t*)&prox_port_cfg[targ->tx_port_queue[i].port].eth_addr))) | ((uint64_t)ETYPE_IPv4 << (64 - 16));
			task->src_mac_from_dpdk_port[targ->tx_port_queue[0].port] = task->src_mac[i];
			plogx_dbg("src_mac = %lx for port %d %d\n", task->src_mac[i], i, targ->tx_port_queue[i].port);
		}
	}

	struct prox_port_cfg *port = find_reachable_port(targ);
	if (port) {
		task->offload_crc = port->requested_tx_offload & (DEV_TX_OFFLOAD_IPV4_CKSUM | DEV_TX_OFFLOAD_UDP_CKSUM);
	}
}

/* Basic static nat. */
static struct task_init task_init_nat = {
	.mode = CGNAT,
	.mode_str = "cgnat",
	.early_init = early_init_task_nat,
	.init = init_task_nat,
	.handle = handle_nat_bulk,
#ifdef SOFT_CRC
	.flag_features = TASK_FEATURE_TXQ_FLAGS_NOOFFLOADS|TASK_FEATURE_ROUTING|TASK_FEATURE_ZERO_RX,
#else
	.flag_features = TASK_FEATURE_ROUTING|TASK_FEATURE_ZERO_RX,
#endif
	.size = sizeof(struct task_nat),
};

__attribute__((constructor)) static void reg_task_nat(void)
{
	reg_task(&task_init_nat);
}