Add ping vnf using vhost-user & virtio-user

Change-Id: If1138e90dd5ebf3c4748dc9cc998c93bb037c4c7
Signed-off-by: Guo Ruijing <ruijing.guo@intel.com>

10 files changed, 352 insertions, 0 deletions

diff --git a/src/vnf/virtio-user-ping/01-add-single-file.patch b/src/vnf/virtio-user-ping/01-add-single-file.patch
new file mode 100644
index 0000000..a686502
--- /dev/null
+++ b/src/vnf/virtio-user-ping/01-add-single-file.patch
@@ -0,0 +1,171 @@
+Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
+---
+ lib/librte_eal/common/eal_common_options.c | 18 +++++++++++
+ lib/librte_eal/common/eal_internal_cfg.h   |  1 +
+ lib/librte_eal/common/eal_options.h        |  2 ++
+ lib/librte_eal/linuxapp/eal/eal.c          |  4 +--
+ lib/librte_eal/linuxapp/eal/eal_memory.c   | 49 +++++++++++++++++++++++++-----
+ 5 files changed, 64 insertions(+), 10 deletions(-)
+
+diff --git a/lib/librte_eal/common/eal_common_options.c b/lib/librte_eal/common/eal_common_options.c
+index f470195..4ad41b3 100644
+--- a/lib/librte_eal/common/eal_common_options.c
++++ b/lib/librte_eal/common/eal_common_options.c
+@@ -95,6 +95,7 @@ eal_long_options[] = {
+ 	{OPT_VFIO_INTR,         1, NULL, OPT_VFIO_INTR_NUM        },
+ 	{OPT_VMWARE_TSC_MAP,    0, NULL, OPT_VMWARE_TSC_MAP_NUM   },
+ 	{OPT_XEN_DOM0,          0, NULL, OPT_XEN_DOM0_NUM         },
++	{OPT_SINGLE_FILE,       0, NULL, OPT_SINGLE_FILE_NUM      },
+ 	{0,                     0, NULL, 0                        }
+ };
+ 
+@@ -933,6 +934,10 @@ eal_parse_common_option(int opt, const char *optarg,
+ 		core_parsed = 1;
+ 		break;
+ 
++	case OPT_SINGLE_FILE_NUM:
++		conf->single_file = 1;
++		break;
++
+ 	/* don't know what to do, leave this to caller */
+ 	default:
+ 		return 1;
+@@ -1025,6 +1030,17 @@ eal_check_common_options(struct internal_config *internal_cfg)
+ 		return -1;
+ 	}
+ 
++	if (internal_cfg->single_file && internal_cfg->force_sockets == 1) {
++		RTE_LOG(ERR, EAL, "Option --"OPT_SINGLE_FILE" cannot "
++			"be specified together with --"OPT_SOCKET_MEM"\n");
++		return -1;
++	}
++	if (internal_cfg->single_file && internal_cfg->hugepage_unlink) {
++		RTE_LOG(ERR, EAL, "Option --"OPT_HUGE_UNLINK" cannot "
++			"be specified together with --"OPT_SINGLE_FILE"\n");
++		return -1;
++	}
++
+ 	if (rte_eal_devargs_type_count(RTE_DEVTYPE_WHITELISTED_PCI) != 0 &&
+ 		rte_eal_devargs_type_count(RTE_DEVTYPE_BLACKLISTED_PCI) != 0) {
+ 		RTE_LOG(ERR, EAL, "Options blacklist (-b) and whitelist (-w) "
+@@ -1056,6 +1072,8 @@ eal_common_usage(void)
+ 	       "  -n CHANNELS         Number of memory channels\n"
+ 	       "  -m MB               Memory to allocate (see also --"OPT_SOCKET_MEM")\n"
+ 	       "  -r RANKS            Force number of memory ranks (don't detect)\n"
++	       "  --"OPT_SINGLE_FILE" Create single file for shared memory, and \n"
++	       "                      do not promise physical contiguity of memseg\n"
+ 	       "  -b, --"OPT_PCI_BLACKLIST" Add a PCI device in black list.\n"
+ 	       "                      Prevent EAL from using this PCI device. The argument\n"
+ 	       "                      format is <domain:bus:devid.func>.\n"
+diff --git a/lib/librte_eal/common/eal_internal_cfg.h b/lib/librte_eal/common/eal_internal_cfg.h
+index 7b7e8c8..82b0f97 100644
+--- a/lib/librte_eal/common/eal_internal_cfg.h
++++ b/lib/librte_eal/common/eal_internal_cfg.h
+@@ -61,6 +61,7 @@ struct hugepage_info {
+  */
+ struct internal_config {
+ 	volatile size_t memory;           /**< amount of asked memory */
++	volatile unsigned single_file;    /**< map all hugepages in single file */
+ 	volatile unsigned force_nchannel; /**< force number of channels */
+ 	volatile unsigned force_nrank;    /**< force number of ranks */
+ 	volatile unsigned no_hugetlbfs;   /**< true to disable hugetlbfs */
+diff --git a/lib/librte_eal/common/eal_options.h b/lib/librte_eal/common/eal_options.h
+index a881c62..e5da14a 100644
+--- a/lib/librte_eal/common/eal_options.h
++++ b/lib/librte_eal/common/eal_options.h
+@@ -83,6 +83,8 @@ enum {
+ 	OPT_VMWARE_TSC_MAP_NUM,
+ #define OPT_XEN_DOM0          "xen-dom0"
+ 	OPT_XEN_DOM0_NUM,
++#define OPT_SINGLE_FILE       "single-file"
++	OPT_SINGLE_FILE_NUM,
+ 	OPT_LONG_MAX_NUM
+ };
+ 
+diff --git a/lib/librte_eal/linuxapp/eal/eal.c b/lib/librte_eal/linuxapp/eal/eal.c
+index 7c78f2d..b6f2ca2 100644
+--- a/lib/librte_eal/linuxapp/eal/eal.c
++++ b/lib/librte_eal/linuxapp/eal/eal.c
+@@ -839,6 +839,8 @@ rte_eal_init(int argc, char **argv)
+ 	}
+ #endif
+ 
++	eal_thread_init_master(rte_config.master_lcore);
++
+ 	if (rte_eal_memory_init() < 0) {
+ 		rte_eal_init_alert("Cannot init memory\n");
+ 		rte_errno = ENOMEM;
+@@ -877,8 +879,6 @@ rte_eal_init(int argc, char **argv)
+ 	if (eal_plugins_init() < 0)
+ 		rte_eal_init_alert("Cannot init plugins\n");
+ 
+-	eal_thread_init_master(rte_config.master_lcore);
+-
+ 	ret = eal_thread_dump_affinity(cpuset, RTE_CPU_AFFINITY_STR_LEN);
+ 
+ 	RTE_LOG(DEBUG, EAL, "Master lcore %u is ready (tid=%x;cpuset=[%s%s])\n", diff --git a/lib/librte_eal/linuxapp/eal/eal_memory.c b/lib/librte_eal/linuxapp/eal/eal_memory.c
+index 618a09b..70c6536 100644
+--- a/lib/librte_eal/linuxapp/eal/eal_memory.c
++++ b/lib/librte_eal/linuxapp/eal/eal_memory.c
+@@ -982,20 +982,53 @@ rte_eal_hugepage_init(void)
+ 	/* get pointer to global configuration */
+ 	mcfg = rte_eal_get_configuration()->mem_config;
+ 
+-	/* hugetlbfs can be disabled */
+-	if (internal_config.no_hugetlbfs) {
+-		addr = mmap(NULL, internal_config.memory, PROT_READ | PROT_WRITE,
+-				MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
++	/* when hugetlbfs is disabled or single-file option is specified */
++	if (internal_config.no_hugetlbfs || internal_config.single_file) {
++		int fd;
++		uint64_t pagesize;
++		unsigned socket_id = rte_socket_id();
++		char filepath[MAX_HUGEPAGE_PATH];
++
++		if (internal_config.no_hugetlbfs) {
++			eal_get_hugefile_path(filepath, sizeof(filepath),
++					"/dev/shm", 0);
++			pagesize = RTE_PGSIZE_4K;
++		} else {
++			struct hugepage_info *hpi;
++
++			hpi = &internal_config.hugepage_info[0];
++			eal_get_hugefile_path(filepath, sizeof(filepath),
++					hpi->hugedir, 0);
++			pagesize = hpi->hugepage_sz;
++		}
++		fd = open(filepath, O_CREAT | O_RDWR, S_IRUSR | S_IWUSR);
++		if (fd < 0) {
++			RTE_LOG(ERR, EAL, "%s: open %s failed: %s\n",
++					__func__, filepath, strerror(errno));
++			return -1;
++		}
++
++		if (ftruncate(fd, internal_config.memory) < 0) {
++			RTE_LOG(ERR, EAL, "ftuncate %s failed: %s\n",
++					filepath, strerror(errno));
++			return -1;
++		}
++
++		addr = mmap(NULL, internal_config.memory,
++				PROT_READ | PROT_WRITE,
++				MAP_SHARED | MAP_POPULATE, fd, 0);
+ 		if (addr == MAP_FAILED) {
+-			RTE_LOG(ERR, EAL, "%s: mmap() failed: %s\n", __func__,
+-					strerror(errno));
++			RTE_LOG(ERR, EAL, "%s: mmap() failed: %s\n",
++					__func__, strerror(errno));
+ 			return -1;
+ 		}
+ 		mcfg->memseg[0].phys_addr = (phys_addr_t)(uintptr_t)addr;
+ 		mcfg->memseg[0].addr = addr;
+-		mcfg->memseg[0].hugepage_sz = RTE_PGSIZE_4K;
++		mcfg->memseg[0].hugepage_sz = pagesize;
+ 		mcfg->memseg[0].len = internal_config.memory;
+-		mcfg->memseg[0].socket_id = 0;
++		mcfg->memseg[0].socket_id = socket_id;
++
++		close(fd);
+ 		return 0;
+ 	}
+ 
diff --git a/src/vnf/virtio-user-ping/02-fix-nohuge-option.patch b/src/vnf/virtio-user-ping/02-fix-nohuge-option.patch
new file mode 100644
index 0000000..3243eaf
--- /dev/null
+++ b/src/vnf/virtio-user-ping/02-fix-nohuge-option.patch
@@ -0,0 +1,19 @@
+Signed-off-by: Jianfeng Tan <jianfeng.tan@intel.com>
+---
+ lib/librte_eal/bsdapp/eal/eal.c | 4 +---
+ 1 file changed, 1 insertion(+), 3 deletions(-)
+
+diff --git a/lib/librte_eal/bsdapp/eal/eal.c b/lib/librte_eal/bsdapp/eal/eal.c index 05f0c1f..b62ef69 100644
+--- a/lib/librte_eal/bsdapp/eal/eal.c
++++ b/lib/librte_eal/bsdapp/eal/eal.c
+@@ -546,9 +546,7 @@ rte_eal_init(int argc, char **argv)
+ 	}
+ 
+ 	if (internal_config.memory == 0 && internal_config.force_sockets == 0) {
+-		if (internal_config.no_hugetlbfs)
+-			internal_config.memory = MEMSIZE_IF_NO_HUGE_PAGE;
+-		else
++		if (!internal_config.no_hugetlbfs)
+ 			internal_config.memory = eal_get_hugepage_mem_size();
+ 	}
+
diff --git a/src/vnf/virtio-user-ping/Dockerfile b/src/vnf/virtio-user-ping/Dockerfile
new file mode 100644
index 0000000..8b09548
--- /dev/null
+++ b/src/vnf/virtio-user-ping/Dockerfile
@@ -0,0 +1,12 @@
+FROM ubuntu:16.04
+
+RUN apt-get update
+RUN apt-get -y install sudo apt-transport-https devscripts git wget vim net-tools
+ADD 01-add-single-file.patch /root
+ADD 02-fix-nohuge-option.patch /root
+ADD virtio-user.patch /root
+ADD build_vpp.sh  /root
+ADD setup_vpp.sh  /root
+ADD startup.conf /root
+ADD setup_virtio_user.sh /root
+RUN /root/build_vpp.sh
diff --git a/src/vnf/virtio-user-ping/Vagrantfile b/src/vnf/virtio-user-ping/Vagrantfile
new file mode 100644
index 0000000..3f5a477
--- /dev/null
+++ b/src/vnf/virtio-user-ping/Vagrantfile
@@ -0,0 +1,22 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+
+  config.vm.box = "bento/ubuntu-16.04"
+  
+  config.vm.define "ovs-virtio" do | h |
+    config.vm.host_name = "ovs-virtio"
+    config.vm.provision "shell", path: "setup_ovs_virtio.sh", privileged: false
+    config.vm.provider :virtualbox do |v|
+      v.customize ["modifyvm", :id, "--memory", 8192]
+      v.customize ["modifyvm", :id, "--cpus", 4]
+      v.customize "post-boot",["controlvm", :id, "setlinkstate1", "on"]
+      v.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.1", "1"]
+      v.customize ["setextradata", :id, "VBoxInternal/CPUM/SSE4.2", "1"]
+    end
+  end
+end
diff --git a/src/vnf/virtio-user-ping/build_vpp.sh b/src/vnf/virtio-user-ping/build_vpp.sh
new file mode 100755
index 0000000..9404bb1
--- /dev/null
+++ b/src/vnf/virtio-user-ping/build_vpp.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+sudo apt-get update
+sudo apt-get -y install sudo apt-transport-https devscripts git wget vim net-tools
+cd /root
+git clone https://gerrit.fd.io/r/vpp
+cd vpp
+git checkout stable/1707
+cp  ../01-add-single-file.patch  dpdk/dpdk-17.05_patches
+cp  ../02-fix-nohuge-option.patch  dpdk/dpdk-17.05_patches
+patch -p1 < ../virtio-user.patch
+make UNATTENDED=yes install-dep || true
+make bootstrap
+make build; find . -type f | grep "install.*bin" | xargs -I {} cp {} /usr/bin/
diff --git a/src/vnf/virtio-user-ping/setup_ovs_virtio.sh b/src/vnf/virtio-user-ping/setup_ovs_virtio.sh
new file mode 100755
index 0000000..a565589
--- /dev/null
+++ b/src/vnf/virtio-user-ping/setup_ovs_virtio.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+sudo sysctl -w vm.nr_hugepages=1024
+sudo mount -t hugetlbfs -o pagesize=2M none /dev/hugepages
+
+sudo apt-get update -y
+sudo apt-get install -y openvswitch-switch-dpdk
+sudo update-alternatives --set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk
+
+echo "DPDK_OPTS='--dpdk -c 0x1 -n 4 -m 1024 --vhost-owner docker --vhost-perm 0664'" | sudo tee -a /etc/default/openvswitch-switch
+sudo service dpdk restart
+sudo service openvswitch-switch restart
+sleep 10
+
+sudo ovs-vsctl add-br br-dpdk -- set bridge br-dpdk datapath_type=netdev
+sudo ovs-vsctl add-port br-dpdk vhost-user-1 -- set Interface vhost-user-1 type=dpdkvhostuser
+sudo ifconfig br-dpdk 192.168.3.1/24 up
+
+sudo sysctl -w vm.nr_hugepages=2048
+sudo apt-get install -y docker.io
+sudo docker build -t vpp /vagrant/
+sudo docker run -itd -v /dev/hugepages/:/dev/hugepages/ -v /var/run/openvswitch:/var/run/openvswitch -v /vagrant:/vagrant vpp /root/setup_vpp.sh
+sleep 20
+ping -c4 192.168.3.2
diff --git a/src/vnf/virtio-user-ping/setup_virtio_user.sh b/src/vnf/virtio-user-ping/setup_virtio_user.sh
new file mode 100755
index 0000000..03e987f
--- /dev/null
+++ b/src/vnf/virtio-user-ping/setup_virtio_user.sh
@@ -0,0 +1,11 @@
+#!/bin/bash
+id=$(ip a | grep dummy | cut -f2 -d"-" | cut -f1 -d":")
+sed -i "s/vhost-user-1/vhost-user-$id/" /root/startup.conf
+mkdir -p /run/vpp
+vpp -c /root/startup.conf &
+sleep 10
+chmod 777 /run/vpp/cli.sock
+vppctl set int state VirtioUser0/0/0 up
+ip=$(cut -f6 -d":" /var/run/cni/netconf-$id | cut -f1 -d" ")
+vppctl set int ip add VirtioUser0/0/0 $ip/24
+sleep 1000000
diff --git a/src/vnf/virtio-user-ping/setup_vpp.sh b/src/vnf/virtio-user-ping/setup_vpp.sh
new file mode 100755
index 0000000..eef672c
--- /dev/null
+++ b/src/vnf/virtio-user-ping/setup_vpp.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+mkdir -p /run/vpp
+vpp -c /root/startup.conf &
+sleep 10
+chmod 777 /run/vpp/cli.sock
+vppctl set int state VirtioUser0/0/0 up
+vppctl set int ip add VirtioUser0/0/0 192.168.3.2/24
+sleep 1000000
diff --git a/src/vnf/virtio-user-ping/startup.conf b/src/vnf/virtio-user-ping/startup.conf
new file mode 100644
index 0000000..ce8badc
--- /dev/null
+++ b/src/vnf/virtio-user-ping/startup.conf
@@ -0,0 +1,23 @@
+unix {
+  nodaemon
+  log /tmp/vpp.log
+  full-coredump
+  cli-listen /run/vpp/cli.sock
+}
+
+api-trace {
+  on
+}
+
+cpu {
+}
+
+plugins
+{
+  path /root/vpp/build-root/install-vpp_debug-native/vpp/lib64/vpp_plugins
+}
+
+dpdk {
+  huge-dir /dev/hugepages
+  virtio-user /var/run/openvswitch/vhost-user-1
+}
diff --git a/src/vnf/virtio-user-ping/virtio-user.patch b/src/vnf/virtio-user-ping/virtio-user.patch
new file mode 100644
index 0000000..2504785
--- /dev/null
+++ b/src/vnf/virtio-user-ping/virtio-user.patch
@@ -0,0 +1,49 @@
+diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c
+index 68c55f3..1acafcc 100755
+--- a/src/plugins/dpdk/device/init.c
++++ b/src/plugins/dpdk/device/init.c
+@@ -827,6 +827,7 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
+   u8 huge_dir = 0;
+   u8 file_prefix = 0;
+   u8 *socket_mem = 0;
++  u8 *virtio_user = 0;
+ 
+   conf->device_config_index_by_pci_addr = hash_create (0, sizeof (uword));
+   log_level = RTE_LOG_NOTICE;
+@@ -852,6 +853,17 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
+       else if (unformat (input, "no-multi-seg"))
+ 	conf->no_multi_seg = 1;
+ 
++
++      /* hardcode here for quick poc */
++      else if (unformat (input, "virtio-user %s", &virtio_user))
++        {
++           vec_add1 (conf->eal_init_args, (u8*)"-m 1024");
++           vec_add1 (conf->eal_init_args, (u8*)"--no-pci");
++           vec_add1 (conf->eal_init_args, (u8*)"--single-file");
++           tmp = format (0, "--vdev=virtio_user0,path=%s", virtio_user);
++           vec_add1 (conf->eal_init_args, (u8*)tmp);
++        }
++
+       else if (unformat (input, "dev default %U", unformat_vlib_cli_sub_input,
+ 			 &sub_input))
+ 	{
+@@ -1168,18 +1180,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input)
+ 
+ #undef _
+ 
+-  /* set master-lcore */
+-  tmp = format (0, "--master-lcore%c", 0);
+-  vec_add1 (conf->eal_init_args, tmp);
+-  tmp = format (0, "%u%c", tm->main_lcore, 0);
+-  vec_add1 (conf->eal_init_args, tmp);
+-
+-  /* set socket-mem */
+-  tmp = format (0, "--socket-mem%c", 0);
+-  vec_add1 (conf->eal_init_args, tmp);
+-  tmp = format (0, "%s%c", socket_mem, 0);
+-  vec_add1 (conf->eal_init_args, tmp);
+-
+   /* NULL terminate the "argv" vector, in case of stupidity */
+   vec_add1 (conf->eal_init_args, 0);
+   _vec_len (conf->eal_init_args) -= 1;