Adding centos.json to be used with packer to generate a VM with PROX

Based on a CentOS cloud image downloadable from the web, you can now use
packer to create an image that has DPDK and PROX installed in the
proper directories. The VM is also tuned for guest VNFs.
A service is also created to deal with the proper isolated_cpu settings
for tuned. The service will also make sure uio and igb_uio are loaded.
The prox_user_data.sh is not needed any more and hence deleted.
ctrl-c of runrapid.py will now also stop prox in all the VMs so a new
test can be started.

Change-Id: I16cc59878e2d4912757f42e05f98d51dff5feb89
Signed-off-by: Luc Provoost <luc.provoost@intel.com>

16 files changed, 328 insertions, 84 deletions

diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/README b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/README
index 2dac5b69..cc940fc6 100644
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/README
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/README
@@ -21,8 +21,9 @@ rapid is a set of files offering an easy way to do a sanity check of the
 dataplane performance of an OpenStack environment.
 
 Copy the files in a directory on a machine that can run the OpenStack CLI
-commands and that can reach the OpenStack public network. Also create a qcow2
-image in the same directory with the following characteristics:
+commands and that can reach the OpenStack public network.
+One option is to create a qcow2 image in the same directory with the
+following characteristics:
 * Name of the qcow2 file should be: rapidVM.qcow2
   This default name can be overruled on the rapid command line (--image_file)
 * Should have DPDK and PROX installed. PROX should be in /root/prox/ directory
@@ -31,8 +32,19 @@ image in the same directory with the following characteristics:
 * Compile prox with 'make crc=soft'. This is a workaround for some cases where the crc calculation offload is not working as expected.
 * Compile dpdk to support AESN-NI Multi Buffer Crypto Poll Mode Driver: http://dpdk.org/doc/guides/cryptodevs/aesni_mb.html
 
-Source the openrc file of the OpenStack environment so that the OpenStack CLI
-commands can be run:
+Alternatively, you can create an image in OpenStack using Packer and then specify
+the image to be used as a parameter to createrapid.py. First step is to upload an
+existing CentOs cloud image from the internet into OpenStack.
+Check out: https://cloud.centos.org/centos/7/images/
+You should now source the proper .rc file so Packer can connect to your OpenStack.
+There are 2 files: centos.json and deploycentos.sh, allowing you to create
+an image automatically. Run
+  # packer build centos.json
+Note that this procedure is not only installing the necessary tools to run PROX,
+but also does some system optimizations (tuned).
+
+Now that the image exists in OpenStack, source the openrc file of the OpenStack
+environment so that the OpenStack CLI commands can be run:
   # source openrc
 Now you can run the createrapid.py file. Use help for more info on the usage:
   # ./createrapid.py --help
@@ -45,6 +57,10 @@ You can use floating IP addresses by specifying the floating IP network
 --floating_network NETWORK
 or directly connect throught the INTERNAL_NETWORK by using the following parameter:
 --floating_network NO
+/etc/resolv.conf will contain DNS info from the "best" interface. Since we are
+deploying VMs with multiple interface on different networks, this info might be
+taken from the "wrong" network (e.g. the dataplane network). It is good to provide
+also a working DNS on that network.
 
 Now you can run the runrapid.py file. Use help for more info on the usage:
   # ./runrapid.py --help
@@ -88,8 +104,11 @@ An example of the env file generated by createrapid.py can be found below.
 Note that this file can be created manually in case the stack is created in a
 different way (not using the createrapid.py). This can be useful in case you are
 not using OpenStack as a VIM or when using special configurations that cannot be
-achieved using createrapid.py. Only the [Mx] sections are used as
-input for runrapid.py.
+achieved using creater needed for runrapid are:
+* all info in the [Mx] sections
+* the key information in the [OpenStack] section
+* the total_number_of_vms information in the [rapid] section
+
 [DEFAULT]
 admin_ip = none
 
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/basicrapid.test b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/basicrapid.test
index 6a9998c6..84f8192e 100644
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/basicrapid.test
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/basicrapid.test
@@ -46,9 +46,8 @@ config_file = swap.cfg
 group1cores = [1]
 
 [test1]
-cmd=run_speedtest(sock[0],sock[1])
+cmd=run_speedtest(socks[0],socks[1])
 [test2]
-cmd=run_sizetest(sock[0],sock[1])
+cmd=run_sizetest(socks[0],socks[1])
 [test3]
-cmd=run_flowtest(sock[0],sock[1])
-
+cmd=run_flowtest(socks[0],socks[1])
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/centos.json b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/centos.json
new file mode 100644
index 00000000..bdca7256
--- /dev/null
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/centos.json
@@ -0,0 +1,33 @@
+{
+"_Copyright": "Copyright (c) 2010-2018 Intel Corporation",
+"_License":   "SPDX-License-Identifier: Apache-2.0",
+"builders": [
+   {
+"type": "openstack",
+"ssh_username": "centos",
+"image_name": "newrapidVM",
+"source_image": "aa0f8333-d8e1-42db-8210-6b39d9ee7267",
+"flavor": "7aaec028-eccc-4612-8f9b-29a1e039943a",
+"networks": "97bbe96f-ee1b-4789-ace8-e922f76bea3f",
+"use_floating_ip": true,
+"floating_ip_pool": "1f564356-8c23-41e2-8292-7696ffd98adf",
+"security_groups": "8fd498fc-5bd8-4be6-94e4-a2cc3b02d151"
+   }
+],
+"provisioners": [
+   {
+   "type": "file",
+   "source": "./check_prox_system_setup.sh",
+   "destination": "/home/centos/"
+   },
+   {
+   "type": "file",
+   "source": "./check-prox-system-setup.service",
+   "destination": "/home/centos/"
+   },
+   {
+   "type": "shell",
+   "script": "deploycentos.sh"
+   }
+]
+}
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/check-prox-system-setup.service b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/check-prox-system-setup.service
new file mode 100644
index 00000000..a55e0c08
--- /dev/null
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/check-prox-system-setup.service
@@ -0,0 +1,11 @@
+[Unit]
+Description=Check PROX system setup (isolated_cores, uio)
+DefaultDependencies=no
+After=multi-user.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/libexec/check_prox_system_setup.sh
+
+[Install]
+WantedBy=multi-user.target
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/check_prox_system_setup.sh b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/check_prox_system_setup.sh
new file mode 100755
index 00000000..48999510
--- /dev/null
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/check_prox_system_setup.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+##
+## Copyright (c) 2010-2018 Intel Corporation
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+NCPUS="$(lscpu | egrep '^CPU\(s\):' | awk '{ print $2 }')"
+MAXCOREID="$((NCPUS-1))"
+
+filename="/etc/tuned/realtime-virtual-guest-variables.conf"
+logfile="/home/centos/prox_system_setup.log"
+if [ -f "$filename" ]
+then
+        while read -r line
+        do
+                case $line in
+                        isolated_cores=1-$MAXCOREID*)
+                                echo "Isolated CPU(s) OK, no reboot: $line">>$logfile
+				modprobe uio
+				insmod /root/dpdk/build/kmod/igb_uio.ko
+                                exit 0
+                        ;;
+                        isolated_cores=*)
+                                echo "Isolated CPU(s) NOK, change the config and reboot: $line">>$logfile
+                                sed -i "/^isolated_cores=.*/c\isolated_cores=1-$MAXCOREID" $filename
+				tuned-adm profile realtime-virtual-guest
+                                reboot
+				exit 0
+                        ;;
+                        *)
+                                echo "$line"
+                        ;;
+                esac
+        done < "$filename"
+        echo "isolated_cores=1-$MAXCOREID" >> $filename
+	echo "No Isolated CPU(s) defined in config, line added: $line">>$logfile
+	tuned-adm profile realtime-virtual-guest
+        reboot
+else
+        echo "$filename not found.">>$logfile
+fi
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/createrapid.py b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/createrapid.py
index ffba5013..f21b91d7 100755
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/createrapid.py
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/createrapid.py
@@ -31,7 +31,7 @@ from logging import handlers
 from prox_ctrl import prox_ctrl
 import ConfigParser
 
-version="18.3.27"
+version="18.6.15"
 stack = "rapid" #Default string for stack. This is not an OpenStack Heat stack, just a group of VMs
 vms = "rapidVMs" #Default string for vms file
 key = "prox" # default name for kay
@@ -296,7 +296,8 @@ else:
 		log.exception("Failed to create subnet: " + subnet)
 		raise Exception("Failed to create subnet: " + subnet)
 
-
+ServerToBeCreated=[]
+ServerName=[]
 config = ConfigParser.RawConfigParser()
 vmconfig = ConfigParser.RawConfigParser()
 vmconfig.read(vms+'.vms')
@@ -306,16 +307,18 @@ for vm in range(1, int(total_number_of_VMs)+1):
 	flavor_meta_data = vmconfig.get('VM%d'%vm, 'flavor_meta_data')
 	boot_info = vmconfig.get('VM%d'%vm, 'boot_info')
 	SRIOV_port = vmconfig.get('VM%d'%vm, 'SRIOV_port')
-	server_name = '%s-VM%d'%(stack,vm)
+	ServerName.append('%s-VM%d'%(stack,vm))
 	flavor_name = '%s-VM%d-flavor'%(stack,vm)
-	log.debug("Checking server: "+server_name)
-	cmd = 'openstack server show '+server_name
+	log.debug("Checking server: "+ServerName[-1])
+	cmd = 'openstack server show '+ServerName[-1]
 	log.debug (cmd)
 	cmd = cmd + ' |grep "\sname\s" | tr -s " " | cut -d" " -f 4'
 	ServerExist = subprocess.check_output(cmd , shell=True).strip()
-	if ServerExist == server_name:
-		log.info("Server ("+server_name+") already active")
+	if ServerExist == ServerName[-1]:
+		log.info("Server ("+ServerName[-1]+") already active")
+		ServerToBeCreated.append("no")
 	else:
+		ServerToBeCreated.append("yes")
 		# Checking if the flavor already exists, if not create it
 		log.debug("Checking flavor: "+flavor_name)
 		cmd = 'openstack flavor show '+flavor_name
@@ -353,24 +356,25 @@ for vm in range(1, int(total_number_of_VMs)+1):
 		else:
 			wait = ' '
 		log.info("Creating server...")
-		cmd = 'openstack server create --flavor %s --key-name %s --image %s %s %s%s%s'%(flavor_name,key,image,nic_info,boot_info,wait,server_name)
+		cmd = 'openstack server create --flavor %s --key-name %s --image %s %s %s%s%s'%(flavor_name,key,image,nic_info,boot_info,wait,ServerName[-1])
 		log.debug(cmd)
 		cmd = cmd + ' |grep "\sname\s" | tr -s " " | cut -d" " -f 4'
 		ServerExist = subprocess.check_output(cmd , shell=True).strip()
-		if floating_network <> 'NO':
+if floating_network <> 'NO':
+	for vm in range(0, int(total_number_of_VMs)):
+		if ServerToBeCreated[vm] =="yes":
 			log.info('Creating floating IP ...')
 			cmd = 'openstack floating ip create  ' + floating_network
 			log.debug(cmd)
 			cmd = cmd + ' |grep "floating_ip_address " | tr -s " " | cut -d"|" -f 3'
 			vmAdminIP = subprocess.check_output(cmd , shell=True).strip()
 			log.info('Associating floating IP ...')
-			cmd = 'openstack server add floating ip %s %s'%(server_name,vmAdminIP)
+			cmd = 'openstack server add floating ip %s %s'%(ServerName[vm],vmAdminIP)
 			log.debug(cmd)
 			output = subprocess.check_output(cmd , shell=True).strip()
-			print (output)
+		
 for vm in range(1, int(total_number_of_VMs)+1):
-	server_name = '%s-VM%d'%(stack,vm)
-	cmd = 'openstack server show %s'%(server_name)
+	cmd = 'openstack server show %s'%(ServerName[vm-1])
 	log.debug(cmd)
 	output = subprocess.check_output(cmd , shell=True).strip()
         searchString = '.*%s.*?([0-9]*\.[0-9]*\.[0-9]*\.[0-9]*)' %(dataplane_network)
@@ -381,15 +385,15 @@ for vm in range(1, int(total_number_of_VMs)+1):
 	vmAdminIP = matchObj.group(2)
 	if vmAdminIP == None:
 		vmAdminIP = matchObj.group(1)
-	cmd = 'openstack port list |grep  %s | tr -s " " | cut -d"|" -f 4'%(vmDPIP)
+	cmd = 'openstack port list |egrep  "\\b%s\\b" | tr -s " " | cut -d"|" -f 4'%(vmDPIP)
 	log.debug(cmd)
 	vmDPmac = subprocess.check_output(cmd , shell=True).strip()
 	config.add_section('M%d'%vm)
-	config.set('M%d'%vm, 'name', server_name)
+	config.set('M%d'%vm, 'name', ServerName[vm-1])
 	config.set('M%d'%vm, 'admin_ip', vmAdminIP)
 	config.set('M%d'%vm, 'dp_ip', vmDPIP)
 	config.set('M%d'%vm, 'dp_mac', vmDPmac)
-	log.info('%s: (admin IP: %s), (dataplane IP: %s), (dataplane MAC: %s)' % (server_name,vmAdminIP,vmDPIP,vmDPmac))
+	log.info('%s: (admin IP: %s), (dataplane IP: %s), (dataplane MAC: %s)' % (ServerName[vm-1],vmAdminIP,vmDPIP,vmDPmac))
 
 config.add_section('OpenStack')
 config.set('OpenStack', 'stack', stack)
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/deploycentos.sh b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/deploycentos.sh
new file mode 100644
index 00000000..89921d5d
--- /dev/null
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/deploycentos.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+
+##
+## Copyright (c) 2010-2018 Intel Corporation
+##
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+##     http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+sudo sh -c '(echo "export RTE_TARGET=\"build\"";echo "export RTE_SDK=\"/root/dpdk\"";echo "export AESNI_MULTI_BUFFER_LIB_PATH=\"/home/centos/intel-ipsec-mb-0.48\"";) >> /root/.bashrc'
+sudo yum install deltarpm -y
+sudo yum update -y
+sudo yum-config-manager --add-repo http://www.nasm.us/nasm.repo
+sudo yum install git wget gcc unzip libpcap-devel ncurses-devel libedit-devel lua-devel kernel-devel iperf3 pciutils numactl-devel vim tuna openssl-devel nasm -y
+# Enabling root ssh access
+sudo sed -i '/disable_root: 1/c\disable_root: 0' /etc/cloud/cloud.cfg
+# The following line is commented since this was a workaround for a problem with the content of /etc/resolv.conf.
+# That file could contain DNS information coming from the dataplane which might be wrong. A solution is to confire the correct DNS for the dataplne
+# in OpenStack.  DNS might be slowing down ssh access. We don't need that for our dataplane benchmarking purposes
+# sudo sed -i '/#UseDNS yes/c\UseDNS no' /etc/ssh/sshd_config
+
+# Mounting huge pages to be used by DPDK
+sudo mkdir -p /mnt/huge
+sudo umount `awk '/hugetlbfs/ { print $2 }' /proc/mounts` >/dev/null 2>&1
+sudo mount -t hugetlbfs nodev /mnt/huge/
+sudo sh -c '(echo "vm.nr_hugepages = 1024") > /etc/sysctl.conf'
+
+# Downloading the Multi-buffer library
+wget https://github.com/01org/intel-ipsec-mb/archive/v0.48.zip
+unzip v0.48.zip
+export  AESNI_MULTI_BUFFER_LIB_PATH=/home/centos/intel-ipsec-mb-0.48
+cd $AESNI_MULTI_BUFFER_LIB_PATH
+make -j8
+# Clone and compile DPDK
+cd /home/centos/
+git clone http://dpdk.org/git/dpdk
+cd dpdk
+git checkout v17.11
+export RTE_TARGET=build
+export RTE_SDK=/home/centos/dpdk
+make config T=x86_64-native-linuxapp-gcc
+# The next sed lines make sure that we can compile DPDK 17.11 with a relatively new OS. Using a newer DPDK (18.5) should also resolve this issue
+sudo sed -i '/CONFIG_RTE_LIBRTE_KNI=y/c\CONFIG_RTE_LIBRTE_KNI=n' /home/centos/dpdk/build/.config
+sudo sed -i '/CONFIG_RTE_LIBRTE_PMD_KNI=y/c\CONFIG_RTE_LIBRTE_PMD_KNI=n' /home/centos/dpdk/build/.config
+sudo sed -i '/CONFIG_RTE_KNI_KMOD=y/c\CONFIG_RTE_KNI_KMOD=n' /home/centos/dpdk/build/.config
+sudo sed -i '/CONFIG_RTE_KNI_PREEMPT_DEFAULT=y/c\CONFIG_RTE_KNI_PREEMPT_DEFAULT=n' /home/centos/dpdk/build/.config
+# Compile with MB library
+sudo sed -i '/CONFIG_RTE_LIBRTE_PMD_AESNI_MB=n/c\CONFIG_RTE_LIBRTE_PMD_AESNI_MB=y' /home/centos/dpdk/build/.config
+make -j8 
+cd /home/centos
+# Copy everything to root since the scripts are assuming /root as the directory for PROX
+sudo cp -r dpdk /root/
+
+# Clone and compile PROX
+git clone https://git.opnfv.org/samplevnf
+cp -r /home/centos/samplevnf/VNFs/DPPD-PROX /home/centos/prox
+cd /home/centos/prox
+make -j8
+cd /home/centos
+# Copy everything to root since the scripts are assuming /root as the directory for PROX
+sudo cp -r /home/centos/prox /root/
+
+# Enabling tuned with the realtime-virtual-guest profile
+wget http://linuxsoft.cern.ch/cern/centos/7/rt/x86_64/Packages/tuned-profiles-realtime-2.8.0-5.el7_4.2.noarch.rpm
+wget http://linuxsoft.cern.ch/cern/centos/7/rt/x86_64/Packages/tuned-profiles-nfv-guest-2.8.0-5.el7_4.2.noarch.rpm
+# Install with --nodeps. The latest CentOS cloud images come with a tuned version higher than 2.8. These 2 packages however
+# do not depend on v2.8 and also work with tuned 2.9. Need to be careful in the future
+sudo rpm -ivh /home/centos/tuned-profiles-realtime-2.8.0-5.el7_4.2.noarch.rpm --nodeps
+sudo rpm -ivh /home/centos/tuned-profiles-nfv-guest-2.8.0-5.el7_4.2.noarch.rpm --nodeps
+# Although we do no know how many cores the VM will have when begin deployed for real testing, we already put a number for the
+# isolated CPUs so we can start the realtime-virtual-guest profile. If we don't, that command will fail.
+# When the VM will be instantiated, the check_kernel_params service will check for the real number of cores available to this VM 
+# and update the realtime-virtual-guest-variables.conf accordingly.
+echo "isolated_cores=1" | sudo tee -a /etc/tuned/realtime-virtual-guest-variables.conf
+sudo tuned-adm profile realtime-virtual-guest
+
+# Install the check_tuned_params service to make sure that the grub cmd line has the right cpus in isolcpu. The actual number of cpu's
+# assigned to this VM depends on the flavor used. We don't know at this time what that will be.
+sudo cp -r /home/centos/check_prox_system_setup.sh /usr/local/libexec/
+sudo cp -r /home/centos/check-prox-system-setup.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable check-prox-system-setup.service
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/devbind.sh b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/devbind.sh
index adc184e3..f9fc03ce 100755
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/devbind.sh
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/devbind.sh
@@ -24,3 +24,4 @@ then
 else
        echo Assuming port is already bound to DPDK
 fi
+exit 0
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/impair.cfg b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/impair.cfg
index e8b3801d..f52cc5d9 100644
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/impair.cfg
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/impair.cfg
@@ -42,6 +42,5 @@ sub mode=l3
 rx port=if0
 tx port=if0
 delay us=10
-probability=100
+probability=80
 local ipv4=${local_ip}
-
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/impair.test b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/impair.test
index 3042e722..27078c53 100644
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/impair.test
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/impair.test
@@ -52,4 +52,5 @@ config_file = swap.cfg
 group1cores = [1]
 
 [test1]
-cmd=run_speedtest(sock[0],sock[2])
+cmd=run_impairtest(socks[0],socks[2],8)
+# The last parameter specifies the speed at which this test will run
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/irq.test b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/irq.test
index 3ad014d5..2d413c1d 100644
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/irq.test
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/irq.test
@@ -48,9 +48,9 @@ group1cores = [1,2,3]
 
 
 [test1]
-cmd=run_irqtest(sock[0])
+cmd=run_irqtest(socks[0])
 [test2]
-cmd=run_irqtest(sock[1])
+cmd=run_irqtest(socks[1])
 [test3]
-cmd=run_irqtest(sock[2])
+cmd=run_irqtest(socks[2])
 
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/l2zeroloss.test b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/l2zeroloss.test
index 1ea7f0a2..3d3bb13f 100644
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/l2zeroloss.test
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/l2zeroloss.test
@@ -45,9 +45,8 @@ config_file = l2swap.cfg
 group1cores = [1]
 
 [test1]
-cmd=run_speedtest(sock[0],sock[1])
+cmd=run_speedtest(socks[0],socks[1])
 [test2]
-cmd=run_sizetest(sock[0],sock[1])
+cmd=run_sizetest(socks[0],socks[1])
 [test3]
-cmd=run_flowtest(sock[0],sock[1])
-
+cmd=run_flowtest(socks[0],socks[1])
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/prox_user_data.sh b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/prox_user_data.sh
deleted file mode 100755
index f211934a..00000000
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/prox_user_data.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-
-##
-## Copyright (c) 2010-2017 Intel Corporation
-##
-## Licensed under the Apache License, Version 2.0 (the "License");
-## you may not use this file except in compliance with the License.
-## You may obtain a copy of the License at
-##
-##     http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-##
-
-echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
-mount  -t hugetlbfs nodev /mnt/huge
-modprobe uio
-insmod /root/dpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
-iptables -F
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/rapidVMs.vms b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/rapidVMs.vms
index cf7b2c8d..1e45ed02 100644
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/rapidVMs.vms
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/rapidVMs.vms
@@ -19,8 +19,10 @@
 total_number_of_vms=3
 flavor_info=--ram 4096 --disk 20 --vcpus 4
 ;flavor_meta_data=--property hw:mem_page_size=large --property hw:cpu_policy=dedicated --property hw:cpu_thread_policy=isolate --property hw:numa_nodes=1 --property hw:numa_cpus.0=0,1,2,3 --property hw:numa_mempolicy=strict --property hw:numa_mem.0=4096
-flavor_meta_data=--property hw:mem_page_size=large --property hw:cpu_policy=dedicated --property hw:cpu_thread_policy=isolate
-boot_info=--availability-zone nova --user-data prox_user_data.sh --security-group default
+;flavor_meta_data=--property hw:mem_page_size=large --property hw:cpu_policy=dedicated --property hw:cpu_thread_policy=isolate
+;flavor_meta_data=--property hw:mem_page_size=large --property hw:cpu_policy=dedicated
+flavor_meta_data=--property hw:mem_page_size=large --property hw:cpu_policy=dedicated --property hw:cpu_realtime=yes --property hw:cpu_realtime_mask=^0
+boot_info=--availability-zone nova --security-group default
 SRIOV_port=NO
 
 [VM1]
@@ -29,3 +31,30 @@ SRIOV_port=NO
 
 [VM3]
 
+[VM4]
+
+[VM5]
+
+[VM6]
+
+[VM7]
+
+[VM8]
+
+[VM9]
+
+[VM10]
+
+[VM11]
+
+[VM12]
+
+[VM13]
+
+[VM14]
+
+[VM15]
+
+[VM16]
+
+
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/runrapid.py b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/runrapid.py
index 0f523cc0..e9684416 100755
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/runrapid.py
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/runrapid.py
@@ -31,8 +31,9 @@ from logging import handlers
 from prox_ctrl import prox_ctrl
 import ConfigParser
 import ast
+import atexit
 
-version="18.3.27"
+version="18.6.15"
 env = "rapid" #Default string for environment
 test = "basicrapid" #Default string for test
 loglevel="DEBUG" # sets log level for writing to file
@@ -182,7 +183,7 @@ def connect_client(client):
 	log.debug("Connected to VM on %s" % client.ip())
 
 def run_iteration(gensock,sutsock):
-	sleep_time = 2
+	sleep_time = 3
 	# Sleep_time is needed to be able to do accurate measurements to check for packet loss. We need to make this time large enough so that we do not take the first measurement while some packets from the previous tests migth still be in flight
 	time.sleep(sleep_time)
 	abs_old_rx, abs_old_tx, abs_old_drop, abs_old_tsc, abs_tsc_hz = gensock.core_stats(genstatcores)
@@ -260,7 +261,7 @@ def run_speedtest(gensock,sutsock):
 	gensock.set_size(gencores,0,size) # This is setting the frame size
 	gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
 	gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
-	# This will only work when using sending UDP packets. For different protocls and ehternet types, we would need a differnt calculation
+	# This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
         while (maxspeed-minspeed > ACCURACY):
                 attempts += 1
                 print('Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
@@ -416,10 +417,11 @@ def run_irqtest(sock):
 	for j,bucket in enumerate(buckets,start=1):
 		irq[0][j] = '<'+ bucket
 	irq[0][-1] = '>'+ buckets [-2]
+	sock.start(irqcores)
+	time.sleep(2)
 	for j,bucket in enumerate(buckets,start=1):
 		for i,irqcore in enumerate(irqcores,start=1):
 			old_irq[i][j] = sock.irq_stats(irqcore,j-1)
-	sock.start(irqcores)
 	time.sleep(float(runtime))
 	sock.stop(irqcores)
 	for i,irqcore in enumerate(irqcores,start=1):
@@ -429,19 +431,41 @@ def run_irqtest(sock):
 			if diff == 0:
 				irq[i][j] = '0'
 			else:
-				irq[i][j] = diff/float(runtime)
+				irq[i][j] = str(round(diff/float(runtime), 2))
 	log.info('\n'.join([''.join(['{:>12}'.format(item) for item in row]) for row in irq]))
 
+def run_impairtest(gensock,sutsock,speed):
+        log.info("+-----------------------------------------------------------------------------------------------------------------------------------------------------------------+")
+        log.info("| Generator is sending UDP (1 flow) packets (64 bytes) to SUT via GW dropping and delaying packets. SUT sends packets back. Use ctrl-c to stop the test           |")
+	log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
+	log.info("| Test   |  Speed requested   | Sent to NIC    |  Sent by Gen   | Forward by SUT |  Rec. by Gen   |  Avg. Latency  |  Max. Latency  |  Packets Lost  | Loss Ratio |")
+	log.info("+--------+--------------------+----------------+----------------+----------------+----------------+----------------+----------------+----------------+------------+")
+	size=60
+	attempts = 0
+	gensock.set_size(gencores,0,size) # This is setting the frame size
+	gensock.set_value(gencores,0,16,(size-14),2) # 18 is the difference between the frame size and IP size = size of (MAC addresses, ethertype and FCS)
+	gensock.set_value(gencores,0,38,(size-34),2) # 38 is the difference between the frame size and UDP size = 18 + size of IP header (=20)
+	# This will only work when using sending UDP packets. For different protocols and ethernet types, we would need a different calculation
+        gensock.speed(speed, gencores)
+        while True:
+                attempts += 1
+                print('Measurement ongoing at speed: ' + str(round(speed,2)) + '%      ',end='\r')
+                sys.stdout.flush()
+                time.sleep(1)
+                # Get statistics now that the generation is stable and NO ARP messages any more
+		pps_req_tx,pps_tx,pps_sut_tx_str,pps_rx,lat_avg,lat_max, abs_dropped, abs_tx = run_iteration(gensock,sutsock)
+		drop_rate = 100.0*abs_dropped/abs_tx
+	        log.info('|{:>7}'.format(str(attempts))+" | " + '{:>5.1f}'.format(speed) + '% ' +'{:>6.3f}'.format(get_pps(speed,size)) + ' Mpps | '+ '{:>9.3f}'.format(pps_req_tx)+' Mpps | '+ '{:>9.3f}'.format(pps_tx) +' Mpps | ' + '{:>9}'.format(pps_sut_tx_str) +' Mpps | '+ '{:>9.3f}'.format(pps_rx)+' Mpps | '+ '{:>9.0f}'.format(lat_avg)+' us   | '+ '{:>9.0f}'.format(lat_max)+' us   | '+ '{:>14d}'.format(abs_dropped)+ ' |''{:>9.2f}'.format(drop_rate)+ '%  |')
 
 def init_test():
 # Running at low speed to make sure the ARP messages can get through.
 # If not doing this, the ARP message could be dropped by a switch in overload and then the test will not give proper results
 # Note hoever that if we would run the test steps during a very long time, the ARP would expire in the switch.
 # PROX will send a new ARP request every seconds so chances are very low that they will all fail to get through
-	sock[0].speed(0.01, gencores)
-	sock[0].start(genstatcores)
+	socks[0].speed(0.01, gencores)
+	socks[0].start(genstatcores)
 	time.sleep(2)
-	sock[0].stop(gencores)
+	socks[0].stop(gencores)
 
 global sutstatcores
 global genstatcores
@@ -450,6 +474,9 @@ global gencores
 global irqcores
 global DROP_RATE_TRESHOLD
 global ACCURACY
+global required_number_of_test_machines
+clients =[]
+socks =[]
 vmDPIP =[]
 vmAdminIP =[]
 vmDPmac =[]
@@ -527,12 +554,19 @@ for vm in range(1, int(required_number_of_test_machines)+1):
 		irqcores = group1cores
 	f.close
 #####################################################################################
-client =[]
-sock =[]
+def exit_handler():
+	log.debug ('exit cleanup')
+	for sock in socks:
+        	sock.quit()
+	for client in clients:
+	        client.close()
+	sys.exit(0)
+
+atexit.register(exit_handler)
 
 for vm in range(0, int(required_number_of_test_machines)):
-	client.append(prox_ctrl(vmAdminIP[machine_index[vm]], key+'.pem','root'))
-	connect_client(client[-1])
+	clients.append(prox_ctrl(vmAdminIP[machine_index[vm]], key+'.pem','root'))
+	connect_client(clients[-1])
 # Creating script to bind the right network interface to the poll mode driver
 	devbindfile = "devbindvm%d.sh"%(vm+1)
 	with open("devbind.sh") as f:
@@ -541,20 +575,20 @@ for vm in range(0, int(required_number_of_test_machines)):
 			f.write(newText)
 	st = os.stat(devbindfile)
 	os.chmod(devbindfile, st.st_mode | stat.S_IEXEC)
-	client[-1].scp_put('./%s'%devbindfile, '/root/devbind.sh')
+	clients[-1].scp_put('./%s'%devbindfile, '/root/devbind.sh')
 	cmd = '/root/devbind.sh'
-	client[-1].run_cmd(cmd)
+	clients[-1].run_cmd(cmd)
 	log.debug("devbind.sh running on VM%d"%(vm+1))
-	client[-1].scp_put('./%s'%config_file[vm], '/root/%s'%config_file[vm])
-	client[-1].scp_put('./parameters%d.lua'%(vm+1), '/root/parameters.lua')
+	clients[-1].scp_put('./%s'%config_file[vm], '/root/%s'%config_file[vm])
+	clients[-1].scp_put('./parameters%d.lua'%(vm+1), '/root/parameters.lua')
 	log.debug("Starting PROX on VM%d"%(vm+1))
 	if script_control[vm] == 'true':
 		cmd = '/root/prox/build/prox -e -t -o cli -f /root/%s'%config_file[vm]
 	else:
 		cmd = '/root/prox/build/prox -t -o cli -f /root/%s'%config_file[vm]
 	if configonly == False:
-		client[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
-		sock.append(connect_socket(client[-1]))
+		clients[-1].fork_cmd(cmd, 'PROX Testing on TestM%d'%(vm+1))
+		socks.append(connect_socket(clients[-1]))
 if configonly:
 	sys.exit()
 init_code = testconfig.get('DEFAULT', 'init_code')
@@ -569,6 +603,3 @@ for vm in range(1, int(number_of_tests)+1):
 	cmd=testconfig.get('test%d'%vm,'cmd')
 	eval(cmd)
 ####################################################
-for vm in range(0, int(required_number_of_test_machines)):
-	sock[vm].quit()
-	client[vm].close()
diff --git a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/secgw.test b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/secgw.test
index 1ac171a6..faaeadf9 100644
--- a/VNFs/DPPD-PROX/helper-scripts/openstackrapid/secgw.test
+++ b/VNFs/DPPD-PROX/helper-scripts/openstackrapid/secgw.test
@@ -53,7 +53,7 @@ config_file = secgw2.cfg
 group1cores = [1]
 
 [test1]
-cmd=run_speedtest(sock[0],sock[2])
+cmd=run_speedtest(socks[0],socks[2])
 
 [test2]
-cmd=run_sizetest(sock[0],sock[2])
+cmd=run_sizetest(socks[0],socks[2])