summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorYichen Wang <yicwang@cisco.com>2017-11-22 16:26:29 -0800
committerYichen Wang <yicwang@cisco.com>2017-11-24 01:52:53 -0800
commit1ce98c7510f810a5ee790523a0dcbf429961adda (patch)
treed29558354fdef0b51a1487fa67ad15022a1c1ad4
parent58d63a026082a7e3207e409d51e919f959627e56 (diff)
Add support to use vswitch to handle V2V in PVVP SRIOV scenario
1. Add support to use vswitch to handle V2V in PVVP SRIOV scenario 2. Update nfvbenchvm to 0.5: (1) Update VPP to 17.10; (2) Update DPDK testpmd to 17.08; (3) Change kernel to based on longterm lineup; Change-Id: I944489579a4cd92d17075e80870bbdb32512a150 Signed-off-by: Yichen Wang <yicwang@cisco.com>
-rw-r--r--nfvbench/cfg.default.yaml7
-rw-r--r--nfvbench/chain_clients.py36
-rw-r--r--nfvbench/nfvbench.py25
-rw-r--r--nfvbench/nfvbenchvm/nfvbenchvm.conf2
-rw-r--r--nfvbenchvm/README.rst28
-rwxr-xr-xnfvbenchvm/dib/build-image.sh2
-rw-r--r--nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo2
-rw-r--r--nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml2
-rwxr-xr-xnfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel2
-rwxr-xr-xnfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script5
-rwxr-xr-xnfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup2
-rw-r--r--nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local25
-rw-r--r--nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf2
13 files changed, 99 insertions, 41 deletions
diff --git a/nfvbench/cfg.default.yaml b/nfvbench/cfg.default.yaml
index 83dd5ac..a8bdc2b 100644
--- a/nfvbench/cfg.default.yaml
+++ b/nfvbench/cfg.default.yaml
@@ -253,6 +253,11 @@ internal_networks:
segmentation_id:
physical_network:
+# In the scenario of PVVP + SRIOV, there is choice of how the traffic will be
+# handled in the middle network. The default (false) will use vswitch, while
+# SRIOV can be used by toggling below setting.
+use_sriov_middle_net: false
+
# EXT chain only. Names of edge networks which will be used to send traffic via traffic generator.
external_networks:
left: 'nfvbench-net0'
@@ -400,4 +405,4 @@ factory_class: 'BasicFactory'
# Custom label added for every perf record generated during this run.
# Can be overriden by --user-label
-user_label: \ No newline at end of file
+user_label:
diff --git a/nfvbench/chain_clients.py b/nfvbench/chain_clients.py
index d9a39af..57b15ee 100644
--- a/nfvbench/chain_clients.py
+++ b/nfvbench/chain_clients.py
@@ -145,11 +145,11 @@ class BasicStageClient(object):
LOG.info('Created network: %s.', name)
return network
- def _create_port(self, net):
+ def _create_port(self, net, vnic_type='normal'):
body = {
"port": {
'network_id': net['id'],
- 'binding:vnic_type': 'direct' if self.config.sriov else 'normal'
+ 'binding:vnic_type': vnic_type
}
}
port = self.neutron.create_port(body)
@@ -305,7 +305,7 @@ class BasicStageClient(object):
else:
LOG.error('Unable to delete flavor: %s', self.config.flavor_type)
- def get_config_file(self, chain_index, src_mac, dst_mac):
+ def get_config_file(self, chain_index, src_mac, dst_mac, intf_mac1, intf_mac2):
boot_script_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'nfvbenchvm/', self.nfvbenchvm_config_name)
@@ -317,6 +317,8 @@ class BasicStageClient(object):
vm_config = {
'forwarder': self.config.vm_forwarder,
+ 'intf_mac1': intf_mac1,
+ 'intf_mac2': intf_mac2,
'tg_gateway1_ip': self.config.traffic_generator.tg_gateway_ip_addrs[0],
'tg_gateway2_ip': self.config.traffic_generator.tg_gateway_ip_addrs[1],
'tg_net1': self.config.traffic_generator.ip_addrs[0],
@@ -479,11 +481,13 @@ class PVPStageClient(BasicStageClient):
if reusable_vm:
self.vms.append(reusable_vm)
else:
+ vnic_type = 'direct' if self.config.sriov else 'normal'
+ ports = [self._create_port(net, vnic_type) for net in self.nets]
config_file = self.get_config_file(chain_index,
self.config.generator_config.src_device.mac,
- self.config.generator_config.dst_device.mac)
-
- ports = [self._create_port(net) for net in self.nets]
+ self.config.generator_config.dst_device.mac,
+ ports[0]['mac_address'],
+ ports[1]['mac_address'])
self.created_ports.extend(ports)
self.vms.append(self._create_server(name, ports, az, config_file))
self._ensure_vms_active()
@@ -542,11 +546,15 @@ class PVVPStageClient(BasicStageClient):
if reusable_vm0 and reusable_vm1:
self.vms.extend([reusable_vm0, reusable_vm1])
else:
- vm0_port_net0 = self._create_port(vm0_nets[0])
- vm0_port_net2 = self._create_port(vm0_nets[1])
+ edge_vnic_type = 'direct' if self.config.sriov else 'normal'
+ middle_vnic_type = 'direct' \
+ if self.config.sriov and self.config.use_sriov_middle_net \
+ else 'normal'
+ vm0_port_net0 = self._create_port(vm0_nets[0], edge_vnic_type)
+ vm0_port_net2 = self._create_port(vm0_nets[1], middle_vnic_type)
- vm1_port_net2 = self._create_port(vm1_nets[1])
- vm1_port_net1 = self._create_port(vm1_nets[0])
+ vm1_port_net2 = self._create_port(vm1_nets[1], middle_vnic_type)
+ vm1_port_net1 = self._create_port(vm1_nets[0], edge_vnic_type)
self.created_ports.extend([vm0_port_net0,
vm0_port_net2,
@@ -558,10 +566,14 @@ class PVVPStageClient(BasicStageClient):
# TG0 (net0) -> VM0 (net2) -> VM1 (net2) -> TG1 (net1)
config_file0 = self.get_config_file(chain_index,
self.config.generator_config.src_device.mac,
- vm1_port_net2['mac_address'])
+ vm1_port_net2['mac_address'],
+ vm0_port_net0['mac_address'],
+ vm0_port_net2['mac_address'])
config_file1 = self.get_config_file(chain_index,
vm0_port_net2['mac_address'],
- self.config.generator_config.dst_device.mac)
+ self.config.generator_config.dst_device.mac,
+ vm1_port_net2['mac_address'],
+ vm1_port_net1['mac_address'])
self.vms.append(self._create_server(name0,
[vm0_port_net0, vm0_port_net2],
diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py
index d1bd0d9..4c9f56c 100644
--- a/nfvbench/nfvbench.py
+++ b/nfvbench/nfvbench.py
@@ -284,6 +284,12 @@ def parse_opts_from_cli():
action='store_true',
help='Use SRIOV (no vswitch - requires SRIOV support in compute nodes)')
+ parser.add_argument('--use-sriov-middle-net', dest='use_sriov_middle_net',
+ default=None,
+ action='store_true',
+ help='Use SRIOV to handle the middle network traffic '
+ '(PVVP with SRIOV only)')
+
parser.add_argument('-d', '--debug', dest='debug',
action='store_true',
default=None,
@@ -491,20 +497,29 @@ def main():
config.sriov = True
if opts.log_file:
config.log_file = opts.log_file
+ if opts.service_chain:
+ config.service_chain = opts.service_chain
+ if opts.service_chain_count:
+ config.service_chain_count = opts.service_chain_count
- # show running config in json format
- if opts.show_config:
- print json.dumps(config, sort_keys=True, indent=4)
- sys.exit(0)
+ if opts.use_sriov_middle_net:
+ if (not config.sriov) or (not config.service_chain == ChainType.PVVP):
+ raise Exception("--use-sriov-middle-net is only valid for PVVP with SRIOV")
+ config.use_sriov_middle_net = True
if config.sriov and config.service_chain != ChainType.EXT:
# if sriov is requested (does not apply to ext chains)
# make sure the physnet names are specified
check_physnet("left", config.internal_networks.left)
check_physnet("right", config.internal_networks.right)
- if config.service_chain == ChainType.PVVP:
+ if config.service_chain == ChainType.PVVP and config.use_sriov_middle_net:
check_physnet("middle", config.internal_networks.middle)
+ # show running config in json format
+ if opts.show_config:
+ print json.dumps(config, sort_keys=True, indent=4)
+ sys.exit(0)
+
# update the config in the config plugin as it might have changed
# in a copy of the dict (config plugin still holds the original dict)
config_plugin.set_config(config)
diff --git a/nfvbench/nfvbenchvm/nfvbenchvm.conf b/nfvbench/nfvbenchvm/nfvbenchvm.conf
index 0b76244..3bc6ace 100644
--- a/nfvbench/nfvbenchvm/nfvbenchvm.conf
+++ b/nfvbench/nfvbenchvm/nfvbenchvm.conf
@@ -1,4 +1,6 @@
FORWARDER={forwarder}
+INTF_MAC1={intf_mac1}
+INTF_MAC2={intf_mac2}
TG_MAC1={tg_mac1}
TG_MAC2={tg_mac2}
VNF_GATEWAY1_CIDR={vnf_gateway1_cidr}
diff --git a/nfvbenchvm/README.rst b/nfvbenchvm/README.rst
index baa6996..1bf0bbf 100644
--- a/nfvbenchvm/README.rst
+++ b/nfvbenchvm/README.rst
@@ -1,4 +1,4 @@
-NFVBENCH VM IMAGE FOR OPENSTACK
+NFVBENCH VM IMAGE FOR OPENSTACK
+++++++++++++++++++++++++++++++
This repo will build a centos 7 image with testpmd and VPP installed.
@@ -18,7 +18,7 @@ Pre-requisites
Build the image
---------------
- cd dib
-- update the version number for the image (if needed) by modifying __version__ in build-image.sh
+- update the version number for the image (if needed) by modifying __version__ in build-image.sh
- setup your http_proxy if needed
- bash build-image.sh
@@ -48,15 +48,17 @@ nfvbenchvm config file is located at ``/etc/nfvbenchvm.conf``.
.. code-block:: bash
- FORWARDER=VPP
- TG_MAC0=00:10:94:00:0A:00
- TG_MAC1=00:11:94:00:0A:00
- VNF1_GATEWAY_CIDR=1.1.0.2/8
- VNF2_GATEWAY_CIDR=2.2.0.2/8
- TG1_NET=10.0.0.0/8
- TG2_NET=20.0.0.0/8
- TG1_GATEWAY_IP=1.1.0.100
- TG1_GATEWAY_IP=2.2.0.100
+ FORWARDER=testpmd
+ INTF_MAC1=FA:16:3E:A2:30:41
+ INTF_MAC2=FA:16:3E:10:DA:10
+ TG_MAC1=00:10:94:00:0A:00
+ TG_MAC2=00:11:94:00:0A:00
+ VNF_GATEWAY1_CIDR=1.1.0.2/8
+ VNF_GATEWAY2_CIDR=2.2.0.2/8
+ TG_NET1=10.0.0.0/8
+ TG_NET2=20.0.0.0/8
+ TG_GATEWAY1_IP=1.1.0.100
+ TG_GATEWAY2_IP=2.2.0.100
Launching nfvbenchvm VM
@@ -79,6 +81,6 @@ To check if VPP is running, you can run this command in VNC console:
Hardcoded Username and Password
--------------------------------
-- Username: nfvbench
-- Password: nfvbench
+- Username: nfvbench
+- Password: nfvbench
diff --git a/nfvbenchvm/dib/build-image.sh b/nfvbenchvm/dib/build-image.sh
index 9326762..605db54 100755
--- a/nfvbenchvm/dib/build-image.sh
+++ b/nfvbenchvm/dib/build-image.sh
@@ -11,7 +11,7 @@ set -e
gs_url=artifacts.opnfv.org/nfvbench/images
# image version number
-__version__=0.4
+__version__=0.5
image_name=nfvbenchvm_centos-$__version__
# if image exists skip building
diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo b/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo
index a26aa42..3ad12fb 100644
--- a/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo
+++ b/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo
@@ -1,5 +1,5 @@
[fdio-release]
name=fd.io release branch latest merge
-baseurl=https://nexus.fd.io/content/repositories/fd.io.stable.1707.centos7/
+baseurl=https://nexus.fd.io/content/repositories/fd.io.stable.1710.centos7/
enabled=1
gpgcheck=0
diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml b/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml
index 36e0196..e3184c7 100644
--- a/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml
+++ b/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml
@@ -6,6 +6,8 @@ screen:
telnet:
python-devel:
libyaml-devel:
+numactl-libs:
+numactl-devel:
vpp:
vpp-plugins:
kernel-firmware:
diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel
index d884f79..8094006 100755
--- a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel
+++ b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel
@@ -8,7 +8,7 @@ fi
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm
yum remove -y kernel-firmware kernel-headers kernel-devel
-yum install -y --enablerepo=elrepo-kernel kernel-ml kernel-ml-headers kernel-ml-devel
+yum install -y --enablerepo=elrepo-kernel kernel-lt kernel-lt-headers kernel-lt-devel
# gcc will be removed with old kernel as dependency, so reinstalling it back
yum install -y gcc
diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script
index acdc6a3..2136c3a 100755
--- a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script
+++ b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script
@@ -1,7 +1,7 @@
#!/bin/bash
-DPDK=dpdk-17.05.2
-DPDK_UNTAR=dpdk-stable-17.05.2
+DPDK=dpdk-17.08
+DPDK_UNTAR=dpdk-17.08
# pick up the kernel version for the target image
kernel_version=`ls -t /lib/modules | awk 'NR==1 {print}'`
@@ -19,6 +19,7 @@ cp usertools/dpdk-devbind.py ../dpdk
# cp tools/dpdk_nic_bind.py ../dpdk/dpdk-devbind.py
cp x86_64-native-linuxapp-gcc/app/testpmd ../dpdk
cp x86_64-native-linuxapp-gcc/kmod/igb_uio.ko ../dpdk
+echo "set promisc all off" > /dpdk/testpmd_cmd.txt
cd ..
rm -f $DPDK.tar.xz
diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup
index 83d4fc5..14e9f27 100755
--- a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup
+++ b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup
@@ -1,3 +1,3 @@
#!/bin/bash
-yum erase -y python-devel libyaml-devel kernel-devel kernel-headers kernel-ml-headers kernel-ml-devel gcc
+yum erase -y python-devel libyaml-devel numactl-devel kernel-devel kernel-headers kernel-lt-headers kernel-lt-devel gcc
diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local b/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local
index 596cfdb..8aab1ae 100644
--- a/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local
+++ b/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local
@@ -34,6 +34,25 @@ for irq in `ls /proc/irq/`; do
done
tuna -c $(seq -s, 1 1 $WORKER_CORES) --isolate
+# Sometimes the interfaces on the loopback VM will use different drivers, e.g.
+# one from vswitch which is virtio based, one is from SRIOV VF. In this case,
+# we have to make sure the forwarder uses them in the right order, which is
+# especially important if the VM is in a PVVP chain.
+if [ -z $INTF_MAC1 ] && [ -z $INTF_MAC2 ]; then
+ NET_PATH=/sys/class/net
+ EXP_INTF_1=$(for f in $(ls $NET_PATH/); do if [ ! $(grep -o "$INTF_MAC1" $NET_PATH/$f/address) ]; then break; fi; done)
+ EXP_PCI_ADDRESS_1=$(basename $(readlink $NET_PATH/$EXP_INTF_1/device))
+ EXP_INTF_2=$(for f in $(ls $NET_PATH/); do if [ ! $(grep -o "$INTF_MAC2" $NET_PATH/$f/address) ]; then break; fi; done)
+ EXP_PCI_ADDRESS_2=$(basename $(readlink $NET_PATH/$EXP_INTF_2/device))
+ if [ "$PCI_ADDRESS_1" == "$EXP_PCI_ADDRESS_2" ] && [ "$PCI_ADDRESS_2" == "$EXP_PCI_ADDRESS_1" ]; then
+ # Interfaces are not coming in the expected order:
+ # (1) Swap the MAC in the case of testpmd;
+ # (2) Swap the interface order in the case of VPP;
+ TEMP=$PCI_ADDRESS_1; PCI_ADDRESS_1=$PCI_ADDRESS_2; PCI_ADDRESS_2=$TEMP
+ TEMP=$TG_MAC1; TG_MAC1=$TG_MAC2; TG_MAC2=$TEMP
+ fi
+fi
+
# Configure the forwarder
if [ "$FORWARDER" == "testpmd" ]; then
echo "Configuring testpmd..."
@@ -56,8 +75,9 @@ if [ "$FORWARDER" == "testpmd" ]; then
--eth-peer=0,$TG_MAC1 \
--eth-peer=1,$TG_MAC2 \
--forward-mode=mac \
- --nb-cores=$WORKER_CORES\
- --max-pkt-len 9000
+ --nb-cores=$WORKER_CORES \
+ --max-pkt-len=9000 \
+ --cmdline-file=/dpdk/testpmd_cmd.txt
else
echo "Configuring vpp..."
cp /vpp/startup.conf /etc/vpp/startup.conf
@@ -81,6 +101,5 @@ else
sed -i "s/{{TG_NET2}}/${TG_NET2//\//\/}/g" /etc/vpp/vm.conf
sed -i "s/{{TG_GATEWAY1_IP}}/${TG_GATEWAY1_IP}/g" /etc/vpp/vm.conf
sed -i "s/{{TG_GATEWAY2_IP}}/${TG_GATEWAY2_IP}/g" /etc/vpp/vm.conf
-
service vpp restart
fi
diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf b/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf
index 811eee1..4306fe9 100644
--- a/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf
+++ b/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf
@@ -3,7 +3,7 @@ unix {
log /tmp/vpp.log
full-coredump
startup-config /etc/vpp/vm.conf
- cli-listen localhost:5002
+ cli-listen /run/vpp/cli.sock
}
api-trace {