diff options
| -rw-r--r-- | nfvbench/cfg.default.yaml | 7 | ||||
| -rw-r--r-- | nfvbench/chain_clients.py | 36 | ||||
| -rw-r--r-- | nfvbench/nfvbench.py | 25 | ||||
| -rw-r--r-- | nfvbench/nfvbenchvm/nfvbenchvm.conf | 2 | ||||
| -rw-r--r-- | nfvbenchvm/README.rst | 28 | ||||
| -rwxr-xr-x | nfvbenchvm/dib/build-image.sh | 2 | ||||
| -rw-r--r-- | nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo | 2 | ||||
| -rw-r--r-- | nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml | 2 | ||||
| -rwxr-xr-x | nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel | 2 | ||||
| -rwxr-xr-x | nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script | 5 | ||||
| -rwxr-xr-x | nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup | 2 | ||||
| -rw-r--r-- | nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local | 25 | ||||
| -rw-r--r-- | nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf | 2 |
13 files changed, 99 insertions, 41 deletions
diff --git a/nfvbench/cfg.default.yaml b/nfvbench/cfg.default.yaml index 83dd5ac..a8bdc2b 100644 --- a/nfvbench/cfg.default.yaml +++ b/nfvbench/cfg.default.yaml @@ -253,6 +253,11 @@ internal_networks: segmentation_id: physical_network: +# In the scenario of PVVP + SRIOV, there is choice of how the traffic will be +# handled in the middle network. The default (false) will use vswitch, while +# SRIOV can be used by toggling below setting. +use_sriov_middle_net: false + # EXT chain only. Names of edge networks which will be used to send traffic via traffic generator. external_networks: left: 'nfvbench-net0' @@ -400,4 +405,4 @@ factory_class: 'BasicFactory' # Custom label added for every perf record generated during this run. # Can be overriden by --user-label -user_label:
\ No newline at end of file +user_label: diff --git a/nfvbench/chain_clients.py b/nfvbench/chain_clients.py index d9a39af..57b15ee 100644 --- a/nfvbench/chain_clients.py +++ b/nfvbench/chain_clients.py @@ -145,11 +145,11 @@ class BasicStageClient(object): LOG.info('Created network: %s.', name) return network - def _create_port(self, net): + def _create_port(self, net, vnic_type='normal'): body = { "port": { 'network_id': net['id'], - 'binding:vnic_type': 'direct' if self.config.sriov else 'normal' + 'binding:vnic_type': vnic_type } } port = self.neutron.create_port(body) @@ -305,7 +305,7 @@ class BasicStageClient(object): else: LOG.error('Unable to delete flavor: %s', self.config.flavor_type) - def get_config_file(self, chain_index, src_mac, dst_mac): + def get_config_file(self, chain_index, src_mac, dst_mac, intf_mac1, intf_mac2): boot_script_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'nfvbenchvm/', self.nfvbenchvm_config_name) @@ -317,6 +317,8 @@ class BasicStageClient(object): vm_config = { 'forwarder': self.config.vm_forwarder, + 'intf_mac1': intf_mac1, + 'intf_mac2': intf_mac2, 'tg_gateway1_ip': self.config.traffic_generator.tg_gateway_ip_addrs[0], 'tg_gateway2_ip': self.config.traffic_generator.tg_gateway_ip_addrs[1], 'tg_net1': self.config.traffic_generator.ip_addrs[0], @@ -479,11 +481,13 @@ class PVPStageClient(BasicStageClient): if reusable_vm: self.vms.append(reusable_vm) else: + vnic_type = 'direct' if self.config.sriov else 'normal' + ports = [self._create_port(net, vnic_type) for net in self.nets] config_file = self.get_config_file(chain_index, self.config.generator_config.src_device.mac, - self.config.generator_config.dst_device.mac) - - ports = [self._create_port(net) for net in self.nets] + self.config.generator_config.dst_device.mac, + ports[0]['mac_address'], + ports[1]['mac_address']) self.created_ports.extend(ports) self.vms.append(self._create_server(name, ports, az, config_file)) self._ensure_vms_active() @@ -542,11 +546,15 @@ class PVVPStageClient(BasicStageClient): if reusable_vm0 and reusable_vm1: self.vms.extend([reusable_vm0, reusable_vm1]) else: - vm0_port_net0 = self._create_port(vm0_nets[0]) - vm0_port_net2 = self._create_port(vm0_nets[1]) + edge_vnic_type = 'direct' if self.config.sriov else 'normal' + middle_vnic_type = 'direct' \ + if self.config.sriov and self.config.use_sriov_middle_net \ + else 'normal' + vm0_port_net0 = self._create_port(vm0_nets[0], edge_vnic_type) + vm0_port_net2 = self._create_port(vm0_nets[1], middle_vnic_type) - vm1_port_net2 = self._create_port(vm1_nets[1]) - vm1_port_net1 = self._create_port(vm1_nets[0]) + vm1_port_net2 = self._create_port(vm1_nets[1], middle_vnic_type) + vm1_port_net1 = self._create_port(vm1_nets[0], edge_vnic_type) self.created_ports.extend([vm0_port_net0, vm0_port_net2, @@ -558,10 +566,14 @@ class PVVPStageClient(BasicStageClient): # TG0 (net0) -> VM0 (net2) -> VM1 (net2) -> TG1 (net1) config_file0 = self.get_config_file(chain_index, self.config.generator_config.src_device.mac, - vm1_port_net2['mac_address']) + vm1_port_net2['mac_address'], + vm0_port_net0['mac_address'], + vm0_port_net2['mac_address']) config_file1 = self.get_config_file(chain_index, vm0_port_net2['mac_address'], - self.config.generator_config.dst_device.mac) + self.config.generator_config.dst_device.mac, + vm1_port_net2['mac_address'], + vm1_port_net1['mac_address']) self.vms.append(self._create_server(name0, [vm0_port_net0, vm0_port_net2], diff --git a/nfvbench/nfvbench.py b/nfvbench/nfvbench.py index d1bd0d9..4c9f56c 100644 --- a/nfvbench/nfvbench.py +++ b/nfvbench/nfvbench.py @@ -284,6 +284,12 @@ def parse_opts_from_cli(): action='store_true', help='Use SRIOV (no vswitch - requires SRIOV support in compute nodes)') + parser.add_argument('--use-sriov-middle-net', dest='use_sriov_middle_net', + default=None, + action='store_true', + help='Use SRIOV to handle the middle network traffic ' + '(PVVP with SRIOV only)') + parser.add_argument('-d', '--debug', dest='debug', action='store_true', default=None, @@ -491,20 +497,29 @@ def main(): config.sriov = True if opts.log_file: config.log_file = opts.log_file + if opts.service_chain: + config.service_chain = opts.service_chain + if opts.service_chain_count: + config.service_chain_count = opts.service_chain_count - # show running config in json format - if opts.show_config: - print json.dumps(config, sort_keys=True, indent=4) - sys.exit(0) + if opts.use_sriov_middle_net: + if (not config.sriov) or (not config.service_chain == ChainType.PVVP): + raise Exception("--use-sriov-middle-net is only valid for PVVP with SRIOV") + config.use_sriov_middle_net = True if config.sriov and config.service_chain != ChainType.EXT: # if sriov is requested (does not apply to ext chains) # make sure the physnet names are specified check_physnet("left", config.internal_networks.left) check_physnet("right", config.internal_networks.right) - if config.service_chain == ChainType.PVVP: + if config.service_chain == ChainType.PVVP and config.use_sriov_middle_net: check_physnet("middle", config.internal_networks.middle) + # show running config in json format + if opts.show_config: + print json.dumps(config, sort_keys=True, indent=4) + sys.exit(0) + # update the config in the config plugin as it might have changed # in a copy of the dict (config plugin still holds the original dict) config_plugin.set_config(config) diff --git a/nfvbench/nfvbenchvm/nfvbenchvm.conf b/nfvbench/nfvbenchvm/nfvbenchvm.conf index 0b76244..3bc6ace 100644 --- a/nfvbench/nfvbenchvm/nfvbenchvm.conf +++ b/nfvbench/nfvbenchvm/nfvbenchvm.conf @@ -1,4 +1,6 @@ FORWARDER={forwarder} +INTF_MAC1={intf_mac1} +INTF_MAC2={intf_mac2} TG_MAC1={tg_mac1} TG_MAC2={tg_mac2} VNF_GATEWAY1_CIDR={vnf_gateway1_cidr} diff --git a/nfvbenchvm/README.rst b/nfvbenchvm/README.rst index baa6996..1bf0bbf 100644 --- a/nfvbenchvm/README.rst +++ b/nfvbenchvm/README.rst @@ -1,4 +1,4 @@ -NFVBENCH VM IMAGE FOR OPENSTACK +NFVBENCH VM IMAGE FOR OPENSTACK +++++++++++++++++++++++++++++++ This repo will build a centos 7 image with testpmd and VPP installed. @@ -18,7 +18,7 @@ Pre-requisites Build the image --------------- - cd dib -- update the version number for the image (if needed) by modifying __version__ in build-image.sh +- update the version number for the image (if needed) by modifying __version__ in build-image.sh - setup your http_proxy if needed - bash build-image.sh @@ -48,15 +48,17 @@ nfvbenchvm config file is located at ``/etc/nfvbenchvm.conf``. .. code-block:: bash - FORWARDER=VPP - TG_MAC0=00:10:94:00:0A:00 - TG_MAC1=00:11:94:00:0A:00 - VNF1_GATEWAY_CIDR=1.1.0.2/8 - VNF2_GATEWAY_CIDR=2.2.0.2/8 - TG1_NET=10.0.0.0/8 - TG2_NET=20.0.0.0/8 - TG1_GATEWAY_IP=1.1.0.100 - TG1_GATEWAY_IP=2.2.0.100 + FORWARDER=testpmd + INTF_MAC1=FA:16:3E:A2:30:41 + INTF_MAC2=FA:16:3E:10:DA:10 + TG_MAC1=00:10:94:00:0A:00 + TG_MAC2=00:11:94:00:0A:00 + VNF_GATEWAY1_CIDR=1.1.0.2/8 + VNF_GATEWAY2_CIDR=2.2.0.2/8 + TG_NET1=10.0.0.0/8 + TG_NET2=20.0.0.0/8 + TG_GATEWAY1_IP=1.1.0.100 + TG_GATEWAY2_IP=2.2.0.100 Launching nfvbenchvm VM @@ -79,6 +81,6 @@ To check if VPP is running, you can run this command in VNC console: Hardcoded Username and Password -------------------------------- -- Username: nfvbench -- Password: nfvbench +- Username: nfvbench +- Password: nfvbench diff --git a/nfvbenchvm/dib/build-image.sh b/nfvbenchvm/dib/build-image.sh index 9326762..605db54 100755 --- a/nfvbenchvm/dib/build-image.sh +++ b/nfvbenchvm/dib/build-image.sh @@ -11,7 +11,7 @@ set -e gs_url=artifacts.opnfv.org/nfvbench/images # image version number -__version__=0.4 +__version__=0.5 image_name=nfvbenchvm_centos-$__version__ # if image exists skip building diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo b/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo index a26aa42..3ad12fb 100644 --- a/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo +++ b/nfvbenchvm/dib/elements/nfvbenchvm/fdio-release.repo @@ -1,5 +1,5 @@ [fdio-release] name=fd.io release branch latest merge -baseurl=https://nexus.fd.io/content/repositories/fd.io.stable.1707.centos7/ +baseurl=https://nexus.fd.io/content/repositories/fd.io.stable.1710.centos7/ enabled=1 gpgcheck=0 diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml b/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml index 36e0196..e3184c7 100644 --- a/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml +++ b/nfvbenchvm/dib/elements/nfvbenchvm/package-installs.yaml @@ -6,6 +6,8 @@ screen: telnet: python-devel: libyaml-devel: +numactl-libs: +numactl-devel: vpp: vpp-plugins: kernel-firmware: diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel index d884f79..8094006 100755 --- a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel +++ b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/01-update-kernel @@ -8,7 +8,7 @@ fi rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm yum remove -y kernel-firmware kernel-headers kernel-devel -yum install -y --enablerepo=elrepo-kernel kernel-ml kernel-ml-headers kernel-ml-devel +yum install -y --enablerepo=elrepo-kernel kernel-lt kernel-lt-headers kernel-lt-devel # gcc will be removed with old kernel as dependency, so reinstalling it back yum install -y gcc diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script index acdc6a3..2136c3a 100755 --- a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script +++ b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/02-testpmd-script @@ -1,7 +1,7 @@ #!/bin/bash -DPDK=dpdk-17.05.2 -DPDK_UNTAR=dpdk-stable-17.05.2 +DPDK=dpdk-17.08 +DPDK_UNTAR=dpdk-17.08 # pick up the kernel version for the target image kernel_version=`ls -t /lib/modules | awk 'NR==1 {print}'` @@ -19,6 +19,7 @@ cp usertools/dpdk-devbind.py ../dpdk # cp tools/dpdk_nic_bind.py ../dpdk/dpdk-devbind.py cp x86_64-native-linuxapp-gcc/app/testpmd ../dpdk cp x86_64-native-linuxapp-gcc/kmod/igb_uio.ko ../dpdk +echo "set promisc all off" > /dpdk/testpmd_cmd.txt cd .. rm -f $DPDK.tar.xz diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup index 83d4fc5..14e9f27 100755 --- a/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup +++ b/nfvbenchvm/dib/elements/nfvbenchvm/post-install.d/99-cleanup @@ -1,3 +1,3 @@ #!/bin/bash -yum erase -y python-devel libyaml-devel kernel-devel kernel-headers kernel-ml-headers kernel-ml-devel gcc +yum erase -y python-devel libyaml-devel numactl-devel kernel-devel kernel-headers kernel-lt-headers kernel-lt-devel gcc diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local b/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local index 596cfdb..8aab1ae 100644 --- a/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local +++ b/nfvbenchvm/dib/elements/nfvbenchvm/static/etc/rc.d/rc.local @@ -34,6 +34,25 @@ for irq in `ls /proc/irq/`; do done tuna -c $(seq -s, 1 1 $WORKER_CORES) --isolate +# Sometimes the interfaces on the loopback VM will use different drivers, e.g. +# one from vswitch which is virtio based, one is from SRIOV VF. In this case, +# we have to make sure the forwarder uses them in the right order, which is +# especially important if the VM is in a PVVP chain. +if [ -z $INTF_MAC1 ] && [ -z $INTF_MAC2 ]; then + NET_PATH=/sys/class/net + EXP_INTF_1=$(for f in $(ls $NET_PATH/); do if [ ! $(grep -o "$INTF_MAC1" $NET_PATH/$f/address) ]; then break; fi; done) + EXP_PCI_ADDRESS_1=$(basename $(readlink $NET_PATH/$EXP_INTF_1/device)) + EXP_INTF_2=$(for f in $(ls $NET_PATH/); do if [ ! $(grep -o "$INTF_MAC2" $NET_PATH/$f/address) ]; then break; fi; done) + EXP_PCI_ADDRESS_2=$(basename $(readlink $NET_PATH/$EXP_INTF_2/device)) + if [ "$PCI_ADDRESS_1" == "$EXP_PCI_ADDRESS_2" ] && [ "$PCI_ADDRESS_2" == "$EXP_PCI_ADDRESS_1" ]; then + # Interfaces are not coming in the expected order: + # (1) Swap the MAC in the case of testpmd; + # (2) Swap the interface order in the case of VPP; + TEMP=$PCI_ADDRESS_1; PCI_ADDRESS_1=$PCI_ADDRESS_2; PCI_ADDRESS_2=$TEMP + TEMP=$TG_MAC1; TG_MAC1=$TG_MAC2; TG_MAC2=$TEMP + fi +fi + # Configure the forwarder if [ "$FORWARDER" == "testpmd" ]; then echo "Configuring testpmd..." @@ -56,8 +75,9 @@ if [ "$FORWARDER" == "testpmd" ]; then --eth-peer=0,$TG_MAC1 \ --eth-peer=1,$TG_MAC2 \ --forward-mode=mac \ - --nb-cores=$WORKER_CORES\ - --max-pkt-len 9000 + --nb-cores=$WORKER_CORES \ + --max-pkt-len=9000 \ + --cmdline-file=/dpdk/testpmd_cmd.txt else echo "Configuring vpp..." cp /vpp/startup.conf /etc/vpp/startup.conf @@ -81,6 +101,5 @@ else sed -i "s/{{TG_NET2}}/${TG_NET2//\//\/}/g" /etc/vpp/vm.conf sed -i "s/{{TG_GATEWAY1_IP}}/${TG_GATEWAY1_IP}/g" /etc/vpp/vm.conf sed -i "s/{{TG_GATEWAY2_IP}}/${TG_GATEWAY2_IP}/g" /etc/vpp/vm.conf - service vpp restart fi diff --git a/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf b/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf index 811eee1..4306fe9 100644 --- a/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf +++ b/nfvbenchvm/dib/elements/nfvbenchvm/static/vpp/startup.conf @@ -3,7 +3,7 @@ unix { log /tmp/vpp.log full-coredump startup-config /etc/vpp/vm.conf - cli-listen localhost:5002 + cli-listen /run/vpp/cli.sock } api-trace { |