diff options
| -rw-r--r-- | conf/07_loadgen.conf | 18 | ||||
| -rw-r--r-- | docs/testing/developer/devguide/requirements/vswitchperf_ltd.rst | 108 | ||||
| -rw-r--r-- | docs/testing/user/configguide/installation.rst | 2 | ||||
| -rwxr-xr-x | systems/centos/build_base_machine.sh | 12 | ||||
| -rwxr-xr-x | systems/centos/prepare_python_env.sh | 4 | ||||
| -rwxr-xr-x | systems/rhel/7.2/build_base_machine.sh | 31 | ||||
| -rwxr-xr-x | systems/rhel/7.2/prepare_python_env.sh | 6 | ||||
| -rwxr-xr-x | systems/rhel/7.3/build_base_machine.sh | 29 | ||||
| -rwxr-xr-x | systems/rhel/7.3/prepare_python_env.sh | 4 | ||||
| -rw-r--r-- | testcases/testcase.py | 4 | ||||
| -rw-r--r-- | tools/load_gen/stressorvm/__init__.py | 16 | ||||
| -rw-r--r-- | tools/load_gen/stressorvm/stressor_vm.py | 117 |
12 files changed, 284 insertions, 67 deletions
diff --git a/conf/07_loadgen.conf b/conf/07_loadgen.conf index e7349a5d..0b2cc1e6 100644 --- a/conf/07_loadgen.conf +++ b/conf/07_loadgen.conf @@ -15,7 +15,23 @@ LOADGEN_DIR = os.path.join(ROOT_DIR, 'tools/load_gen') ###################################################### -# LOADGEN tool: one of DummyLoadGen, Stress, StressNg +# LOADGEN tool: one of DummyLoadGen, Stress, StressNg, +# and StressorVM ###################################################### LOADGEN = "DummyLoadGen" ###################################################### + + +###################################################### +# StressorVm specific COnfiguration +###################################################### +NN_COUNT = 1 +NN_MEMORY = ['4096'] +NN_SMP = ['2'] +NN_IMAGE = ['/home/opnfv/stressng-images/stressng-high-TypeE.qemu'] +NN_SHARED_DRIVE_TYPE = ['scsi'] +NN_BOOT_DRIVE_TYPE = ['scsi'] +NN_CORE_BINDING = [('9','10')] +NN_NICS_NR = ['2'] +NN_BASE_VNC_PORT = 4 +NN_LOG_FILE = 'nnqemu.log' diff --git a/docs/testing/developer/devguide/requirements/vswitchperf_ltd.rst b/docs/testing/developer/devguide/requirements/vswitchperf_ltd.rst index e1372520..c703ff40 100644 --- a/docs/testing/developer/devguide/requirements/vswitchperf_ltd.rst +++ b/docs/testing/developer/devguide/requirements/vswitchperf_ltd.rst @@ -413,7 +413,21 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoak **Title**: RFC 2889 X% packet loss Max Forwarding Rate Soak Test - **Prerequisite Test** LTD.Throughput.RFC2544.PacketLossRatio + **Prerequisite Tests**: + + LTD.Throughput.RFC2544.PacketLossRatio will determine the offered load and + frame size for which the maximum theoretical throughput of the interface + has not been achieved. As described in RFC 2544 section 24, the final + determination of the benchmark SHOULD be conducted using a full length + trial, and for this purpose the duration is 5 minutes with zero loss ratio. + + It is also essential to verify that the Traffic Generator has sufficient + stability to conduct Soak tests. Therefore, a prerequisite is to perform + this test with the DUT removed and replaced with a cross-over cable (or + other equivalent very low overhead method such as a loopback in a HW switch), + so that the traffic generator (and any other network involved) can be tested + over the Soak period. Note that this test may be challenging for software- + based traffic generators. **Priority**: @@ -422,12 +436,19 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoak The aim of this test is to understand the Max Forwarding Rate stability over an extended test duration in order to uncover any outliers. To allow for an extended test duration, the test should ideally run for 24 hours - or, if this is not possible, for at least 6 hours. For this test, each frame - size must be sent at the highest Throughput rate with X% packet loss, as - determined in the prerequisite test. The default loss percentages to be - tested are: - X = 0% - X = 10^-7% + or if this is not possible, for at least 6 hours. - Note: Other values can be tested if required by the user. + For this test, one frame size must be sent at the highest frame rate with + X% packet loss ratio, as determined in the prerequisite test (a short trial). + The loss ratio shall be measured and recorded every 5 minutes during the test + (it may be sufficient to collect lost frame counts and divide by the number + of frames sent in 5 minutes to see if a threshold has been crossed, + and accept some small inaccuracy in the threshold evaluation, not the result). + The default loss ratio is X = 0% and loss ratio > 10^-7% is the default + threshold to terminate the test early (or inform the test operator of + the failure status). + + Note: Other values of X and loss threshold can be tested if required by the user. **Expected Result**: @@ -441,13 +462,13 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoak and reporting any time intervals with packet loss. The `RFC2889 <https://www.rfc-editor.org/rfc/rfc2289.txt>`__ Forwarding Rate shall be measured in each interval. - An interval of 60s is suggested. + An interval of 300s is suggested. - CPU and memory utilization may also be collected as part of this test, to determine the vSwitch's performance footprint on the system. - The `RFC5481 <https://www.rfc-editor.org/rfc/rfc5481.txt>`__ PDV form of delay variation on the traffic flow, - using the 99th percentile. + using the 99th percentile, may also be collected. .. 3.2.2.1.7 @@ -457,7 +478,22 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoakFrameModification **Title**: RFC 2889 Max Forwarding Rate Soak Test with Frame Modification **Prerequisite Test**: + LTD.Throughput.RFC2544.PacketLossRatioFrameModification (0% Packet Loss) + will determine the offered load and + frame size for which the maximum theoretical throughput of the interface + has not been achieved. As described in RFC 2544 section 24, the final + determination of the benchmark SHOULD be conducted using a full length + trial, and for this purpose the duration is 5 minutes with zero loss ratio. + + It is also essential to verify that the Traffic Generator has sufficient + stability to conduct Soak tests. Therefore, a prerequisite is to perform + this test with the DUT removed and replaced with a cross-over cable (or + other equivalent very low overhead method such as a loopback in a HW switch), + so that the traffic generator (and any other network involved) can be tested + over the Soak period. Note that this test may be challenging for software- + based traffic generators. + **Priority**: @@ -466,9 +502,19 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoakFrameModification The aim of this test is to understand the Max Forwarding Rate stability over an extended test duration in order to uncover any outliers. To allow for an extended test duration, the test should ideally run for 24 hours or, if - this is not possible, for at least 6 hour. For this test, each frame - size must be sent at the highest Throughput rate with 0% packet loss, as - determined in the prerequisite test. + this is not possible, for at least 6 hours. + + For this test, one frame size must be sent at the highest frame rate with + X% packet loss ratio, as determined in the prerequisite test (a short trial). + The loss ratio shall be measured and recorded every 5 minutes during the test + (it may be sufficient to collect lost frame counts and divide by the number + of frames sent in 5 minutes to see if a threshold has been crossed, + and accept some small inaccuracy in the threshold evaluation, not the result). + The default loss ratio is X = 0% and loss ratio > 10^-7% is the default + threshold to terminate the test early (or inform the test operator of + the failure status). + + Note: Other values of X and loss threshold can be tested if required by the user. During this test, the DUT must perform the following operations on the traffic flow: @@ -498,13 +544,13 @@ Test ID: LTD.Throughput.RFC2889.MaxForwardingRateSoakFrameModification and reporting any time intervals with packet loss. The `RFC2889 <https://www.rfc-editor.org/rfc/rfc2289.txt>`__ Forwarding Rate shall be measured in each interval. - An interval of 60s is suggested. + An interval of 300s is suggested. - CPU and memory utilization may also be collected as part of this test, to determine the vSwitch's performance footprint on the system. - The `RFC5481 <https://www.rfc-editor.org/rfc/rfc5481.txt>`__ PDV form of delay variation on the traffic flow, using the 99th - percentile. + percentile, may also be collected. .. 3.2.2.1.8 @@ -1150,7 +1196,22 @@ Test ID: LTD.PacketDelayVariation.RFC3393.Soak **Title**: Packet Delay Variation Soak Test - **Prerequisite Tests**: LTD.Throughput.RFC2544.PacketLossRatio (0% Packet Loss) + **Prerequisite Tests**: + + LTD.Throughput.RFC2544.PacketLossRatio will determine the offered load and + frame size for which the maximum theoretical throughput of the interface + has not been achieved. As described in RFC 2544 section 24, the final + determination of the benchmark SHOULD be conducted using a full length + trial, and for this purpose the duration is 5 minutes with zero loss ratio. + + It is also essential to verify that the Traffic Generator has sufficient + stability to conduct Soak tests. Therefore, a prerequisite is to perform + this test with the DUT removed and replaced with a cross-over cable (or + other equivalent very low overhead method such as a loopback in a HW switch), + so that the traffic generator (and any other network involved) can be tested + over the Soak period. Note that this test may be challenging for software- + based traffic generators. + **Priority**: @@ -1160,9 +1221,20 @@ Test ID: LTD.PacketDelayVariation.RFC3393.Soak variation for different frame sizes over an extended test duration and to determine if there are any outliers. To allow for an extended test duration, the test should ideally run for 24 hours or, if this is not - possible, for at least 6 hour. For this test, each frame size must be - sent at the highest possible throughput with 0% packet loss, as - determined in the prerequisite test. + possible, for at least 6 hours. + + For this test, one frame size must be sent at the highest frame rate with + X% packet loss ratio, as determined in the prerequisite test (a short trial). + The loss ratio shall be measured and recorded every 5 minutes during the test + (it may be sufficient to collect lost frame counts and divide by the number + of frames sent in 5 minutes to see if a threshold has been crossed, + and accept some small inaccuracy in the threshold evaluation, not the result). + The default loss ratio is X = 0% and loss ratio > 10^-7% is the default + threshold to terminate the test early (or inform the test operator of + the failure status). + + Note: Other values of X and loss threshold can be tested if required by the user. + **Expected Result**: @@ -1173,7 +1245,7 @@ Test ID: LTD.PacketDelayVariation.RFC3393.Soak - The packet delay variation value for traffic passing through the DUT. - The `RFC5481 <https://www.rfc-editor.org/rfc/rfc5481.txt>`__ PDV form of delay variation on the traffic flow, - using the 99th percentile, for each 60s interval during the test. + using the 99th percentile, for each 300s interval during the test. - CPU and memory utilization may also be collected as part of this test, to determine the vSwitch's performance footprint on the system. diff --git a/docs/testing/user/configguide/installation.rst b/docs/testing/user/configguide/installation.rst index 7f4d640b..51588007 100644 --- a/docs/testing/user/configguide/installation.rst +++ b/docs/testing/user/configguide/installation.rst @@ -202,7 +202,7 @@ new shell session. Its activation is specific to your OS: .. code:: bash - $ scl enable python33 bash + $ scl enable rh-python34 bash $ source $HOME/vsperfenv/bin/activate * Fedora and Ubuntu diff --git a/systems/centos/build_base_machine.sh b/systems/centos/build_base_machine.sh index f2efb541..a45b0c3d 100755 --- a/systems/centos/build_base_machine.sh +++ b/systems/centos/build_base_machine.sh @@ -60,6 +60,8 @@ pixman-devel socat numactl numactl-devel +libpng-devel +freetype-devel # install gvim vim-X11 @@ -68,13 +70,13 @@ vim-X11 epel-release " | grep -v ^#) -# install SCL for python33 -sudo yum -y install centos-release-scl +# install SCL for python34 +sudo yum -y install centos-release-scl-rh -# install python33 packages and git-review tool +# install python34 packages and git-review tool yum -y install $(echo " -python33 -python33-python-tkinter +rh-python34 +rh-python34-python-tkinter git-review " | grep -v ^#) # prevent ovs vanilla from building from source due to kernel incompatibilities diff --git a/systems/centos/prepare_python_env.sh b/systems/centos/prepare_python_env.sh index 8bce53cc..ac7ccba4 100755 --- a/systems/centos/prepare_python_env.sh +++ b/systems/centos/prepare_python_env.sh @@ -21,8 +21,8 @@ if [ -d "$VSPERFENV_DIR" ] ; then exit fi -scl enable python33 " -virtualenv "$VSPERFENV_DIR" --python /usr/bin/python3 +scl enable rh-python34 " +virtualenv "$VSPERFENV_DIR" --python /opt/rh/rh-python34/root/usr/bin/python3 source "$VSPERFENV_DIR"/bin/activate pip install -r ../requirements.txt pip install pylint diff --git a/systems/rhel/7.2/build_base_machine.sh b/systems/rhel/7.2/build_base_machine.sh index 9eb8bbd2..858092df 100755 --- a/systems/rhel/7.2/build_base_machine.sh +++ b/systems/rhel/7.2/build_base_machine.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Build a base machine for RHEL 7.2 +# Build a base machine for RHEL 7.3 # # Copyright 2016 OPNFV, Intel Corporation & Red Hat Inc. # @@ -52,6 +52,7 @@ pkglist=( wget\ numactl\ numactl-devel\ + libpng-devel ) # python tools for proper QEMU, DPDK, and OVS make @@ -78,28 +79,24 @@ if [ "${#failedinstall[*]}" -gt 0 ]; then exit 1 fi -# install SCL for python33 by adding a repo to find its location to install it -cat <<'EOT' >> /etc/yum.repos.d/python33.repo -[rhscl-python33-el7] -name=Copr repo for python33-el7 owned by rhscl -baseurl=https://copr-be.cloud.fedoraproject.org/results/rhscl/python33-el7/epel-7-$basearch/ -type=rpm-md -skip_if_unavailable=True -gpgcheck=1 -gpgkey=https://copr-be.cloud.fedoraproject.org/results/rhscl/python33-el7/pubkey.gpg -repo_gpgcheck=0 +# install SCL for python34 by adding a repo to find its location to install it +cat <<'EOT' >> /etc/yum.repos.d/python34.repo +[centos-sclo-rh] +name=CentOS-7 - SCLo rh +baseurl=http://mirror.centos.org/centos/7/sclo/$basearch/rh/ +gpgcheck=0 enabled=1 -enabled_metadata=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-SCLo EOT -# install python33 packages and git-review tool +# install python34 packages and git-review tool yum -y install $(echo " -python33 -python33-python-tkinter +rh-python34 +rh-python34-python-tkinter " | grep -v ^#) -# cleanup python 33 repo file -rm -f /etc/yum.repos.d/python33.repo +# cleanup python 34 repo file +rm -f /etc/yum.repos.d/python34.repo # Create hugepage dirs mkdir -p /dev/hugepages diff --git a/systems/rhel/7.2/prepare_python_env.sh b/systems/rhel/7.2/prepare_python_env.sh index fb5882f1..e137aaab 100755 --- a/systems/rhel/7.2/prepare_python_env.sh +++ b/systems/rhel/7.2/prepare_python_env.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Prepare Python environment for vsperf execution on Red Hat 7.2 systems. +# Prepare Python environment for vsperf execution on RHEL 7.3 systems. # # Copyright 2016-2017 OPNFV, Intel Corporation, Red Hat Inc. # @@ -21,8 +21,8 @@ if [ -d "$VSPERFENV_DIR" ] ; then exit fi -scl enable python33 " -virtualenv "$VSPERFENV_DIR" --python /opt/rh/python33/root/usr/bin/python3 +scl enable rh-python34 " +virtualenv "$VSPERFENV_DIR" --python /opt/rh/rh-python34/root/usr/bin/python3 source "$VSPERFENV_DIR"/bin/activate pip install -r ../requirements.txt pip install pylint diff --git a/systems/rhel/7.3/build_base_machine.sh b/systems/rhel/7.3/build_base_machine.sh index 5a9b4b2e..58953e23 100755 --- a/systems/rhel/7.3/build_base_machine.sh +++ b/systems/rhel/7.3/build_base_machine.sh @@ -52,6 +52,7 @@ pkglist=( wget\ numactl\ numactl-devel\ + libpng-devel ) # python tools for proper QEMU, DPDK, and OVS make @@ -78,28 +79,24 @@ if [ "${#failedinstall[*]}" -gt 0 ]; then exit 1 fi -# install SCL for python33 by adding a repo to find its location to install it -cat <<'EOT' >> /etc/yum.repos.d/python33.repo -[rhscl-python33-el7] -name=Copr repo for python33-el7 owned by rhscl -baseurl=https://copr-be.cloud.fedoraproject.org/results/rhscl/python33-el7/epel-7-$basearch/ -type=rpm-md -skip_if_unavailable=True -gpgcheck=1 -gpgkey=https://copr-be.cloud.fedoraproject.org/results/rhscl/python33-el7/pubkey.gpg -repo_gpgcheck=0 +# install SCL for python34 by adding a repo to find its location to install it +cat <<'EOT' >> /etc/yum.repos.d/python34.repo +[centos-sclo-rh] +name=CentOS-7 - SCLo rh +baseurl=http://mirror.centos.org/centos/7/sclo/$basearch/rh/ +gpgcheck=0 enabled=1 -enabled_metadata=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-SCLo EOT -# install python33 packages and git-review tool +# install python34 packages and git-review tool yum -y install $(echo " -python33 -python33-python-tkinter +rh-python34 +rh-python34-python-tkinter " | grep -v ^#) -# cleanup python 33 repo file -rm -f /etc/yum.repos.d/python33.repo +# cleanup python 34 repo file +rm -f /etc/yum.repos.d/python34.repo # Create hugepage dirs mkdir -p /dev/hugepages diff --git a/systems/rhel/7.3/prepare_python_env.sh b/systems/rhel/7.3/prepare_python_env.sh index b573bb9f..e137aaab 100755 --- a/systems/rhel/7.3/prepare_python_env.sh +++ b/systems/rhel/7.3/prepare_python_env.sh @@ -21,8 +21,8 @@ if [ -d "$VSPERFENV_DIR" ] ; then exit fi -scl enable python33 " -virtualenv "$VSPERFENV_DIR" --python /opt/rh/python33/root/usr/bin/python3 +scl enable rh-python34 " +virtualenv "$VSPERFENV_DIR" --python /opt/rh/rh-python34/root/usr/bin/python3 source "$VSPERFENV_DIR"/bin/activate pip install -r ../requirements.txt pip install pylint diff --git a/testcases/testcase.py b/testcases/testcase.py index 5d4a6ea9..ebf1e797 100644 --- a/testcases/testcase.py +++ b/testcases/testcase.py @@ -348,8 +348,8 @@ class TestCase(object): self.run_initialize() try: - with self._vswitch_ctl, self._loadgen: - with self._vnf_ctl, self._collector: + with self._vswitch_ctl: + with self._vnf_ctl, self._collector, self._loadgen: if not self._vswitch_none: self._add_flows() diff --git a/tools/load_gen/stressorvm/__init__.py b/tools/load_gen/stressorvm/__init__.py new file mode 100644 index 00000000..6a22d81c --- /dev/null +++ b/tools/load_gen/stressorvm/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2017-2018 Spirent Communications +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Package with wrapper for Stressor-VMs +""" diff --git a/tools/load_gen/stressorvm/stressor_vm.py b/tools/load_gen/stressorvm/stressor_vm.py new file mode 100644 index 00000000..410f10e3 --- /dev/null +++ b/tools/load_gen/stressorvm/stressor_vm.py @@ -0,0 +1,117 @@ +# Copyright 2017-2018 Spirent Communications. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Wrapper file to create and manage Stressor-VM as loadgen +""" + +import logging +import os +from tools import tasks +from tools.load_gen.load_gen import ILoadGenerator +from conf import settings as S + + +class QemuVM(tasks.Process): + """ + Class for controling an instance of QEMU + """ + def __init__(self, index): + self._running = False + self._logger = logging.getLogger(__name__) + self._number = index + pnumber = int(S.getValue('NN_BASE_VNC_PORT')) + self._number + cpumask = ",".join(S.getValue('NN_CORE_BINDING')[self._number]) + self._monitor = '%s/vm%dmonitor' % ('/tmp', pnumber) + self._logfile = (os.path.join(S.getValue('LOG_DIR'), + S.getValue('NN_LOG_FILE')) + + str(self._number)) + self._log_prefix = 'vnf_%d_cmd : ' % pnumber + name = 'NN%d' % index + vnc = ':%d' % pnumber + self._shared_dir = '%s/qemu%d_share' % ('/tmp', pnumber) + if not os.path.exists(self._shared_dir): + try: + os.makedirs(self._shared_dir) + except OSError as exp: + raise OSError("Failed to create shared directory %s: %s", + self._shared_dir, exp) + + self.nics_nr = S.getValue('NN_NICS_NR')[self._number] + self.image = S.getValue('NN_IMAGE')[self._number] + self._cmd = ['sudo', '-E', 'taskset', '-c', cpumask, + S.getValue('TOOLS')['qemu-system'], + '-m', S.getValue('NN_MEMORY')[self._number], + '-smp', S.getValue('NN_SMP')[self._number], + '-cpu', 'host,migratable=off', + '-drive', 'if={},file='.format( + S.getValue('NN_BOOT_DRIVE_TYPE')[self._number]) + + self.image, '-boot', + 'c', '--enable-kvm', + '-monitor', 'unix:%s,server,nowait' % self._monitor, + '-nographic', '-vnc', str(vnc), '-name', name, + '-snapshot', '-net none', '-no-reboot', + '-drive', + 'if=%s,format=raw,file=fat:rw:%s,snapshot=off' % + (S.getValue('NN_SHARED_DRIVE_TYPE')[self._number], + self._shared_dir) + ] + + def start(self): + """ + Start QEMU instance + """ + super(QemuVM, self).start() + self._running = True + + def stop(self, sig, slp): + """ + Stops VNF instance. + """ + if self._running: + self._logger.info('Killing VNF...') + # force termination of VNF and wait to terminate; It will avoid + # sporadic reboot of host. + super(QemuVM, self).kill(signal=sig, sleep=slp) + # remove shared dir if it exists to avoid issues with file consistency + if os.path.exists(self._shared_dir): + tasks.run_task(['rm', '-f', '-r', self._shared_dir], self._logger, + 'Removing content of shared directory...', True) + self._running = False + + +# pylint: disable=super-init-not-called +class StressorVM(ILoadGenerator): + """ + Wrapper Class for Load-Generation through stressor-vm + """ + # pylint: disable=unused-argument + def __init__(self, config): + self.qvm_list = [] + for vmindex in range(int(S.getValue('NN_COUNT'))): + qvm = QemuVM(vmindex) + self.qvm_list.append(qvm) + + def start(self): + """Start stressor VMs + """ + for nvm in self.qvm_list: + nvm.start() + + def kill(self, signal='-9', sleep=2): + """ + Stop Stressor VMs + """ + for nvm in self.qvm_list: + nvm.stop(signal, sleep) |