Support of configurable background load and implementation of LTD.CPU.RFC2544.0PacketLoss and LTD.Memory.RFC2544.0PacketLoss

Every testcase can be configured to run background load
on the DUT. Load is generated by chosen command line tool,
which must be installed according to the installation.md.
Currently 'stress' and 'stress-ng' tools are supported,
but support for new tool can be easily added.
Load can be configured by testcase configuration option 'load'.
Details about supported load generator options can be found
in conf/01_testcases.conf.

JIRA: VSPERF-23
JIRA: VSPERF-24
Change-Id: I3e1aebc0d41357f5a2764015c41ffc7e50fdbf8b
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Billy O Mahony <billy.o.mahony@intel.com>
Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>

13 files changed, 393 insertions, 8 deletions

diff --git a/conf/01_testcases.conf b/conf/01_testcases.conf
index 82252fb2..a95a4b54 100755
--- a/conf/01_testcases.conf
+++ b/conf/01_testcases.conf
@@ -25,7 +25,40 @@
 # "Deployment": "p2p",             # One of the supported deployment scenarios.
 # "Description": "Lorem ipsum..."  # Optional. A human-readable string
 #                                  # describing the test.
-# "MultiStream": [true|false],
+# "biDirectional": [true|false],   # Specifies if genearted traffic will be
+#                                  # full-duplex (true) or half-duplex (false)
+# "MultiStream": 0-65535           # Optional. Defines number of flows simulated
+#                                  # by traffic generator. Value 0 disables
+#                                  # MultiStream feature
+# "Load": dictionary               # Optional. Configures background load
+#                                  # during testcase execution.
+#   Descritopn of "Load" dictionary keys, their meanings and available values:
+#
+#   "tool": "stress"               # One of the supported load generators.
+#   "load": 0-100                  # percentage of cores which should be
+#                                  # utilized by load generator
+#                                  # e.g. load = 70%, detected cpu cores = 14 =>
+#                                  # round(14*0.7)=10, i.e. 10 instances of load
+#                                  # generator will be executed
+#   "reserved": 0- (Default 0)     # Optional. Defines number of cores reserved
+#                                  # for vsperf
+#                                  # e.g. load = 80%, detected cpu cores = 14,
+#                                  # reserved = 4 => round((14-4)*0.8)=8,
+#                                  # i.e. 8 load gen. instances will be executed
+#   "pattern" : "c"                # stress/stress-ng specific; Number of 'c',
+#                                  # 'm' and 'i' defines ratio between cpu, mem
+#                                  # and io workers respectively
+#                                  # e.g. "ccccmmi" => ratio among workers types
+#                                  # will be 3:2:1, so in case that 12 stress
+#                                  # instances should be executed, then 6 cpu,
+#                                  # 4 memory and 2 io workers will be executed
+#   "load_memory": 0-100           # Optional. Defines percentage of the system
+#                                  # memory, which should be utilized by memory
+#                                  # workers (if they are part of "pattern").
+#                                  # if not specified then default stress(-ng)
+#                                  # value will be used
+#   "options": ""                  # Optional. Additional command line options
+#                                  # to be passed to the load generator.
 # "Test Modifier": [FrameMod|Other],
 # "Dependency": [Test_Case_Name |None],
 
@@ -70,8 +103,6 @@ PERFORMANCE_TESTS = [
         "Deployment": "p2p",
         "biDirectional": "True",
         "Description": "LTD.Scalability.RFC2544.0PacketLoss",
-        # MultiStream defines number of flows simulated by traffic generator
-        # allowed range: 0-65535; value 0 disables MultiStream feature
         "MultiStream": "8000",
     },
     {
@@ -90,4 +121,33 @@ PERFORMANCE_TESTS = [
         "Description": "LTD.Throughput.RFC2544.BackToBackFrames",
         "biDirectional": "True",
     },
+    {
+        "Name": "phy2phy_cpu_load",
+        "Traffic Type": "rfc2544",
+        "Collector": "cpu",
+        "Deployment": "p2p",
+        "biDirectional": "True",
+        "Description": "LTD.CPU.RFC2544.0PacketLoss",
+        "Load" : {
+            "tool" : "stress-ng",
+            "load" : 100,
+            "reserved" : 4,
+            "pattern" : "c",
+        }
+    },
+    {
+        "Name": "phy2phy_mem_load",
+        "Traffic Type": "rfc2544",
+        "Collector": "cpu",
+        "Deployment": "p2p",
+        "biDirectional": "True",
+        "Description": "LTD.Memory.RFC2544.0PacketLoss",
+        "Load" : {
+            "tool" : "stress-ng",
+            "load" : 50,
+            "pattern" : "m",
+            "load_memory" : 80,
+        }
+    },
+
 ]
diff --git a/core/component_factory.py b/core/component_factory.py
index 1fe0964e..f92de204 100644
--- a/core/component_factory.py
+++ b/core/component_factory.py
@@ -21,6 +21,9 @@ from core.vswitch_controller_pvp import VswitchControllerPVP
 from core.vnf_controller_p2p import VnfControllerP2P
 from core.vnf_controller_pvp import VnfControllerPVP
 from core.collector_controller import CollectorController
+from tools.load_gen.stress.stress import Stress
+from tools.load_gen.stress_ng.stress_ng import StressNg
+from tools.load_gen.dummy.dummy import DummyLoadGen
 
 
 def __init__():
@@ -98,3 +101,22 @@ def create_collector(collector, collector_class):
     if "cpu" in collector or "memory" in collector:
         return CollectorController(collector_class)
 
+def create_loadgen(loadgen_type, loadgen_cfg):
+    """Return a new ILoadGenerator for the loadgen type.
+
+    The returned load generator has the given loadgen type and loadgen
+    generator class.
+
+    :param loadgen_type: Name of loadgen type
+    :param loadgen_class: Reference to load generator class to be used.
+    :return: A new ILoadGenerator class
+    """
+    loadgen_type = loadgen_type.lower()
+    if loadgen_type.find("dummy") >= 0:
+        return DummyLoadGen(loadgen_cfg)
+    elif loadgen_type.find("stress-ng") >= 0:
+        return StressNg(loadgen_cfg)
+    elif loadgen_type.find("stress") >= 0:
+        return Stress(loadgen_cfg)
+
+
diff --git a/docs/NEWS.rst b/docs/NEWS.rst
index 223d3c8a..3f01e371 100644
--- a/docs/NEWS.rst
+++ b/docs/NEWS.rst
@@ -4,6 +4,9 @@ New
 ---
 - Backport and enhancement of reporting
 - PVP deployment scenario testing using vhost-cuse as guest access method
+- Implementation of LTD.Scalability.RFC2544.0PacketLoss testcase
+- Support for background load generation with command line tools like stress
+  and stress-ng
 
 July 2015
 =========
@@ -51,6 +54,5 @@ New
 Missing
 -------
 
--  Report generation is currently disabled
 -  xmlunit output is currently disabled
 -  VNF support.
diff --git a/docs/installation.rst b/docs/installation.rst
index e9a3d115..90de7a0a 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -1,5 +1,5 @@
-Installing toit
-===============
+Installing vswitchperf
+======================
 
 The test suite requires Python 3.3 and relies on a number of other
 packages. These need to be installed for the test suite to function. To
@@ -18,6 +18,14 @@ Enable Software Collections (SCL)
    yum -y install scl-utils
    yum -y install https://www.softwarecollections.org/en/scls/rhscl/python33/epel-7-x86_64/download/rhscl-python33-epel-7-x86_64.noarch.rpm
 
+(Optional) Enable Repoforge (for stress)
+----------------------------------------
+Allows optional installation of stress tool, which is required by load tests.
+
+  .. code-block:: console
+
+   yum -y install http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el7.rf.x86_64.rpm
+
 System packages
 -----------------
 There are a number of packages that must be installed using `yum`. These can be installed like so:
@@ -26,6 +34,11 @@ There are a number of packages that must be installed using `yum`. These can be
 
    yum -y --exclude=python33-mod_wsgi* install python33-* pciutils
 
+Optional installation of stress tool
+
+  .. code-block:: console
+
+   yum -y install stress
 
 Python 3 Packages
 -----------------
diff --git a/testcases/testcase.py b/testcases/testcase.py
index 6191a117..16e222bf 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -44,6 +44,15 @@ class TestCase(object):
         self._collector = cfg['Collector']
         self._bidir = cfg['biDirectional']
         self._frame_mod = cfg.get('Frame Modification', None)
+
+        # check if test requires background load and which generator it uses
+        self._load_cfg = cfg.get('Load', None)
+        if self._load_cfg and 'tool' in self._load_cfg:
+            self._loadgen = self._load_cfg['tool']
+        else:
+            # background load is not requested, so use dummy implementation
+            self._loadgen = "Dummy"
+
         if self._frame_mod:
             self._frame_mod = self._frame_mod.lower()
         self._results_dir = results_dir
@@ -71,12 +80,15 @@ class TestCase(object):
         collector_ctl = component_factory.create_collector(
             self._collector,
             loader.get_collector_class())
-
+        loadgen = component_factory.create_loadgen(
+            self._loadgen,
+            self._load_cfg)
 
         self._logger.debug("Setup:")
         collector_ctl.log_cpu_stats()
-        with vswitch_ctl:
+        with vswitch_ctl, loadgen:
             with vnf_ctl:
+                vnf_ctl.start()
                 traffic = {'traffic_type': self._traffic_type,
                            'bidir': self._bidir,
                            'multistream': self._multistream}
diff --git a/tools/load_gen/__init__.py b/tools/load_gen/__init__.py
new file mode 100644
index 00000000..2eee60bd
--- /dev/null
+++ b/tools/load_gen/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Package with wrappers of load generators
+"""
diff --git a/tools/load_gen/dummy/__init__.py b/tools/load_gen/dummy/__init__.py
new file mode 100644
index 00000000..834e9f63
--- /dev/null
+++ b/tools/load_gen/dummy/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Package with wrapper of dummy load generator
+"""
diff --git a/tools/load_gen/dummy/dummy.py b/tools/load_gen/dummy/dummy.py
new file mode 100644
index 00000000..74fc33b7
--- /dev/null
+++ b/tools/load_gen/dummy/dummy.py
@@ -0,0 +1,31 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Class with implementation of wrapper of dummy load generator"""
+
+from tools.load_gen.load_gen import ILoadGenerator
+
+class DummyLoadGen(ILoadGenerator):
+    """Dummy load generator, which doesn't generate any load"""
+    def __init__(self, stress_config):
+        """Initialise process state."""
+        pass
+
+    def start(self):
+        """Start stress load if it was requested"""
+        pass
+
+    def kill(self):
+        """Kill stress load if it is active"""
+        pass
diff --git a/tools/load_gen/load_gen.py b/tools/load_gen/load_gen.py
new file mode 100644
index 00000000..b55a643c
--- /dev/null
+++ b/tools/load_gen/load_gen.py
@@ -0,0 +1,22 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generic interface VSPERF uses for controlling a load generators
+"""
+
+from tools import tasks
+
+class ILoadGenerator(tasks.CustomProcess):
+    """Interface class that is implemented by specific classes of load
+    generators is derived from CustomProcess"""
diff --git a/tools/load_gen/stress/__init__.py b/tools/load_gen/stress/__init__.py
new file mode 100644
index 00000000..6b04c74d
--- /dev/null
+++ b/tools/load_gen/stress/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Package with wrapper of stress tool
+"""
diff --git a/tools/load_gen/stress/stress.py b/tools/load_gen/stress/stress.py
new file mode 100644
index 00000000..4a53f2d2
--- /dev/null
+++ b/tools/load_gen/stress/stress.py
@@ -0,0 +1,124 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module with implementation of wrapper around the stress tool
+"""
+
+import logging
+import subprocess
+import copy
+from tools import tasks
+from tools import systeminfo
+from tools.load_gen.load_gen import ILoadGenerator
+
+class Stress(ILoadGenerator):
+    """Wrapper around stress tool, which generates load based on testcase
+    configuration parameter 'load'
+    """
+    _process_args = {
+        'cmd': ['sudo', 'stress'],
+        'timeout': 5,
+        'logfile': '/tmp/stress.log',
+        'expect': r'stress: info:',
+        'name': 'stress'
+    }
+    _logger = logging.getLogger(__name__)
+
+    def __init__(self, stress_config):
+        self._running = False
+        # copy stress process setings before its modification
+        process_args = copy.deepcopy(self._process_args)
+        # check if load is requested and correctly configured
+        if not (stress_config and 'load' in stress_config and
+                'pattern' in stress_config and stress_config['load'] > 0):
+            self._logger.error('stress test is not enabled')
+            return
+
+        if stress_config['load'] < 0 or stress_config['load'] > 100:
+            self._logger.error('defined load %s is out of range 0-100',
+                               stress_config['load'])
+            return
+
+        # check if load tool binary is available
+        if not ('tool' in stress_config) or subprocess.call("which " + stress_config['tool'], shell=True) > 0:
+            self._logger.error("stress tool binary '%s' is not available", stress_config['tool'])
+            return
+
+        # calculate requested load details and load split among different
+        # types of workers
+        cpus = systeminfo.get_cpu_cores()
+        if 'reserved' in stress_config:
+            cpus = cpus - int(stress_config['reserved'])
+            if cpus < 1:
+                cpus = 1
+
+        workers = round(cpus/100 * int(stress_config['load']))
+        cmd_dict = {}
+        p_types = {}
+        p_total = 0
+        for p_type in ('c', 'i', 'm'):
+            p_count = stress_config['pattern'].lower().count(p_type)
+            if p_count > 0:
+                p_types[p_type] = p_count
+                p_total += p_count
+        if p_total < 1:
+            self._logger.error('stress test pattern does not contain any of ' \
+                               'c, i or m pattern types')
+            return
+        for p_type in p_types:
+            cmd_dict['-'+p_type] = round(workers* p_types[p_type] / p_total)
+
+        # check for memory load in case that memory workers are detected
+        # in case of error or 0%, memory size is not specified and default
+        # amount of memory will be used by stress tool
+        if '-m' in cmd_dict and cmd_dict['-m'] > 0:
+            if 'load_memory' in stress_config and \
+                stress_config['load_memory'] > 0 and \
+                stress_config['load_memory'] <= 100:
+
+                mem = systeminfo.get_memory_bytes()
+                if mem:
+                    cmd_dict['--vm-bytes'] = round(int(mem) / 100 * \
+                        stress_config['load_memory'] / cmd_dict['-m'])
+
+        # append stress arguments to cmd list used by parent class Process
+        for key, value in cmd_dict.items():
+            process_args['cmd'].append(key)
+            process_args['cmd'].append(str(value))
+
+        # append load generator options if specified
+        if 'options' in stress_config:
+            process_args['cmd'].append(stress_config['options'])
+
+        # initialize load generator and remember it
+        super(Stress, self).__init__(**process_args)
+        self._running = True
+
+    def start(self):
+        """Start stress load if it was requested
+        """
+        if self._running:
+            super(Stress, self).start()
+
+    def kill(self):
+        """
+        Kill stress load if it is active
+        """
+        if self._running and self._child and self._child.isalive():
+            tasks.run_task(['sudo', 'pkill', self._proc_name],
+                           self._logger)
+
+        self._logger.info(
+            'Log available at %s', self._logfile)
+        self._running = False
diff --git a/tools/load_gen/stress_ng/__init__.py b/tools/load_gen/stress_ng/__init__.py
new file mode 100644
index 00000000..eb530a4d
--- /dev/null
+++ b/tools/load_gen/stress_ng/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Package with wrapper of stress-ng tool
+"""
diff --git a/tools/load_gen/stress_ng/stress_ng.py b/tools/load_gen/stress_ng/stress_ng.py
new file mode 100644
index 00000000..c2592dd1
--- /dev/null
+++ b/tools/load_gen/stress_ng/stress_ng.py
@@ -0,0 +1,35 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module with implementation of wrapper around the stress-ng tool
+"""
+
+import logging
+from tools.load_gen.stress.stress import Stress
+
+class StressNg(Stress):
+    """Wrapper around stress-ng tool, which generates load based on
+    testcase configuration parameter 'load'
+    """
+    _process_args = {
+        'cmd': ['sudo', 'stress-ng'],
+        'timeout': 5,
+        'logfile': '/tmp/stress-ng.log',
+        'expect': r'stress-ng: info:',
+        'name': 'stress-ng'
+    }
+    _logger = logging.getLogger(__name__)
+
+    def __init__(self, stress_config):
+        super(StressNg, self).__init__(stress_config)