Merge "Support of configurable background load and implementation of LTD.CPU.RFC2544.0PacketLoss and LTD.Memory.RFC2544.0PacketLoss"

13 files changed, 393 insertions, 8 deletions

diff --git a/conf/01_testcases.conf b/conf/01_testcases.conf
index 82252fb2..a95a4b54 100755
--- a/conf/01_testcases.conf
+++ b/conf/01_testcases.conf
@@ -25,7 +25,40 @@
 # "Deployment": "p2p",             # One of the supported deployment scenarios.
 # "Description": "Lorem ipsum..."  # Optional. A human-readable string
 #                                  # describing the test.
-# "MultiStream": [true|false],
+# "biDirectional": [true|false],   # Specifies if genearted traffic will be
+#                                  # full-duplex (true) or half-duplex (false)
+# "MultiStream": 0-65535           # Optional. Defines number of flows simulated
+#                                  # by traffic generator. Value 0 disables
+#                                  # MultiStream feature
+# "Load": dictionary               # Optional. Configures background load
+#                                  # during testcase execution.
+#   Descritopn of "Load" dictionary keys, their meanings and available values:
+#
+#   "tool": "stress"               # One of the supported load generators.
+#   "load": 0-100                  # percentage of cores which should be
+#                                  # utilized by load generator
+#                                  # e.g. load = 70%, detected cpu cores = 14 =>
+#                                  # round(14*0.7)=10, i.e. 10 instances of load
+#                                  # generator will be executed
+#   "reserved": 0- (Default 0)     # Optional. Defines number of cores reserved
+#                                  # for vsperf
+#                                  # e.g. load = 80%, detected cpu cores = 14,
+#                                  # reserved = 4 => round((14-4)*0.8)=8,
+#                                  # i.e. 8 load gen. instances will be executed
+#   "pattern" : "c"                # stress/stress-ng specific; Number of 'c',
+#                                  # 'm' and 'i' defines ratio between cpu, mem
+#                                  # and io workers respectively
+#                                  # e.g. "ccccmmi" => ratio among workers types
+#                                  # will be 3:2:1, so in case that 12 stress
+#                                  # instances should be executed, then 6 cpu,
+#                                  # 4 memory and 2 io workers will be executed
+#   "load_memory": 0-100           # Optional. Defines percentage of the system
+#                                  # memory, which should be utilized by memory
+#                                  # workers (if they are part of "pattern").
+#                                  # if not specified then default stress(-ng)
+#                                  # value will be used
+#   "options": ""                  # Optional. Additional command line options
+#                                  # to be passed to the load generator.
 # "Test Modifier": [FrameMod|Other],
 # "Dependency": [Test_Case_Name |None],
 
@@ -70,8 +103,6 @@ PERFORMANCE_TESTS = [
         "Deployment": "p2p",
         "biDirectional": "True",
         "Description": "LTD.Scalability.RFC2544.0PacketLoss",
-        # MultiStream defines number of flows simulated by traffic generator
-        # allowed range: 0-65535; value 0 disables MultiStream feature
         "MultiStream": "8000",
     },
     {
@@ -90,4 +121,33 @@ PERFORMANCE_TESTS = [
         "Description": "LTD.Throughput.RFC2544.BackToBackFrames",
         "biDirectional": "True",
     },
+    {
+        "Name": "phy2phy_cpu_load",
+        "Traffic Type": "rfc2544",
+        "Collector": "cpu",
+        "Deployment": "p2p",
+        "biDirectional": "True",
+        "Description": "LTD.CPU.RFC2544.0PacketLoss",
+        "Load" : {
+            "tool" : "stress-ng",
+            "load" : 100,
+            "reserved" : 4,
+            "pattern" : "c",
+        }
+    },
+    {
+        "Name": "phy2phy_mem_load",
+        "Traffic Type": "rfc2544",
+        "Collector": "cpu",
+        "Deployment": "p2p",
+        "biDirectional": "True",
+        "Description": "LTD.Memory.RFC2544.0PacketLoss",
+        "Load" : {
+            "tool" : "stress-ng",
+            "load" : 50,
+            "pattern" : "m",
+            "load_memory" : 80,
+        }
+    },
+
 ]
diff --git a/core/component_factory.py b/core/component_factory.py
index 1fe0964e..f92de204 100644
--- a/core/component_factory.py
+++ b/core/component_factory.py
@@ -21,6 +21,9 @@ from core.vswitch_controller_pvp import VswitchControllerPVP
 from core.vnf_controller_p2p import VnfControllerP2P
 from core.vnf_controller_pvp import VnfControllerPVP
 from core.collector_controller import CollectorController
+from tools.load_gen.stress.stress import Stress
+from tools.load_gen.stress_ng.stress_ng import StressNg
+from tools.load_gen.dummy.dummy import DummyLoadGen
 
 
 def __init__():
@@ -98,3 +101,22 @@ def create_collector(collector, collector_class):
     if "cpu" in collector or "memory" in collector:
         return CollectorController(collector_class)
 
+def create_loadgen(loadgen_type, loadgen_cfg):
+    """Return a new ILoadGenerator for the loadgen type.
+
+    The returned load generator has the given loadgen type and loadgen
+    generator class.
+
+    :param loadgen_type: Name of loadgen type
+    :param loadgen_class: Reference to load generator class to be used.
+    :return: A new ILoadGenerator class
+    """
+    loadgen_type = loadgen_type.lower()
+    if loadgen_type.find("dummy") >= 0:
+        return DummyLoadGen(loadgen_cfg)
+    elif loadgen_type.find("stress-ng") >= 0:
+        return StressNg(loadgen_cfg)
+    elif loadgen_type.find("stress") >= 0:
+        return Stress(loadgen_cfg)
+
+
diff --git a/docs/NEWS.rst b/docs/NEWS.rst
index 223d3c8a..3f01e371 100644
--- a/docs/NEWS.rst
+++ b/docs/NEWS.rst
@@ -4,6 +4,9 @@ New
 ---
 - Backport and enhancement of reporting
 - PVP deployment scenario testing using vhost-cuse as guest access method
+- Implementation of LTD.Scalability.RFC2544.0PacketLoss testcase
+- Support for background load generation with command line tools like stress
+  and stress-ng
 
 July 2015
 =========
@@ -51,6 +54,5 @@ New
 Missing
 -------
 
--  Report generation is currently disabled
 -  xmlunit output is currently disabled
 -  VNF support.
diff --git a/docs/installation.rst b/docs/installation.rst
index e9a3d115..90de7a0a 100644
--- a/docs/installation.rst
+++ b/docs/installation.rst
@@ -1,5 +1,5 @@
-Installing toit
-===============
+Installing vswitchperf
+======================
 
 The test suite requires Python 3.3 and relies on a number of other
 packages. These need to be installed for the test suite to function. To
@@ -18,6 +18,14 @@ Enable Software Collections (SCL)
    yum -y install scl-utils
    yum -y install https://www.softwarecollections.org/en/scls/rhscl/python33/epel-7-x86_64/download/rhscl-python33-epel-7-x86_64.noarch.rpm
 
+(Optional) Enable Repoforge (for stress)
+----------------------------------------
+Allows optional installation of stress tool, which is required by load tests.
+
+  .. code-block:: console
+
+   yum -y install http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el7.rf.x86_64.rpm
+
 System packages
 -----------------
 There are a number of packages that must be installed using `yum`. These can be installed like so:
@@ -26,6 +34,11 @@ There are a number of packages that must be installed using `yum`. These can be
 
    yum -y --exclude=python33-mod_wsgi* install python33-* pciutils
 
+Optional installation of stress tool
+
+  .. code-block:: console
+
+   yum -y install stress
 
 Python 3 Packages
 -----------------
diff --git a/testcases/testcase.py b/testcases/testcase.py
index 6191a117..16e222bf 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -44,6 +44,15 @@ class TestCase(object):
         self._collector = cfg['Collector']
         self._bidir = cfg['biDirectional']
         self._frame_mod = cfg.get('Frame Modification', None)
+
+        # check if test requires background load and which generator it uses
+        self._load_cfg = cfg.get('Load', None)
+        if self._load_cfg and 'tool' in self._load_cfg:
+            self._loadgen = self._load_cfg['tool']
+        else:
+            # background load is not requested, so use dummy implementation
+            self._loadgen = "Dummy"
+
         if self._frame_mod:
             self._frame_mod = self._frame_mod.lower()
         self._results_dir = results_dir
@@ -71,12 +80,15 @@ class TestCase(object):
         collector_ctl = component_factory.create_collector(
             self._collector,
             loader.get_collector_class())
-
+        loadgen = component_factory.create_loadgen(
+            self._loadgen,
+            self._load_cfg)
 
         self._logger.debug("Setup:")
         collector_ctl.log_cpu_stats()
-        with vswitch_ctl:
+        with vswitch_ctl, loadgen:
             with vnf_ctl:
+                vnf_ctl.start()
                 traffic = {'traffic_type': self._traffic_type,
                            'bidir': self._bidir,
                            'multistream': self._multistream}
diff --git a/tools/load_gen/__init__.py b/tools/load_gen/__init__.py
new file mode 100644
index 00000000..2eee60bd
--- /dev/null
+++ b/tools/load_gen/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Package with wrappers of load generators
+"""
diff --git a/tools/load_gen/dummy/__init__.py b/tools/load_gen/dummy/__init__.py
new file mode 100644
index 00000000..834e9f63
--- /dev/null
+++ b/tools/load_gen/dummy/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Package with wrapper of dummy load generator
+"""
diff --git a/tools/load_gen/dummy/dummy.py b/tools/load_gen/dummy/dummy.py
new file mode 100644
index 00000000..74fc33b7
--- /dev/null
+++ b/tools/load_gen/dummy/dummy.py
@@ -0,0 +1,31 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Class with implementation of wrapper of dummy load generator"""
+
+from tools.load_gen.load_gen import ILoadGenerator
+
+class DummyLoadGen(ILoadGenerator):
+    """Dummy load generator, which doesn't generate any load"""
+    def __init__(self, stress_config):
+        """Initialise process state."""
+        pass
+
+    def start(self):
+        """Start stress load if it was requested"""
+        pass
+
+    def kill(self):
+        """Kill stress load if it is active"""
+        pass
diff --git a/tools/load_gen/load_gen.py b/tools/load_gen/load_gen.py
new file mode 100644
index 00000000..b55a643c
--- /dev/null
+++ b/tools/load_gen/load_gen.py
@@ -0,0 +1,22 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generic interface VSPERF uses for controlling a load generators
+"""
+
+from tools import tasks
+
+class ILoadGenerator(tasks.CustomProcess):
+    """Interface class that is implemented by specific classes of load
+    generators is derived from CustomProcess"""
diff --git a/tools/load_gen/stress/__init__.py b/tools/load_gen/stress/__init__.py
new file mode 100644
index 00000000..6b04c74d
--- /dev/null
+++ b/tools/load_gen/stress/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Package with wrapper of stress tool
+"""
diff --git a/tools/load_gen/stress/stress.py b/tools/load_gen/stress/stress.py
new file mode 100644
index 00000000..4a53f2d2
--- /dev/null
+++ b/tools/load_gen/stress/stress.py
@@ -0,0 +1,124 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module with implementation of wrapper around the stress tool
+"""
+
+import logging
+import subprocess
+import copy
+from tools import tasks
+from tools import systeminfo
+from tools.load_gen.load_gen import ILoadGenerator
+
+class Stress(ILoadGenerator):
+    """Wrapper around stress tool, which generates load based on testcase
+    configuration parameter 'load'
+    """
+    _process_args = {
+        'cmd': ['sudo', 'stress'],
+        'timeout': 5,
+        'logfile': '/tmp/stress.log',
+        'expect': r'stress: info:',
+        'name': 'stress'
+    }
+    _logger = logging.getLogger(__name__)
+
+    def __init__(self, stress_config):
+        self._running = False
+        # copy stress process setings before its modification
+        process_args = copy.deepcopy(self._process_args)
+        # check if load is requested and correctly configured
+        if not (stress_config and 'load' in stress_config and
+                'pattern' in stress_config and stress_config['load'] > 0):
+            self._logger.error('stress test is not enabled')
+            return
+
+        if stress_config['load'] < 0 or stress_config['load'] > 100:
+            self._logger.error('defined load %s is out of range 0-100',
+                               stress_config['load'])
+            return
+
+        # check if load tool binary is available
+        if not ('tool' in stress_config) or subprocess.call("which " + stress_config['tool'], shell=True) > 0:
+            self._logger.error("stress tool binary '%s' is not available", stress_config['tool'])
+            return
+
+        # calculate requested load details and load split among different
+        # types of workers
+        cpus = systeminfo.get_cpu_cores()
+        if 'reserved' in stress_config:
+            cpus = cpus - int(stress_config['reserved'])
+            if cpus < 1:
+                cpus = 1
+
+        workers = round(cpus/100 * int(stress_config['load']))
+        cmd_dict = {}
+        p_types = {}
+        p_total = 0
+        for p_type in ('c', 'i', 'm'):
+            p_count = stress_config['pattern'].lower().count(p_type)
+            if p_count > 0:
+                p_types[p_type] = p_count
+                p_total += p_count
+        if p_total < 1:
+            self._logger.error('stress test pattern does not contain any of ' \
+                               'c, i or m pattern types')
+            return
+        for p_type in p_types:
+            cmd_dict['-'+p_type] = round(workers* p_types[p_type] / p_total)
+
+        # check for memory load in case that memory workers are detected
+        # in case of error or 0%, memory size is not specified and default
+        # amount of memory will be used by stress tool
+        if '-m' in cmd_dict and cmd_dict['-m'] > 0:
+            if 'load_memory' in stress_config and \
+                stress_config['load_memory'] > 0 and \
+                stress_config['load_memory'] <= 100:
+
+                mem = systeminfo.get_memory_bytes()
+                if mem:
+                    cmd_dict['--vm-bytes'] = round(int(mem) / 100 * \
+                        stress_config['load_memory'] / cmd_dict['-m'])
+
+        # append stress arguments to cmd list used by parent class Process
+        for key, value in cmd_dict.items():
+            process_args['cmd'].append(key)
+            process_args['cmd'].append(str(value))
+
+        # append load generator options if specified
+        if 'options' in stress_config:
+            process_args['cmd'].append(stress_config['options'])
+
+        # initialize load generator and remember it
+        super(Stress, self).__init__(**process_args)
+        self._running = True
+
+    def start(self):
+        """Start stress load if it was requested
+        """
+        if self._running:
+            super(Stress, self).start()
+
+    def kill(self):
+        """
+        Kill stress load if it is active
+        """
+        if self._running and self._child and self._child.isalive():
+            tasks.run_task(['sudo', 'pkill', self._proc_name],
+                           self._logger)
+
+        self._logger.info(
+            'Log available at %s', self._logfile)
+        self._running = False
diff --git a/tools/load_gen/stress_ng/__init__.py b/tools/load_gen/stress_ng/__init__.py
new file mode 100644
index 00000000..eb530a4d
--- /dev/null
+++ b/tools/load_gen/stress_ng/__init__.py
@@ -0,0 +1,16 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Package with wrapper of stress-ng tool
+"""
diff --git a/tools/load_gen/stress_ng/stress_ng.py b/tools/load_gen/stress_ng/stress_ng.py
new file mode 100644
index 00000000..c2592dd1
--- /dev/null
+++ b/tools/load_gen/stress_ng/stress_ng.py
@@ -0,0 +1,35 @@
+# Copyright 2015 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module with implementation of wrapper around the stress-ng tool
+"""
+
+import logging
+from tools.load_gen.stress.stress import Stress
+
+class StressNg(Stress):
+    """Wrapper around stress-ng tool, which generates load based on
+    testcase configuration parameter 'load'
+    """
+    _process_args = {
+        'cmd': ['sudo', 'stress-ng'],
+        'timeout': 5,
+        'logfile': '/tmp/stress-ng.log',
+        'expect': r'stress-ng: info:',
+        'name': 'stress-ng'
+    }
+    _logger = logging.getLogger(__name__)
+
+    def __init__(self, stress_config):
+        super(StressNg, self).__init__(stress_config)