vsperf: Initial integration with VSPERF

New scenario class allows to execute vsperf installed inside VM.
Vsperf is executed in trafficgen mode. It means, it will only
configure and execute traffic by external HW traffic generator
based on custom configuration file. After traffic generator stops,
then test results will be converted from vsperf CSV format into
JSON and passed to yardstick for further processing.
Currently, traffic is passed only through external bridge. In
the future, test scenarios will launch VNFs and traffic will
be properly routed to them. Proper routing can be ensured
by OVS flows configuration via setup-script executed during
setup phase.
Testcase definition yaml files inside vsperf/ directory won't be
pushed to the yardstick, but they will be stored inside vsperf
repository. They are part of the draft review only to show
how it is done.

JIRA: VSPERF-288

Change-Id: I13a519ed39091fe89d1a43cc522738044fb3c609
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: <sunshine.wang@huawei.com>
Reviewed-by: <lvjing5@huawei.com>
Reviewed-by: <jean.gaoliang@huawei.com>
Reviewed-by: <david.j.chou@intel.com>

2 files changed, 361 insertions, 0 deletions

diff --git a/tests/unit/benchmark/scenarios/networking/test_vsperf.py b/tests/unit/benchmark/scenarios/networking/test_vsperf.py
new file mode 100644
index 000000000..cb5c09ab3
--- /dev/null
+++ b/tests/unit/benchmark/scenarios/networking/test_vsperf.py
@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+
+# Copyright 2016 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Unittest for yardstick.benchmark.scenarios.networking.vsperf.Vsperf
+
+import mock
+import unittest
+import os
+import subprocess
+
+from yardstick.benchmark.scenarios.networking import vsperf
+
+
+@mock.patch('yardstick.benchmark.scenarios.networking.vsperf.subprocess')
+@mock.patch('yardstick.benchmark.scenarios.networking.vsperf.ssh')
+@mock.patch("__builtin__.open", return_value=None)
+class VsperfTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.ctx = {
+            "host": {
+                "ip": "10.229.47.137",
+                "user": "ubuntu",
+                "password": "ubuntu",
+            },
+        }
+        self.args = {
+            'options': {
+                'testname': 'rfc2544_p2p_continuous',
+                'traffic_type': 'continuous',
+                'pkt_sizes': '64',
+                'bidirectional': 'True',
+                'iload': 100,
+                'duration': 29,
+                'trafficgen_port1': 'eth1',
+                'trafficgen_port2': 'eth3',
+                'external_bridge': 'br-ex',
+                'conf-file': 'vsperf-yardstick.conf',
+                'setup-script': 'setup_yardstick.sh',
+            },
+            'sla': {
+                'metrics': 'throughput_rx_fps',
+                'throughput_rx_fps': 500000,
+                'action': 'monitor',
+            }
+        }
+
+    def test_vsperf_setup(self, mock_open, mock_ssh, mock_subprocess):
+        p = vsperf.Vsperf(self.args, self.ctx)
+        mock_ssh.SSH().execute.return_value = (0, '', '')
+        mock_subprocess.call().execute.return_value = None
+
+        p.setup()
+        self.assertIsNotNone(p.client)
+        self.assertEqual(p.setup_done, True)
+
+    def test_vsperf_teardown(self, mock_open, mock_ssh, mock_subprocess):
+        p = vsperf.Vsperf(self.args, self.ctx)
+
+        # setup() specific mocks
+        mock_ssh.SSH().execute.return_value = (0, '', '')
+        mock_subprocess.call().execute.return_value = None
+
+        p.setup()
+        self.assertIsNotNone(p.client)
+        self.assertEqual(p.setup_done, True)
+
+        p.teardown()
+        self.assertEqual(p.setup_done, False)
+
+    def test_vsperf_run_ok(self, mock_open, mock_ssh, mock_subprocess):
+        p = vsperf.Vsperf(self.args, self.ctx)
+
+        # setup() specific mocks
+        mock_ssh.SSH().execute.return_value = (0, '', '')
+        mock_subprocess.call().execute.return_value = None
+
+        # run() specific mocks
+        mock_ssh.SSH().execute.return_value = (0, '', '')
+        mock_ssh.SSH().execute.return_value = (0, 'throughput_rx_fps\r\n14797660.000\r\n', '')
+
+        result = {}
+        p.run(result)
+
+        self.assertEqual(result['throughput_rx_fps'], '14797660.000')
+
+    def test_vsperf_run_falied_vsperf_execution(self, mock_open, mock_ssh, mock_subprocess):
+        p = vsperf.Vsperf(self.args, self.ctx)
+
+        # setup() specific mocks
+        mock_ssh.SSH().execute.return_value = (0, '', '')
+        mock_subprocess.call().execute.return_value = None
+
+        # run() specific mocks
+        mock_ssh.SSH().execute.return_value = (1, '', '')
+
+        result = {}
+        self.assertRaises(RuntimeError, p.run, result)
+
+    def test_vsperf_run_falied_csv_report(self, mock_open, mock_ssh, mock_subprocess):
+        p = vsperf.Vsperf(self.args, self.ctx)
+
+        # setup() specific mocks
+        mock_ssh.SSH().execute.return_value = (0, '', '')
+        mock_subprocess.call().execute.return_value = None
+
+        # run() specific mocks
+        mock_ssh.SSH().execute.return_value = (0, '', '')
+        mock_ssh.SSH().execute.return_value = (1, '', '')
+
+        result = {}
+        self.assertRaises(RuntimeError, p.run, result)
+
+
+def main():
+    unittest.main()
+
+if __name__ == '__main__':
+    main()
diff --git a/yardstick/benchmark/scenarios/networking/vsperf.py b/yardstick/benchmark/scenarios/networking/vsperf.py
new file mode 100644
index 000000000..d3123083a
--- /dev/null
+++ b/yardstick/benchmark/scenarios/networking/vsperf.py
@@ -0,0 +1,229 @@
+# Copyright 2016 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" Vsperf specific scenario definition """
+
+import logging
+import os
+import subprocess
+import csv
+
+import yardstick.ssh as ssh
+from yardstick.benchmark.scenarios import base
+
+LOG = logging.getLogger(__name__)
+
+
+class Vsperf(base.Scenario):
+    """Execute vsperf with defined parameters
+
+  Parameters:
+    traffic_type - to specify the type of traffic executed by traffic generator
+    the valid values are "rfc2544", "continuous", "back2back"
+        type:    string
+        default: "rfc2544"
+    pkt_sizes - a packet size for which test should be executed;
+        Multiple packet sizes can be tested by modification of Sequence runner
+        section inside TC YAML definition.
+        type:    string
+        default: "64"
+    duration - sets duration for which traffic will be generated
+        type:    int
+        default: 30
+    bidirectional - speficies if traffic will be uni (False) or bi-directional
+        (True)
+        type:    string
+        default: False
+    iload - specifies frame rate
+        type:    string
+        default: 100
+    rfc2544_trials - the number of trials performed for each packet size
+        type:    string
+        default: NA
+    multistream - the number of simulated streams
+        type:    string
+        default: 0 (disabled)
+    stream_type - specifies network layer used for multistream simulation
+        the valid values are "L4", "L3" and "L2"
+        type:    string
+        default: "L4"
+    conf-file - path to the vsperf configuration file, which will be uploaded
+        to the VM
+        type:   string
+        default: NA
+    setup-script - path to the setup script, which will be executed during
+        setup and teardown phases
+        type:   string
+        default: NA
+    trafficgen_port1 - specifies device name of 1st interface connected to
+        the trafficgen
+        type:   string
+        default: NA
+    trafficgen_port2 - specifies device name of 2nd interface connected to
+        the trafficgen
+        type:   string
+        default: NA
+    external_bridge - specifies name of external bridge configured in OVS
+        type:   string
+        default: "br-ex"
+
+    """
+    __scenario_type__ = "Vsperf"
+
+    VSPERF_CONF = '~/vsperf-yardstick.conf'
+
+    def __init__(self, scenario_cfg, context_cfg):
+        self.scenario_cfg = scenario_cfg
+        self.context_cfg = context_cfg
+        self.setup_done = False
+        self.client = None
+        self.tg_port1 = self.scenario_cfg['options'].get('trafficgen_port1',
+                                                         None)
+        self.tg_port2 = self.scenario_cfg['options'].get('trafficgen_port2',
+                                                         None)
+        self.br_ex = self.scenario_cfg['options'].get('external_bridge',
+                                                      'br-ex')
+        self.vsperf_conf = os.path.expanduser(
+            self.scenario_cfg['options'].get('conf-file', Vsperf.VSPERF_CONF))
+        self.setup_script = self.scenario_cfg['options'].get('setup-script',
+                                                             None)
+        if self.setup_script:
+            self.setup_script = os.path.expanduser(self.setup_script)
+
+    def setup(self):
+        '''scenario setup'''
+        vsperf = self.context_cfg['host']
+        vsperf_user = vsperf.get('user', 'ubuntu')
+        vsperf_password = vsperf.get('password', 'ubuntu')
+        vsperf_ip = vsperf.get('ip', None)
+
+        # add trafficgen interfaces to the external bridge
+        if self.tg_port1:
+            subprocess.call('sudo bash -c "ovs-vsctl add-port %s %s"' %
+                            (self.br_ex, self.tg_port1), shell=True)
+        if self.tg_port2:
+            subprocess.call('sudo bash -c "ovs-vsctl add-port %s %s"' %
+                            (self.br_ex, self.tg_port2), shell=True)
+
+        # copy vsperf conf to VM
+        LOG.info("user:%s, host:%s", vsperf_user, vsperf_ip)
+        self.client = ssh.SSH(vsperf_user, vsperf_ip,
+                              password=vsperf_password)
+        # traffic generation could last long
+        self.client.wait(timeout=1800)
+
+        # copy script to host
+        self.client.run("cat > ~/vsperf.conf",
+                        stdin=open(self.vsperf_conf, "rb"))
+
+        # execute external setup script
+        if self.setup_script:
+            cmd = "%s setup" % (self.setup_script)
+            LOG.info("Execute setup script \"%s\"", cmd)
+            subprocess.call(cmd, shell=True)
+
+        self.setup_done = True
+
+    def run(self, result):
+        """ execute the vsperf benchmark and return test results
+            within result dictionary
+        """
+        def add_test_params(options, option, default_value):
+            """return parameter and its value as a string to be passed
+               to the VSPERF inside --test-params argument
+
+               Parameters:
+                options - dictionary with scenario options
+                option  - a name of option to be added to the string
+                default_value - value to be used in case that option
+                    is not defined inside scenario options
+            """
+            if option in options:
+                return "%s=%s" % (option, options[option])
+            elif default_value is not None:
+                return "%s=%s" % (option, default_value)
+            else:
+                return None
+
+        if not self.setup_done:
+            self.setup()
+
+        # remove results from previous tests
+        self.client.execute("rm -rf /tmp/results*")
+
+        # get vsperf options
+        options = self.scenario_cfg['options']
+        test_params = []
+        test_params.append(add_test_params(options, "traffic_type", "rfc2544"))
+        test_params.append(add_test_params(options, "pkt_sizes", "64"))
+        test_params.append(add_test_params(options, "duration", None))
+        test_params.append(add_test_params(options, "bidirectional", "False"))
+        test_params.append(add_test_params(options, "iload", 100))
+        test_params.append(add_test_params(options, "rfc2544_trials", None))
+        test_params.append(add_test_params(options, "multistream", None))
+        test_params.append(add_test_params(options, "stream_type", None))
+
+        # execute vsperf
+        cmd = "source ~/vsperfenv/bin/activate ; cd vswitchperf ; "
+        cmd += "./vsperf --mode trafficgen --conf-file ~/vsperf.conf "
+        cmd += "--test-params=\"%s\"" % (';'.join(filter(None, test_params)))
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+
+        if status:
+            raise RuntimeError(stderr)
+
+        # get test results
+        cmd = "cat /tmp/results*/result.csv"
+        LOG.debug("Executing command: %s", cmd)
+        status, stdout, stderr = self.client.execute(cmd)
+
+        if status:
+            raise RuntimeError(stderr)
+
+        # convert result.csv to JSON format
+        reader = csv.DictReader(stdout.split('\r\n'))
+        result.update(reader.next())
+
+        # sla check; go through all defined SLAs and check if values measured
+        # by VSPERF are higher then those defined by SLAs
+        if 'sla' in self.scenario_cfg and \
+           'metrics' in self.scenario_cfg['sla']:
+            for metric in self.scenario_cfg['sla']['metrics'].split(','):
+                assert metric in result, \
+                    '%s is not collected by VSPERF' % (metric)
+                assert metric in self.scenario_cfg['sla'], \
+                    '%s is not defined in SLA' % (metric)
+                vs_res = float(result[metric])
+                sla_res = float(self.scenario_cfg['sla'][metric])
+                assert vs_res >= sla_res, \
+                    'VSPERF_%s(%f) < SLA_%s(%f)' % \
+                    (metric, vs_res, metric, sla_res)
+
+    def teardown(self):
+        """cleanup after the test execution"""
+        # remove trafficgen interfaces from the external bridge
+        if self.tg_port1:
+            subprocess.call('sudo bash -c "ovs-vsctl del-port %s %s"' %
+                            (self.br_ex, self.tg_port1), shell=True)
+        if self.tg_port2:
+            subprocess.call('sudo bash -c "ovs-vsctl del-port %s %s"' %
+                            (self.br_ex, self.tg_port2), shell=True)
+
+        # execute external setup script
+        if self.setup_script:
+            cmd = "%s teardown" % (self.setup_script)
+            LOG.info("Execute setup script \"%s\"", cmd)
+            subprocess.call(cmd, shell=True)
+
+        self.setup_done = False