yardstick: CLI params support for yardstick TCs

Vswitchperf specific class in yardstick was updated to reflect
new functionality of --test-params. These changes have impact
on vsperf documentation and yardstick samples.
As part of this fix following changes were made:
* trafficgen mode can be executed with default configuration
* yardstick specific TC samples were updated
* dummy traffic generator was enhanced to support back2back
* dummy traffic generator was enhanced to accept traffic
results from configuration

JIRA: VSPERF-422
JIRA: VSPERF-205

Change-Id: Ibeaa6ef46bde453e3ca5dee6e4568fc8c3853521
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Bill Michalowski <bmichalo@redhat.com>
Reviewed-by: Antonio Fischetti <antonio.fischetti@intel.com>
Reviewed-by: <sridhar.rao@spirent.com>

5 files changed, 204 insertions, 97 deletions

diff --git a/yardstick/tests/p2p_cont.yaml b/yardstick/tests/p2p_cont.yaml
deleted file mode 100644
index 5cd63793..00000000
--- a/yardstick/tests/p2p_cont.yaml
+++ /dev/null
@@ -1,74 +0,0 @@
-# Copyright 2016 Intel Corporation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# VSPERF specific configuration file for execution of continuous traffic.
-# Traffic executed by traffic generator is forwarded directly between
-# interfaces connected to the traffic generator. So test will only
-# benchmark the performance of OVS external bridge at controller node.
-# For details about supported test options see following file from
-# the yardstick repository:
-#
-# yardstick/benchmark/scenarios/networking/vsperf.py
-
-schema: "yardstick:task:0.1"
-
-scenarios:
--
-  type: Vsperf
-  options:
-    testname: 'rfc2544_p2p_continuous'
-    traffic_type: 'continuous'
-    pkt_sizes: '64'
-    bidirectional: 'True'
-    iload: 100
-    duration: 30
-    trafficgen_port1: 'eth1'
-    trafficgen_port2: 'eth3'
-    external_bridge: 'br-ex'
-    conf-file: '~/vsperf-yardstick.conf'
-
-  host: vsperf.demo
-
-  runner:
-    type: Sequence
-    scenario_option_name: pkt_sizes
-    sequence:
-    - 64
-    - 128
-    - 512
-    - 1024
-    - 1518
-  sla:
-    metrics: 'throughput_rx_fps'
-    throughput_rx_fps: 500000
-    action: monitor
-
-context:
-  name: demo
-  image: vsperf
-  flavor: vsperf-flavor
-  user: ubuntu
-
-  placement_groups:
-    pgrp1:
-      policy: "availability"
-
-  servers:
-    vsperf:
-      floating_ip: true
-      placement: "pgrp1"
-
-  networks:
-    test:
-      cidr: '10.0.0.0/24'
diff --git a/yardstick/tests/p2p_tput.yaml b/yardstick/tests/p2p_rfc2544_throughput_sample.yaml
index 7a172450..b633732b 100644
--- a/yardstick/tests/p2p_tput.yaml
+++ b/yardstick/tests/p2p_rfc2544_throughput_sample.yaml
@@ -16,10 +16,10 @@
 # traffic. Traffic executed by traffic generator is forwarded directly
 # between interfaces connected to the traffic generator. So test will only
 # benchmark the performance of OVS external bridge at controller node.
-# For details about supported test options see following file from
-# the yardstick repository:
+# Details about supported test options and test case execution can be
+# found in VSPERF documentation:
 #
-# yardstick/benchmark/scenarios/networking/vsperf.py
+#   http://artifacts.opnfv.org/vswitchperf/docs/userguide/yardstick.html
 
 schema: "yardstick:task:0.1"
 
@@ -27,22 +27,22 @@ scenarios:
 -
   type: Vsperf
   options:
-    testname: 'rfc2544_p2p_tput'
+    testname: 'p2p_rfc2544_throughput'
     traffic_type: 'rfc2544'
-    pkt_sizes: '64'
+    frame_size: '64'
     bidirectional: 'True'
     iload: 100
-    duration: 30
+    test_params: 'TRAFFICGEN_DURATION=30;'
     trafficgen_port1: 'eth1'
     trafficgen_port2: 'eth3'
     external_bridge: 'br-ex'
-    conf-file: '~/vsperf-yardstick.conf'
+    conf_file: '~/vsperf-yardstick.conf'
 
   host: vsperf.demo
 
   runner:
     type: Sequence
-    scenario_option_name: pkt_sizes
+    scenario_option_name: frame_size
     sequence:
     - 64
     - 128
@@ -50,6 +50,10 @@ scenarios:
     - 1024
     - 1518
   sla:
+    # The throughput SLA (or any other SLA) cannot be set to a meaningful
+    # value without knowledge of the server and networking environment,
+    # possibly including prior testing in that environment to establish
+    # a baseline SLA level under well-understood circumstances.
     metrics: 'throughput_rx_fps'
     throughput_rx_fps: 500000
     action: monitor
diff --git a/yardstick/tests/p2p_back2back.yaml b/yardstick/tests/rfc2544_back2back_dummy.yaml
index 39ab03e1..a3265c52 100644
--- a/yardstick/tests/p2p_back2back.yaml
+++ b/yardstick/tests/rfc2544_back2back_dummy.yaml
@@ -12,14 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# VSPERF specific configuration file for execution of back2back traffic.
-# Traffic executed by traffic generator is forwarded directly between
-# interfaces connected to the traffic generator. So test will only
-# benchmark the performance of OVS external bridge at controller node.
-# For details about supported test options see following file from
-# the yardstick repository:
+# VSPERF specific configuration file for execution of RFC2544 back2back
+# traffic.
+# This test uses a dummy traffic generator, which does not execute a real
+# traffic. The purpose of this test is to demonstrate vsperf execution
+# by yardstick environment. Provided 'test_params' option can modified
+# to use a real traffic generator.
+# Details about supported test options and test case execution can be
+# found in VSPERF documentation:
 #
-# yardstick/benchmark/scenarios/networking/vsperf.py
+#   http://artifacts.opnfv.org/vswitchperf/docs/userguide/yardstick.html
 
 schema: "yardstick:task:0.1"
 
@@ -27,22 +29,21 @@ scenarios:
 -
   type: Vsperf
   options:
-    testname: 'rfc2544_p2p_back2back'
+    testname: 'rfc2544_back2back'
     traffic_type: 'back2back'
-    pkt_sizes: '64'
+    frame_size: '64'
     bidirectional: 'True'
     iload: 100
-    duration: 30
-    trafficgen_port1: 'eth1'
-    trafficgen_port2: 'eth3'
-    external_bridge: 'br-ex'
-    conf-file: '~/vsperf-yardstick.conf'
+    test_params: 'TRAFFICGEN="Dummy";
+        TRAFFICGEN_DUMMY_RESULTS={
+            "b2b frames":"10000",
+            "b2b frame loss %":"0.0"}'
 
   host: vsperf.demo
 
   runner:
     type: Sequence
-    scenario_option_name: pkt_sizes
+    scenario_option_name: frame_size
     sequence:
     - 64
     - 128
@@ -50,6 +51,10 @@ scenarios:
     - 1024
     - 1518
   sla:
+    # The throughput SLA (or any other SLA) cannot be set to a meaningful
+    # value without knowledge of the server and networking environment,
+    # possibly including prior testing in that environment to establish
+    # a baseline SLA level under well-understood circumstances.
     metrics: 'b2b_frames'
     b2b_frames: 1
     action: monitor
diff --git a/yardstick/tests/rfc2544_continuous_dummy.yaml b/yardstick/tests/rfc2544_continuous_dummy.yaml
new file mode 100644
index 00000000..fea0af3f
--- /dev/null
+++ b/yardstick/tests/rfc2544_continuous_dummy.yaml
@@ -0,0 +1,86 @@
+# Copyright 2016 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# VSPERF specific configuration file for execution of RFC2544 continuous
+# traffic.
+# This test uses a dummy traffic generator, which does not execute a real
+# traffic. The purpose of this test is to demonstrate vsperf execution
+# by yardstick environment. Provided 'test_params' option can modified
+# to use a real traffic generator.
+# Details about supported test options and test case execution can be
+# found in VSPERF documentation:
+#
+#   http://artifacts.opnfv.org/vswitchperf/docs/userguide/yardstick.html
+
+schema: "yardstick:task:0.1"
+
+scenarios:
+-
+  type: Vsperf
+  options:
+    testname: 'rfc2544_continuous'
+    traffic_type: 'continuous'
+    frame_size: '64'
+    bidirectional: 'True'
+    iload: 100
+    test_params: 'TRAFFICGEN_DURATION=30;
+        TRAFFICGEN="Dummy";
+        TRAFFICGEN_DUMMY_RESULTS={
+            "frames tx":15000000,
+            "frames rx":15000000,
+            "tx rate %":100,
+            "rx rate %":100,
+            "min latency":1,
+            "max latency":15,
+            "avg latency":2.5,
+            "frameloss %":0,}'
+
+  host: vsperf.demo
+
+  runner:
+    type: Sequence
+    scenario_option_name: frame_size
+    sequence:
+    - 64
+    - 128
+    - 512
+    - 1024
+    - 1518
+  sla:
+    # The throughput SLA (or any other SLA) cannot be set to a meaningful
+    # value without knowledge of the server and networking environment,
+    # possibly including prior testing in that environment to establish
+    # a baseline SLA level under well-understood circumstances.
+    metrics: 'throughput_rx_fps'
+    throughput_rx_fps: 500000
+    action: monitor
+
+context:
+  name: demo
+  image: vsperf
+  flavor: vsperf-flavor
+  user: ubuntu
+
+  placement_groups:
+    pgrp1:
+      policy: "availability"
+
+  servers:
+    vsperf:
+      floating_ip: true
+      placement: "pgrp1"
+
+  networks:
+    test:
+      cidr: '10.0.0.0/24'
diff --git a/yardstick/tests/rfc2544_throughput_dummy.yaml b/yardstick/tests/rfc2544_throughput_dummy.yaml
new file mode 100644
index 00000000..81755c50
--- /dev/null
+++ b/yardstick/tests/rfc2544_throughput_dummy.yaml
@@ -0,0 +1,86 @@
+# Copyright 2016 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# VSPERF specific configuration file for execution of RFC2544 throughput
+# traffic.
+# This test uses a dummy traffic generator, which does not execute a real
+# traffic. The purpose of this test is to demonstrate vsperf execution
+# by yardstick environment. Provided 'test_params' option can modified
+# to use a real traffic generator.
+# Details about supported test options and test case execution can be
+# found in VSPERF documentation:
+#
+#   http://artifacts.opnfv.org/vswitchperf/docs/userguide/yardstick.html
+
+schema: "yardstick:task:0.1"
+
+scenarios:
+-
+  type: Vsperf
+  options:
+    testname: 'rfc2544_throughput'
+    traffic_type: 'rfc2544'
+    frame_size: '64'
+    bidirectional: 'True'
+    iload: 100
+    test_params: 'TRAFFICGEN_DURATION=30;
+        TRAFFICGEN="Dummy";
+        TRAFFICGEN_DUMMY_RESULTS={
+            "frames tx":15000000,
+            "frames rx":15000000,
+            "tx rate %":100,
+            "rx rate %":100,
+            "min latency":1,
+            "max latency":15,
+            "avg latency":2.5,
+            "frameloss %":0,}'
+
+  host: vsperf.demo
+
+  runner:
+    type: Sequence
+    scenario_option_name: frame_size
+    sequence:
+    - 64
+    - 128
+    - 512
+    - 1024
+    - 1518
+  sla:
+    # The throughput SLA (or any other SLA) cannot be set to a meaningful
+    # value without knowledge of the server and networking environment,
+    # possibly including prior testing in that environment to establish
+    # a baseline SLA level under well-understood circumstances.
+    metrics: 'throughput_rx_fps'
+    throughput_rx_fps: 500000
+    action: monitor
+
+context:
+  name: demo
+  image: vsperf
+  flavor: vsperf-flavor
+  user: ubuntu
+
+  placement_groups:
+    pgrp1:
+      policy: "availability"
+
+  servers:
+    vsperf:
+      floating_ip: true
+      placement: "pgrp1"
+
+  networks:
+    test:
+      cidr: '10.0.0.0/24'