vpp: Reporting update related to VPP

VSPERF reports were updated to contain vSwitch name.
In case of VPP, it is not possible to read its version
if VPP is not running. Thus VSPERF was enhanced
to support tool version check during its runtime. This
mechanism can be enhanced to support e.g. VNF in the future.

JIRA: VSPERF-496

Change-Id: I74b47505e35340eead165d9b588e9cc15c058bbf
Signed-off-by: Martin Klozik <martinx.klozik@intel.com>
Reviewed-by: Al Morton <acmorton@att.com>
Reviewed-by: Christian Trautman <ctrautma@redhat.com>
Reviewed-by: Sridhar Rao <sridhar.rao@spirent.com>
Reviewed-by: Trevor Cooper <trevor.cooper@intel.com>

13 files changed, 159 insertions, 51 deletions

diff --git a/core/results/results_constants.py b/core/results/results_constants.py
index 2c733fe4..f6fbaad2 100644
--- a/core/results/results_constants.py
+++ b/core/results/results_constants.py
@@ -21,6 +21,7 @@ class ResultsConstants(object):
     ID = 'id'
     PACKET_SIZE = 'packet_size'
     DEPLOYMENT = 'deployment'
+    VSWITCH = 'vswitch'
     TRAFFIC_TYPE = 'traffic_type'
     GUEST_LOOPBACK = 'guest_loopback_app'
     TUNNEL_TYPE = 'tunnel_type'
diff --git a/testcases/performance.py b/testcases/performance.py
index 0be99724..a82b5d1c 100644
--- a/testcases/performance.py
+++ b/testcases/performance.py
@@ -34,4 +34,4 @@ class PerformanceTestCase(TestCase):
     def run_report(self):
         super(PerformanceTestCase, self).run_report()
         if self._tc_results:
-            report.generate(self._output_file, self._tc_results, self._collector.get_results(), self._type)
+            report.generate(self)
diff --git a/testcases/testcase.py b/testcases/testcase.py
index 4fbf9c04..5446d0d9 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -65,6 +65,7 @@ class TestCase(object):
         self._settings_original = {}
         self._settings_paths_modified = False
         self._testcast_run_time = None
+        self._versions = []
         # initialization of step driven specific members
         self._step_check = False    # by default don't check result for step driven testcases
         self._step_vnf_list = {}
@@ -303,6 +304,8 @@ class TestCase(object):
                     if not self._vswitch_none:
                         self._add_flows()
 
+                    self._versions += self._vswitch_ctl.get_vswitch().get_version()
+
                     with self._traffic_ctl:
                         # execute test based on TestSteps definition if needed...
                         if self.step_run():
@@ -358,6 +361,7 @@ class TestCase(object):
         for item in results:
             item[ResultsConstants.ID] = self.name
             item[ResultsConstants.DEPLOYMENT] = self.deployment
+            item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
             if self._traffic['multistream']:
@@ -774,3 +778,36 @@ class TestCase(object):
 
         # all steps processed without any issue
         return True
+
+    #
+    # get methods for TestCase members, which needs to be publicly available
+    #
+    def get_output_file(self):
+        """Return content of self._output_file member
+        """
+        return self._output_file
+
+    def get_desc(self):
+        """Return content of self.desc member
+        """
+        return self.desc
+
+    def get_versions(self):
+        """Return content of self.versions member
+        """
+        return self._versions
+
+    def get_traffic(self):
+        """Return content of self._traffic member
+        """
+        return self._traffic
+
+    def get_tc_results(self):
+        """Return content of self._tc_results member
+        """
+        return self._tc_results
+
+    def get_collector(self):
+        """Return content of self._collector member
+        """
+        return self._collector
diff --git a/tools/pkt_gen/ixia/ixia.py b/tools/pkt_gen/ixia/ixia.py
index 13c1a9a7..e768be06 100755
--- a/tools/pkt_gen/ixia/ixia.py
+++ b/tools/pkt_gen/ixia/ixia.py
@@ -77,7 +77,7 @@ def configure_env():
     os.environ['IXIA_LOGS_DIR'] = '/tmp/Ixia/Logs'
     os.environ['IXIA_TCL_DIR'] = os.path.expandvars('$IxiaLibPath')
     os.environ['IXIA_SAMPLES'] = os.path.expandvars('$IxiaLibPath/ixTcl1.0')
-    os.environ['IXIA_VERSION'] = systeminfo.get_version('Ixia')['version']
+    os.environ['IXIA_VERSION'] = systeminfo.get_version('Ixia').get()['version']
 
 
 def _build_set_cmds(values, prefix='dict set'):
diff --git a/tools/report/report.jinja b/tools/report/report.jinja
index 90cd43d7..8fe32d8f 100644
--- a/tools/report/report.jinja
+++ b/tools/report/report.jinja
@@ -91,18 +91,18 @@ Below is the environment that the test was performed in:
 - CPU cores: {{tests[0].env.cpu_cores}}
 - Memory: {{tests[0].env.memory}}
 - Virtual Switch Set-up: {{tests[0].deployment}}
-- vswitchperf: GIT tag: {{tests[0].env.vsperf['git_tag']}}
-- Traffic Generator: {{tests[0].env.traffic_gen['name']}}, Version: {{tests[0].env.traffic_gen['version']}}, GIT tag: {{tests[0].env.traffic_gen['git_tag']}}
-- vSwitch: {{tests[0].env.vswitch['name']}}, Version: {{tests[0].env.vswitch['version']}}, GIT tag: {{tests[0].env.vswitch['git_tag']}}
+- vswitchperf: GIT tag: {{tests[0].env.vsperf.get()['git_tag']}}
+- Traffic Generator: {{tests[0].env.traffic_gen.get()['name']}}, Version: {{tests[0].env.traffic_gen.get()['version']}}, GIT tag: {{tests[0].env.traffic_gen.get()['git_tag']}}
+- vSwitch: {{tests[0].env.vswitch.get()['name']}}, Version: {{tests[0].env.vswitch.get()['version']}}, GIT tag: {{tests[0].env.vswitch.get()['git_tag']}}
 {%- if 'dpdk' in tests[0].env %}
-- DPDK Version: {{tests[0].env.dpdk['version']}}, GIT tag: {{tests[0].env.dpdk['git_tag']}}
+- DPDK Version: {{tests[0].env.dpdk.get()['version']}}, GIT tag: {{tests[0].env.dpdk.get()['git_tag']}}
 {%- endif %}
 {%- if 'vnf' in tests[0].env %}
-- VNF: {{tests[0].env.vnf['name']}}, Version: {{tests[0].env.vnf['version']}}, GIT tag: {{tests[0].env.vnf['git_tag']}}
+- VNF: {{tests[0].env.vnf.get()['name']}}, Version: {{tests[0].env.vnf.get()['version']}}, GIT tag: {{tests[0].env.vnf.get()['git_tag']}}
 - VM images:{% for guest_image in tests[0].env.guest_image %}
     - {{guest_image}}{% endfor %}
 - VM loopback apps:{% for loopback_app in tests[0].env.loopback_app %}
-    - {{loopback_app['name']}}, Version: {{loopback_app['version']}}, GIT tag: {{loopback_app['git_tag']}}{% endfor %}
+    - {{loopback_app.get()['name']}}, Version: {{loopback_app.get()['version']}}, GIT tag: {{loopback_app.get()['git_tag']}}{% endfor %}
 {%- endif %}
 
 For each test, a summary of the key test results is provided.
@@ -116,7 +116,7 @@ Below are test details:
 - Deployment: {{ "%s"|format(test.deployment) }}
 - Traffic type: {{ "%s"|format(test.result['type']) }}
 - Packet size: {{ "%s"|format(test.result['packet_size']) }}
-- Bidirectional : {{ "%s"|format(test.conf['biDirectional']) }}
+- Bidirectional : {{ "%s"|format(test.conf['bidir']) }}
 {%- if test.result['tunnel_type'] %}
 - Tunnel type: {{ "%s"|format(test.result['tunnel_type']) }}
 {%- endif %}
diff --git a/tools/report/report.py b/tools/report/report.py
index e2914fdf..4d892075 100644
--- a/tools/report/report.py
+++ b/tools/report/report.py
@@ -18,7 +18,6 @@ vSwitch Characterization Report Generation.
 Generate reports in format defined by X.
 """
 
-import sys
 import os
 import logging
 import jinja2
@@ -31,7 +30,7 @@ _TEMPLATE_FILES = ['report.jinja', 'report_rst.jinja']
 _ROOT_DIR = os.path.normpath(os.path.dirname(os.path.realpath(__file__)))
 
 
-def _get_env(result):
+def _get_env(result, versions):
     """
     Get system configuration.
 
@@ -44,6 +43,16 @@ def _get_env(result):
                 'nic': 'NIC'}
 
     """
+    def _get_version(name, versions):
+        """Returns version of tool with given `name` if version was not read
+        during runtime (not inside given `versions` list), then it will be
+        obtained by call of systeminfo.get_version()
+        """
+        for version in versions:
+            if version.get()['name'] == name:
+                return version
+
+        return systeminfo.get_version(name)
 
     env = {
         'os': systeminfo.get_os(),
@@ -55,11 +64,11 @@ def _get_env(result):
         'platform': systeminfo.get_platform(),
         'vsperf': systeminfo.get_version('vswitchperf'),
         'traffic_gen': systeminfo.get_version(S.getValue('TRAFFICGEN')),
-        'vswitch': systeminfo.get_version(S.getValue('VSWITCH')),
+        'vswitch': _get_version(S.getValue('VSWITCH'), versions),
     }
 
     if S.getValue('VSWITCH').lower().count('dpdk'):
-        env.update({'dpdk': systeminfo.get_version('dpdk')})
+        env.update({'dpdk': _get_version('dpdk', versions)})
 
     if result[ResultsConstants.DEPLOYMENT].count('v'):
         env.update({'vnf': systeminfo.get_version(S.getValue('VNF')),
@@ -71,7 +80,7 @@ def _get_env(result):
     return env
 
 
-def generate(input_file, tc_results, tc_stats, test_type='performance'):
+def generate(testcase):
     """Generate actual report.
 
     Generate a Markdown and RST formatted files using results of tests and some
@@ -84,33 +93,30 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'):
     :param tc_stats: System statistics collected during testcase execution.
         These statistics are overall statistics for all specified packet
         sizes.
+    :param traffic: Dictionary with traffic definition used by TC to control
+        traffic generator.
     :test_type: Specifies type of the testcase. Supported values are
         'performance' and 'integration'.
 
     :returns: Path to generated report
     """
-    output_files = [('.'.join([os.path.splitext(input_file)[0], 'md'])),
-                    ('.'.join([os.path.splitext(input_file)[0], 'rst']))]
+    output_files = [('.'.join([os.path.splitext(testcase.get_output_file())[0], 'md'])),
+                    ('.'.join([os.path.splitext(testcase.get_output_file())[0], 'rst']))]
     template_loader = jinja2.FileSystemLoader(searchpath=_ROOT_DIR)
     template_env = jinja2.Environment(loader=template_loader)
 
     tests = []
     try:
-        for result in tc_results:
-            test_config = {}
-            if test_type == 'performance':
-                for tc_conf in S.getValue('PERFORMANCE_TESTS'):
-                    if tc_conf['Name'] == result[ResultsConstants.ID]:
-                        test_config = tc_conf
-                        break
-            elif test_type == 'integration':
-                for tc_conf in S.getValue('INTEGRATION_TESTS'):
-                    if tc_conf['Name'] == result[ResultsConstants.ID]:
-                        test_config = tc_conf
-                        break
-            else:
-                logging.error("Unsupported test type '%s'. Test details are not known.", test_type)
-
+        # there might be multiple test results, but they are produced
+        # by the same test, only traffic details (e.g. packet size)
+        # differs
+        # in case that multiple TC conf values will be needed, then
+        # testcase refactoring should be made to store updated cfg
+        # options into testcase._cfg dictionary
+        test_config = {'Description' : testcase.get_desc(),
+                       'bidir' : testcase.get_traffic()['bidir']}
+
+        for result in testcase.get_tc_results():
             # pass test results, env details and configuration to template
             tests.append({
                 'ID': result[ResultsConstants.ID].upper(),
@@ -118,8 +124,8 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'):
                 'deployment': result[ResultsConstants.DEPLOYMENT],
                 'conf': test_config,
                 'result': result,
-                'env': _get_env(result),
-                'stats': tc_stats
+                'env': _get_env(result, testcase.get_versions()),
+                'stats': testcase.get_collector().get_results(),
             })
 
             # remove id and deployment from results before rendering
@@ -142,12 +148,6 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'):
 
     except KeyError:
         logging.info("Report: Ignoring file (Wrongly defined columns): %s",
-                     (input_file))
+                     testcase.get_output_file())
         raise
     return output_files
-
-
-if __name__ == '__main__':
-    S.load_from_dir('conf')
-    OUT = generate(sys.argv[1], '', '')
-    print('Test report written to "%s"...' % OUT)
diff --git a/tools/report/report_rst.jinja b/tools/report/report_rst.jinja
index e2cb4c83..eda0c01e 100644
--- a/tools/report/report_rst.jinja
+++ b/tools/report/report_rst.jinja
@@ -23,18 +23,18 @@ Below is the environment that the test was performed in:
 * CPU cores: {{tests[0].env.cpu_cores}}
 * Memory: {{tests[0].env.memory}}
 * Virtual Switch Set-up: {{tests[0].deployment}}
-* vswitchperf: GIT tag: {{tests[0].env.vsperf['git_tag']}}
-* Traffic Generator: {{tests[0].env.traffic_gen['name']}}, Version: {{tests[0].env.traffic_gen['version']}}, GIT tag: {{tests[0].env.traffic_gen['git_tag']}}
-* vSwitch: {{tests[0].env.vswitch['name']}}, Version: {{tests[0].env.vswitch['version']}}, GIT tag: {{tests[0].env.vswitch['git_tag']}}
+* vswitchperf: GIT tag: {{tests[0].env.vsperf.get()['git_tag']}}
+* Traffic Generator: {{tests[0].env.traffic_gen.get()['name']}}, Version: {{tests[0].env.traffic_gen.get()['version']}}, GIT tag: {{tests[0].env.traffic_gen.get()['git_tag']}}
+* vSwitch: {{tests[0].env.vswitch.get()['name']}}, Version: {{tests[0].env.vswitch.get()['version']}}, GIT tag: {{tests[0].env.vswitch.get()['git_tag']}}
 {%- if 'dpdk' in tests[0].env %}
-* DPDK Version: {{tests[0].env.dpdk['version']}}, GIT tag: {{tests[0].env.dpdk['git_tag']}}
+* DPDK Version: {{tests[0].env.dpdk.get()['version']}}, GIT tag: {{tests[0].env.dpdk.get()['git_tag']}}
 {%- endif %}
 {%- if 'vnf' in tests[0].env %}
-* VNF: {{tests[0].env.vnf['name']}}, Version: {{tests[0].env.vnf['version']}}, GIT tag: {{tests[0].env.vnf['git_tag']}}
+* VNF: {{tests[0].env.vnf.get()['name']}}, Version: {{tests[0].env.vnf.get()['version']}}, GIT tag: {{tests[0].env.vnf.get()['git_tag']}}
 * VM images:{% for guest_image in tests[0].env.guest_image %}
     * {{guest_image}}{% endfor %}
 * VM loopback apps:{% for loopback_app in tests[0].env.loopback_app %}
-    * {{loopback_app['name']}}, Version: {{loopback_app['version']}}, GIT tag: {{loopback_app['git_tag']}}{% endfor %}
+    * {{loopback_app.get()['name']}}, Version: {{loopback_app.get()['version']}}, GIT tag: {{loopback_app.get()['git_tag']}}{% endfor %}
 {%- endif %}
 
 Below are test details:
@@ -43,7 +43,7 @@ Below are test details:
 * Description: {{ "%s"|format(tests[0].conf['Description']) }}
 * Deployment: {{ "%s"|format(tests[0].deployment) }}
 * Traffic type: {{ "%s"|format(tests[0].result['type']) }}
-* Bidirectional : {{ "%s"|format(tests[0].conf['biDirectional']) }}
+* Bidirectional : {{ "%s"|format(tests[0].conf['bidir']) }}
 {%- if tests[0].result['tunnel_type'] %}
 * Tunnel type: {{ "%s"|format(tests[0].result['tunnel_type']) }}
 {%- endif %}
diff --git a/tools/systeminfo.py b/tools/systeminfo.py
index d515983d..575dd87e 100644
--- a/tools/systeminfo.py
+++ b/tools/systeminfo.py
@@ -23,6 +23,7 @@ import re
 import distro
 
 from conf import settings as S
+from tools.version import Version
 
 def match_line(file_name, pattern):
     """ loops through given file and returns first line matching given pattern
@@ -328,7 +329,7 @@ def get_version(app_name):
         app_version = 'NA'
         app_git_tag = 'NA'
 
-    return {'name' : app_name, 'version' : app_version, 'git_tag' : app_git_tag}
+    return Version(app_name, app_version, app_git_tag)
 
 def get_loopback_version(loopback_app_name):
     """ Get version of given guest loopback application and its git tag
@@ -337,5 +338,5 @@ def get_loopback_version(loopback_app_name):
         version or git tag are not known or not applicaple, than None is returned for any unknown value
     """
     version = get_version("loopback_{}".format(loopback_app_name))
-    version['name'] = loopback_app_name
+    version.set_value('name', loopback_app_name)
     return version
diff --git a/tools/version.py b/tools/version.py
new file mode 100644
index 00000000..30ba3401
--- /dev/null
+++ b/tools/version.py
@@ -0,0 +1,34 @@
+# Copyright 2017 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generic class to hold sotware version information shown in the report
+"""
+
+class Version(object):
+    """Container to keep software version details
+    """
+    def __init__(self, name, version, git_tag='NA'):
+        """Create Version object with given data
+        """
+        self._version = {'name' : name, 'version' : version, 'git_tag' : git_tag}
+
+    def set_value(self, key, value):
+        """Upate given `key` by given `value`
+        """
+        self._version[key] = value
+
+    def get(self):
+        """Get content of version object
+        """
+        return self._version
diff --git a/vsperf b/vsperf
index fea7997b..e92a0dc1 100755
--- a/vsperf
+++ b/vsperf
@@ -336,7 +336,7 @@ def generate_final_report():
             if retval == 0 and os.path.isfile(test_report):
                 _LOGGER.info('Overall test report written to "%s"', test_report)
             else:
-                _LOGGER.error('Generatrion of overall test report has failed.')
+                _LOGGER.error('Generation of overall test report has failed.')
 
             # remove temporary file
             os.remove(_TEMPLATE_RST['tmp'])
diff --git a/vswitches/ovs.py b/vswitches/ovs.py
index b6d64fb7..12620e22 100644
--- a/vswitches/ovs.py
+++ b/vswitches/ovs.py
@@ -95,6 +95,14 @@ class IVSwitchOvs(IVSwitch, tasks.Process):
         """
         pass
 
+    # Method could be a function
+    # pylint: disable=no-self-use
+    def get_version(self):
+        """See IVswitch for general description
+        """
+        # OVS version can be read offline
+        return []
+
     def stop(self):
         """See IVswitch for general description
         """
diff --git a/vswitches/vpp_dpdk_vhost.py b/vswitches/vpp_dpdk_vhost.py
index d0d9e2ac..68375538 100644
--- a/vswitches/vpp_dpdk_vhost.py
+++ b/vswitches/vpp_dpdk_vhost.py
@@ -25,6 +25,7 @@ from src.dpdk import dpdk
 from conf import settings as S
 from vswitches.vswitch import IVSwitch
 from tools import tasks
+from tools.version import Version
 
 # pylint: disable=too-many-public-methods
 class VppDpdkVhost(IVSwitch, tasks.Process):
@@ -110,7 +111,6 @@ class VppDpdkVhost(IVSwitch, tasks.Process):
         self._logger.debug("VPP CLI args: %s", cli_args)
         return cli_args
 
-
     def start(self):
         """Activates DPDK kernel modules and starts VPP
 
@@ -156,6 +156,25 @@ class VppDpdkVhost(IVSwitch, tasks.Process):
         # has not been terminated yet
         tasks.Process.kill(self, signal, sleep)
 
+    def get_version(self):
+        """See IVswitch for general description
+        """
+        versions = []
+        output = self.run_vppctl(['show', 'version', 'verbose'])
+        if output[1]:
+            self._logger.warning("VPP version can not be read!")
+            return versions
+
+        match = re.search(r'Version:\s*(.+)', output[0])
+        if match:
+            versions.append(Version(S.getValue('VSWITCH'), match.group(1)))
+
+        match = re.search(r'DPDK Version:\s*DPDK (.+)', output[0])
+        if match:
+            versions.append(Version('dpdk', match.group(1)))
+
+        return versions
+
     def add_switch(self, switch_name, dummy_params=None):
         """See IVswitch for general description
         """
diff --git a/vswitches/vswitch.py b/vswitches/vswitch.py
index 893bd1ff..dd69e6d9 100644
--- a/vswitches/vswitch.py
+++ b/vswitches/vswitch.py
@@ -20,6 +20,14 @@ class IVSwitch(object):
 
     Other methods are called only between start() and stop()
     """
+    def get_version(self):
+        """Return version of vSwitch and DPDK (if used by vSwitch)
+           This method should be implemented in case, that version
+           of vswitch or DPDK can be read only during vSwitch runtime.
+           Otherwise it can be implemented inside tools/systeminfo.py.
+        """
+        raise NotImplementedError()
+
     def start(self):
         """Start the vSwitch