Merge "vpp: Reporting update related to VPP"

13 files changed, 159 insertions, 51 deletions

diff --git a/core/results/results_constants.py b/core/results/results_constants.py
index 2c733fe4..f6fbaad2 100644
--- a/core/results/results_constants.py
+++ b/core/results/results_constants.py
@@ -21,6 +21,7 @@ class ResultsConstants(object):
     ID = 'id'
     PACKET_SIZE = 'packet_size'
     DEPLOYMENT = 'deployment'
+    VSWITCH = 'vswitch'
     TRAFFIC_TYPE = 'traffic_type'
     GUEST_LOOPBACK = 'guest_loopback_app'
     TUNNEL_TYPE = 'tunnel_type'
diff --git a/testcases/performance.py b/testcases/performance.py
index 0be99724..a82b5d1c 100644
--- a/testcases/performance.py
+++ b/testcases/performance.py
@@ -34,4 +34,4 @@ class PerformanceTestCase(TestCase):
     def run_report(self):
         super(PerformanceTestCase, self).run_report()
         if self._tc_results:
-            report.generate(self._output_file, self._tc_results, self._collector.get_results(), self._type)
+            report.generate(self)
diff --git a/testcases/testcase.py b/testcases/testcase.py
index d17abb61..75ed1a5c 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -65,6 +65,7 @@ class TestCase(object):
         self._settings_original = {}
         self._settings_paths_modified = False
         self._testcast_run_time = None
+        self._versions = []
         # initialization of step driven specific members
         self._step_check = False    # by default don't check result for step driven testcases
         self._step_vnf_list = {}
@@ -305,6 +306,8 @@ class TestCase(object):
                     if not self._vswitch_none:
                         self._add_flows()
 
+                    self._versions += self._vswitch_ctl.get_vswitch().get_version()
+
                     with self._traffic_ctl:
                         # execute test based on TestSteps definition if needed...
                         if self.step_run():
@@ -360,6 +363,7 @@ class TestCase(object):
         for item in results:
             item[ResultsConstants.ID] = self.name
             item[ResultsConstants.DEPLOYMENT] = self.deployment
+            item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH')
             item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto']
             item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time
             if self._traffic['multistream']:
@@ -776,3 +780,36 @@ class TestCase(object):
 
         # all steps processed without any issue
         return True
+
+    #
+    # get methods for TestCase members, which needs to be publicly available
+    #
+    def get_output_file(self):
+        """Return content of self._output_file member
+        """
+        return self._output_file
+
+    def get_desc(self):
+        """Return content of self.desc member
+        """
+        return self.desc
+
+    def get_versions(self):
+        """Return content of self.versions member
+        """
+        return self._versions
+
+    def get_traffic(self):
+        """Return content of self._traffic member
+        """
+        return self._traffic
+
+    def get_tc_results(self):
+        """Return content of self._tc_results member
+        """
+        return self._tc_results
+
+    def get_collector(self):
+        """Return content of self._collector member
+        """
+        return self._collector
diff --git a/tools/pkt_gen/ixia/ixia.py b/tools/pkt_gen/ixia/ixia.py
index 13c1a9a7..e768be06 100755
--- a/tools/pkt_gen/ixia/ixia.py
+++ b/tools/pkt_gen/ixia/ixia.py
@@ -77,7 +77,7 @@ def configure_env():
     os.environ['IXIA_LOGS_DIR'] = '/tmp/Ixia/Logs'
     os.environ['IXIA_TCL_DIR'] = os.path.expandvars('$IxiaLibPath')
     os.environ['IXIA_SAMPLES'] = os.path.expandvars('$IxiaLibPath/ixTcl1.0')
-    os.environ['IXIA_VERSION'] = systeminfo.get_version('Ixia')['version']
+    os.environ['IXIA_VERSION'] = systeminfo.get_version('Ixia').get()['version']
 
 
 def _build_set_cmds(values, prefix='dict set'):
diff --git a/tools/report/report.jinja b/tools/report/report.jinja
index 90cd43d7..8fe32d8f 100644
--- a/tools/report/report.jinja
+++ b/tools/report/report.jinja
@@ -91,18 +91,18 @@ Below is the environment that the test was performed in:
 - CPU cores: {{tests[0].env.cpu_cores}}
 - Memory: {{tests[0].env.memory}}
 - Virtual Switch Set-up: {{tests[0].deployment}}
-- vswitchperf: GIT tag: {{tests[0].env.vsperf['git_tag']}}
-- Traffic Generator: {{tests[0].env.traffic_gen['name']}}, Version: {{tests[0].env.traffic_gen['version']}}, GIT tag: {{tests[0].env.traffic_gen['git_tag']}}
-- vSwitch: {{tests[0].env.vswitch['name']}}, Version: {{tests[0].env.vswitch['version']}}, GIT tag: {{tests[0].env.vswitch['git_tag']}}
+- vswitchperf: GIT tag: {{tests[0].env.vsperf.get()['git_tag']}}
+- Traffic Generator: {{tests[0].env.traffic_gen.get()['name']}}, Version: {{tests[0].env.traffic_gen.get()['version']}}, GIT tag: {{tests[0].env.traffic_gen.get()['git_tag']}}
+- vSwitch: {{tests[0].env.vswitch.get()['name']}}, Version: {{tests[0].env.vswitch.get()['version']}}, GIT tag: {{tests[0].env.vswitch.get()['git_tag']}}
 {%- if 'dpdk' in tests[0].env %}
-- DPDK Version: {{tests[0].env.dpdk['version']}}, GIT tag: {{tests[0].env.dpdk['git_tag']}}
+- DPDK Version: {{tests[0].env.dpdk.get()['version']}}, GIT tag: {{tests[0].env.dpdk.get()['git_tag']}}
 {%- endif %}
 {%- if 'vnf' in tests[0].env %}
-- VNF: {{tests[0].env.vnf['name']}}, Version: {{tests[0].env.vnf['version']}}, GIT tag: {{tests[0].env.vnf['git_tag']}}
+- VNF: {{tests[0].env.vnf.get()['name']}}, Version: {{tests[0].env.vnf.get()['version']}}, GIT tag: {{tests[0].env.vnf.get()['git_tag']}}
 - VM images:{% for guest_image in tests[0].env.guest_image %}
     - {{guest_image}}{% endfor %}
 - VM loopback apps:{% for loopback_app in tests[0].env.loopback_app %}
-    - {{loopback_app['name']}}, Version: {{loopback_app['version']}}, GIT tag: {{loopback_app['git_tag']}}{% endfor %}
+    - {{loopback_app.get()['name']}}, Version: {{loopback_app.get()['version']}}, GIT tag: {{loopback_app.get()['git_tag']}}{% endfor %}
 {%- endif %}
 
 For each test, a summary of the key test results is provided.
@@ -116,7 +116,7 @@ Below are test details:
 - Deployment: {{ "%s"|format(test.deployment) }}
 - Traffic type: {{ "%s"|format(test.result['type']) }}
 - Packet size: {{ "%s"|format(test.result['packet_size']) }}
-- Bidirectional : {{ "%s"|format(test.conf['biDirectional']) }}
+- Bidirectional : {{ "%s"|format(test.conf['bidir']) }}
 {%- if test.result['tunnel_type'] %}
 - Tunnel type: {{ "%s"|format(test.result['tunnel_type']) }}
 {%- endif %}
diff --git a/tools/report/report.py b/tools/report/report.py
index e2914fdf..4d892075 100644
--- a/tools/report/report.py
+++ b/tools/report/report.py
@@ -18,7 +18,6 @@ vSwitch Characterization Report Generation.
 Generate reports in format defined by X.
 """
 
-import sys
 import os
 import logging
 import jinja2
@@ -31,7 +30,7 @@ _TEMPLATE_FILES = ['report.jinja', 'report_rst.jinja']
 _ROOT_DIR = os.path.normpath(os.path.dirname(os.path.realpath(__file__)))
 
 
-def _get_env(result):
+def _get_env(result, versions):
     """
     Get system configuration.
 
@@ -44,6 +43,16 @@ def _get_env(result):
                 'nic': 'NIC'}
 
     """
+    def _get_version(name, versions):
+        """Returns version of tool with given `name` if version was not read
+        during runtime (not inside given `versions` list), then it will be
+        obtained by call of systeminfo.get_version()
+        """
+        for version in versions:
+            if version.get()['name'] == name:
+                return version
+
+        return systeminfo.get_version(name)
 
     env = {
         'os': systeminfo.get_os(),
@@ -55,11 +64,11 @@ def _get_env(result):
         'platform': systeminfo.get_platform(),
         'vsperf': systeminfo.get_version('vswitchperf'),
         'traffic_gen': systeminfo.get_version(S.getValue('TRAFFICGEN')),
-        'vswitch': systeminfo.get_version(S.getValue('VSWITCH')),
+        'vswitch': _get_version(S.getValue('VSWITCH'), versions),
     }
 
     if S.getValue('VSWITCH').lower().count('dpdk'):
-        env.update({'dpdk': systeminfo.get_version('dpdk')})
+        env.update({'dpdk': _get_version('dpdk', versions)})
 
     if result[ResultsConstants.DEPLOYMENT].count('v'):
         env.update({'vnf': systeminfo.get_version(S.getValue('VNF')),
@@ -71,7 +80,7 @@ def _get_env(result):
     return env
 
 
-def generate(input_file, tc_results, tc_stats, test_type='performance'):
+def generate(testcase):
     """Generate actual report.
 
     Generate a Markdown and RST formatted files using results of tests and some
@@ -84,33 +93,30 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'):
     :param tc_stats: System statistics collected during testcase execution.
         These statistics are overall statistics for all specified packet
         sizes.
+    :param traffic: Dictionary with traffic definition used by TC to control
+        traffic generator.
     :test_type: Specifies type of the testcase. Supported values are
         'performance' and 'integration'.
 
     :returns: Path to generated report
     """
-    output_files = [('.'.join([os.path.splitext(input_file)[0], 'md'])),
-                    ('.'.join([os.path.splitext(input_file)[0], 'rst']))]
+    output_files = [('.'.join([os.path.splitext(testcase.get_output_file())[0], 'md'])),
+                    ('.'.join([os.path.splitext(testcase.get_output_file())[0], 'rst']))]
     template_loader = jinja2.FileSystemLoader(searchpath=_ROOT_DIR)
     template_env = jinja2.Environment(loader=template_loader)
 
     tests = []
     try:
-        for result in tc_results:
-            test_config = {}
-            if test_type == 'performance':
-                for tc_conf in S.getValue('PERFORMANCE_TESTS'):
-                    if tc_conf['Name'] == result[ResultsConstants.ID]:
-                        test_config = tc_conf
-                        break
-            elif test_type == 'integration':
-                for tc_conf in S.getValue('INTEGRATION_TESTS'):
-                    if tc_conf['Name'] == result[ResultsConstants.ID]:
-                        test_config = tc_conf
-                        break
-            else:
-                logging.error("Unsupported test type '%s'. Test details are not known.", test_type)
-
+        # there might be multiple test results, but they are produced
+        # by the same test, only traffic details (e.g. packet size)
+        # differs
+        # in case that multiple TC conf values will be needed, then
+        # testcase refactoring should be made to store updated cfg
+        # options into testcase._cfg dictionary
+        test_config = {'Description' : testcase.get_desc(),
+                       'bidir' : testcase.get_traffic()['bidir']}
+
+        for result in testcase.get_tc_results():
             # pass test results, env details and configuration to template
             tests.append({
                 'ID': result[ResultsConstants.ID].upper(),
@@ -118,8 +124,8 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'):
                 'deployment': result[ResultsConstants.DEPLOYMENT],
                 'conf': test_config,
                 'result': result,
-                'env': _get_env(result),
-                'stats': tc_stats
+                'env': _get_env(result, testcase.get_versions()),
+                'stats': testcase.get_collector().get_results(),
             })
 
             # remove id and deployment from results before rendering
@@ -142,12 +148,6 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'):
 
     except KeyError:
         logging.info("Report: Ignoring file (Wrongly defined columns): %s",
-                     (input_file))
+                     testcase.get_output_file())
         raise
     return output_files
-
-
-if __name__ == '__main__':
-    S.load_from_dir('conf')
-    OUT = generate(sys.argv[1], '', '')
-    print('Test report written to "%s"...' % OUT)
diff --git a/tools/report/report_rst.jinja b/tools/report/report_rst.jinja
index e2cb4c83..eda0c01e 100644
--- a/tools/report/report_rst.jinja
+++ b/tools/report/report_rst.jinja
@@ -23,18 +23,18 @@ Below is the environment that the test was performed in:
 * CPU cores: {{tests[0].env.cpu_cores}}
 * Memory: {{tests[0].env.memory}}
 * Virtual Switch Set-up: {{tests[0].deployment}}
-* vswitchperf: GIT tag: {{tests[0].env.vsperf['git_tag']}}
-* Traffic Generator: {{tests[0].env.traffic_gen['name']}}, Version: {{tests[0].env.traffic_gen['version']}}, GIT tag: {{tests[0].env.traffic_gen['git_tag']}}
-* vSwitch: {{tests[0].env.vswitch['name']}}, Version: {{tests[0].env.vswitch['version']}}, GIT tag: {{tests[0].env.vswitch['git_tag']}}
+* vswitchperf: GIT tag: {{tests[0].env.vsperf.get()['git_tag']}}
+* Traffic Generator: {{tests[0].env.traffic_gen.get()['name']}}, Version: {{tests[0].env.traffic_gen.get()['version']}}, GIT tag: {{tests[0].env.traffic_gen.get()['git_tag']}}
+* vSwitch: {{tests[0].env.vswitch.get()['name']}}, Version: {{tests[0].env.vswitch.get()['version']}}, GIT tag: {{tests[0].env.vswitch.get()['git_tag']}}
 {%- if 'dpdk' in tests[0].env %}
-* DPDK Version: {{tests[0].env.dpdk['version']}}, GIT tag: {{tests[0].env.dpdk['git_tag']}}
+* DPDK Version: {{tests[0].env.dpdk.get()['version']}}, GIT tag: {{tests[0].env.dpdk.get()['git_tag']}}
 {%- endif %}
 {%- if 'vnf' in tests[0].env %}
-* VNF: {{tests[0].env.vnf['name']}}, Version: {{tests[0].env.vnf['version']}}, GIT tag: {{tests[0].env.vnf['git_tag']}}
+* VNF: {{tests[0].env.vnf.get()['name']}}, Version: {{tests[0].env.vnf.get()['version']}}, GIT tag: {{tests[0].env.vnf.get()['git_tag']}}
 * VM images:{% for guest_image in tests[0].env.guest_image %}
     * {{guest_image}}{% endfor %}
 * VM loopback apps:{% for loopback_app in tests[0].env.loopback_app %}
-    * {{loopback_app['name']}}, Version: {{loopback_app['version']}}, GIT tag: {{loopback_app['git_tag']}}{% endfor %}
+    * {{loopback_app.get()['name']}}, Version: {{loopback_app.get()['version']}}, GIT tag: {{loopback_app.get()['git_tag']}}{% endfor %}
 {%- endif %}
 
 Below are test details:
@@ -43,7 +43,7 @@ Below are test details:
 * Description: {{ "%s"|format(tests[0].conf['Description']) }}
 * Deployment: {{ "%s"|format(tests[0].deployment) }}
 * Traffic type: {{ "%s"|format(tests[0].result['type']) }}
-* Bidirectional : {{ "%s"|format(tests[0].conf['biDirectional']) }}
+* Bidirectional : {{ "%s"|format(tests[0].conf['bidir']) }}
 {%- if tests[0].result['tunnel_type'] %}
 * Tunnel type: {{ "%s"|format(tests[0].result['tunnel_type']) }}
 {%- endif %}
diff --git a/tools/systeminfo.py b/tools/systeminfo.py
index d515983d..575dd87e 100644
--- a/tools/systeminfo.py
+++ b/tools/systeminfo.py
@@ -23,6 +23,7 @@ import re
 import distro
 
 from conf import settings as S
+from tools.version import Version
 
 def match_line(file_name, pattern):
     """ loops through given file and returns first line matching given pattern
@@ -328,7 +329,7 @@ def get_version(app_name):
         app_version = 'NA'
         app_git_tag = 'NA'
 
-    return {'name' : app_name, 'version' : app_version, 'git_tag' : app_git_tag}
+    return Version(app_name, app_version, app_git_tag)
 
 def get_loopback_version(loopback_app_name):
     """ Get version of given guest loopback application and its git tag
@@ -337,5 +338,5 @@ def get_loopback_version(loopback_app_name):
         version or git tag are not known or not applicaple, than None is returned for any unknown value
     """
     version = get_version("loopback_{}".format(loopback_app_name))
-    version['name'] = loopback_app_name
+    version.set_value('name', loopback_app_name)
     return version
diff --git a/tools/version.py b/tools/version.py
new file mode 100644
index 00000000..30ba3401
--- /dev/null
+++ b/tools/version.py
@@ -0,0 +1,34 @@
+# Copyright 2017 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generic class to hold sotware version information shown in the report
+"""
+
+class Version(object):
+    """Container to keep software version details
+    """
+    def __init__(self, name, version, git_tag='NA'):
+        """Create Version object with given data
+        """
+        self._version = {'name' : name, 'version' : version, 'git_tag' : git_tag}
+
+    def set_value(self, key, value):
+        """Upate given `key` by given `value`
+        """
+        self._version[key] = value
+
+    def get(self):
+        """Get content of version object
+        """
+        return self._version
diff --git a/vsperf b/vsperf
index da6d15e4..6618bed5 100755
--- a/vsperf
+++ b/vsperf
@@ -336,7 +336,7 @@ def generate_final_report():
             if retval == 0 and os.path.isfile(test_report):
                 _LOGGER.info('Overall test report written to "%s"', test_report)
             else:
-                _LOGGER.error('Generatrion of overall test report has failed.')
+                _LOGGER.error('Generation of overall test report has failed.')
 
             # remove temporary file
             os.remove(_TEMPLATE_RST['tmp'])
diff --git a/vswitches/ovs.py b/vswitches/ovs.py
index b6d64fb7..12620e22 100644
--- a/vswitches/ovs.py
+++ b/vswitches/ovs.py
@@ -95,6 +95,14 @@ class IVSwitchOvs(IVSwitch, tasks.Process):
         """
         pass
 
+    # Method could be a function
+    # pylint: disable=no-self-use
+    def get_version(self):
+        """See IVswitch for general description
+        """
+        # OVS version can be read offline
+        return []
+
     def stop(self):
         """See IVswitch for general description
         """
diff --git a/vswitches/vpp_dpdk_vhost.py b/vswitches/vpp_dpdk_vhost.py
index d0d9e2ac..68375538 100644
--- a/vswitches/vpp_dpdk_vhost.py
+++ b/vswitches/vpp_dpdk_vhost.py
@@ -25,6 +25,7 @@ from src.dpdk import dpdk
 from conf import settings as S
 from vswitches.vswitch import IVSwitch
 from tools import tasks
+from tools.version import Version
 
 # pylint: disable=too-many-public-methods
 class VppDpdkVhost(IVSwitch, tasks.Process):
@@ -110,7 +111,6 @@ class VppDpdkVhost(IVSwitch, tasks.Process):
         self._logger.debug("VPP CLI args: %s", cli_args)
         return cli_args
 
-
     def start(self):
         """Activates DPDK kernel modules and starts VPP
 
@@ -156,6 +156,25 @@ class VppDpdkVhost(IVSwitch, tasks.Process):
         # has not been terminated yet
         tasks.Process.kill(self, signal, sleep)
 
+    def get_version(self):
+        """See IVswitch for general description
+        """
+        versions = []
+        output = self.run_vppctl(['show', 'version', 'verbose'])
+        if output[1]:
+            self._logger.warning("VPP version can not be read!")
+            return versions
+
+        match = re.search(r'Version:\s*(.+)', output[0])
+        if match:
+            versions.append(Version(S.getValue('VSWITCH'), match.group(1)))
+
+        match = re.search(r'DPDK Version:\s*DPDK (.+)', output[0])
+        if match:
+            versions.append(Version('dpdk', match.group(1)))
+
+        return versions
+
     def add_switch(self, switch_name, dummy_params=None):
         """See IVswitch for general description
         """
diff --git a/vswitches/vswitch.py b/vswitches/vswitch.py
index 893bd1ff..dd69e6d9 100644
--- a/vswitches/vswitch.py
+++ b/vswitches/vswitch.py
@@ -20,6 +20,14 @@ class IVSwitch(object):
 
     Other methods are called only between start() and stop()
     """
+    def get_version(self):
+        """Return version of vSwitch and DPDK (if used by vSwitch)
+           This method should be implemented in case, that version
+           of vswitch or DPDK can be read only during vSwitch runtime.
+           Otherwise it can be implemented inside tools/systeminfo.py.
+        """
+        raise NotImplementedError()
+
     def start(self):
         """Start the vSwitch