Merge "vpp: Reporting update related to VPP"

6 files changed, 83 insertions, 48 deletions

diff --git a/tools/pkt_gen/ixia/ixia.py b/tools/pkt_gen/ixia/ixia.py
index 13c1a9a7..e768be06 100755
--- a/tools/pkt_gen/ixia/ixia.py
+++ b/tools/pkt_gen/ixia/ixia.py
@@ -77,7 +77,7 @@ def configure_env():
     os.environ['IXIA_LOGS_DIR'] = '/tmp/Ixia/Logs'
     os.environ['IXIA_TCL_DIR'] = os.path.expandvars('$IxiaLibPath')
     os.environ['IXIA_SAMPLES'] = os.path.expandvars('$IxiaLibPath/ixTcl1.0')
-    os.environ['IXIA_VERSION'] = systeminfo.get_version('Ixia')['version']
+    os.environ['IXIA_VERSION'] = systeminfo.get_version('Ixia').get()['version']
 
 
 def _build_set_cmds(values, prefix='dict set'):
diff --git a/tools/report/report.jinja b/tools/report/report.jinja
index 90cd43d7..8fe32d8f 100644
--- a/tools/report/report.jinja
+++ b/tools/report/report.jinja
@@ -91,18 +91,18 @@ Below is the environment that the test was performed in:
 - CPU cores: {{tests[0].env.cpu_cores}}
 - Memory: {{tests[0].env.memory}}
 - Virtual Switch Set-up: {{tests[0].deployment}}
-- vswitchperf: GIT tag: {{tests[0].env.vsperf['git_tag']}}
-- Traffic Generator: {{tests[0].env.traffic_gen['name']}}, Version: {{tests[0].env.traffic_gen['version']}}, GIT tag: {{tests[0].env.traffic_gen['git_tag']}}
-- vSwitch: {{tests[0].env.vswitch['name']}}, Version: {{tests[0].env.vswitch['version']}}, GIT tag: {{tests[0].env.vswitch['git_tag']}}
+- vswitchperf: GIT tag: {{tests[0].env.vsperf.get()['git_tag']}}
+- Traffic Generator: {{tests[0].env.traffic_gen.get()['name']}}, Version: {{tests[0].env.traffic_gen.get()['version']}}, GIT tag: {{tests[0].env.traffic_gen.get()['git_tag']}}
+- vSwitch: {{tests[0].env.vswitch.get()['name']}}, Version: {{tests[0].env.vswitch.get()['version']}}, GIT tag: {{tests[0].env.vswitch.get()['git_tag']}}
 {%- if 'dpdk' in tests[0].env %}
-- DPDK Version: {{tests[0].env.dpdk['version']}}, GIT tag: {{tests[0].env.dpdk['git_tag']}}
+- DPDK Version: {{tests[0].env.dpdk.get()['version']}}, GIT tag: {{tests[0].env.dpdk.get()['git_tag']}}
 {%- endif %}
 {%- if 'vnf' in tests[0].env %}
-- VNF: {{tests[0].env.vnf['name']}}, Version: {{tests[0].env.vnf['version']}}, GIT tag: {{tests[0].env.vnf['git_tag']}}
+- VNF: {{tests[0].env.vnf.get()['name']}}, Version: {{tests[0].env.vnf.get()['version']}}, GIT tag: {{tests[0].env.vnf.get()['git_tag']}}
 - VM images:{% for guest_image in tests[0].env.guest_image %}
     - {{guest_image}}{% endfor %}
 - VM loopback apps:{% for loopback_app in tests[0].env.loopback_app %}
-    - {{loopback_app['name']}}, Version: {{loopback_app['version']}}, GIT tag: {{loopback_app['git_tag']}}{% endfor %}
+    - {{loopback_app.get()['name']}}, Version: {{loopback_app.get()['version']}}, GIT tag: {{loopback_app.get()['git_tag']}}{% endfor %}
 {%- endif %}
 
 For each test, a summary of the key test results is provided.
@@ -116,7 +116,7 @@ Below are test details:
 - Deployment: {{ "%s"|format(test.deployment) }}
 - Traffic type: {{ "%s"|format(test.result['type']) }}
 - Packet size: {{ "%s"|format(test.result['packet_size']) }}
-- Bidirectional : {{ "%s"|format(test.conf['biDirectional']) }}
+- Bidirectional : {{ "%s"|format(test.conf['bidir']) }}
 {%- if test.result['tunnel_type'] %}
 - Tunnel type: {{ "%s"|format(test.result['tunnel_type']) }}
 {%- endif %}
diff --git a/tools/report/report.py b/tools/report/report.py
index e2914fdf..4d892075 100644
--- a/tools/report/report.py
+++ b/tools/report/report.py
@@ -18,7 +18,6 @@ vSwitch Characterization Report Generation.
 Generate reports in format defined by X.
 """
 
-import sys
 import os
 import logging
 import jinja2
@@ -31,7 +30,7 @@ _TEMPLATE_FILES = ['report.jinja', 'report_rst.jinja']
 _ROOT_DIR = os.path.normpath(os.path.dirname(os.path.realpath(__file__)))
 
 
-def _get_env(result):
+def _get_env(result, versions):
     """
     Get system configuration.
 
@@ -44,6 +43,16 @@ def _get_env(result):
                 'nic': 'NIC'}
 
     """
+    def _get_version(name, versions):
+        """Returns version of tool with given `name` if version was not read
+        during runtime (not inside given `versions` list), then it will be
+        obtained by call of systeminfo.get_version()
+        """
+        for version in versions:
+            if version.get()['name'] == name:
+                return version
+
+        return systeminfo.get_version(name)
 
     env = {
         'os': systeminfo.get_os(),
@@ -55,11 +64,11 @@ def _get_env(result):
         'platform': systeminfo.get_platform(),
         'vsperf': systeminfo.get_version('vswitchperf'),
         'traffic_gen': systeminfo.get_version(S.getValue('TRAFFICGEN')),
-        'vswitch': systeminfo.get_version(S.getValue('VSWITCH')),
+        'vswitch': _get_version(S.getValue('VSWITCH'), versions),
     }
 
     if S.getValue('VSWITCH').lower().count('dpdk'):
-        env.update({'dpdk': systeminfo.get_version('dpdk')})
+        env.update({'dpdk': _get_version('dpdk', versions)})
 
     if result[ResultsConstants.DEPLOYMENT].count('v'):
         env.update({'vnf': systeminfo.get_version(S.getValue('VNF')),
@@ -71,7 +80,7 @@ def _get_env(result):
     return env
 
 
-def generate(input_file, tc_results, tc_stats, test_type='performance'):
+def generate(testcase):
     """Generate actual report.
 
     Generate a Markdown and RST formatted files using results of tests and some
@@ -84,33 +93,30 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'):
     :param tc_stats: System statistics collected during testcase execution.
         These statistics are overall statistics for all specified packet
         sizes.
+    :param traffic: Dictionary with traffic definition used by TC to control
+        traffic generator.
     :test_type: Specifies type of the testcase. Supported values are
         'performance' and 'integration'.
 
     :returns: Path to generated report
     """
-    output_files = [('.'.join([os.path.splitext(input_file)[0], 'md'])),
-                    ('.'.join([os.path.splitext(input_file)[0], 'rst']))]
+    output_files = [('.'.join([os.path.splitext(testcase.get_output_file())[0], 'md'])),
+                    ('.'.join([os.path.splitext(testcase.get_output_file())[0], 'rst']))]
     template_loader = jinja2.FileSystemLoader(searchpath=_ROOT_DIR)
     template_env = jinja2.Environment(loader=template_loader)
 
     tests = []
     try:
-        for result in tc_results:
-            test_config = {}
-            if test_type == 'performance':
-                for tc_conf in S.getValue('PERFORMANCE_TESTS'):
-                    if tc_conf['Name'] == result[ResultsConstants.ID]:
-                        test_config = tc_conf
-                        break
-            elif test_type == 'integration':
-                for tc_conf in S.getValue('INTEGRATION_TESTS'):
-                    if tc_conf['Name'] == result[ResultsConstants.ID]:
-                        test_config = tc_conf
-                        break
-            else:
-                logging.error("Unsupported test type '%s'. Test details are not known.", test_type)
-
+        # there might be multiple test results, but they are produced
+        # by the same test, only traffic details (e.g. packet size)
+        # differs
+        # in case that multiple TC conf values will be needed, then
+        # testcase refactoring should be made to store updated cfg
+        # options into testcase._cfg dictionary
+        test_config = {'Description' : testcase.get_desc(),
+                       'bidir' : testcase.get_traffic()['bidir']}
+
+        for result in testcase.get_tc_results():
             # pass test results, env details and configuration to template
             tests.append({
                 'ID': result[ResultsConstants.ID].upper(),
@@ -118,8 +124,8 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'):
                 'deployment': result[ResultsConstants.DEPLOYMENT],
                 'conf': test_config,
                 'result': result,
-                'env': _get_env(result),
-                'stats': tc_stats
+                'env': _get_env(result, testcase.get_versions()),
+                'stats': testcase.get_collector().get_results(),
             })
 
             # remove id and deployment from results before rendering
@@ -142,12 +148,6 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'):
 
     except KeyError:
         logging.info("Report: Ignoring file (Wrongly defined columns): %s",
-                     (input_file))
+                     testcase.get_output_file())
         raise
     return output_files
-
-
-if __name__ == '__main__':
-    S.load_from_dir('conf')
-    OUT = generate(sys.argv[1], '', '')
-    print('Test report written to "%s"...' % OUT)
diff --git a/tools/report/report_rst.jinja b/tools/report/report_rst.jinja
index e2cb4c83..eda0c01e 100644
--- a/tools/report/report_rst.jinja
+++ b/tools/report/report_rst.jinja
@@ -23,18 +23,18 @@ Below is the environment that the test was performed in:
 * CPU cores: {{tests[0].env.cpu_cores}}
 * Memory: {{tests[0].env.memory}}
 * Virtual Switch Set-up: {{tests[0].deployment}}
-* vswitchperf: GIT tag: {{tests[0].env.vsperf['git_tag']}}
-* Traffic Generator: {{tests[0].env.traffic_gen['name']}}, Version: {{tests[0].env.traffic_gen['version']}}, GIT tag: {{tests[0].env.traffic_gen['git_tag']}}
-* vSwitch: {{tests[0].env.vswitch['name']}}, Version: {{tests[0].env.vswitch['version']}}, GIT tag: {{tests[0].env.vswitch['git_tag']}}
+* vswitchperf: GIT tag: {{tests[0].env.vsperf.get()['git_tag']}}
+* Traffic Generator: {{tests[0].env.traffic_gen.get()['name']}}, Version: {{tests[0].env.traffic_gen.get()['version']}}, GIT tag: {{tests[0].env.traffic_gen.get()['git_tag']}}
+* vSwitch: {{tests[0].env.vswitch.get()['name']}}, Version: {{tests[0].env.vswitch.get()['version']}}, GIT tag: {{tests[0].env.vswitch.get()['git_tag']}}
 {%- if 'dpdk' in tests[0].env %}
-* DPDK Version: {{tests[0].env.dpdk['version']}}, GIT tag: {{tests[0].env.dpdk['git_tag']}}
+* DPDK Version: {{tests[0].env.dpdk.get()['version']}}, GIT tag: {{tests[0].env.dpdk.get()['git_tag']}}
 {%- endif %}
 {%- if 'vnf' in tests[0].env %}
-* VNF: {{tests[0].env.vnf['name']}}, Version: {{tests[0].env.vnf['version']}}, GIT tag: {{tests[0].env.vnf['git_tag']}}
+* VNF: {{tests[0].env.vnf.get()['name']}}, Version: {{tests[0].env.vnf.get()['version']}}, GIT tag: {{tests[0].env.vnf.get()['git_tag']}}
 * VM images:{% for guest_image in tests[0].env.guest_image %}
     * {{guest_image}}{% endfor %}
 * VM loopback apps:{% for loopback_app in tests[0].env.loopback_app %}
-    * {{loopback_app['name']}}, Version: {{loopback_app['version']}}, GIT tag: {{loopback_app['git_tag']}}{% endfor %}
+    * {{loopback_app.get()['name']}}, Version: {{loopback_app.get()['version']}}, GIT tag: {{loopback_app.get()['git_tag']}}{% endfor %}
 {%- endif %}
 
 Below are test details:
@@ -43,7 +43,7 @@ Below are test details:
 * Description: {{ "%s"|format(tests[0].conf['Description']) }}
 * Deployment: {{ "%s"|format(tests[0].deployment) }}
 * Traffic type: {{ "%s"|format(tests[0].result['type']) }}
-* Bidirectional : {{ "%s"|format(tests[0].conf['biDirectional']) }}
+* Bidirectional : {{ "%s"|format(tests[0].conf['bidir']) }}
 {%- if tests[0].result['tunnel_type'] %}
 * Tunnel type: {{ "%s"|format(tests[0].result['tunnel_type']) }}
 {%- endif %}
diff --git a/tools/systeminfo.py b/tools/systeminfo.py
index d515983d..575dd87e 100644
--- a/tools/systeminfo.py
+++ b/tools/systeminfo.py
@@ -23,6 +23,7 @@ import re
 import distro
 
 from conf import settings as S
+from tools.version import Version
 
 def match_line(file_name, pattern):
     """ loops through given file and returns first line matching given pattern
@@ -328,7 +329,7 @@ def get_version(app_name):
         app_version = 'NA'
         app_git_tag = 'NA'
 
-    return {'name' : app_name, 'version' : app_version, 'git_tag' : app_git_tag}
+    return Version(app_name, app_version, app_git_tag)
 
 def get_loopback_version(loopback_app_name):
     """ Get version of given guest loopback application and its git tag
@@ -337,5 +338,5 @@ def get_loopback_version(loopback_app_name):
         version or git tag are not known or not applicaple, than None is returned for any unknown value
     """
     version = get_version("loopback_{}".format(loopback_app_name))
-    version['name'] = loopback_app_name
+    version.set_value('name', loopback_app_name)
     return version
diff --git a/tools/version.py b/tools/version.py
new file mode 100644
index 00000000..30ba3401
--- /dev/null
+++ b/tools/version.py
@@ -0,0 +1,34 @@
+# Copyright 2017 Intel Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Generic class to hold sotware version information shown in the report
+"""
+
+class Version(object):
+    """Container to keep software version details
+    """
+    def __init__(self, name, version, git_tag='NA'):
+        """Create Version object with given data
+        """
+        self._version = {'name' : name, 'version' : version, 'git_tag' : git_tag}
+
+    def set_value(self, key, value):
+        """Upate given `key` by given `value`
+        """
+        self._version[key] = value
+
+    def get(self):
+        """Get content of version object
+        """
+        return self._version