diff options
| -rw-r--r-- | core/results/results_constants.py | 1 | ||||
| -rw-r--r-- | testcases/performance.py | 2 | ||||
| -rw-r--r-- | testcases/testcase.py | 37 | ||||
| -rwxr-xr-x | tools/pkt_gen/ixia/ixia.py | 2 | ||||
| -rw-r--r-- | tools/report/report.jinja | 14 | ||||
| -rw-r--r-- | tools/report/report.py | 62 | ||||
| -rw-r--r-- | tools/report/report_rst.jinja | 14 | ||||
| -rw-r--r-- | tools/systeminfo.py | 5 | ||||
| -rw-r--r-- | tools/version.py | 34 | ||||
| -rwxr-xr-x | vsperf | 2 | ||||
| -rw-r--r-- | vswitches/ovs.py | 8 | ||||
| -rw-r--r-- | vswitches/vpp_dpdk_vhost.py | 21 | ||||
| -rw-r--r-- | vswitches/vswitch.py | 8 |
13 files changed, 159 insertions, 51 deletions
diff --git a/core/results/results_constants.py b/core/results/results_constants.py index 2c733fe4..f6fbaad2 100644 --- a/core/results/results_constants.py +++ b/core/results/results_constants.py @@ -21,6 +21,7 @@ class ResultsConstants(object): ID = 'id' PACKET_SIZE = 'packet_size' DEPLOYMENT = 'deployment' + VSWITCH = 'vswitch' TRAFFIC_TYPE = 'traffic_type' GUEST_LOOPBACK = 'guest_loopback_app' TUNNEL_TYPE = 'tunnel_type' diff --git a/testcases/performance.py b/testcases/performance.py index 0be99724..a82b5d1c 100644 --- a/testcases/performance.py +++ b/testcases/performance.py @@ -34,4 +34,4 @@ class PerformanceTestCase(TestCase): def run_report(self): super(PerformanceTestCase, self).run_report() if self._tc_results: - report.generate(self._output_file, self._tc_results, self._collector.get_results(), self._type) + report.generate(self) diff --git a/testcases/testcase.py b/testcases/testcase.py index 4fbf9c04..5446d0d9 100644 --- a/testcases/testcase.py +++ b/testcases/testcase.py @@ -65,6 +65,7 @@ class TestCase(object): self._settings_original = {} self._settings_paths_modified = False self._testcast_run_time = None + self._versions = [] # initialization of step driven specific members self._step_check = False # by default don't check result for step driven testcases self._step_vnf_list = {} @@ -303,6 +304,8 @@ class TestCase(object): if not self._vswitch_none: self._add_flows() + self._versions += self._vswitch_ctl.get_vswitch().get_version() + with self._traffic_ctl: # execute test based on TestSteps definition if needed... if self.step_run(): @@ -358,6 +361,7 @@ class TestCase(object): for item in results: item[ResultsConstants.ID] = self.name item[ResultsConstants.DEPLOYMENT] = self.deployment + item[ResultsConstants.VSWITCH] = S.getValue('VSWITCH') item[ResultsConstants.TRAFFIC_TYPE] = self._traffic['l3']['proto'] item[ResultsConstants.TEST_RUN_TIME] = self._testcase_run_time if self._traffic['multistream']: @@ -774,3 +778,36 @@ class TestCase(object): # all steps processed without any issue return True + + # + # get methods for TestCase members, which needs to be publicly available + # + def get_output_file(self): + """Return content of self._output_file member + """ + return self._output_file + + def get_desc(self): + """Return content of self.desc member + """ + return self.desc + + def get_versions(self): + """Return content of self.versions member + """ + return self._versions + + def get_traffic(self): + """Return content of self._traffic member + """ + return self._traffic + + def get_tc_results(self): + """Return content of self._tc_results member + """ + return self._tc_results + + def get_collector(self): + """Return content of self._collector member + """ + return self._collector diff --git a/tools/pkt_gen/ixia/ixia.py b/tools/pkt_gen/ixia/ixia.py index 13c1a9a7..e768be06 100755 --- a/tools/pkt_gen/ixia/ixia.py +++ b/tools/pkt_gen/ixia/ixia.py @@ -77,7 +77,7 @@ def configure_env(): os.environ['IXIA_LOGS_DIR'] = '/tmp/Ixia/Logs' os.environ['IXIA_TCL_DIR'] = os.path.expandvars('$IxiaLibPath') os.environ['IXIA_SAMPLES'] = os.path.expandvars('$IxiaLibPath/ixTcl1.0') - os.environ['IXIA_VERSION'] = systeminfo.get_version('Ixia')['version'] + os.environ['IXIA_VERSION'] = systeminfo.get_version('Ixia').get()['version'] def _build_set_cmds(values, prefix='dict set'): diff --git a/tools/report/report.jinja b/tools/report/report.jinja index 90cd43d7..8fe32d8f 100644 --- a/tools/report/report.jinja +++ b/tools/report/report.jinja @@ -91,18 +91,18 @@ Below is the environment that the test was performed in: - CPU cores: {{tests[0].env.cpu_cores}} - Memory: {{tests[0].env.memory}} - Virtual Switch Set-up: {{tests[0].deployment}} -- vswitchperf: GIT tag: {{tests[0].env.vsperf['git_tag']}} -- Traffic Generator: {{tests[0].env.traffic_gen['name']}}, Version: {{tests[0].env.traffic_gen['version']}}, GIT tag: {{tests[0].env.traffic_gen['git_tag']}} -- vSwitch: {{tests[0].env.vswitch['name']}}, Version: {{tests[0].env.vswitch['version']}}, GIT tag: {{tests[0].env.vswitch['git_tag']}} +- vswitchperf: GIT tag: {{tests[0].env.vsperf.get()['git_tag']}} +- Traffic Generator: {{tests[0].env.traffic_gen.get()['name']}}, Version: {{tests[0].env.traffic_gen.get()['version']}}, GIT tag: {{tests[0].env.traffic_gen.get()['git_tag']}} +- vSwitch: {{tests[0].env.vswitch.get()['name']}}, Version: {{tests[0].env.vswitch.get()['version']}}, GIT tag: {{tests[0].env.vswitch.get()['git_tag']}} {%- if 'dpdk' in tests[0].env %} -- DPDK Version: {{tests[0].env.dpdk['version']}}, GIT tag: {{tests[0].env.dpdk['git_tag']}} +- DPDK Version: {{tests[0].env.dpdk.get()['version']}}, GIT tag: {{tests[0].env.dpdk.get()['git_tag']}} {%- endif %} {%- if 'vnf' in tests[0].env %} -- VNF: {{tests[0].env.vnf['name']}}, Version: {{tests[0].env.vnf['version']}}, GIT tag: {{tests[0].env.vnf['git_tag']}} +- VNF: {{tests[0].env.vnf.get()['name']}}, Version: {{tests[0].env.vnf.get()['version']}}, GIT tag: {{tests[0].env.vnf.get()['git_tag']}} - VM images:{% for guest_image in tests[0].env.guest_image %} - {{guest_image}}{% endfor %} - VM loopback apps:{% for loopback_app in tests[0].env.loopback_app %} - - {{loopback_app['name']}}, Version: {{loopback_app['version']}}, GIT tag: {{loopback_app['git_tag']}}{% endfor %} + - {{loopback_app.get()['name']}}, Version: {{loopback_app.get()['version']}}, GIT tag: {{loopback_app.get()['git_tag']}}{% endfor %} {%- endif %} For each test, a summary of the key test results is provided. @@ -116,7 +116,7 @@ Below are test details: - Deployment: {{ "%s"|format(test.deployment) }} - Traffic type: {{ "%s"|format(test.result['type']) }} - Packet size: {{ "%s"|format(test.result['packet_size']) }} -- Bidirectional : {{ "%s"|format(test.conf['biDirectional']) }} +- Bidirectional : {{ "%s"|format(test.conf['bidir']) }} {%- if test.result['tunnel_type'] %} - Tunnel type: {{ "%s"|format(test.result['tunnel_type']) }} {%- endif %} diff --git a/tools/report/report.py b/tools/report/report.py index e2914fdf..4d892075 100644 --- a/tools/report/report.py +++ b/tools/report/report.py @@ -18,7 +18,6 @@ vSwitch Characterization Report Generation. Generate reports in format defined by X. """ -import sys import os import logging import jinja2 @@ -31,7 +30,7 @@ _TEMPLATE_FILES = ['report.jinja', 'report_rst.jinja'] _ROOT_DIR = os.path.normpath(os.path.dirname(os.path.realpath(__file__))) -def _get_env(result): +def _get_env(result, versions): """ Get system configuration. @@ -44,6 +43,16 @@ def _get_env(result): 'nic': 'NIC'} """ + def _get_version(name, versions): + """Returns version of tool with given `name` if version was not read + during runtime (not inside given `versions` list), then it will be + obtained by call of systeminfo.get_version() + """ + for version in versions: + if version.get()['name'] == name: + return version + + return systeminfo.get_version(name) env = { 'os': systeminfo.get_os(), @@ -55,11 +64,11 @@ def _get_env(result): 'platform': systeminfo.get_platform(), 'vsperf': systeminfo.get_version('vswitchperf'), 'traffic_gen': systeminfo.get_version(S.getValue('TRAFFICGEN')), - 'vswitch': systeminfo.get_version(S.getValue('VSWITCH')), + 'vswitch': _get_version(S.getValue('VSWITCH'), versions), } if S.getValue('VSWITCH').lower().count('dpdk'): - env.update({'dpdk': systeminfo.get_version('dpdk')}) + env.update({'dpdk': _get_version('dpdk', versions)}) if result[ResultsConstants.DEPLOYMENT].count('v'): env.update({'vnf': systeminfo.get_version(S.getValue('VNF')), @@ -71,7 +80,7 @@ def _get_env(result): return env -def generate(input_file, tc_results, tc_stats, test_type='performance'): +def generate(testcase): """Generate actual report. Generate a Markdown and RST formatted files using results of tests and some @@ -84,33 +93,30 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'): :param tc_stats: System statistics collected during testcase execution. These statistics are overall statistics for all specified packet sizes. + :param traffic: Dictionary with traffic definition used by TC to control + traffic generator. :test_type: Specifies type of the testcase. Supported values are 'performance' and 'integration'. :returns: Path to generated report """ - output_files = [('.'.join([os.path.splitext(input_file)[0], 'md'])), - ('.'.join([os.path.splitext(input_file)[0], 'rst']))] + output_files = [('.'.join([os.path.splitext(testcase.get_output_file())[0], 'md'])), + ('.'.join([os.path.splitext(testcase.get_output_file())[0], 'rst']))] template_loader = jinja2.FileSystemLoader(searchpath=_ROOT_DIR) template_env = jinja2.Environment(loader=template_loader) tests = [] try: - for result in tc_results: - test_config = {} - if test_type == 'performance': - for tc_conf in S.getValue('PERFORMANCE_TESTS'): - if tc_conf['Name'] == result[ResultsConstants.ID]: - test_config = tc_conf - break - elif test_type == 'integration': - for tc_conf in S.getValue('INTEGRATION_TESTS'): - if tc_conf['Name'] == result[ResultsConstants.ID]: - test_config = tc_conf - break - else: - logging.error("Unsupported test type '%s'. Test details are not known.", test_type) - + # there might be multiple test results, but they are produced + # by the same test, only traffic details (e.g. packet size) + # differs + # in case that multiple TC conf values will be needed, then + # testcase refactoring should be made to store updated cfg + # options into testcase._cfg dictionary + test_config = {'Description' : testcase.get_desc(), + 'bidir' : testcase.get_traffic()['bidir']} + + for result in testcase.get_tc_results(): # pass test results, env details and configuration to template tests.append({ 'ID': result[ResultsConstants.ID].upper(), @@ -118,8 +124,8 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'): 'deployment': result[ResultsConstants.DEPLOYMENT], 'conf': test_config, 'result': result, - 'env': _get_env(result), - 'stats': tc_stats + 'env': _get_env(result, testcase.get_versions()), + 'stats': testcase.get_collector().get_results(), }) # remove id and deployment from results before rendering @@ -142,12 +148,6 @@ def generate(input_file, tc_results, tc_stats, test_type='performance'): except KeyError: logging.info("Report: Ignoring file (Wrongly defined columns): %s", - (input_file)) + testcase.get_output_file()) raise return output_files - - -if __name__ == '__main__': - S.load_from_dir('conf') - OUT = generate(sys.argv[1], '', '') - print('Test report written to "%s"...' % OUT) diff --git a/tools/report/report_rst.jinja b/tools/report/report_rst.jinja index e2cb4c83..eda0c01e 100644 --- a/tools/report/report_rst.jinja +++ b/tools/report/report_rst.jinja @@ -23,18 +23,18 @@ Below is the environment that the test was performed in: * CPU cores: {{tests[0].env.cpu_cores}} * Memory: {{tests[0].env.memory}} * Virtual Switch Set-up: {{tests[0].deployment}} -* vswitchperf: GIT tag: {{tests[0].env.vsperf['git_tag']}} -* Traffic Generator: {{tests[0].env.traffic_gen['name']}}, Version: {{tests[0].env.traffic_gen['version']}}, GIT tag: {{tests[0].env.traffic_gen['git_tag']}} -* vSwitch: {{tests[0].env.vswitch['name']}}, Version: {{tests[0].env.vswitch['version']}}, GIT tag: {{tests[0].env.vswitch['git_tag']}} +* vswitchperf: GIT tag: {{tests[0].env.vsperf.get()['git_tag']}} +* Traffic Generator: {{tests[0].env.traffic_gen.get()['name']}}, Version: {{tests[0].env.traffic_gen.get()['version']}}, GIT tag: {{tests[0].env.traffic_gen.get()['git_tag']}} +* vSwitch: {{tests[0].env.vswitch.get()['name']}}, Version: {{tests[0].env.vswitch.get()['version']}}, GIT tag: {{tests[0].env.vswitch.get()['git_tag']}} {%- if 'dpdk' in tests[0].env %} -* DPDK Version: {{tests[0].env.dpdk['version']}}, GIT tag: {{tests[0].env.dpdk['git_tag']}} +* DPDK Version: {{tests[0].env.dpdk.get()['version']}}, GIT tag: {{tests[0].env.dpdk.get()['git_tag']}} {%- endif %} {%- if 'vnf' in tests[0].env %} -* VNF: {{tests[0].env.vnf['name']}}, Version: {{tests[0].env.vnf['version']}}, GIT tag: {{tests[0].env.vnf['git_tag']}} +* VNF: {{tests[0].env.vnf.get()['name']}}, Version: {{tests[0].env.vnf.get()['version']}}, GIT tag: {{tests[0].env.vnf.get()['git_tag']}} * VM images:{% for guest_image in tests[0].env.guest_image %} * {{guest_image}}{% endfor %} * VM loopback apps:{% for loopback_app in tests[0].env.loopback_app %} - * {{loopback_app['name']}}, Version: {{loopback_app['version']}}, GIT tag: {{loopback_app['git_tag']}}{% endfor %} + * {{loopback_app.get()['name']}}, Version: {{loopback_app.get()['version']}}, GIT tag: {{loopback_app.get()['git_tag']}}{% endfor %} {%- endif %} Below are test details: @@ -43,7 +43,7 @@ Below are test details: * Description: {{ "%s"|format(tests[0].conf['Description']) }} * Deployment: {{ "%s"|format(tests[0].deployment) }} * Traffic type: {{ "%s"|format(tests[0].result['type']) }} -* Bidirectional : {{ "%s"|format(tests[0].conf['biDirectional']) }} +* Bidirectional : {{ "%s"|format(tests[0].conf['bidir']) }} {%- if tests[0].result['tunnel_type'] %} * Tunnel type: {{ "%s"|format(tests[0].result['tunnel_type']) }} {%- endif %} diff --git a/tools/systeminfo.py b/tools/systeminfo.py index d515983d..575dd87e 100644 --- a/tools/systeminfo.py +++ b/tools/systeminfo.py @@ -23,6 +23,7 @@ import re import distro from conf import settings as S +from tools.version import Version def match_line(file_name, pattern): """ loops through given file and returns first line matching given pattern @@ -328,7 +329,7 @@ def get_version(app_name): app_version = 'NA' app_git_tag = 'NA' - return {'name' : app_name, 'version' : app_version, 'git_tag' : app_git_tag} + return Version(app_name, app_version, app_git_tag) def get_loopback_version(loopback_app_name): """ Get version of given guest loopback application and its git tag @@ -337,5 +338,5 @@ def get_loopback_version(loopback_app_name): version or git tag are not known or not applicaple, than None is returned for any unknown value """ version = get_version("loopback_{}".format(loopback_app_name)) - version['name'] = loopback_app_name + version.set_value('name', loopback_app_name) return version diff --git a/tools/version.py b/tools/version.py new file mode 100644 index 00000000..30ba3401 --- /dev/null +++ b/tools/version.py @@ -0,0 +1,34 @@ +# Copyright 2017 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Generic class to hold sotware version information shown in the report +""" + +class Version(object): + """Container to keep software version details + """ + def __init__(self, name, version, git_tag='NA'): + """Create Version object with given data + """ + self._version = {'name' : name, 'version' : version, 'git_tag' : git_tag} + + def set_value(self, key, value): + """Upate given `key` by given `value` + """ + self._version[key] = value + + def get(self): + """Get content of version object + """ + return self._version @@ -336,7 +336,7 @@ def generate_final_report(): if retval == 0 and os.path.isfile(test_report): _LOGGER.info('Overall test report written to "%s"', test_report) else: - _LOGGER.error('Generatrion of overall test report has failed.') + _LOGGER.error('Generation of overall test report has failed.') # remove temporary file os.remove(_TEMPLATE_RST['tmp']) diff --git a/vswitches/ovs.py b/vswitches/ovs.py index b6d64fb7..12620e22 100644 --- a/vswitches/ovs.py +++ b/vswitches/ovs.py @@ -95,6 +95,14 @@ class IVSwitchOvs(IVSwitch, tasks.Process): """ pass + # Method could be a function + # pylint: disable=no-self-use + def get_version(self): + """See IVswitch for general description + """ + # OVS version can be read offline + return [] + def stop(self): """See IVswitch for general description """ diff --git a/vswitches/vpp_dpdk_vhost.py b/vswitches/vpp_dpdk_vhost.py index d0d9e2ac..68375538 100644 --- a/vswitches/vpp_dpdk_vhost.py +++ b/vswitches/vpp_dpdk_vhost.py @@ -25,6 +25,7 @@ from src.dpdk import dpdk from conf import settings as S from vswitches.vswitch import IVSwitch from tools import tasks +from tools.version import Version # pylint: disable=too-many-public-methods class VppDpdkVhost(IVSwitch, tasks.Process): @@ -110,7 +111,6 @@ class VppDpdkVhost(IVSwitch, tasks.Process): self._logger.debug("VPP CLI args: %s", cli_args) return cli_args - def start(self): """Activates DPDK kernel modules and starts VPP @@ -156,6 +156,25 @@ class VppDpdkVhost(IVSwitch, tasks.Process): # has not been terminated yet tasks.Process.kill(self, signal, sleep) + def get_version(self): + """See IVswitch for general description + """ + versions = [] + output = self.run_vppctl(['show', 'version', 'verbose']) + if output[1]: + self._logger.warning("VPP version can not be read!") + return versions + + match = re.search(r'Version:\s*(.+)', output[0]) + if match: + versions.append(Version(S.getValue('VSWITCH'), match.group(1))) + + match = re.search(r'DPDK Version:\s*DPDK (.+)', output[0]) + if match: + versions.append(Version('dpdk', match.group(1))) + + return versions + def add_switch(self, switch_name, dummy_params=None): """See IVswitch for general description """ diff --git a/vswitches/vswitch.py b/vswitches/vswitch.py index 893bd1ff..dd69e6d9 100644 --- a/vswitches/vswitch.py +++ b/vswitches/vswitch.py @@ -20,6 +20,14 @@ class IVSwitch(object): Other methods are called only between start() and stop() """ + def get_version(self): + """Return version of vSwitch and DPDK (if used by vSwitch) + This method should be implemented in case, that version + of vswitch or DPDK can be read only during vSwitch runtime. + Otherwise it can be implemented inside tools/systeminfo.py. + """ + raise NotImplementedError() + def start(self): """Start the vSwitch |