summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Klozik <martinx.klozik@intel.com>2016-01-25 10:34:27 +0000
committerMaryam Tahhan <maryam.tahhan@intel.com>2016-01-25 11:09:20 +0000
commitda081ab2603eb34baf7d7fe7b61f681ef15d865e (patch)
tree52144f7a175ed5f953a6b4356998884f1673ab11
parente840131eef8e493a1b0a4bd4ed2885c958b481f9 (diff)
report: create test report for all executed TCs
Every testcase generates report with its results into separate file. It is required to merge all partial reports into final overall report for whole set of TCs executed by VSPERF. Hugepages are mounted also for TestPMD packet forwarder. Change-Id: Iaa1ab4e08d1637106da804e06b7f30100d609cd6 JIRA: VSPERF-181 Signed-off-by: Martin Klozik <martinx.klozik@intel.com> Reviewed-by: Maryam Tahhan <maryam.tahhan@intel.com>
-rw-r--r--testcases/testcase.py4
-rw-r--r--tools/report/report_foot.rst29
-rw-r--r--tools/report/report_head.rst55
-rw-r--r--tools/report/report_rst.jinja123
-rwxr-xr-xvsperf35
5 files changed, 133 insertions, 113 deletions
diff --git a/testcases/testcase.py b/testcases/testcase.py
index 9c755ea7..d470d951 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -251,7 +251,9 @@ class TestCase(object):
"""
# hugepages are needed by DPDK and Qemu
if not self._hugepages_mounted and \
- (self.deployment.count('v') or S.getValue('VSWITCH').lower().count('dpdk')):
+ (self.deployment.count('v') or \
+ S.getValue('VSWITCH').lower().count('dpdk') or \
+ self._vswitch_none):
hugepages.mount_hugepages()
self._hugepages_mounted = True
diff --git a/tools/report/report_foot.rst b/tools/report/report_foot.rst
new file mode 100644
index 00000000..e1a55718
--- /dev/null
+++ b/tools/report/report_foot.rst
@@ -0,0 +1,29 @@
+.. _rationale:
+
+Rationale for decisions
+=======================
+TODO.
+
+.. _conclusions:
+
+Conclusions and recommendations
+===============================
+TODO.
+
+General
+========
+
+Glossary
+--------
+- NFV - Network Function Virtualization
+- Mbps - 1,000,000bps
+
+Document change procedures and history
+--------------------------------------
+=============================================== ================= =============
+ Document ID Author Date Modified
+=============================================== ================= =============
+`OPNFV_vswitchperf_LTR_ver_1.0_Jan_15_CN_DRAFT` Christopher Nolan 23/01/2015
+`OPNFV_vswitchperf_LTR_ver_1.1_Jan_15_CN_DRAFT` Christopher Nolan 28/01/2015
+=============================================== ================= =============
+
diff --git a/tools/report/report_head.rst b/tools/report/report_head.rst
new file mode 100644
index 00000000..d16614cd
--- /dev/null
+++ b/tools/report/report_head.rst
@@ -0,0 +1,55 @@
+==========================================================================
+CHARACTERIZE VSWITCH PERFORMANCE FOR TELCO NFV USE CASES LEVEL TEST REPORT
+==========================================================================
+
+Introduction
+============
+The objective of the OPNFV project titled **"Characterise vSwitch Performance
+for Telco NFV Use Cases"**, is to evaluate a virtual switch to identify its
+suitability for a Telco Network Function Virtualization (NFV) environment. As
+well as this, the project aims to identify any gaps or bottlenecks in order to
+drive architectural changes to improve virtual switch performance and
+determinism. The purpose of this document is to summarize the results of the
+tests carried out on the virtual switch in the Network Function Virtualization
+Infrastructure (NFVI) and, from these results, provide evaluations and
+recommendations for the virtual switch. Test results will be outlined in
+details-of-LTR_, preceded by the document-identifier_ and the scope_ and
+references_).
+
+This document is currently in draft form.
+
+.. _document-identifier:
+
+Document identifier
+-------------------
+The document id will be used to uniquely identify versions of the LTR. The
+format for the document id will be:
+OPNFV\_vswitchperf\_LTR\_rel\_STATUS, the status is one of: DRAFT, REVIEWED,
+CORRECTED or FINAL. The document id for this version of the LTR is:
+OPNFV\_vswitchperf\_LTR\_Brahmaputra\_DRAFT.
+
+.. _scope:
+
+Scope
+-----
+The scope of this report is to detail the results of the tests that have been
+performed on the virtual switch. This report will also evaluate the results of
+these tests and, based on these evaluations, provide recommendations on the
+suitability of the virtual switch for use in a Telco NFV environment.
+
+.. _references:
+
+References
+----------
+`OPNFV_vswitchperf_LTD_Brahmaputra_REVIEWED
+<http://artifacts.opnfv.org/vswitchperf/docs/requirements/vswitchperf_ltd.html>`__
+
+.. _details-of-LTR:
+
+Details of the Level Test Report
+================================
+This section provides a test-results-overview_. Also included are the rationale_
+and the conclusions_.
+
+.. _test-results-overview:
+
diff --git a/tools/report/report_rst.jinja b/tools/report/report_rst.jinja
index b3c2f2d0..1e90565f 100644
--- a/tools/report/report_rst.jinja
+++ b/tools/report/report_rst.jinja
@@ -1,60 +1,6 @@
-==========================================================================
-CHARACTERIZE VSWITCH PERFORMANCE FOR TELCO NFV USE CASES LEVEL TEST REPORT
-==========================================================================
+Test ID: {{ tests[0].ID }}
+--------------------------
-Introduction
-============
-The objective of the OPNFV project titled **"Characterise vSwitch Performance
-for Telco NFV Use Cases"**, is to evaluate a virtual switch to identify its
-suitability for a Telco Network Function Virtualization (NFV) environment. As
-well as this, the project aims to identify any gaps or bottlenecks in order to
-drive architectural changes to improve virtual switch performance and
-determinism. The purpose of this document is to summarize the results of the
-tests carried out on the virtual switch in the Network Function Virtualization
-Infrastructure (NFVI) and, from these results, provide evaluations and
-recommendations for the virtual switch. Test results will be outlined in
-details-of-LTR_, preceded by the document-identifier_ and the scope_ and
-references_).
-
-This document is currently in draft form.
-
-.. _document-identifier:
-
-Document identifier
--------------------
-The document id will be used to uniquely identify versions of the LTR. The
-format for the document id will be:
-OPNFV\_vswitchperf\_LTR\_rel\_STATUS, the status is one of: DRAFT, REVIEWED,
-CORRECTED or FINAL. The document id for this version of the LTR is:
-OPNFV\_vswitchperf\_LTR\_Brahmaputra\_DRAFT.
-
-.. _scope:
-
-Scope
------
-The scope of this report is to detail the results of the tests that have been
-performed on the virtual switch. This report will also evaluate the results of
-these tests and, based on these evaluations, provide recommendations on the
-suitability of the virtual switch for use in a Telco NFV environment.
-
-.. _references:
-
-References
-----------
-`OPNFV_vswitchperf_LTD_Brahmaputra_REVIEWED
-<http://artifacts.opnfv.org/vswitchperf/docs/requirements/vswitchperf_ltd.html>`__
-
-.. details-of-LTR:
-
-Details of the Level Test Report
-================================
-This section provides a test-results-overview_ as well as
-detailed-test-results_. Also included are the rationale_ and the conclusions_.
-
-.. _test-results-overview:
-
-Overview of test results
-------------------------
Test Environment
~~~~~~~~~~~~~~~~
Below is the environment that the test was performed in:
@@ -80,30 +26,21 @@ Below is the environment that the test was performed in:
* {{loopback_app['name']}}, Version: {{loopback_app['version']}}, GIT tag: {{loopback_app['git_tag']}}{% endfor %}
{%- endif %}
-For each test, a summary of the key test results is provided.
-{% for test in tests %}
-Test ID: {{ test.ID }}
-~~~~~~~~~~~~~~~~~~~~~~~
Below are test details:
-* Test ID: {{ "%s"|format(test.id) }}
-* Description: {{ "%s"|format(test.conf['Description']) }}
-* Deployment: {{ "%s"|format(test.deployment) }}
-* Traffic type: {{ "%s"|format(test.result['type']) }}
-* Packet size: {{ "%s"|format(test.result['packet_size']) }}
-* Bidirectional : {{ "%s"|format(test.conf['biDirectional']) }}
-{% endfor %}
-
-.. _detailed-test-results:
+* Test ID: {{ "%s"|format(tests[0].id) }}
+* Description: {{ "%s"|format(tests[0].conf['Description']) }}
+* Deployment: {{ "%s"|format(tests[0].deployment) }}
+* Traffic type: {{ "%s"|format(tests[0].result['type']) }}
+* Bidirectional : {{ "%s"|format(tests[0].conf['biDirectional']) }}
-Detailed test results
-======================
-A detailed summary of the main results for each test is outlined below.
{% for test in tests %}
-Test ID: {{ test.ID }}
-----------------------
+Test results for packet size: {{ "%s"|format(test.result['packet_size']) }}
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A detailed summary of the main results is outlined below.
+
Results/Metrics Collected
-~~~~~~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^^^^^^
The following are the metrics obtained during this test:
@@ -116,12 +53,10 @@ The following are the metrics obtained during this test:
========================== ==================================
Statistics collected
-~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^
The following system statistics were collected during testcase execution:
-
{% for process in test.stats %}
-
========================== ==================================
Process: {{ '_'.join(process.split('_')[:-1]) }}
-------------------------------------------------------------
@@ -132,8 +67,7 @@ Process: {{ '_'.join(process.split('_')[:-1]) }}
{%- endfor %}
========================== ==================================
-{%- endfor %}
-
+{% endfor %}{% endfor %}
Anomalies
~~~~~~~~~~
@@ -142,34 +76,5 @@ No anomalies were detected during the course of this test.
Testing Activities/Events
~~~~~~~~~~~~~~~~~~~~~~~~~
There were no significant testing activities for this test.
-{% endfor %}
-
-.. _rationale:
-
-Rationale for decisions
-=======================
-TODO.
-
-.. _conclusions:
-
-Conclusions and recommendations
-===============================
-TODO.
-
-General
-========
-
-Glossary
---------
-- NFV - Network Function Virtualization
-- Mbps - 1,000,000bps
-Document change procedures and history
---------------------------------------
-=============================================== ================= =============
- Document ID Author Date Modified
-=============================================== ================= =============
-`OPNFV_vswitchperf_LTR_ver_1.0_Jan_15_CN_DRAFT` Christopher Nolan 23/01/2015
-`OPNFV_vswitchperf_LTR_ver_1.1_Jan_15_CN_DRAFT` Christopher Nolan 28/01/2015
-=============================================== ================= =============
diff --git a/vsperf b/vsperf
index 93772326..4b1d86d0 100755
--- a/vsperf
+++ b/vsperf
@@ -28,6 +28,8 @@ import unittest
import xmlrunner
import locale
import copy
+import glob
+import subprocess
sys.dont_write_bytecode = True
@@ -39,7 +41,6 @@ from tools import tasks
from tools.pkt_gen import trafficgen
from tools.opnfvdashboard import opnfvdashboard
from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
-from conf import get_test_param
import core.component_factory as component_factory
VERBOSITY_LEVELS = {
@@ -50,6 +51,10 @@ VERBOSITY_LEVELS = {
'critical': logging.CRITICAL
}
+_TEMPLATE_RST = {'head' : 'tools/report/report_head.rst',
+ 'foot' : 'tools/report/report_foot.rst',
+ 'final' : 'test_report.rst'
+ }
def parse_arguments():
"""
@@ -262,6 +267,27 @@ def check_and_set_locale():
logging.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
system_locale, locale.getdefaultlocale())
+
+def generate_final_report(path):
+ """ Function will check if partial test results are available
+ and generates final report in rst format.
+ """
+
+ # check if there are any results in rst format
+ rst_results = glob.glob(os.path.join(path, 'result*rst'))
+ if len(rst_results):
+ try:
+ test_report = os.path.join(path, _TEMPLATE_RST['final'])
+ retval = subprocess.call('cat {} {} {} > {}'.format(_TEMPLATE_RST['head'], ' '.join(rst_results),
+ _TEMPLATE_RST['foot'], test_report), shell=True)
+ if retval == 0 and os.path.isfile(test_report):
+ logging.info('Overall test report written to "%s"', test_report)
+ else:
+ logging.error('Generatrion of overall test report has failed.')
+ except subprocess.CalledProcessError:
+ logging.error('Generatrion of overall test report has failed.')
+
+
class MockTestCase(unittest.TestCase):
"""Allow use of xmlrunner to generate Jenkins compatible output without
using xmlrunner to actually run tests.
@@ -313,7 +339,7 @@ def main():
# than both a settings file and environment variables
settings.load_from_dict(args)
- vswitch_none = False
+ vswitch_none = False
# set dpdk and ovs paths accorfing to VNF and VSWITCH
if settings.getValue('VSWITCH').endswith('Vanilla'):
# settings paths for Vanilla
@@ -351,7 +377,7 @@ def main():
# configure vswitch
if args['vswitch']:
- vswitch_none = 'none' == args['vswitch'].strip().lower()
+ vswitch_none = 'none' == args['vswitch'].strip().lower()
if vswitch_none:
settings.setValue('VSWITCH', 'none')
else:
@@ -499,6 +525,9 @@ def main():
suite.addTest(MockTestCase(str(ex), False, test.name))
logger.info("Continuing with next test...")
+ # generate final rst report with results of all executed TCs
+ generate_final_report(results_path)
+
if settings.getValue('XUNIT'):
xmlrunner.XMLTestRunner(
output=settings.getValue('XUNIT_DIR'), outsuffix="",