summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--testcases/testcase.py4
-rw-r--r--tools/report/report_foot.rst29
-rw-r--r--tools/report/report_head.rst55
-rw-r--r--tools/report/report_rst.jinja123
-rwxr-xr-xvsperf35
5 files changed, 133 insertions, 113 deletions
diff --git a/testcases/testcase.py b/testcases/testcase.py
index 9c755ea7..d470d951 100644
--- a/testcases/testcase.py
+++ b/testcases/testcase.py
@@ -251,7 +251,9 @@ class TestCase(object):
"""
# hugepages are needed by DPDK and Qemu
if not self._hugepages_mounted and \
- (self.deployment.count('v') or S.getValue('VSWITCH').lower().count('dpdk')):
+ (self.deployment.count('v') or \
+ S.getValue('VSWITCH').lower().count('dpdk') or \
+ self._vswitch_none):
hugepages.mount_hugepages()
self._hugepages_mounted = True
diff --git a/tools/report/report_foot.rst b/tools/report/report_foot.rst
new file mode 100644
index 00000000..e1a55718
--- /dev/null
+++ b/tools/report/report_foot.rst
@@ -0,0 +1,29 @@
+.. _rationale:
+
+Rationale for decisions
+=======================
+TODO.
+
+.. _conclusions:
+
+Conclusions and recommendations
+===============================
+TODO.
+
+General
+========
+
+Glossary
+--------
+- NFV - Network Function Virtualization
+- Mbps - 1,000,000bps
+
+Document change procedures and history
+--------------------------------------
+=============================================== ================= =============
+ Document ID Author Date Modified
+=============================================== ================= =============
+`OPNFV_vswitchperf_LTR_ver_1.0_Jan_15_CN_DRAFT` Christopher Nolan 23/01/2015
+`OPNFV_vswitchperf_LTR_ver_1.1_Jan_15_CN_DRAFT` Christopher Nolan 28/01/2015
+=============================================== ================= =============
+
diff --git a/tools/report/report_head.rst b/tools/report/report_head.rst
new file mode 100644
index 00000000..d16614cd
--- /dev/null
+++ b/tools/report/report_head.rst
@@ -0,0 +1,55 @@
+==========================================================================
+CHARACTERIZE VSWITCH PERFORMANCE FOR TELCO NFV USE CASES LEVEL TEST REPORT
+==========================================================================
+
+Introduction
+============
+The objective of the OPNFV project titled **"Characterise vSwitch Performance
+for Telco NFV Use Cases"**, is to evaluate a virtual switch to identify its
+suitability for a Telco Network Function Virtualization (NFV) environment. As
+well as this, the project aims to identify any gaps or bottlenecks in order to
+drive architectural changes to improve virtual switch performance and
+determinism. The purpose of this document is to summarize the results of the
+tests carried out on the virtual switch in the Network Function Virtualization
+Infrastructure (NFVI) and, from these results, provide evaluations and
+recommendations for the virtual switch. Test results will be outlined in
+details-of-LTR_, preceded by the document-identifier_ and the scope_ and
+references_).
+
+This document is currently in draft form.
+
+.. _document-identifier:
+
+Document identifier
+-------------------
+The document id will be used to uniquely identify versions of the LTR. The
+format for the document id will be:
+OPNFV\_vswitchperf\_LTR\_rel\_STATUS, the status is one of: DRAFT, REVIEWED,
+CORRECTED or FINAL. The document id for this version of the LTR is:
+OPNFV\_vswitchperf\_LTR\_Brahmaputra\_DRAFT.
+
+.. _scope:
+
+Scope
+-----
+The scope of this report is to detail the results of the tests that have been
+performed on the virtual switch. This report will also evaluate the results of
+these tests and, based on these evaluations, provide recommendations on the
+suitability of the virtual switch for use in a Telco NFV environment.
+
+.. _references:
+
+References
+----------
+`OPNFV_vswitchperf_LTD_Brahmaputra_REVIEWED
+<http://artifacts.opnfv.org/vswitchperf/docs/requirements/vswitchperf_ltd.html>`__
+
+.. _details-of-LTR:
+
+Details of the Level Test Report
+================================
+This section provides a test-results-overview_. Also included are the rationale_
+and the conclusions_.
+
+.. _test-results-overview:
+
diff --git a/tools/report/report_rst.jinja b/tools/report/report_rst.jinja
index b3c2f2d0..1e90565f 100644
--- a/tools/report/report_rst.jinja
+++ b/tools/report/report_rst.jinja
@@ -1,60 +1,6 @@
-==========================================================================
-CHARACTERIZE VSWITCH PERFORMANCE FOR TELCO NFV USE CASES LEVEL TEST REPORT
-==========================================================================
+Test ID: {{ tests[0].ID }}
+--------------------------
-Introduction
-============
-The objective of the OPNFV project titled **"Characterise vSwitch Performance
-for Telco NFV Use Cases"**, is to evaluate a virtual switch to identify its
-suitability for a Telco Network Function Virtualization (NFV) environment. As
-well as this, the project aims to identify any gaps or bottlenecks in order to
-drive architectural changes to improve virtual switch performance and
-determinism. The purpose of this document is to summarize the results of the
-tests carried out on the virtual switch in the Network Function Virtualization
-Infrastructure (NFVI) and, from these results, provide evaluations and
-recommendations for the virtual switch. Test results will be outlined in
-details-of-LTR_, preceded by the document-identifier_ and the scope_ and
-references_).
-
-This document is currently in draft form.
-
-.. _document-identifier:
-
-Document identifier
--------------------
-The document id will be used to uniquely identify versions of the LTR. The
-format for the document id will be:
-OPNFV\_vswitchperf\_LTR\_rel\_STATUS, the status is one of: DRAFT, REVIEWED,
-CORRECTED or FINAL. The document id for this version of the LTR is:
-OPNFV\_vswitchperf\_LTR\_Brahmaputra\_DRAFT.
-
-.. _scope:
-
-Scope
------
-The scope of this report is to detail the results of the tests that have been
-performed on the virtual switch. This report will also evaluate the results of
-these tests and, based on these evaluations, provide recommendations on the
-suitability of the virtual switch for use in a Telco NFV environment.
-
-.. _references:
-
-References
-----------
-`OPNFV_vswitchperf_LTD_Brahmaputra_REVIEWED
-<http://artifacts.opnfv.org/vswitchperf/docs/requirements/vswitchperf_ltd.html>`__
-
-.. details-of-LTR:
-
-Details of the Level Test Report
-================================
-This section provides a test-results-overview_ as well as
-detailed-test-results_. Also included are the rationale_ and the conclusions_.
-
-.. _test-results-overview:
-
-Overview of test results
-------------------------
Test Environment
~~~~~~~~~~~~~~~~
Below is the environment that the test was performed in:
@@ -80,30 +26,21 @@ Below is the environment that the test was performed in:
* {{loopback_app['name']}}, Version: {{loopback_app['version']}}, GIT tag: {{loopback_app['git_tag']}}{% endfor %}
{%- endif %}
-For each test, a summary of the key test results is provided.
-{% for test in tests %}
-Test ID: {{ test.ID }}
-~~~~~~~~~~~~~~~~~~~~~~~
Below are test details:
-* Test ID: {{ "%s"|format(test.id) }}
-* Description: {{ "%s"|format(test.conf['Description']) }}
-* Deployment: {{ "%s"|format(test.deployment) }}
-* Traffic type: {{ "%s"|format(test.result['type']) }}
-* Packet size: {{ "%s"|format(test.result['packet_size']) }}
-* Bidirectional : {{ "%s"|format(test.conf['biDirectional']) }}
-{% endfor %}
-
-.. _detailed-test-results:
+* Test ID: {{ "%s"|format(tests[0].id) }}
+* Description: {{ "%s"|format(tests[0].conf['Description']) }}
+* Deployment: {{ "%s"|format(tests[0].deployment) }}
+* Traffic type: {{ "%s"|format(tests[0].result['type']) }}
+* Bidirectional : {{ "%s"|format(tests[0].conf['biDirectional']) }}
-Detailed test results
-======================
-A detailed summary of the main results for each test is outlined below.
{% for test in tests %}
-Test ID: {{ test.ID }}
-----------------------
+Test results for packet size: {{ "%s"|format(test.result['packet_size']) }}
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+A detailed summary of the main results is outlined below.
+
Results/Metrics Collected
-~~~~~~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^^^^^^
The following are the metrics obtained during this test:
@@ -116,12 +53,10 @@ The following are the metrics obtained during this test:
========================== ==================================
Statistics collected
-~~~~~~~~~~~~~~~~~~~~
+^^^^^^^^^^^^^^^^^^^^
The following system statistics were collected during testcase execution:
-
{% for process in test.stats %}
-
========================== ==================================
Process: {{ '_'.join(process.split('_')[:-1]) }}
-------------------------------------------------------------
@@ -132,8 +67,7 @@ Process: {{ '_'.join(process.split('_')[:-1]) }}
{%- endfor %}
========================== ==================================
-{%- endfor %}
-
+{% endfor %}{% endfor %}
Anomalies
~~~~~~~~~~
@@ -142,34 +76,5 @@ No anomalies were detected during the course of this test.
Testing Activities/Events
~~~~~~~~~~~~~~~~~~~~~~~~~
There were no significant testing activities for this test.
-{% endfor %}
-
-.. _rationale:
-
-Rationale for decisions
-=======================
-TODO.
-
-.. _conclusions:
-
-Conclusions and recommendations
-===============================
-TODO.
-
-General
-========
-
-Glossary
---------
-- NFV - Network Function Virtualization
-- Mbps - 1,000,000bps
-Document change procedures and history
---------------------------------------
-=============================================== ================= =============
- Document ID Author Date Modified
-=============================================== ================= =============
-`OPNFV_vswitchperf_LTR_ver_1.0_Jan_15_CN_DRAFT` Christopher Nolan 23/01/2015
-`OPNFV_vswitchperf_LTR_ver_1.1_Jan_15_CN_DRAFT` Christopher Nolan 28/01/2015
-=============================================== ================= =============
diff --git a/vsperf b/vsperf
index 93772326..4b1d86d0 100755
--- a/vsperf
+++ b/vsperf
@@ -28,6 +28,8 @@ import unittest
import xmlrunner
import locale
import copy
+import glob
+import subprocess
sys.dont_write_bytecode = True
@@ -39,7 +41,6 @@ from tools import tasks
from tools.pkt_gen import trafficgen
from tools.opnfvdashboard import opnfvdashboard
from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS
-from conf import get_test_param
import core.component_factory as component_factory
VERBOSITY_LEVELS = {
@@ -50,6 +51,10 @@ VERBOSITY_LEVELS = {
'critical': logging.CRITICAL
}
+_TEMPLATE_RST = {'head' : 'tools/report/report_head.rst',
+ 'foot' : 'tools/report/report_foot.rst',
+ 'final' : 'test_report.rst'
+ }
def parse_arguments():
"""
@@ -262,6 +267,27 @@ def check_and_set_locale():
logging.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s",
system_locale, locale.getdefaultlocale())
+
+def generate_final_report(path):
+ """ Function will check if partial test results are available
+ and generates final report in rst format.
+ """
+
+ # check if there are any results in rst format
+ rst_results = glob.glob(os.path.join(path, 'result*rst'))
+ if len(rst_results):
+ try:
+ test_report = os.path.join(path, _TEMPLATE_RST['final'])
+ retval = subprocess.call('cat {} {} {} > {}'.format(_TEMPLATE_RST['head'], ' '.join(rst_results),
+ _TEMPLATE_RST['foot'], test_report), shell=True)
+ if retval == 0 and os.path.isfile(test_report):
+ logging.info('Overall test report written to "%s"', test_report)
+ else:
+ logging.error('Generatrion of overall test report has failed.')
+ except subprocess.CalledProcessError:
+ logging.error('Generatrion of overall test report has failed.')
+
+
class MockTestCase(unittest.TestCase):
"""Allow use of xmlrunner to generate Jenkins compatible output without
using xmlrunner to actually run tests.
@@ -313,7 +339,7 @@ def main():
# than both a settings file and environment variables
settings.load_from_dict(args)
- vswitch_none = False
+ vswitch_none = False
# set dpdk and ovs paths accorfing to VNF and VSWITCH
if settings.getValue('VSWITCH').endswith('Vanilla'):
# settings paths for Vanilla
@@ -351,7 +377,7 @@ def main():
# configure vswitch
if args['vswitch']:
- vswitch_none = 'none' == args['vswitch'].strip().lower()
+ vswitch_none = 'none' == args['vswitch'].strip().lower()
if vswitch_none:
settings.setValue('VSWITCH', 'none')
else:
@@ -499,6 +525,9 @@ def main():
suite.addTest(MockTestCase(str(ex), False, test.name))
logger.info("Continuing with next test...")
+ # generate final rst report with results of all executed TCs
+ generate_final_report(results_path)
+
if settings.getValue('XUNIT'):
xmlrunner.XMLTestRunner(
output=settings.getValue('XUNIT_DIR'), outsuffix="",