diff options
| -rw-r--r-- | testcases/testcase.py | 4 | ||||
| -rw-r--r-- | tools/report/report_foot.rst | 29 | ||||
| -rw-r--r-- | tools/report/report_head.rst | 55 | ||||
| -rw-r--r-- | tools/report/report_rst.jinja | 123 | ||||
| -rwxr-xr-x | vsperf | 35 |
5 files changed, 133 insertions, 113 deletions
diff --git a/testcases/testcase.py b/testcases/testcase.py index 9c755ea7..d470d951 100644 --- a/testcases/testcase.py +++ b/testcases/testcase.py @@ -251,7 +251,9 @@ class TestCase(object): """ # hugepages are needed by DPDK and Qemu if not self._hugepages_mounted and \ - (self.deployment.count('v') or S.getValue('VSWITCH').lower().count('dpdk')): + (self.deployment.count('v') or \ + S.getValue('VSWITCH').lower().count('dpdk') or \ + self._vswitch_none): hugepages.mount_hugepages() self._hugepages_mounted = True diff --git a/tools/report/report_foot.rst b/tools/report/report_foot.rst new file mode 100644 index 00000000..e1a55718 --- /dev/null +++ b/tools/report/report_foot.rst @@ -0,0 +1,29 @@ +.. _rationale: + +Rationale for decisions +======================= +TODO. + +.. _conclusions: + +Conclusions and recommendations +=============================== +TODO. + +General +======== + +Glossary +-------- +- NFV - Network Function Virtualization +- Mbps - 1,000,000bps + +Document change procedures and history +-------------------------------------- +=============================================== ================= ============= + Document ID Author Date Modified +=============================================== ================= ============= +`OPNFV_vswitchperf_LTR_ver_1.0_Jan_15_CN_DRAFT` Christopher Nolan 23/01/2015 +`OPNFV_vswitchperf_LTR_ver_1.1_Jan_15_CN_DRAFT` Christopher Nolan 28/01/2015 +=============================================== ================= ============= + diff --git a/tools/report/report_head.rst b/tools/report/report_head.rst new file mode 100644 index 00000000..d16614cd --- /dev/null +++ b/tools/report/report_head.rst @@ -0,0 +1,55 @@ +========================================================================== +CHARACTERIZE VSWITCH PERFORMANCE FOR TELCO NFV USE CASES LEVEL TEST REPORT +========================================================================== + +Introduction +============ +The objective of the OPNFV project titled **"Characterise vSwitch Performance +for Telco NFV Use Cases"**, is to evaluate a virtual switch to identify its +suitability for a Telco Network Function Virtualization (NFV) environment. As +well as this, the project aims to identify any gaps or bottlenecks in order to +drive architectural changes to improve virtual switch performance and +determinism. The purpose of this document is to summarize the results of the +tests carried out on the virtual switch in the Network Function Virtualization +Infrastructure (NFVI) and, from these results, provide evaluations and +recommendations for the virtual switch. Test results will be outlined in +details-of-LTR_, preceded by the document-identifier_ and the scope_ and +references_). + +This document is currently in draft form. + +.. _document-identifier: + +Document identifier +------------------- +The document id will be used to uniquely identify versions of the LTR. The +format for the document id will be: +OPNFV\_vswitchperf\_LTR\_rel\_STATUS, the status is one of: DRAFT, REVIEWED, +CORRECTED or FINAL. The document id for this version of the LTR is: +OPNFV\_vswitchperf\_LTR\_Brahmaputra\_DRAFT. + +.. _scope: + +Scope +----- +The scope of this report is to detail the results of the tests that have been +performed on the virtual switch. This report will also evaluate the results of +these tests and, based on these evaluations, provide recommendations on the +suitability of the virtual switch for use in a Telco NFV environment. + +.. _references: + +References +---------- +`OPNFV_vswitchperf_LTD_Brahmaputra_REVIEWED +<http://artifacts.opnfv.org/vswitchperf/docs/requirements/vswitchperf_ltd.html>`__ + +.. _details-of-LTR: + +Details of the Level Test Report +================================ +This section provides a test-results-overview_. Also included are the rationale_ +and the conclusions_. + +.. _test-results-overview: + diff --git a/tools/report/report_rst.jinja b/tools/report/report_rst.jinja index b3c2f2d0..1e90565f 100644 --- a/tools/report/report_rst.jinja +++ b/tools/report/report_rst.jinja @@ -1,60 +1,6 @@ -========================================================================== -CHARACTERIZE VSWITCH PERFORMANCE FOR TELCO NFV USE CASES LEVEL TEST REPORT -========================================================================== +Test ID: {{ tests[0].ID }} +-------------------------- -Introduction -============ -The objective of the OPNFV project titled **"Characterise vSwitch Performance -for Telco NFV Use Cases"**, is to evaluate a virtual switch to identify its -suitability for a Telco Network Function Virtualization (NFV) environment. As -well as this, the project aims to identify any gaps or bottlenecks in order to -drive architectural changes to improve virtual switch performance and -determinism. The purpose of this document is to summarize the results of the -tests carried out on the virtual switch in the Network Function Virtualization -Infrastructure (NFVI) and, from these results, provide evaluations and -recommendations for the virtual switch. Test results will be outlined in -details-of-LTR_, preceded by the document-identifier_ and the scope_ and -references_). - -This document is currently in draft form. - -.. _document-identifier: - -Document identifier -------------------- -The document id will be used to uniquely identify versions of the LTR. The -format for the document id will be: -OPNFV\_vswitchperf\_LTR\_rel\_STATUS, the status is one of: DRAFT, REVIEWED, -CORRECTED or FINAL. The document id for this version of the LTR is: -OPNFV\_vswitchperf\_LTR\_Brahmaputra\_DRAFT. - -.. _scope: - -Scope ------ -The scope of this report is to detail the results of the tests that have been -performed on the virtual switch. This report will also evaluate the results of -these tests and, based on these evaluations, provide recommendations on the -suitability of the virtual switch for use in a Telco NFV environment. - -.. _references: - -References ----------- -`OPNFV_vswitchperf_LTD_Brahmaputra_REVIEWED -<http://artifacts.opnfv.org/vswitchperf/docs/requirements/vswitchperf_ltd.html>`__ - -.. details-of-LTR: - -Details of the Level Test Report -================================ -This section provides a test-results-overview_ as well as -detailed-test-results_. Also included are the rationale_ and the conclusions_. - -.. _test-results-overview: - -Overview of test results ------------------------- Test Environment ~~~~~~~~~~~~~~~~ Below is the environment that the test was performed in: @@ -80,30 +26,21 @@ Below is the environment that the test was performed in: * {{loopback_app['name']}}, Version: {{loopback_app['version']}}, GIT tag: {{loopback_app['git_tag']}}{% endfor %} {%- endif %} -For each test, a summary of the key test results is provided. -{% for test in tests %} -Test ID: {{ test.ID }} -~~~~~~~~~~~~~~~~~~~~~~~ Below are test details: -* Test ID: {{ "%s"|format(test.id) }} -* Description: {{ "%s"|format(test.conf['Description']) }} -* Deployment: {{ "%s"|format(test.deployment) }} -* Traffic type: {{ "%s"|format(test.result['type']) }} -* Packet size: {{ "%s"|format(test.result['packet_size']) }} -* Bidirectional : {{ "%s"|format(test.conf['biDirectional']) }} -{% endfor %} - -.. _detailed-test-results: +* Test ID: {{ "%s"|format(tests[0].id) }} +* Description: {{ "%s"|format(tests[0].conf['Description']) }} +* Deployment: {{ "%s"|format(tests[0].deployment) }} +* Traffic type: {{ "%s"|format(tests[0].result['type']) }} +* Bidirectional : {{ "%s"|format(tests[0].conf['biDirectional']) }} -Detailed test results -====================== -A detailed summary of the main results for each test is outlined below. {% for test in tests %} -Test ID: {{ test.ID }} ----------------------- +Test results for packet size: {{ "%s"|format(test.result['packet_size']) }} +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +A detailed summary of the main results is outlined below. + Results/Metrics Collected -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ The following are the metrics obtained during this test: @@ -116,12 +53,10 @@ The following are the metrics obtained during this test: ========================== ================================== Statistics collected -~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^ The following system statistics were collected during testcase execution: - {% for process in test.stats %} - ========================== ================================== Process: {{ '_'.join(process.split('_')[:-1]) }} ------------------------------------------------------------- @@ -132,8 +67,7 @@ Process: {{ '_'.join(process.split('_')[:-1]) }} {%- endfor %} ========================== ================================== -{%- endfor %} - +{% endfor %}{% endfor %} Anomalies ~~~~~~~~~~ @@ -142,34 +76,5 @@ No anomalies were detected during the course of this test. Testing Activities/Events ~~~~~~~~~~~~~~~~~~~~~~~~~ There were no significant testing activities for this test. -{% endfor %} - -.. _rationale: - -Rationale for decisions -======================= -TODO. - -.. _conclusions: - -Conclusions and recommendations -=============================== -TODO. - -General -======== - -Glossary --------- -- NFV - Network Function Virtualization -- Mbps - 1,000,000bps -Document change procedures and history --------------------------------------- -=============================================== ================= ============= - Document ID Author Date Modified -=============================================== ================= ============= -`OPNFV_vswitchperf_LTR_ver_1.0_Jan_15_CN_DRAFT` Christopher Nolan 23/01/2015 -`OPNFV_vswitchperf_LTR_ver_1.1_Jan_15_CN_DRAFT` Christopher Nolan 28/01/2015 -=============================================== ================= ============= @@ -28,6 +28,8 @@ import unittest import xmlrunner import locale import copy +import glob +import subprocess sys.dont_write_bytecode = True @@ -39,7 +41,6 @@ from tools import tasks from tools.pkt_gen import trafficgen from tools.opnfvdashboard import opnfvdashboard from tools.pkt_gen.trafficgen.trafficgenhelper import TRAFFIC_DEFAULTS -from conf import get_test_param import core.component_factory as component_factory VERBOSITY_LEVELS = { @@ -50,6 +51,10 @@ VERBOSITY_LEVELS = { 'critical': logging.CRITICAL } +_TEMPLATE_RST = {'head' : 'tools/report/report_head.rst', + 'foot' : 'tools/report/report_foot.rst', + 'final' : 'test_report.rst' + } def parse_arguments(): """ @@ -262,6 +267,27 @@ def check_and_set_locale(): logging.warning("Locale was not properly configured. Default values were set. Old locale: %s, New locale: %s", system_locale, locale.getdefaultlocale()) + +def generate_final_report(path): + """ Function will check if partial test results are available + and generates final report in rst format. + """ + + # check if there are any results in rst format + rst_results = glob.glob(os.path.join(path, 'result*rst')) + if len(rst_results): + try: + test_report = os.path.join(path, _TEMPLATE_RST['final']) + retval = subprocess.call('cat {} {} {} > {}'.format(_TEMPLATE_RST['head'], ' '.join(rst_results), + _TEMPLATE_RST['foot'], test_report), shell=True) + if retval == 0 and os.path.isfile(test_report): + logging.info('Overall test report written to "%s"', test_report) + else: + logging.error('Generatrion of overall test report has failed.') + except subprocess.CalledProcessError: + logging.error('Generatrion of overall test report has failed.') + + class MockTestCase(unittest.TestCase): """Allow use of xmlrunner to generate Jenkins compatible output without using xmlrunner to actually run tests. @@ -313,7 +339,7 @@ def main(): # than both a settings file and environment variables settings.load_from_dict(args) - vswitch_none = False + vswitch_none = False # set dpdk and ovs paths accorfing to VNF and VSWITCH if settings.getValue('VSWITCH').endswith('Vanilla'): # settings paths for Vanilla @@ -351,7 +377,7 @@ def main(): # configure vswitch if args['vswitch']: - vswitch_none = 'none' == args['vswitch'].strip().lower() + vswitch_none = 'none' == args['vswitch'].strip().lower() if vswitch_none: settings.setValue('VSWITCH', 'none') else: @@ -499,6 +525,9 @@ def main(): suite.addTest(MockTestCase(str(ex), False, test.name)) logger.info("Continuing with next test...") + # generate final rst report with results of all executed TCs + generate_final_report(results_path) + if settings.getValue('XUNIT'): xmlrunner.XMLTestRunner( output=settings.getValue('XUNIT_DIR'), outsuffix="", |