summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMartin Klozik <martinx.klozik@intel.com>2017-09-01 15:36:35 +0100
committerMartin Klozik <martinx.klozik@intel.com>2017-09-01 16:00:27 +0100
commitbc150cc8b5a06a79b5c2308407718b2ba9a04a2b (patch)
treeba7d9deb4f4f67eb435c66972e41be884aa5d91b
parent7608b5f5b73af88bd22e1825a013cbcd123050ac (diff)
opnfvresultdb: Report failed TCs into result DB
In following cases, TC will be reported as FAIL in result DB: * CSV file with VSPERF results is missing (e.g. execution of vSwitch, VNF or trafficgen has failed) * 0 RX FPS is detected for at least one packet size during RFC2544 Throughput test (e.g. vSwitch is not properly configured due insufficient hugepages size) JIRA: VSPERF-488 Change-Id: I453bbe32db87497644e1464e74ef597f0f0f323c Signed-off-by: Martin Klozik <martinx.klozik@intel.com> Reviewed-by: Mars Toktonaliev <mars.toktonaliev@nokia.com> Reviewed-by: Morgan Richomme <morgan.richomme@orange.com> Reviewed-by: Al Morton <acmorton@att.com> Reviewed-by: Christian Trautman <ctrautma@redhat.com> Reviewed-by: Sridhar Rao <sridhar.rao@spirent.com> Reviewed-by: Trevor Cooper <trevor.cooper@intel.com>
-rw-r--r--tools/opnfvdashboard/opnfvdashboard.py77
-rwxr-xr-xvsperf10
2 files changed, 60 insertions, 27 deletions
diff --git a/tools/opnfvdashboard/opnfvdashboard.py b/tools/opnfvdashboard/opnfvdashboard.py
index cf2fd1b0..a77a9c3a 100644
--- a/tools/opnfvdashboard/opnfvdashboard.py
+++ b/tools/opnfvdashboard/opnfvdashboard.py
@@ -17,10 +17,14 @@ vsperf2dashboard
import os
import csv
+import copy
import logging
+from datetime import datetime as dt
import requests
-def results2opnfv_dashboard(results_path, int_data):
+_DETAILS = {"64": '', "128": '', "512": '', "1024": '', "1518": ''}
+
+def results2opnfv_dashboard(tc_names, results_path, int_data):
"""
the method open the csv file with results and calls json encoder
"""
@@ -31,23 +35,33 @@ def results2opnfv_dashboard(results_path, int_data):
resfile = results_path + '/' + test
with open(resfile, 'r') as in_file:
reader = csv.DictReader(in_file)
- _push_results(reader, int_data)
+ tc_data = _prepare_results(reader, int_data)
+ _push_results(tc_data)
+ tc_names.remove(tc_data['id'])
+
+ # report TCs without results as FAIL
+ if tc_names:
+ tc_fail = copy.deepcopy(int_data)
+ tc_fail['start_time'] = dt.now().strftime('%Y-%m-%d %H:%M:%S')
+ tc_fail['stop_time'] = tc_fail['start_time']
+ tc_fail['criteria'] = 'FAIL'
+ tc_fail['version'] = 'N/A'
+ tc_fail['details'] = copy.deepcopy(_DETAILS)
+ for tc_name in tc_names:
+ tc_fail['dashboard_id'] = "{}_{}".format(tc_name, tc_fail['vswitch'])
+ _push_results(tc_fail)
-def _push_results(reader, int_data):
+def _prepare_results(reader, int_data):
"""
- the method encodes results and sends them into opnfv dashboard
+ the method prepares dashboard details for passed testcases
"""
- db_url = int_data['db_url']
- url = db_url + "/results"
- casename = ""
version_vswitch = ""
version_dpdk = ""
- version = ""
allowed_pkt = ["64", "128", "512", "1024", "1518"]
- details = {"64": '', "128": '', "512": '', "1024": '', "1518": ''}
- test_start = None
- test_stop = None
vswitch = None
+ details = copy.deepcopy(_DETAILS)
+ tc_data = copy.deepcopy(int_data)
+ tc_data['criteria'] = 'PASS'
for row_reader in reader:
if allowed_pkt.count(row_reader['packet_size']) == 0:
@@ -56,20 +70,25 @@ def _push_results(reader, int_data):
# test execution time includes all frame sizes, so start & stop time
# is the same (repeated) for every framesize in CSV file
- if test_start is None:
- test_start = row_reader['start_time']
- test_stop = row_reader['stop_time']
+ if not 'test_start' in tc_data:
+ tc_data['start_time'] = row_reader['start_time']
+ tc_data['stop_time'] = row_reader['stop_time']
+ tc_data['id'] = row_reader['id']
# CI job executes/reports TCs per vswitch type
vswitch = row_reader['vswitch']
- casename = "{}_{}".format(row_reader['id'], row_reader['vswitch'].lower())
+ tc_data['dashboard_id'] = "{}_{}".format(row_reader['id'], row_reader['vswitch'].lower())
if "back2back" in row_reader['id']:
+ # 0 B2B frames is quite common, so we can't mark such TC as FAIL
details[row_reader['packet_size']] = row_reader['b2b_frames']
else:
details[row_reader['packet_size']] = row_reader['throughput_rx_fps']
+ # 0 PPS is definitelly a failure
+ if float(row_reader['throughput_rx_fps']) == 0:
+ tc_data['criteria'] = 'FAIL'
# Create version field
- with open(int_data['pkg_list'], 'r') as pkg_file:
+ with open(tc_data['pkg_list'], 'r') as pkg_file:
for line in pkg_file:
if "OVS_TAG" in line and vswitch.startswith('Ovs'):
version_vswitch = line.replace(' ', '')
@@ -83,23 +102,33 @@ def _push_results(reader, int_data):
version_dpdk = line.replace(' ', '')
version_dpdk = " DPDK {}".format(
version_dpdk.replace('DPDK_TAG?=', ''))
- version = version_vswitch.replace('\n', '') + version_dpdk.replace('\n', '')
+
+ tc_data['details'] = details
+ tc_data['version'] = version_vswitch.replace('\n', '') + version_dpdk.replace('\n', '')
+
+ return tc_data
+
+def _push_results(int_data):
+ """
+ the method sends testcase details into dashboard database
+ """
+ url = int_data['db_url'] + "/results"
# Build body
body = {"project_name": "vsperf",
"scenario": "vsperf",
- "start_date": test_start,
- "stop_date": test_stop,
- "case_name": casename,
+ "start_date": int_data['start_time'],
+ "stop_date": int_data['stop_time'],
+ "case_name": int_data['dashboard_id'],
"pod_name": int_data['pod'],
"installer": int_data['installer'],
- "version": version,
+ "version": int_data['version'],
"build_tag": int_data['build_tag'],
"criteria": int_data['criteria'],
- "details": details}
+ "details": int_data['details']}
my_data = requests.post(url, json=body)
- logging.info("Results for %s sent to opnfv, http response: %s", casename, my_data)
- logging.debug("opnfv url: %s", db_url)
+ logging.info("Results for %s sent to opnfv, http response: %s", int_data['dashboard_id'], my_data)
+ logging.debug("opnfv url: %s", int_data['db_url'])
logging.debug("the body sent to opnfv")
logging.debug(body)
diff --git a/vsperf b/vsperf
index 36c12df9..6efe53da 100755
--- a/vsperf
+++ b/vsperf
@@ -708,12 +708,16 @@ def main():
pkg_list = settings.getValue('PACKAGE_LIST')
int_data = {'pod': pod_name,
- 'criteria': "PASS",
'build_tag': get_build_tag(),
'installer': installer_name,
'pkg_list': pkg_list,
- 'db_url': opnfv_url}
- opnfvdashboard.results2opnfv_dashboard(results_path, int_data)
+ 'db_url': opnfv_url,
+ # pass vswitch name from configuration to be used for failed
+ # TCs; In case of successful TCs it is safer to use vswitch
+ # name from CSV as TC can override global configuration
+ 'vswitch': str(settings.getValue('VSWITCH')).lower()}
+ tc_names = [tc['Name'] for tc in selected_tests]
+ opnfvdashboard.results2opnfv_dashboard(tc_names, results_path, int_data)
# cleanup before exit
vsperf_finalize()