diff options
Diffstat (limited to 'utils/test/reporting')
| -rwxr-xr-x | utils/test/reporting/docker/reporting.sh | 20 | ||||
| -rwxr-xr-x | utils/test/reporting/functest/reporting-status.py | 6 | ||||
| -rw-r--r-- | utils/test/reporting/functest/template/index-status-tmpl.html | 8 | ||||
| -rw-r--r-- | utils/test/reporting/functest/testCase.py | 59 | ||||
| -rw-r--r-- | utils/test/reporting/js/trend-qtip.js | 76 | ||||
| -rw-r--r-- | utils/test/reporting/qtip/__init__.py | 0 | ||||
| -rw-r--r-- | utils/test/reporting/qtip/index.html | 51 | ||||
| -rw-r--r-- | utils/test/reporting/qtip/reporting-status.py | 110 | ||||
| -rw-r--r-- | utils/test/reporting/qtip/template/index-status-tmpl.html | 86 | ||||
| -rw-r--r-- | utils/test/reporting/reporting.yaml | 2 | ||||
| -rw-r--r-- | utils/test/reporting/storperf/template/index-status-tmpl.html | 1 | ||||
| -rw-r--r-- | utils/test/reporting/utils/reporting_utils.py | 37 | ||||
| -rw-r--r-- | utils/test/reporting/yardstick/template/index-status-tmpl.html | 8 |
13 files changed, 399 insertions, 65 deletions
diff --git a/utils/test/reporting/docker/reporting.sh b/utils/test/reporting/docker/reporting.sh index 1de13ae32..49f4517f7 100755 --- a/utils/test/reporting/docker/reporting.sh +++ b/utils/test/reporting/docker/reporting.sh @@ -3,7 +3,7 @@ export PYTHONPATH="${PYTHONPATH}:." export CONFIG_REPORTING_YAML=./reporting.yaml -declare -a versions=(colorado master) +declare -a versions=(danube master) declare -a projects=(functest storperf yardstick) project=$1 @@ -32,6 +32,22 @@ cp -Rf js display # yardstick | # storperf | +function report_project() +{ + project=$1 + dir=$2 + type=$3 + echo "********************************" + echo " $project reporting " + echo "********************************" + python ./$dir/reporting-$type.py + if [ $? ]; then + echo "$project reporting $type...OK" + else + echo "$project reporting $type...KO" + fi +} + if [ -z "$1" ]; then echo "********************************" echo " Functest reporting " @@ -60,6 +76,8 @@ if [ -z "$1" ]; then python ./storperf/reporting-status.py echo "Storperf reporting status...OK" + report_project "QTIP" "qtip" "status" + else if [ -z "$2" ]; then reporting_type="status" diff --git a/utils/test/reporting/functest/reporting-status.py b/utils/test/reporting/functest/reporting-status.py index 94e7f2f3e..e700e047f 100755 --- a/utils/test/reporting/functest/reporting-status.py +++ b/utils/test/reporting/functest/reporting-status.py @@ -163,14 +163,13 @@ for version in versions: test_case.isRunnable)) time.sleep(1) if test_case.isRunnable: - dbName = test_case.getDbName() name = test_case.getName() displayName = test_case.getDisplayName() project = test_case.getProject() nb_test_runnable_for_this_scenario += 1 logger.info(" Searching results for case %s " % (displayName)) - result = rp_utils.getResult(dbName, installer, + result = rp_utils.getResult(name, installer, s, version) # if no result set the value to 0 if result < 0: @@ -197,13 +196,12 @@ for version in versions: test_case.isRunnable)) time.sleep(1) if test_case.isRunnable: - dbName = test_case.getDbName() name = test_case.getName() displayName = test_case.getDisplayName() project = test_case.getProject() logger.info(" Searching results for case %s " % (displayName)) - result = rp_utils.getResult(dbName, installer, + result = rp_utils.getResult(name, installer, s, version) # at least 1 result for the test if result > -1: diff --git a/utils/test/reporting/functest/template/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html index ebacfd159..cc4edaac5 100644 --- a/utils/test/reporting/functest/template/index-status-tmpl.html +++ b/utils/test/reporting/functest/template/index-status-tmpl.html @@ -134,13 +134,13 @@ $(document).ready(function (){ <tr class="tr-weather-weather"> {% for test in items[scenario] -%} {% if test.getCriteria() > 2 -%} - <td><img src="../../../img/weather-clear.png"></td> + <td><img src="../../img/weather-clear.png"></td> {%- elif test.getCriteria() > 1 -%} - <td><img src="../../../img/weather-few-clouds.png"></td> + <td><img src="../../img/weather-few-clouds.png"></td> {%- elif test.getCriteria() > 0 -%} - <td><img src="../../../img/weather-overcast.png"></td> + <td><img src="../../img/weather-overcast.png"></td> {%- elif test.getCriteria() > -1 -%} - <td><img src="../../../img/weather-storm.png"></td> + <td><img src="../../img/weather-storm.png"></td> {%- endif %} {%- endfor %} </tr> diff --git a/utils/test/reporting/functest/testCase.py b/utils/test/reporting/functest/testCase.py index c89e619c0..9834f0753 100644 --- a/utils/test/reporting/functest/testCase.py +++ b/utils/test/reporting/functest/testCase.py @@ -33,27 +33,29 @@ class TestCase(object): 'bgpvpn': 'bgpvpn', 'rally_full': 'Rally (full)', 'vims': 'vIMS', - 'doctor': 'Doctor', + 'doctor-notification': 'Doctor', 'promise': 'Promise', 'moon': 'Moon', 'copper': 'Copper', 'security_scan': 'Security', 'multisite': 'Multisite', - 'domino': 'Domino', - 'odl-sfc': 'SFC', + 'domino-multinode': 'Domino', + 'functest-odl-sfc': 'SFC', 'onos_sfc': 'SFC', - 'parser': 'Parser', + 'parser-basics': 'Parser', 'connection_check': 'Health (connection)', 'api_check': 'Health (api)', 'snaps_smoke': 'SNAPS', 'snaps_health_check': 'Health (dhcp)', - 'netready': 'Netready', + 'gluon_vping': 'Netready', 'fds': 'FDS', 'cloudify_ims': 'vIMS (Cloudify)', 'orchestra_ims': 'OpenIMS (OpenBaton)', 'opera_ims': 'vIMS (Open-O)', 'vyos_vrouter': 'vyos', - 'barometer': 'Barometer'} + 'barometercollectd': 'Barometer', + 'odl_netvirt': 'Netvirt', + 'security_scan': 'Security'} try: self.displayName = display_name_matrix[self.name] except: @@ -119,50 +121,5 @@ class TestCase(object): ";IsRunnable" + str(self.isRunnable)) return testcase - def getDbName(self): - # Correspondance name of the test case / name in the DB - # ideally we should modify the DB to avoid such interface.... - # '<name in the config>':'<name in the DB>' - # I know it is uggly... - test_match_matrix = {'healthcheck': 'healthcheck', - 'vping_ssh': 'vping_ssh', - 'vping_userdata': 'vping_userdata', - 'odl': 'odl', - 'onos': 'onos', - 'ocl': 'ocl', - 'tempest_smoke_serial': 'tempest_smoke_serial', - 'tempest_full_parallel': 'tempest_full_parallel', - 'tempest_defcore': 'tempest_defcore', - 'refstack_defcore': 'refstack_defcore', - 'rally_sanity': 'rally_sanity', - 'bgpvpn': 'bgpvpn', - 'rally_full': 'rally_full', - 'vims': 'vims', - 'doctor': 'doctor-notification', - 'promise': 'promise', - 'moon': 'moon_authentication', - 'copper': 'copper-notification', - 'security_scan': 'security', - 'multisite': 'multisite', - 'domino': 'domino-multinode', - 'odl-sfc': 'functest-odl-sfc', - 'onos_sfc': 'onos_sfc', - 'parser': 'parser-basics', - 'connection_check': 'connection_check', - 'api_check': 'api_check', - 'snaps_smoke': 'snaps_smoke', - 'snaps_health_check': 'snaps_health_check', - 'netready': 'gluon_vping', - 'fds': 'fds', - 'cloudify_ims': 'cloudify_ims', - 'orchestra_ims': 'orchestra_ims', - 'opera_ims': 'opera_ims', - 'vyos_vrouter': 'vyos_vrouter', - 'barometer': 'barometercollectd'} - try: - return test_match_matrix[self.name] - except: - return "unknown" - def getDisplayName(self): return self.displayName diff --git a/utils/test/reporting/js/trend-qtip.js b/utils/test/reporting/js/trend-qtip.js new file mode 100644 index 000000000..d4c8735d9 --- /dev/null +++ b/utils/test/reporting/js/trend-qtip.js @@ -0,0 +1,76 @@ +// ****************************************** +// Trend line for reporting +// based on scenario_history.txt +// where data looks like +// date,scenario,installer,detail,score +// 2016-09-22 13:12,os-nosdn-fdio-noha,apex,4/12,33.0 +// 2016-09-22 13:13,os-odl_l2-fdio-noha,apex,12/15,80.0 +// 2016-09-22 13:13,os-odl_l2-sfc-noha,apex,18/24,75.0 +// ..... +// ****************************************** +// Set the dimensions of the canvas / graph +var trend_margin = {top: 20, right: 30, bottom: 50, left: 40}, + trend_width = 300 - trend_margin.left - trend_margin.right, + trend_height = 130 - trend_margin.top - trend_margin.bottom; + +// Parse the date / time +var parseDate = d3.time.format("%Y-%m-%d %H:%M").parse; + +// Set the ranges +var trend_x = d3.time.scale().range([0, trend_width]); +var trend_y = d3.scale.linear().range([trend_height, 0]); + +// Define the axes +var trend_xAxis = d3.svg.axis().scale(trend_x) + .orient("bottom").ticks(2).tickFormat(d3.time.format("%m-%d")); + +var trend_yAxis = d3.svg.axis().scale(trend_y) + .orient("left").ticks(4, "s"); + +// Define the line +var valueline = d3.svg.line() + .x(function(d) { return trend_x(d.date); }) + .y(function(d) { return trend_y(d.score); }); + +var trend = function(container, trend_data) { + + var trend_svg = d3.select(container) + .append("svg") + .attr("width", trend_width + trend_margin.left + trend_margin.right) + .attr("height", trend_height + trend_margin.top + trend_margin.bottom) + .attr("style", "font-size: small") + .append("g") + .attr("transform", + "translate(" + trend_margin.left + "," + trend_margin.top + ")"); + + // Scale the range of the data + trend_x.domain(d3.extent(trend_data, function(d) { return d.date; })); + trend_y.domain([0, d3.max(trend_data, function(d) { return d.score; })]); + + // Add the X Axis + trend_svg.append("g") + .attr("class", "x axis") + .attr("transform", "translate(0," + trend_height + ")") + .call(trend_xAxis); + + // Add the Y Axis + trend_svg.append("g") + .attr("class", "y axis") + .call(trend_yAxis); + + // Add the valueline path. + trend_svg.append("path") + .attr("class", "line") + .attr("d", valueline(trend_data)) + .attr("stroke", "steelblue") + .attr("fill", "none"); + + trend_svg.selectAll(".dot") + .data(trend_data) + .enter().append("circle") + .attr("r", 2.5) + .attr("cx", function(d) { return trend_x(d.date); }) + .attr("cy", function(d) { return trend_y(d.score); }); + + return trend; +} diff --git a/utils/test/reporting/qtip/__init__.py b/utils/test/reporting/qtip/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/utils/test/reporting/qtip/__init__.py diff --git a/utils/test/reporting/qtip/index.html b/utils/test/reporting/qtip/index.html new file mode 100644 index 000000000..0f9df8564 --- /dev/null +++ b/utils/test/reporting/qtip/index.html @@ -0,0 +1,51 @@ + <html> + <head> + <meta charset="utf-8"> + <!-- Bootstrap core CSS --> + <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css" rel="stylesheet"> + <link href="default.css" rel="stylesheet"> + <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1/jquery.min.js"></script> + <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> + <script type="text/javascript"> + $(document).ready(function (){ + $(".btn-more").click(function() { + $(this).hide(); + $(this).parent().find(".panel-default").show(); + }); + }) + </script> + </head> + <body> + <div class="container"> + <div class="masthead"> + <h3 class="text-muted">QTIP reporting page</h3> + <nav> + <ul class="nav nav-justified"> + <li class="active"><a href="http://testresults.opnfv.org/reporting/index.html">Home</a></li> + <li><a href="index-status-apex.html">Apex</a></li> + <li><a href="index-status-compass.html">Compass</a></li> + <li><a href="index-status-fuel.html">Fuel</a></li> + <li><a href="index-status-joid.html">Joid</a></li> + </ul> + </nav> + </div> +<div class="row"> + <div class="col-md-1"></div> + <div class="col-md-10"> + <div class="page-main"> + <h2>QTIP</h2> + QTIP is used in OPNFV for verifying the OPNFV infrastructure and some of the OPNFV features. + <br>The QTIP framework is deployed in several OPNFV community labs. + <br>It is installer, infrastructure and application independent. + + <h2>Useful Links</h2> + <li><a href="https://wiki.opnfv.org/download/attachments/5734608/qtip%20in%20depth.pdf?version=1&modificationDate=1463410431000&api=v2">QTIP in Depth</a></li> + <li><a href="https://git.opnfv.org/cgit/qtip">QTIP Repo</a></li> + <li><a href="https://wiki.opnfv.org/display/qtip">QTIP Project</a></li> + <li><a href="https://build.opnfv.org/ci/view/qtip/">QTIP Jenkins page</a></li> + <li><a href="https://jira.opnfv.org/browse/QTIP-119?jql=project%20%3D%20QTIP">JIRA</a></li> + + </div> + </div> + <div class="col-md-1"></div> +</div> diff --git a/utils/test/reporting/qtip/reporting-status.py b/utils/test/reporting/qtip/reporting-status.py new file mode 100644 index 000000000..5967cf6b9 --- /dev/null +++ b/utils/test/reporting/qtip/reporting-status.py @@ -0,0 +1,110 @@ +#!/usr/bin/python +# +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +import datetime +import os + +import jinja2 +import utils.reporting_utils as rp_utils +import utils.scenarioResult as sr + +installers = rp_utils.get_config('general.installers') +versions = rp_utils.get_config('general.versions') +PERIOD = rp_utils.get_config('general.period') + +# Logger +logger = rp_utils.getLogger("Qtip-Status") +reportingDate = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") + +logger.info("*******************************************") +logger.info("* Generating reporting scenario status *") +logger.info("* Data retention = %s days *" % PERIOD) +logger.info("* *") +logger.info("*******************************************") + + +def prepare_profile_file(version): + profile_dir = './display/{}/qtip'.format(version) + if not os.path.exists(profile_dir): + os.makedirs(profile_dir) + + profile_file = '{}/scenario_history.txt'.format(profile_dir, version) + if not os.path.exists(profile_file): + with open(profile_file, 'w') as f: + info = 'date,scenario,installer,details,score\n' + f.write(info) + f.close() + return profile_file + + +def profile_results(results, installer, profile_fd): + result_criterias = {} + for s_p, s_p_result in results.iteritems(): + ten_criteria = len(s_p_result) + ten_score = sum(s_p_result) + + LASTEST_TESTS = rp_utils.get_config( + 'general.nb_iteration_tests_success_criteria') + four_result = s_p_result[:LASTEST_TESTS] + four_criteria = len(four_result) + four_score = sum(four_result) + + s_four_score = str(four_score / four_criteria) + s_ten_score = str(ten_score / ten_criteria) + + info = '{},{},{},{},{}\n'.format(reportingDate, + s_p, + installer, + s_ten_score, + s_four_score) + profile_fd.write(info) + result_criterias[s_p] = sr.ScenarioResult('OK', + s_four_score, + s_ten_score, + '100') + + logger.info("--------------------------") + return result_criterias + + +def render_html(prof_results, installer, version): + template_loader = jinja2.FileSystemLoader(".") + template_env = jinja2.Environment(loader=template_loader, + autoescape=True) + + template_file = "./qtip/template/index-status-tmpl.html" + template = template_env.get_template(template_file) + + render_outcome = template.render(prof_results=prof_results, + installer=installer, + period=PERIOD, + version=version, + date=reportingDate) + + with open('./display/{}/qtip/status-{}.html'.format(version, installer), + 'wb') as fh: + fh.write(render_outcome) + + +def render_reporter(): + for version in versions: + profile_file = prepare_profile_file(version) + profile_fd = open(profile_file, 'a') + for installer in installers: + results = rp_utils.getQtipResults(version, installer) + prof_results = profile_results(results, installer, profile_fd) + render_html(prof_results=prof_results, + installer=installer, + version=version) + profile_fd.close() + logger.info("Manage export CSV") + rp_utils.generate_csv(profile_file) + logger.info("CSV generated...") + +if __name__ == '__main__': + render_reporter() diff --git a/utils/test/reporting/qtip/template/index-status-tmpl.html b/utils/test/reporting/qtip/template/index-status-tmpl.html new file mode 100644 index 000000000..26da36ceb --- /dev/null +++ b/utils/test/reporting/qtip/template/index-status-tmpl.html @@ -0,0 +1,86 @@ + <html> + <head> + <meta charset="utf-8"> + <!-- Bootstrap core CSS --> + <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css" rel="stylesheet"> + <link href="../../css/default.css" rel="stylesheet"> + <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1/jquery.min.js"></script> + <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script> + <script type="text/javascript" src="http://d3js.org/d3.v2.min.js"></script> + <script type="text/javascript" src="../../js/trend-qtip.js"></script> + <script> + // trend line management + d3.csv("./scenario_history.csv", function(data) { + // *************************************** + // Create the trend line + {% for scenario in prof_results.keys() -%} + // for scenario {{scenario}} + // Filter results + var trend{{loop.index}} = data.filter(function(row) { + return row["scenario"]=="{{scenario}}" && row["installer"]=="{{installer}}"; + }) + // Parse the date + trend{{loop.index}}.forEach(function(d) { + d.date = parseDate(d.date); + d.score = +d.score + }); + // Draw the trend line + var mytrend = trend("#trend_svg{{loop.index}}",trend{{loop.index}}) + // **************************************** + {%- endfor %} + }); + </script> + <script type="text/javascript"> + $(document).ready(function (){ + $(".btn-more").click(function() { + $(this).hide(); + $(this).parent().find(".panel-default").show(); + }); + }) + </script> + </head> + <body> + <div class="container"> + <div class="masthead"> + <h3 class="text-muted">QTIP status page ({{version}}, {{date}})</h3> + <nav> + <ul class="nav nav-justified"> + <li class="active"><a href="http://testresults.opnfv.org/reporting/index.html">Home</a></li> + <li><a href="index-status-apex.html">Apex</a></li> + <li><a href="index-status-compass.html">Compass</a></li> + <li><a href="index-status-fuel.html">Fuel</a></li> + <li><a href="index-status-joid.html">Joid</a></li> + </ul> + </nav> + </div> +<div class="row"> + <div class="col-md-1"></div> + <div class="col-md-10"> + <div class="page-header"> + <h2>{{installer}}</h2> + </div> + + <div class="scenario-overview"> + <div class="panel-heading"><h4><b>List of last scenarios ({{version}}) run over the last {{period}} days </b></h4></div> + <table class="table"> + <tr> + <th width="25%">Pod/Scenario</th> + <th width="25%">Trend</th> + <th width="25%">Last 4 Iterations</th> + <th width="25%">Last 10 Days</th> + </tr> + {% for scenario,result in prof_results.iteritems() -%} + <tr class="tr-ok"> + <td>{{scenario}}</td> + <td><div id="trend_svg{{loop.index}}"></div></td> + <td>{{prof_results[scenario].getFourDaysScore()}}</td> + <td>{{prof_results[scenario].getTenDaysScore()}}</td> + </tr> + {%- endfor %} + </table> + </div> + + + </div> + <div class="col-md-1"></div> +</div> diff --git a/utils/test/reporting/reporting.yaml b/utils/test/reporting/reporting.yaml index 73781eb56..1692f481d 100644 --- a/utils/test/reporting/reporting.yaml +++ b/utils/test/reporting/reporting.yaml @@ -60,6 +60,8 @@ storperf: log_level: ERROR qtip: + log_level: ERROR + period: 1 bottleneck: diff --git a/utils/test/reporting/storperf/template/index-status-tmpl.html b/utils/test/reporting/storperf/template/index-status-tmpl.html index e0fcc6828..e872272c3 100644 --- a/utils/test/reporting/storperf/template/index-status-tmpl.html +++ b/utils/test/reporting/storperf/template/index-status-tmpl.html @@ -70,7 +70,6 @@ <li class="active"><a href="http://testresults.opnfv.org/reporting/index.html">Home</a></li> <li><a href="status-apex.html">Apex</a></li> <li><a href="status-compass.html">Compass</a></li> - <li><a href="status-daisy.html">Daisy</a></li> <li><a href="status-fuel.html">Fuel</a></li> <li><a href="status-joid.html">Joid</a></li> </ul> diff --git a/utils/test/reporting/utils/reporting_utils.py b/utils/test/reporting/utils/reporting_utils.py index 98ade2c30..599a93818 100644 --- a/utils/test/reporting/utils/reporting_utils.py +++ b/utils/test/reporting/utils/reporting_utils.py @@ -199,6 +199,35 @@ def getScenarioStatus(installer, version): return result_dict +def getQtipResults(version, installer): + period = get_config('qtip.period') + url_base = get_config('testapi.url') + + url = ("http://" + url_base + "?project=qtip" + + "&installer=" + installer + + "&version=" + version + "&period=" + str(period)) + request = Request(url) + + try: + response = urlopen(request) + k = response.read() + response.close() + results = json.loads(k)['results'] + except URLError as e: + print('Got an error code:', e) + + result_dict = {} + if results: + for r in results: + key = '{}/{}'.format(r['pod_name'], r['scenario']) + if key not in result_dict.keys(): + result_dict[key] = [] + result_dict[key].append(r['details']['score']) + + # return scenario_results + return result_dict + + def getNbtestOk(results): nb_test_ok = 0 for r in results: @@ -406,6 +435,14 @@ def export_csv(scenario_file_name, installer, version): scenario_installer_file.close +def generate_csv(scenario_file): + import shutil + # csv + # generate sub files based on scenario_history.txt + csv_file = scenario_file.replace('txt', 'csv') + shutil.copy2(scenario_file, csv_file) + + def export_pdf(pdf_path, pdf_doc_name): try: pdfkit.from_file(pdf_path, pdf_doc_name) diff --git a/utils/test/reporting/yardstick/template/index-status-tmpl.html b/utils/test/reporting/yardstick/template/index-status-tmpl.html index b6f237a1a..77ba9502f 100644 --- a/utils/test/reporting/yardstick/template/index-status-tmpl.html +++ b/utils/test/reporting/yardstick/template/index-status-tmpl.html @@ -68,10 +68,10 @@ <nav> <ul class="nav nav-justified"> <li class="active"><a href="http://testresults.opnfv.org/reporting/index.html">Home</a></li> - <li><a href="index-status-apex.html">Apex</a></li> - <li><a href="index-status-compass.html">Compass</a></li> - <li><a href="index-status-fuel.html">Fuel</a></li> - <li><a href="index-status-joid.html">Joid</a></li> + <li><a href="status-apex.html">Apex</a></li> + <li><a href="status-compass.html">Compass</a></li> + <li><a href="status-fuel.html">Fuel</a></li> + <li><a href="status-joid.html">Joid</a></li> </ul> </nav> </div> |