summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMorgan Richomme <morgan.richomme@orange.com>2016-03-09 17:21:05 +0100
committerMorgan Richomme <morgan.richomme@orange.com>2016-03-09 17:21:05 +0100
commite17429d4c54a5c559429c0f5ffcd7da90fa2ebc7 (patch)
tree274148f1c575553bbd451a8db81b0c92c60c760f
parent41679c5e444c7479b3fbad4b9e6749623c907275 (diff)
Add automated reporting for Tempest
Change-Id: I3cc411c2d0d22a96333bf0b978ad8dd833744e8c Signed-off-by: Morgan Richomme <morgan.richomme@orange.com>
Diffstat
-rw-r--r--utils/test/result_collection_api/tools/reporting/index-tempest-tmpl.html90
-rw-r--r--utils/test/result_collection_api/tools/reporting/reporting-tempest.py98
2 files changed, 188 insertions, 0 deletions
diff --git a/utils/test/result_collection_api/tools/reporting/index-tempest-tmpl.html b/utils/test/result_collection_api/tools/reporting/index-tempest-tmpl.html
new file mode 100644
index 000000000..24d87bee2
--- /dev/null
+++ b/utils/test/result_collection_api/tools/reporting/index-tempest-tmpl.html
@@ -0,0 +1,90 @@
+ <html>
+ <head>
+ <meta charset="utf-8">
+ <!-- Bootstrap core CSS -->
+ <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css" rel="stylesheet">
+ <link href="default.css" rel="stylesheet">
+ <script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1/jquery.min.js"></script>
+ <script type="text/javascript" src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
+ <script type="text/javascript">
+ $(document).ready(function (){
+ $(".btn-more").click(function() {
+ $(this).hide();
+ $(this).parent().find(".panel-default").show();
+ });
+ })
+ </script>
+ </head>
+ <body>
+ <div class="container">
+ <div class="masthead">
+ <h3 class="text-muted">Tempest status page</h3>
+ <nav>
+ <ul class="nav nav-justified">
+ <li class="active"><a href="#">Home</a></li>
+ <li><a href="index-tempest-apex.html">Apex</a></li>
+ <li><a href="index-tempest-compass.html">Compass</a></li>
+ <li><a href="index-tempest-fuel.html">Fuel</a></li>
+ <li><a href="index-tempest-joid.html">Joid</a></li>
+ </ul>
+ </nav>
+ </div>
+<div class="row">
+ <div class="col-md-1"></div>
+ <div class="col-md-10">
+ <div class="page-header">
+ <h2>{{installer}}</h2>
+ </div>
+ {% for scenario_name, results in scenario_results.iteritems() -%}
+ <div class="scenario-part">
+ <div class="page-header">
+ <h3><span class="glyphicon glyphicon-chevron-right"> <b>{{scenario_name}}</b></h3>
+ </div>
+ {% for result in results -%}
+ {% if loop.index > 2 -%}
+ <div class="panel panel-default" hidden>
+ {%- else -%}
+ <div class="panel panel-default">
+ {%- endif %}
+ <div class="panel-heading">
+ <div class="progress-bar" role="progressbar" aria-valuenow="{{result.pr_step_ok}}" aria-valuemin="0" aria-valuemax="100" style="width: {{result.pr_step_ok}}%"></div>
+ <span class="panel-header-item">
+ <h4><b>{{result.creation_date}}</b></h4>
+ </span>
+ <span class="badge panel-pod-name">{{result.pod_name}}</span>
+ </div>
+ <table class="table">
+ <tr>
+ <th width="20%">Item</th>
+ <th width="10%">Result</th>
+ <th width="10%">Status</th>
+ <th width="60%">Errors</th>
+ </tr>
+ {% for item in items -%}
+ {% if item in result.details.keys() -%}
+ {% if result.criteria[item] -%}
+ <tr class="tr-ok">
+ <td>{{item}}</td>
+ <td>{{result.details[item]}}</td>
+ <td><span class="glyphicon glyphicon-ok"></td>
+ <td>{{result.errors[item]}}</td>
+ </tr>
+ {%- else -%}
+ <tr class="tr-danger">
+ <td>{{item}}</td>
+ <td>{{result.details[item]}}</td>
+ <td><span class="glyphicon glyphicon-remove"></td>
+ <td>{{result.errors[item]}}</td>
+ </tr>
+ {%- endif %}
+ {%- endif %}
+ {%- endfor %}
+ </table>
+ </div>
+ {%- endfor %}
+ <button type="button" class="btn btn-more">More than two</button>
+ </div>
+ {%- endfor %}
+ </div>
+ <div class="col-md-1"></div>
+</div>
diff --git a/utils/test/result_collection_api/tools/reporting/reporting-tempest.py b/utils/test/result_collection_api/tools/reporting/reporting-tempest.py
new file mode 100644
index 000000000..6d90f863c
--- /dev/null
+++ b/utils/test/result_collection_api/tools/reporting/reporting-tempest.py
@@ -0,0 +1,98 @@
+from urllib2 import Request, urlopen, URLError
+import json
+import jinja2
+import os
+
+installers = ["apex", "compass", "fuel", "joid"]
+items = ["tests", "Success rate", "duration"]
+
+for installer in installers:
+ # we consider the Tempest results of the last 7 days
+ url = "http://testresults.opnfv.org/testapi/results?case=Tempest"
+ request = Request(url + '&period=7&installer=' + installer)
+
+ try:
+ response = urlopen(request)
+ k = response.read()
+ results = json.loads(k)
+ except URLError, e:
+ print 'No kittez. Got an error code:', e
+
+ test_results = results['test_results']
+ test_results.reverse()
+
+ scenario_results = {}
+ criteria = {}
+ errors = {}
+
+ for r in test_results:
+ # Retrieve all the scenarios per installer
+ if not r['version'] in scenario_results.keys():
+ scenario_results[r['version']] = []
+ scenario_results[r['version']].append(r)
+
+ for s, s_result in scenario_results.items():
+ scenario_results[s] = s_result[0:5]
+ # For each scenario, we build a result object to deal with
+ # results, criteria and error handling
+ for result in scenario_results[s]:
+ result["creation_date"] = result["creation_date"].split(".")[0]
+
+ # retrieve results
+ # ****************
+ nb_tests_run = result['details']['tests']
+ if nb_tests_run != 0:
+ success_rate = 100*(int(result['details']['tests']) - int(result['details']['failures']))/int(result['details']['tests'])
+ else:
+ success_rate = 0
+
+ result['details']["tests"] = nb_tests_run
+ result['details']["Success rate"] = str(success_rate) + "%"
+
+ # Criteria management
+ # *******************
+ crit_tests = False
+ crit_rate = False
+ crit_time = False
+
+ # Expect that at least 200 tests are run
+ if nb_tests_run > 200:
+ crit_tests = True
+
+ # Expect that at least 90% of success
+ if success_rate > 90:
+ crit_rate = True
+
+ if result['details']['duration'] < 900:
+ crit_time = True
+
+ result['criteria'] = {'tests': crit_tests,
+ 'Success rate': crit_rate,
+ 'duration': crit_time}
+
+ # error management
+ # ****************
+
+ # TODO get information from artefact based on build tag
+ # to identify errors of the associated run
+ # build tag needed to wget errors on the artifacts
+ # the idea is to list the tests in errors and provide the link
+ # towards complete artifact
+ # another option will be to put the errors in the DB
+ # (in the detail section)...
+ result['errors'] = {'tests': "",
+ 'Success rate': "",
+ 'duration': ""}
+
+ templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
+ templateEnv = jinja2.Environment(loader=templateLoader)
+
+ TEMPLATE_FILE = "index-tempest-tmpl.html"
+ template = templateEnv.get_template(TEMPLATE_FILE)
+
+ outputText = template.render(scenario_results=scenario_results,
+ items=items,
+ installer=installer)
+
+ with open("index-tempest-" + installer + ".html", "wb") as fh:
+ fh.write(outputText)