summaryrefslogtreecommitdiffstats
path: root/utils/test/reporting
diff options
context:
space:
mode:
Diffstat (limited to 'utils/test/reporting')
-rw-r--r--utils/test/reporting/functest/default.css (renamed from utils/test/reporting/default.css)0
-rw-r--r--utils/test/reporting/functest/img/icon-nok.pngbin0 -> 2317 bytes
-rw-r--r--utils/test/reporting/functest/img/icon-ok.pngbin0 -> 4063 bytes
-rw-r--r--utils/test/reporting/functest/img/weather-clear.png (renamed from utils/test/reporting/img/weather-clear.png)bin1560 -> 1560 bytes
-rw-r--r--utils/test/reporting/functest/img/weather-few-clouds.png (renamed from utils/test/reporting/img/weather-few-clouds.png)bin1927 -> 1927 bytes
-rw-r--r--utils/test/reporting/functest/img/weather-overcast.png (renamed from utils/test/reporting/img/weather-overcast.png)bin1588 -> 1588 bytes
-rw-r--r--utils/test/reporting/functest/img/weather-storm.png (renamed from utils/test/reporting/img/weather-storm.png)bin2137 -> 2137 bytes
-rw-r--r--utils/test/reporting/functest/index.html (renamed from utils/test/reporting/index.html)0
-rw-r--r--utils/test/reporting/functest/reporting-status.py (renamed from utils/test/reporting/reporting-status.py)211
-rw-r--r--utils/test/reporting/functest/reporting-tempest.py (renamed from utils/test/reporting/reporting-tempest.py)4
-rw-r--r--utils/test/reporting/functest/reporting-vims.py (renamed from utils/test/reporting/reporting-vims.py)4
-rw-r--r--utils/test/reporting/functest/template/index-status-tmpl.html (renamed from utils/test/reporting/index-status-tmpl.html)25
-rw-r--r--utils/test/reporting/functest/template/index-tempest-tmpl.html (renamed from utils/test/reporting/index-tempest-tmpl.html)0
-rw-r--r--utils/test/reporting/functest/template/index-vims-tmpl.html (renamed from utils/test/reporting/index-vims-tmpl.html)0
14 files changed, 154 insertions, 90 deletions
diff --git a/utils/test/reporting/default.css b/utils/test/reporting/functest/default.css
index 0e330e965..0e330e965 100644
--- a/utils/test/reporting/default.css
+++ b/utils/test/reporting/functest/default.css
diff --git a/utils/test/reporting/functest/img/icon-nok.png b/utils/test/reporting/functest/img/icon-nok.png
new file mode 100644
index 000000000..526b5294b
--- /dev/null
+++ b/utils/test/reporting/functest/img/icon-nok.png
Binary files differ
diff --git a/utils/test/reporting/functest/img/icon-ok.png b/utils/test/reporting/functest/img/icon-ok.png
new file mode 100644
index 000000000..3a9de2e89
--- /dev/null
+++ b/utils/test/reporting/functest/img/icon-ok.png
Binary files differ
diff --git a/utils/test/reporting/img/weather-clear.png b/utils/test/reporting/functest/img/weather-clear.png
index a0d967750..a0d967750 100644
--- a/utils/test/reporting/img/weather-clear.png
+++ b/utils/test/reporting/functest/img/weather-clear.png
Binary files differ
diff --git a/utils/test/reporting/img/weather-few-clouds.png b/utils/test/reporting/functest/img/weather-few-clouds.png
index acfa78398..acfa78398 100644
--- a/utils/test/reporting/img/weather-few-clouds.png
+++ b/utils/test/reporting/functest/img/weather-few-clouds.png
Binary files differ
diff --git a/utils/test/reporting/img/weather-overcast.png b/utils/test/reporting/functest/img/weather-overcast.png
index 4296246d0..4296246d0 100644
--- a/utils/test/reporting/img/weather-overcast.png
+++ b/utils/test/reporting/functest/img/weather-overcast.png
Binary files differ
diff --git a/utils/test/reporting/img/weather-storm.png b/utils/test/reporting/functest/img/weather-storm.png
index 956f0e20f..956f0e20f 100644
--- a/utils/test/reporting/img/weather-storm.png
+++ b/utils/test/reporting/functest/img/weather-storm.png
Binary files differ
diff --git a/utils/test/reporting/index.html b/utils/test/reporting/functest/index.html
index af4033567..af4033567 100644
--- a/utils/test/reporting/index.html
+++ b/utils/test/reporting/functest/index.html
diff --git a/utils/test/reporting/reporting-status.py b/utils/test/reporting/functest/reporting-status.py
index e15bac9f7..9271717bb 100644
--- a/utils/test/reporting/reporting-status.py
+++ b/utils/test/reporting/functest/reporting-status.py
@@ -1,9 +1,11 @@
from urllib2 import Request, urlopen, URLError
+import datetime
import json
import jinja2
import os
import re
import requests
+import sys
import time
import yaml
@@ -12,18 +14,21 @@ functest_test_list = ['vPing', 'vPing_userdata',
'Tempest', 'Rally',
'ODL', 'ONOS', 'vIMS']
# functest_test_list = ['vPing']
-# functest_test_list = []
companion_test_list = ['doctor/doctor-notification', 'promise/promise']
# companion_test_list = []
installers = ["apex", "compass", "fuel", "joid"]
-# installers = ["apex"]
+# installers = ["fuel"]
+versions = ["brahmaputra", "master"]
+# versions = ["master"]
PERIOD = 10
+MAX_SCENARIO_CRITERIA = 18
# Correspondance between the name of the test case and the name in the DB
# ideally we should modify the DB to avoid such interface....
# '<name in the DB':'<name in the config'>
# I know it is uggly...
-test_match_matrix = {'vPing': 'vping_ssh',
+test_match_matrix = {'healthcheck': 'healthcheck',
+ 'vPing': 'vping_ssh',
'vPing_userdata': 'vping_userdata',
'ODL': 'odl',
'ONOS': 'onos',
@@ -100,7 +105,21 @@ class TestCase(object):
self.isRunnable = is_runnable
-def getApiResults(case, installer, scenario):
+class ScenarioResult(object):
+ def __init__(self, status, score=0):
+ self.status = status
+ self.score = score
+
+ def getStatus(self):
+ return self.status
+
+ def getScore(self):
+ return self.score
+
+# *****************************************************************************
+
+
+def getApiResults(case, installer, scenario, version):
case = case.getName()
results = json.dumps([])
# to remove proxy (to be removed at the end for local test only)
@@ -109,9 +128,9 @@ def getApiResults(case, installer, scenario):
# urllib2.install_opener(opener)
# url = "http://127.0.0.1:8000/results?case=" + case + \
# "&period=30&installer=" + installer
- url = "http://testresults.opnfv.org/testapi/results?case=" + case + \
- "&period=" + str(PERIOD) + "&installer=" + installer + \
- "&scenario=" + scenario
+ url = ("http://testresults.opnfv.org/testapi/results?case=" + case +
+ "&period=" + str(PERIOD) + "&installer=" + installer +
+ "&scenario=" + scenario + "&version=" + version)
request = Request(url)
try:
@@ -124,11 +143,12 @@ def getApiResults(case, installer, scenario):
return results
-def getScenarios(case, installer):
+def getScenarios(case, installer, version):
case = case.getName()
url = "http://testresults.opnfv.org/testapi/results?case=" + case + \
- "&period=" + str(PERIOD) + "&installer=" + installer
+ "&period=" + str(PERIOD) + "&installer=" + installer + \
+ "&version=" + version
request = Request(url)
try:
@@ -147,9 +167,9 @@ def getScenarios(case, installer):
for r in test_results:
# Retrieve all the scenarios per installer
- if not r['version'] in scenario_results.keys():
- scenario_results[r['version']] = []
- scenario_results[r['version']].append(r)
+ if not r['scenario'] in scenario_results.keys():
+ scenario_results[r['scenario']] = []
+ scenario_results[r['scenario']].append(r)
return scenario_results
@@ -174,10 +194,10 @@ def getNbtestOk(results):
return nb_test_ok
-def getResult(testCase, installer, scenario):
+def getResult(testCase, installer, scenario, version):
# retrieve raw results
- results = getApiResults(testCase, installer, scenario)
+ results = getApiResults(testCase, installer, scenario, version)
# let's concentrate on test results only
test_results = results['test_results']
@@ -230,77 +250,114 @@ def getResult(testCase, installer, scenario):
# ******************************************************************************
# ******************************************************************************
-# as the criteria are all difference, we shall use a common way to indicate
-# the criteria
-# 100 = 100% = all the test must be OK
-# 90 = 90% = all the test must be above 90% of success rate
-# TODO harmonize success criteria
-# some criteria could be the duration, the success rate, the packet loss,...
-# to be done case by case
-# TODo create TestCriteria Object
-
-
# init just tempest to get the list of scenarios
# as all the scenarios run Tempest
tempest = TestCase("Tempest", "functest", -1)
# Retrieve the Functest configuration to detect which tests are relevant
# according to the installer, scenario
-response = requests.get('https://git.opnfv.org/cgit/functest/plain/testcases/config_functest.yaml')
+cf = "https://git.opnfv.org/cgit/functest/plain/ci/config_functest.yaml"
+response = requests.get(cf)
functest_yaml_config = yaml.load(response.text)
print "****************************************"
print "* Generating reporting..... *"
+print ("* Data retention = %s days *" % PERIOD)
+print "* *"
print "****************************************"
-# For all the installers
-for installer in installers:
- # get scenarios
- scenario_results = getScenarios(tempest, installer)
- scenario_stats = getScenarioStats(scenario_results)
-
- items = {}
- # For all the scenarios get results
- for s, s_result in scenario_results.items():
- testCases = []
- # For each scenario declare the test cases
- # Functest cases
- for test_case in functest_test_list:
- testCases.append(TestCase(test_case, "functest"))
-
- # project/case
- for test_case in companion_test_list:
- test_split = test_case.split("/")
- test_project = test_split[0]
- test_case = test_split[1]
- testCases.append(TestCase(test_case, test_project))
-
- # Check if test case is runnable according to the installer, scenario
- for test_case in testCases:
- test_case.checkRunnable(installer, s, functest_yaml_config)
- # print "testcase %s is %s" % (test_case.getName(),
- # test_case.isRunnable)
-
- print "--------------------------"
- print "%s / %s:" % (installer, s)
- for testCase in testCases:
- time.sleep(1)
- if testCase.isRunnable:
- print " Searching results for case %s " % testCase.getName()
- result = getResult(testCase, installer, s)
- testCase.setCriteria(result)
- items[s] = testCases
- print "--------------------------"
- print "****************************************"
- templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
- templateEnv = jinja2.Environment(loader=templateLoader)
-
- TEMPLATE_FILE = "index-status-tmpl.html"
- template = templateEnv.get_template(TEMPLATE_FILE)
-
- outputText = template.render(scenario_stats=scenario_stats,
- items=items,
- installer=installer,
- period=PERIOD)
-
- with open("index-status-" + installer + ".html", "wb") as fh:
- fh.write(outputText)
+
+# For all the versions
+for version in versions:
+ # For all the installers
+ for installer in installers:
+ # get scenarios
+ scenario_results = getScenarios(tempest, installer, version)
+ scenario_stats = getScenarioStats(scenario_results)
+ items = {}
+ scenario_result_criteria = {}
+
+ # For all the scenarios get results
+ for s, s_result in scenario_results.items():
+ testCases = []
+ # Green or Red light for a given scenario
+ nb_test_runnable_for_this_scenario = 0
+ scenario_score = 0
+
+ # For each scenario declare the test cases
+ # Functest cases
+ for test_case in functest_test_list:
+ testCases.append(TestCase(test_case, "functest"))
+
+ # project/case
+ for test_case in companion_test_list:
+ test_split = test_case.split("/")
+ test_project = test_split[0]
+ test_case = test_split[1]
+ testCases.append(TestCase(test_case, test_project))
+
+ # Check if test case is runnable / installer, scenario
+ try:
+ for test_case in testCases:
+ test_case.checkRunnable(installer, s, functest_yaml_config)
+ # print "testcase %s is %s" % (test_case.getName(),
+ # test_case.isRunnable)
+ print ("installer %s, version %s, scenario %s:" %
+ (installer, version, s))
+ for testCase in testCases:
+ time.sleep(1)
+ if testCase.isRunnable:
+ nb_test_runnable_for_this_scenario += 1
+ print (" Searching results for case %s " %
+ (testCase.getName()))
+ result = getResult(testCase, installer, s, version)
+ testCase.setCriteria(result)
+ items[s] = testCases
+ scenario_score = scenario_score + result
+ except:
+ print ("installer %s, version %s, scenario %s" %
+ (installer, version, s))
+ print "No data available , error %s " % (sys.exc_info()[0])
+
+ # the validation criteria = nb runnable tests x 3
+ scenario_criteria = nb_test_runnable_for_this_scenario * 3
+ # if 0 runnable tests set criteria at a high value
+ if scenario_criteria < 1:
+ scenario_criteria = MAX_SCENARIO_CRITERIA
+
+ s_score = str(scenario_score) + "/" + str(scenario_criteria)
+ s_status = "KO"
+ if scenario_score < scenario_criteria:
+ print (">>>> scenario not OK, score = %s/%s" %
+ (scenario_score, scenario_criteria))
+ s_status = "KO"
+ else:
+ print ">>>>> scenario OK, save the information"
+ s_status = "OK"
+ with open("./release/" + version +
+ "/validated_scenario_history.txt", "a") as f:
+ time_format = "%Y-%m-%d %H:%M"
+ info = (datetime.datetime.now().strftime(time_format) +
+ ";" + installer + ";" + s + "\n")
+ f.write(info)
+
+ scenario_result_criteria[s] = ScenarioResult(s_status, s_score)
+ print "--------------------------"
+
+ templateLoader = jinja2.FileSystemLoader(os.path.dirname
+ (os.path.abspath
+ (__file__)))
+ templateEnv = jinja2.Environment(loader=templateLoader)
+
+ TEMPLATE_FILE = "./template/index-status-tmpl.html"
+ template = templateEnv.get_template(TEMPLATE_FILE)
+
+ outputText = template.render(scenario_stats=scenario_stats,
+ scenario_results=scenario_result_criteria,
+ items=items,
+ installer=installer,
+ period=PERIOD,
+ version=version)
+
+ with open("./release/" + version +
+ "/index-status-" + installer + ".html", "wb") as fh:
+ fh.write(outputText)
diff --git a/utils/test/reporting/reporting-tempest.py b/utils/test/reporting/functest/reporting-tempest.py
index 944b42809..563e53010 100644
--- a/utils/test/reporting/reporting-tempest.py
+++ b/utils/test/reporting/functest/reporting-tempest.py
@@ -88,12 +88,12 @@ for installer in installers:
templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
templateEnv = jinja2.Environment(loader=templateLoader)
- TEMPLATE_FILE = "index-tempest-tmpl.html"
+ TEMPLATE_FILE = "./template/index-tempest-tmpl.html"
template = templateEnv.get_template(TEMPLATE_FILE)
outputText = template.render(scenario_results=scenario_results,
items=items,
installer=installer)
- with open("index-tempest-" + installer + ".html", "wb") as fh:
+ with open("./release/index-tempest-" + installer + ".html", "wb") as fh:
fh.write(outputText)
diff --git a/utils/test/reporting/reporting-vims.py b/utils/test/reporting/functest/reporting-vims.py
index cf43f3ebc..78ca9f5b3 100644
--- a/utils/test/reporting/reporting-vims.py
+++ b/utils/test/reporting/functest/reporting-vims.py
@@ -72,12 +72,12 @@ for installer in installers:
templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
templateEnv = jinja2.Environment( loader=templateLoader )
- TEMPLATE_FILE = "index-vims-tmpl.html"
+ TEMPLATE_FILE = "./template/index-vims-tmpl.html"
template = templateEnv.get_template( TEMPLATE_FILE )
outputText = template.render( scenario_results = scenario_results, step_order = step_order, installer = installer)
- with open("index-vims" + installer + ".html", "wb") as fh:
+ with open("./release/index-vims" + installer + ".html", "wb") as fh:
fh.write(outputText)
diff --git a/utils/test/reporting/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html
index 0fd470d6f..7a0656b74 100644
--- a/utils/test/reporting/index-status-tmpl.html
+++ b/utils/test/reporting/functest/template/index-status-tmpl.html
@@ -18,7 +18,7 @@
<body>
<div class="container">
<div class="masthead">
- <h3 class="text-muted">Functest status page</h3>
+ <h3 class="text-muted">Functest status page ({{version}})</h3>
<nav>
<ul class="nav nav-justified">
<li class="active"><a href="index.html">Home</a></li>
@@ -37,15 +37,23 @@
</div>
<div class="scenario-overview">
- <div class="panel-heading"><h4><b>List of last scenarios run over the last {{period}} days </b></h4></div>
+ <div class="panel-heading"><h4><b>List of last scenarios ({{version}}) run over the last {{period}} days </b></h4></div>
<table class="table">
<tr>
- <th width="80%">Scenario</th>
- <th width="20%">Iteration</th>
+ <th width="60%">Scenario</th>
+ <th width="20%">Status</th>
+ <th width="10%">Score</th>
+ <th width="10%">Iteration</th>
</tr>
{% for scenario,iteration in scenario_stats.iteritems() -%}
<tr class="tr-ok">
<td>{{scenario}}</td>
+ <td>{%if scenario_results[scenario].getStatus() is sameas "OK" -%}
+ <img src="../../img/icon-ok.png">
+ {%- else -%}
+ <img src="../../img/icon-nok.png">
+ {%- endif %}</td>
+ <td>{{scenario_results[scenario].getScore()}}</td>
<td>{{iteration}}</td>
</tr>
{%- endfor %}
@@ -53,7 +61,6 @@
</div>
-
{% for scenario, iteration in scenario_stats.iteritems() -%}
<div class="scenario-part">
<div class="page-header">
@@ -75,13 +82,13 @@
{% if test.isRunnable is sameas false -%}
<td>N.R</td>
{% elif test.getCriteria() > 2 -%}
- <td><img src="./img/weather-clear.png"></td>
+ <td><img src="../../img/weather-clear.png"></td>
{%- elif test.getCriteria() > 1 -%}
- <td><img src="./img/weather-few-clouds.png"></td>
+ <td><img src="../../img/weather-few-clouds.png"></td>
{%- elif test.getCriteria() > 0 -%}
- <td><img src="./img/weather-overcast.png"></td>
+ <td><img src="../../img/weather-overcast.png"></td>
{%- else -%}
- <td><img src="./img/weather-storm.png"></td>
+ <td><img src="../../img/weather-storm.png"></td>
{%- endif %}
{%- endfor %}
</tr>
diff --git a/utils/test/reporting/index-tempest-tmpl.html b/utils/test/reporting/functest/template/index-tempest-tmpl.html
index be0b79734..be0b79734 100644
--- a/utils/test/reporting/index-tempest-tmpl.html
+++ b/utils/test/reporting/functest/template/index-tempest-tmpl.html
diff --git a/utils/test/reporting/index-vims-tmpl.html b/utils/test/reporting/functest/template/index-vims-tmpl.html
index 8858182c1..8858182c1 100644
--- a/utils/test/reporting/index-vims-tmpl.html
+++ b/utils/test/reporting/functest/template/index-vims-tmpl.html