summaryrefslogtreecommitdiffstats
path: root/utils
diff options
context:
space:
mode:
authorMorgan Richomme <morgan.richomme@orange.com>2016-04-26 14:24:56 +0200
committerMorgan Richomme <morgan.richomme@orange.com>2016-04-26 15:15:50 +0200
commit42c0c75a9e3564758aaaccadec9e1bad42b283eb (patch)
treeb4e407feb611c70b0b9dc5da7b7da1cf587cc7f6 /utils
parent9bd71f3a00944616299fa2168105b57d74e549ea (diff)
Functest reporting refactoring
integrate the notion of version (brahmaputra, master, ..) change dir structure Change-Id: Ieb8aed811ed4e1ab8738fb02e4db411da3d07ca2 Signed-off-by: Morgan Richomme <morgan.richomme@orange.com>
Diffstat (limited to 'utils')
-rw-r--r--utils/test/reporting/functest/default.css (renamed from utils/test/reporting/default.css)0
-rw-r--r--utils/test/reporting/functest/img/weather-clear.png (renamed from utils/test/reporting/img/weather-clear.png)bin1560 -> 1560 bytes
-rw-r--r--utils/test/reporting/functest/img/weather-few-clouds.png (renamed from utils/test/reporting/img/weather-few-clouds.png)bin1927 -> 1927 bytes
-rw-r--r--utils/test/reporting/functest/img/weather-overcast.png (renamed from utils/test/reporting/img/weather-overcast.png)bin1588 -> 1588 bytes
-rw-r--r--utils/test/reporting/functest/img/weather-storm.png (renamed from utils/test/reporting/img/weather-storm.png)bin2137 -> 2137 bytes
-rw-r--r--utils/test/reporting/functest/index.html (renamed from utils/test/reporting/index.html)0
-rw-r--r--utils/test/reporting/functest/reporting-status.py (renamed from utils/test/reporting/reporting-status.py)147
-rw-r--r--utils/test/reporting/functest/reporting-tempest.py (renamed from utils/test/reporting/reporting-tempest.py)4
-rw-r--r--utils/test/reporting/functest/reporting-vims.py (renamed from utils/test/reporting/reporting-vims.py)4
-rw-r--r--utils/test/reporting/functest/template/index-status-tmpl.html (renamed from utils/test/reporting/index-status-tmpl.html)12
-rw-r--r--utils/test/reporting/functest/template/index-tempest-tmpl.html (renamed from utils/test/reporting/index-tempest-tmpl.html)0
-rw-r--r--utils/test/reporting/functest/template/index-vims-tmpl.html (renamed from utils/test/reporting/index-vims-tmpl.html)0
12 files changed, 84 insertions, 83 deletions
diff --git a/utils/test/reporting/default.css b/utils/test/reporting/functest/default.css
index 0e330e965..0e330e965 100644
--- a/utils/test/reporting/default.css
+++ b/utils/test/reporting/functest/default.css
diff --git a/utils/test/reporting/img/weather-clear.png b/utils/test/reporting/functest/img/weather-clear.png
index a0d967750..a0d967750 100644
--- a/utils/test/reporting/img/weather-clear.png
+++ b/utils/test/reporting/functest/img/weather-clear.png
Binary files differ
diff --git a/utils/test/reporting/img/weather-few-clouds.png b/utils/test/reporting/functest/img/weather-few-clouds.png
index acfa78398..acfa78398 100644
--- a/utils/test/reporting/img/weather-few-clouds.png
+++ b/utils/test/reporting/functest/img/weather-few-clouds.png
Binary files differ
diff --git a/utils/test/reporting/img/weather-overcast.png b/utils/test/reporting/functest/img/weather-overcast.png
index 4296246d0..4296246d0 100644
--- a/utils/test/reporting/img/weather-overcast.png
+++ b/utils/test/reporting/functest/img/weather-overcast.png
Binary files differ
diff --git a/utils/test/reporting/img/weather-storm.png b/utils/test/reporting/functest/img/weather-storm.png
index 956f0e20f..956f0e20f 100644
--- a/utils/test/reporting/img/weather-storm.png
+++ b/utils/test/reporting/functest/img/weather-storm.png
Binary files differ
diff --git a/utils/test/reporting/index.html b/utils/test/reporting/functest/index.html
index af4033567..af4033567 100644
--- a/utils/test/reporting/index.html
+++ b/utils/test/reporting/functest/index.html
diff --git a/utils/test/reporting/reporting-status.py b/utils/test/reporting/functest/reporting-status.py
index e15bac9f7..9e6aeb1ad 100644
--- a/utils/test/reporting/reporting-status.py
+++ b/utils/test/reporting/functest/reporting-status.py
@@ -4,6 +4,7 @@ import jinja2
import os
import re
import requests
+import sys
import time
import yaml
@@ -12,11 +13,12 @@ functest_test_list = ['vPing', 'vPing_userdata',
'Tempest', 'Rally',
'ODL', 'ONOS', 'vIMS']
# functest_test_list = ['vPing']
-# functest_test_list = []
companion_test_list = ['doctor/doctor-notification', 'promise/promise']
# companion_test_list = []
installers = ["apex", "compass", "fuel", "joid"]
-# installers = ["apex"]
+# installers = ["fuel"]
+versions = ["brahmaputra", "master"]
+# versions = ["master"]
PERIOD = 10
# Correspondance between the name of the test case and the name in the DB
@@ -100,7 +102,7 @@ class TestCase(object):
self.isRunnable = is_runnable
-def getApiResults(case, installer, scenario):
+def getApiResults(case, installer, scenario, version):
case = case.getName()
results = json.dumps([])
# to remove proxy (to be removed at the end for local test only)
@@ -111,7 +113,7 @@ def getApiResults(case, installer, scenario):
# "&period=30&installer=" + installer
url = "http://testresults.opnfv.org/testapi/results?case=" + case + \
"&period=" + str(PERIOD) + "&installer=" + installer + \
- "&scenario=" + scenario
+ "&scenario=" + scenario + "&version=" + version
request = Request(url)
try:
@@ -124,11 +126,12 @@ def getApiResults(case, installer, scenario):
return results
-def getScenarios(case, installer):
+def getScenarios(case, installer, version):
case = case.getName()
url = "http://testresults.opnfv.org/testapi/results?case=" + case + \
- "&period=" + str(PERIOD) + "&installer=" + installer
+ "&period=" + str(PERIOD) + "&installer=" + installer + \
+ "&version=" + version
request = Request(url)
try:
@@ -147,9 +150,9 @@ def getScenarios(case, installer):
for r in test_results:
# Retrieve all the scenarios per installer
- if not r['version'] in scenario_results.keys():
- scenario_results[r['version']] = []
- scenario_results[r['version']].append(r)
+ if not r['scenario'] in scenario_results.keys():
+ scenario_results[r['scenario']] = []
+ scenario_results[r['scenario']].append(r)
return scenario_results
@@ -174,10 +177,10 @@ def getNbtestOk(results):
return nb_test_ok
-def getResult(testCase, installer, scenario):
+def getResult(testCase, installer, scenario, version):
# retrieve raw results
- results = getApiResults(testCase, installer, scenario)
+ results = getApiResults(testCase, installer, scenario, version)
# let's concentrate on test results only
test_results = results['test_results']
@@ -230,16 +233,6 @@ def getResult(testCase, installer, scenario):
# ******************************************************************************
# ******************************************************************************
-# as the criteria are all difference, we shall use a common way to indicate
-# the criteria
-# 100 = 100% = all the test must be OK
-# 90 = 90% = all the test must be above 90% of success rate
-# TODO harmonize success criteria
-# some criteria could be the duration, the success rate, the packet loss,...
-# to be done case by case
-# TODo create TestCriteria Object
-
-
# init just tempest to get the list of scenarios
# as all the scenarios run Tempest
tempest = TestCase("Tempest", "functest", -1)
@@ -252,55 +245,63 @@ functest_yaml_config = yaml.load(response.text)
print "****************************************"
print "* Generating reporting..... *"
print "****************************************"
-# For all the installers
-for installer in installers:
- # get scenarios
- scenario_results = getScenarios(tempest, installer)
- scenario_stats = getScenarioStats(scenario_results)
-
- items = {}
- # For all the scenarios get results
- for s, s_result in scenario_results.items():
- testCases = []
- # For each scenario declare the test cases
- # Functest cases
- for test_case in functest_test_list:
- testCases.append(TestCase(test_case, "functest"))
-
- # project/case
- for test_case in companion_test_list:
- test_split = test_case.split("/")
- test_project = test_split[0]
- test_case = test_split[1]
- testCases.append(TestCase(test_case, test_project))
-
- # Check if test case is runnable according to the installer, scenario
- for test_case in testCases:
- test_case.checkRunnable(installer, s, functest_yaml_config)
- # print "testcase %s is %s" % (test_case.getName(),
- # test_case.isRunnable)
-
- print "--------------------------"
- print "%s / %s:" % (installer, s)
- for testCase in testCases:
- time.sleep(1)
- if testCase.isRunnable:
- print " Searching results for case %s " % testCase.getName()
- result = getResult(testCase, installer, s)
- testCase.setCriteria(result)
- items[s] = testCases
- print "--------------------------"
- print "****************************************"
- templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
- templateEnv = jinja2.Environment(loader=templateLoader)
-
- TEMPLATE_FILE = "index-status-tmpl.html"
- template = templateEnv.get_template(TEMPLATE_FILE)
-
- outputText = template.render(scenario_stats=scenario_stats,
- items=items,
- installer=installer,
- period=PERIOD)
-
- with open("index-status-" + installer + ".html", "wb") as fh:
- fh.write(outputText)
+# For all the versions
+for version in versions:
+ # For all the installers
+ for installer in installers:
+ # get scenarios
+ scenario_results = getScenarios(tempest, installer, version)
+ scenario_stats = getScenarioStats(scenario_results)
+
+ items = {}
+ # For all the scenarios get results
+ for s, s_result in scenario_results.items():
+ testCases = []
+ # For each scenario declare the test cases
+ # Functest cases
+ for test_case in functest_test_list:
+ testCases.append(TestCase(test_case, "functest"))
+
+ # project/case
+ for test_case in companion_test_list:
+ test_split = test_case.split("/")
+ test_project = test_split[0]
+ test_case = test_split[1]
+ testCases.append(TestCase(test_case, test_project))
+
+ # Check if test case is runnable / installer, scenario
+ try:
+ for test_case in testCases:
+ test_case.checkRunnable(installer, s, functest_yaml_config)
+ # print "testcase %s is %s" % (test_case.getName(),
+ # test_case.isRunnable)
+ print "--------------------------"
+ print "installer %s, version %s, scenario %s:" % (installer, version, s)
+ for testCase in testCases:
+ time.sleep(1)
+ if testCase.isRunnable:
+ print " Searching results for case %s " % (testCase.getName())
+ result = getResult(testCase, installer, s, version)
+ testCase.setCriteria(result)
+ items[s] = testCases
+ print "--------------------------"
+ except:
+ print "installer %s, version %s, scenario %s" % (installer, version, s)
+ print "No data available , error %s " % (sys.exc_info()[0])
+
+ print "****************************************"
+ templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
+ templateEnv = jinja2.Environment(loader=templateLoader)
+
+ TEMPLATE_FILE = "./template/index-status-tmpl.html"
+ template = templateEnv.get_template(TEMPLATE_FILE)
+
+ outputText = template.render(scenario_stats=scenario_stats,
+ items=items,
+ installer=installer,
+ period=PERIOD,
+ version=version)
+
+ with open("./release/" + version +
+ "/index-status-" + installer + ".html", "wb") as fh:
+ fh.write(outputText)
diff --git a/utils/test/reporting/reporting-tempest.py b/utils/test/reporting/functest/reporting-tempest.py
index 944b42809..563e53010 100644
--- a/utils/test/reporting/reporting-tempest.py
+++ b/utils/test/reporting/functest/reporting-tempest.py
@@ -88,12 +88,12 @@ for installer in installers:
templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
templateEnv = jinja2.Environment(loader=templateLoader)
- TEMPLATE_FILE = "index-tempest-tmpl.html"
+ TEMPLATE_FILE = "./template/index-tempest-tmpl.html"
template = templateEnv.get_template(TEMPLATE_FILE)
outputText = template.render(scenario_results=scenario_results,
items=items,
installer=installer)
- with open("index-tempest-" + installer + ".html", "wb") as fh:
+ with open("./release/index-tempest-" + installer + ".html", "wb") as fh:
fh.write(outputText)
diff --git a/utils/test/reporting/reporting-vims.py b/utils/test/reporting/functest/reporting-vims.py
index cf43f3ebc..78ca9f5b3 100644
--- a/utils/test/reporting/reporting-vims.py
+++ b/utils/test/reporting/functest/reporting-vims.py
@@ -72,12 +72,12 @@ for installer in installers:
templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__)))
templateEnv = jinja2.Environment( loader=templateLoader )
- TEMPLATE_FILE = "index-vims-tmpl.html"
+ TEMPLATE_FILE = "./template/index-vims-tmpl.html"
template = templateEnv.get_template( TEMPLATE_FILE )
outputText = template.render( scenario_results = scenario_results, step_order = step_order, installer = installer)
- with open("index-vims" + installer + ".html", "wb") as fh:
+ with open("./release/index-vims" + installer + ".html", "wb") as fh:
fh.write(outputText)
diff --git a/utils/test/reporting/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html
index 0fd470d6f..604f2c8e4 100644
--- a/utils/test/reporting/index-status-tmpl.html
+++ b/utils/test/reporting/functest/template/index-status-tmpl.html
@@ -18,7 +18,7 @@
<body>
<div class="container">
<div class="masthead">
- <h3 class="text-muted">Functest status page</h3>
+ <h3 class="text-muted">Functest status page ({{version}})</h3>
<nav>
<ul class="nav nav-justified">
<li class="active"><a href="index.html">Home</a></li>
@@ -37,7 +37,7 @@
</div>
<div class="scenario-overview">
- <div class="panel-heading"><h4><b>List of last scenarios run over the last {{period}} days </b></h4></div>
+ <div class="panel-heading"><h4><b>List of last scenarios ({{version}}) run over the last {{period}} days </b></h4></div>
<table class="table">
<tr>
<th width="80%">Scenario</th>
@@ -75,13 +75,13 @@
{% if test.isRunnable is sameas false -%}
<td>N.R</td>
{% elif test.getCriteria() > 2 -%}
- <td><img src="./img/weather-clear.png"></td>
+ <td><img src="../../img/weather-clear.png"></td>
{%- elif test.getCriteria() > 1 -%}
- <td><img src="./img/weather-few-clouds.png"></td>
+ <td><img src="../../img/weather-few-clouds.png"></td>
{%- elif test.getCriteria() > 0 -%}
- <td><img src="./img/weather-overcast.png"></td>
+ <td><img src="../../img/weather-overcast.png"></td>
{%- else -%}
- <td><img src="./img/weather-storm.png"></td>
+ <td><img src="../../img/weather-storm.png"></td>
{%- endif %}
{%- endfor %}
</tr>
diff --git a/utils/test/reporting/index-tempest-tmpl.html b/utils/test/reporting/functest/template/index-tempest-tmpl.html
index be0b79734..be0b79734 100644
--- a/utils/test/reporting/index-tempest-tmpl.html
+++ b/utils/test/reporting/functest/template/index-tempest-tmpl.html
diff --git a/utils/test/reporting/index-vims-tmpl.html b/utils/test/reporting/functest/template/index-vims-tmpl.html
index 8858182c1..8858182c1 100644
--- a/utils/test/reporting/index-vims-tmpl.html
+++ b/utils/test/reporting/functest/template/index-vims-tmpl.html