summaryrefslogtreecommitdiffstats
path: root/utils/test/reporting
diff options
context:
space:
mode:
Diffstat (limited to 'utils/test/reporting')
-rwxr-xr-xutils/test/reporting/functest/reporting-status.py407
-rw-r--r--utils/test/reporting/functest/template/index-status-tmpl.html22
-rw-r--r--utils/test/reporting/reporting.yaml7
-rw-r--r--utils/test/reporting/utils/reporting_utils.py39
4 files changed, 266 insertions, 209 deletions
diff --git a/utils/test/reporting/functest/reporting-status.py b/utils/test/reporting/functest/reporting-status.py
index af1d1d8a5..94e7f2f3e 100755
--- a/utils/test/reporting/functest/reporting-status.py
+++ b/utils/test/reporting/functest/reporting-status.py
@@ -9,10 +9,8 @@
import datetime
import jinja2
import os
-import requests
import sys
import time
-import yaml
import testCase as tc
import scenarioResult as sr
@@ -43,9 +41,7 @@ log_level = rp_utils.get_config('general.log.log_level')
exclude_noha = rp_utils.get_config('functest.exclude_noha')
exclude_virtual = rp_utils.get_config('functest.exclude_virtual')
-response = requests.get(cf)
-
-functest_yaml_config = yaml.safe_load(response.text)
+functest_yaml_config = rp_utils.getFunctestConfig()
logger.info("*******************************************")
logger.info("* *")
@@ -69,128 +65,117 @@ config_tiers = functest_yaml_config.get("tiers")
for tier in config_tiers:
if tier['order'] >= 0 and tier['order'] < 2:
for case in tier['testcases']:
- if case['name'] not in blacklist:
- testValid.append(tc.TestCase(case['name'],
+ if case['case_name'] not in blacklist:
+ testValid.append(tc.TestCase(case['case_name'],
"functest",
case['dependencies']))
elif tier['order'] == 2:
for case in tier['testcases']:
- if case['name'] not in blacklist:
- testValid.append(tc.TestCase(case['name'],
- case['name'],
+ if case['case_name'] not in blacklist:
+ testValid.append(tc.TestCase(case['case_name'],
+ case['case_name'],
case['dependencies']))
elif tier['order'] > 2:
for case in tier['testcases']:
- if case['name'] not in blacklist:
- otherTestCases.append(tc.TestCase(case['name'],
+ if case['case_name'] not in blacklist:
+ otherTestCases.append(tc.TestCase(case['case_name'],
"functest",
case['dependencies']))
logger.debug("Functest reporting start")
+
# For all the versions
for version in versions:
# For all the installers
+ scenario_directory = "./display/" + version + "/functest/"
+ scenario_file_name = scenario_directory + "scenario_history.txt"
+
+ # check that the directory exists, if not create it
+ # (first run on new version)
+ if not os.path.exists(scenario_directory):
+ os.makedirs(scenario_directory)
+
+ # initiate scenario file if it does not exist
+ if not os.path.isfile(scenario_file_name):
+ with open(scenario_file_name, "a") as my_file:
+ logger.debug("Create scenario file: %s" % scenario_file_name)
+ my_file.write("date,scenario,installer,detail,score\n")
+
for installer in installers:
+
# get scenarios
scenario_results = rp_utils.getScenarios(healthcheck,
installer,
version)
- scenario_stats = rp_utils.getScenarioStats(scenario_results)
- items = {}
- scenario_result_criteria = {}
- scenario_directory = "./display/" + version + "/functest/"
- scenario_file_name = scenario_directory + "scenario_history.txt"
-
- # check that the directory exists, if not create it
- # (first run on new version)
- if not os.path.exists(scenario_directory):
- os.makedirs(scenario_directory)
-
- # initiate scenario file if it does not exist
- if not os.path.isfile(scenario_file_name):
- with open(scenario_file_name, "a") as my_file:
- logger.debug("Create scenario file: %s" % scenario_file_name)
- my_file.write("date,scenario,installer,detail,score\n")
-
- # For all the scenarios get results
- for s, s_result in scenario_results.items():
- logger.info("---------------------------------")
- logger.info("installer %s, version %s, scenario %s:" %
- (installer, version, s))
- logger.debug("Scenario results: %s" % s_result)
-
- # Green or Red light for a given scenario
- nb_test_runnable_for_this_scenario = 0
- scenario_score = 0
- # url of the last jenkins log corresponding to a given
- # scenario
- s_url = ""
- if len(s_result) > 0:
- build_tag = s_result[len(s_result)-1]['build_tag']
- logger.debug("Build tag: %s" % build_tag)
- s_url = rp_utils.getJenkinsUrl(build_tag)
- if s_url is None:
- s_url = "http://testresultS.opnfv.org/reporting"
- logger.info("last jenkins url: %s" % s_url)
- testCases2BeDisplayed = []
- # Check if test case is runnable / installer, scenario
- # for the test case used for Scenario validation
- try:
- # 1) Manage the test cases for the scenario validation
- # concretely Tiers 0-3
- for test_case in testValid:
- test_case.checkRunnable(installer, s,
- test_case.getConstraints())
- logger.debug("testcase %s (%s) is %s" %
- (test_case.getDisplayName(),
- test_case.getName(),
- test_case.isRunnable))
- time.sleep(1)
- if test_case.isRunnable:
- dbName = test_case.getDbName()
- name = test_case.getName()
- displayName = test_case.getDisplayName()
- project = test_case.getProject()
- nb_test_runnable_for_this_scenario += 1
- logger.info(" Searching results for case %s " %
- (displayName))
- result = rp_utils.getResult(dbName, installer,
- s, version)
- # if no result set the value to 0
- if result < 0:
- result = 0
- logger.info(" >>>> Test score = " + str(result))
- test_case.setCriteria(result)
- test_case.setIsRunnable(True)
- testCases2BeDisplayed.append(tc.TestCase(name,
- project,
- "",
- result,
- True,
- 1))
- scenario_score = scenario_score + result
-
- # 2) Manage the test cases for the scenario qualification
- # concretely Tiers > 3
- for test_case in otherTestCases:
- test_case.checkRunnable(installer, s,
- test_case.getConstraints())
- logger.debug("testcase %s (%s) is %s" %
- (test_case.getDisplayName(),
- test_case.getName(),
- test_case.isRunnable))
- time.sleep(1)
- if test_case.isRunnable:
- dbName = test_case.getDbName()
- name = test_case.getName()
- displayName = test_case.getDisplayName()
- project = test_case.getProject()
- logger.info(" Searching results for case %s " %
- (displayName))
- result = rp_utils.getResult(dbName, installer,
- s, version)
- # at least 1 result for the test
- if result > -1:
+
+ # get nb of supported architecture (x86, aarch64)
+ architectures = rp_utils.getArchitectures(scenario_results)
+ logger.info("Supported architectures: {}".format(architectures))
+
+ for architecture in architectures:
+ logger.info("architecture: {}".format(architecture))
+ # Consider only the results for the selected architecture
+ # i.e drop x86 for aarch64 and vice versa
+ filter_results = rp_utils.filterArchitecture(scenario_results,
+ architecture)
+ scenario_stats = rp_utils.getScenarioStats(filter_results)
+ items = {}
+ scenario_result_criteria = {}
+
+ # in case of more than 1 architecture supported
+ # precise the architecture
+ installer_display = installer
+ if (len(architectures) > 1):
+ installer_display = installer + "@" + architecture
+
+ # For all the scenarios get results
+ for s, s_result in filter_results.items():
+ logger.info("---------------------------------")
+ logger.info("installer %s, version %s, scenario %s:" %
+ (installer, version, s))
+ logger.debug("Scenario results: %s" % s_result)
+
+ # Green or Red light for a given scenario
+ nb_test_runnable_for_this_scenario = 0
+ scenario_score = 0
+ # url of the last jenkins log corresponding to a given
+ # scenario
+ s_url = ""
+ if len(s_result) > 0:
+ build_tag = s_result[len(s_result)-1]['build_tag']
+ logger.debug("Build tag: %s" % build_tag)
+ s_url = rp_utils.getJenkinsUrl(build_tag)
+ if s_url is None:
+ s_url = "http://testresultS.opnfv.org/reporting"
+ logger.info("last jenkins url: %s" % s_url)
+ testCases2BeDisplayed = []
+ # Check if test case is runnable / installer, scenario
+ # for the test case used for Scenario validation
+ try:
+ # 1) Manage the test cases for the scenario validation
+ # concretely Tiers 0-3
+ for test_case in testValid:
+ test_case.checkRunnable(installer, s,
+ test_case.getConstraints())
+ logger.debug("testcase %s (%s) is %s" %
+ (test_case.getDisplayName(),
+ test_case.getName(),
+ test_case.isRunnable))
+ time.sleep(1)
+ if test_case.isRunnable:
+ dbName = test_case.getDbName()
+ name = test_case.getName()
+ displayName = test_case.getDisplayName()
+ project = test_case.getProject()
+ nb_test_runnable_for_this_scenario += 1
+ logger.info(" Searching results for case %s " %
+ (displayName))
+ result = rp_utils.getResult(dbName, installer,
+ s, version)
+ # if no result set the value to 0
+ if result < 0:
+ result = 0
+ logger.info(" >>>> Test score = " + str(result))
test_case.setCriteria(result)
test_case.setIsRunnable(True)
testCases2BeDisplayed.append(tc.TestCase(name,
@@ -198,91 +183,127 @@ for version in versions:
"",
result,
True,
- 4))
- else:
- logger.debug("No results found")
-
- items[s] = testCases2BeDisplayed
- except:
- logger.error("Error: installer %s, version %s, scenario %s" %
- (installer, version, s))
- logger.error("No data available: %s " % (sys.exc_info()[0]))
-
- # **********************************************
- # Evaluate the results for scenario validation
- # **********************************************
- # the validation criteria = nb runnable tests x 3
- # because each test case = 0,1,2 or 3
- scenario_criteria = nb_test_runnable_for_this_scenario * 3
- # if 0 runnable tests set criteria at a high value
- if scenario_criteria < 1:
- scenario_criteria = 50 # conf.MAX_SCENARIO_CRITERIA
-
- s_score = str(scenario_score) + "/" + str(scenario_criteria)
- s_score_percent = rp_utils.getScenarioPercent(scenario_score,
- scenario_criteria)
-
- s_status = "KO"
- if scenario_score < scenario_criteria:
- logger.info(">>>> scenario not OK, score = %s/%s" %
- (scenario_score, scenario_criteria))
+ 1))
+ scenario_score = scenario_score + result
+
+ # 2) Manage the test cases for the scenario qualification
+ # concretely Tiers > 3
+ for test_case in otherTestCases:
+ test_case.checkRunnable(installer, s,
+ test_case.getConstraints())
+ logger.debug("testcase %s (%s) is %s" %
+ (test_case.getDisplayName(),
+ test_case.getName(),
+ test_case.isRunnable))
+ time.sleep(1)
+ if test_case.isRunnable:
+ dbName = test_case.getDbName()
+ name = test_case.getName()
+ displayName = test_case.getDisplayName()
+ project = test_case.getProject()
+ logger.info(" Searching results for case %s " %
+ (displayName))
+ result = rp_utils.getResult(dbName, installer,
+ s, version)
+ # at least 1 result for the test
+ if result > -1:
+ test_case.setCriteria(result)
+ test_case.setIsRunnable(True)
+ testCases2BeDisplayed.append(tc.TestCase(
+ name,
+ project,
+ "",
+ result,
+ True,
+ 4))
+ else:
+ logger.debug("No results found")
+
+ items[s] = testCases2BeDisplayed
+ except:
+ logger.error("Error: installer %s, version %s, scenario %s"
+ % (installer, version, s))
+ logger.error("No data available: %s" % (sys.exc_info()[0]))
+
+ # **********************************************
+ # Evaluate the results for scenario validation
+ # **********************************************
+ # the validation criteria = nb runnable tests x 3
+ # because each test case = 0,1,2 or 3
+ scenario_criteria = nb_test_runnable_for_this_scenario * 3
+ # if 0 runnable tests set criteria at a high value
+ if scenario_criteria < 1:
+ scenario_criteria = 50 # conf.MAX_SCENARIO_CRITERIA
+
+ s_score = str(scenario_score) + "/" + str(scenario_criteria)
+ s_score_percent = rp_utils.getScenarioPercent(
+ scenario_score,
+ scenario_criteria)
+
s_status = "KO"
- else:
- logger.info(">>>>> scenario OK, save the information")
- s_status = "OK"
- path_validation_file = ("./display/" + version +
- "/functest/" +
- "validated_scenario_history.txt")
- with open(path_validation_file, "a") as f:
- time_format = "%Y-%m-%d %H:%M"
- info = (datetime.datetime.now().strftime(time_format) +
- ";" + installer + ";" + s + "\n")
+ if scenario_score < scenario_criteria:
+ logger.info(">>>> scenario not OK, score = %s/%s" %
+ (scenario_score, scenario_criteria))
+ s_status = "KO"
+ else:
+ logger.info(">>>>> scenario OK, save the information")
+ s_status = "OK"
+ path_validation_file = ("./display/" + version +
+ "/functest/" +
+ "validated_scenario_history.txt")
+ with open(path_validation_file, "a") as f:
+ time_format = "%Y-%m-%d %H:%M"
+ info = (datetime.datetime.now().strftime(time_format) +
+ ";" + installer_display + ";" + s + "\n")
+ f.write(info)
+
+ # Save daily results in a file
+ with open(scenario_file_name, "a") as f:
+ info = (reportingDate + "," + s + "," + installer_display +
+ "," + s_score + "," +
+ str(round(s_score_percent)) + "\n")
f.write(info)
- # Save daily results in a file
- with open(scenario_file_name, "a") as f:
- info = (reportingDate + "," + s + "," + installer +
- "," + s_score + "," +
- str(round(s_score_percent)) + "\n")
- f.write(info)
-
- scenario_result_criteria[s] = sr.ScenarioResult(s_status,
- s_score,
- s_score_percent,
- s_url)
- logger.info("--------------------------")
-
- templateLoader = jinja2.FileSystemLoader(".")
- templateEnv = jinja2.Environment(
- loader=templateLoader, autoescape=True)
-
- TEMPLATE_FILE = "./functest/template/index-status-tmpl.html"
- template = templateEnv.get_template(TEMPLATE_FILE)
-
- outputText = template.render(scenario_stats=scenario_stats,
- scenario_results=scenario_result_criteria,
- items=items,
- installer=installer,
- period=period,
- version=version,
- date=reportingDate)
-
- with open("./display/" + version +
- "/functest/status-" + installer + ".html", "wb") as fh:
- fh.write(outputText)
-
- logger.info("Manage export CSV & PDF")
- rp_utils.export_csv(scenario_file_name, installer, version)
- logger.error("CSV generated...")
-
- # Generate outputs for export
- # pdf
- # TODO Change once web site updated...use the current one
- # to test pdf production
- url_pdf = rp_utils.get_config('general.url')
- pdf_path = ("./display/" + version +
- "/functest/status-" + installer + ".html")
- pdf_doc_name = ("./display/" + version +
- "/functest/status-" + installer + ".pdf")
- rp_utils.export_pdf(pdf_path, pdf_doc_name)
- logger.info("PDF generated...")
+ scenario_result_criteria[s] = sr.ScenarioResult(
+ s_status,
+ s_score,
+ s_score_percent,
+ s_url)
+ logger.info("--------------------------")
+
+ templateLoader = jinja2.FileSystemLoader(".")
+ templateEnv = jinja2.Environment(
+ loader=templateLoader, autoescape=True)
+
+ TEMPLATE_FILE = "./functest/template/index-status-tmpl.html"
+ template = templateEnv.get_template(TEMPLATE_FILE)
+
+ outputText = template.render(
+ scenario_stats=scenario_stats,
+ scenario_results=scenario_result_criteria,
+ items=items,
+ installer=installer_display,
+ period=period,
+ version=version,
+ date=reportingDate)
+
+ with open("./display/" + version +
+ "/functest/status-" +
+ installer_display + ".html", "wb") as fh:
+ fh.write(outputText)
+
+ logger.info("Manage export CSV & PDF")
+ rp_utils.export_csv(scenario_file_name, installer_display, version)
+ logger.error("CSV generated...")
+
+ # Generate outputs for export
+ # pdf
+ # TODO Change once web site updated...use the current one
+ # to test pdf production
+ url_pdf = rp_utils.get_config('general.url')
+ pdf_path = ("./display/" + version +
+ "/functest/status-" + installer_display + ".html")
+ pdf_doc_name = ("./display/" + version +
+ "/functest/status-" + installer_display + ".pdf")
+ rp_utils.export_pdf(pdf_path, pdf_doc_name)
+ logger.info("PDF generated...")
diff --git a/utils/test/reporting/functest/template/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html
index 52046c37f..ebacfd159 100644
--- a/utils/test/reporting/functest/template/index-status-tmpl.html
+++ b/utils/test/reporting/functest/template/index-status-tmpl.html
@@ -15,27 +15,27 @@
{% for scenario in scenario_stats.iteritems() -%}
var gaugeScenario{{loop.index}} = gauge('#gaugeScenario{{loop.index}}');
{%- endfor %}
-
+
// assign success rate to the gauge
function updateReadings() {
{% for scenario,iteration in scenario_stats.iteritems() -%}
gaugeScenario{{loop.index}}.update({{scenario_results[scenario].getScorePercent()}});
{%- endfor %}
}
- updateReadings();
+ updateReadings();
}
-
+
// trend line management
- d3.csv("./scenario_history.csv", function(data) {
+ d3.csv("./scenario_history.txt", function(data) {
// ***************************************
// Create the trend line
{% for scenario,iteration in scenario_stats.iteritems() -%}
- // for scenario {{scenario}}
+ // for scenario {{scenario}}
// Filter results
- var trend{{loop.index}} = data.filter(function(row) {
+ var trend{{loop.index}} = data.filter(function(row) {
return row["scenario"]=="{{scenario}}" && row["installer"]=="{{installer}}";
})
- // Parse the date
+ // Parse the date
trend{{loop.index}}.forEach(function(d) {
d.date = parseDate(d.date);
d.score = +d.score
@@ -44,7 +44,7 @@
var mytrend = trend("#trend_svg{{loop.index}}",trend{{loop.index}})
// ****************************************
{%- endfor %}
- });
+ });
if ( !window.isLoaded ) {
window.addEventListener("load", function() {
onDocumentReady();
@@ -61,7 +61,7 @@ $(document).ready(function (){
});
})
</script>
-
+
</head>
<body>
<div class="container">
@@ -72,8 +72,8 @@ $(document).ready(function (){
<li class="active"><a href="../../index.html">Home</a></li>
<li><a href="status-apex.html">Apex</a></li>
<li><a href="status-compass.html">Compass</a></li>
- <li><a href="status-daisy.html">Daisy</a></li>
- <li><a href="status-fuel.html">Fuel</a></li>
+ <li><a href="status-fuel@x86.html">fuel@x86</a></li>
+ <li><a href="status-fuel@aarch64.html">fuel@aarch64</a></li>
<li><a href="status-joid.html">Joid</a></li>
</ul>
</nav>
diff --git a/utils/test/reporting/reporting.yaml b/utils/test/reporting/reporting.yaml
index 6af87c121..1692f481d 100644
--- a/utils/test/reporting/reporting.yaml
+++ b/utils/test/reporting/reporting.yaml
@@ -3,7 +3,6 @@ general:
installers:
- apex
- compass
- - daisy
- fuel
- joid
@@ -37,7 +36,6 @@ functest:
blacklist:
- ovno
- security_scan
- - rally_sanity
- healthcheck
- odl_netvirt
- aaa
@@ -45,13 +43,12 @@ functest:
- orchestra_ims
- juju_epc
- orchestra
- - promise
max_scenario_criteria: 50
test_conf: https://git.opnfv.org/cgit/functest/plain/functest/ci/testcases.yaml
log_level: ERROR
jenkins_url: https://build.opnfv.org/ci/view/functest/job/
- exclude_noha: "False"
- exclude_virtual: "False"
+ exclude_noha: False
+ exclude_virtual: False
yardstick:
test_conf: https://git.opnfv.org/cgit/yardstick/plain/tests/ci/report_config.yaml
diff --git a/utils/test/reporting/utils/reporting_utils.py b/utils/test/reporting/utils/reporting_utils.py
index 0eae59163..599a93818 100644
--- a/utils/test/reporting/utils/reporting_utils.py
+++ b/utils/test/reporting/utils/reporting_utils.py
@@ -10,6 +10,7 @@ from urllib2 import Request, urlopen, URLError
import logging
import json
import os
+import requests
import pdfkit
import yaml
@@ -328,6 +329,44 @@ def getScenarioPercent(scenario_score, scenario_criteria):
# *********
+# Functest
+# *********
+def getFunctestConfig(version=""):
+ config_file = get_config('functest.test_conf') + version
+ response = requests.get(config_file)
+ return yaml.safe_load(response.text)
+
+
+def getArchitectures(scenario_results):
+ supported_arch = ['x86']
+ if (len(scenario_results) > 0):
+ for scenario_result in scenario_results.values():
+ for value in scenario_result:
+ if ("armband" in value['build_tag']):
+ supported_arch.append('aarch64')
+ return supported_arch
+ return supported_arch
+
+
+def filterArchitecture(results, architecture):
+ filtered_results = {}
+ for name, results in results.items():
+ filtered_values = []
+ for value in results:
+ if (architecture is "x86"):
+ # drop aarch64 results
+ if ("armband" not in value['build_tag']):
+ filtered_values.append(value)
+ elif(architecture is "aarch64"):
+ # drop x86 results
+ if ("armband" in value['build_tag']):
+ filtered_values.append(value)
+ if (len(filtered_values) > 0):
+ filtered_results[name] = filtered_values
+ return filtered_results
+
+
+# *********
# Yardstick
# *********
def subfind(given_list, pattern_list):