diff options
Diffstat (limited to 'utils/test/reporting')
| -rw-r--r-- | utils/test/reporting/functest/default.css (renamed from utils/test/reporting/default.css) | 0 | ||||
| -rw-r--r-- | utils/test/reporting/functest/img/weather-clear.png (renamed from utils/test/reporting/img/weather-clear.png) | bin | 1560 -> 1560 bytes | |||
| -rw-r--r-- | utils/test/reporting/functest/img/weather-few-clouds.png (renamed from utils/test/reporting/img/weather-few-clouds.png) | bin | 1927 -> 1927 bytes | |||
| -rw-r--r-- | utils/test/reporting/functest/img/weather-overcast.png (renamed from utils/test/reporting/img/weather-overcast.png) | bin | 1588 -> 1588 bytes | |||
| -rw-r--r-- | utils/test/reporting/functest/img/weather-storm.png (renamed from utils/test/reporting/img/weather-storm.png) | bin | 2137 -> 2137 bytes | |||
| -rw-r--r-- | utils/test/reporting/functest/index.html (renamed from utils/test/reporting/index.html) | 0 | ||||
| -rw-r--r-- | utils/test/reporting/functest/reporting-status.py (renamed from utils/test/reporting/reporting-status.py) | 147 | ||||
| -rw-r--r-- | utils/test/reporting/functest/reporting-tempest.py (renamed from utils/test/reporting/reporting-tempest.py) | 4 | ||||
| -rw-r--r-- | utils/test/reporting/functest/reporting-vims.py (renamed from utils/test/reporting/reporting-vims.py) | 4 | ||||
| -rw-r--r-- | utils/test/reporting/functest/template/index-status-tmpl.html (renamed from utils/test/reporting/index-status-tmpl.html) | 12 | ||||
| -rw-r--r-- | utils/test/reporting/functest/template/index-tempest-tmpl.html (renamed from utils/test/reporting/index-tempest-tmpl.html) | 0 | ||||
| -rw-r--r-- | utils/test/reporting/functest/template/index-vims-tmpl.html (renamed from utils/test/reporting/index-vims-tmpl.html) | 0 |
12 files changed, 84 insertions, 83 deletions
diff --git a/utils/test/reporting/default.css b/utils/test/reporting/functest/default.css index 0e330e965..0e330e965 100644 --- a/utils/test/reporting/default.css +++ b/utils/test/reporting/functest/default.css diff --git a/utils/test/reporting/img/weather-clear.png b/utils/test/reporting/functest/img/weather-clear.png Binary files differindex a0d967750..a0d967750 100644 --- a/utils/test/reporting/img/weather-clear.png +++ b/utils/test/reporting/functest/img/weather-clear.png diff --git a/utils/test/reporting/img/weather-few-clouds.png b/utils/test/reporting/functest/img/weather-few-clouds.png Binary files differindex acfa78398..acfa78398 100644 --- a/utils/test/reporting/img/weather-few-clouds.png +++ b/utils/test/reporting/functest/img/weather-few-clouds.png diff --git a/utils/test/reporting/img/weather-overcast.png b/utils/test/reporting/functest/img/weather-overcast.png Binary files differindex 4296246d0..4296246d0 100644 --- a/utils/test/reporting/img/weather-overcast.png +++ b/utils/test/reporting/functest/img/weather-overcast.png diff --git a/utils/test/reporting/img/weather-storm.png b/utils/test/reporting/functest/img/weather-storm.png Binary files differindex 956f0e20f..956f0e20f 100644 --- a/utils/test/reporting/img/weather-storm.png +++ b/utils/test/reporting/functest/img/weather-storm.png diff --git a/utils/test/reporting/index.html b/utils/test/reporting/functest/index.html index af4033567..af4033567 100644 --- a/utils/test/reporting/index.html +++ b/utils/test/reporting/functest/index.html diff --git a/utils/test/reporting/reporting-status.py b/utils/test/reporting/functest/reporting-status.py index e15bac9f7..9e6aeb1ad 100644 --- a/utils/test/reporting/reporting-status.py +++ b/utils/test/reporting/functest/reporting-status.py @@ -4,6 +4,7 @@ import jinja2 import os import re import requests +import sys import time import yaml @@ -12,11 +13,12 @@ functest_test_list = ['vPing', 'vPing_userdata', 'Tempest', 'Rally', 'ODL', 'ONOS', 'vIMS'] # functest_test_list = ['vPing'] -# functest_test_list = [] companion_test_list = ['doctor/doctor-notification', 'promise/promise'] # companion_test_list = [] installers = ["apex", "compass", "fuel", "joid"] -# installers = ["apex"] +# installers = ["fuel"] +versions = ["brahmaputra", "master"] +# versions = ["master"] PERIOD = 10 # Correspondance between the name of the test case and the name in the DB @@ -100,7 +102,7 @@ class TestCase(object): self.isRunnable = is_runnable -def getApiResults(case, installer, scenario): +def getApiResults(case, installer, scenario, version): case = case.getName() results = json.dumps([]) # to remove proxy (to be removed at the end for local test only) @@ -111,7 +113,7 @@ def getApiResults(case, installer, scenario): # "&period=30&installer=" + installer url = "http://testresults.opnfv.org/testapi/results?case=" + case + \ "&period=" + str(PERIOD) + "&installer=" + installer + \ - "&scenario=" + scenario + "&scenario=" + scenario + "&version=" + version request = Request(url) try: @@ -124,11 +126,12 @@ def getApiResults(case, installer, scenario): return results -def getScenarios(case, installer): +def getScenarios(case, installer, version): case = case.getName() url = "http://testresults.opnfv.org/testapi/results?case=" + case + \ - "&period=" + str(PERIOD) + "&installer=" + installer + "&period=" + str(PERIOD) + "&installer=" + installer + \ + "&version=" + version request = Request(url) try: @@ -147,9 +150,9 @@ def getScenarios(case, installer): for r in test_results: # Retrieve all the scenarios per installer - if not r['version'] in scenario_results.keys(): - scenario_results[r['version']] = [] - scenario_results[r['version']].append(r) + if not r['scenario'] in scenario_results.keys(): + scenario_results[r['scenario']] = [] + scenario_results[r['scenario']].append(r) return scenario_results @@ -174,10 +177,10 @@ def getNbtestOk(results): return nb_test_ok -def getResult(testCase, installer, scenario): +def getResult(testCase, installer, scenario, version): # retrieve raw results - results = getApiResults(testCase, installer, scenario) + results = getApiResults(testCase, installer, scenario, version) # let's concentrate on test results only test_results = results['test_results'] @@ -230,16 +233,6 @@ def getResult(testCase, installer, scenario): # ****************************************************************************** # ****************************************************************************** -# as the criteria are all difference, we shall use a common way to indicate -# the criteria -# 100 = 100% = all the test must be OK -# 90 = 90% = all the test must be above 90% of success rate -# TODO harmonize success criteria -# some criteria could be the duration, the success rate, the packet loss,... -# to be done case by case -# TODo create TestCriteria Object - - # init just tempest to get the list of scenarios # as all the scenarios run Tempest tempest = TestCase("Tempest", "functest", -1) @@ -252,55 +245,63 @@ functest_yaml_config = yaml.load(response.text) print "****************************************" print "* Generating reporting..... *" print "****************************************" -# For all the installers -for installer in installers: - # get scenarios - scenario_results = getScenarios(tempest, installer) - scenario_stats = getScenarioStats(scenario_results) - - items = {} - # For all the scenarios get results - for s, s_result in scenario_results.items(): - testCases = [] - # For each scenario declare the test cases - # Functest cases - for test_case in functest_test_list: - testCases.append(TestCase(test_case, "functest")) - - # project/case - for test_case in companion_test_list: - test_split = test_case.split("/") - test_project = test_split[0] - test_case = test_split[1] - testCases.append(TestCase(test_case, test_project)) - - # Check if test case is runnable according to the installer, scenario - for test_case in testCases: - test_case.checkRunnable(installer, s, functest_yaml_config) - # print "testcase %s is %s" % (test_case.getName(), - # test_case.isRunnable) - - print "--------------------------" - print "%s / %s:" % (installer, s) - for testCase in testCases: - time.sleep(1) - if testCase.isRunnable: - print " Searching results for case %s " % testCase.getName() - result = getResult(testCase, installer, s) - testCase.setCriteria(result) - items[s] = testCases - print "--------------------------" - print "****************************************" - templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) - templateEnv = jinja2.Environment(loader=templateLoader) - - TEMPLATE_FILE = "index-status-tmpl.html" - template = templateEnv.get_template(TEMPLATE_FILE) - - outputText = template.render(scenario_stats=scenario_stats, - items=items, - installer=installer, - period=PERIOD) - - with open("index-status-" + installer + ".html", "wb") as fh: - fh.write(outputText) +# For all the versions +for version in versions: + # For all the installers + for installer in installers: + # get scenarios + scenario_results = getScenarios(tempest, installer, version) + scenario_stats = getScenarioStats(scenario_results) + + items = {} + # For all the scenarios get results + for s, s_result in scenario_results.items(): + testCases = [] + # For each scenario declare the test cases + # Functest cases + for test_case in functest_test_list: + testCases.append(TestCase(test_case, "functest")) + + # project/case + for test_case in companion_test_list: + test_split = test_case.split("/") + test_project = test_split[0] + test_case = test_split[1] + testCases.append(TestCase(test_case, test_project)) + + # Check if test case is runnable / installer, scenario + try: + for test_case in testCases: + test_case.checkRunnable(installer, s, functest_yaml_config) + # print "testcase %s is %s" % (test_case.getName(), + # test_case.isRunnable) + print "--------------------------" + print "installer %s, version %s, scenario %s:" % (installer, version, s) + for testCase in testCases: + time.sleep(1) + if testCase.isRunnable: + print " Searching results for case %s " % (testCase.getName()) + result = getResult(testCase, installer, s, version) + testCase.setCriteria(result) + items[s] = testCases + print "--------------------------" + except: + print "installer %s, version %s, scenario %s" % (installer, version, s) + print "No data available , error %s " % (sys.exc_info()[0]) + + print "****************************************" + templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) + templateEnv = jinja2.Environment(loader=templateLoader) + + TEMPLATE_FILE = "./template/index-status-tmpl.html" + template = templateEnv.get_template(TEMPLATE_FILE) + + outputText = template.render(scenario_stats=scenario_stats, + items=items, + installer=installer, + period=PERIOD, + version=version) + + with open("./release/" + version + + "/index-status-" + installer + ".html", "wb") as fh: + fh.write(outputText) diff --git a/utils/test/reporting/reporting-tempest.py b/utils/test/reporting/functest/reporting-tempest.py index 944b42809..563e53010 100644 --- a/utils/test/reporting/reporting-tempest.py +++ b/utils/test/reporting/functest/reporting-tempest.py @@ -88,12 +88,12 @@ for installer in installers: templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) templateEnv = jinja2.Environment(loader=templateLoader) - TEMPLATE_FILE = "index-tempest-tmpl.html" + TEMPLATE_FILE = "./template/index-tempest-tmpl.html" template = templateEnv.get_template(TEMPLATE_FILE) outputText = template.render(scenario_results=scenario_results, items=items, installer=installer) - with open("index-tempest-" + installer + ".html", "wb") as fh: + with open("./release/index-tempest-" + installer + ".html", "wb") as fh: fh.write(outputText) diff --git a/utils/test/reporting/reporting-vims.py b/utils/test/reporting/functest/reporting-vims.py index cf43f3ebc..78ca9f5b3 100644 --- a/utils/test/reporting/reporting-vims.py +++ b/utils/test/reporting/functest/reporting-vims.py @@ -72,12 +72,12 @@ for installer in installers: templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) templateEnv = jinja2.Environment( loader=templateLoader ) - TEMPLATE_FILE = "index-vims-tmpl.html" + TEMPLATE_FILE = "./template/index-vims-tmpl.html" template = templateEnv.get_template( TEMPLATE_FILE ) outputText = template.render( scenario_results = scenario_results, step_order = step_order, installer = installer) - with open("index-vims" + installer + ".html", "wb") as fh: + with open("./release/index-vims" + installer + ".html", "wb") as fh: fh.write(outputText) diff --git a/utils/test/reporting/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html index 0fd470d6f..604f2c8e4 100644 --- a/utils/test/reporting/index-status-tmpl.html +++ b/utils/test/reporting/functest/template/index-status-tmpl.html @@ -18,7 +18,7 @@ <body> <div class="container"> <div class="masthead"> - <h3 class="text-muted">Functest status page</h3> + <h3 class="text-muted">Functest status page ({{version}})</h3> <nav> <ul class="nav nav-justified"> <li class="active"><a href="index.html">Home</a></li> @@ -37,7 +37,7 @@ </div> <div class="scenario-overview"> - <div class="panel-heading"><h4><b>List of last scenarios run over the last {{period}} days </b></h4></div> + <div class="panel-heading"><h4><b>List of last scenarios ({{version}}) run over the last {{period}} days </b></h4></div> <table class="table"> <tr> <th width="80%">Scenario</th> @@ -75,13 +75,13 @@ {% if test.isRunnable is sameas false -%} <td>N.R</td> {% elif test.getCriteria() > 2 -%} - <td><img src="./img/weather-clear.png"></td> + <td><img src="../../img/weather-clear.png"></td> {%- elif test.getCriteria() > 1 -%} - <td><img src="./img/weather-few-clouds.png"></td> + <td><img src="../../img/weather-few-clouds.png"></td> {%- elif test.getCriteria() > 0 -%} - <td><img src="./img/weather-overcast.png"></td> + <td><img src="../../img/weather-overcast.png"></td> {%- else -%} - <td><img src="./img/weather-storm.png"></td> + <td><img src="../../img/weather-storm.png"></td> {%- endif %} {%- endfor %} </tr> diff --git a/utils/test/reporting/index-tempest-tmpl.html b/utils/test/reporting/functest/template/index-tempest-tmpl.html index be0b79734..be0b79734 100644 --- a/utils/test/reporting/index-tempest-tmpl.html +++ b/utils/test/reporting/functest/template/index-tempest-tmpl.html diff --git a/utils/test/reporting/index-vims-tmpl.html b/utils/test/reporting/functest/template/index-vims-tmpl.html index 8858182c1..8858182c1 100644 --- a/utils/test/reporting/index-vims-tmpl.html +++ b/utils/test/reporting/functest/template/index-vims-tmpl.html |
