diff options
Diffstat (limited to 'utils/test/reporting/functest')
| -rw-r--r-- | utils/test/reporting/functest/img/icon-nok.png | bin | 0 -> 2317 bytes | |||
| -rw-r--r-- | utils/test/reporting/functest/img/icon-ok.png | bin | 0 -> 4063 bytes | |||
| -rw-r--r-- | utils/test/reporting/functest/reporting-status.py | 64 | ||||
| -rw-r--r-- | utils/test/reporting/functest/template/index-status-tmpl.html | 13 |
4 files changed, 68 insertions, 9 deletions
diff --git a/utils/test/reporting/functest/img/icon-nok.png b/utils/test/reporting/functest/img/icon-nok.png Binary files differnew file mode 100644 index 000000000..526b5294b --- /dev/null +++ b/utils/test/reporting/functest/img/icon-nok.png diff --git a/utils/test/reporting/functest/img/icon-ok.png b/utils/test/reporting/functest/img/icon-ok.png Binary files differnew file mode 100644 index 000000000..3a9de2e89 --- /dev/null +++ b/utils/test/reporting/functest/img/icon-ok.png diff --git a/utils/test/reporting/functest/reporting-status.py b/utils/test/reporting/functest/reporting-status.py index 2eb0f50d5..e6ea1b4b8 100644 --- a/utils/test/reporting/functest/reporting-status.py +++ b/utils/test/reporting/functest/reporting-status.py @@ -1,4 +1,5 @@ from urllib2 import Request, urlopen, URLError +import datetime import json import jinja2 import os @@ -20,12 +21,14 @@ installers = ["apex", "compass", "fuel", "joid"] versions = ["brahmaputra", "master"] # versions = ["master"] PERIOD = 10 +MAX_SCENARIO_CRITERIA = 18 # Correspondance between the name of the test case and the name in the DB # ideally we should modify the DB to avoid such interface.... # '<name in the DB':'<name in the config'> # I know it is uggly... -test_match_matrix = {'vPing': 'vping_ssh', +test_match_matrix = {'healthcheck': 'healthcheck', + 'vPing': 'vping_ssh', 'vPing_userdata': 'vping_userdata', 'ODL': 'odl', 'ONOS': 'onos', @@ -102,6 +105,20 @@ class TestCase(object): self.isRunnable = is_runnable +class ScenarioResult(object): + def __init__(self, status, score=0): + self.status = status + self.score = score + + def getStatus(self): + return self.status + + def getScore(self): + return self.score + +# ***************************************************************************** + + def getApiResults(case, installer, scenario, version): case = case.getName() results = json.dumps([]) @@ -245,7 +262,10 @@ functest_yaml_config = yaml.load(response.text) print "****************************************" print "* Generating reporting..... *" +print ("* Data retention = %s days *" % PERIOD) +print "* *" print "****************************************" + # For all the versions for version in versions: # For all the installers @@ -253,11 +273,16 @@ for version in versions: # get scenarios scenario_results = getScenarios(tempest, installer, version) scenario_stats = getScenarioStats(scenario_results) - items = {} + scenario_result_criteria = {} + # For all the scenarios get results for s, s_result in scenario_results.items(): testCases = [] + # Green or Red light for a given scenario + nb_test_runnable_for_this_scenario = 0 + scenario_score = 0 + # For each scenario declare the test cases # Functest cases for test_case in functest_test_list: @@ -276,31 +301,58 @@ for version in versions: test_case.checkRunnable(installer, s, functest_yaml_config) # print "testcase %s is %s" % (test_case.getName(), # test_case.isRunnable) - print "--------------------------" print ("installer %s, version %s, scenario %s:" % (installer, version, s)) for testCase in testCases: time.sleep(1) if testCase.isRunnable: + nb_test_runnable_for_this_scenario += 1 print (" Searching results for case %s " % (testCase.getName())) result = getResult(testCase, installer, s, version) testCase.setCriteria(result) items[s] = testCases - print "--------------------------" + scenario_score = scenario_score + result except: print ("installer %s, version %s, scenario %s" % (installer, version, s)) print "No data available , error %s " % (sys.exc_info()[0]) - print "****************************************" - templateLoader = jinja2.FileSystemLoader(os.path.dirname(os.path.abspath(__file__))) + # the validation criteria = nb runnable tests x 3 + scenario_criteria = nb_test_runnable_for_this_scenario * 3 + # if 0 runnable tests set criteria at a high value + if scenario_criteria < 1: + scenario_criteria = MAX_SCENARIO_CRITERIA + + s_score = str(scenario_score) + "/" + str(scenario_criteria) + s_status = "KO" + if scenario_score < scenario_criteria: + print (">>>> scenario not OK, score = %s/%s" % + (scenario_score, scenario_criteria)) + s_status = "KO" + else: + print ">>>>> scenario OK, save the information" + s_status = "OK" + with open("./release/" + version + + "/validated_scenario_history.txt", "a") as f: + time_format = "%Y-%m-%d %H:%M" + info = (datetime.datetime.now().strftime(time_format) + + ";" + installer + ";" + s + "\n") + f.write(info) + + scenario_result_criteria[s] = ScenarioResult(s_status, s_score) + print "--------------------------" + + templateLoader = jinja2.FileSystemLoader(os.path.dirname + (os.path.abspath + (__file__))) templateEnv = jinja2.Environment(loader=templateLoader) TEMPLATE_FILE = "./template/index-status-tmpl.html" template = templateEnv.get_template(TEMPLATE_FILE) outputText = template.render(scenario_stats=scenario_stats, + scenario_results=scenario_result_criteria, items=items, installer=installer, period=PERIOD, diff --git a/utils/test/reporting/functest/template/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html index 604f2c8e4..7a0656b74 100644 --- a/utils/test/reporting/functest/template/index-status-tmpl.html +++ b/utils/test/reporting/functest/template/index-status-tmpl.html @@ -40,12 +40,20 @@ <div class="panel-heading"><h4><b>List of last scenarios ({{version}}) run over the last {{period}} days </b></h4></div> <table class="table"> <tr> - <th width="80%">Scenario</th> - <th width="20%">Iteration</th> + <th width="60%">Scenario</th> + <th width="20%">Status</th> + <th width="10%">Score</th> + <th width="10%">Iteration</th> </tr> {% for scenario,iteration in scenario_stats.iteritems() -%} <tr class="tr-ok"> <td>{{scenario}}</td> + <td>{%if scenario_results[scenario].getStatus() is sameas "OK" -%} + <img src="../../img/icon-ok.png"> + {%- else -%} + <img src="../../img/icon-nok.png"> + {%- endif %}</td> + <td>{{scenario_results[scenario].getScore()}}</td> <td>{{iteration}}</td> </tr> {%- endfor %} @@ -53,7 +61,6 @@ </div> - {% for scenario, iteration in scenario_stats.iteritems() -%} <div class="scenario-part"> <div class="page-header"> |
