diff options
Diffstat (limited to 'utils/test/reporting/functest')
18 files changed, 99 insertions, 23 deletions
diff --git a/utils/test/reporting/functest/img/gauge_0.png b/utils/test/reporting/functest/img/gauge_0.png Binary files differnew file mode 100644 index 000000000..ecefc0e66 --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_0.png diff --git a/utils/test/reporting/functest/img/gauge_100.png b/utils/test/reporting/functest/img/gauge_100.png Binary files differnew file mode 100644 index 000000000..e199e1561 --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_100.png diff --git a/utils/test/reporting/functest/img/gauge_16.7.png b/utils/test/reporting/functest/img/gauge_16.7.png Binary files differnew file mode 100644 index 000000000..3e3993c3b --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_16.7.png diff --git a/utils/test/reporting/functest/img/gauge_25.png b/utils/test/reporting/functest/img/gauge_25.png Binary files differnew file mode 100644 index 000000000..4923659b9 --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_25.png diff --git a/utils/test/reporting/functest/img/gauge_33.3.png b/utils/test/reporting/functest/img/gauge_33.3.png Binary files differnew file mode 100644 index 000000000..364574b4a --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_33.3.png diff --git a/utils/test/reporting/functest/img/gauge_41.7.png b/utils/test/reporting/functest/img/gauge_41.7.png Binary files differnew file mode 100644 index 000000000..8c3e910fa --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_41.7.png diff --git a/utils/test/reporting/functest/img/gauge_50.png b/utils/test/reporting/functest/img/gauge_50.png Binary files differnew file mode 100644 index 000000000..2874b9fcf --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_50.png diff --git a/utils/test/reporting/functest/img/gauge_58.3.png b/utils/test/reporting/functest/img/gauge_58.3.png Binary files differnew file mode 100644 index 000000000..beedc8aa9 --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_58.3.png diff --git a/utils/test/reporting/functest/img/gauge_66.7.png b/utils/test/reporting/functest/img/gauge_66.7.png Binary files differnew file mode 100644 index 000000000..93f44d133 --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_66.7.png diff --git a/utils/test/reporting/functest/img/gauge_75.png b/utils/test/reporting/functest/img/gauge_75.png Binary files differnew file mode 100644 index 000000000..9fc261ff8 --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_75.png diff --git a/utils/test/reporting/functest/img/gauge_8.3.png b/utils/test/reporting/functest/img/gauge_8.3.png Binary files differnew file mode 100644 index 000000000..59f86571e --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_8.3.png diff --git a/utils/test/reporting/functest/img/gauge_83.3.png b/utils/test/reporting/functest/img/gauge_83.3.png Binary files differnew file mode 100644 index 000000000..27ae4ec54 --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_83.3.png diff --git a/utils/test/reporting/functest/img/gauge_91.7.png b/utils/test/reporting/functest/img/gauge_91.7.png Binary files differnew file mode 100644 index 000000000..280865714 --- /dev/null +++ b/utils/test/reporting/functest/img/gauge_91.7.png diff --git a/utils/test/reporting/functest/reporting-status.py b/utils/test/reporting/functest/reporting-status.py index 0b80fe49a..90699bd61 100755 --- a/utils/test/reporting/functest/reporting-status.py +++ b/utils/test/reporting/functest/reporting-status.py @@ -24,6 +24,7 @@ logger = utils.getLogger("Status") # Initialization testValid = [] otherTestCases = [] +reportingDate = datetime.datetime.now().strftime("%Y-%m-%d %H:%M") # init just tempest to get the list of scenarios # as all the scenarios run Tempest @@ -82,18 +83,26 @@ for version in conf.versions: # For all the scenarios get results for s, s_result in scenario_results.items(): + logger.info("---------------------------------") + logger.info("installer %s, version %s, scenario %s:" % + (installer, version, s)) + logger.debug("Scenario results: %s" % s_result) + # Green or Red light for a given scenario nb_test_runnable_for_this_scenario = 0 scenario_score = 0 - + # url of the last jenkins log corresponding to a given + # scenario + s_url = "" + if len(s_result) > 0: + build_tag = s_result[len(s_result)-1]['build_tag'] + logger.debug("Build tag: %s" % build_tag) + s_url = s_url = utils.getJenkinsUrl(build_tag) + logger.info("last jenkins url: %s" % s_url) testCases2BeDisplayed = [] # Check if test case is runnable / installer, scenario # for the test case used for Scenario validation try: - logger.info("---------------------------------") - logger.info("installer %s, version %s, scenario %s:" % - (installer, version, s)) - # 1) Manage the test cases for the scenario validation # concretely Tiers 0-3 for test_case in testValid: @@ -175,6 +184,8 @@ for version in conf.versions: scenario_criteria = conf.MAX_SCENARIO_CRITERIA s_score = str(scenario_score) + "/" + str(scenario_criteria) + s_score_percent = float( + scenario_score) / float(scenario_criteria) * 100 s_status = "KO" if scenario_score < scenario_criteria: logger.info(">>>> scenario not OK, score = %s/%s" % @@ -183,7 +194,8 @@ for version in conf.versions: else: logger.info(">>>>> scenario OK, save the information") s_status = "OK" - path_validation_file = (conf.REPORTING_PATH + "/release/" + version + + path_validation_file = (conf.REPORTING_PATH + + "/release/" + version + "/validated_scenario_history.txt") with open(path_validation_file, "a") as f: time_format = "%Y-%m-%d %H:%M" @@ -191,11 +203,25 @@ for version in conf.versions: ";" + installer + ";" + s + "\n") f.write(info) - scenario_result_criteria[s] = sr.ScenarioResult(s_status, s_score) + # Save daily results in a file + path_validation_file = (conf.REPORTING_PATH + + "/release/" + version + + "/scenario_history.txt") + with open(path_validation_file, "a") as f: + info = (reportingDate + "," + s + "," + installer + + "," + s_score + "," + + str(round(s_score_percent)) + "\n") + f.write(info) + + scenario_result_criteria[s] = sr.ScenarioResult(s_status, + s_score, + s_score_percent, + s_url) logger.info("--------------------------") templateLoader = jinja2.FileSystemLoader(conf.REPORTING_PATH) - templateEnv = jinja2.Environment(loader=templateLoader, autoescape=True) + templateEnv = jinja2.Environment( + loader=templateLoader, autoescape=True) TEMPLATE_FILE = "/template/index-status-tmpl.html" template = templateEnv.get_template(TEMPLATE_FILE) @@ -205,7 +231,8 @@ for version in conf.versions: items=items, installer=installer, period=conf.PERIOD, - version=version) + version=version, + date=reportingDate) with open(conf.REPORTING_PATH + "/release/" + version + "/index-status-" + installer + ".html", "wb") as fh: diff --git a/utils/test/reporting/functest/reportingConf.py b/utils/test/reporting/functest/reportingConf.py index b0e4cf7a1..e1c4b61a8 100644 --- a/utils/test/reporting/functest/reportingConf.py +++ b/utils/test/reporting/functest/reportingConf.py @@ -12,10 +12,10 @@ installers = ["apex", "compass", "fuel", "joid"] # list of test cases declared in testcases.yaml but that must not be # taken into account for the scoring -blacklist = ["ovno", "security_scan", 'odl-sfc'] +blacklist = ["ovno", "security_scan"] # versions = ["brahmaputra", "master"] versions = ["master", "colorado"] -PERIOD = 50 +PERIOD = 10 MAX_SCENARIO_CRITERIA = 50 # get the last 5 test results to determinate the success criteria NB_TESTS = 5 diff --git a/utils/test/reporting/functest/reportingUtils.py b/utils/test/reporting/functest/reportingUtils.py index f02620430..9ba02e821 100644 --- a/utils/test/reporting/functest/reportingUtils.py +++ b/utils/test/reporting/functest/reportingUtils.py @@ -139,7 +139,7 @@ def getResult(testCase, installer, scenario, version): # print "Nb test OK (last 10 days):"+ str(nbTestOk) # check that we have at least 4 runs if len(scenario_results) < 1: - # No results available + # No results available test_result_indicator = -1 elif nbTestOk < 1: test_result_indicator = 0 @@ -158,3 +158,21 @@ def getResult(testCase, installer, scenario, version): else: test_result_indicator = 2 return test_result_indicator + + +def getJenkinsUrl(build_tag): + # e.g. jenkins-functest-apex-apex-daily-colorado-daily-colorado-246 + # id = 246 + # note it is linked to jenkins format + # if this format changes...function to be adapted.... + url_base = "https://build.opnfv.org/ci/view/functest/job/" + jenkins_url = "" + try: + build_id = [int(s) for s in build_tag.split("-") if s.isdigit()] + jenkins_path = filter(lambda c: not c.isdigit(), build_tag) + url_id = jenkins_path[8:-1] + "/" + str(build_id[0]) + jenkins_url = url_base + url_id + "/console" + except: + print 'Impossible to get jenkins url:' + + return jenkins_url diff --git a/utils/test/reporting/functest/scenarioResult.py b/utils/test/reporting/functest/scenarioResult.py index 743346adc..5a54eed96 100644 --- a/utils/test/reporting/functest/scenarioResult.py +++ b/utils/test/reporting/functest/scenarioResult.py @@ -9,12 +9,21 @@ class ScenarioResult(object): - def __init__(self, status, score=0): + + def __init__(self, status, score=0, score_percent=0, url_lastrun=''): self.status = status self.score = score + self.score_percent = score_percent + self.url_lastrun = url_lastrun def getStatus(self): return self.status def getScore(self): return self.score + + def getScorePercent(self): + return self.score_percent + + def getUrlLastRun(self): + return self.url_lastrun diff --git a/utils/test/reporting/functest/template/index-status-tmpl.html b/utils/test/reporting/functest/template/index-status-tmpl.html index da2213bc0..67c23491a 100644 --- a/utils/test/reporting/functest/template/index-status-tmpl.html +++ b/utils/test/reporting/functest/template/index-status-tmpl.html @@ -18,7 +18,7 @@ <body> <div class="container"> <div class="masthead"> - <h3 class="text-muted">Functest status page ({{version}})</h3> + <h3 class="text-muted">Functest status page ({{version}}, {{date}})</h3> <nav> <ul class="nav nav-justified"> <li class="active"><a href="http://testresults.opnfv.org/reporting/index.html">Home</a></li> @@ -47,12 +47,34 @@ </tr> {% for scenario,iteration in scenario_stats.iteritems() -%} <tr class="tr-ok"> - <td>{{scenario}}</td> - <td>{%if scenario_results[scenario].getStatus() is sameas "OK" -%} - <img src="../../img/icon-ok.png"> - {%- else -%} - <img src="../../img/icon-nok.png"> - {%- endif %}</td> + <td><a href={{scenario_results[scenario].getUrlLastRun()}}>{{scenario}}</a></td> + <td>{%if scenario_results[scenario].getScorePercent() < 8.3 -%} + <img src="../../img/gauge_0.png"> + {%elif scenario_results[scenario].getScorePercent() < 16.7 -%} + <img src="../../img/gauge_8.3.png"> + {%elif scenario_results[scenario].getScorePercent() < 25 -%} + <img src="../../img/gauge_16.7.png"> + {%elif scenario_results[scenario].getScorePercent() < 33.3 -%} + <img src="../../img/gauge_25.png"> + {%elif scenario_results[scenario].getScorePercent() < 41.7 -%} + <img src="../../img/gauge_33.3.png"> + {%elif scenario_results[scenario].getScorePercent() < 50 -%} + <img src="../../img/gauge_41.7.png"> + {%elif scenario_results[scenario].getScorePercent() < 58.3 -%} + <img src="../../img/gauge_50.png"> + {%elif scenario_results[scenario].getScorePercent() < 66.7 -%} + <img src="../../img/gauge_58.3.png"> + {%elif scenario_results[scenario].getScorePercent() < 75 -%} + <img src="../../img/gauge_66.7.png"> + {%elif scenario_results[scenario].getScorePercent() < 83.3 -%} + <img src="../../img/gauge_75.png"> + {%elif scenario_results[scenario].getScorePercent() < 91.7 -%} + <img src="../../img/gauge_83.3.png"> + {%elif scenario_results[scenario].getScorePercent() < 100 -%} + <img src="../../img/gauge_91.7.png"> + {%- else -%} + <img src="../../img/gauge_100.png"> + {%- endif %}</td> <td>{{scenario_results[scenario].getScore()}}</td> <td>{{iteration}}</td> </tr> @@ -78,14 +100,14 @@ {% if test.getCriteria() > -1 -%} {{test.getDisplayName() }} {%- endif %} - {% if test.getTier() > 3 -%} + {% if test.getTier() > 3 -%} * {%- endif %} - </th> + </th> {%- endfor %} </tr> <tr class="tr-weather-weather"> - {% for test in items[scenario] -%} + {% for test in items[scenario] -%} {% if test.getCriteria() > 2 -%} <td><img src="../../img/weather-clear.png"></td> {%- elif test.getCriteria() > 1 -%} |
