From 6860c975e30daf3045f9565ea8ab82aacc2ed0f2 Mon Sep 17 00:00:00 2001 From: Morgan Richomme Date: Tue, 17 Oct 2017 11:54:44 +0200 Subject: Add new function to get indivial test score for aarch64 as architecture is not considered as a constraints (unlike scenario or installer) we need to work on the build tag to retrieve the data but it is not possible to use regex directly in the API so retrieveing the accurate results require extra processing * based on build_tag, filter the results properly per scenario * build a table build_tag time for a given scenario * keep only the last 4 results * evaluation criteria field It would simplified if architecture could be declared as a constraint then querying the resuls will not change and not require intermediate steps JIRA: FUNCTEST-880 Change-Id: Id4fe4f56babf0964d41e299905c804e0d837c3d7 Signed-off-by: Morgan Richomme --- .../reporting/functest/reporting-status.py | 20 ++++++++--- utils/test/reporting/reporting/reporting.yaml | 1 - .../reporting/reporting/utils/reporting_utils.py | 40 ++++++++++++++++++++-- 3 files changed, 54 insertions(+), 7 deletions(-) (limited to 'utils') diff --git a/utils/test/reporting/reporting/functest/reporting-status.py b/utils/test/reporting/reporting/functest/reporting-status.py index c71e00f3b..592f92996 100755 --- a/utils/test/reporting/reporting/functest/reporting-status.py +++ b/utils/test/reporting/reporting/functest/reporting-status.py @@ -172,8 +172,13 @@ for version in versions: nb_test_runnable_for_this_scenario += 1 LOGGER.info(" Searching results for case %s ", displayName) - result = rp_utils.getResult(name, installer, - s, version) + if "fuel" in installer: + result = rp_utils.getCaseScoreFromBuildTag( + name, + s_result) + else: + result = rp_utils.getCaseScore(name, installer, + s, version) # if no result set the value to 0 if result < 0: result = 0 @@ -204,8 +209,13 @@ for version in versions: project = test_case.getProject() LOGGER.info(" Searching results for case %s ", displayName) - result = rp_utils.getResult(name, installer, - s, version) + if "fuel" in installer: + result = rp_utils.getCaseScoreFromBuildTag( + name, + s_result) + else: + result = rp_utils.getCaseScore(name, installer, + s, version) # at least 1 result for the test if result > -1: test_case.setCriteria(result) @@ -240,6 +250,8 @@ for version in versions: # 2 iterations : max score = 20 (10x2) # 3 iterations : max score = 20 # 4 or more iterations : max score = 30 (1x30) + LOGGER.info("Number of iterations for this scenario: %s", + len(s_result)) if len(s_result) > 3: k_score = 3 elif len(s_result) < 2: diff --git a/utils/test/reporting/reporting/reporting.yaml b/utils/test/reporting/reporting/reporting.yaml index 1e2e9a476..ec5c2ef38 100644 --- a/utils/test/reporting/reporting/reporting.yaml +++ b/utils/test/reporting/reporting/reporting.yaml @@ -10,7 +10,6 @@ general: versions: - master - euphrates - - danube log: log_file: reporting.log diff --git a/utils/test/reporting/reporting/utils/reporting_utils.py b/utils/test/reporting/reporting/utils/reporting_utils.py index 65267ca11..8dc4f0933 100644 --- a/utils/test/reporting/reporting/utils/reporting_utils.py +++ b/utils/test/reporting/reporting/utils/reporting_utils.py @@ -6,7 +6,6 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -from urllib2 import Request, urlopen, URLError import logging import json import os @@ -14,6 +13,8 @@ import requests import pdfkit import yaml +from urllib2 import Request, urlopen, URLError + # ---------------------------------------------------------- # @@ -284,7 +285,7 @@ def getNbtestOk(results): return nb_test_ok -def getResult(testCase, installer, scenario, version): +def getCaseScore(testCase, installer, scenario, version): """ Get Result for a given Functest Testcase """ @@ -343,6 +344,41 @@ def getResult(testCase, installer, scenario, version): return test_result_indicator +def getCaseScoreFromBuildTag(testCase, s_results): + """ + Get Results for a given Functest Testcase with arch filtering + """ + url_base = get_config('testapi.url') + nb_tests = get_config('general.nb_iteration_tests_success_criteria') + test_result_indicator = 0 + # architecture is not a result field...so we cannot use getResult as it is + res_matrix = [] + try: + for s_result in s_results: + build_tag = s_result['build_tag'] + d = s_result['start_date'] + res_matrix.append({'date': d, + 'build_tag': build_tag}) + # sort res_matrix + filter_res_matrix = sorted(res_matrix, key=lambda k: k['date'], + reverse=True)[:nb_tests] + for my_res in filter_res_matrix: + url = ("http://" + url_base + "?case=" + testCase + + "&build_tag=" + my_res['build_tag']) + request = Request(url) + response = urlopen(request) + k = response.read() + results = json.loads(k) + if "PASS" in results['results'][0]['criteria']: + test_result_indicator += 1 + except: + print "No results found for this case" + if test_result_indicator > 3: + test_result_indicator = 3 + + return test_result_indicator + + def getJenkinsUrl(build_tag): """ Get Jenkins url_base corespoding to the last test CI run -- cgit 1.2.3-korg