diff options
Diffstat (limited to 'utils/test')
12 files changed, 109 insertions, 18 deletions
diff --git a/utils/test/reporting/reporting/functest/reporting-status.py b/utils/test/reporting/reporting/functest/reporting-status.py index c71e00f3b..592f92996 100755 --- a/utils/test/reporting/reporting/functest/reporting-status.py +++ b/utils/test/reporting/reporting/functest/reporting-status.py @@ -172,8 +172,13 @@ for version in versions: nb_test_runnable_for_this_scenario += 1 LOGGER.info(" Searching results for case %s ", displayName) - result = rp_utils.getResult(name, installer, - s, version) + if "fuel" in installer: + result = rp_utils.getCaseScoreFromBuildTag( + name, + s_result) + else: + result = rp_utils.getCaseScore(name, installer, + s, version) # if no result set the value to 0 if result < 0: result = 0 @@ -204,8 +209,13 @@ for version in versions: project = test_case.getProject() LOGGER.info(" Searching results for case %s ", displayName) - result = rp_utils.getResult(name, installer, - s, version) + if "fuel" in installer: + result = rp_utils.getCaseScoreFromBuildTag( + name, + s_result) + else: + result = rp_utils.getCaseScore(name, installer, + s, version) # at least 1 result for the test if result > -1: test_case.setCriteria(result) @@ -240,6 +250,8 @@ for version in versions: # 2 iterations : max score = 20 (10x2) # 3 iterations : max score = 20 # 4 or more iterations : max score = 30 (1x30) + LOGGER.info("Number of iterations for this scenario: %s", + len(s_result)) if len(s_result) > 3: k_score = 3 elif len(s_result) < 2: diff --git a/utils/test/reporting/reporting/reporting.yaml b/utils/test/reporting/reporting/reporting.yaml index 1e2e9a476..8123d0135 100644 --- a/utils/test/reporting/reporting/reporting.yaml +++ b/utils/test/reporting/reporting/reporting.yaml @@ -10,7 +10,6 @@ general: versions: - master - euphrates - - danube log: log_file: reporting.log @@ -38,6 +37,8 @@ functest: blacklist: - odl_netvirt - juju_epc + - tempest_full_parallel + - rally_full max_scenario_criteria: 50 test_conf: https://git.opnfv.org/cgit/functest/plain/functest/ci/testcases.yaml log_level: ERROR diff --git a/utils/test/reporting/reporting/utils/reporting_utils.py b/utils/test/reporting/reporting/utils/reporting_utils.py index 65267ca11..58a0c6233 100644 --- a/utils/test/reporting/reporting/utils/reporting_utils.py +++ b/utils/test/reporting/reporting/utils/reporting_utils.py @@ -6,7 +6,6 @@ # # http://www.apache.org/licenses/LICENSE-2.0 # -from urllib2 import Request, urlopen, URLError import logging import json import os @@ -14,6 +13,8 @@ import requests import pdfkit import yaml +from urllib2 import Request, urlopen, URLError + # ---------------------------------------------------------- # @@ -113,7 +114,8 @@ def getScenarios(project, case, installer, version): """ Get the list of Scenarios """ - + test_results = None + scenario_results = None period = get_config('general.period') url_base = get_config('testapi.url') @@ -284,7 +286,7 @@ def getNbtestOk(results): return nb_test_ok -def getResult(testCase, installer, scenario, version): +def getCaseScore(testCase, installer, scenario, version): """ Get Result for a given Functest Testcase """ @@ -343,6 +345,41 @@ def getResult(testCase, installer, scenario, version): return test_result_indicator +def getCaseScoreFromBuildTag(testCase, s_results): + """ + Get Results for a given Functest Testcase with arch filtering + """ + url_base = get_config('testapi.url') + nb_tests = get_config('general.nb_iteration_tests_success_criteria') + test_result_indicator = 0 + # architecture is not a result field...so we cannot use getResult as it is + res_matrix = [] + try: + for s_result in s_results: + build_tag = s_result['build_tag'] + d = s_result['start_date'] + res_matrix.append({'date': d, + 'build_tag': build_tag}) + # sort res_matrix + filter_res_matrix = sorted(res_matrix, key=lambda k: k['date'], + reverse=True)[:nb_tests] + for my_res in filter_res_matrix: + url = ("http://" + url_base + "?case=" + testCase + + "&build_tag=" + my_res['build_tag']) + request = Request(url) + response = urlopen(request) + k = response.read() + results = json.loads(k) + if "PASS" in results['results'][0]['criteria']: + test_result_indicator += 1 + except: + print "No results found for this case" + if test_result_indicator > 2: + test_result_indicator = test_result_indicator - 1 + + return test_result_indicator + + def getJenkinsUrl(build_tag): """ Get Jenkins url_base corespoding to the last test CI run diff --git a/utils/test/testapi/etc/config.ini b/utils/test/testapi/etc/config.ini index 8d0bde20b..86cb0caa7 100644 --- a/utils/test/testapi/etc/config.ini +++ b/utils/test/testapi/etc/config.ini @@ -16,7 +16,8 @@ results_per_page = 20 # With debug_on set to true, error traces will be shown in HTTP responses debug = True -authenticate = False +token_check = False +authenticate = True [ui] url = http://localhost:8000 diff --git a/utils/test/testapi/opnfv_testapi/cmd/server.py b/utils/test/testapi/opnfv_testapi/cmd/server.py index b7d3caa20..011a6cd6e 100644 --- a/utils/test/testapi/opnfv_testapi/cmd/server.py +++ b/utils/test/testapi/opnfv_testapi/cmd/server.py @@ -42,7 +42,7 @@ def make_app(): return swagger.Application( url_mappings.mappings, debug=CONF.api_debug, - auth=CONF.api_authenticate, + auth=CONF.api_token_check, cookie_secret='opnfv-testapi', ) diff --git a/utils/test/testapi/opnfv_testapi/common/check.py b/utils/test/testapi/opnfv_testapi/common/check.py index e80b1c6b7..667578fed 100644 --- a/utils/test/testapi/opnfv_testapi/common/check.py +++ b/utils/test/testapi/opnfv_testapi/common/check.py @@ -14,13 +14,14 @@ from tornado import gen from opnfv_testapi.common import constants from opnfv_testapi.common import message from opnfv_testapi.common import raises +from opnfv_testapi.common.config import CONF from opnfv_testapi.db import api as dbapi def is_authorized(method): @functools.wraps(method) def wrapper(self, *args, **kwargs): - if self.table in ['pods']: + if CONF.api_authenticate and self.table in ['pods']: testapi_id = self.get_secure_cookie(constants.TESTAPI_ID) if not testapi_id: raises.Unauthorized(message.not_login()) @@ -102,6 +103,19 @@ def carriers_exist(xstep): return wrap +def values_check(xstep): + @functools.wraps(xstep) + def wrap(self, *args, **kwargs): + checks = kwargs.pop('values_check', {}) + if checks: + for field, check, options in checks: + if not check(field, options): + raises.BadRequest(message.invalid_value(field, options)) + ret = yield gen.coroutine(xstep)(self, *args, **kwargs) + raise gen.Return(ret) + return wrap + + def new_not_exists(xstep): @functools.wraps(xstep) def wrap(self, *args, **kwargs): diff --git a/utils/test/testapi/opnfv_testapi/common/message.py b/utils/test/testapi/opnfv_testapi/common/message.py index 8b5c3fb7a..3e14f7258 100644 --- a/utils/test/testapi/opnfv_testapi/common/message.py +++ b/utils/test/testapi/opnfv_testapi/common/message.py @@ -26,6 +26,10 @@ def missing(name): return '{} Missing'.format(name) +def invalid_value(name, options): + return '{} must be in {}'.format(name, options) + + def exist(key, value): return '{} [{}] {}'.format(key, value, exist_base) diff --git a/utils/test/testapi/opnfv_testapi/resources/handlers.py b/utils/test/testapi/opnfv_testapi/resources/handlers.py index 8e5dab235..6c7a819c9 100644 --- a/utils/test/testapi/opnfv_testapi/resources/handlers.py +++ b/utils/test/testapi/opnfv_testapi/resources/handlers.py @@ -79,6 +79,7 @@ class GenericApiHandler(web.RequestHandler): @check.valid_token @check.no_body @check.miss_fields + @check.values_check @check.carriers_exist @check.new_not_exists def _create(self, **kwargs): diff --git a/utils/test/testapi/opnfv_testapi/resources/result_handlers.py b/utils/test/testapi/opnfv_testapi/resources/result_handlers.py index e202f5c2c..a25852856 100644 --- a/utils/test/testapi/opnfv_testapi/resources/result_handlers.py +++ b/utils/test/testapi/opnfv_testapi/resources/result_handlers.py @@ -215,12 +215,18 @@ class ResultsCLHandler(GenericResultHandler): return {'project_name': self.json_args.get('project_name'), 'name': self.json_args.get('case_name')} + def options_check(field, options): + return self.json_args.get(field).upper() in options + miss_fields = ['pod_name', 'project_name', 'case_name'] carriers = [('pods', pod_query), ('projects', project_query), ('testcases', testcase_query)] + values_check = [('criteria', options_check, ['PASS', 'FAIL'])] - self._create(miss_fields=miss_fields, carriers=carriers) + self._create(miss_fields=miss_fields, + carriers=carriers, + values_check=values_check) class ResultsUploadHandler(ResultsCLHandler): diff --git a/utils/test/testapi/opnfv_testapi/tests/unit/common/test_config.py b/utils/test/testapi/opnfv_testapi/tests/unit/common/test_config.py index ea2297275..6d160ce1d 100644 --- a/utils/test/testapi/opnfv_testapi/tests/unit/common/test_config.py +++ b/utils/test/testapi/opnfv_testapi/tests/unit/common/test_config.py @@ -12,7 +12,8 @@ def test_config_normal(mocker, config_normal): assert CONF.mongo_dbname == 'test_results_collection' assert CONF.api_port == 8000 assert CONF.api_debug is True - assert CONF.api_authenticate is False + assert CONF.api_token_check is False + assert CONF.api_authenticate is True assert CONF.ui_url == 'http://localhost:8000' diff --git a/utils/test/testapi/opnfv_testapi/tests/unit/resources/test_result.py b/utils/test/testapi/opnfv_testapi/tests/unit/resources/test_result.py index 1df31f36c..6c1a07a91 100644 --- a/utils/test/testapi/opnfv_testapi/tests/unit/resources/test_result.py +++ b/utils/test/testapi/opnfv_testapi/tests/unit/resources/test_result.py @@ -62,7 +62,7 @@ class TestResultBase(base.TestBase): self.version = 'C' self.build_tag = 'v3.0' self.scenario = 'odl-l2' - self.criteria = 'passed' + self.criteria = 'PASS' self.trust_indicator = result_models.TI(0.7) self.start_date = str(datetime.now()) self.stop_date = str(datetime.now() + timedelta(minutes=1)) @@ -170,6 +170,13 @@ class TestResultCreate(TestResultBase): req.case_name = None return req + @executor.create(httplib.BAD_REQUEST, + message.invalid_value('criteria', ['PASS', 'FAIL'])) + def test_invalid_criteria(self): + req = self.req_d + req.criteria = 'invalid' + return req + @executor.create(httplib.FORBIDDEN, message.not_found_base) def test_noPod(self): req = self.req_d diff --git a/utils/test/testapi/opnfv_testapi/ui/auth/user.py b/utils/test/testapi/opnfv_testapi/ui/auth/user.py index ab86007f1..ff2c2a993 100644 --- a/utils/test/testapi/opnfv_testapi/ui/auth/user.py +++ b/utils/test/testapi/opnfv_testapi/ui/auth/user.py @@ -1,5 +1,6 @@ from opnfv_testapi.common import constants from opnfv_testapi.common import raises +from opnfv_testapi.common.config import CONF from opnfv_testapi.resources import handlers from opnfv_testapi.resources import models @@ -19,8 +20,14 @@ class UserHandler(handlers.GenericApiHandler): self.table_cls = User def get(self): - username = self.get_secure_cookie(constants.TESTAPI_ID) - if username: - self._get_one(query={'user': username}) + if CONF.api_authenticate: + username = self.get_secure_cookie(constants.TESTAPI_ID) + if username: + self._get_one(query={'user': username}) + else: + raises.Unauthorized('Unauthorized') else: - raises.Unauthorized('Unauthorized') + self.finish_request(User('anonymous', + 'anonymous@linuxfoundation.com', + 'anonymous lf', + constants.TESTAPI_USERS).format()) |