diff options
Diffstat (limited to 'utils/test')
9 files changed, 122 insertions, 37 deletions
diff --git a/utils/test/reporting/functest/reporting-tempest.py b/utils/test/reporting/functest/reporting-tempest.py index 70fb57702..e9662d5c2 100644 --- a/utils/test/reporting/functest/reporting-tempest.py +++ b/utils/test/reporting/functest/reporting-tempest.py @@ -7,10 +7,10 @@ installers = ["apex", "compass", "fuel", "joid"] items = ["tests", "Success rate", "duration"] PERIOD = 7 - +print "Generate Tempest automatic reporting" for installer in installers: # we consider the Tempest results of the last PERIOD days - url = "http://testresults.opnfv.org/testapi/results?case=Tempest" + url = "http://testresults.opnfv.org/test/api/v1/results?case=Tempest" request = Request(url + '&period=' + str(PERIOD) + '&installer=' + installer + '&version=master') @@ -21,7 +21,7 @@ for installer in installers: except URLError, e: print 'No kittez. Got an error code:', e - test_results = results['test_results'] + test_results = results['results'] test_results.reverse() scenario_results = {} @@ -41,7 +41,7 @@ for installer in installers: # For each scenario, we build a result object to deal with # results, criteria and error handling for result in scenario_results[s]: - result["creation_date"] = result["creation_date"].split(".")[0] + result["start_date"] = result["start_date"].split(".")[0] # retrieve results # **************** @@ -96,5 +96,7 @@ for installer in installers: items=items, installer=installer) - with open("./release/index-tempest-" + installer + ".html", "wb") as fh: + with open("./release/master/index-tempest-" + + installer + ".html", "wb") as fh: fh.write(outputText) +print "Tempest automatic reporting Done" diff --git a/utils/test/reporting/functest/reporting-vims.py b/utils/test/reporting/functest/reporting-vims.py index 78ca9f5b3..5ba92d6c3 100644 --- a/utils/test/reporting/functest/reporting-vims.py +++ b/utils/test/reporting/functest/reporting-vims.py @@ -24,7 +24,7 @@ installers = ["fuel", "compass", "joid", "apex"] step_order = ["initialisation", "orchestrator", "vIMS", "sig_test"] for installer in installers: - request = Request('http://testresults.opnfv.org/testapi/results?case=vIMS&installer=' + installer) + request = Request('http://testresults.opnfv.org/test/api/v1/results?case=vIMS&installer=' + installer) try: response = urlopen(request) @@ -33,7 +33,7 @@ for installer in installers: except URLError, e: print 'No kittez. Got an error code:', e - test_results = results['test_results'] + test_results = results['results'] test_results.reverse() scenario_results = {} @@ -45,7 +45,7 @@ for installer in installers: for s, s_result in scenario_results.items(): scenario_results[s] = s_result[0:5] for result in scenario_results[s]: - result["creation_date"] = result["creation_date"].split(".")[0] + result["start_date"] = result["start_date"].split(".")[0] sig_test = result['details']['sig_test']['result'] if not sig_test == "" and isinstance(sig_test, list): format_result = sig_test_format(sig_test) @@ -77,7 +77,7 @@ for installer in installers: outputText = template.render( scenario_results = scenario_results, step_order = step_order, installer = installer) - with open("./release/index-vims" + installer + ".html", "wb") as fh: + with open("./release/master/index-vims-" + installer + ".html", "wb") as fh: fh.write(outputText) diff --git a/utils/test/reporting/functest/reportingConf.py b/utils/test/reporting/functest/reportingConf.py index e4c4548c8..e390a18d2 100644 --- a/utils/test/reporting/functest/reportingConf.py +++ b/utils/test/reporting/functest/reportingConf.py @@ -11,7 +11,7 @@ # **************************************************** installers = ["apex", "compass", "fuel", "joid"] # installers = ["compass"] -versions = ["brahmaputra", "master"] -# versions = ["master"] +# versions = ["brahmaputra", "master"] +versions = ["master"] PERIOD = 10 MAX_SCENARIO_CRITERIA = 18 diff --git a/utils/test/reporting/functest/reportingUtils.py b/utils/test/reporting/functest/reportingUtils.py index f8d64aa88..7170b8398 100644 --- a/utils/test/reporting/functest/reportingUtils.py +++ b/utils/test/reporting/functest/reportingUtils.py @@ -19,7 +19,7 @@ def getApiResults(case, installer, scenario, version): # urllib2.install_opener(opener) # url = "http://127.0.0.1:8000/results?case=" + case + \ # "&period=30&installer=" + installer - url = ("http://testresults.opnfv.org/testapi/results?case=" + case + + url = ("http://testresults.opnfv.org/test/api/v1/results?case=" + case + "&period=" + str(reportingConf.PERIOD) + "&installer=" + installer + "&scenario=" + scenario + "&version=" + version) request = Request(url) @@ -37,7 +37,8 @@ def getApiResults(case, installer, scenario, version): def getScenarios(case, installer, version): case = case.getName() - url = ("http://testresults.opnfv.org/testapi/results?case=" + case + + print case + url = ("http://testresults.opnfv.org/test/api/v1/results?case=" + case + "&period=" + str(reportingConf.PERIOD) + "&installer=" + installer + "&version=" + version) request = Request(url) @@ -46,11 +47,10 @@ def getScenarios(case, installer, version): response = urlopen(request) k = response.read() results = json.loads(k) + test_results = results['results'] except URLError, e: print 'Got an error code:', e - test_results = results['test_results'] - if test_results is not None: test_results.reverse() @@ -78,7 +78,7 @@ def getNbtestOk(results): for r in results: for k, v in r.iteritems(): try: - if "passed" in v: + if "PASS" in v: nb_test_ok += 1 except: print "Cannot retrieve test status" @@ -90,7 +90,7 @@ def getResult(testCase, installer, scenario, version): # retrieve raw results results = getApiResults(testCase, installer, scenario, version) # let's concentrate on test results only - test_results = results['test_results'] + test_results = results['results'] # if results found, analyze them if test_results is not None: @@ -106,7 +106,7 @@ def getResult(testCase, installer, scenario, version): for r in test_results: # print r["creation_date"] # print r["criteria"] - scenario_results.append({r["creation_date"]: r["criteria"]}) + scenario_results.append({r["start_date"]: r["criteria"]}) # sort results scenario_results.sort() # 4 levels for the results diff --git a/utils/test/result_collection_api/opnfv_testapi/resources/handlers.py b/utils/test/result_collection_api/opnfv_testapi/resources/handlers.py index df920c48f..873701103 100644 --- a/utils/test/result_collection_api/opnfv_testapi/resources/handlers.py +++ b/utils/test/result_collection_api/opnfv_testapi/resources/handlers.py @@ -96,8 +96,10 @@ class GenericApiHandler(RequestHandler): code, message = error(data) raise HTTPError(code, message) - data.creation_date = datetime.now() - _id = yield self._eval_db(self.table, 'insert', data.format()) + if self.table != 'results': + data.creation_date = datetime.now() + _id = yield self._eval_db(self.table, 'insert', data.format(), + check_keys=False) if 'name' in self.json_args: resource = data.name else: @@ -173,7 +175,8 @@ class GenericApiHandler(RequestHandler): edit_request.update(self._update_requests(data)) """ Updating the DB """ - yield self._eval_db(self.table, 'update', query, edit_request) + yield self._eval_db(self.table, 'update', query, edit_request, + check_keys=False) edit_request['_id'] = str(data._id) self.finish_request(edit_request) @@ -214,8 +217,8 @@ class GenericApiHandler(RequestHandler): query[key] = new return equal, query - def _eval_db(self, table, method, *args): - return eval('self.db.%s.%s(*args)' % (table, method)) + def _eval_db(self, table, method, *args, **kwargs): + return eval('self.db.%s.%s(*args, **kwargs)' % (table, method)) def _eval_db_find_one(self, query, table=None): if table is None: diff --git a/utils/test/result_collection_api/opnfv_testapi/tests/unit/fake_pymongo.py b/utils/test/result_collection_api/opnfv_testapi/tests/unit/fake_pymongo.py index ef9c719be..6ab98c720 100644 --- a/utils/test/result_collection_api/opnfv_testapi/tests/unit/fake_pymongo.py +++ b/utils/test/result_collection_api/opnfv_testapi/tests/unit/fake_pymongo.py @@ -80,11 +80,15 @@ class MemDb(object): return_one = True docs = [docs] + if check_keys: + for doc in docs: + self._check_keys(doc) + ids = [] for doc in docs: if '_id' not in doc: doc['_id'] = str(ObjectId()) - if not check_keys or not self._find_one(doc['_id']): + if not self._find_one(doc['_id']): ids.append(doc['_id']) self.contents.append(doc_or_docs) @@ -131,8 +135,12 @@ class MemDb(object): def find(self, *args): return MemCursor(self._find(*args)) - def _update(self, spec, document): + def _update(self, spec, document, check_keys=True): updated = False + + if check_keys: + self._check_keys(document) + for index in range(len(self.contents)): content = self.contents[index] if self._in(content, spec): @@ -142,8 +150,8 @@ class MemDb(object): self.contents[index] = content return updated - def update(self, spec, document): - return thread_execute(self._update, spec, document) + def update(self, spec, document, check_keys=True): + return thread_execute(self._update, spec, document, check_keys) def _remove(self, spec_or_id=None): if spec_or_id is None: @@ -163,6 +171,17 @@ class MemDb(object): def clear(self): self._remove() + def _check_keys(self, doc): + for key in doc.keys(): + print('key', key, 'value', doc.get(key)) + if '.' in key: + raise NameError('key {} must not contain .'.format(key)) + if key.startswith('$'): + raise NameError('key {} must not start with $'.format(key)) + if isinstance(doc.get(key), dict): + self._check_keys(doc.get(key)) + + pods = MemDb() projects = MemDb() testcases = MemDb() diff --git a/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_fake_pymongo.py b/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_fake_pymongo.py index 9bc311cf3..27382f089 100644 --- a/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_fake_pymongo.py +++ b/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_fake_pymongo.py @@ -53,25 +53,70 @@ class MyTest(AsyncHTTPTestCase): user = yield self.db.pods.find_one({'_id': '1'}) self.assertEqual(user.get('name', None), 'new_test1') + def test_update_dot_error(self): + self._update_assert({'_id': '1', 'name': {'1. name': 'test1'}}, + 'key 1. name must not contain .') + + def test_update_dot_no_error(self): + self._update_assert({'_id': '1', 'name': {'1. name': 'test1'}}, + None, + check_keys=False) + + def test_update_dollar_error(self): + self._update_assert({'_id': '1', 'name': {'$name': 'test1'}}, + 'key $name must not start with $') + + def test_update_dollar_no_error(self): + self._update_assert({'_id': '1', 'name': {'$name': 'test1'}}, + None, + check_keys=False) + @gen_test def test_remove(self): yield self.db.pods.remove({'_id': '1'}) user = yield self.db.pods.find_one({'_id': '1'}) self.assertIsNone(user) - @gen_test - def test_insert_check_keys(self): - yield self.db.pods.insert({'_id': '1', 'name': 'test1'}, - check_keys=False) - cursor = self.db.pods.find({'_id': '1'}) - names = [] - while (yield cursor.fetch_next): - ob = cursor.next_object() - names.append(ob.get('name')) - self.assertItemsEqual(names, ['test1', 'test1']) + def test_insert_dot_error(self): + self._insert_assert({'_id': '1', '2. name': 'test1'}, + 'key 2. name must not contain .') + + def test_insert_dot_no_error(self): + self._insert_assert({'_id': '1', '2. name': 'test1'}, + None, + check_keys=False) + + def test_insert_dollar_error(self): + self._insert_assert({'_id': '1', '$name': 'test1'}, + 'key $name must not start with $') + + def test_insert_dollar_no_error(self): + self._insert_assert({'_id': '1', '$name': 'test1'}, + None, + check_keys=False) def _clear(self): self.db.pods.clear() + def _update_assert(self, docs, error=None, **kwargs): + self._db_assert('update', error, {'_id': '1'}, docs, **kwargs) + + def _insert_assert(self, docs, error=None, **kwargs): + self._db_assert('insert', error, docs, **kwargs) + + @gen_test + def _db_assert(self, method, error, *args, **kwargs): + name_error = None + try: + yield self._eval_pods_db(method, *args, **kwargs) + except NameError as err: + name_error = err.args[0] + finally: + self.assertEqual(name_error, error) + + def _eval_pods_db(self, method, *args, **kwargs): + return eval('self.db.pods.%s(*args, **kwargs)' % method) + + if __name__ == '__main__': unittest.main() diff --git a/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_result.py b/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_result.py index dbc4431ce..bba3b228f 100644 --- a/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_result.py +++ b/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_result.py @@ -7,6 +7,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import unittest +import copy from opnfv_testapi.common.constants import HTTP_OK, HTTP_BAD_REQUEST, \ HTTP_NOT_FOUND @@ -161,6 +162,13 @@ class TestResultCreate(TestResultBase): self.assertEqual(code, HTTP_OK) self.assert_href(body) + def test_key_with_doc(self): + req = copy.deepcopy(self.req_d) + req.details = {'1.name': 'dot_name'} + (code, body) = self.create(req) + self.assertEqual(code, HTTP_OK) + self.assert_href(body) + class TestResultGet(TestResultBase): def test_getOne(self): diff --git a/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_testcase.py b/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_testcase.py index a145c00da..cb767844a 100644 --- a/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_testcase.py +++ b/utils/test/result_collection_api/opnfv_testapi/tests/unit/test_testcase.py @@ -7,6 +7,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## import unittest +import copy from test_base import TestBase from opnfv_testapi.resources.testcase_models import TestcaseCreateRequest, \ @@ -168,6 +169,13 @@ class TestCaseUpdate(TestCaseBase): self.assertEqual(_id, new_body._id) self.assert_update_body(self.req_d, new_body, self.update_e) + def test_with_dollar(self): + self.create_d() + update = copy.deepcopy(self.update_d) + update.description = {'2. change': 'dollar change'} + code, body = self.update(update, self.req_d.name) + self.assertEqual(code, HTTP_OK) + class TestCaseDelete(TestCaseBase): def test_notFound(self): |