From 8afb6df913ce350090b8374b652ac9ca07c0d077 Mon Sep 17 00:00:00 2001 From: Guy Rodrigue Koffi Date: Thu, 1 Oct 2015 10:56:55 +0200 Subject: Updates for result_collection_api - pod identification : RELENG-33 - add filters on results : RELENG-32 / RELENG-31 - fix bugs on db connection Change-Id: Ie720bb8028e5bfabd914c57df323b81d40c47fcd Signed-off-by: Guy Rodrigue Koffi --- utils/test/result_collection_api/common/config.py | 6 ++ utils/test/result_collection_api/config.ini | 8 +- .../result_collection_api/resources/handlers.py | 120 ++++++++++++++++++--- .../test/result_collection_api/resources/models.py | 32 ++++-- .../result_collection_api/result_collection_api.py | 7 +- 5 files changed, 143 insertions(+), 30 deletions(-) (limited to 'utils/test') diff --git a/utils/test/result_collection_api/common/config.py b/utils/test/result_collection_api/common/config.py index a0d0757a5..a62e80593 100644 --- a/utils/test/result_collection_api/common/config.py +++ b/utils/test/result_collection_api/common/config.py @@ -47,6 +47,7 @@ class APIConfig: def __init__(self): self._default_config_location = "config.ini" self.mongo_url = None + self.mongo_dbname = None self.api_port = None self.api_debug_on = None self._parser = None @@ -87,13 +88,18 @@ class APIConfig: # Linking attributes to keys from file with their sections obj.mongo_url = obj._get_parameter("mongo", "url") + obj.mongo_dbname = obj._get_parameter("mongo", "dbname") + obj.api_port = obj._get_int_parameter("api", "port") obj.api_debug_on = obj._get_bool_parameter("api", "debug") + return obj def __str__(self): return "mongo_url = %s \n" \ + "mongo_dbname = %s \n" \ "api_port = %s \n" \ "api_debug_on = %s \n" % (self.mongo_url, + self.mongo_dbname, self.api_port, self.api_debug_on) diff --git a/utils/test/result_collection_api/config.ini b/utils/test/result_collection_api/config.ini index e00b56c5f..f703cc6c4 100644 --- a/utils/test/result_collection_api/config.ini +++ b/utils/test/result_collection_api/config.ini @@ -1,10 +1,12 @@ +# to add a new parameter in the config file, +# the CONF object in config.ini must be updated [mongo] # URL of the mongo DB # Mongo auth url => mongodb://user1:pwd1@host1/?authSource=db1 -url = mongodb://127.0.0.1:27017/test_results_collection - +url = mongodb://127.0.0.1:27017/ +dbname = test_results_collection [api] # Listening port -port = 80 +port = 8000 # With debug_on set to true, error traces will be shown in HTTP responses debug = True \ No newline at end of file diff --git a/utils/test/result_collection_api/resources/handlers.py b/utils/test/result_collection_api/resources/handlers.py index 3faba5aeb..f8b26d090 100644 --- a/utils/test/result_collection_api/resources/handlers.py +++ b/utils/test/result_collection_api/resources/handlers.py @@ -11,7 +11,7 @@ import json from tornado.web import RequestHandler, asynchronous, HTTPError from tornado import gen -from datetime import datetime +from datetime import datetime, timedelta from models import Pod, TestProject, TestCase, TestResult from common.constants import DEFAULT_REPRESENTATION, HTTP_BAD_REQUEST, \ @@ -60,6 +60,8 @@ class PodHandler(GenericApiHandler): """ Handle the requests about the POD Platforms HTTP Methdods : - GET : Get PODS + - POST : Create a pod + - DELETE : DELETE POD """ def initialize(self): @@ -68,19 +70,15 @@ class PodHandler(GenericApiHandler): @asynchronous @gen.coroutine - def get(self, pod_id=None): + def get(self, pod_name=None): """ Get all pods or a single pod :param pod_id: """ - - if pod_id is None: - pod_id = "" - get_request = dict() - if len(pod_id) > 0: - get_request["_id"] = int(pod_id) + if pod_name is not None: + get_request["name"] = pod_name res = [] cursor = self.db.pod.find(get_request) @@ -98,6 +96,69 @@ class PodHandler(GenericApiHandler): self.finish_request(answer) + @asynchronous + @gen.coroutine + def post(self): + """ Create a POD""" + + if self.json_args is None: + raise HTTPError(HTTP_BAD_REQUEST) + + query = {"name": self.json_args.get("name")} + + # check for existing name in db + mongo_dict = yield self.db.pod.find_one(query) + if mongo_dict is not None: + raise HTTPError(HTTP_FORBIDDEN, + "{} already exists as a pod".format( + self.json_args.get("name"))) + + pod = Pod.pod_from_dict(self.json_args) + pod.creation_date = datetime.now() + + future = self.db.pod.insert(pod.format()) + result = yield future + pod._id = result + + meta = dict() + meta["success"] = True + meta["uri"] = "/pods/{}".format(pod.name) + + answer = dict() + answer["pod"] = pod.format_http() + answer["meta"] = meta + + self.finish_request(answer) + + @asynchronous + @gen.coroutine + def delete(self, pod_name): + """ Remove a POD + + # check for an existing pod to be deleted + mongo_dict = yield self.db.pod.find_one( + {'name': pod_name}) + pod = TestProject.pod(mongo_dict) + if pod is None: + raise HTTPError(HTTP_NOT_FOUND, + "{} could not be found as a pod to be deleted" + .format(pod_name)) + + # just delete it, or maybe save it elsewhere in a future + res = yield self.db.test_projects.remove( + {'name': pod_name}) + + meta = dict() + meta["success"] = True + meta["deletion-data"] = res + + answer = dict() + answer["meta"] = meta + + self.finish_request(answer) + """ + pass + class TestProjectHandler(GenericApiHandler): """ @@ -440,18 +501,26 @@ class TestResultsHandler(GenericApiHandler): Available filters for this request are : - project : project name - case : case name - - pod : pod ID + - pod : pod name + - version : platform version (Arno-R1, ...) + - installer (fuel, ...) + - period : x (x last days) + :param result_id: Get a result by ID :raise HTTPError - GET /results/project=functest&case=keystone.catalog&pod=1 + GET /results/project=functest&case=vPing&version=Arno-R1 \ + &pod=pod_name&period=15 => get results with optional filters """ project_arg = self.get_query_argument("project", None) case_arg = self.get_query_argument("case", None) pod_arg = self.get_query_argument("pod", None) + version_arg = self.get_query_argument("version", None) + installer_arg = self.get_query_argument("installer", None) + period_arg = self.get_query_argument("period", None) # prepare request get_request = dict() @@ -463,15 +532,34 @@ class TestResultsHandler(GenericApiHandler): get_request["case_name"] = case_arg if pod_arg is not None: - get_request["pod_id"] = int(pod_arg) + get_request["pod_name"] = pod_arg + + if version_arg is not None: + get_request["version"] = version_arg + + if installer_arg is not None: + get_request["installer"] = installer_arg + + if period_arg is not None: + try: + period_arg = int(period_arg) + except: + raise HTTPError(HTTP_BAD_REQUEST) + + if period_arg > 0: + period = datetime.now() - timedelta(days=period_arg) + obj = {"$gte": period} + get_request["creation_date"] = obj else: get_request["_id"] = result_id + print get_request res = [] # fetching results cursor = self.db.test_results.find(get_request) while (yield cursor.fetch_next): - test_result = TestResult.test_result_from_dict(cursor.next_object()) + test_result = TestResult.test_result_from_dict( + cursor.next_object()) res.append(test_result.format_http()) # building meta object @@ -502,7 +590,9 @@ class TestResultsHandler(GenericApiHandler): raise HTTPError(HTTP_BAD_REQUEST) if self.json_args.get("case_name") is None: raise HTTPError(HTTP_BAD_REQUEST) - if self.json_args.get("pod_id") is None: + # check for pod_name instead of id, + # keeping id for current implementations + if self.json_args.get("pod_name") is None: raise HTTPError(HTTP_BAD_REQUEST) # TODO : replace checks with jsonschema @@ -524,11 +614,11 @@ class TestResultsHandler(GenericApiHandler): # check for pod mongo_dict = yield self.db.pod.find_one( - {"_id": self.json_args.get("pod_id")}) + {"name": self.json_args.get("pod_name")}) if mongo_dict is None: raise HTTPError(HTTP_NOT_FOUND, "Could not find POD [{}] " - .format(self.json_args.get("pod_id"))) + .format(self.json_args.get("pod_name"))) # convert payload to object test_result = TestResult.test_result_from_dict(self.json_args) diff --git a/utils/test/result_collection_api/resources/models.py b/utils/test/result_collection_api/resources/models.py index 82c17fda3..3b4d843f3 100644 --- a/utils/test/result_collection_api/resources/models.py +++ b/utils/test/result_collection_api/resources/models.py @@ -22,13 +22,19 @@ class Pod: p = Pod() p._id = pod_dict.get('_id') - p.creation_date = pod_dict.get('creation_date') + p.creation_date = str(pod_dict.get('creation_date')) p.name = pod_dict.get('name') return p def format(self): return { - "_id": self._id, + "name": self.name, + "creation_date": str(self.creation_date), + } + + def format_http(self): + return { + "_id": str(self._id), "name": self.name, "creation_date": str(self.creation_date), } @@ -82,6 +88,7 @@ class TestCase: self.name = None self.project_name = None self.description = None + self.url = None self.creation_date = None @staticmethod @@ -96,6 +103,7 @@ class TestCase: t.creation_date = testcase_dict.get('creation_date') t.name = testcase_dict.get('name') t.description = testcase_dict.get('description') + t.url = testcase_dict.get('url') return t @@ -104,7 +112,8 @@ class TestCase: "name": self.name, "description": self.description, "project_name": self.project_name, - "creation_date": str(self.creation_date) + "creation_date": str(self.creation_date), + "url": self.url } def format_http(self, test_project=None): @@ -113,6 +122,7 @@ class TestCase: "name": self.name, "description": self.description, "creation_date": str(self.creation_date), + "url": self.url, } if test_project is not None: res["test_project"] = test_project @@ -127,9 +137,9 @@ class TestResult: self._id = None self.case_name = None self.project_name = None - self.pod_id = None + self.pod_name = None self.installer = None - self.platform_version = None + self.version = None self.description = None self.creation_date = None self.details = None @@ -143,11 +153,13 @@ class TestResult: t = TestResult() t._id = test_result_dict.get('_id') t.case_name = test_result_dict.get('case_name') + t.pod_name = test_result_dict.get('pod_name') t.project_name = test_result_dict.get('project_name') - t.pod_id = test_result_dict.get('pod_id') t.description = test_result_dict.get('description') t.creation_date = str(test_result_dict.get('creation_date')) t.details = test_result_dict.get('details') + t.version = test_result_dict.get('version') + t.installer = test_result_dict.get('installer') return t @@ -155,9 +167,11 @@ class TestResult: return { "case_name": self.case_name, "project_name": self.project_name, - "pod_id": self.pod_id, + "pod_name": self.pod_name, "description": self.description, "creation_date": str(self.creation_date), + "version": self.version, + "installer": self.installer, "details": self.details, } @@ -166,8 +180,10 @@ class TestResult: "_id": str(self._id), "case_name": self.case_name, "project_name": self.project_name, - "pod_id": self.pod_id, + "pod_name": self.pod_name, "description": self.description, "creation_date": str(self.creation_date), + "version": self.version, + "installer": self.installer, "details": self.details, } diff --git a/utils/test/result_collection_api/result_collection_api.py b/utils/test/result_collection_api/result_collection_api.py index c04e0343b..49695772d 100644 --- a/utils/test/result_collection_api/result_collection_api.py +++ b/utils/test/result_collection_api/result_collection_api.py @@ -15,14 +15,12 @@ Pre-requisites: We can launch the API with this file TODOs : - - use POD name instead of id - logging - json args validation with schemes - POST/PUT/DELETE for PODs - POST/PUT/GET/DELETE for installers, platforms (enrich results info) - count cases for GET on test_projects - count results for GET on cases - - provide filtering on requests - include objects - swagger documentation - setup file @@ -48,7 +46,8 @@ args = parser.parse_args() CONF = APIConfig().parse(args.config_file) # connecting to MongoDB server, and choosing database -db = motor.MotorClient(CONF.mongo_url) +client = motor.MotorClient(CONF.mongo_url) +db = client[CONF.mongo_dbname] def make_app(): @@ -61,7 +60,7 @@ def make_app(): # GET /pods => Get all pods # GET /pods/1 => Get details on POD 1 (r"/pods", PodHandler), - (r"/pods/(\d*)", PodHandler), + (r"/pods/([^/]+)", PodHandler), # few examples: # GET /test_projects -- cgit 1.2.3-korg