diff options
author | 2016-11-10 10:59:26 +0800 | |
---|---|---|
committer | 2016-11-10 14:50:17 +0800 | |
commit | 9edb0f9e48cc923ed443d3d66886713cde9b628d (patch) | |
tree | 1463ca978963b6570ea143131d64d0b919a0b1f3 /qtip/api | |
parent | 939299c36b3cf79f198d4a4849345eabd878973b (diff) |
Adjust folder structure
Note: this patchset may break existing features, but it will help
to accelerate the refactoring work.
JIRA: QTIP-131
Change-Id: Ie0cd9d185e6b02316878daef905e26f4e533a66b
Signed-off-by: Yujun Zhang <zhang.yujunz@zte.com.cn>
Diffstat (limited to 'qtip/api')
-rw-r--r-- | qtip/api/__init__.py | 0 | ||||
-rw-r--r-- | qtip/api/db.py | 98 | ||||
-rw-r--r-- | qtip/api/qtip_server.py | 206 | ||||
-rw-r--r-- | qtip/api/result_handler.py | 22 |
4 files changed, 326 insertions, 0 deletions
diff --git a/qtip/api/__init__.py b/qtip/api/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/qtip/api/__init__.py diff --git a/qtip/api/db.py b/qtip/api/db.py new file mode 100644 index 00000000..24fc27a5 --- /dev/null +++ b/qtip/api/db.py @@ -0,0 +1,98 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from datetime import datetime +from operator import add +import uuid + +jobs = {} +threads = {} + + +def create_job(args): + if len(filter(lambda x: jobs[x]['state'] == 'processing', jobs.keys())) > 0: + return None + else: + job = {'job_id': str(uuid.uuid4()), + 'installer_type': args["installer_type"], + 'installer_ip': args["installer_ip"], + 'pod_name': args["pod_name"], + 'suite_name': args["suite_name"], + 'max_minutes': args["max_minutes"], + 'type': args["type"], + 'testdb_url': args["testdb_url"], + 'node_name': args["node_name"], + 'start_time': str(datetime.now()), + 'end_time': None, + 'state': 'processing', + 'state_detail': [], + 'result': None, + 'result_detail': []} + jobs[job['job_id']] = job + return job['job_id'] + + +def delete_job(job_id): + if job_id in threads: + stop_thread(job_id) + if job_id in jobs: + jobs[job_id]['end_time'] = str(datetime.now()) + jobs[job_id]['state'] = 'terminated' + return True + else: + return False + + +def get_job_info(job_id): + if job_id in jobs: + return jobs[job_id] + else: + return None + + +def finish_job(job_id): + jobs[job_id]['end_time'] = str(datetime.now()) + jobs[job_id]['state'] = 'finished' + jobs[job_id]['result'] = reduce(add, map(lambda x: x['result'], + jobs[job_id]['result_detail'])) + del threads[job_id] + + +def update_job_state_detail(job_id, state_detail): + jobs[job_id]['state_detail'] = state_detail + + +def update_job_result_detail(job_id, benchmark, result): + result['benchmark'] = benchmark + jobs[job_id]['result_detail'].append(result) + + +def is_job_timeout(job_id): + period = datetime.now() - datetime.strptime(jobs[job_id]['start_time'], + "%Y-%m-%d %H:%M:%S.%f") + return True if jobs[job_id]['max_minutes'] * 60 < period.total_seconds()\ + else False + + +def start_thread(job_id, thread, thread_stop): + threads[job_id] = {'thread': thread, + 'thread_stop': thread_stop} + thread.start() + + +def stop_thread(job_id): + if threads[job_id]['thread'].isAlive(): + threads[job_id]['thread_stop'].set() + threads[job_id]['thread'].join() + if job_id in threads: + del threads[job_id] + + +def update_benchmark_state(job_id, benchmark, benchmark_state): + filter(lambda x: x["benchmark"] == benchmark, + get_job_info(job_id)["state_detail"])[0]['state'] = benchmark_state diff --git a/qtip/api/qtip_server.py b/qtip/api/qtip_server.py new file mode 100644 index 00000000..537b2c05 --- /dev/null +++ b/qtip/api/qtip_server.py @@ -0,0 +1,206 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +from flask import Flask, abort +from flask_restful import Api, Resource, fields, reqparse +from flask_restful_swagger import swagger +import threading +from copy import copy +import db +import utils.args_handler as args_handler +import restful_server.result_handler as result_handler + + +app = Flask(__name__) +api = swagger.docs(Api(app), apiVersion='0.1') + + +@swagger.model +class JobModel: + resource_fields = { + 'installer_type': fields.String, + 'installer_ip': fields.String, + 'max_minutes': fields.Integer, + 'pod_name': fields.String, + 'suite_name': fields.String, + 'type': fields.String, + 'benchmark_name': fields.String, + 'testdb_url': fields.String, + 'node_name': fields.String + } + required = ['installer_type', 'installer_ip'] + + +@swagger.model +class JobResponseModel: + resource_fields = { + 'job_id': fields.String + } + + +class Job(Resource): + @swagger.operation( + notes='get a job by ID', + nickname='get', + parameters=[], + responseMessages=[ + { + "code": 200, + "message": "Job detail info." + }, + { + "code": 404, + "message": "Can't not find the job id XXXXXXX" + } + ] + ) + def get(self, id): + ret = db.get_job_info(id) + return ret if ret else abort(404, " Can't not find the job id %s" % id) + + @swagger.operation( + notes='delete a job by ID', + nickname='delete', + parameters=[], + responseMessages=[ + { + "code": 200, + "message": "Delete successfully" + }, + { + "code": 404, + "message": "Can not find job_id XXXXXXXXX" + } + ] + ) + def delete(self, id): + ret = db.delete_job(id) + return {'result': "Delete successfully"} if ret else abort(404, "Can not find job_id %s" % id) + + +class JobList(Resource): + @swagger.operation( + note='create a job with parameters', + nickname='create', + parameters=[ + { + "name": "body", + "description": """ +"installer_type": The installer type, for example fuel, compass.., + +"installer_ip": The installer ip of the pod, + +"max_minutes": If specified, the maximum duration in minutes +for any single test iteration, default is '60', + +"pod_name": If specified, the Pod name, default is 'default', + +"suite_name": If specified, Test suite name, for example 'compute', 'network', 'storage', +default is 'compute', + +"type": BM or VM,default is 'BM', + +"benchmark_name": If specified, benchmark name in suite, for example 'dhrystone_bm.yaml', +default is all benchmarks in suite with specified type, + +"testdb_url": test db http url, for example 'http://testresults.opnfv.org/test/api/v1', + +"node_name": node name reported to test db + """, + "required": True, + "type": "JobModel", + "paramType": "body" + } + ], + type=JobResponseModel.__name__, + responseMessages=[ + { + "code": 200, + "message": "Job submitted" + }, + { + "code": 400, + "message": "Missing configuration data" + }, + { + "code": 409, + "message": "It already has one job running now!" + } + ] + ) + def post(self): + parser = reqparse.RequestParser() + parser.add_argument('installer_type', type=str, required=True, help='installer_type is required') + parser.add_argument('installer_ip', type=str, required=True, help='installer_ip is required') + parser.add_argument('max_minutes', type=int, required=False, default=60, help='max_minutes should be integer') + parser.add_argument('pod_name', type=str, required=False, default='default', help='pod_name should be string') + parser.add_argument('suite_name', type=str, required=False, default='compute', help='suite_name should be string') + parser.add_argument('type', type=str, required=False, default='BM', help='type should be BM, VM and ALL') + parser.add_argument('benchmark_name', type=str, required=False, default='all', help='benchmark_name should be string') + parser.add_argument('testdb_url', type=str, required=False, default=None, + help='testdb_url should be test db http url,for example http://testresults.opnfv.org/test/api/v1') + parser.add_argument('node_name', type=str, required=False, default=None, help='node_name should be string') + args = parser.parse_args() + if not args_handler.check_suite(args["suite_name"]): + return abort(404, 'message:Test suite {0} does not exist under benchmarks/suite'.format(args["suite_name"])) + if not args_handler.check_lab_name(args["pod_name"]): + return abort(404, 'message: You have specified a lab {0}\ + that is not present in test_cases'.format(args['pod_name'])) + + job_id = db.create_job(args) + if not job_id: + return abort(409, 'message:It already has one job running now!') + + benchmarks = args_handler.get_files_in_suite(args["suite_name"], + args["type"].lower()) + test_cases = args_handler.get_files_in_test_case(args["pod_name"], + args["suite_name"], + args["type"].lower()) + benchmarks_list = filter(lambda x: x in test_cases, benchmarks) + if args["benchmark_name"] in benchmarks_list: + benchmarks_list = [args["benchmark_name"]] + if (args["benchmark_name"] is not 'all') and args["benchmark_name"] not in benchmarks_list: + return abort(404, 'message: Benchmark name {0} does not exist in suit {1}'.format(args["benchmark_name"], + args["suite_name"])) + state_detail = map(lambda x: {'benchmark': x, 'state': 'idle'}, benchmarks_list) + db.update_job_state_detail(job_id, copy(state_detail)) + thread_stop = threading.Event() + post_thread = threading.Thread(target=self.thread_post, args=(args["installer_type"], + benchmarks_list, + args["pod_name"], + args["suite_name"], + job_id, + args["testdb_url"], + args["node_name"], + thread_stop)) + db.start_thread(job_id, post_thread, thread_stop) + return {'job_id': str(job_id)} + + def thread_post(self, installer_type, benchmarks_list, pod_name, suite_name, + job_id, testdb_url, node_name, stop_event): + for benchmark in benchmarks_list: + if db.is_job_timeout(job_id) or stop_event.is_set(): + break + db.update_benchmark_state(job_id, benchmark, 'processing') + result = args_handler.prepare_and_run_benchmark(installer_type, + '/home', + args_handler.get_benchmark_path(pod_name, + suite_name, + benchmark)) + db.update_job_result_detail(job_id, benchmark, copy(result)) + db.update_benchmark_state(job_id, benchmark, 'finished') + if (result_handler.dump_suite_result(suite_name) and testdb_url): + result_handler.push_suite_result_to_db(suite_name, testdb_url, installer_type, node_name) + db.finish_job(job_id) + + +api.add_resource(JobList, '/api/v1.0/jobs') +api.add_resource(Job, '/api/v1.0/jobs/<string:id>') + +if __name__ == "__main__": + app.run(host='0.0.0.0') diff --git a/qtip/api/result_handler.py b/qtip/api/result_handler.py new file mode 100644 index 00000000..200330cb --- /dev/null +++ b/qtip/api/result_handler.py @@ -0,0 +1,22 @@ +############################################################################## +# Copyright (c) 2016 ZTE Corp and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +import json +import scripts.ref_results.suite_result as suite_result +import dashboard.pushtoDB as push_to_db + + +def dump_suite_result(suite_name): + return suite_result.get_suite_result(suite_name) + + +def push_suite_result_to_db(suite_name, test_db_url, installer_type, node_name): + with open('results/{0}_result.json'.format(suite_name), 'r') as result_file: + j = json.load(result_file) + push_to_db.push_results_to_db(test_db_url, '{0}_test_suite'.format(suite_name), + j, installer_type, node_name) |