diff options
| -rw-r--r-- | functest/api/base.py | 4 | ||||
| -rw-r--r-- | functest/api/common/api_utils.py | 10 | ||||
| -rw-r--r-- | functest/api/common/error.py | 24 | ||||
| -rw-r--r-- | functest/api/common/thread.py | 52 | ||||
| -rw-r--r-- | functest/api/database/__init__.py | 0 | ||||
| -rw-r--r-- | functest/api/database/db.py | 26 | ||||
| -rw-r--r-- | functest/api/database/v1/__init__.py | 0 | ||||
| -rw-r--r-- | functest/api/database/v1/handlers.py | 43 | ||||
| -rw-r--r-- | functest/api/database/v1/models.py | 33 | ||||
| -rw-r--r-- | functest/api/resources/v1/creds.py | 38 | ||||
| -rw-r--r-- | functest/api/resources/v1/envs.py | 8 | ||||
| -rw-r--r-- | functest/api/resources/v1/tasks.py | 58 | ||||
| -rw-r--r-- | functest/api/resources/v1/testcases.py | 69 | ||||
| -rw-r--r-- | functest/api/server.py | 50 | ||||
| -rw-r--r-- | functest/api/urls.py | 16 | ||||
| -rw-r--r-- | functest/utils/openstack_utils.py | 9 |
16 files changed, 403 insertions, 37 deletions
diff --git a/functest/api/base.py b/functest/api/base.py index efeab8243..ffc567860 100644 --- a/functest/api/base.py +++ b/functest/api/base.py @@ -17,7 +17,7 @@ import logging from flask import request from flask_restful import Resource -from functest.api.common import api_utils, error +from functest.api.common import api_utils LOGGER = logging.getLogger(__name__) @@ -58,7 +58,7 @@ class ApiResource(Resource): try: return getattr(self, action)(args) except AttributeError: - error.result_handler(status=1, data='No such action') + api_utils.result_handler(status=1, data='No such action') # Import modules from package "functest.api.resources" diff --git a/functest/api/common/api_utils.py b/functest/api/common/api_utils.py index f518e777c..d85acf927 100644 --- a/functest/api/common/api_utils.py +++ b/functest/api/common/api_utils.py @@ -18,6 +18,7 @@ import os import sys from oslo_utils import importutils +from flask import jsonify import six import functest @@ -89,3 +90,12 @@ def change_obj_to_dict(obj): for key, value in vars(obj).items(): dic.update({key: value}) return dic + + +def result_handler(status, data): + """ Return the json format of result in dict """ + result = { + 'status': status, + 'result': data + } + return jsonify(result) diff --git a/functest/api/common/error.py b/functest/api/common/error.py deleted file mode 100644 index d00452252..000000000 --- a/functest/api/common/error.py +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python - -# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 - -""" -Used to handle results - -""" - -from flask import jsonify - - -def result_handler(status, data): - """ Return the json format of result in dict """ - result = { - 'status': status, - 'result': data - } - return jsonify(result) diff --git a/functest/api/common/thread.py b/functest/api/common/thread.py new file mode 100644 index 000000000..fb60aaac7 --- /dev/null +++ b/functest/api/common/thread.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +""" +Used to handle multi-thread tasks +""" + +import logging +import threading + +from oslo_serialization import jsonutils + + +LOGGER = logging.getLogger(__name__) + + +class TaskThread(threading.Thread): + """ Task Thread Class """ + + def __init__(self, target, args, handler): + super(TaskThread, self).__init__(target=target, args=args) + self.target = target + self.args = args + self.handler = handler + + def run(self): + """ Override the function run: run testcase and update database """ + update_data = {'task_id': self.args.get('task_id'), + 'status': 'IN PROGRESS'} + self.handler.insert(update_data) + + LOGGER.info('Starting running test case') + + try: + data = self.target(self.args) + except Exception as err: # pylint: disable=broad-except + LOGGER.exception('Task Failed') + update_data = {'status': 'FAIL', 'error': str(err)} + self.handler.update_attr(self.args.get('task_id'), update_data) + else: + LOGGER.info('Task Finished') + LOGGER.debug('Result: %s', data) + new_data = {'status': 'FINISHED', + 'result': jsonutils.dumps(data.get('result', {}))} + + self.handler.update_attr(self.args.get('task_id'), new_data) diff --git a/functest/api/database/__init__.py b/functest/api/database/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/functest/api/database/__init__.py diff --git a/functest/api/database/db.py b/functest/api/database/db.py new file mode 100644 index 000000000..ea861ddbd --- /dev/null +++ b/functest/api/database/db.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +""" +Create database to store task results using sqlalchemy +""" + +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import scoped_session, sessionmaker + + +SQLITE = 'sqlite:////tmp/functest.db' + +ENGINE = create_engine(SQLITE, convert_unicode=True) +DB_SESSION = scoped_session(sessionmaker(autocommit=False, + autoflush=False, + bind=ENGINE)) +BASE = declarative_base() +BASE.query = DB_SESSION.query_property() diff --git a/functest/api/database/v1/__init__.py b/functest/api/database/v1/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/functest/api/database/v1/__init__.py diff --git a/functest/api/database/v1/handlers.py b/functest/api/database/v1/handlers.py new file mode 100644 index 000000000..7bd286ded --- /dev/null +++ b/functest/api/database/v1/handlers.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python + +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +""" +Used to handle tasks: insert the task info into database and update it +""" + +from functest.api.database.db import DB_SESSION +from functest.api.database.v1.models import Tasks + + +class TasksHandler(object): + """ Tasks Handler Class """ + + def insert(self, kwargs): # pylint: disable=no-self-use + """ To insert the task info into database """ + task = Tasks(**kwargs) + DB_SESSION.add(task) # pylint: disable=maybe-no-member + DB_SESSION.commit() # pylint: disable=maybe-no-member + return task + + def get_task_by_taskid(self, task_id): # pylint: disable=no-self-use + """ Obtain the task by task id """ + # pylint: disable=maybe-no-member + task = Tasks.query.filter_by(task_id=task_id).first() + if not task: + raise ValueError + + return task + + def update_attr(self, task_id, attr): + """ Update the required attributes of the task """ + task = self.get_task_by_taskid(task_id) + + for key, value in attr.items(): + setattr(task, key, value) + DB_SESSION.commit() # pylint: disable=maybe-no-member diff --git a/functest/api/database/v1/models.py b/functest/api/database/v1/models.py new file mode 100644 index 000000000..c5de91bca --- /dev/null +++ b/functest/api/database/v1/models.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +""" +Define tables for tasks +""" + +from sqlalchemy import Column +from sqlalchemy import Integer +from sqlalchemy import String +from sqlalchemy import Text + +from functest.api.database.db import BASE + + +class Tasks(BASE): # pylint: disable=too-few-public-methods, no-init + """ Create a table for tasks""" + + __tablename__ = 'tasks' + id = Column(Integer, primary_key=True) # pylint: disable=invalid-name + task_id = Column(String(50)) + status = Column(Integer) + error = Column(String(120)) + result = Column(Text) + + def __repr__(self): + return '<Task %r>' % Tasks.task_id diff --git a/functest/api/resources/v1/creds.py b/functest/api/resources/v1/creds.py index e402d7e3a..45e4559f4 100644 --- a/functest/api/resources/v1/creds.py +++ b/functest/api/resources/v1/creds.py @@ -11,13 +11,19 @@ Resources to handle openstack related requests """ +import collections +import logging + from flask import jsonify from functest.api.base import ApiResource +from functest.api.common import api_utils from functest.cli.commands.cli_os import OpenStack from functest.utils import openstack_utils as os_utils from functest.utils.constants import CONST +LOGGER = logging.getLogger(__name__) + class V1Creds(ApiResource): """ V1Creds Resource class""" @@ -27,3 +33,35 @@ class V1Creds(ApiResource): os_utils.source_credentials(CONST.__getattribute__('openstack_creds')) credentials_show = OpenStack.show_credentials() return jsonify(credentials_show) + + def post(self): + """ Used to handle post request """ + return self._dispatch_post() + + def update_openrc(self, args): # pylint: disable=no-self-use + """ Used to update the OpenStack RC file """ + try: + openrc_vars = args['openrc'] + except KeyError: + return api_utils.result_handler( + status=0, data='openrc must be provided') + else: + if not isinstance(openrc_vars, collections.Mapping): + return api_utils.result_handler( + status=0, data='args should be a dict') + + lines = ['export {}={}\n'.format(k, v) for k, v in openrc_vars.items()] + + rc_file = CONST.__getattribute__('openstack_creds') + with open(rc_file, 'w') as creds_file: + creds_file.writelines(lines) + + LOGGER.info("Sourcing the OpenStack RC file...") + try: + os_utils.source_credentials(rc_file) + except Exception as err: # pylint: disable=broad-except + LOGGER.exception('Failed to source the OpenStack RC file') + return api_utils.result_handler(status=0, data=str(err)) + + return api_utils.result_handler( + status=0, data='Update openrc successfully') diff --git a/functest/api/resources/v1/envs.py b/functest/api/resources/v1/envs.py index 35bffb04f..9c455198d 100644 --- a/functest/api/resources/v1/envs.py +++ b/functest/api/resources/v1/envs.py @@ -14,6 +14,7 @@ from flask import jsonify from functest.api.base import ApiResource from functest.cli.commands.cli_env import Env +from functest.api.common import api_utils import functest.utils.functest_utils as ft_utils @@ -31,4 +32,9 @@ class V1Envs(ApiResource): def prepare(self, args): # pylint: disable=no-self-use, unused-argument """ Prepare environment """ - ft_utils.execute_command("prepare_env start") + try: + ft_utils.execute_command("prepare_env start") + except Exception as err: # pylint: disable=broad-except + return api_utils.result_handler(status=1, data=str(err)) + return api_utils.result_handler( + status=0, data="Prepare env successfully") diff --git a/functest/api/resources/v1/tasks.py b/functest/api/resources/v1/tasks.py new file mode 100644 index 000000000..7086e7075 --- /dev/null +++ b/functest/api/resources/v1/tasks.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python + +# Copyright (c) 2017 Huawei Technologies Co.,Ltd and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 + +""" +Resources to retrieve the task results +""" + + +import json +import logging +import uuid + +from flask import jsonify + +from functest.api.base import ApiResource +from functest.api.common import api_utils +from functest.api.database.v1.handlers import TasksHandler + + +LOGGER = logging.getLogger(__name__) + + +class V1Tasks(ApiResource): + """ V1Tasks Resource class""" + + def get(self, task_id): # pylint: disable=no-self-use + """ GET the result of the task id """ + try: + uuid.UUID(task_id) + except ValueError: + return api_utils.result_handler(status=1, data='Invalid task id') + + task_handler = TasksHandler() + try: + task = task_handler.get_task_by_taskid(task_id) + except ValueError: + return api_utils.result_handler(status=1, data='No such task id') + + status = task.status + LOGGER.debug('Task status is: %s', status) + + if status not in ['IN PROGRESS', 'FAIL', 'FINISHED']: + return api_utils.result_handler(status=1, + data='internal server error') + if status == 'IN PROGRESS': + result = {'status': status, 'result': ''} + elif status == 'FAIL': + result = {'status': status, 'error': task.error} + else: + result = {'status': status, 'result': json.loads(task.result)} + + return jsonify(result) diff --git a/functest/api/resources/v1/testcases.py b/functest/api/resources/v1/testcases.py index c3b8217a1..f146c24ce 100644 --- a/functest/api/resources/v1/testcases.py +++ b/functest/api/resources/v1/testcases.py @@ -11,11 +11,20 @@ Resources to handle testcase related requests """ +import os +import logging +import uuid + from flask import abort, jsonify from functest.api.base import ApiResource -from functest.api.common import api_utils +from functest.api.common import api_utils, thread from functest.cli.commands.cli_testcase import Testcase +from functest.api.database.v1.handlers import TasksHandler +from functest.utils.constants import CONST +import functest.utils.functest_utils as ft_utils + +LOGGER = logging.getLogger(__name__) class V1Testcases(ApiResource): @@ -46,3 +55,61 @@ class V1Testcase(ApiResource): result.update(testcase_info) result.update({'dependency': dependency_dict}) return jsonify(result) + + def post(self): + """ Used to handle post request """ + return self._dispatch_post() + + def run_test_case(self, args): + """ Run a testcase """ + try: + case_name = args['testcase'] + except KeyError: + return api_utils.result_handler( + status=1, data='testcase name must be provided') + + task_id = str(uuid.uuid4()) + + task_args = {'testcase': case_name, 'task_id': task_id} + + task_args.update(args.get('opts', {})) + + task_thread = thread.TaskThread(self._run, task_args, TasksHandler()) + task_thread.start() + + results = {'testcase': case_name, 'task_id': task_id} + return jsonify(results) + + def _run(self, args): # pylint: disable=no-self-use + """ The built_in function to run a test case """ + + case_name = args.get('testcase') + + if not os.path.isfile(CONST.__getattribute__('env_active')): + raise Exception("Functest environment is not ready.") + else: + try: + cmd = "run_tests -t {}".format(case_name) + runner = ft_utils.execute_command(cmd) + except Exception: # pylint: disable=broad-except + result = 'FAIL' + LOGGER.exception("Running test case %s failed!", case_name) + if runner == os.EX_OK: + result = 'PASS' + else: + result = 'FAIL' + + env_info = { + 'installer': CONST.__getattribute__('INSTALLER_TYPE'), + 'scenario': CONST.__getattribute__('DEPLOY_SCENARIO'), + 'build_tag': CONST.__getattribute__('BUILD_TAG'), + 'ci_loop': CONST.__getattribute__('CI_LOOP') + } + result = { + 'task_id': args.get('task_id'), + 'case_name': case_name, + 'env_info': env_info, + 'result': result + } + + return {'result': result} diff --git a/functest/api/server.py b/functest/api/server.py index e246333ef..1d47b0dcb 100644 --- a/functest/api/server.py +++ b/functest/api/server.py @@ -12,6 +12,7 @@ Used to launch Functest RestApi """ +import inspect import logging import socket from urlparse import urljoin @@ -21,12 +22,28 @@ from flask import Flask from flask_restful import Api from functest.api.base import ApiResource -from functest.api.urls import URLPATTERNS from functest.api.common import api_utils +from functest.api.database.db import BASE +from functest.api.database.db import DB_SESSION +from functest.api.database.db import ENGINE +from functest.api.database.v1 import models +from functest.api.urls import URLPATTERNS LOGGER = logging.getLogger(__name__) +APP = Flask(__name__) +API = Api(APP) + + +@APP.teardown_request +def shutdown_session(exception=None): # pylint: disable=unused-argument + """ + To be called at the end of each request whether it is successful + or an exception is raised + """ + DB_SESSION.remove() + def get_resource(resource_name): """ Obtain the required resource according to resource name """ @@ -41,7 +58,7 @@ def get_endpoint(url): return urljoin('http://{}:5000'.format(address), url) -def api_add_resource(api): +def api_add_resource(): """ The resource has multiple URLs and you can pass multiple URLs to the add_resource() method on the Api object. Each one will be routed to @@ -49,19 +66,38 @@ def api_add_resource(api): """ for url_pattern in URLPATTERNS: try: - api.add_resource( + API.add_resource( get_resource(url_pattern.target), url_pattern.url, endpoint=get_endpoint(url_pattern.url)) except StopIteration: LOGGER.error('url resource not found: %s', url_pattern.url) +def init_db(): + """ + Import all modules here that might define models so that + they will be registered properly on the metadata, and then + create a database + """ + def func(subcls): + """ To check the subclasses of BASE""" + try: + if issubclass(subcls[1], BASE): + return True + except TypeError: + pass + return False + # pylint: disable=bad-builtin + subclses = filter(func, inspect.getmembers(models, inspect.isclass)) + LOGGER.debug('Import models: %s', [subcls[1] for subcls in subclses]) + BASE.metadata.create_all(bind=ENGINE) + + def main(): """Entry point""" logging.config.fileConfig(pkg_resources.resource_filename( 'functest', 'ci/logging.ini')) LOGGER.info('Starting Functest server') - app = Flask(__name__) - api = Api(app) - api_add_resource(api) - app.run(host='0.0.0.0') + api_add_resource() + init_db() + APP.run(host='0.0.0.0') diff --git a/functest/api/urls.py b/functest/api/urls.py index ca45b4beb..f7bcae389 100644 --- a/functest/api/urls.py +++ b/functest/api/urls.py @@ -32,6 +32,10 @@ URLPATTERNS = [ # GET /api/v1/functest/openstack/credentials => GET credentials Url('/api/v1/functest/openstack/credentials', 'v1_creds'), + # POST /api/v1/functest/openstack/action + # {"action":"update_openrc", "args": {"openrc": {}}} => Update openrc + Url('/api/v1/functest/openstack/action', 'v1_creds'), + # GET /api/v1/functest/testcases => GET all testcases Url('/api/v1/functest/testcases', 'v1_test_cases'), @@ -39,6 +43,11 @@ URLPATTERNS = [ # => GET the info of one testcase Url('/api/v1/functest/testcases/<testcase_name>', 'v1_testcase'), + # POST /api/v1/functest/testcases/action + # {"action":"run_test_case", "args": {"opts": {}, "testcase": "vping_ssh"}} + # => Run a testcase + Url('/api/v1/functest/testcases/action', 'v1_testcase'), + # GET /api/v1/functest/testcases => GET all tiers Url('/api/v1/functest/tiers', 'v1_tiers'), @@ -48,5 +57,10 @@ URLPATTERNS = [ # GET /api/v1/functest/tiers/<tier_name>/testcases # => GET all testcases within given tier - Url('/api/v1/functest/tiers/<tier_name>/testcases', 'v1_testcases_in_tier') + Url('/api/v1/functest/tiers/<tier_name>/testcases', + 'v1_testcases_in_tier'), + + # GET /api/v1/functest/tasks/<task_id> + # => GET the result of the task id + Url('/api/v1/functest/tasks/<task_id>', 'v1_tasks') ] diff --git a/functest/utils/openstack_utils.py b/functest/utils/openstack_utils.py index 335f14cd1..8b59c9547 100644 --- a/functest/utils/openstack_utils.py +++ b/functest/utils/openstack_utils.py @@ -22,8 +22,8 @@ from heatclient import client as heatclient from novaclient import client as novaclient from keystoneclient import client as keystoneclient from neutronclient.neutron import client as neutronclient -from functest.utils.constants import CONST +from functest.utils.constants import CONST import functest.utils.functest_utils as ft_utils logger = logging.getLogger(__name__) @@ -713,6 +713,8 @@ def get_private_net(neutron_client): def get_external_net(neutron_client): + if (hasattr(CONST, 'EXTERNAL_NETWORK')): + return CONST.__getattribute__('EXTERNAL_NETWORK') for network in neutron_client.list_networks()['networks']: if network['router:external']: return network['name'] @@ -720,6 +722,11 @@ def get_external_net(neutron_client): def get_external_net_id(neutron_client): + if (hasattr(CONST, 'EXTERNAL_NETWORK')): + networks = neutron_client.list_networks( + name=CONST.__getattribute__('EXTERNAL_NETWORK')) + net_id = networks['networks'][0]['id'] + return net_id for network in neutron_client.list_networks()['networks']: if network['router:external']: return network['id'] |