diff options
Diffstat (limited to 'dashboard/backend')
| -rwxr-xr-x | dashboard/backend/dovetail/__init__.py | 8 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/api/__init__.py | 29 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/api/api.py | 183 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/api/exception_handler.py | 93 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/api/utils.py | 20 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/db/__init__.py | 8 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/db/api.py | 72 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/db/database.py | 182 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/db/exception.py | 121 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/db/models.py | 105 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/db/utils.py | 478 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/utils/__init__.py | 8 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/utils/flags.py | 82 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/utils/logsetting.py | 98 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/utils/setting_wrapper.py | 18 | ||||
| -rwxr-xr-x | dashboard/backend/dovetail/utils/util.py | 71 | ||||
| -rwxr-xr-x | dashboard/backend/install_db.py | 55 | ||||
| -rwxr-xr-x | dashboard/backend/wsgi.py | 35 |
18 files changed, 0 insertions, 1666 deletions
diff --git a/dashboard/backend/dovetail/__init__.py b/dashboard/backend/dovetail/__init__.py deleted file mode 100755 index 6dbd8d79..00000000 --- a/dashboard/backend/dovetail/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## diff --git a/dashboard/backend/dovetail/api/__init__.py b/dashboard/backend/dovetail/api/__init__.py deleted file mode 100755 index f9c4e5a2..00000000 --- a/dashboard/backend/dovetail/api/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import datetime -import logging - -from flask import Flask - -from dovetail.utils import util - -logging.info('flask app: begin to init') - -app = Flask(__name__) -app.debug = True -logging.info('flask app config:%s', app.config) - -app.config['REMEMBER_COOKIE_DURATION'] = ( - datetime.timedelta( - seconds=util.parse_time_interval('2h') - ) -) - -logging.info('flask app: finish init') diff --git a/dashboard/backend/dovetail/api/api.py b/dashboard/backend/dovetail/api/api.py deleted file mode 100755 index 7839b893..00000000 --- a/dashboard/backend/dovetail/api/api.py +++ /dev/null @@ -1,183 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import logging - -from dovetail.api import utils -from dovetail.api import exception_handler -from dovetail.db import api as db_api - -from flask import Flask -from flask import request - -import json - -app = Flask(__name__) - - -@app.after_request -def after_request(response): - response.headers.add('Access-Control-Allow-Origin', '*') - response.headers.add( - 'Access-Control-Allow-Headers', - 'Content-Type, Authorization') - response.headers.add('Aceess-Control-Allow-Methods', 'GET,PUT,DELETE,POST') - return response - -# test - - -@app.route("/test", methods=['GET']) -def test(): - """backend api test""" - logging.info('test functest') - resp = utils.make_json_response( - 200, {'test': 20} - ) - return resp - - -# settings -@app.route("/clear", methods=['POST']) -def clear_settings(): - """ clear all settings data on backend server """ - logging.info('clear all settings') - - return utils.make_json_response( - 200, {} - ) - - -@app.route("/settings", methods=['GET']) -def list_settings(): - """list settings""" - logging.info('list settings') - global settings - return utils.make_json_response(200, settings) - - -@app.route("/settings", methods=['POST']) -def add_settings(): - pass - - -@app.route("/settings", methods=['POST']) -def remove_settings(): - pass - - -@app.route("/testcases", methods=['GET']) -def get_testcases(): - pass - - -@app.route("/results/<test_id>", methods=['GET']) -def show_result(test_id): - data = _get_request_args() - return utils.make_json_response( - 200, - db_api.get_result( - test_id, **data - ) - ) - - -@app.route("/results", methods=['GET']) -def list_results(): - data = _get_request_args() - return utils.make_json_response( - 200, - db_api.list_results( - **data - ) - ) - - -@app.route("/results", methods=['POST']) -def add_result(): - data = _get_request_data() - ret_code = 200 - json_object = json.loads(data) - logging.debug('json_object:%s', (json_object)) - if not db_api.store_result(**json_object): - ret_code = 500 - resp = utils.make_json_response( - ret_code, data - ) - return resp - - -@app.route("/results/<test_id>", methods=['DELETE']) -def remove_results(test_id): - data = _get_request_data() - logging.debug('data:%s', data) - response = db_api.del_result( - test_id, **data - ) - return utils.make_json_response( - 200, response - ) - - -def _get_request_data(): - """Convert reqeust data from string to python dict. - - If the request data is not json formatted, raises - exception_handler.BadRequest. - If the request data is not json formatted dict, raises - exception_handler.BadRequest - If the request data is empty, return default as empty dict. - - Usage: It is used to add or update a single resource. - """ - if request.data: - try: - data = json.loads(request.data) - except Exception: - raise exception_handler.BadRequest( - 'request data is not json formatted: %s' % request.data - ) - if not isinstance(data, dict): - raise exception_handler.BadRequest( - 'request data is not json formatted dict: %s' % request.data - ) - - return request.data - else: - return {} - - -def _get_request_args(**kwargs): - """Get request args as dict. - - The value in the dict is converted to expected type. - - Args: - kwargs: for each key, the value is the type converter. - """ - args = dict(request.args) - for key, value in args.items(): - if key in kwargs: - converter = kwargs[key] - if isinstance(value, list): - args[key] = [converter(item) for item in value] - else: - args[key] = converter(value) - return args - - -''' -@app.teardown_appcontext -def shutdown_session(exception=None): - db_session.remove() -''' -# user login/logout - -if __name__ == '__main__': - app.run(host='127.0.0.1') diff --git a/dashboard/backend/dovetail/api/exception_handler.py b/dashboard/backend/dovetail/api/exception_handler.py deleted file mode 100755 index b7ce592a..00000000 --- a/dashboard/backend/dovetail/api/exception_handler.py +++ /dev/null @@ -1,93 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -"""Exceptions for RESTful API.""" -import traceback - -from dovetail.api import app -from dovetail.api import utils - - -class HTTPException(Exception): - - def __init__(self, message, status_code): - super(HTTPException, self).__init__(message) - self.traceback = traceback.format_exc() - self.status_code = status_code - - def to_dict(self): - return {'message': str(self)} - - -class ItemNotFound(HTTPException): - """Define the exception for referring non-existing object.""" - - def __init__(self, message): - super(ItemNotFound, self).__init__(message, 410) - - -class BadRequest(HTTPException): - """Define the exception for invalid/missing parameters. - - User making a request in invalid state cannot be processed. - """ - - def __init__(self, message): - super(BadRequest, self).__init__(message, 400) - - -class Unauthorized(HTTPException): - """Define the exception for invalid user login.""" - - def __init__(self, message): - super(Unauthorized, self).__init__(message, 401) - - -class UserDisabled(HTTPException): - """Define the exception for disabled users.""" - - def __init__(self, message): - super(UserDisabled, self).__init__(message, 403) - - -class Forbidden(HTTPException): - """Define the exception for invalid permissions.""" - - def __init__(self, message): - super(Forbidden, self).__init__(message, 403) - - -class BadMethod(HTTPException): - """Define the exception for invoking unsupported methods.""" - - def __init__(self, message): - super(BadMethod, self).__init__(message, 405) - - -class ConflictObject(HTTPException): - """Define the exception for creating an existing object.""" - - def __init__(self, message): - super(ConflictObject, self).__init__(message, 409) - - -@app.errorhandler(Exception) -def handle_exception(error): - if hasattr(error, 'to_dict'): - response = error.to_dict() - else: - response = {'message': str(error)} - if app.debug and hasattr(error, 'traceback'): - response['traceback'] = error.traceback - - status_code = 400 - if hasattr(error, 'status_code'): - status_code = error.status_code - - return utils.make_json_response(status_code, response) diff --git a/dashboard/backend/dovetail/api/utils.py b/dashboard/backend/dovetail/api/utils.py deleted file mode 100755 index dbe8d082..00000000 --- a/dashboard/backend/dovetail/api/utils.py +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import json -from flask import make_response - - -def make_json_response(status_code, data): - """Wrap json format to the reponse object.""" - - result = json.dumps(data, indent=4, default=lambda x: None) + '\r\n' - resp = make_response(result, status_code) - resp.headers['Content-type'] = 'application/json' - return resp diff --git a/dashboard/backend/dovetail/db/__init__.py b/dashboard/backend/dovetail/db/__init__.py deleted file mode 100755 index 6dbd8d79..00000000 --- a/dashboard/backend/dovetail/db/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## diff --git a/dashboard/backend/dovetail/db/api.py b/dashboard/backend/dovetail/db/api.py deleted file mode 100755 index 631ed2a3..00000000 --- a/dashboard/backend/dovetail/db/api.py +++ /dev/null @@ -1,72 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -""" -Defines interface for DB access. -""" - -import logging - -from dovetail.db import database -from dovetail.db import utils -from dovetail.db import models - - -@database.run_in_session() -def store_result(exception_when_existing=True, - session=None, **kwargs): - """Storing results into database. - - :param data: Dict describes test results. - """ - logging.debug('store_result:%s', kwargs) - result = utils.add_db_object( - session, models.Result, exception_when_existing, - **kwargs) - - return result - - -@database.run_in_session() -@utils.wrap_to_dict() -def list_results(session=None, **filters): - """Get all results - """ - logging.debug('session:%s', session) - results = utils.list_db_objects( - session, models.Result, **filters - ) - return results - - -@database.run_in_session() -@utils.wrap_to_dict() -def get_result(test_id, exception_when_missing=True, - session=None, **kwargs): - """Get specific result with the test_id - - :param test_id: the unique serial number for the test - """ - return _get_result(test_id, session, - exception_when_missing=exception_when_missing, **kwargs) - - -def _get_result(test_id, session=None, **kwargs): - return utils.get_db_object( - session, models.Result, test_id=test_id, **kwargs) - - -@database.run_in_session() -def del_result(test_id, session=None, **kwargs): - """Delete a results from database - - :param test_id: the unique serial number for the test - """ - return utils.del_db_objects(session, models.Result, - test_id=test_id, **kwargs) diff --git a/dashboard/backend/dovetail/db/database.py b/dashboard/backend/dovetail/db/database.py deleted file mode 100755 index bc09d3bd..00000000 --- a/dashboard/backend/dovetail/db/database.py +++ /dev/null @@ -1,182 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import logging -import functools - -from threading import local - -from sqlalchemy import create_engine -from sqlalchemy.exc import IntegrityError -from sqlalchemy.exc import OperationalError -from sqlalchemy.orm import scoped_session -from sqlalchemy.orm import sessionmaker -from sqlalchemy.pool import StaticPool - -from contextlib import contextmanager -from dovetail.db import exception -from dovetail.db import models - -ENGINE = None -SESSION = sessionmaker(autocommit=False, autoflush=False) -SCOPED_SESSION = None -SESSION_HOLDER = local() - -SQLALCHEMY_DATABASE_URI = "mysql://root:%s@localhost:3306/dovetail" % ('root') - - -def init(database_url=None): - """Initialize database. - - :param database_url: string, database url. - """ - global ENGINE - global SCOPED_SESSION - if not database_url: - database_url = SQLALCHEMY_DATABASE_URI - logging.info('init database %s', database_url) - print("database init %s" % database_url) - ENGINE = create_engine( - database_url, convert_unicode=True, - poolclass=StaticPool - ) - SESSION.configure(bind=ENGINE) - SCOPED_SESSION = scoped_session(SESSION) - models.BASE.query = SCOPED_SESSION.query_property() - - -def in_session(): - """check if in database session scope.""" - bool(hasattr(SESSION_HOLDER, 'session')) - - -@contextmanager -def session(exception_when_in_session=True): - """database session scope. - - To operate database, it should be called in database session. - If not exception_when_in_session, the with session statement support - nested session and only the out most session commit/rollback the - transaction. - """ - if not ENGINE: - init() - - nested_session = False - if hasattr(SESSION_HOLDER, 'session'): - if exception_when_in_session: - logging.error('we are already in session') - raise exception.DatabaseException('session already exist') - else: - new_session = SESSION_HOLDER.session - nested_session = True - logging.log( - logging.DEBUG, - 'reuse session %s', nested_session - ) - else: - new_session = SCOPED_SESSION() - setattr(SESSION_HOLDER, 'session', new_session) - logging.log( - logging.DEBUG, - 'enter session %s', new_session - ) - try: - yield new_session - if not nested_session: - new_session.commit() - except Exception as error: - if not nested_session: - new_session.rollback() - logging.error('failed to commit session') - logging.exception(error) - if isinstance(error, IntegrityError): - for item in error.statement.split(): - if item.islower(): - object = item - break - raise exception.DuplicatedRecord( - '%s in %s' % (error.orig, object) - ) - elif isinstance(error, OperationalError): - raise exception.DatabaseException( - 'operation error in database' - ) - elif isinstance(error, exception.DatabaseException): - raise error - else: - raise exception.DatabaseException(str(error)) - finally: - if not nested_session: - new_session.close() - SCOPED_SESSION.remove() - delattr(SESSION_HOLDER, 'session') - logging.log( - logging.DEBUG, - 'exit session %s', new_session - ) - - -def current_session(): - """Get the current session scope when it is called. - - :return: database session. - :raises: DatabaseException when it is not in session. - """ - try: - return SESSION_HOLDER.session - except Exception as error: - logging.error('It is not in the session scope') - logging.exception(error) - if isinstance(error, exception.DatabaseException): - raise error - else: - raise exception.DatabaseException(str(error)) - - -def run_in_session(exception_when_in_session=True): - """Decorator to make sure the decorated function run in session. - - When not exception_when_in_session, the run_in_session can be - decorated several times. - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - my_session = kwargs.get('session') - if my_session is not None: - return func(*args, **kwargs) - else: - with session( - exception_when_in_session=exception_when_in_session - ) as my_session: - kwargs['session'] = my_session - return func(*args, **kwargs) - except Exception as error: - logging.error( - 'got exception with func %s args %s kwargs %s', - func, args, kwargs - ) - logging.exception(error) - raise error - return wrapper - return decorator - - -@run_in_session() -def create_db(session=None): - """Create database.""" - models.BASE.metadata.create_all(bind=ENGINE) - print('create_db') - - -def drop_db(): - """Drop database.""" - models.BASE.metadata.drop_all(bind=ENGINE) diff --git a/dashboard/backend/dovetail/db/exception.py b/dashboard/backend/dovetail/db/exception.py deleted file mode 100755 index 4acc5fbd..00000000 --- a/dashboard/backend/dovetail/db/exception.py +++ /dev/null @@ -1,121 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -"""Custom exception""" -import traceback - - -class DatabaseException(Exception): - """Base class for all database exceptions.""" - - def __init__(self, message): - super(DatabaseException, self).__init__(message) - self.traceback = traceback.format_exc() - self.status_code = 400 - - def to_dict(self): - return {'message': str(self)} - - -class RecordNotExists(DatabaseException): - """Define the exception for referring non-existing object in DB.""" - - def __init__(self, message): - super(RecordNotExists, self).__init__(message) - self.status_code = 404 - - -class DuplicatedRecord(DatabaseException): - """Define the exception for trying to insert an existing object in DB.""" - - def __init__(self, message): - super(DuplicatedRecord, self).__init__(message) - self.status_code = 409 - - -class Unauthorized(DatabaseException): - """Define the exception for invalid user login.""" - - def __init__(self, message): - super(Unauthorized, self).__init__(message) - self.status_code = 401 - - -class UserDisabled(DatabaseException): - """Define the exception that a disabled user tries to do some operations. - - """ - - def __init__(self, message): - super(UserDisabled, self).__init__(message) - self.status_code = 403 - - -class Forbidden(DatabaseException): - """Define the exception that a user is trying to make some action - - without the right permission. - - """ - - def __init__(self, message): - super(Forbidden, self).__init__(message) - self.status_code = 403 - - -class NotAcceptable(DatabaseException): - """The data is not acceptable.""" - - def __init__(self, message): - super(NotAcceptable, self).__init__(message) - self.status_code = 406 - - -class InvalidParameter(DatabaseException): - """Define the exception that the request has invalid or missing parameters. - - """ - - def __init__(self, message): - super(InvalidParameter, self).__init__(message) - self.status_code = 400 - - -class InvalidResponse(DatabaseException): - """Define the exception that the response is invalid. - - """ - - def __init__(self, message): - super(InvalidResponse, self).__init__(message) - self.status_code = 400 - - -class MultiDatabaseException(DatabaseException): - """Define the exception composites with multi exceptions.""" - - def __init__(self, exceptions): - super(MultiDatabaseException, self).__init__('multi exceptions') - self.exceptions = exceptions - self.status_code = 400 - - @property - def traceback(self): - tracebacks = [] - for exception in self.exceptions: - tracebacks.append(exception.trackback) - - def to_dict(self): - dict_info = super(MultiDatabaseException, self).to_dict() - dict_info.update({ - 'exceptions': [ - exception.to_dict() for exception in self.exceptions - ] - }) - return dict_info diff --git a/dashboard/backend/dovetail/db/models.py b/dashboard/backend/dovetail/db/models.py deleted file mode 100755 index e0f3ffa3..00000000 --- a/dashboard/backend/dovetail/db/models.py +++ /dev/null @@ -1,105 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import datetime - -from sqlalchemy import Column, Integer, String, DateTime -from sqlalchemy.ext.declarative import declarative_base - -from dovetail.utils import util -from dovetail.db import exception - -BASE = declarative_base() - - -class MarkTimestamp(object): - created = Column(DateTime, default=lambda: datetime.datetime.now()) - updated = Column(DateTime, default=lambda: datetime.datetime.now(), - onupdate=lambda: datetime.datetime.now()) - - -class ModelHandler(object): - - def initialize(self): - self.update() - - def update(self): - pass - - @staticmethod - def type_check(value, column_type): - if value is None: - return True - if not hasattr(column_type, 'python_type'): - return True - column_python_type = column_type.python_type - if isinstance(value, column_python_type): - return True - if issubclass(column_python_type, basestring): - return isinstance(value, basestring) - if column_python_type in [int, long]: - return type(value) in [int, long] - if column_python_type in [float]: - return type(value) in [float] - if column_python_type in [bool]: - return type(value) in [bool] - return False - - def validate(self): - columns = self.__mapper__.columns - for key, column in columns.items(): - value = getattr(self, key) - if not self.type_check(value, column.type): - raise exception.InvalidParameter( - 'column %s value %r type is unexpected: %s' % ( - key, value, column.type - ) - ) - - def to_dict(self): - """General function to convert record to dict. - - Convert all columns not starting with '_' to - {<column_name>: <column_value>} - """ - keys = self.__mapper__.columns.keys() - dict_info = {} - for key in keys: - if key.startswith('_'): - continue - value = getattr(self, key) - if value is not None: - if isinstance(value, datetime.datetime): - value = util.format_datetime(value) - dict_info[key] = value - return dict_info - - -class Result(BASE, MarkTimestamp, ModelHandler): - __tablename__ = 'result' - id = Column(Integer, primary_key=True) - test_id = Column(String(120), unique=True) - name = Column(String(120)) - data = Column(String(64000)) - - def __init__(self, **kwargs): - super(Result, self).__init__(**kwargs) - - def __repr__(self): - return '<Result %r>' % (self.name) - - def __str__(self): - return 'Result[%s:%s]' % (self.name, self.test_id) - - def to_dict(self): - dict_info = super(Result, self).to_dict() - dict_info['name'] = self.name - dict_info['test_id'] = self.test_id - dict_info['data'] = self.data - return dict_info diff --git a/dashboard/backend/dovetail/db/utils.py b/dashboard/backend/dovetail/db/utils.py deleted file mode 100755 index 4bb0026d..00000000 --- a/dashboard/backend/dovetail/db/utils.py +++ /dev/null @@ -1,478 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -"""Utilities for database.""" - - -import functools -import inspect -import logging - -from sqlalchemy import and_ -from sqlalchemy import or_ - -from dovetail.db import exception -from dovetail.db import models - - -def add_db_object(session, table, exception_when_existing=True, - *args, **kwargs): - """Create db object. - - If not exception_when_existing and the db object exists, - Instead of raising exception, updating the existing db object. - """ - if not session: - raise exception.DatabaseException('session param is None') - with session.begin(subtransactions=True): - logging.debug( - 'session %s add object %s atributes %s to table %s', - id(session), args, kwargs, table.__name__) - argspec = inspect.getargspec(table.__init__) - arg_names = argspec.args[1:] - arg_defaults = argspec.defaults - if not arg_defaults: - arg_defaults = [] - if not ( - len(arg_names) - len(arg_defaults) <= len(args) <= len(arg_names) - ): - raise exception.InvalidParameter( - 'arg names %s does not match arg values %s' % ( - arg_names, args) - ) - db_keys = dict(zip(arg_names, args)) - logging.debug('db_keys:%s', db_keys) - if db_keys: - db_object = session.query(table).filter_by(**db_keys).first() - else: - logging.debug('db object is None') - db_object = None - - new_object = False - if db_object: - logging.debug( - 'got db object %s: %s', db_keys, db_object - ) - if exception_when_existing: - raise exception.DuplicatedRecord( - '%s exists in table %s' % (db_keys, table.__name__) - ) - else: - db_object = table(**db_keys) - new_object = True - - for key, value in kwargs.items(): - setattr(db_object, key, value) - - logging.debug('db_object:%s', db_object) - if new_object: - session.add(db_object) - session.flush() - db_object.initialize() - db_object.validate() - logging.debug( - 'session %s db object %s added', id(session), db_object - ) - return db_object - - -def list_db_objects(session, table, order_by=[], **filters): - """List db objects. - - If order by given, the db objects should be sorted by the ordered keys. - """ - if not session: - raise exception.DatabaseException('session param is None') - with session.begin(subtransactions=True): - logging.debug( - 'session %s list db objects by filters %s in table %s', - id(session), filters, table.__name__ - ) - db_objects = model_order_by( - model_filter( - model_query(session, table), - table, - **filters - ), - table, - order_by - ).all() - logging.debug( - 'session %s got listed db objects: %s', - id(session), db_objects - ) - return db_objects - - -def get_db_object(session, table, exception_when_missing=True, **kwargs): - """Get db object. - - If not exception_when_missing and the db object can not be found, - return None instead of raising exception. - """ - if not session: - raise exception.DatabaseException('session param is None') - with session.begin(subtransactions=True): - logging.debug( - 'session %s get db object %s from table %s', - id(session), kwargs, table.__name__) - db_object = model_filter( - model_query(session, table), table, **kwargs - ).first() - logging.debug( - 'session %s got db object %s', id(session), db_object - ) - if db_object: - return db_object - - if not exception_when_missing: - return None - - raise exception.RecordNotExists( - 'Cannot find the record in table %s: %s' % ( - table.__name__, kwargs - ) - ) - - -def del_db_objects(session, table, **filters): - """delete db objects.""" - if not session: - raise exception.DatabaseException('session param is None') - with session.begin(subtransactions=True): - logging.debug( - 'session %s delete db objects by filters %s in table %s', - id(session), filters, table.__name__ - ) - query = model_filter( - model_query(session, table), table, **filters - ) - db_objects = query.all() - query.delete(synchronize_session=False) - logging.debug( - 'session %s db objects %s deleted', id(session), db_objects - ) - return db_objects - - -def model_order_by(query, model, order_by): - """append order by into sql query model.""" - if not order_by: - return query - order_by_cols = [] - for key in order_by: - if isinstance(key, tuple): - key, is_desc = key - else: - is_desc = False - if isinstance(key, basestring): - if hasattr(model, key): - col_attr = getattr(model, key) - else: - continue - else: - col_attr = key - if is_desc: - order_by_cols.append(col_attr.desc()) - else: - order_by_cols.append(col_attr) - return query.order_by(*order_by_cols) - - -def _model_condition(col_attr, value): - """Generate condition for one column. - - Example for col_attr is name: - value is 'a': name == 'a' - value is ['a']: name == 'a' - value is ['a', 'b']: name == 'a' or name == 'b' - value is {'eq': 'a'}: name == 'a' - value is {'lt': 'a'}: name < 'a' - value is {'le': 'a'}: name <= 'a' - value is {'gt': 'a'}: name > 'a' - value is {'ge': 'a'}: name >= 'a' - value is {'ne': 'a'}: name != 'a' - value is {'in': ['a', 'b']}: name in ['a', 'b'] - value is {'notin': ['a', 'b']}: name not in ['a', 'b'] - value is {'startswith': 'abc'}: name like 'abc%' - value is {'endswith': 'abc'}: name like '%abc' - value is {'like': 'abc'}: name like '%abc%' - value is {'between': ('a', 'c')}: name >= 'a' and name <= 'c' - value is [{'lt': 'a'}]: name < 'a' - value is [{'lt': 'a'}, {'gt': c'}]: name < 'a' or name > 'c' - value is {'lt': 'c', 'gt': 'a'}: name > 'a' and name < 'c' - - If value is a list, the condition is the or relationship among - conditions of each item. - If value is dict and there are multi keys in the dict, the relationship - is and conditions of each key. - Otherwise the condition is to compare the column with the value. - """ - if isinstance(value, list): - basetype_values = [] - composite_values = [] - for item in value: - if isinstance(item, (list, dict)): - composite_values.append(item) - else: - basetype_values.append(item) - conditions = [] - if basetype_values: - if len(basetype_values) == 1: - condition = (col_attr == basetype_values[0]) - else: - condition = col_attr.in_(basetype_values) - conditions.append(condition) - for composite_value in composite_values: - condition = _model_condition(col_attr, composite_value) - if condition is not None: - conditions.append(condition) - if not conditions: - return None - if len(conditions) == 1: - return conditions[0] - return or_(*conditions) - elif isinstance(value, dict): - conditions = [] - if 'eq' in value: - conditions.append(_model_condition_func( - col_attr, value['eq'], - lambda attr, data: attr == data, - lambda attr, data, item_condition_func: attr.in_(data) - )) - if 'lt' in value: - conditions.append(_model_condition_func( - col_attr, value['lt'], - lambda attr, data: attr < data, - _one_item_list_condition_func - )) - if 'gt' in value: - conditions.append(_model_condition_func( - col_attr, value['gt'], - lambda attr, data: attr > data, - _one_item_list_condition_func - )) - if 'le' in value: - conditions.append(_model_condition_func( - col_attr, value['le'], - lambda attr, data: attr <= data, - _one_item_list_condition_func - )) - if 'ge' in value: - conditions.append(_model_condition_func( - col_attr, value['ge'], - lambda attr, data: attr >= data, - _one_item_list_condition_func - )) - if 'ne' in value: - conditions.append(_model_condition_func( - col_attr, value['ne'], - lambda attr, data: attr != data, - lambda attr, data, item_condition_func: attr.notin_(data) - )) - if 'in' in value: - conditions.append(col_attr.in_(value['in'])) - if 'notin' in value: - conditions.append(col_attr.notin_(value['notin'])) - if 'startswith' in value: - conditions.append(_model_condition_func( - col_attr, value['startswith'], - lambda attr, data: attr.like('%s%%' % data) - )) - if 'endswith' in value: - conditions.append(_model_condition_func( - col_attr, value['endswith'], - lambda attr, data: attr.like('%%%s' % data) - )) - if 'like' in value: - conditions.append(_model_condition_func( - col_attr, value['like'], - lambda attr, data: attr.like('%%%s%%' % data) - )) - conditions = [ - condition - for condition in conditions - if condition is not None - ] - if not conditions: - return None - if len(conditions) == 1: - return conditions[0] - return and_(conditions) - else: - condition = (col_attr == value) - return condition - - -def _default_list_condition_func(col_attr, value, condition_func): - """The default condition func for a list of data. - - Given the condition func for single item of data, this function - wrap the condition_func and return another condition func using - or_ to merge the conditions of each single item to deal with a - list of data item. - - Args: - col_attr: the colomn name - value: the column value need to be compared. - condition_func: the sqlalchemy condition object like == - - Examples: - col_attr is name, value is ['a', 'b', 'c'] and - condition_func is ==, the returned condition is - name == 'a' or name == 'b' or name == 'c' - """ - conditions = [] - for sub_value in value: - condition = condition_func(col_attr, sub_value) - if condition is not None: - conditions.append(condition) - if conditions: - return or_(*conditions) - else: - return None - - -def _one_item_list_condition_func(col_attr, value, condition_func): - """The wrapper condition func to deal with one item data list. - - For simplification, it is used to reduce generating too complex - sql conditions. - """ - if value: - return condition_func(col_attr, value[0]) - else: - return None - - -def _model_condition_func( - col_attr, value, - item_condition_func, - list_condition_func=_default_list_condition_func -): - """Return sql condition based on value type.""" - if isinstance(value, list): - if not value: - return None - if len(value) == 1: - return item_condition_func(col_attr, value) - return list_condition_func( - col_attr, value, item_condition_func - ) - else: - return item_condition_func(col_attr, value) - - -def model_filter(query, model, **filters): - """Append conditons to query for each possible column.""" - for key, value in filters.items(): - if isinstance(key, basestring): - if hasattr(model, key): - col_attr = getattr(model, key) - else: - continue - else: - col_attr = key - - condition = _model_condition(col_attr, value) - if condition is not None: - query = query.filter(condition) - return query - - -def model_query(session, model): - """model query. - - Return sqlalchemy query object. - """ - if not issubclass(model, models.BASE): - raise exception.DatabaseException("model should be sublass of BASE!") - - return session.query(model) - - -def wrap_to_dict(support_keys=[], **filters): - """Decrator to convert returned object to dict. - - The details is decribed in _wrapper_dict. - """ - def decorator(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - return _wrapper_dict( - func(*args, **kwargs), support_keys, **filters - ) - return wrapper - return decorator - - -def _wrapper_dict(data, support_keys, **filters): - """Helper for warpping db object into dictionary. - - If data is list, convert it to a list of dict - If data is Base model, convert it to dict - for the data as a dict, filter it with the supported keys. - For each filter_key, filter_value in filters, also filter - data[filter_key] by filter_value recursively if it exists. - - Example: - data is models.Switch, it will be converted to - { - 'id': 1, 'ip': '10.0.0.1', 'ip_int': 123456, - 'credentials': {'version': 2, 'password': 'abc'} - } - Then if support_keys are ['id', 'ip', 'credentials'], - it will be filtered to { - 'id': 1, 'ip': '10.0.0.1', - 'credentials': {'version': 2, 'password': 'abc'} - } - Then if filters is {'credentials': ['version']}, - it will be filtered to { - 'id': 1, 'ip': '10.0.0.1', - 'credentials': {'version': 2} - } - """ - logging.debug( - 'wrap dict %s by support_keys=%s filters=%s', - data, support_keys, filters - ) - if isinstance(data, list): - return [ - _wrapper_dict(item, support_keys, **filters) - for item in data - ] - if isinstance(data, models.ModelHandler): - data = data.to_dict() - if not isinstance(data, dict): - raise exception.InvalidResponse( - 'response %s type is not dict' % data - ) - info = {} - try: - if len(support_keys) == 0: - support_keys = data.keys() - for key in support_keys: - if key in data and data[key] is not None: - if key in filters: - filter_keys = filters[key] - if isinstance(filter_keys, dict): - info[key] = _wrapper_dict( - data[key], filter_keys.keys(), - **filter_keys - ) - else: - info[key] = _wrapper_dict( - data[key], filter_keys - ) - else: - info[key] = data[key] - return info - except Exception as error: - logging.exception(error) - raise error diff --git a/dashboard/backend/dovetail/utils/__init__.py b/dashboard/backend/dovetail/utils/__init__.py deleted file mode 100755 index 6dbd8d79..00000000 --- a/dashboard/backend/dovetail/utils/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## diff --git a/dashboard/backend/dovetail/utils/flags.py b/dashboard/backend/dovetail/utils/flags.py deleted file mode 100755 index dd10670b..00000000 --- a/dashboard/backend/dovetail/utils/flags.py +++ /dev/null @@ -1,82 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import sys - -from optparse import OptionParser - - -class Flags(object): - """Class to store flags.""" - - PARSER = OptionParser() - PARSED_OPTIONS = None - - @classmethod - def parse_args(cls): - """parse args.""" - (options, argv) = Flags.PARSER.parse_args() - sys.argv = [sys.argv[0]] + argv - Flags.PARSED_OPTIONS = options - - def __getattr__(self, name): - if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name): - return getattr(Flags.PARSED_OPTIONS, name) - - for option in Flags.PARSER.option_list: - if option.dest == name: - return option.default - - raise AttributeError('Option instance has no attribute %s' % name) - - def __setattr__(self, name, value): - if Flags.PARSED_OPTIONS and hasattr(Flags.PARSED_OPTIONS, name): - setattr(Flags.PARSED_OPTIONS, name, value) - return - - for option in Flags.PARSER.option_list: - if option.dest == name: - option.default = value - return - - object.__setattr__(self, name, value) - - -OPTIONS = Flags() - - -def init(): - """Init flag parsing.""" - OPTIONS.parse_args() - - -def add(flagname, **kwargs): - """Add a flag name and its setting. - - :param flagname: flag name declared in cmd as --<flagname>=... - :type flagname: str - """ - Flags.PARSER.add_option('--%s' % flagname, - dest=flagname, **kwargs) - - -def add_bool(flagname, default=True, **kwargs): - """Add a bool flag name and its setting. - - :param flagname: flag name declared in cmd as --[no]<flagname>. - :type flagname: str - :param default: default value - :type default: bool - """ - Flags.PARSER.add_option('--%s' % flagname, - dest=flagname, default=default, - action="store_true", **kwargs) - Flags.PARSER.add_option('--no%s' % flagname, - dest=flagname, - action="store_false", **kwargs) diff --git a/dashboard/backend/dovetail/utils/logsetting.py b/dashboard/backend/dovetail/utils/logsetting.py deleted file mode 100755 index 27255688..00000000 --- a/dashboard/backend/dovetail/utils/logsetting.py +++ /dev/null @@ -1,98 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -import logging -import logging.handlers -import os -import os.path -import sys - -from dovetail.utils import flags -from dovetail.utils import setting_wrapper as setting - - -flags.add('loglevel', - help='logging level', default=setting.DEFAULT_LOGLEVEL) -flags.add('logdir', - help='logging directory', default=setting.DEFAULT_LOGDIR) -flags.add('logfile', - help='logging filename', default=None) -flags.add('log_interval', type='int', - help='log interval', default=setting.DEFAULT_LOGINTERVAL) -flags.add('log_interval_unit', - help='log interval unit', default=setting.DEFAULT_LOGINTERVAL_UNIT) -flags.add('log_format', - help='log format', default=setting.DEFAULT_LOGFORMAT) -flags.add('log_backup_count', type='int', - help='log backup count', default=setting.DEFAULT_LOGBACKUPCOUNT) - - -# mapping str setting in flag --loglevel to logging level. -LOGLEVEL_MAPPING = { - 'finest': logging.DEBUG - 2, # more detailed log. - 'fine': logging.DEBUG - 1, # detailed log. - 'debug': logging.DEBUG, - 'info': logging.INFO, - 'warning': logging.WARNING, - 'error': logging.ERROR, - 'critical': logging.CRITICAL, -} - - -logging.addLevelName(LOGLEVEL_MAPPING['fine'], 'fine') -logging.addLevelName(LOGLEVEL_MAPPING['finest'], 'finest') - - -# disable logging when logsetting.init not called -logging.getLogger().setLevel(logging.CRITICAL) - - -def getLevelByName(level_name): - """Get log level by level name.""" - return LOGLEVEL_MAPPING[level_name] - - -def init(): - """Init loggsetting. It should be called after flags.init.""" - loglevel = flags.OPTIONS.loglevel.lower() - logdir = flags.OPTIONS.logdir - logfile = flags.OPTIONS.logfile - logger = logging.getLogger() - if logger.handlers: - for handler in logger.handlers: - logger.removeHandler(handler) - - if logdir: - if not logfile: - logfile = './%s.log' % os.path.basename(sys.argv[0]) - - handler = logging.handlers.TimedRotatingFileHandler( - os.path.join(logdir, logfile), - when=flags.OPTIONS.log_interval_unit, - interval=flags.OPTIONS.log_interval, - backupCount=flags.OPTIONS.log_backup_count) - else: - if not logfile: - handler = logging.StreamHandler(sys.stderr) - else: - handler = logging.handlers.TimedRotatingFileHandler( - logfile, - when=flags.OPTIONS.log_interval_unit, - interval=flags.OPTIONS.log_interval, - backupCount=flags.OPTIONS.log_backup_count) - - if loglevel in LOGLEVEL_MAPPING: - logger.setLevel(LOGLEVEL_MAPPING[loglevel]) - handler.setLevel(LOGLEVEL_MAPPING[loglevel]) - - formatter = logging.Formatter( - flags.OPTIONS.log_format) - - handler.setFormatter(formatter) - logger.addHandler(handler) diff --git a/dashboard/backend/dovetail/utils/setting_wrapper.py b/dashboard/backend/dovetail/utils/setting_wrapper.py deleted file mode 100755 index bb390ada..00000000 --- a/dashboard/backend/dovetail/utils/setting_wrapper.py +++ /dev/null @@ -1,18 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -DEFAULT_LOGLEVEL = 'debug' -DEFAULT_LOGDIR = '/var/log/dovetail/' -DEFAULT_LOGINTERVAL = 30 -DEFAULT_LOGINTERVAL_UNIT = 'M' -DEFAULT_LOGFORMAT = ( - '%(asctime)s - %(filename)s - %(lineno)d - %(levelname)s - %(message)s') -DEFAULT_LOGBACKUPCOUNT = 10 -WEB_LOGFILE = 'dovetail_web.log' diff --git a/dashboard/backend/dovetail/utils/util.py b/dashboard/backend/dovetail/utils/util.py deleted file mode 100755 index bfd257d7..00000000 --- a/dashboard/backend/dovetail/utils/util.py +++ /dev/null @@ -1,71 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - - -import datetime -import re -import sys - - -def format_datetime(date_time): - """Generate string from datetime object.""" - return date_time.strftime("%Y-%m-%d %H:%M:%S") - - -def parse_time_interval(time_interval_str): - """parse string of time interval to time interval. - - supported time interval unit: ['d', 'w', 'h', 'm', 's'] - Examples: - time_interval_str: '3d 2h' time interval to 3 days and 2 hours. - """ - if not time_interval_str: - return 0 - - time_interval_tuple = [ - time_interval_element - for time_interval_element in time_interval_str.split(' ') - if time_interval_element - ] - time_interval_dict = {} - time_interval_unit_mapping = { - 'd': 'days', - 'w': 'weeks', - 'h': 'hours', - 'm': 'minutes', - 's': 'seconds' - } - for time_interval_element in time_interval_tuple: - mat = re.match(r'^([+-]?\d+)(w|d|h|m|s).*', time_interval_element) - if not mat: - continue - - time_interval_value = int(mat.group(1)) - time_interval_unit = time_interval_unit_mapping[mat.group(2)] - time_interval_dict[time_interval_unit] = ( - time_interval_dict.get(time_interval_unit, 0) + time_interval_value - ) - - time_interval = datetime.timedelta(**time_interval_dict) - if sys.version_info[0:2] > (2, 6): - return time_interval.total_seconds() - else: - return ( - time_interval.microseconds + ( - time_interval.seconds + time_interval.days * 24 * 3600 - ) * 1e6 - ) / 1e6 - - -def pretty_print(*contents): - """pretty print contents.""" - if len(contents) == 0: - print "" - else: - print "\n".join(content for content in contents) diff --git a/dashboard/backend/install_db.py b/dashboard/backend/install_db.py deleted file mode 100755 index d37a4099..00000000 --- a/dashboard/backend/install_db.py +++ /dev/null @@ -1,55 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -# create db in new env -from dovetail.utils import flags -from dovetail.utils import logsetting -from dovetail.utils import setting_wrapper as setting - -from flask_script import Manager - -from dovetail.db import database -from dovetail.api.api import app - -import os - -app_manager = Manager(app, usage="Perform database operations") - -# flags.init() -curr_path = os.path.dirname(os.path.abspath(__file__)) -logdir = os.path.join(curr_path, 'log') -if not os.path.exists(logdir): - os.makedirs(logdir) - -flags.OPTIONS.logdir = logdir -flags.OPTIONS.logfile = setting.WEB_LOGFILE -logsetting.init() - - -@app_manager.command -def createdb(): - """Creates database from sqlalchemy models.""" - database.init() - try: - database.drop_db() - except Exception: - pass - - database.create_db() - - -@app_manager.command -def dropdb(): - """Drops database from sqlalchemy models.""" - database.init() - database.drop_db() - - -if __name__ == "__main__": - app_manager.run() diff --git a/dashboard/backend/wsgi.py b/dashboard/backend/wsgi.py deleted file mode 100755 index 088299d7..00000000 --- a/dashboard/backend/wsgi.py +++ /dev/null @@ -1,35 +0,0 @@ -##############################################################################
-# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-from dovetail.utils import flags
-from dovetail.utils import logsetting
-from dovetail.utils import setting_wrapper as setting
-
-from dovetail.api.api import app
-
-import os
-import logging
-
-gunicorn_error_logger = logging.getLogger('gunicorn.error')
-app.logger.handlers.extend(gunicorn_error_logger.handlers)
-app.logger.setLevel(logging.DEBUG)
-
-# flags.init()
-# logdir = setting.DEFAULT_LOGDIR
-curr_path = os.path.dirname(os.path.abspath(__file__))
-logdir = os.path.join(curr_path, 'log')
-if not os.path.exists(logdir):
- os.makedirs(logdir)
-
-flags.OPTIONS.logdir = logdir
-flags.OPTIONS.logfile = setting.WEB_LOGFILE
-logsetting.init()
-
-
-if __name__ == "__main__":
- app.run()
|