Backend rest api mechanism

JIRA:DOVETAIL-63

provide rest api as the dashboard backend
1. using gunicorn as rest api server
2. using flask as rest api framework
3. using sqlalchemy as mysql database driver
4. implement basic report CRUD operations
5. implement basic session management in  database operations

Change-Id: Ifbd251462396c2cb414b1ae9150cfc1e2e2d00c0
Signed-off-by: Leo Wang <grakiss.wanglei@huawei.com>

1 files changed, 478 insertions, 0 deletions

diff --git a/dashboard/backend/dovetail/db/utils.py b/dashboard/backend/dovetail/db/utils.py
new file mode 100755
index 00000000..5e788a71
--- /dev/null
+++ b/dashboard/backend/dovetail/db/utils.py
@@ -0,0 +1,478 @@
+##############################################################################
+# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+"""Utilities for database."""
+
+
+import functools
+import inspect
+import logging
+
+from sqlalchemy import and_
+from sqlalchemy import or_
+
+from dovetail.db import exception
+from dovetail.db import models
+
+
+def add_db_object(session, table, exception_when_existing=True,
+                  *args, **kwargs):
+    """Create db object.
+
+    If not exception_when_existing and the db object exists,
+    Instead of raising exception, updating the existing db object.
+    """
+    if not session:
+        raise exception.DatabaseException('session param is None')
+    with session.begin(subtransactions=True):
+        logging.debug(
+            'session %s add object %s atributes %s to table %s',
+            id(session), args, kwargs, table.__name__)
+        argspec = inspect.getargspec(table.__init__)
+        arg_names = argspec.args[1:]
+        arg_defaults = argspec.defaults
+        if not arg_defaults:
+            arg_defaults = []
+        if not (
+            len(arg_names) - len(arg_defaults) <= len(args) <= len(arg_names)
+        ):
+            raise exception.InvalidParameter(
+                'arg names %s does not match arg values %s' % (
+                    arg_names, args)
+            )
+        db_keys = dict(zip(arg_names, args))
+        logging.debug('db_keys:%s' % db_keys)
+        if db_keys:
+            db_object = session.query(table).filter_by(**db_keys).first()
+        else:
+            logging.debug('db object is None')
+            db_object = None
+
+        new_object = False
+        if db_object:
+            logging.debug(
+                'got db object %s: %s', db_keys, db_object
+            )
+            if exception_when_existing:
+                raise exception.DuplicatedRecord(
+                    '%s exists in table %s' % (db_keys, table.__name__)
+                )
+        else:
+            db_object = table(**db_keys)
+            new_object = True
+
+        for key, value in kwargs.items():
+            setattr(db_object, key, value)
+
+        logging.debug('db_object:%s' % db_object)
+        if new_object:
+            session.add(db_object)
+        session.flush()
+        db_object.initialize()
+        db_object.validate()
+        logging.debug(
+            'session %s db object %s added', id(session), db_object
+        )
+        return db_object
+
+
+def list_db_objects(session, table, order_by=[], **filters):
+    """List db objects.
+
+    If order by given, the db objects should be sorted by the ordered keys.
+    """
+    if not session:
+        raise exception.DatabaseException('session param is None')
+    with session.begin(subtransactions=True):
+        logging.debug(
+            'session %s list db objects by filters %s in table %s',
+            id(session), filters, table.__name__
+        )
+        db_objects = model_order_by(
+            model_filter(
+                model_query(session, table),
+                table,
+                **filters
+            ),
+            table,
+            order_by
+        ).all()
+        logging.debug(
+            'session %s got listed db objects: %s',
+            id(session), db_objects
+        )
+        return db_objects
+
+
+def get_db_object(session, table, exception_when_missing=True, **kwargs):
+    """Get db object.
+
+    If not exception_when_missing and the db object can not be found,
+    return None instead of raising exception.
+    """
+    if not session:
+        raise exception.DatabaseException('session param is None')
+    with session.begin(subtransactions=True):
+        logging.debug(
+            'session %s get db object %s from table %s',
+            id(session), kwargs, table.__name__)
+        db_object = model_filter(
+            model_query(session, table), table, **kwargs
+        ).first()
+        logging.debug(
+            'session %s got db object %s', id(session), db_object
+        )
+        if db_object:
+            return db_object
+
+        if not exception_when_missing:
+            return None
+
+        raise exception.RecordNotExists(
+            'Cannot find the record in table %s: %s' % (
+                table.__name__, kwargs
+            )
+        )
+
+
+def del_db_objects(session, table, **filters):
+    """delete db objects."""
+    if not session:
+        raise exception.DatabaseException('session param is None')
+    with session.begin(subtransactions=True):
+        logging.debug(
+            'session %s delete db objects by filters %s in table %s',
+            id(session), filters, table.__name__
+        )
+        query = model_filter(
+            model_query(session, table), table, **filters
+        )
+        db_objects = query.all()
+        query.delete(synchronize_session=False)
+        logging.debug(
+            'session %s db objects %s deleted', id(session), db_objects
+        )
+        return db_objects
+
+
+def model_order_by(query, model, order_by):
+    """append order by into sql query model."""
+    if not order_by:
+        return query
+    order_by_cols = []
+    for key in order_by:
+        if isinstance(key, tuple):
+            key, is_desc = key
+        else:
+            is_desc = False
+        if isinstance(key, basestring):
+            if hasattr(model, key):
+                col_attr = getattr(model, key)
+            else:
+                continue
+        else:
+            col_attr = key
+        if is_desc:
+            order_by_cols.append(col_attr.desc())
+        else:
+            order_by_cols.append(col_attr)
+    return query.order_by(*order_by_cols)
+
+
+def _model_condition(col_attr, value):
+    """Generate condition for one column.
+
+    Example for col_attr is name:
+        value is 'a': name == 'a'
+        value is ['a']: name == 'a'
+        value is ['a', 'b']: name == 'a' or name == 'b'
+        value is {'eq': 'a'}: name == 'a'
+        value is {'lt': 'a'}: name < 'a'
+        value is {'le': 'a'}: name <= 'a'
+        value is {'gt': 'a'}: name > 'a'
+        value is {'ge': 'a'}: name >= 'a'
+        value is {'ne': 'a'}: name != 'a'
+        value is {'in': ['a', 'b']}: name in ['a', 'b']
+        value is {'notin': ['a', 'b']}: name not in ['a', 'b']
+        value is {'startswith': 'abc'}: name like 'abc%'
+        value is {'endswith': 'abc'}: name like '%abc'
+        value is {'like': 'abc'}: name like '%abc%'
+        value is {'between': ('a', 'c')}: name >= 'a' and name <= 'c'
+        value is [{'lt': 'a'}]: name < 'a'
+        value is [{'lt': 'a'}, {'gt': c'}]: name < 'a' or name > 'c'
+        value is {'lt': 'c', 'gt': 'a'}: name > 'a' and name < 'c'
+
+    If value is a list, the condition is the or relationship among
+    conditions of each item.
+    If value is dict and there are multi keys in the dict, the relationship
+    is and conditions of each key.
+    Otherwise the condition is to compare the column with the value.
+    """
+    if isinstance(value, list):
+        basetype_values = []
+        composite_values = []
+        for item in value:
+            if isinstance(item, (list, dict)):
+                composite_values.append(item)
+            else:
+                basetype_values.append(item)
+        conditions = []
+        if basetype_values:
+            if len(basetype_values) == 1:
+                condition = (col_attr == basetype_values[0])
+            else:
+                condition = col_attr.in_(basetype_values)
+            conditions.append(condition)
+        for composite_value in composite_values:
+            condition = _model_condition(col_attr, composite_value)
+            if condition is not None:
+                conditions.append(condition)
+        if not conditions:
+            return None
+        if len(conditions) == 1:
+            return conditions[0]
+        return or_(*conditions)
+    elif isinstance(value, dict):
+        conditions = []
+        if 'eq' in value:
+            conditions.append(_model_condition_func(
+                col_attr, value['eq'],
+                lambda attr, data: attr == data,
+                lambda attr, data, item_condition_func: attr.in_(data)
+            ))
+        if 'lt' in value:
+            conditions.append(_model_condition_func(
+                col_attr, value['lt'],
+                lambda attr, data: attr < data,
+                _one_item_list_condition_func
+            ))
+        if 'gt' in value:
+            conditions.append(_model_condition_func(
+                col_attr, value['gt'],
+                lambda attr, data: attr > data,
+                _one_item_list_condition_func
+            ))
+        if 'le' in value:
+            conditions.append(_model_condition_func(
+                col_attr, value['le'],
+                lambda attr, data: attr <= data,
+                _one_item_list_condition_func
+            ))
+        if 'ge' in value:
+            conditions.append(_model_condition_func(
+                col_attr, value['ge'],
+                lambda attr, data: attr >= data,
+                _one_item_list_condition_func
+            ))
+        if 'ne' in value:
+            conditions.append(_model_condition_func(
+                col_attr, value['ne'],
+                lambda attr, data: attr != data,
+                lambda attr, data, item_condition_func: attr.notin_(data)
+            ))
+        if 'in' in value:
+            conditions.append(col_attr.in_(value['in']))
+        if 'notin' in value:
+            conditions.append(col_attr.notin_(value['notin']))
+        if 'startswith' in value:
+            conditions.append(_model_condition_func(
+                col_attr, value['startswith'],
+                lambda attr, data: attr.like('%s%%' % data)
+            ))
+        if 'endswith' in value:
+            conditions.append(_model_condition_func(
+                col_attr, value['endswith'],
+                lambda attr, data: attr.like('%%%s' % data)
+            ))
+        if 'like' in value:
+            conditions.append(_model_condition_func(
+                col_attr, value['like'],
+                lambda attr, data: attr.like('%%%s%%' % data)
+            ))
+        conditions = [
+            condition
+            for condition in conditions
+            if condition is not None
+        ]
+        if not conditions:
+            return None
+        if len(conditions) == 1:
+            return conditions[0]
+        return and_(conditions)
+    else:
+        condition = (col_attr == value)
+        return condition
+
+
+def _default_list_condition_func(col_attr, value, condition_func):
+    """The default condition func for a list of data.
+
+    Given the condition func for single item of data, this function
+    wrap the condition_func and return another condition func using
+    or_ to merge the conditions of each single item to deal with a
+    list of data item.
+
+    Args:
+       col_attr: the colomn name
+       value: the column value need to be compared.
+       condition_func: the sqlalchemy condition object like ==
+
+    Examples:
+       col_attr is name, value is ['a', 'b', 'c'] and
+       condition_func is ==, the returned condition is
+       name == 'a' or name == 'b' or name == 'c'
+    """
+    conditions = []
+    for sub_value in value:
+        condition = condition_func(col_attr, sub_value)
+        if condition is not None:
+            conditions.append(condition)
+    if conditions:
+        return or_(*conditions)
+    else:
+        return None
+
+
+def _one_item_list_condition_func(col_attr, value, condition_func):
+    """The wrapper condition func to deal with one item data list.
+
+    For simplification, it is used to reduce generating too complex
+    sql conditions.
+    """
+    if value:
+        return condition_func(col_attr, value[0])
+    else:
+        return None
+
+
+def _model_condition_func(
+    col_attr, value,
+    item_condition_func,
+    list_condition_func=_default_list_condition_func
+):
+    """Return sql condition based on value type."""
+    if isinstance(value, list):
+        if not value:
+            return None
+        if len(value) == 1:
+            return item_condition_func(col_attr, value)
+        return list_condition_func(
+            col_attr, value, item_condition_func
+        )
+    else:
+        return item_condition_func(col_attr, value)
+
+
+def model_filter(query, model, **filters):
+    """Append conditons to query for each possible column."""
+    for key, value in filters.items():
+        if isinstance(key, basestring):
+            if hasattr(model, key):
+                col_attr = getattr(model, key)
+            else:
+                continue
+        else:
+            col_attr = key
+
+        condition = _model_condition(col_attr, value)
+        if condition is not None:
+            query = query.filter(condition)
+    return query
+
+
+def model_query(session, model):
+    """model query.
+
+    Return sqlalchemy query object.
+    """
+    if not issubclass(model, models.BASE):
+        raise exception.DatabaseException("model should be sublass of BASE!")
+
+    return session.query(model)
+
+
+def wrap_to_dict(support_keys=[], **filters):
+    """Decrator to convert returned object to dict.
+
+    The details is decribed in _wrapper_dict.
+    """
+    def decorator(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            return _wrapper_dict(
+                func(*args, **kwargs), support_keys, **filters
+            )
+        return wrapper
+    return decorator
+
+
+def _wrapper_dict(data, support_keys, **filters):
+    """Helper for warpping db object into dictionary.
+
+    If data is list, convert it to a list of dict
+    If data is Base model, convert it to dict
+    for the data as a dict, filter it with the supported keys.
+    For each filter_key, filter_value  in filters, also filter
+    data[filter_key] by filter_value recursively if it exists.
+
+    Example:
+       data is models.Switch, it will be converted to
+       {
+           'id': 1, 'ip': '10.0.0.1', 'ip_int': 123456,
+           'credentials': {'version': 2, 'password': 'abc'}
+       }
+       Then if support_keys are ['id', 'ip', 'credentials'],
+       it will be filtered to {
+           'id': 1, 'ip': '10.0.0.1',
+           'credentials': {'version': 2, 'password': 'abc'}
+       }
+       Then if filters is {'credentials': ['version']},
+       it will be filtered to {
+           'id': 1, 'ip': '10.0.0.1',
+           'credentials': {'version': 2}
+       }
+    """
+    logging.debug(
+        'wrap dict %s by support_keys=%s filters=%s',
+        data, support_keys, filters
+    )
+    if isinstance(data, list):
+        return [
+            _wrapper_dict(item, support_keys, **filters)
+            for item in data
+        ]
+    if isinstance(data, models.ModelHandler):
+        data = data.to_dict()
+    if not isinstance(data, dict):
+        raise exception.InvalidResponse(
+            'response %s type is not dict' % data
+        )
+    info = {}
+    try:
+        if len(support_keys) == 0:
+            support_keys = data.keys()
+        for key in support_keys:
+            if key in data and data[key] is not None:
+                if key in filters:
+                    filter_keys = filters[key]
+                    if isinstance(filter_keys, dict):
+                        info[key] = _wrapper_dict(
+                            data[key], filter_keys.keys(),
+                            **filter_keys
+                        )
+                    else:
+                        info[key] = _wrapper_dict(
+                            data[key], filter_keys
+                        )
+                else:
+                    info[key] = data[key]
+        return info
+    except Exception as error:
+        logging.exception(error)
+        raise error