Merge compass-tasks-osa and compass-tasks-k8s

JIRA: COMPASS-568

rename compass-tasks to compass-tasks-base.
add both osa and k8s support in compass-tasks

Change-Id: I438f5b17e509d4cb751ced0ffe640ec70899882f
Signed-off-by: Harry Huang <huangxiangyu5@huawei.com>

1 files changed, 731 insertions, 0 deletions

diff --git a/compass-tasks-base/db/api/metadata_holder.py b/compass-tasks-base/db/api/metadata_holder.py
new file mode 100644
index 0000000..24afc67
--- /dev/null
+++ b/compass-tasks-base/db/api/metadata_holder.py
@@ -0,0 +1,731 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Metadata related object holder."""
+import logging
+
+from compass.db.api import adapter as adapter_api
+from compass.db.api import adapter_holder as adapter_holder_api
+from compass.db.api import database
+from compass.db.api import metadata as metadata_api
+from compass.db.api import permission
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+from compass.utils import setting_wrapper as setting
+from compass.utils import util
+
+
+RESP_METADATA_FIELDS = [
+    'os_config', 'package_config'
+]
+RESP_UI_METADATA_FIELDS = [
+    'os_global_config', 'flavor_config'
+]
+
+
+def load_metadatas(force_reload=False):
+    """Load metadatas."""
+    # TODO(xicheng): today we load metadata in memory as it original
+    # format in files in metadata.py. We get these inmemory metadata
+    # and do some translation, store the translated metadata into memory
+    # too in metadata_holder.py. api can only access the global inmemory
+    # data in metadata_holder.py.
+    _load_os_metadatas(force_reload=force_reload)
+    _load_package_metadatas(force_reload=force_reload)
+    _load_flavor_metadatas(force_reload=force_reload)
+    _load_os_metadata_ui_converters(force_reload=force_reload)
+    _load_flavor_metadata_ui_converters(force_reload=force_reload)
+
+
+def _load_os_metadata_ui_converters(force_reload=False):
+    global OS_METADATA_UI_CONVERTERS
+    if force_reload or OS_METADATA_UI_CONVERTERS is None:
+        logging.info('load os metadatas ui converters into memory')
+        OS_METADATA_UI_CONVERTERS = (
+            metadata_api.get_oses_metadata_ui_converters_internal(
+                force_reload=force_reload
+            )
+        )
+
+
+def _load_os_metadatas(force_reload=False):
+    """Load os metadata from inmemory db and map it by os_id."""
+    global OS_METADATA_MAPPING
+    if force_reload or OS_METADATA_MAPPING is None:
+        logging.info('load os metadatas into memory')
+        OS_METADATA_MAPPING = metadata_api.get_oses_metadata_internal(
+            force_reload=force_reload
+        )
+
+
+def _load_flavor_metadata_ui_converters(force_reload=False):
+    """Load flavor metadata ui converters from inmemory db.
+
+    The loaded metadata is mapped by flavor id.
+    """
+    global FLAVOR_METADATA_UI_CONVERTERS
+    if force_reload or FLAVOR_METADATA_UI_CONVERTERS is None:
+        logging.info('load flavor metadata ui converters into memory')
+        FLAVOR_METADATA_UI_CONVERTERS = {}
+        adapters_flavors_metadata_ui_converters = (
+            metadata_api.get_flavors_metadata_ui_converters_internal(
+                force_reload=force_reload
+            )
+        )
+        for adapter_name, adapter_flavors_metadata_ui_converters in (
+            adapters_flavors_metadata_ui_converters.items()
+        ):
+            for flavor_name, flavor_metadata_ui_converter in (
+                adapter_flavors_metadata_ui_converters.items()
+            ):
+                FLAVOR_METADATA_UI_CONVERTERS[
+                    '%s:%s' % (adapter_name, flavor_name)
+                ] = flavor_metadata_ui_converter
+
+
+@util.deprecated
+def _load_package_metadatas(force_reload=False):
+    """Load deployable package metadata from inmemory db."""
+    global PACKAGE_METADATA_MAPPING
+    if force_reload or PACKAGE_METADATA_MAPPING is None:
+        logging.info('load package metadatas into memory')
+        PACKAGE_METADATA_MAPPING = (
+            metadata_api.get_packages_metadata_internal(
+                force_reload=force_reload
+            )
+        )
+
+
+def _load_flavor_metadatas(force_reload=False):
+    """Load flavor metadata from inmemory db.
+
+    The loaded metadata are mapped by flavor id.
+    """
+    global FLAVOR_METADATA_MAPPING
+    if force_reload or FLAVOR_METADATA_MAPPING is None:
+        logging.info('load flavor metadatas into memory')
+        FLAVOR_METADATA_MAPPING = {}
+        adapters_flavors_metadata = (
+            metadata_api.get_flavors_metadata_internal(
+                force_reload=force_reload
+            )
+        )
+        for adapter_name, adapter_flavors_metadata in (
+            adapters_flavors_metadata.items()
+        ):
+            for flavor_name, flavor_metadata in (
+                adapter_flavors_metadata.items()
+            ):
+                FLAVOR_METADATA_MAPPING[
+                    '%s:%s' % (adapter_name, flavor_name)
+                ] = flavor_metadata
+
+
+OS_METADATA_MAPPING = None
+PACKAGE_METADATA_MAPPING = None
+FLAVOR_METADATA_MAPPING = None
+OS_METADATA_UI_CONVERTERS = None
+FLAVOR_METADATA_UI_CONVERTERS = None
+
+
+def validate_os_config(
+    config, os_id, whole_check=False, **kwargs
+):
+    """Validate os config."""
+    load_metadatas()
+    if os_id not in OS_METADATA_MAPPING:
+        raise exception.InvalidParameter(
+            'os %s is not found in os metadata mapping' % os_id
+        )
+    _validate_config(
+        '', config, OS_METADATA_MAPPING[os_id],
+        whole_check, **kwargs
+    )
+
+
+@util.deprecated
+def validate_package_config(
+    config, adapter_id, whole_check=False, **kwargs
+):
+    """Validate package config."""
+    load_metadatas()
+    if adapter_id not in PACKAGE_METADATA_MAPPING:
+        raise exception.InvalidParameter(
+            'adapter %s is not found in package metedata mapping' % adapter_id
+        )
+    _validate_config(
+        '', config, PACKAGE_METADATA_MAPPING[adapter_id],
+        whole_check, **kwargs
+    )
+
+
+def validate_flavor_config(
+    config, flavor_id, whole_check=False, **kwargs
+):
+    """Validate flavor config."""
+    load_metadatas()
+    if not flavor_id:
+        logging.info('There is no flavor, skipping flavor validation...')
+    elif flavor_id not in FLAVOR_METADATA_MAPPING:
+        raise exception.InvalidParameter(
+            'flavor %s is not found in flavor metedata mapping' % flavor_id
+        )
+    else:
+        _validate_config(
+            '', config, FLAVOR_METADATA_MAPPING[flavor_id],
+            whole_check, **kwargs
+        )
+
+
+def _filter_metadata(metadata, **kwargs):
+    """Filter metadata before return it to api.
+
+    Some metadata fields are not json compatible or
+    only used in db/api internally.
+    We should strip these fields out before return to api.
+    """
+    if not isinstance(metadata, dict):
+        return metadata
+    filtered_metadata = {}
+    for key, value in metadata.items():
+        if key == '_self':
+            filtered_metadata[key] = {
+                'name': value['name'],
+                'description': value.get('description', None),
+                'default_value': value.get('default_value', None),
+                'is_required': value.get('is_required', False),
+                'required_in_whole_config': value.get(
+                    'required_in_whole_config', False),
+                'js_validator': value.get('js_validator', None),
+                'options': value.get('options', None),
+                'required_in_options': value.get(
+                    'required_in_options', False),
+                'field_type': value.get(
+                    'field_type_data', 'str'),
+                'display_type': value.get('display_type', None),
+                'mapping_to': value.get('mapping_to', None)
+            }
+        else:
+            filtered_metadata[key] = _filter_metadata(value, **kwargs)
+    return filtered_metadata
+
+
+@util.deprecated
+def _get_package_metadata(adapter_id):
+    """get package metadata."""
+    load_metadatas()
+    if adapter_id not in PACKAGE_METADATA_MAPPING:
+        raise exception.RecordNotExists(
+            'adpater %s does not exist' % adapter_id
+        )
+    return _filter_metadata(
+        PACKAGE_METADATA_MAPPING[adapter_id]
+    )
+
+
+@util.deprecated
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+    permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_package_metadata(adapter_id, user=None, session=None, **kwargs):
+    """Get package metadata from adapter."""
+    return {
+        'package_config': _get_package_metadata(adapter_id)
+    }
+
+
+def _get_flavor_metadata(flavor_id):
+    """get flavor metadata."""
+    load_metadatas()
+    if not flavor_id:
+        logging.info('There is no flavor id, skipping...')
+    elif flavor_id not in FLAVOR_METADATA_MAPPING:
+        raise exception.RecordNotExists(
+            'flavor %s does not exist' % flavor_id
+        )
+    else:
+        return _filter_metadata(FLAVOR_METADATA_MAPPING[flavor_id])
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+    permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_flavor_metadata(flavor_id, user=None, session=None, **kwargs):
+    """Get flavor metadata by flavor."""
+    return {
+        'package_config': _get_flavor_metadata(flavor_id)
+    }
+
+
+def _get_os_metadata(os_id):
+    """get os metadata."""
+    load_metadatas()
+    if os_id not in OS_METADATA_MAPPING:
+        raise exception.RecordNotExists(
+            'os %s does not exist' % os_id
+        )
+    return _filter_metadata(OS_METADATA_MAPPING[os_id])
+
+
+def _get_os_metadata_ui_converter(os_id):
+    """get os metadata ui converter."""
+    load_metadatas()
+    if os_id not in OS_METADATA_UI_CONVERTERS:
+        raise exception.RecordNotExists(
+            'os %s does not exist' % os_id
+        )
+    return OS_METADATA_UI_CONVERTERS[os_id]
+
+
+def _get_flavor_metadata_ui_converter(flavor_id):
+    """get flavor metadata ui converter."""
+    load_metadatas()
+    if flavor_id not in FLAVOR_METADATA_UI_CONVERTERS:
+        raise exception.RecordNotExists(
+            'flavor %s does not exist' % flavor_id
+        )
+    return FLAVOR_METADATA_UI_CONVERTERS[flavor_id]
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+    permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_os_metadata(os_id, user=None, session=None, **kwargs):
+    """get os metadatas."""
+    return {'os_config': _get_os_metadata(os_id)}
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+    permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS)
+def get_os_ui_metadata(os_id, user=None, session=None, **kwargs):
+    """Get os metadata ui converter by os."""
+    metadata = _get_os_metadata(os_id)
+    metadata_ui_converter = _get_os_metadata_ui_converter(os_id)
+    return _get_ui_metadata(metadata, metadata_ui_converter)
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+    permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS)
+def get_flavor_ui_metadata(flavor_id, user=None, session=None, **kwargs):
+    """Get flavor ui metadata by flavor."""
+    metadata = _get_flavor_metadata(flavor_id)
+    metadata_ui_converter = _get_flavor_metadata_ui_converter(flavor_id)
+    return _get_ui_metadata(metadata, metadata_ui_converter)
+
+
+def _get_ui_metadata(metadata, metadata_ui_converter):
+    """convert metadata to ui metadata.
+
+     Args:
+        metadata: metadata we defined in metadata files.
+        metadata_ui_converter: metadata ui converter defined in metadata
+                               mapping files. Used to convert orignal
+                               metadata to ui understandable metadata.
+
+     Returns:
+        ui understandable metadata.
+     """
+    ui_metadata = {}
+    ui_metadata[metadata_ui_converter['mapped_name']] = []
+    for mapped_child in metadata_ui_converter['mapped_children']:
+        data_dict = {}
+        for ui_key, ui_value in mapped_child.items():
+            for key, value in ui_value.items():
+                if 'data' == key:
+                    result_data = []
+                    _get_ui_metadata_data(
+                        metadata[ui_key], value, result_data
+                    )
+                    data_dict['data'] = result_data
+                else:
+                    data_dict[key] = value
+        ui_metadata[metadata_ui_converter['mapped_name']].append(data_dict)
+    return ui_metadata
+
+
+def _get_ui_metadata_data(metadata, config, result_data):
+    """Get ui metadata data and fill to result."""
+    data_dict = {}
+    for key, config_value in config.items():
+        if isinstance(config_value, dict) and key != 'content_data':
+            if key in metadata.keys():
+                _get_ui_metadata_data(metadata[key], config_value, result_data)
+            else:
+                _get_ui_metadata_data(metadata, config_value, result_data)
+        elif isinstance(config_value, list):
+            option_list = []
+            for item in config_value:
+                if isinstance(item, dict):
+                    option_list.append(item)
+                    data_dict[key] = option_list
+                else:
+                    if isinstance(metadata['_self'][item], bool):
+                        data_dict[item] = str(metadata['_self'][item]).lower()
+                    else:
+                        data_dict[item] = metadata['_self'][item]
+        else:
+            data_dict[key] = config_value
+    if data_dict:
+        result_data.append(data_dict)
+    return result_data
+
+
+@util.deprecated
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+    permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_package_os_metadata(
+    adapter_id, os_id,
+    user=None, session=None, **kwargs
+):
+    """Get metadata by adapter and os."""
+    adapter = adapter_holder_api.get_adapter(
+        adapter_id, user=user, session=session
+    )
+    os_ids = [os['id'] for os in adapter['supported_oses']]
+    if os_id not in os_ids:
+        raise exception.InvalidParameter(
+            'os %s is not in the supported os list of adapter %s' % (
+                os_id, adapter_id
+            )
+        )
+    metadatas = {}
+    metadatas['os_config'] = _get_os_metadata(
+        os_id
+    )
+    metadatas['package_config'] = _get_package_metadata(
+        adapter_id
+    )
+    return metadatas
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+    permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_flavor_os_metadata(
+    flavor_id, os_id,
+    user=None, session=None, **kwargs
+):
+    """Get metadata by flavor and os."""
+    flavor = adapter_holder_api.get_flavor(
+        flavor_id, user=user, session=session
+    )
+    adapter_id = flavor['adapter_id']
+    adapter = adapter_holder_api.get_adapter(
+        adapter_id, user=user, session=session
+    )
+    os_ids = [os['id'] for os in adapter['supported_oses']]
+    if os_id not in os_ids:
+        raise exception.InvalidParameter(
+            'os %s is not in the supported os list of adapter %s' % (
+                os_id, adapter_id
+            )
+        )
+    metadatas = {}
+    metadatas['os_config'] = _get_os_metadata(
+        session, os_id
+    )
+    metadatas['package_config'] = _get_flavor_metadata(
+        session, flavor_id
+    )
+    return metadatas
+
+
+def _validate_self(
+    config_path, config_key, config,
+    metadata, whole_check,
+    **kwargs
+):
+    """validate config by metadata self section."""
+    logging.debug('validate config self %s', config_path)
+    if '_self' not in metadata:
+        if isinstance(config, dict):
+            _validate_config(
+                config_path, config, metadata, whole_check, **kwargs
+            )
+        return
+    field_type = metadata['_self'].get('field_type', basestring)
+    if not isinstance(config, field_type):
+        raise exception.InvalidParameter(
+            '%s config type is not %s: %s' % (config_path, field_type, config)
+        )
+    is_required = metadata['_self'].get(
+        'is_required', False
+    )
+    required_in_whole_config = metadata['_self'].get(
+        'required_in_whole_config', False
+    )
+    if isinstance(config, basestring):
+        if config == '' and not is_required and not required_in_whole_config:
+            # ignore empty config when it is optional
+            return
+    required_in_options = metadata['_self'].get(
+        'required_in_options', False
+    )
+    options = metadata['_self'].get('options', None)
+    if required_in_options:
+        if field_type in [int, basestring, float, bool]:
+            if options and config not in options:
+                raise exception.InvalidParameter(
+                    '%s config is not in %s: %s' % (
+                        config_path, options, config
+                    )
+                )
+        elif field_type in [list, tuple]:
+            if options and not set(config).issubset(set(options)):
+                raise exception.InvalidParameter(
+                    '%s config is not in %s: %s' % (
+                        config_path, options, config
+                    )
+                )
+        elif field_type == dict:
+            if options and not set(config.keys()).issubset(set(options)):
+                raise exception.InvalidParameter(
+                    '%s config is not in %s: %s' % (
+                        config_path, options, config
+                    )
+                )
+    validator = metadata['_self'].get('validator', None)
+    logging.debug('validate by validator %s', validator)
+    if validator:
+        if not validator(config_key, config, **kwargs):
+            raise exception.InvalidParameter(
+                '%s config is invalid' % config_path
+            )
+    if isinstance(config, dict):
+        _validate_config(
+            config_path, config, metadata, whole_check, **kwargs
+        )
+
+
+def _validate_config(
+    config_path, config, metadata, whole_check,
+    **kwargs
+):
+    """validate config by metadata."""
+    logging.debug('validate config %s', config_path)
+    generals = {}
+    specified = {}
+    for key, value in metadata.items():
+        if key.startswith('$'):
+            generals[key] = value
+        elif key.startswith('_'):
+            pass
+        else:
+            specified[key] = value
+    config_keys = set(config.keys())
+    specified_keys = set(specified.keys())
+    intersect_keys = config_keys & specified_keys
+    not_found_keys = config_keys - specified_keys
+    redundant_keys = specified_keys - config_keys
+    for key in redundant_keys:
+        if '_self' not in specified[key]:
+            continue
+        if specified[key]['_self'].get('is_required', False):
+            raise exception.InvalidParameter(
+                '%s/%s does not find but it is required' % (
+                    config_path, key
+                )
+            )
+        if (
+            whole_check and
+            specified[key]['_self'].get(
+                'required_in_whole_config', False
+            )
+        ):
+            raise exception.InvalidParameter(
+                '%s/%s does not find but it is required in whole config' % (
+                    config_path, key
+                )
+            )
+    for key in intersect_keys:
+        _validate_self(
+            '%s/%s' % (config_path, key),
+            key, config[key], specified[key], whole_check,
+            **kwargs
+        )
+    for key in not_found_keys:
+        if not generals:
+            raise exception.InvalidParameter(
+                'key %s missing in metadata %s' % (
+                    key, config_path
+                )
+            )
+        for general_key, general_value in generals.items():
+            _validate_self(
+                '%s/%s' % (config_path, key),
+                key, config[key], general_value, whole_check,
+                **kwargs
+            )
+
+
+def _autofill_self_config(
+    config_path, config_key, config,
+    metadata,
+    **kwargs
+):
+    """Autofill config by metadata self section."""
+    if '_self' not in metadata:
+        if isinstance(config, dict):
+            _autofill_config(
+                config_path, config, metadata, **kwargs
+            )
+        return config
+    logging.debug(
+        'autofill %s by metadata %s', config_path, metadata['_self']
+    )
+    autofill_callback = metadata['_self'].get(
+        'autofill_callback', None
+    )
+    autofill_callback_params = metadata['_self'].get(
+        'autofill_callback_params', {}
+    )
+    callback_params = dict(kwargs)
+    if autofill_callback_params:
+        callback_params.update(autofill_callback_params)
+    default_value = metadata['_self'].get(
+        'default_value', None
+    )
+    if default_value is not None:
+        callback_params['default_value'] = default_value
+    options = metadata['_self'].get(
+        'options', None
+    )
+    if options is not None:
+        callback_params['options'] = options
+    if autofill_callback:
+        config = autofill_callback(
+            config_key, config, **callback_params
+        )
+    if config is None:
+        new_config = {}
+    else:
+        new_config = config
+    if isinstance(new_config, dict):
+        _autofill_config(
+            config_path, new_config, metadata, **kwargs
+        )
+        if new_config:
+            config = new_config
+    return config
+
+
+def _autofill_config(
+    config_path, config, metadata, **kwargs
+):
+    """autofill config by metadata."""
+    generals = {}
+    specified = {}
+    for key, value in metadata.items():
+        if key.startswith('$'):
+            generals[key] = value
+        elif key.startswith('_'):
+            pass
+        else:
+            specified[key] = value
+    config_keys = set(config.keys())
+    specified_keys = set(specified.keys())
+    intersect_keys = config_keys & specified_keys
+    not_found_keys = config_keys - specified_keys
+    redundant_keys = specified_keys - config_keys
+    for key in redundant_keys:
+        self_config = _autofill_self_config(
+            '%s/%s' % (config_path, key),
+            key, None, specified[key], **kwargs
+        )
+        if self_config is not None:
+            config[key] = self_config
+    for key in intersect_keys:
+        config[key] = _autofill_self_config(
+            '%s/%s' % (config_path, key),
+            key, config[key], specified[key],
+            **kwargs
+        )
+    for key in not_found_keys:
+        for general_key, general_value in generals.items():
+            config[key] = _autofill_self_config(
+                '%s/%s' % (config_path, key),
+                key, config[key], general_value,
+                **kwargs
+            )
+    return config
+
+
+def autofill_os_config(
+    config, os_id, **kwargs
+):
+    load_metadatas()
+    if os_id not in OS_METADATA_MAPPING:
+        raise exception.InvalidParameter(
+            'os %s is not found in os metadata mapping' % os_id
+        )
+
+    return _autofill_config(
+        '', config, OS_METADATA_MAPPING[os_id], **kwargs
+    )
+
+
+def autofill_package_config(
+    config, adapter_id, **kwargs
+):
+    load_metadatas()
+    if adapter_id not in PACKAGE_METADATA_MAPPING:
+        raise exception.InvalidParameter(
+            'adapter %s is not found in package metadata mapping' % adapter_id
+        )
+
+    return _autofill_config(
+        '', config, PACKAGE_METADATA_MAPPING[adapter_id], **kwargs
+    )
+
+
+def autofill_flavor_config(
+    config, flavor_id, **kwargs
+):
+    load_metadatas()
+    if not flavor_id:
+        logging.info('There is no flavor, skipping...')
+    elif flavor_id not in FLAVOR_METADATA_MAPPING:
+        raise exception.InvalidParameter(
+            'flavor %s is not found in flavor metadata mapping' % flavor_id
+        )
+    else:
+        return _autofill_config(
+            '', config, FLAVOR_METADATA_MAPPING[flavor_id], **kwargs
+        )