Merge compass-tasks-osa and compass-tasks-k8s

JIRA: COMPASS-568

rename compass-tasks to compass-tasks-base.
add both osa and k8s support in compass-tasks

Change-Id: I438f5b17e509d4cb751ced0ffe640ec70899882f
Signed-off-by: Harry Huang <huangxiangyu5@huawei.com>

1 files changed, 0 insertions, 731 deletions

diff --git a/compass-tasks/db/api/metadata_holder.py b/compass-tasks/db/api/metadata_holder.py
deleted file mode 100644
index 24afc67..0000000
--- a/compass-tasks/db/api/metadata_holder.py
+++ /dev/null
@@ -1,731 +0,0 @@
-# Copyright 2014 Huawei Technologies Co. Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Metadata related object holder."""
-import logging
-
-from compass.db.api import adapter as adapter_api
-from compass.db.api import adapter_holder as adapter_holder_api
-from compass.db.api import database
-from compass.db.api import metadata as metadata_api
-from compass.db.api import permission
-from compass.db.api import user as user_api
-from compass.db.api import utils
-from compass.db import exception
-from compass.db import models
-from compass.utils import setting_wrapper as setting
-from compass.utils import util
-
-
-RESP_METADATA_FIELDS = [
-    'os_config', 'package_config'
-]
-RESP_UI_METADATA_FIELDS = [
-    'os_global_config', 'flavor_config'
-]
-
-
-def load_metadatas(force_reload=False):
-    """Load metadatas."""
-    # TODO(xicheng): today we load metadata in memory as it original
-    # format in files in metadata.py. We get these inmemory metadata
-    # and do some translation, store the translated metadata into memory
-    # too in metadata_holder.py. api can only access the global inmemory
-    # data in metadata_holder.py.
-    _load_os_metadatas(force_reload=force_reload)
-    _load_package_metadatas(force_reload=force_reload)
-    _load_flavor_metadatas(force_reload=force_reload)
-    _load_os_metadata_ui_converters(force_reload=force_reload)
-    _load_flavor_metadata_ui_converters(force_reload=force_reload)
-
-
-def _load_os_metadata_ui_converters(force_reload=False):
-    global OS_METADATA_UI_CONVERTERS
-    if force_reload or OS_METADATA_UI_CONVERTERS is None:
-        logging.info('load os metadatas ui converters into memory')
-        OS_METADATA_UI_CONVERTERS = (
-            metadata_api.get_oses_metadata_ui_converters_internal(
-                force_reload=force_reload
-            )
-        )
-
-
-def _load_os_metadatas(force_reload=False):
-    """Load os metadata from inmemory db and map it by os_id."""
-    global OS_METADATA_MAPPING
-    if force_reload or OS_METADATA_MAPPING is None:
-        logging.info('load os metadatas into memory')
-        OS_METADATA_MAPPING = metadata_api.get_oses_metadata_internal(
-            force_reload=force_reload
-        )
-
-
-def _load_flavor_metadata_ui_converters(force_reload=False):
-    """Load flavor metadata ui converters from inmemory db.
-
-    The loaded metadata is mapped by flavor id.
-    """
-    global FLAVOR_METADATA_UI_CONVERTERS
-    if force_reload or FLAVOR_METADATA_UI_CONVERTERS is None:
-        logging.info('load flavor metadata ui converters into memory')
-        FLAVOR_METADATA_UI_CONVERTERS = {}
-        adapters_flavors_metadata_ui_converters = (
-            metadata_api.get_flavors_metadata_ui_converters_internal(
-                force_reload=force_reload
-            )
-        )
-        for adapter_name, adapter_flavors_metadata_ui_converters in (
-            adapters_flavors_metadata_ui_converters.items()
-        ):
-            for flavor_name, flavor_metadata_ui_converter in (
-                adapter_flavors_metadata_ui_converters.items()
-            ):
-                FLAVOR_METADATA_UI_CONVERTERS[
-                    '%s:%s' % (adapter_name, flavor_name)
-                ] = flavor_metadata_ui_converter
-
-
-@util.deprecated
-def _load_package_metadatas(force_reload=False):
-    """Load deployable package metadata from inmemory db."""
-    global PACKAGE_METADATA_MAPPING
-    if force_reload or PACKAGE_METADATA_MAPPING is None:
-        logging.info('load package metadatas into memory')
-        PACKAGE_METADATA_MAPPING = (
-            metadata_api.get_packages_metadata_internal(
-                force_reload=force_reload
-            )
-        )
-
-
-def _load_flavor_metadatas(force_reload=False):
-    """Load flavor metadata from inmemory db.
-
-    The loaded metadata are mapped by flavor id.
-    """
-    global FLAVOR_METADATA_MAPPING
-    if force_reload or FLAVOR_METADATA_MAPPING is None:
-        logging.info('load flavor metadatas into memory')
-        FLAVOR_METADATA_MAPPING = {}
-        adapters_flavors_metadata = (
-            metadata_api.get_flavors_metadata_internal(
-                force_reload=force_reload
-            )
-        )
-        for adapter_name, adapter_flavors_metadata in (
-            adapters_flavors_metadata.items()
-        ):
-            for flavor_name, flavor_metadata in (
-                adapter_flavors_metadata.items()
-            ):
-                FLAVOR_METADATA_MAPPING[
-                    '%s:%s' % (adapter_name, flavor_name)
-                ] = flavor_metadata
-
-
-OS_METADATA_MAPPING = None
-PACKAGE_METADATA_MAPPING = None
-FLAVOR_METADATA_MAPPING = None
-OS_METADATA_UI_CONVERTERS = None
-FLAVOR_METADATA_UI_CONVERTERS = None
-
-
-def validate_os_config(
-    config, os_id, whole_check=False, **kwargs
-):
-    """Validate os config."""
-    load_metadatas()
-    if os_id not in OS_METADATA_MAPPING:
-        raise exception.InvalidParameter(
-            'os %s is not found in os metadata mapping' % os_id
-        )
-    _validate_config(
-        '', config, OS_METADATA_MAPPING[os_id],
-        whole_check, **kwargs
-    )
-
-
-@util.deprecated
-def validate_package_config(
-    config, adapter_id, whole_check=False, **kwargs
-):
-    """Validate package config."""
-    load_metadatas()
-    if adapter_id not in PACKAGE_METADATA_MAPPING:
-        raise exception.InvalidParameter(
-            'adapter %s is not found in package metedata mapping' % adapter_id
-        )
-    _validate_config(
-        '', config, PACKAGE_METADATA_MAPPING[adapter_id],
-        whole_check, **kwargs
-    )
-
-
-def validate_flavor_config(
-    config, flavor_id, whole_check=False, **kwargs
-):
-    """Validate flavor config."""
-    load_metadatas()
-    if not flavor_id:
-        logging.info('There is no flavor, skipping flavor validation...')
-    elif flavor_id not in FLAVOR_METADATA_MAPPING:
-        raise exception.InvalidParameter(
-            'flavor %s is not found in flavor metedata mapping' % flavor_id
-        )
-    else:
-        _validate_config(
-            '', config, FLAVOR_METADATA_MAPPING[flavor_id],
-            whole_check, **kwargs
-        )
-
-
-def _filter_metadata(metadata, **kwargs):
-    """Filter metadata before return it to api.
-
-    Some metadata fields are not json compatible or
-    only used in db/api internally.
-    We should strip these fields out before return to api.
-    """
-    if not isinstance(metadata, dict):
-        return metadata
-    filtered_metadata = {}
-    for key, value in metadata.items():
-        if key == '_self':
-            filtered_metadata[key] = {
-                'name': value['name'],
-                'description': value.get('description', None),
-                'default_value': value.get('default_value', None),
-                'is_required': value.get('is_required', False),
-                'required_in_whole_config': value.get(
-                    'required_in_whole_config', False),
-                'js_validator': value.get('js_validator', None),
-                'options': value.get('options', None),
-                'required_in_options': value.get(
-                    'required_in_options', False),
-                'field_type': value.get(
-                    'field_type_data', 'str'),
-                'display_type': value.get('display_type', None),
-                'mapping_to': value.get('mapping_to', None)
-            }
-        else:
-            filtered_metadata[key] = _filter_metadata(value, **kwargs)
-    return filtered_metadata
-
-
-@util.deprecated
-def _get_package_metadata(adapter_id):
-    """get package metadata."""
-    load_metadatas()
-    if adapter_id not in PACKAGE_METADATA_MAPPING:
-        raise exception.RecordNotExists(
-            'adpater %s does not exist' % adapter_id
-        )
-    return _filter_metadata(
-        PACKAGE_METADATA_MAPPING[adapter_id]
-    )
-
-
-@util.deprecated
-@utils.supported_filters([])
-@database.run_in_session()
-@user_api.check_user_permission(
-    permission.PERMISSION_LIST_METADATAS
-)
-@utils.wrap_to_dict(RESP_METADATA_FIELDS)
-def get_package_metadata(adapter_id, user=None, session=None, **kwargs):
-    """Get package metadata from adapter."""
-    return {
-        'package_config': _get_package_metadata(adapter_id)
-    }
-
-
-def _get_flavor_metadata(flavor_id):
-    """get flavor metadata."""
-    load_metadatas()
-    if not flavor_id:
-        logging.info('There is no flavor id, skipping...')
-    elif flavor_id not in FLAVOR_METADATA_MAPPING:
-        raise exception.RecordNotExists(
-            'flavor %s does not exist' % flavor_id
-        )
-    else:
-        return _filter_metadata(FLAVOR_METADATA_MAPPING[flavor_id])
-
-
-@utils.supported_filters([])
-@database.run_in_session()
-@user_api.check_user_permission(
-    permission.PERMISSION_LIST_METADATAS
-)
-@utils.wrap_to_dict(RESP_METADATA_FIELDS)
-def get_flavor_metadata(flavor_id, user=None, session=None, **kwargs):
-    """Get flavor metadata by flavor."""
-    return {
-        'package_config': _get_flavor_metadata(flavor_id)
-    }
-
-
-def _get_os_metadata(os_id):
-    """get os metadata."""
-    load_metadatas()
-    if os_id not in OS_METADATA_MAPPING:
-        raise exception.RecordNotExists(
-            'os %s does not exist' % os_id
-        )
-    return _filter_metadata(OS_METADATA_MAPPING[os_id])
-
-
-def _get_os_metadata_ui_converter(os_id):
-    """get os metadata ui converter."""
-    load_metadatas()
-    if os_id not in OS_METADATA_UI_CONVERTERS:
-        raise exception.RecordNotExists(
-            'os %s does not exist' % os_id
-        )
-    return OS_METADATA_UI_CONVERTERS[os_id]
-
-
-def _get_flavor_metadata_ui_converter(flavor_id):
-    """get flavor metadata ui converter."""
-    load_metadatas()
-    if flavor_id not in FLAVOR_METADATA_UI_CONVERTERS:
-        raise exception.RecordNotExists(
-            'flavor %s does not exist' % flavor_id
-        )
-    return FLAVOR_METADATA_UI_CONVERTERS[flavor_id]
-
-
-@utils.supported_filters([])
-@database.run_in_session()
-@user_api.check_user_permission(
-    permission.PERMISSION_LIST_METADATAS
-)
-@utils.wrap_to_dict(RESP_METADATA_FIELDS)
-def get_os_metadata(os_id, user=None, session=None, **kwargs):
-    """get os metadatas."""
-    return {'os_config': _get_os_metadata(os_id)}
-
-
-@utils.supported_filters([])
-@database.run_in_session()
-@user_api.check_user_permission(
-    permission.PERMISSION_LIST_METADATAS
-)
-@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS)
-def get_os_ui_metadata(os_id, user=None, session=None, **kwargs):
-    """Get os metadata ui converter by os."""
-    metadata = _get_os_metadata(os_id)
-    metadata_ui_converter = _get_os_metadata_ui_converter(os_id)
-    return _get_ui_metadata(metadata, metadata_ui_converter)
-
-
-@utils.supported_filters([])
-@database.run_in_session()
-@user_api.check_user_permission(
-    permission.PERMISSION_LIST_METADATAS
-)
-@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS)
-def get_flavor_ui_metadata(flavor_id, user=None, session=None, **kwargs):
-    """Get flavor ui metadata by flavor."""
-    metadata = _get_flavor_metadata(flavor_id)
-    metadata_ui_converter = _get_flavor_metadata_ui_converter(flavor_id)
-    return _get_ui_metadata(metadata, metadata_ui_converter)
-
-
-def _get_ui_metadata(metadata, metadata_ui_converter):
-    """convert metadata to ui metadata.
-
-     Args:
-        metadata: metadata we defined in metadata files.
-        metadata_ui_converter: metadata ui converter defined in metadata
-                               mapping files. Used to convert orignal
-                               metadata to ui understandable metadata.
-
-     Returns:
-        ui understandable metadata.
-     """
-    ui_metadata = {}
-    ui_metadata[metadata_ui_converter['mapped_name']] = []
-    for mapped_child in metadata_ui_converter['mapped_children']:
-        data_dict = {}
-        for ui_key, ui_value in mapped_child.items():
-            for key, value in ui_value.items():
-                if 'data' == key:
-                    result_data = []
-                    _get_ui_metadata_data(
-                        metadata[ui_key], value, result_data
-                    )
-                    data_dict['data'] = result_data
-                else:
-                    data_dict[key] = value
-        ui_metadata[metadata_ui_converter['mapped_name']].append(data_dict)
-    return ui_metadata
-
-
-def _get_ui_metadata_data(metadata, config, result_data):
-    """Get ui metadata data and fill to result."""
-    data_dict = {}
-    for key, config_value in config.items():
-        if isinstance(config_value, dict) and key != 'content_data':
-            if key in metadata.keys():
-                _get_ui_metadata_data(metadata[key], config_value, result_data)
-            else:
-                _get_ui_metadata_data(metadata, config_value, result_data)
-        elif isinstance(config_value, list):
-            option_list = []
-            for item in config_value:
-                if isinstance(item, dict):
-                    option_list.append(item)
-                    data_dict[key] = option_list
-                else:
-                    if isinstance(metadata['_self'][item], bool):
-                        data_dict[item] = str(metadata['_self'][item]).lower()
-                    else:
-                        data_dict[item] = metadata['_self'][item]
-        else:
-            data_dict[key] = config_value
-    if data_dict:
-        result_data.append(data_dict)
-    return result_data
-
-
-@util.deprecated
-@utils.supported_filters([])
-@database.run_in_session()
-@user_api.check_user_permission(
-    permission.PERMISSION_LIST_METADATAS
-)
-@utils.wrap_to_dict(RESP_METADATA_FIELDS)
-def get_package_os_metadata(
-    adapter_id, os_id,
-    user=None, session=None, **kwargs
-):
-    """Get metadata by adapter and os."""
-    adapter = adapter_holder_api.get_adapter(
-        adapter_id, user=user, session=session
-    )
-    os_ids = [os['id'] for os in adapter['supported_oses']]
-    if os_id not in os_ids:
-        raise exception.InvalidParameter(
-            'os %s is not in the supported os list of adapter %s' % (
-                os_id, adapter_id
-            )
-        )
-    metadatas = {}
-    metadatas['os_config'] = _get_os_metadata(
-        os_id
-    )
-    metadatas['package_config'] = _get_package_metadata(
-        adapter_id
-    )
-    return metadatas
-
-
-@utils.supported_filters([])
-@database.run_in_session()
-@user_api.check_user_permission(
-    permission.PERMISSION_LIST_METADATAS
-)
-@utils.wrap_to_dict(RESP_METADATA_FIELDS)
-def get_flavor_os_metadata(
-    flavor_id, os_id,
-    user=None, session=None, **kwargs
-):
-    """Get metadata by flavor and os."""
-    flavor = adapter_holder_api.get_flavor(
-        flavor_id, user=user, session=session
-    )
-    adapter_id = flavor['adapter_id']
-    adapter = adapter_holder_api.get_adapter(
-        adapter_id, user=user, session=session
-    )
-    os_ids = [os['id'] for os in adapter['supported_oses']]
-    if os_id not in os_ids:
-        raise exception.InvalidParameter(
-            'os %s is not in the supported os list of adapter %s' % (
-                os_id, adapter_id
-            )
-        )
-    metadatas = {}
-    metadatas['os_config'] = _get_os_metadata(
-        session, os_id
-    )
-    metadatas['package_config'] = _get_flavor_metadata(
-        session, flavor_id
-    )
-    return metadatas
-
-
-def _validate_self(
-    config_path, config_key, config,
-    metadata, whole_check,
-    **kwargs
-):
-    """validate config by metadata self section."""
-    logging.debug('validate config self %s', config_path)
-    if '_self' not in metadata:
-        if isinstance(config, dict):
-            _validate_config(
-                config_path, config, metadata, whole_check, **kwargs
-            )
-        return
-    field_type = metadata['_self'].get('field_type', basestring)
-    if not isinstance(config, field_type):
-        raise exception.InvalidParameter(
-            '%s config type is not %s: %s' % (config_path, field_type, config)
-        )
-    is_required = metadata['_self'].get(
-        'is_required', False
-    )
-    required_in_whole_config = metadata['_self'].get(
-        'required_in_whole_config', False
-    )
-    if isinstance(config, basestring):
-        if config == '' and not is_required and not required_in_whole_config:
-            # ignore empty config when it is optional
-            return
-    required_in_options = metadata['_self'].get(
-        'required_in_options', False
-    )
-    options = metadata['_self'].get('options', None)
-    if required_in_options:
-        if field_type in [int, basestring, float, bool]:
-            if options and config not in options:
-                raise exception.InvalidParameter(
-                    '%s config is not in %s: %s' % (
-                        config_path, options, config
-                    )
-                )
-        elif field_type in [list, tuple]:
-            if options and not set(config).issubset(set(options)):
-                raise exception.InvalidParameter(
-                    '%s config is not in %s: %s' % (
-                        config_path, options, config
-                    )
-                )
-        elif field_type == dict:
-            if options and not set(config.keys()).issubset(set(options)):
-                raise exception.InvalidParameter(
-                    '%s config is not in %s: %s' % (
-                        config_path, options, config
-                    )
-                )
-    validator = metadata['_self'].get('validator', None)
-    logging.debug('validate by validator %s', validator)
-    if validator:
-        if not validator(config_key, config, **kwargs):
-            raise exception.InvalidParameter(
-                '%s config is invalid' % config_path
-            )
-    if isinstance(config, dict):
-        _validate_config(
-            config_path, config, metadata, whole_check, **kwargs
-        )
-
-
-def _validate_config(
-    config_path, config, metadata, whole_check,
-    **kwargs
-):
-    """validate config by metadata."""
-    logging.debug('validate config %s', config_path)
-    generals = {}
-    specified = {}
-    for key, value in metadata.items():
-        if key.startswith('$'):
-            generals[key] = value
-        elif key.startswith('_'):
-            pass
-        else:
-            specified[key] = value
-    config_keys = set(config.keys())
-    specified_keys = set(specified.keys())
-    intersect_keys = config_keys & specified_keys
-    not_found_keys = config_keys - specified_keys
-    redundant_keys = specified_keys - config_keys
-    for key in redundant_keys:
-        if '_self' not in specified[key]:
-            continue
-        if specified[key]['_self'].get('is_required', False):
-            raise exception.InvalidParameter(
-                '%s/%s does not find but it is required' % (
-                    config_path, key
-                )
-            )
-        if (
-            whole_check and
-            specified[key]['_self'].get(
-                'required_in_whole_config', False
-            )
-        ):
-            raise exception.InvalidParameter(
-                '%s/%s does not find but it is required in whole config' % (
-                    config_path, key
-                )
-            )
-    for key in intersect_keys:
-        _validate_self(
-            '%s/%s' % (config_path, key),
-            key, config[key], specified[key], whole_check,
-            **kwargs
-        )
-    for key in not_found_keys:
-        if not generals:
-            raise exception.InvalidParameter(
-                'key %s missing in metadata %s' % (
-                    key, config_path
-                )
-            )
-        for general_key, general_value in generals.items():
-            _validate_self(
-                '%s/%s' % (config_path, key),
-                key, config[key], general_value, whole_check,
-                **kwargs
-            )
-
-
-def _autofill_self_config(
-    config_path, config_key, config,
-    metadata,
-    **kwargs
-):
-    """Autofill config by metadata self section."""
-    if '_self' not in metadata:
-        if isinstance(config, dict):
-            _autofill_config(
-                config_path, config, metadata, **kwargs
-            )
-        return config
-    logging.debug(
-        'autofill %s by metadata %s', config_path, metadata['_self']
-    )
-    autofill_callback = metadata['_self'].get(
-        'autofill_callback', None
-    )
-    autofill_callback_params = metadata['_self'].get(
-        'autofill_callback_params', {}
-    )
-    callback_params = dict(kwargs)
-    if autofill_callback_params:
-        callback_params.update(autofill_callback_params)
-    default_value = metadata['_self'].get(
-        'default_value', None
-    )
-    if default_value is not None:
-        callback_params['default_value'] = default_value
-    options = metadata['_self'].get(
-        'options', None
-    )
-    if options is not None:
-        callback_params['options'] = options
-    if autofill_callback:
-        config = autofill_callback(
-            config_key, config, **callback_params
-        )
-    if config is None:
-        new_config = {}
-    else:
-        new_config = config
-    if isinstance(new_config, dict):
-        _autofill_config(
-            config_path, new_config, metadata, **kwargs
-        )
-        if new_config:
-            config = new_config
-    return config
-
-
-def _autofill_config(
-    config_path, config, metadata, **kwargs
-):
-    """autofill config by metadata."""
-    generals = {}
-    specified = {}
-    for key, value in metadata.items():
-        if key.startswith('$'):
-            generals[key] = value
-        elif key.startswith('_'):
-            pass
-        else:
-            specified[key] = value
-    config_keys = set(config.keys())
-    specified_keys = set(specified.keys())
-    intersect_keys = config_keys & specified_keys
-    not_found_keys = config_keys - specified_keys
-    redundant_keys = specified_keys - config_keys
-    for key in redundant_keys:
-        self_config = _autofill_self_config(
-            '%s/%s' % (config_path, key),
-            key, None, specified[key], **kwargs
-        )
-        if self_config is not None:
-            config[key] = self_config
-    for key in intersect_keys:
-        config[key] = _autofill_self_config(
-            '%s/%s' % (config_path, key),
-            key, config[key], specified[key],
-            **kwargs
-        )
-    for key in not_found_keys:
-        for general_key, general_value in generals.items():
-            config[key] = _autofill_self_config(
-                '%s/%s' % (config_path, key),
-                key, config[key], general_value,
-                **kwargs
-            )
-    return config
-
-
-def autofill_os_config(
-    config, os_id, **kwargs
-):
-    load_metadatas()
-    if os_id not in OS_METADATA_MAPPING:
-        raise exception.InvalidParameter(
-            'os %s is not found in os metadata mapping' % os_id
-        )
-
-    return _autofill_config(
-        '', config, OS_METADATA_MAPPING[os_id], **kwargs
-    )
-
-
-def autofill_package_config(
-    config, adapter_id, **kwargs
-):
-    load_metadatas()
-    if adapter_id not in PACKAGE_METADATA_MAPPING:
-        raise exception.InvalidParameter(
-            'adapter %s is not found in package metadata mapping' % adapter_id
-        )
-
-    return _autofill_config(
-        '', config, PACKAGE_METADATA_MAPPING[adapter_id], **kwargs
-    )
-
-
-def autofill_flavor_config(
-    config, flavor_id, **kwargs
-):
-    load_metadatas()
-    if not flavor_id:
-        logging.info('There is no flavor, skipping...')
-    elif flavor_id not in FLAVOR_METADATA_MAPPING:
-        raise exception.InvalidParameter(
-            'flavor %s is not found in flavor metadata mapping' % flavor_id
-        )
-    else:
-        return _autofill_config(
-            '', config, FLAVOR_METADATA_MAPPING[flavor_id], **kwargs
-        )