diff options
Diffstat (limited to 'compass-tasks/db/api/metadata_holder.py')
| -rw-r--r-- | compass-tasks/db/api/metadata_holder.py | 731 |
1 files changed, 731 insertions, 0 deletions
diff --git a/compass-tasks/db/api/metadata_holder.py b/compass-tasks/db/api/metadata_holder.py new file mode 100644 index 0000000..24afc67 --- /dev/null +++ b/compass-tasks/db/api/metadata_holder.py @@ -0,0 +1,731 @@ +# Copyright 2014 Huawei Technologies Co. Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Metadata related object holder.""" +import logging + +from compass.db.api import adapter as adapter_api +from compass.db.api import adapter_holder as adapter_holder_api +from compass.db.api import database +from compass.db.api import metadata as metadata_api +from compass.db.api import permission +from compass.db.api import user as user_api +from compass.db.api import utils +from compass.db import exception +from compass.db import models +from compass.utils import setting_wrapper as setting +from compass.utils import util + + +RESP_METADATA_FIELDS = [ + 'os_config', 'package_config' +] +RESP_UI_METADATA_FIELDS = [ + 'os_global_config', 'flavor_config' +] + + +def load_metadatas(force_reload=False): + """Load metadatas.""" + # TODO(xicheng): today we load metadata in memory as it original + # format in files in metadata.py. We get these inmemory metadata + # and do some translation, store the translated metadata into memory + # too in metadata_holder.py. api can only access the global inmemory + # data in metadata_holder.py. + _load_os_metadatas(force_reload=force_reload) + _load_package_metadatas(force_reload=force_reload) + _load_flavor_metadatas(force_reload=force_reload) + _load_os_metadata_ui_converters(force_reload=force_reload) + _load_flavor_metadata_ui_converters(force_reload=force_reload) + + +def _load_os_metadata_ui_converters(force_reload=False): + global OS_METADATA_UI_CONVERTERS + if force_reload or OS_METADATA_UI_CONVERTERS is None: + logging.info('load os metadatas ui converters into memory') + OS_METADATA_UI_CONVERTERS = ( + metadata_api.get_oses_metadata_ui_converters_internal( + force_reload=force_reload + ) + ) + + +def _load_os_metadatas(force_reload=False): + """Load os metadata from inmemory db and map it by os_id.""" + global OS_METADATA_MAPPING + if force_reload or OS_METADATA_MAPPING is None: + logging.info('load os metadatas into memory') + OS_METADATA_MAPPING = metadata_api.get_oses_metadata_internal( + force_reload=force_reload + ) + + +def _load_flavor_metadata_ui_converters(force_reload=False): + """Load flavor metadata ui converters from inmemory db. + + The loaded metadata is mapped by flavor id. + """ + global FLAVOR_METADATA_UI_CONVERTERS + if force_reload or FLAVOR_METADATA_UI_CONVERTERS is None: + logging.info('load flavor metadata ui converters into memory') + FLAVOR_METADATA_UI_CONVERTERS = {} + adapters_flavors_metadata_ui_converters = ( + metadata_api.get_flavors_metadata_ui_converters_internal( + force_reload=force_reload + ) + ) + for adapter_name, adapter_flavors_metadata_ui_converters in ( + adapters_flavors_metadata_ui_converters.items() + ): + for flavor_name, flavor_metadata_ui_converter in ( + adapter_flavors_metadata_ui_converters.items() + ): + FLAVOR_METADATA_UI_CONVERTERS[ + '%s:%s' % (adapter_name, flavor_name) + ] = flavor_metadata_ui_converter + + +@util.deprecated +def _load_package_metadatas(force_reload=False): + """Load deployable package metadata from inmemory db.""" + global PACKAGE_METADATA_MAPPING + if force_reload or PACKAGE_METADATA_MAPPING is None: + logging.info('load package metadatas into memory') + PACKAGE_METADATA_MAPPING = ( + metadata_api.get_packages_metadata_internal( + force_reload=force_reload + ) + ) + + +def _load_flavor_metadatas(force_reload=False): + """Load flavor metadata from inmemory db. + + The loaded metadata are mapped by flavor id. + """ + global FLAVOR_METADATA_MAPPING + if force_reload or FLAVOR_METADATA_MAPPING is None: + logging.info('load flavor metadatas into memory') + FLAVOR_METADATA_MAPPING = {} + adapters_flavors_metadata = ( + metadata_api.get_flavors_metadata_internal( + force_reload=force_reload + ) + ) + for adapter_name, adapter_flavors_metadata in ( + adapters_flavors_metadata.items() + ): + for flavor_name, flavor_metadata in ( + adapter_flavors_metadata.items() + ): + FLAVOR_METADATA_MAPPING[ + '%s:%s' % (adapter_name, flavor_name) + ] = flavor_metadata + + +OS_METADATA_MAPPING = None +PACKAGE_METADATA_MAPPING = None +FLAVOR_METADATA_MAPPING = None +OS_METADATA_UI_CONVERTERS = None +FLAVOR_METADATA_UI_CONVERTERS = None + + +def validate_os_config( + config, os_id, whole_check=False, **kwargs +): + """Validate os config.""" + load_metadatas() + if os_id not in OS_METADATA_MAPPING: + raise exception.InvalidParameter( + 'os %s is not found in os metadata mapping' % os_id + ) + _validate_config( + '', config, OS_METADATA_MAPPING[os_id], + whole_check, **kwargs + ) + + +@util.deprecated +def validate_package_config( + config, adapter_id, whole_check=False, **kwargs +): + """Validate package config.""" + load_metadatas() + if adapter_id not in PACKAGE_METADATA_MAPPING: + raise exception.InvalidParameter( + 'adapter %s is not found in package metedata mapping' % adapter_id + ) + _validate_config( + '', config, PACKAGE_METADATA_MAPPING[adapter_id], + whole_check, **kwargs + ) + + +def validate_flavor_config( + config, flavor_id, whole_check=False, **kwargs +): + """Validate flavor config.""" + load_metadatas() + if not flavor_id: + logging.info('There is no flavor, skipping flavor validation...') + elif flavor_id not in FLAVOR_METADATA_MAPPING: + raise exception.InvalidParameter( + 'flavor %s is not found in flavor metedata mapping' % flavor_id + ) + else: + _validate_config( + '', config, FLAVOR_METADATA_MAPPING[flavor_id], + whole_check, **kwargs + ) + + +def _filter_metadata(metadata, **kwargs): + """Filter metadata before return it to api. + + Some metadata fields are not json compatible or + only used in db/api internally. + We should strip these fields out before return to api. + """ + if not isinstance(metadata, dict): + return metadata + filtered_metadata = {} + for key, value in metadata.items(): + if key == '_self': + filtered_metadata[key] = { + 'name': value['name'], + 'description': value.get('description', None), + 'default_value': value.get('default_value', None), + 'is_required': value.get('is_required', False), + 'required_in_whole_config': value.get( + 'required_in_whole_config', False), + 'js_validator': value.get('js_validator', None), + 'options': value.get('options', None), + 'required_in_options': value.get( + 'required_in_options', False), + 'field_type': value.get( + 'field_type_data', 'str'), + 'display_type': value.get('display_type', None), + 'mapping_to': value.get('mapping_to', None) + } + else: + filtered_metadata[key] = _filter_metadata(value, **kwargs) + return filtered_metadata + + +@util.deprecated +def _get_package_metadata(adapter_id): + """get package metadata.""" + load_metadatas() + if adapter_id not in PACKAGE_METADATA_MAPPING: + raise exception.RecordNotExists( + 'adpater %s does not exist' % adapter_id + ) + return _filter_metadata( + PACKAGE_METADATA_MAPPING[adapter_id] + ) + + +@util.deprecated +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_METADATA_FIELDS) +def get_package_metadata(adapter_id, user=None, session=None, **kwargs): + """Get package metadata from adapter.""" + return { + 'package_config': _get_package_metadata(adapter_id) + } + + +def _get_flavor_metadata(flavor_id): + """get flavor metadata.""" + load_metadatas() + if not flavor_id: + logging.info('There is no flavor id, skipping...') + elif flavor_id not in FLAVOR_METADATA_MAPPING: + raise exception.RecordNotExists( + 'flavor %s does not exist' % flavor_id + ) + else: + return _filter_metadata(FLAVOR_METADATA_MAPPING[flavor_id]) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_METADATA_FIELDS) +def get_flavor_metadata(flavor_id, user=None, session=None, **kwargs): + """Get flavor metadata by flavor.""" + return { + 'package_config': _get_flavor_metadata(flavor_id) + } + + +def _get_os_metadata(os_id): + """get os metadata.""" + load_metadatas() + if os_id not in OS_METADATA_MAPPING: + raise exception.RecordNotExists( + 'os %s does not exist' % os_id + ) + return _filter_metadata(OS_METADATA_MAPPING[os_id]) + + +def _get_os_metadata_ui_converter(os_id): + """get os metadata ui converter.""" + load_metadatas() + if os_id not in OS_METADATA_UI_CONVERTERS: + raise exception.RecordNotExists( + 'os %s does not exist' % os_id + ) + return OS_METADATA_UI_CONVERTERS[os_id] + + +def _get_flavor_metadata_ui_converter(flavor_id): + """get flavor metadata ui converter.""" + load_metadatas() + if flavor_id not in FLAVOR_METADATA_UI_CONVERTERS: + raise exception.RecordNotExists( + 'flavor %s does not exist' % flavor_id + ) + return FLAVOR_METADATA_UI_CONVERTERS[flavor_id] + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_METADATA_FIELDS) +def get_os_metadata(os_id, user=None, session=None, **kwargs): + """get os metadatas.""" + return {'os_config': _get_os_metadata(os_id)} + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS) +def get_os_ui_metadata(os_id, user=None, session=None, **kwargs): + """Get os metadata ui converter by os.""" + metadata = _get_os_metadata(os_id) + metadata_ui_converter = _get_os_metadata_ui_converter(os_id) + return _get_ui_metadata(metadata, metadata_ui_converter) + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS) +def get_flavor_ui_metadata(flavor_id, user=None, session=None, **kwargs): + """Get flavor ui metadata by flavor.""" + metadata = _get_flavor_metadata(flavor_id) + metadata_ui_converter = _get_flavor_metadata_ui_converter(flavor_id) + return _get_ui_metadata(metadata, metadata_ui_converter) + + +def _get_ui_metadata(metadata, metadata_ui_converter): + """convert metadata to ui metadata. + + Args: + metadata: metadata we defined in metadata files. + metadata_ui_converter: metadata ui converter defined in metadata + mapping files. Used to convert orignal + metadata to ui understandable metadata. + + Returns: + ui understandable metadata. + """ + ui_metadata = {} + ui_metadata[metadata_ui_converter['mapped_name']] = [] + for mapped_child in metadata_ui_converter['mapped_children']: + data_dict = {} + for ui_key, ui_value in mapped_child.items(): + for key, value in ui_value.items(): + if 'data' == key: + result_data = [] + _get_ui_metadata_data( + metadata[ui_key], value, result_data + ) + data_dict['data'] = result_data + else: + data_dict[key] = value + ui_metadata[metadata_ui_converter['mapped_name']].append(data_dict) + return ui_metadata + + +def _get_ui_metadata_data(metadata, config, result_data): + """Get ui metadata data and fill to result.""" + data_dict = {} + for key, config_value in config.items(): + if isinstance(config_value, dict) and key != 'content_data': + if key in metadata.keys(): + _get_ui_metadata_data(metadata[key], config_value, result_data) + else: + _get_ui_metadata_data(metadata, config_value, result_data) + elif isinstance(config_value, list): + option_list = [] + for item in config_value: + if isinstance(item, dict): + option_list.append(item) + data_dict[key] = option_list + else: + if isinstance(metadata['_self'][item], bool): + data_dict[item] = str(metadata['_self'][item]).lower() + else: + data_dict[item] = metadata['_self'][item] + else: + data_dict[key] = config_value + if data_dict: + result_data.append(data_dict) + return result_data + + +@util.deprecated +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_METADATA_FIELDS) +def get_package_os_metadata( + adapter_id, os_id, + user=None, session=None, **kwargs +): + """Get metadata by adapter and os.""" + adapter = adapter_holder_api.get_adapter( + adapter_id, user=user, session=session + ) + os_ids = [os['id'] for os in adapter['supported_oses']] + if os_id not in os_ids: + raise exception.InvalidParameter( + 'os %s is not in the supported os list of adapter %s' % ( + os_id, adapter_id + ) + ) + metadatas = {} + metadatas['os_config'] = _get_os_metadata( + os_id + ) + metadatas['package_config'] = _get_package_metadata( + adapter_id + ) + return metadatas + + +@utils.supported_filters([]) +@database.run_in_session() +@user_api.check_user_permission( + permission.PERMISSION_LIST_METADATAS +) +@utils.wrap_to_dict(RESP_METADATA_FIELDS) +def get_flavor_os_metadata( + flavor_id, os_id, + user=None, session=None, **kwargs +): + """Get metadata by flavor and os.""" + flavor = adapter_holder_api.get_flavor( + flavor_id, user=user, session=session + ) + adapter_id = flavor['adapter_id'] + adapter = adapter_holder_api.get_adapter( + adapter_id, user=user, session=session + ) + os_ids = [os['id'] for os in adapter['supported_oses']] + if os_id not in os_ids: + raise exception.InvalidParameter( + 'os %s is not in the supported os list of adapter %s' % ( + os_id, adapter_id + ) + ) + metadatas = {} + metadatas['os_config'] = _get_os_metadata( + session, os_id + ) + metadatas['package_config'] = _get_flavor_metadata( + session, flavor_id + ) + return metadatas + + +def _validate_self( + config_path, config_key, config, + metadata, whole_check, + **kwargs +): + """validate config by metadata self section.""" + logging.debug('validate config self %s', config_path) + if '_self' not in metadata: + if isinstance(config, dict): + _validate_config( + config_path, config, metadata, whole_check, **kwargs + ) + return + field_type = metadata['_self'].get('field_type', basestring) + if not isinstance(config, field_type): + raise exception.InvalidParameter( + '%s config type is not %s: %s' % (config_path, field_type, config) + ) + is_required = metadata['_self'].get( + 'is_required', False + ) + required_in_whole_config = metadata['_self'].get( + 'required_in_whole_config', False + ) + if isinstance(config, basestring): + if config == '' and not is_required and not required_in_whole_config: + # ignore empty config when it is optional + return + required_in_options = metadata['_self'].get( + 'required_in_options', False + ) + options = metadata['_self'].get('options', None) + if required_in_options: + if field_type in [int, basestring, float, bool]: + if options and config not in options: + raise exception.InvalidParameter( + '%s config is not in %s: %s' % ( + config_path, options, config + ) + ) + elif field_type in [list, tuple]: + if options and not set(config).issubset(set(options)): + raise exception.InvalidParameter( + '%s config is not in %s: %s' % ( + config_path, options, config + ) + ) + elif field_type == dict: + if options and not set(config.keys()).issubset(set(options)): + raise exception.InvalidParameter( + '%s config is not in %s: %s' % ( + config_path, options, config + ) + ) + validator = metadata['_self'].get('validator', None) + logging.debug('validate by validator %s', validator) + if validator: + if not validator(config_key, config, **kwargs): + raise exception.InvalidParameter( + '%s config is invalid' % config_path + ) + if isinstance(config, dict): + _validate_config( + config_path, config, metadata, whole_check, **kwargs + ) + + +def _validate_config( + config_path, config, metadata, whole_check, + **kwargs +): + """validate config by metadata.""" + logging.debug('validate config %s', config_path) + generals = {} + specified = {} + for key, value in metadata.items(): + if key.startswith('$'): + generals[key] = value + elif key.startswith('_'): + pass + else: + specified[key] = value + config_keys = set(config.keys()) + specified_keys = set(specified.keys()) + intersect_keys = config_keys & specified_keys + not_found_keys = config_keys - specified_keys + redundant_keys = specified_keys - config_keys + for key in redundant_keys: + if '_self' not in specified[key]: + continue + if specified[key]['_self'].get('is_required', False): + raise exception.InvalidParameter( + '%s/%s does not find but it is required' % ( + config_path, key + ) + ) + if ( + whole_check and + specified[key]['_self'].get( + 'required_in_whole_config', False + ) + ): + raise exception.InvalidParameter( + '%s/%s does not find but it is required in whole config' % ( + config_path, key + ) + ) + for key in intersect_keys: + _validate_self( + '%s/%s' % (config_path, key), + key, config[key], specified[key], whole_check, + **kwargs + ) + for key in not_found_keys: + if not generals: + raise exception.InvalidParameter( + 'key %s missing in metadata %s' % ( + key, config_path + ) + ) + for general_key, general_value in generals.items(): + _validate_self( + '%s/%s' % (config_path, key), + key, config[key], general_value, whole_check, + **kwargs + ) + + +def _autofill_self_config( + config_path, config_key, config, + metadata, + **kwargs +): + """Autofill config by metadata self section.""" + if '_self' not in metadata: + if isinstance(config, dict): + _autofill_config( + config_path, config, metadata, **kwargs + ) + return config + logging.debug( + 'autofill %s by metadata %s', config_path, metadata['_self'] + ) + autofill_callback = metadata['_self'].get( + 'autofill_callback', None + ) + autofill_callback_params = metadata['_self'].get( + 'autofill_callback_params', {} + ) + callback_params = dict(kwargs) + if autofill_callback_params: + callback_params.update(autofill_callback_params) + default_value = metadata['_self'].get( + 'default_value', None + ) + if default_value is not None: + callback_params['default_value'] = default_value + options = metadata['_self'].get( + 'options', None + ) + if options is not None: + callback_params['options'] = options + if autofill_callback: + config = autofill_callback( + config_key, config, **callback_params + ) + if config is None: + new_config = {} + else: + new_config = config + if isinstance(new_config, dict): + _autofill_config( + config_path, new_config, metadata, **kwargs + ) + if new_config: + config = new_config + return config + + +def _autofill_config( + config_path, config, metadata, **kwargs +): + """autofill config by metadata.""" + generals = {} + specified = {} + for key, value in metadata.items(): + if key.startswith('$'): + generals[key] = value + elif key.startswith('_'): + pass + else: + specified[key] = value + config_keys = set(config.keys()) + specified_keys = set(specified.keys()) + intersect_keys = config_keys & specified_keys + not_found_keys = config_keys - specified_keys + redundant_keys = specified_keys - config_keys + for key in redundant_keys: + self_config = _autofill_self_config( + '%s/%s' % (config_path, key), + key, None, specified[key], **kwargs + ) + if self_config is not None: + config[key] = self_config + for key in intersect_keys: + config[key] = _autofill_self_config( + '%s/%s' % (config_path, key), + key, config[key], specified[key], + **kwargs + ) + for key in not_found_keys: + for general_key, general_value in generals.items(): + config[key] = _autofill_self_config( + '%s/%s' % (config_path, key), + key, config[key], general_value, + **kwargs + ) + return config + + +def autofill_os_config( + config, os_id, **kwargs +): + load_metadatas() + if os_id not in OS_METADATA_MAPPING: + raise exception.InvalidParameter( + 'os %s is not found in os metadata mapping' % os_id + ) + + return _autofill_config( + '', config, OS_METADATA_MAPPING[os_id], **kwargs + ) + + +def autofill_package_config( + config, adapter_id, **kwargs +): + load_metadatas() + if adapter_id not in PACKAGE_METADATA_MAPPING: + raise exception.InvalidParameter( + 'adapter %s is not found in package metadata mapping' % adapter_id + ) + + return _autofill_config( + '', config, PACKAGE_METADATA_MAPPING[adapter_id], **kwargs + ) + + +def autofill_flavor_config( + config, flavor_id, **kwargs +): + load_metadatas() + if not flavor_id: + logging.info('There is no flavor, skipping...') + elif flavor_id not in FLAVOR_METADATA_MAPPING: + raise exception.InvalidParameter( + 'flavor %s is not found in flavor metadata mapping' % flavor_id + ) + else: + return _autofill_config( + '', config, FLAVOR_METADATA_MAPPING[flavor_id], **kwargs + ) |