diff options
Diffstat (limited to 'compass-tasks/db/api/metadata_holder.py')
| -rw-r--r-- | compass-tasks/db/api/metadata_holder.py | 731 |
1 files changed, 0 insertions, 731 deletions
diff --git a/compass-tasks/db/api/metadata_holder.py b/compass-tasks/db/api/metadata_holder.py deleted file mode 100644 index 24afc67..0000000 --- a/compass-tasks/db/api/metadata_holder.py +++ /dev/null @@ -1,731 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Metadata related object holder.""" -import logging - -from compass.db.api import adapter as adapter_api -from compass.db.api import adapter_holder as adapter_holder_api -from compass.db.api import database -from compass.db.api import metadata as metadata_api -from compass.db.api import permission -from compass.db.api import user as user_api -from compass.db.api import utils -from compass.db import exception -from compass.db import models -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -RESP_METADATA_FIELDS = [ - 'os_config', 'package_config' -] -RESP_UI_METADATA_FIELDS = [ - 'os_global_config', 'flavor_config' -] - - -def load_metadatas(force_reload=False): - """Load metadatas.""" - # TODO(xicheng): today we load metadata in memory as it original - # format in files in metadata.py. We get these inmemory metadata - # and do some translation, store the translated metadata into memory - # too in metadata_holder.py. api can only access the global inmemory - # data in metadata_holder.py. - _load_os_metadatas(force_reload=force_reload) - _load_package_metadatas(force_reload=force_reload) - _load_flavor_metadatas(force_reload=force_reload) - _load_os_metadata_ui_converters(force_reload=force_reload) - _load_flavor_metadata_ui_converters(force_reload=force_reload) - - -def _load_os_metadata_ui_converters(force_reload=False): - global OS_METADATA_UI_CONVERTERS - if force_reload or OS_METADATA_UI_CONVERTERS is None: - logging.info('load os metadatas ui converters into memory') - OS_METADATA_UI_CONVERTERS = ( - metadata_api.get_oses_metadata_ui_converters_internal( - force_reload=force_reload - ) - ) - - -def _load_os_metadatas(force_reload=False): - """Load os metadata from inmemory db and map it by os_id.""" - global OS_METADATA_MAPPING - if force_reload or OS_METADATA_MAPPING is None: - logging.info('load os metadatas into memory') - OS_METADATA_MAPPING = metadata_api.get_oses_metadata_internal( - force_reload=force_reload - ) - - -def _load_flavor_metadata_ui_converters(force_reload=False): - """Load flavor metadata ui converters from inmemory db. - - The loaded metadata is mapped by flavor id. - """ - global FLAVOR_METADATA_UI_CONVERTERS - if force_reload or FLAVOR_METADATA_UI_CONVERTERS is None: - logging.info('load flavor metadata ui converters into memory') - FLAVOR_METADATA_UI_CONVERTERS = {} - adapters_flavors_metadata_ui_converters = ( - metadata_api.get_flavors_metadata_ui_converters_internal( - force_reload=force_reload - ) - ) - for adapter_name, adapter_flavors_metadata_ui_converters in ( - adapters_flavors_metadata_ui_converters.items() - ): - for flavor_name, flavor_metadata_ui_converter in ( - adapter_flavors_metadata_ui_converters.items() - ): - FLAVOR_METADATA_UI_CONVERTERS[ - '%s:%s' % (adapter_name, flavor_name) - ] = flavor_metadata_ui_converter - - -@util.deprecated -def _load_package_metadatas(force_reload=False): - """Load deployable package metadata from inmemory db.""" - global PACKAGE_METADATA_MAPPING - if force_reload or PACKAGE_METADATA_MAPPING is None: - logging.info('load package metadatas into memory') - PACKAGE_METADATA_MAPPING = ( - metadata_api.get_packages_metadata_internal( - force_reload=force_reload - ) - ) - - -def _load_flavor_metadatas(force_reload=False): - """Load flavor metadata from inmemory db. - - The loaded metadata are mapped by flavor id. - """ - global FLAVOR_METADATA_MAPPING - if force_reload or FLAVOR_METADATA_MAPPING is None: - logging.info('load flavor metadatas into memory') - FLAVOR_METADATA_MAPPING = {} - adapters_flavors_metadata = ( - metadata_api.get_flavors_metadata_internal( - force_reload=force_reload - ) - ) - for adapter_name, adapter_flavors_metadata in ( - adapters_flavors_metadata.items() - ): - for flavor_name, flavor_metadata in ( - adapter_flavors_metadata.items() - ): - FLAVOR_METADATA_MAPPING[ - '%s:%s' % (adapter_name, flavor_name) - ] = flavor_metadata - - -OS_METADATA_MAPPING = None -PACKAGE_METADATA_MAPPING = None -FLAVOR_METADATA_MAPPING = None -OS_METADATA_UI_CONVERTERS = None -FLAVOR_METADATA_UI_CONVERTERS = None - - -def validate_os_config( - config, os_id, whole_check=False, **kwargs -): - """Validate os config.""" - load_metadatas() - if os_id not in OS_METADATA_MAPPING: - raise exception.InvalidParameter( - 'os %s is not found in os metadata mapping' % os_id - ) - _validate_config( - '', config, OS_METADATA_MAPPING[os_id], - whole_check, **kwargs - ) - - -@util.deprecated -def validate_package_config( - config, adapter_id, whole_check=False, **kwargs -): - """Validate package config.""" - load_metadatas() - if adapter_id not in PACKAGE_METADATA_MAPPING: - raise exception.InvalidParameter( - 'adapter %s is not found in package metedata mapping' % adapter_id - ) - _validate_config( - '', config, PACKAGE_METADATA_MAPPING[adapter_id], - whole_check, **kwargs - ) - - -def validate_flavor_config( - config, flavor_id, whole_check=False, **kwargs -): - """Validate flavor config.""" - load_metadatas() - if not flavor_id: - logging.info('There is no flavor, skipping flavor validation...') - elif flavor_id not in FLAVOR_METADATA_MAPPING: - raise exception.InvalidParameter( - 'flavor %s is not found in flavor metedata mapping' % flavor_id - ) - else: - _validate_config( - '', config, FLAVOR_METADATA_MAPPING[flavor_id], - whole_check, **kwargs - ) - - -def _filter_metadata(metadata, **kwargs): - """Filter metadata before return it to api. - - Some metadata fields are not json compatible or - only used in db/api internally. - We should strip these fields out before return to api. - """ - if not isinstance(metadata, dict): - return metadata - filtered_metadata = {} - for key, value in metadata.items(): - if key == '_self': - filtered_metadata[key] = { - 'name': value['name'], - 'description': value.get('description', None), - 'default_value': value.get('default_value', None), - 'is_required': value.get('is_required', False), - 'required_in_whole_config': value.get( - 'required_in_whole_config', False), - 'js_validator': value.get('js_validator', None), - 'options': value.get('options', None), - 'required_in_options': value.get( - 'required_in_options', False), - 'field_type': value.get( - 'field_type_data', 'str'), - 'display_type': value.get('display_type', None), - 'mapping_to': value.get('mapping_to', None) - } - else: - filtered_metadata[key] = _filter_metadata(value, **kwargs) - return filtered_metadata - - -@util.deprecated -def _get_package_metadata(adapter_id): - """get package metadata.""" - load_metadatas() - if adapter_id not in PACKAGE_METADATA_MAPPING: - raise exception.RecordNotExists( - 'adpater %s does not exist' % adapter_id - ) - return _filter_metadata( - PACKAGE_METADATA_MAPPING[adapter_id] - ) - - -@util.deprecated -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_METADATA_FIELDS) -def get_package_metadata(adapter_id, user=None, session=None, **kwargs): - """Get package metadata from adapter.""" - return { - 'package_config': _get_package_metadata(adapter_id) - } - - -def _get_flavor_metadata(flavor_id): - """get flavor metadata.""" - load_metadatas() - if not flavor_id: - logging.info('There is no flavor id, skipping...') - elif flavor_id not in FLAVOR_METADATA_MAPPING: - raise exception.RecordNotExists( - 'flavor %s does not exist' % flavor_id - ) - else: - return _filter_metadata(FLAVOR_METADATA_MAPPING[flavor_id]) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_METADATA_FIELDS) -def get_flavor_metadata(flavor_id, user=None, session=None, **kwargs): - """Get flavor metadata by flavor.""" - return { - 'package_config': _get_flavor_metadata(flavor_id) - } - - -def _get_os_metadata(os_id): - """get os metadata.""" - load_metadatas() - if os_id not in OS_METADATA_MAPPING: - raise exception.RecordNotExists( - 'os %s does not exist' % os_id - ) - return _filter_metadata(OS_METADATA_MAPPING[os_id]) - - -def _get_os_metadata_ui_converter(os_id): - """get os metadata ui converter.""" - load_metadatas() - if os_id not in OS_METADATA_UI_CONVERTERS: - raise exception.RecordNotExists( - 'os %s does not exist' % os_id - ) - return OS_METADATA_UI_CONVERTERS[os_id] - - -def _get_flavor_metadata_ui_converter(flavor_id): - """get flavor metadata ui converter.""" - load_metadatas() - if flavor_id not in FLAVOR_METADATA_UI_CONVERTERS: - raise exception.RecordNotExists( - 'flavor %s does not exist' % flavor_id - ) - return FLAVOR_METADATA_UI_CONVERTERS[flavor_id] - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_METADATA_FIELDS) -def get_os_metadata(os_id, user=None, session=None, **kwargs): - """get os metadatas.""" - return {'os_config': _get_os_metadata(os_id)} - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS) -def get_os_ui_metadata(os_id, user=None, session=None, **kwargs): - """Get os metadata ui converter by os.""" - metadata = _get_os_metadata(os_id) - metadata_ui_converter = _get_os_metadata_ui_converter(os_id) - return _get_ui_metadata(metadata, metadata_ui_converter) - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS) -def get_flavor_ui_metadata(flavor_id, user=None, session=None, **kwargs): - """Get flavor ui metadata by flavor.""" - metadata = _get_flavor_metadata(flavor_id) - metadata_ui_converter = _get_flavor_metadata_ui_converter(flavor_id) - return _get_ui_metadata(metadata, metadata_ui_converter) - - -def _get_ui_metadata(metadata, metadata_ui_converter): - """convert metadata to ui metadata. - - Args: - metadata: metadata we defined in metadata files. - metadata_ui_converter: metadata ui converter defined in metadata - mapping files. Used to convert orignal - metadata to ui understandable metadata. - - Returns: - ui understandable metadata. - """ - ui_metadata = {} - ui_metadata[metadata_ui_converter['mapped_name']] = [] - for mapped_child in metadata_ui_converter['mapped_children']: - data_dict = {} - for ui_key, ui_value in mapped_child.items(): - for key, value in ui_value.items(): - if 'data' == key: - result_data = [] - _get_ui_metadata_data( - metadata[ui_key], value, result_data - ) - data_dict['data'] = result_data - else: - data_dict[key] = value - ui_metadata[metadata_ui_converter['mapped_name']].append(data_dict) - return ui_metadata - - -def _get_ui_metadata_data(metadata, config, result_data): - """Get ui metadata data and fill to result.""" - data_dict = {} - for key, config_value in config.items(): - if isinstance(config_value, dict) and key != 'content_data': - if key in metadata.keys(): - _get_ui_metadata_data(metadata[key], config_value, result_data) - else: - _get_ui_metadata_data(metadata, config_value, result_data) - elif isinstance(config_value, list): - option_list = [] - for item in config_value: - if isinstance(item, dict): - option_list.append(item) - data_dict[key] = option_list - else: - if isinstance(metadata['_self'][item], bool): - data_dict[item] = str(metadata['_self'][item]).lower() - else: - data_dict[item] = metadata['_self'][item] - else: - data_dict[key] = config_value - if data_dict: - result_data.append(data_dict) - return result_data - - -@util.deprecated -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_METADATA_FIELDS) -def get_package_os_metadata( - adapter_id, os_id, - user=None, session=None, **kwargs -): - """Get metadata by adapter and os.""" - adapter = adapter_holder_api.get_adapter( - adapter_id, user=user, session=session - ) - os_ids = [os['id'] for os in adapter['supported_oses']] - if os_id not in os_ids: - raise exception.InvalidParameter( - 'os %s is not in the supported os list of adapter %s' % ( - os_id, adapter_id - ) - ) - metadatas = {} - metadatas['os_config'] = _get_os_metadata( - os_id - ) - metadatas['package_config'] = _get_package_metadata( - adapter_id - ) - return metadatas - - -@utils.supported_filters([]) -@database.run_in_session() -@user_api.check_user_permission( - permission.PERMISSION_LIST_METADATAS -) -@utils.wrap_to_dict(RESP_METADATA_FIELDS) -def get_flavor_os_metadata( - flavor_id, os_id, - user=None, session=None, **kwargs -): - """Get metadata by flavor and os.""" - flavor = adapter_holder_api.get_flavor( - flavor_id, user=user, session=session - ) - adapter_id = flavor['adapter_id'] - adapter = adapter_holder_api.get_adapter( - adapter_id, user=user, session=session - ) - os_ids = [os['id'] for os in adapter['supported_oses']] - if os_id not in os_ids: - raise exception.InvalidParameter( - 'os %s is not in the supported os list of adapter %s' % ( - os_id, adapter_id - ) - ) - metadatas = {} - metadatas['os_config'] = _get_os_metadata( - session, os_id - ) - metadatas['package_config'] = _get_flavor_metadata( - session, flavor_id - ) - return metadatas - - -def _validate_self( - config_path, config_key, config, - metadata, whole_check, - **kwargs -): - """validate config by metadata self section.""" - logging.debug('validate config self %s', config_path) - if '_self' not in metadata: - if isinstance(config, dict): - _validate_config( - config_path, config, metadata, whole_check, **kwargs - ) - return - field_type = metadata['_self'].get('field_type', basestring) - if not isinstance(config, field_type): - raise exception.InvalidParameter( - '%s config type is not %s: %s' % (config_path, field_type, config) - ) - is_required = metadata['_self'].get( - 'is_required', False - ) - required_in_whole_config = metadata['_self'].get( - 'required_in_whole_config', False - ) - if isinstance(config, basestring): - if config == '' and not is_required and not required_in_whole_config: - # ignore empty config when it is optional - return - required_in_options = metadata['_self'].get( - 'required_in_options', False - ) - options = metadata['_self'].get('options', None) - if required_in_options: - if field_type in [int, basestring, float, bool]: - if options and config not in options: - raise exception.InvalidParameter( - '%s config is not in %s: %s' % ( - config_path, options, config - ) - ) - elif field_type in [list, tuple]: - if options and not set(config).issubset(set(options)): - raise exception.InvalidParameter( - '%s config is not in %s: %s' % ( - config_path, options, config - ) - ) - elif field_type == dict: - if options and not set(config.keys()).issubset(set(options)): - raise exception.InvalidParameter( - '%s config is not in %s: %s' % ( - config_path, options, config - ) - ) - validator = metadata['_self'].get('validator', None) - logging.debug('validate by validator %s', validator) - if validator: - if not validator(config_key, config, **kwargs): - raise exception.InvalidParameter( - '%s config is invalid' % config_path - ) - if isinstance(config, dict): - _validate_config( - config_path, config, metadata, whole_check, **kwargs - ) - - -def _validate_config( - config_path, config, metadata, whole_check, - **kwargs -): - """validate config by metadata.""" - logging.debug('validate config %s', config_path) - generals = {} - specified = {} - for key, value in metadata.items(): - if key.startswith('$'): - generals[key] = value - elif key.startswith('_'): - pass - else: - specified[key] = value - config_keys = set(config.keys()) - specified_keys = set(specified.keys()) - intersect_keys = config_keys & specified_keys - not_found_keys = config_keys - specified_keys - redundant_keys = specified_keys - config_keys - for key in redundant_keys: - if '_self' not in specified[key]: - continue - if specified[key]['_self'].get('is_required', False): - raise exception.InvalidParameter( - '%s/%s does not find but it is required' % ( - config_path, key - ) - ) - if ( - whole_check and - specified[key]['_self'].get( - 'required_in_whole_config', False - ) - ): - raise exception.InvalidParameter( - '%s/%s does not find but it is required in whole config' % ( - config_path, key - ) - ) - for key in intersect_keys: - _validate_self( - '%s/%s' % (config_path, key), - key, config[key], specified[key], whole_check, - **kwargs - ) - for key in not_found_keys: - if not generals: - raise exception.InvalidParameter( - 'key %s missing in metadata %s' % ( - key, config_path - ) - ) - for general_key, general_value in generals.items(): - _validate_self( - '%s/%s' % (config_path, key), - key, config[key], general_value, whole_check, - **kwargs - ) - - -def _autofill_self_config( - config_path, config_key, config, - metadata, - **kwargs -): - """Autofill config by metadata self section.""" - if '_self' not in metadata: - if isinstance(config, dict): - _autofill_config( - config_path, config, metadata, **kwargs - ) - return config - logging.debug( - 'autofill %s by metadata %s', config_path, metadata['_self'] - ) - autofill_callback = metadata['_self'].get( - 'autofill_callback', None - ) - autofill_callback_params = metadata['_self'].get( - 'autofill_callback_params', {} - ) - callback_params = dict(kwargs) - if autofill_callback_params: - callback_params.update(autofill_callback_params) - default_value = metadata['_self'].get( - 'default_value', None - ) - if default_value is not None: - callback_params['default_value'] = default_value - options = metadata['_self'].get( - 'options', None - ) - if options is not None: - callback_params['options'] = options - if autofill_callback: - config = autofill_callback( - config_key, config, **callback_params - ) - if config is None: - new_config = {} - else: - new_config = config - if isinstance(new_config, dict): - _autofill_config( - config_path, new_config, metadata, **kwargs - ) - if new_config: - config = new_config - return config - - -def _autofill_config( - config_path, config, metadata, **kwargs -): - """autofill config by metadata.""" - generals = {} - specified = {} - for key, value in metadata.items(): - if key.startswith('$'): - generals[key] = value - elif key.startswith('_'): - pass - else: - specified[key] = value - config_keys = set(config.keys()) - specified_keys = set(specified.keys()) - intersect_keys = config_keys & specified_keys - not_found_keys = config_keys - specified_keys - redundant_keys = specified_keys - config_keys - for key in redundant_keys: - self_config = _autofill_self_config( - '%s/%s' % (config_path, key), - key, None, specified[key], **kwargs - ) - if self_config is not None: - config[key] = self_config - for key in intersect_keys: - config[key] = _autofill_self_config( - '%s/%s' % (config_path, key), - key, config[key], specified[key], - **kwargs - ) - for key in not_found_keys: - for general_key, general_value in generals.items(): - config[key] = _autofill_self_config( - '%s/%s' % (config_path, key), - key, config[key], general_value, - **kwargs - ) - return config - - -def autofill_os_config( - config, os_id, **kwargs -): - load_metadatas() - if os_id not in OS_METADATA_MAPPING: - raise exception.InvalidParameter( - 'os %s is not found in os metadata mapping' % os_id - ) - - return _autofill_config( - '', config, OS_METADATA_MAPPING[os_id], **kwargs - ) - - -def autofill_package_config( - config, adapter_id, **kwargs -): - load_metadatas() - if adapter_id not in PACKAGE_METADATA_MAPPING: - raise exception.InvalidParameter( - 'adapter %s is not found in package metadata mapping' % adapter_id - ) - - return _autofill_config( - '', config, PACKAGE_METADATA_MAPPING[adapter_id], **kwargs - ) - - -def autofill_flavor_config( - config, flavor_id, **kwargs -): - load_metadatas() - if not flavor_id: - logging.info('There is no flavor, skipping...') - elif flavor_id not in FLAVOR_METADATA_MAPPING: - raise exception.InvalidParameter( - 'flavor %s is not found in flavor metadata mapping' % flavor_id - ) - else: - return _autofill_config( - '', config, FLAVOR_METADATA_MAPPING[flavor_id], **kwargs - ) |