diff options
Diffstat (limited to 'compass-tasks/db/api/metadata.py')
| -rw-r--r-- | compass-tasks/db/api/metadata.py | 517 |
1 files changed, 0 insertions, 517 deletions
diff --git a/compass-tasks/db/api/metadata.py b/compass-tasks/db/api/metadata.py deleted file mode 100644 index 16310c8..0000000 --- a/compass-tasks/db/api/metadata.py +++ /dev/null @@ -1,517 +0,0 @@ -# Copyright 2014 Huawei Technologies Co. Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Metadata related database operations.""" -import copy -import logging -import string - -from compass.db.api import adapter as adapter_api -from compass.db.api import database -from compass.db.api import utils -from compass.db import callback as metadata_callback -from compass.db import exception -from compass.db import models -from compass.db import validator as metadata_validator - - -from compass.utils import setting_wrapper as setting -from compass.utils import util - - -OS_FIELDS = None -PACKAGE_FIELDS = None -FLAVOR_FIELDS = None -OSES_METADATA = None -PACKAGES_METADATA = None -FLAVORS_METADATA = None -OSES_METADATA_UI_CONVERTERS = None -FLAVORS_METADATA_UI_CONVERTERS = None - - -def _get_field_from_configuration(configs): - """Get fields from configurations.""" - fields = {} - for config in configs: - if not isinstance(config, dict): - raise exception.InvalidParameter( - 'config %s is not dict' % config - ) - field_name = config['NAME'] - fields[field_name] = { - 'name': field_name, - 'id': field_name, - 'field_type': config.get('FIELD_TYPE', basestring), - 'display_type': config.get('DISPLAY_TYPE', 'text'), - 'validator': config.get('VALIDATOR', None), - 'js_validator': config.get('JS_VALIDATOR', None), - 'description': config.get('DESCRIPTION', field_name) - } - return fields - - -def _get_os_fields_from_configuration(): - """Get os fields from os field config dir.""" - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.OS_FIELD_DIR, - env_locals=env_locals - ) - return _get_field_from_configuration( - configs - ) - - -def _get_package_fields_from_configuration(): - """Get package fields from package field config dir.""" - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.PACKAGE_FIELD_DIR, - env_locals=env_locals - ) - return _get_field_from_configuration( - configs - ) - - -def _get_flavor_fields_from_configuration(): - """Get flavor fields from flavor field config dir.""" - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.FLAVOR_FIELD_DIR, - env_locals=env_locals - ) - return _get_field_from_configuration( - configs - ) - - -def _get_metadata_from_configuration( - path, name, config, - fields, **kwargs -): - """Recursively get metadata from configuration. - - Args: - path: used to indicate the path to the root element. - mainly for trouble shooting. - name: the key of the metadata section. - config: the value of the metadata section. - fields: all fields defined in os fields or package fields dir. - """ - if not isinstance(config, dict): - raise exception.InvalidParameter( - '%s config %s is not dict' % (path, config) - ) - metadata_self = config.get('_self', {}) - if 'field' in metadata_self: - field_name = metadata_self['field'] - field = fields[field_name] - else: - field = {} - # mapping to may contain $ like $partition. Here we replace the - # $partition to the key of the correspendent config. The backend then - # can use this kind of feature to support multi partitions when we - # only declare the partition metadata in one place. - mapping_to_template = metadata_self.get('mapping_to', None) - if mapping_to_template: - mapping_to = string.Template( - mapping_to_template - ).safe_substitute( - **kwargs - ) - else: - mapping_to = None - self_metadata = { - 'name': name, - 'display_name': metadata_self.get('display_name', name), - 'field_type': field.get('field_type', dict), - 'display_type': field.get('display_type', None), - 'description': metadata_self.get( - 'description', field.get('description', None) - ), - 'is_required': metadata_self.get('is_required', False), - 'required_in_whole_config': metadata_self.get( - 'required_in_whole_config', False), - 'mapping_to': mapping_to, - 'validator': metadata_self.get( - 'validator', field.get('validator', None) - ), - 'js_validator': metadata_self.get( - 'js_validator', field.get('js_validator', None) - ), - 'default_value': metadata_self.get('default_value', None), - 'default_callback': metadata_self.get('default_callback', None), - 'default_callback_params': metadata_self.get( - 'default_callback_params', {}), - 'options': metadata_self.get('options', None), - 'options_callback': metadata_self.get('options_callback', None), - 'options_callback_params': metadata_self.get( - 'options_callback_params', {}), - 'autofill_callback': metadata_self.get( - 'autofill_callback', None), - 'autofill_callback_params': metadata_self.get( - 'autofill_callback_params', {}), - 'required_in_options': metadata_self.get( - 'required_in_options', False) - } - self_metadata.update(kwargs) - metadata = {'_self': self_metadata} - # Key extension used to do two things: - # one is to return the extended metadata that $<something> - # will be replace to possible extensions. - # The other is to record the $<something> to extended value - # and used in future mapping_to subsititution. - # TODO(grace): select proper name instead of key_extensions if - # you think it is better. - # Suppose key_extension is {'$partition': ['/var', '/']} for $partition - # the metadata for $partition will be mapped to { - # '/var': ..., '/': ...} and kwargs={'partition': '/var'} and - # kwargs={'partition': '/'} will be parsed to recursive metadata parsing - # for sub metadata under '/var' and '/'. Then in the metadata parsing - # for the sub metadata, this kwargs will be used to substitute mapping_to. - key_extensions = metadata_self.get('key_extensions', {}) - general_keys = [] - for key, value in config.items(): - if key.startswith('_'): - continue - if key in key_extensions: - if not key.startswith('$'): - raise exception.InvalidParameter( - '%s subkey %s should start with $' % ( - path, key - ) - ) - extended_keys = key_extensions[key] - for extended_key in extended_keys: - if extended_key.startswith('$'): - raise exception.InvalidParameter( - '%s extended key %s should not start with $' % ( - path, extended_key - ) - ) - sub_kwargs = dict(kwargs) - sub_kwargs[key[1:]] = extended_key - metadata[extended_key] = _get_metadata_from_configuration( - '%s/%s' % (path, extended_key), extended_key, value, - fields, **sub_kwargs - ) - else: - if key.startswith('$'): - general_keys.append(key) - metadata[key] = _get_metadata_from_configuration( - '%s/%s' % (path, key), key, value, - fields, **kwargs - ) - if len(general_keys) > 1: - raise exception.InvalidParameter( - 'foud multi general keys in %s: %s' % ( - path, general_keys - ) - ) - return metadata - - -def _get_oses_metadata_from_configuration(): - """Get os metadata from os metadata config dir.""" - oses_metadata = {} - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.OS_METADATA_DIR, - env_locals=env_locals - ) - for config in configs: - os_name = config['OS'] - os_metadata = oses_metadata.setdefault(os_name, {}) - for key, value in config['METADATA'].items(): - os_metadata[key] = _get_metadata_from_configuration( - key, key, value, OS_FIELDS - ) - - oses = adapter_api.OSES - parents = {} - for os_name, os in oses.items(): - parent = os.get('parent', None) - parents[os_name] = parent - for os_name, os in oses.items(): - oses_metadata[os_name] = util.recursive_merge_dict( - os_name, oses_metadata, parents - ) - return oses_metadata - - -def _get_packages_metadata_from_configuration(): - """Get package metadata from package metadata config dir.""" - packages_metadata = {} - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.PACKAGE_METADATA_DIR, - env_locals=env_locals - ) - for config in configs: - adapter_name = config['ADAPTER'] - package_metadata = packages_metadata.setdefault(adapter_name, {}) - for key, value in config['METADATA'].items(): - package_metadata[key] = _get_metadata_from_configuration( - key, key, value, PACKAGE_FIELDS - ) - adapters = adapter_api.ADAPTERS - parents = {} - for adapter_name, adapter in adapters.items(): - parent = adapter.get('parent', None) - parents[adapter_name] = parent - for adapter_name, adapter in adapters.items(): - packages_metadata[adapter_name] = util.recursive_merge_dict( - adapter_name, packages_metadata, parents - ) - return packages_metadata - - -def _get_flavors_metadata_from_configuration(): - """Get flavor metadata from flavor metadata config dir.""" - flavors_metadata = {} - env_locals = {} - env_locals.update(metadata_validator.VALIDATOR_LOCALS) - env_locals.update(metadata_callback.CALLBACK_LOCALS) - configs = util.load_configs( - setting.FLAVOR_METADATA_DIR, - env_locals=env_locals - ) - for config in configs: - adapter_name = config['ADAPTER'] - flavor_name = config['FLAVOR'] - flavor_metadata = flavors_metadata.setdefault( - adapter_name, {} - ).setdefault(flavor_name, {}) - for key, value in config['METADATA'].items(): - flavor_metadata[key] = _get_metadata_from_configuration( - key, key, value, FLAVOR_FIELDS - ) - - packages_metadata = PACKAGES_METADATA - adapters_flavors = adapter_api.ADAPTERS_FLAVORS - for adapter_name, adapter_flavors in adapters_flavors.items(): - package_metadata = packages_metadata.get(adapter_name, {}) - for flavor_name, flavor in adapter_flavors.items(): - flavor_metadata = flavors_metadata.setdefault( - adapter_name, {} - ).setdefault(flavor_name, {}) - util.merge_dict(flavor_metadata, package_metadata, override=False) - return flavors_metadata - - -def _filter_metadata(metadata, **kwargs): - if not isinstance(metadata, dict): - return metadata - filtered_metadata = {} - for key, value in metadata.items(): - if key == '_self': - default_value = value.get('default_value', None) - if default_value is None: - default_callback_params = value.get( - 'default_callback_params', {} - ) - callback_params = dict(kwargs) - if default_callback_params: - callback_params.update(default_callback_params) - default_callback = value.get('default_callback', None) - if default_callback: - default_value = default_callback(key, **callback_params) - options = value.get('options', None) - if options is None: - options_callback_params = value.get( - 'options_callback_params', {} - ) - callback_params = dict(kwargs) - if options_callback_params: - callback_params.update(options_callback_params) - - options_callback = value.get('options_callback', None) - if options_callback: - options = options_callback(key, **callback_params) - filtered_metadata[key] = value - if default_value is not None: - filtered_metadata[key]['default_value'] = default_value - if options is not None: - filtered_metadata[key]['options'] = options - else: - filtered_metadata[key] = _filter_metadata(value, **kwargs) - return filtered_metadata - - -def _load_metadata(force_reload=False): - """Load metadata information into memory. - - If force_reload, the metadata information will be reloaded - even if the metadata is already loaded. - """ - adapter_api.load_adapters_internal(force_reload=force_reload) - global OS_FIELDS - if force_reload or OS_FIELDS is None: - OS_FIELDS = _get_os_fields_from_configuration() - global PACKAGE_FIELDS - if force_reload or PACKAGE_FIELDS is None: - PACKAGE_FIELDS = _get_package_fields_from_configuration() - global FLAVOR_FIELDS - if force_reload or FLAVOR_FIELDS is None: - FLAVOR_FIELDS = _get_flavor_fields_from_configuration() - global OSES_METADATA - if force_reload or OSES_METADATA is None: - OSES_METADATA = _get_oses_metadata_from_configuration() - global PACKAGES_METADATA - if force_reload or PACKAGES_METADATA is None: - PACKAGES_METADATA = _get_packages_metadata_from_configuration() - global FLAVORS_METADATA - if force_reload or FLAVORS_METADATA is None: - FLAVORS_METADATA = _get_flavors_metadata_from_configuration() - global OSES_METADATA_UI_CONVERTERS - if force_reload or OSES_METADATA_UI_CONVERTERS is None: - OSES_METADATA_UI_CONVERTERS = ( - _get_oses_metadata_ui_converters_from_configuration() - ) - global FLAVORS_METADATA_UI_CONVERTERS - if force_reload or FLAVORS_METADATA_UI_CONVERTERS is None: - FLAVORS_METADATA_UI_CONVERTERS = ( - _get_flavors_metadata_ui_converters_from_configuration() - ) - - -def _get_oses_metadata_ui_converters_from_configuration(): - """Get os metadata ui converters from os metadata mapping config dir. - - os metadata ui converter is used to convert os metadata to - the format UI can understand and show. - """ - oses_metadata_ui_converters = {} - configs = util.load_configs(setting.OS_MAPPING_DIR) - for config in configs: - os_name = config['OS'] - oses_metadata_ui_converters[os_name] = config.get('CONFIG_MAPPING', {}) - - oses = adapter_api.OSES - parents = {} - for os_name, os in oses.items(): - parent = os.get('parent', None) - parents[os_name] = parent - for os_name, os in oses.items(): - oses_metadata_ui_converters[os_name] = util.recursive_merge_dict( - os_name, oses_metadata_ui_converters, parents - ) - return oses_metadata_ui_converters - - -def _get_flavors_metadata_ui_converters_from_configuration(): - """Get flavor metadata ui converters from flavor mapping config dir.""" - flavors_metadata_ui_converters = {} - configs = util.load_configs(setting.FLAVOR_MAPPING_DIR) - for config in configs: - adapter_name = config['ADAPTER'] - flavor_name = config['FLAVOR'] - flavors_metadata_ui_converters.setdefault( - adapter_name, {} - )[flavor_name] = config.get('CONFIG_MAPPING', {}) - adapters = adapter_api.ADAPTERS - parents = {} - for adapter_name, adapter in adapters.items(): - parent = adapter.get('parent', None) - parents[adapter_name] = parent - for adapter_name, adapter in adapters.items(): - flavors_metadata_ui_converters[adapter_name] = ( - util.recursive_merge_dict( - adapter_name, flavors_metadata_ui_converters, parents - ) - ) - return flavors_metadata_ui_converters - - -def get_packages_metadata_internal(force_reload=False): - """Get deployable package metadata.""" - _load_metadata(force_reload=force_reload) - metadata_mapping = {} - adapters = adapter_api.ADAPTERS - for adapter_name, adapter in adapters.items(): - if adapter.get('deployable'): - metadata_mapping[adapter_name] = _filter_metadata( - PACKAGES_METADATA.get(adapter_name, {}) - ) - else: - logging.info( - 'ignore metadata since its adapter %s is not deployable', - adapter_name - ) - return metadata_mapping - - -def get_flavors_metadata_internal(force_reload=False): - """Get deployable flavor metadata.""" - _load_metadata(force_reload=force_reload) - metadata_mapping = {} - adapters_flavors = adapter_api.ADAPTERS_FLAVORS - for adapter_name, adapter_flavors in adapters_flavors.items(): - adapter = adapter_api.ADAPTERS[adapter_name] - if not adapter.get('deployable'): - logging.info( - 'ignore metadata since its adapter %s is not deployable', - adapter_name - ) - continue - for flavor_name, flavor in adapter_flavors.items(): - flavor_metadata = FLAVORS_METADATA.get( - adapter_name, {} - ).get(flavor_name, {}) - metadata = _filter_metadata(flavor_metadata) - metadata_mapping.setdefault( - adapter_name, {} - )[flavor_name] = metadata - return metadata_mapping - - -def get_flavors_metadata_ui_converters_internal(force_reload=False): - """Get usable flavor metadata ui converters.""" - _load_metadata(force_reload=force_reload) - return FLAVORS_METADATA_UI_CONVERTERS - - -def get_oses_metadata_internal(force_reload=False): - """Get deployable os metadata.""" - _load_metadata(force_reload=force_reload) - metadata_mapping = {} - oses = adapter_api.OSES - for os_name, os in oses.items(): - if os.get('deployable'): - metadata_mapping[os_name] = _filter_metadata( - OSES_METADATA.get(os_name, {}) - ) - else: - logging.info( - 'ignore metadata since its os %s is not deployable', - os_name - ) - return metadata_mapping - - -def get_oses_metadata_ui_converters_internal(force_reload=False): - """Get usable os metadata ui converters.""" - _load_metadata(force_reload=force_reload) - return OSES_METADATA_UI_CONVERTERS |