summaryrefslogtreecommitdiffstats
path: root/compass-tasks/db/api/metadata_holder.py
diff options
context:
space:
mode:
Diffstat (limited to 'compass-tasks/db/api/metadata_holder.py')
-rw-r--r--compass-tasks/db/api/metadata_holder.py731
1 files changed, 731 insertions, 0 deletions
diff --git a/compass-tasks/db/api/metadata_holder.py b/compass-tasks/db/api/metadata_holder.py
new file mode 100644
index 0000000..24afc67
--- /dev/null
+++ b/compass-tasks/db/api/metadata_holder.py
@@ -0,0 +1,731 @@
+# Copyright 2014 Huawei Technologies Co. Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Metadata related object holder."""
+import logging
+
+from compass.db.api import adapter as adapter_api
+from compass.db.api import adapter_holder as adapter_holder_api
+from compass.db.api import database
+from compass.db.api import metadata as metadata_api
+from compass.db.api import permission
+from compass.db.api import user as user_api
+from compass.db.api import utils
+from compass.db import exception
+from compass.db import models
+from compass.utils import setting_wrapper as setting
+from compass.utils import util
+
+
+RESP_METADATA_FIELDS = [
+ 'os_config', 'package_config'
+]
+RESP_UI_METADATA_FIELDS = [
+ 'os_global_config', 'flavor_config'
+]
+
+
+def load_metadatas(force_reload=False):
+ """Load metadatas."""
+ # TODO(xicheng): today we load metadata in memory as it original
+ # format in files in metadata.py. We get these inmemory metadata
+ # and do some translation, store the translated metadata into memory
+ # too in metadata_holder.py. api can only access the global inmemory
+ # data in metadata_holder.py.
+ _load_os_metadatas(force_reload=force_reload)
+ _load_package_metadatas(force_reload=force_reload)
+ _load_flavor_metadatas(force_reload=force_reload)
+ _load_os_metadata_ui_converters(force_reload=force_reload)
+ _load_flavor_metadata_ui_converters(force_reload=force_reload)
+
+
+def _load_os_metadata_ui_converters(force_reload=False):
+ global OS_METADATA_UI_CONVERTERS
+ if force_reload or OS_METADATA_UI_CONVERTERS is None:
+ logging.info('load os metadatas ui converters into memory')
+ OS_METADATA_UI_CONVERTERS = (
+ metadata_api.get_oses_metadata_ui_converters_internal(
+ force_reload=force_reload
+ )
+ )
+
+
+def _load_os_metadatas(force_reload=False):
+ """Load os metadata from inmemory db and map it by os_id."""
+ global OS_METADATA_MAPPING
+ if force_reload or OS_METADATA_MAPPING is None:
+ logging.info('load os metadatas into memory')
+ OS_METADATA_MAPPING = metadata_api.get_oses_metadata_internal(
+ force_reload=force_reload
+ )
+
+
+def _load_flavor_metadata_ui_converters(force_reload=False):
+ """Load flavor metadata ui converters from inmemory db.
+
+ The loaded metadata is mapped by flavor id.
+ """
+ global FLAVOR_METADATA_UI_CONVERTERS
+ if force_reload or FLAVOR_METADATA_UI_CONVERTERS is None:
+ logging.info('load flavor metadata ui converters into memory')
+ FLAVOR_METADATA_UI_CONVERTERS = {}
+ adapters_flavors_metadata_ui_converters = (
+ metadata_api.get_flavors_metadata_ui_converters_internal(
+ force_reload=force_reload
+ )
+ )
+ for adapter_name, adapter_flavors_metadata_ui_converters in (
+ adapters_flavors_metadata_ui_converters.items()
+ ):
+ for flavor_name, flavor_metadata_ui_converter in (
+ adapter_flavors_metadata_ui_converters.items()
+ ):
+ FLAVOR_METADATA_UI_CONVERTERS[
+ '%s:%s' % (adapter_name, flavor_name)
+ ] = flavor_metadata_ui_converter
+
+
+@util.deprecated
+def _load_package_metadatas(force_reload=False):
+ """Load deployable package metadata from inmemory db."""
+ global PACKAGE_METADATA_MAPPING
+ if force_reload or PACKAGE_METADATA_MAPPING is None:
+ logging.info('load package metadatas into memory')
+ PACKAGE_METADATA_MAPPING = (
+ metadata_api.get_packages_metadata_internal(
+ force_reload=force_reload
+ )
+ )
+
+
+def _load_flavor_metadatas(force_reload=False):
+ """Load flavor metadata from inmemory db.
+
+ The loaded metadata are mapped by flavor id.
+ """
+ global FLAVOR_METADATA_MAPPING
+ if force_reload or FLAVOR_METADATA_MAPPING is None:
+ logging.info('load flavor metadatas into memory')
+ FLAVOR_METADATA_MAPPING = {}
+ adapters_flavors_metadata = (
+ metadata_api.get_flavors_metadata_internal(
+ force_reload=force_reload
+ )
+ )
+ for adapter_name, adapter_flavors_metadata in (
+ adapters_flavors_metadata.items()
+ ):
+ for flavor_name, flavor_metadata in (
+ adapter_flavors_metadata.items()
+ ):
+ FLAVOR_METADATA_MAPPING[
+ '%s:%s' % (adapter_name, flavor_name)
+ ] = flavor_metadata
+
+
+OS_METADATA_MAPPING = None
+PACKAGE_METADATA_MAPPING = None
+FLAVOR_METADATA_MAPPING = None
+OS_METADATA_UI_CONVERTERS = None
+FLAVOR_METADATA_UI_CONVERTERS = None
+
+
+def validate_os_config(
+ config, os_id, whole_check=False, **kwargs
+):
+ """Validate os config."""
+ load_metadatas()
+ if os_id not in OS_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'os %s is not found in os metadata mapping' % os_id
+ )
+ _validate_config(
+ '', config, OS_METADATA_MAPPING[os_id],
+ whole_check, **kwargs
+ )
+
+
+@util.deprecated
+def validate_package_config(
+ config, adapter_id, whole_check=False, **kwargs
+):
+ """Validate package config."""
+ load_metadatas()
+ if adapter_id not in PACKAGE_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'adapter %s is not found in package metedata mapping' % adapter_id
+ )
+ _validate_config(
+ '', config, PACKAGE_METADATA_MAPPING[adapter_id],
+ whole_check, **kwargs
+ )
+
+
+def validate_flavor_config(
+ config, flavor_id, whole_check=False, **kwargs
+):
+ """Validate flavor config."""
+ load_metadatas()
+ if not flavor_id:
+ logging.info('There is no flavor, skipping flavor validation...')
+ elif flavor_id not in FLAVOR_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'flavor %s is not found in flavor metedata mapping' % flavor_id
+ )
+ else:
+ _validate_config(
+ '', config, FLAVOR_METADATA_MAPPING[flavor_id],
+ whole_check, **kwargs
+ )
+
+
+def _filter_metadata(metadata, **kwargs):
+ """Filter metadata before return it to api.
+
+ Some metadata fields are not json compatible or
+ only used in db/api internally.
+ We should strip these fields out before return to api.
+ """
+ if not isinstance(metadata, dict):
+ return metadata
+ filtered_metadata = {}
+ for key, value in metadata.items():
+ if key == '_self':
+ filtered_metadata[key] = {
+ 'name': value['name'],
+ 'description': value.get('description', None),
+ 'default_value': value.get('default_value', None),
+ 'is_required': value.get('is_required', False),
+ 'required_in_whole_config': value.get(
+ 'required_in_whole_config', False),
+ 'js_validator': value.get('js_validator', None),
+ 'options': value.get('options', None),
+ 'required_in_options': value.get(
+ 'required_in_options', False),
+ 'field_type': value.get(
+ 'field_type_data', 'str'),
+ 'display_type': value.get('display_type', None),
+ 'mapping_to': value.get('mapping_to', None)
+ }
+ else:
+ filtered_metadata[key] = _filter_metadata(value, **kwargs)
+ return filtered_metadata
+
+
+@util.deprecated
+def _get_package_metadata(adapter_id):
+ """get package metadata."""
+ load_metadatas()
+ if adapter_id not in PACKAGE_METADATA_MAPPING:
+ raise exception.RecordNotExists(
+ 'adpater %s does not exist' % adapter_id
+ )
+ return _filter_metadata(
+ PACKAGE_METADATA_MAPPING[adapter_id]
+ )
+
+
+@util.deprecated
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_package_metadata(adapter_id, user=None, session=None, **kwargs):
+ """Get package metadata from adapter."""
+ return {
+ 'package_config': _get_package_metadata(adapter_id)
+ }
+
+
+def _get_flavor_metadata(flavor_id):
+ """get flavor metadata."""
+ load_metadatas()
+ if not flavor_id:
+ logging.info('There is no flavor id, skipping...')
+ elif flavor_id not in FLAVOR_METADATA_MAPPING:
+ raise exception.RecordNotExists(
+ 'flavor %s does not exist' % flavor_id
+ )
+ else:
+ return _filter_metadata(FLAVOR_METADATA_MAPPING[flavor_id])
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_flavor_metadata(flavor_id, user=None, session=None, **kwargs):
+ """Get flavor metadata by flavor."""
+ return {
+ 'package_config': _get_flavor_metadata(flavor_id)
+ }
+
+
+def _get_os_metadata(os_id):
+ """get os metadata."""
+ load_metadatas()
+ if os_id not in OS_METADATA_MAPPING:
+ raise exception.RecordNotExists(
+ 'os %s does not exist' % os_id
+ )
+ return _filter_metadata(OS_METADATA_MAPPING[os_id])
+
+
+def _get_os_metadata_ui_converter(os_id):
+ """get os metadata ui converter."""
+ load_metadatas()
+ if os_id not in OS_METADATA_UI_CONVERTERS:
+ raise exception.RecordNotExists(
+ 'os %s does not exist' % os_id
+ )
+ return OS_METADATA_UI_CONVERTERS[os_id]
+
+
+def _get_flavor_metadata_ui_converter(flavor_id):
+ """get flavor metadata ui converter."""
+ load_metadatas()
+ if flavor_id not in FLAVOR_METADATA_UI_CONVERTERS:
+ raise exception.RecordNotExists(
+ 'flavor %s does not exist' % flavor_id
+ )
+ return FLAVOR_METADATA_UI_CONVERTERS[flavor_id]
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_os_metadata(os_id, user=None, session=None, **kwargs):
+ """get os metadatas."""
+ return {'os_config': _get_os_metadata(os_id)}
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS)
+def get_os_ui_metadata(os_id, user=None, session=None, **kwargs):
+ """Get os metadata ui converter by os."""
+ metadata = _get_os_metadata(os_id)
+ metadata_ui_converter = _get_os_metadata_ui_converter(os_id)
+ return _get_ui_metadata(metadata, metadata_ui_converter)
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_UI_METADATA_FIELDS)
+def get_flavor_ui_metadata(flavor_id, user=None, session=None, **kwargs):
+ """Get flavor ui metadata by flavor."""
+ metadata = _get_flavor_metadata(flavor_id)
+ metadata_ui_converter = _get_flavor_metadata_ui_converter(flavor_id)
+ return _get_ui_metadata(metadata, metadata_ui_converter)
+
+
+def _get_ui_metadata(metadata, metadata_ui_converter):
+ """convert metadata to ui metadata.
+
+ Args:
+ metadata: metadata we defined in metadata files.
+ metadata_ui_converter: metadata ui converter defined in metadata
+ mapping files. Used to convert orignal
+ metadata to ui understandable metadata.
+
+ Returns:
+ ui understandable metadata.
+ """
+ ui_metadata = {}
+ ui_metadata[metadata_ui_converter['mapped_name']] = []
+ for mapped_child in metadata_ui_converter['mapped_children']:
+ data_dict = {}
+ for ui_key, ui_value in mapped_child.items():
+ for key, value in ui_value.items():
+ if 'data' == key:
+ result_data = []
+ _get_ui_metadata_data(
+ metadata[ui_key], value, result_data
+ )
+ data_dict['data'] = result_data
+ else:
+ data_dict[key] = value
+ ui_metadata[metadata_ui_converter['mapped_name']].append(data_dict)
+ return ui_metadata
+
+
+def _get_ui_metadata_data(metadata, config, result_data):
+ """Get ui metadata data and fill to result."""
+ data_dict = {}
+ for key, config_value in config.items():
+ if isinstance(config_value, dict) and key != 'content_data':
+ if key in metadata.keys():
+ _get_ui_metadata_data(metadata[key], config_value, result_data)
+ else:
+ _get_ui_metadata_data(metadata, config_value, result_data)
+ elif isinstance(config_value, list):
+ option_list = []
+ for item in config_value:
+ if isinstance(item, dict):
+ option_list.append(item)
+ data_dict[key] = option_list
+ else:
+ if isinstance(metadata['_self'][item], bool):
+ data_dict[item] = str(metadata['_self'][item]).lower()
+ else:
+ data_dict[item] = metadata['_self'][item]
+ else:
+ data_dict[key] = config_value
+ if data_dict:
+ result_data.append(data_dict)
+ return result_data
+
+
+@util.deprecated
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_package_os_metadata(
+ adapter_id, os_id,
+ user=None, session=None, **kwargs
+):
+ """Get metadata by adapter and os."""
+ adapter = adapter_holder_api.get_adapter(
+ adapter_id, user=user, session=session
+ )
+ os_ids = [os['id'] for os in adapter['supported_oses']]
+ if os_id not in os_ids:
+ raise exception.InvalidParameter(
+ 'os %s is not in the supported os list of adapter %s' % (
+ os_id, adapter_id
+ )
+ )
+ metadatas = {}
+ metadatas['os_config'] = _get_os_metadata(
+ os_id
+ )
+ metadatas['package_config'] = _get_package_metadata(
+ adapter_id
+ )
+ return metadatas
+
+
+@utils.supported_filters([])
+@database.run_in_session()
+@user_api.check_user_permission(
+ permission.PERMISSION_LIST_METADATAS
+)
+@utils.wrap_to_dict(RESP_METADATA_FIELDS)
+def get_flavor_os_metadata(
+ flavor_id, os_id,
+ user=None, session=None, **kwargs
+):
+ """Get metadata by flavor and os."""
+ flavor = adapter_holder_api.get_flavor(
+ flavor_id, user=user, session=session
+ )
+ adapter_id = flavor['adapter_id']
+ adapter = adapter_holder_api.get_adapter(
+ adapter_id, user=user, session=session
+ )
+ os_ids = [os['id'] for os in adapter['supported_oses']]
+ if os_id not in os_ids:
+ raise exception.InvalidParameter(
+ 'os %s is not in the supported os list of adapter %s' % (
+ os_id, adapter_id
+ )
+ )
+ metadatas = {}
+ metadatas['os_config'] = _get_os_metadata(
+ session, os_id
+ )
+ metadatas['package_config'] = _get_flavor_metadata(
+ session, flavor_id
+ )
+ return metadatas
+
+
+def _validate_self(
+ config_path, config_key, config,
+ metadata, whole_check,
+ **kwargs
+):
+ """validate config by metadata self section."""
+ logging.debug('validate config self %s', config_path)
+ if '_self' not in metadata:
+ if isinstance(config, dict):
+ _validate_config(
+ config_path, config, metadata, whole_check, **kwargs
+ )
+ return
+ field_type = metadata['_self'].get('field_type', basestring)
+ if not isinstance(config, field_type):
+ raise exception.InvalidParameter(
+ '%s config type is not %s: %s' % (config_path, field_type, config)
+ )
+ is_required = metadata['_self'].get(
+ 'is_required', False
+ )
+ required_in_whole_config = metadata['_self'].get(
+ 'required_in_whole_config', False
+ )
+ if isinstance(config, basestring):
+ if config == '' and not is_required and not required_in_whole_config:
+ # ignore empty config when it is optional
+ return
+ required_in_options = metadata['_self'].get(
+ 'required_in_options', False
+ )
+ options = metadata['_self'].get('options', None)
+ if required_in_options:
+ if field_type in [int, basestring, float, bool]:
+ if options and config not in options:
+ raise exception.InvalidParameter(
+ '%s config is not in %s: %s' % (
+ config_path, options, config
+ )
+ )
+ elif field_type in [list, tuple]:
+ if options and not set(config).issubset(set(options)):
+ raise exception.InvalidParameter(
+ '%s config is not in %s: %s' % (
+ config_path, options, config
+ )
+ )
+ elif field_type == dict:
+ if options and not set(config.keys()).issubset(set(options)):
+ raise exception.InvalidParameter(
+ '%s config is not in %s: %s' % (
+ config_path, options, config
+ )
+ )
+ validator = metadata['_self'].get('validator', None)
+ logging.debug('validate by validator %s', validator)
+ if validator:
+ if not validator(config_key, config, **kwargs):
+ raise exception.InvalidParameter(
+ '%s config is invalid' % config_path
+ )
+ if isinstance(config, dict):
+ _validate_config(
+ config_path, config, metadata, whole_check, **kwargs
+ )
+
+
+def _validate_config(
+ config_path, config, metadata, whole_check,
+ **kwargs
+):
+ """validate config by metadata."""
+ logging.debug('validate config %s', config_path)
+ generals = {}
+ specified = {}
+ for key, value in metadata.items():
+ if key.startswith('$'):
+ generals[key] = value
+ elif key.startswith('_'):
+ pass
+ else:
+ specified[key] = value
+ config_keys = set(config.keys())
+ specified_keys = set(specified.keys())
+ intersect_keys = config_keys & specified_keys
+ not_found_keys = config_keys - specified_keys
+ redundant_keys = specified_keys - config_keys
+ for key in redundant_keys:
+ if '_self' not in specified[key]:
+ continue
+ if specified[key]['_self'].get('is_required', False):
+ raise exception.InvalidParameter(
+ '%s/%s does not find but it is required' % (
+ config_path, key
+ )
+ )
+ if (
+ whole_check and
+ specified[key]['_self'].get(
+ 'required_in_whole_config', False
+ )
+ ):
+ raise exception.InvalidParameter(
+ '%s/%s does not find but it is required in whole config' % (
+ config_path, key
+ )
+ )
+ for key in intersect_keys:
+ _validate_self(
+ '%s/%s' % (config_path, key),
+ key, config[key], specified[key], whole_check,
+ **kwargs
+ )
+ for key in not_found_keys:
+ if not generals:
+ raise exception.InvalidParameter(
+ 'key %s missing in metadata %s' % (
+ key, config_path
+ )
+ )
+ for general_key, general_value in generals.items():
+ _validate_self(
+ '%s/%s' % (config_path, key),
+ key, config[key], general_value, whole_check,
+ **kwargs
+ )
+
+
+def _autofill_self_config(
+ config_path, config_key, config,
+ metadata,
+ **kwargs
+):
+ """Autofill config by metadata self section."""
+ if '_self' not in metadata:
+ if isinstance(config, dict):
+ _autofill_config(
+ config_path, config, metadata, **kwargs
+ )
+ return config
+ logging.debug(
+ 'autofill %s by metadata %s', config_path, metadata['_self']
+ )
+ autofill_callback = metadata['_self'].get(
+ 'autofill_callback', None
+ )
+ autofill_callback_params = metadata['_self'].get(
+ 'autofill_callback_params', {}
+ )
+ callback_params = dict(kwargs)
+ if autofill_callback_params:
+ callback_params.update(autofill_callback_params)
+ default_value = metadata['_self'].get(
+ 'default_value', None
+ )
+ if default_value is not None:
+ callback_params['default_value'] = default_value
+ options = metadata['_self'].get(
+ 'options', None
+ )
+ if options is not None:
+ callback_params['options'] = options
+ if autofill_callback:
+ config = autofill_callback(
+ config_key, config, **callback_params
+ )
+ if config is None:
+ new_config = {}
+ else:
+ new_config = config
+ if isinstance(new_config, dict):
+ _autofill_config(
+ config_path, new_config, metadata, **kwargs
+ )
+ if new_config:
+ config = new_config
+ return config
+
+
+def _autofill_config(
+ config_path, config, metadata, **kwargs
+):
+ """autofill config by metadata."""
+ generals = {}
+ specified = {}
+ for key, value in metadata.items():
+ if key.startswith('$'):
+ generals[key] = value
+ elif key.startswith('_'):
+ pass
+ else:
+ specified[key] = value
+ config_keys = set(config.keys())
+ specified_keys = set(specified.keys())
+ intersect_keys = config_keys & specified_keys
+ not_found_keys = config_keys - specified_keys
+ redundant_keys = specified_keys - config_keys
+ for key in redundant_keys:
+ self_config = _autofill_self_config(
+ '%s/%s' % (config_path, key),
+ key, None, specified[key], **kwargs
+ )
+ if self_config is not None:
+ config[key] = self_config
+ for key in intersect_keys:
+ config[key] = _autofill_self_config(
+ '%s/%s' % (config_path, key),
+ key, config[key], specified[key],
+ **kwargs
+ )
+ for key in not_found_keys:
+ for general_key, general_value in generals.items():
+ config[key] = _autofill_self_config(
+ '%s/%s' % (config_path, key),
+ key, config[key], general_value,
+ **kwargs
+ )
+ return config
+
+
+def autofill_os_config(
+ config, os_id, **kwargs
+):
+ load_metadatas()
+ if os_id not in OS_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'os %s is not found in os metadata mapping' % os_id
+ )
+
+ return _autofill_config(
+ '', config, OS_METADATA_MAPPING[os_id], **kwargs
+ )
+
+
+def autofill_package_config(
+ config, adapter_id, **kwargs
+):
+ load_metadatas()
+ if adapter_id not in PACKAGE_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'adapter %s is not found in package metadata mapping' % adapter_id
+ )
+
+ return _autofill_config(
+ '', config, PACKAGE_METADATA_MAPPING[adapter_id], **kwargs
+ )
+
+
+def autofill_flavor_config(
+ config, flavor_id, **kwargs
+):
+ load_metadatas()
+ if not flavor_id:
+ logging.info('There is no flavor, skipping...')
+ elif flavor_id not in FLAVOR_METADATA_MAPPING:
+ raise exception.InvalidParameter(
+ 'flavor %s is not found in flavor metadata mapping' % flavor_id
+ )
+ else:
+ return _autofill_config(
+ '', config, FLAVOR_METADATA_MAPPING[flavor_id], **kwargs
+ )