From f977b94e012828cd2a905b9b841aff33c4b732f8 Mon Sep 17 00:00:00 2001 From: shangxdy Date: Wed, 14 Sep 2016 14:36:33 +0800 Subject: Update the upstream of tosco-parser and heat-translator to stable release 0.6/0.5 Currently the parser is based on dev branch of upstream projects, include tosco-parser and heat-translator, for the colorado release of parser, it should be based on a stable version, so it's necessary to update the upstream version to 0.6/tosca-parser and 0.5/heat-translator. JIRA:PARSER-106 Change-Id: I8fb043068d25188c47e5648e1b66184446ac82d6 Signed-off-by: shangxdy --- .../translator/hot/syntax/hot_output.py | 7 +- .../translator/hot/syntax/hot_resource.py | 36 +-- .../translator/hot/syntax/hot_template.py | 1 - .../hot/tosca/tests/test_tosca_compute.py | 21 +- .../translator/hot/tosca/tosca_block_storage.py | 2 +- .../hot/tosca/tosca_block_storage_attachment.py | 2 +- .../translator/hot/tosca/tosca_compute.py | 13 +- .../translator/hot/translate_node_templates.py | 287 ++++----------------- .../translator/hot/translate_outputs.py | 14 +- 9 files changed, 97 insertions(+), 286 deletions(-) (limited to 'tosca2heat/heat-translator/translator/hot') diff --git a/tosca2heat/heat-translator/translator/hot/syntax/hot_output.py b/tosca2heat/heat-translator/translator/hot/syntax/hot_output.py index a41208a..ad77fb3 100644 --- a/tosca2heat/heat-translator/translator/hot/syntax/hot_output.py +++ b/tosca2heat/heat-translator/translator/hot/syntax/hot_output.py @@ -21,8 +21,5 @@ class HotOutput(object): self.description = description def get_dict_output(self): - if self.description: - return {self.name: {'value': self.value, - 'description': self.description}} - else: - return {self.name: {'value': self.value}} + return {self.name: {'value': self.value, + 'description': self.description}} diff --git a/tosca2heat/heat-translator/translator/hot/syntax/hot_resource.py b/tosca2heat/heat-translator/translator/hot/syntax/hot_resource.py index 7b83906..54e0d96 100644 --- a/tosca2heat/heat-translator/translator/hot/syntax/hot_resource.py +++ b/tosca2heat/heat-translator/translator/hot/syntax/hot_resource.py @@ -103,7 +103,7 @@ class HotResource(object): # scenarios and cannot be fixed or hard coded here operations_deploy_sequence = ['create', 'configure', 'start'] - operations = HotResource.get_all_operations(self.nodetemplate) + operations = HotResource._get_all_operations(self.nodetemplate) # create HotResource for each operation used for deployment: # create, start, configure @@ -140,15 +140,7 @@ class HotResource(object): # hosting_server is None if requirements is None hosting_on_server = (hosting_server.name if hosting_server else None) - base_type = HotResource.get_base_type( - self.nodetemplate.type_definition).type - # handle interfaces directly defined on a compute - if hosting_on_server is None \ - and base_type == 'tosca.nodes.Compute': - hosting_on_server = self.name - - if operation.name == reserve_current and \ - base_type != 'tosca.nodes.Compute': + if operation.name == reserve_current: deploy_resource = self self.name = deploy_name self.type = 'OS::Heat::SoftwareDeployment' @@ -156,7 +148,7 @@ class HotResource(object): 'server': {'get_resource': hosting_on_server}, 'signal_transport': 'HEAT_SIGNAL'} - deploy_lookup[operation] = self + deploy_lookup[operation.name] = self else: sd_config = {'config': {'get_resource': config_name}, 'server': {'get_resource': @@ -168,7 +160,7 @@ class HotResource(object): 'OS::Heat::SoftwareDeployment', sd_config) hot_resources.append(deploy_resource) - deploy_lookup[operation] = deploy_resource + deploy_lookup[operation.name] = deploy_resource lifecycle_inputs = self._get_lifecycle_inputs(operation) if lifecycle_inputs: deploy_resource.properties['input_values'] = \ @@ -178,34 +170,24 @@ class HotResource(object): # in operations_deploy_sequence # TODO(anyone): find some better way to encode this implicit sequence group = {} - op_index_max = -1 for op, hot in deploy_lookup.items(): # position to determine potential preceding nodes - op_index = operations_deploy_sequence.index(op.name) - if op_index > op_index_max: - op_index_max = op_index - for preceding_op_name in \ + op_index = operations_deploy_sequence.index(op) + for preceding_op in \ reversed(operations_deploy_sequence[:op_index]): - preceding_hot = deploy_lookup.get( - operations.get(preceding_op_name)) + preceding_hot = deploy_lookup.get(preceding_op) if preceding_hot: hot.depends_on.append(preceding_hot) hot.depends_on_nodes.append(preceding_hot) group[preceding_hot] = hot break - if op_index_max >= 0: - last_deploy = deploy_lookup.get(operations.get( - operations_deploy_sequence[op_index_max])) - else: - last_deploy = None - # save this dependency chain in the set of HOT resources self.group_dependencies.update(group) for hot in hot_resources: hot.group_dependencies.update(group) - return hot_resources, deploy_lookup, last_deploy + return hot_resources def handle_connectsto(self, tosca_source, tosca_target, hot_source, hot_target, config_location, operation): @@ -353,7 +335,7 @@ class HotResource(object): return tosca_props @staticmethod - def get_all_operations(node): + def _get_all_operations(node): operations = {} for operation in node.interfaces: operations[operation.name] = operation diff --git a/tosca2heat/heat-translator/translator/hot/syntax/hot_template.py b/tosca2heat/heat-translator/translator/hot/syntax/hot_template.py index 0403562..4263c4d 100644 --- a/tosca2heat/heat-translator/translator/hot/syntax/hot_template.py +++ b/tosca2heat/heat-translator/translator/hot/syntax/hot_template.py @@ -77,7 +77,6 @@ class HotTemplate(object): dict_output.update({self.OUTPUTS: all_outputs}) yaml.add_representer(OrderedDict, self.represent_ordereddict) - yaml.add_representer(dict, self.represent_ordereddict) yaml_string = yaml.dump(dict_output, default_flow_style=False) # get rid of the '' from yaml.dump around numbers yaml_string = yaml_string.replace('\'', '') diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py b/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py index 743074b..408ee8b 100644 --- a/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py +++ b/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py @@ -16,6 +16,7 @@ from mock import patch from toscaparser.nodetemplate import NodeTemplate from toscaparser.tests.base import TestCase +from toscaparser.utils.gettextutils import _ import toscaparser.utils.yamlparser from translator.hot.tosca.tosca_compute import ToscaCompute @@ -26,12 +27,22 @@ class ToscaComputeTest(TestCase): nodetemplates = (toscaparser.utils.yamlparser. simple_parse(tpl_snippet)['node_templates']) name = list(nodetemplates.keys())[0] - nodetemplate = NodeTemplate(name, nodetemplates) - nodetemplate.validate() - toscacompute = ToscaCompute(nodetemplate) - toscacompute.handle_properties() + try: + nodetemplate = NodeTemplate(name, nodetemplates) + nodetemplate.validate() + toscacompute = ToscaCompute(nodetemplate) + toscacompute.handle_properties() + if not self._compare_properties(toscacompute.properties, + expectedprops): + raise Exception(_("Hot Properties are not" + " same as expected properties")) + except Exception: + # for time being rethrowing. Will be handled future based + # on new development in Glance and Graffiti + raise - self.assertDictEqual(expectedprops, toscacompute.properties) + def _compare_properties(self, hotprops, expectedprops): + return all(item in hotprops.items() for item in expectedprops.items()) def test_node_compute_with_host_and_os_capabilities(self): tpl_snippet = ''' diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py index 924ff9d..d4b2f44 100644 --- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py +++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py @@ -67,5 +67,5 @@ class ToscaBlockStorage(HotResource): # attribute for the matching resource. Unless there is additional # runtime support, this should be a one to one mapping. if attribute == 'volume_id': - attr['get_resource'] = self.name + attr['get_resource'] = args[0] return attr diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py index 2242c2d..71b9822 100644 --- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py +++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py @@ -50,4 +50,4 @@ class ToscaBlockStorageAttachment(HotResource): self.properties.pop('device') def handle_life_cycle(self): - return None, None, None + pass diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py index 16d0518..8a959d1 100644 --- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py +++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py @@ -84,14 +84,6 @@ class ToscaCompute(HotResource): ('architecture', 'distribution', 'type', 'version') toscatype = 'tosca.nodes.Compute' - ALLOWED_NOVA_SERVER_PROPS = \ - ('admin_pass', 'availability_zone', 'block_device_mapping', - 'block_device_mapping_v2', 'config_drive', 'diskConfig', 'flavor', - 'flavor_update_policy', 'image', 'image_update_policy', 'key_name', - 'metadata', 'name', 'networks', 'personality', 'reservation_id', - 'scheduler_hints', 'security_groups', 'software_config_transport', - 'user_data', 'user_data_format', 'user_data_update_policy') - def __init__(self, nodetemplate): super(ToscaCompute, self).__init__(nodetemplate, type='OS::Nova::Server') @@ -107,8 +99,7 @@ class ToscaCompute(HotResource): self.properties['software_config_transport'] = 'POLL_SERVER_HEAT' tosca_props = self.get_tosca_props() for key, value in tosca_props.items(): - if key in self.ALLOWED_NOVA_SERVER_PROPS: - self.properties[key] = value + self.properties[key] = value # To be reorganized later based on new development in Glance and Graffiti def translate_compute_flavor_and_image(self, @@ -264,7 +255,7 @@ class ToscaCompute(HotResource): images = IMAGES if translator.common.utils.check_for_env_variables(): resp = self._populate_image_dict() - if resp and len(resp.keys()) > 0: + if len(resp.keys()) > 0: images = resp match_all = images.keys() architecture = properties.get(self.ARCHITECTURE) diff --git a/tosca2heat/heat-translator/translator/hot/translate_node_templates.py b/tosca2heat/heat-translator/translator/hot/translate_node_templates.py index 4dd9556..0aefd48 100644 --- a/tosca2heat/heat-translator/translator/hot/translate_node_templates.py +++ b/tosca2heat/heat-translator/translator/hot/translate_node_templates.py @@ -16,8 +16,6 @@ import logging import os import six -from collections import OrderedDict -from toscaparser.functions import Concat from toscaparser.functions import GetAttribute from toscaparser.functions import GetInput from toscaparser.functions import GetProperty @@ -27,7 +25,6 @@ from toscaparser.utils.gettextutils import _ from translator.common.exception import ToscaClassAttributeError from translator.common.exception import ToscaClassImportError from translator.common.exception import ToscaModImportError -from translator.common import utils from translator.conf.config import ConfigProvider as translatorConfig from translator.hot.syntax.hot_resource import HotResource from translator.hot.tosca.tosca_block_storage_attachment import ( @@ -135,8 +132,6 @@ log = logging.getLogger('heat-translator') TOSCA_TO_HOT_TYPE = _generate_type_map() -BASE_TYPES = six.string_types + six.integer_types + (dict, OrderedDict) - class TranslateNodeTemplates(object): '''Translate TOSCA NodeTemplates to Heat Resources.''' @@ -151,9 +146,6 @@ class TranslateNodeTemplates(object): log.debug(_('Mapping between TOSCA nodetemplate and HOT resource.')) self.hot_lookup = {} self.policies = self.tosca.topology_template.policies - # stores the last deploy of generated behavior for a resource - # useful to satisfy underlying dependencies between interfaces - self.last_deploy_map = {} def translate(self): return self._translate_nodetemplates() @@ -231,14 +223,9 @@ class TranslateNodeTemplates(object): # into multiple HOT resources and may change their name lifecycle_resources = [] for resource in self.hot_resources: - expanded_resources, deploy_lookup, last_deploy = resource.\ - handle_life_cycle() - if expanded_resources: - lifecycle_resources += expanded_resources - if deploy_lookup: - self.hot_lookup.update(deploy_lookup) - if last_deploy: - self.last_deploy_map[resource] = last_deploy + expanded = resource.handle_life_cycle() + if expanded: + lifecycle_resources += expanded self.hot_resources += lifecycle_resources # Handle configuration from ConnectsTo relationship in the TOSCA node: @@ -270,9 +257,7 @@ class TranslateNodeTemplates(object): # if the source of dependency is a server and the # relationship type is 'tosca.relationships.HostedOn', # add dependency as properties.server - base_type = HotResource.get_base_type( - node_depend.type_definition) - if base_type.type == 'tosca.nodes.Compute' and \ + if node_depend.type == 'tosca.nodes.Compute' and \ node.related[node_depend].type == \ node.type_definition.HOSTEDON: self.hot_lookup[node].properties['server'] = \ @@ -285,13 +270,6 @@ class TranslateNodeTemplates(object): self.hot_lookup[node].depends_on_nodes.append( self.hot_lookup[node_depend].top_of_chain()) - last_deploy = self.last_deploy_map.get( - self.hot_lookup[node_depend]) - if last_deploy and \ - last_deploy not in self.hot_lookup[node].depends_on: - self.hot_lookup[node].depends_on.append(last_deploy) - self.hot_lookup[node].depends_on_nodes.append(last_deploy) - # handle hosting relationship for resource in self.hot_resources: resource.handle_hosting() @@ -321,202 +299,53 @@ class TranslateNodeTemplates(object): inputs = resource.properties.get('input_values') if inputs: for name, value in six.iteritems(inputs): - inputs[name] = self.translate_param_value(value, resource) - - # remove resources without type defined - # for example a SoftwareComponent without interfaces - # would fall in this case - to_remove = [] - for resource in self.hot_resources: - if resource.type is None: - to_remove.append(resource) - - for resource in to_remove: - self.hot_resources.remove(resource) + inputs[name] = self._translate_input(value, resource) return self.hot_resources - def translate_param_value(self, param_value, resource): - tosca_template = None - if resource: - tosca_template = resource.nodetemplate - + def _translate_input(self, input_value, resource): get_property_args = None - if isinstance(param_value, GetProperty): - get_property_args = param_value.args + if isinstance(input_value, GetProperty): + get_property_args = input_value.args # to remove when the parser is fixed to return GetProperty - elif isinstance(param_value, dict) and 'get_property' in param_value: - get_property_args = param_value['get_property'] + if isinstance(input_value, dict) and 'get_property' in input_value: + get_property_args = input_value['get_property'] if get_property_args is not None: - tosca_target, prop_name, prop_arg = \ - self.decipher_get_operation(get_property_args, - tosca_template) - if tosca_target: - prop_value = tosca_target.get_property_value(prop_name) - if prop_value: - prop_value = self.translate_param_value( - prop_value, resource) - return self._unfold_value(prop_value, prop_arg) - get_attr_args = None - if isinstance(param_value, GetAttribute): - get_attr_args = param_value.result().args - # to remove when the parser is fixed to return GetAttribute - elif isinstance(param_value, dict) and 'get_attribute' in param_value: - get_attr_args = param_value['get_attribute'] - if get_attr_args is not None: + hot_target = self._find_hot_resource_for_tosca( + get_property_args[0], resource) + if hot_target: + props = hot_target.get_tosca_props() + prop_name = get_property_args[1] + if prop_name in props: + return props[prop_name] + elif isinstance(input_value, GetAttribute): # for the attribute # get the proper target type to perform the translation - tosca_target, attr_name, attr_arg = \ - self.decipher_get_operation(get_attr_args, tosca_template) - attr_args = [] - if attr_arg: - attr_args += attr_arg - if tosca_target: - if tosca_target in self.hot_lookup: - attr_value = self.hot_lookup[tosca_target].\ - get_hot_attribute(attr_name, attr_args) - attr_value = self.translate_param_value( - attr_value, resource) - return self._unfold_value(attr_value, attr_arg) - elif isinstance(param_value, dict) and 'get_artifact' in param_value: - get_artifact_args = param_value['get_artifact'] - tosca_target, artifact_name, _ = \ - self.decipher_get_operation(get_artifact_args, - tosca_template) - - if tosca_target: - artifacts = self.get_all_artifacts(tosca_target) - if artifact_name in artifacts: - artifact = artifacts[artifact_name] - if artifact.get('type', None) == 'tosca.artifacts.File': - return {'get_file': artifact.get('file')} - get_input_args = None - if isinstance(param_value, GetInput): - get_input_args = param_value.args - elif isinstance(param_value, dict) and 'get_input' in param_value: - get_input_args = param_value['get_input'] - if get_input_args is not None: - if isinstance(get_input_args, list) \ - and len(get_input_args) == 1: - return {'get_param': self.translate_param_value( - get_input_args[0], resource)} - else: - return {'get_param': self.translate_param_value( - get_input_args, resource)} - elif isinstance(param_value, dict) \ - and 'get_operation_output' in param_value: - res = self._translate_get_operation_output_function( - param_value['get_operation_output'], tosca_template) - if res: - return res - concat_list = None - if isinstance(param_value, Concat): - concat_list = param_value.args - elif isinstance(param_value, dict) and 'concat' in param_value: - concat_list = param_value['concat'] - if concat_list is not None: - res = self._translate_concat_function(concat_list, resource) - if res: - return res - - if isinstance(param_value, list): - translated_list = [] - for elem in param_value: - translated_elem = self.translate_param_value(elem, resource) - if translated_elem: - translated_list.append(translated_elem) - return translated_list - - if isinstance(param_value, BASE_TYPES): - return param_value - - return None - - def _translate_concat_function(self, concat_list, resource): - str_replace_template = '' - str_replace_params = {} - index = 0 - for elem in concat_list: - str_replace_template += '$s' + str(index) - str_replace_params['$s' + str(index)] = \ - self.translate_param_value(elem, resource) - index += 1 - - return {'str_replace': { - 'template': str_replace_template, - 'params': str_replace_params - }} - - def _translate_get_operation_output_function(self, args, tosca_template): - tosca_target = self._find_tosca_node(args[0], - tosca_template) - if tosca_target and len(args) >= 4: - operations = HotResource.get_all_operations(tosca_target) - # ignore Standard interface name, - # it is the only one supported in the translator anyway - op_name = args[2] - output_name = args[3] - if op_name in operations: - operation = operations[op_name] - if operation in self.hot_lookup: - matching_deploy = self.hot_lookup[operation] - matching_config_name = matching_deploy.\ - properties['config']['get_resource'] - matching_config = self.find_hot_resource( - matching_config_name) - if matching_config: - outputs = matching_config.properties.get('outputs') - if outputs is None: - outputs = [] - outputs.append({'name': output_name}) - matching_config.properties['outputs'] = outputs - return {'get_attr': [ - matching_deploy.name, - output_name - ]} - - @staticmethod - def _unfold_value(value, value_arg): - if value_arg is not None: - if isinstance(value, dict): - val = value.get(value_arg) - if val is not None: - return val - - index = utils.str_to_num(value_arg) - if isinstance(value, list) and index is not None: - return value[index] - return value - - def decipher_get_operation(self, args, current_tosca_node): - tosca_target = self._find_tosca_node(args[0], - current_tosca_node) - new_target = None - if tosca_target and len(args) > 2: - cap_or_req_name = args[1] - cap = tosca_target.get_capability(cap_or_req_name) - if cap: - new_target = cap + args = input_value.result().args + hot_target = self._find_hot_resource_for_tosca(args[0], resource) + + return hot_target.get_hot_attribute(args[1], args) + # most of artifacts logic should move to the parser + elif isinstance(input_value, dict) and 'get_artifact' in input_value: + get_artifact_args = input_value['get_artifact'] + + hot_target = self._find_hot_resource_for_tosca( + get_artifact_args[0], resource) + artifacts = TranslateNodeTemplates.get_all_artifacts( + hot_target.nodetemplate) + + if get_artifact_args[1] in artifacts: + artifact = artifacts[get_artifact_args[1]] + if artifact.get('type', None) == 'tosca.artifacts.File': + return {'get_file': artifact.get('file')} + elif isinstance(input_value, GetInput): + if isinstance(input_value.args, list) \ + and len(input_value.args) == 1: + return {'get_param': input_value.args[0]} else: - for req in tosca_target.requirements: - if cap_or_req_name in req: - new_target = self._find_tosca_node( - req[cap_or_req_name]) - cap = new_target.get_capability(cap_or_req_name) - if cap: - new_target = cap - break - - if new_target: - tosca_target = new_target - - prop_name = args[2] - prop_arg = args[3] if len(args) >= 4 else None - else: - prop_name = args[1] - prop_arg = args[2] if len(args) >= 3 else None + return {'get_param': input_value.args} - return tosca_target, prop_name, prop_arg + return input_value @staticmethod def get_all_artifacts(nodetemplate): @@ -591,29 +420,23 @@ class TranslateNodeTemplates(object): if resource.name == name: return resource - def _find_tosca_node(self, tosca_name, current_tosca_template=None): - tosca_node = None - if tosca_name == 'SELF': - tosca_node = current_tosca_template - if tosca_name == 'HOST' and current_tosca_template: - for req in current_tosca_template.requirements: - if 'host' in req: - tosca_node = self._find_tosca_node(req['host']) - - if tosca_node is None: - for node in self.nodetemplates: - if node.name == tosca_name: - tosca_node = node - break - return tosca_node + def _find_tosca_node(self, tosca_name): + for node in self.nodetemplates: + if node.name == tosca_name: + return node def _find_hot_resource_for_tosca(self, tosca_name, current_hot_resource=None): - current_tosca_resource = current_hot_resource.nodetemplate \ - if current_hot_resource else None - tosca_node = self._find_tosca_node(tosca_name, current_tosca_resource) - if tosca_node: - return self.hot_lookup[tosca_node] + if tosca_name == 'SELF': + return current_hot_resource + if tosca_name == 'HOST' and current_hot_resource is not None: + for req in current_hot_resource.nodetemplate.requirements: + if 'host' in req: + return self._find_hot_resource_for_tosca(req['host']) + + for node in self.nodetemplates: + if node.name == tosca_name: + return self.hot_lookup[node] return None diff --git a/tosca2heat/heat-translator/translator/hot/translate_outputs.py b/tosca2heat/heat-translator/translator/hot/translate_outputs.py index 87ec02a..4197cdd 100644 --- a/tosca2heat/heat-translator/translator/hot/translate_outputs.py +++ b/tosca2heat/heat-translator/translator/hot/translate_outputs.py @@ -33,8 +33,16 @@ class TranslateOutputs(object): def _translate_outputs(self): hot_outputs = [] for output in self.outputs: - hot_value = self.nodes.translate_param_value(output.value, None) - if hot_value is not None: - hot_outputs.append(HotOutput(output.name, hot_value, + if output.value.name == 'get_attribute': + get_parameters = output.value.args + hot_target = self.nodes.find_hot_resource(get_parameters[0]) + hot_value = hot_target.get_hot_attribute(get_parameters[1], + get_parameters) + hot_outputs.append(HotOutput(output.name, + hot_value, + output.description)) + else: + hot_outputs.append(HotOutput(output.name, + output.value, output.description)) return hot_outputs -- cgit 1.2.3-korg