diff options
80 files changed, 2404 insertions, 656 deletions
diff --git a/tosca2heat/README.md b/tosca2heat/README.md index d2f36a7..cfd4293 100644 --- a/tosca2heat/README.md +++ b/tosca2heat/README.md @@ -2,3 +2,4 @@ Basic version information: 1. tosca-paser is based the version of 0.5 in openstack community 2. heat-translator is based the version of 0.5 in openstack community +3. the last bugs from openstack is at July 7, 2016. diff --git a/tosca2heat/heat-translator/.gitignore b/tosca2heat/heat-translator/.gitignore new file mode 100644 index 0000000..7dfa8ae --- /dev/null +++ b/tosca2heat/heat-translator/.gitignore @@ -0,0 +1,53 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +.testrepository + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.idea +*.iml
\ No newline at end of file diff --git a/tosca2heat/heat-translator/.gitreview b/tosca2heat/heat-translator/.gitreview new file mode 100644 index 0000000..59046fa --- /dev/null +++ b/tosca2heat/heat-translator/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=openstack/heat-translator.git diff --git a/tosca2heat/heat-translator/.mailmap b/tosca2heat/heat-translator/.mailmap new file mode 100644 index 0000000..cc92f17 --- /dev/null +++ b/tosca2heat/heat-translator/.mailmap @@ -0,0 +1,3 @@ +# Format is: +# <preferred e-mail> <other e-mail 1> +# <preferred e-mail> <other e-mail 2>
\ No newline at end of file diff --git a/tosca2heat/heat-translator/doc/source/usage.rst b/tosca2heat/heat-translator/doc/source/usage.rst index ebad0e8..e8f132a 100644 --- a/tosca2heat/heat-translator/doc/source/usage.rst +++ b/tosca2heat/heat-translator/doc/source/usage.rst @@ -65,10 +65,11 @@ In this case, you can simply run translation via CLI entry point:: Things To Consider ------------------ -* When deploying the translated template with Heat, please ensure that you have image registered in the Glance. The Heat-Translator - project sets flavor and image from a pre-defined set of values (as listed in /home/openstack/heat-translator/translator/hot/tosca/tosca_compute.py) - with the best possible match to the constraints defined in the TOSCA template. If there is no possible match found, a null value is set currently. - Per the future plan, an image and flavor will be provided from an online repository. +* When use Heat-Translator in an OpenStack environment, please ensure that you have one or more preferred flavors and images available in your OpenStack + environment. To find an appropriate flavor and image, that meets constraints defined in the TOSCA template for the HOST and OS capabilities of TOSCA Compute node, + the Heat-Translator project first runs a query against Nova flavors and Glance images. During the query call, it uses the metadata of flavors and images. + If call to Nova or Glance can not be made or no flavor or image is found, the Heat-Translator project will set flavor and image from a pre-defined set of values (as listed in /home/openstack/heat-translator/translator/hot/tosca/tosca_compute.py) + with the best possible match to the constraints defined in the TOSCA template. * The ``key_name`` property of Nova server is irrelevant to the TOSCA specification and can not be used in TOSCA template. In order to use it in the translated templates, the user must provide it via parameters, and the heat-translator will set it to all resources of ``OS::Nova::Server`` type. * Since properties of TOSCA Compute OS and HOST capabilities are optional, the user should make sure that either they set these properties correctly diff --git a/tosca2heat/heat-translator/requirements.txt b/tosca2heat/heat-translator/requirements.txt index e54d0ab..b9792da 100644 --- a/tosca2heat/heat-translator/requirements.txt +++ b/tosca2heat/heat-translator/requirements.txt @@ -2,9 +2,9 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. pbr>=1.6 # Apache-2.0 -Babel>=1.3 # BSD +Babel>=2.3.4 # BSD cliff!=1.16.0,!=1.17.0,>=1.15.0 # Apache-2.0 PyYAML>=3.1.0 # MIT python-dateutil>=2.4.2 # BSD six>=1.9.0 # MIT -tosca-parser>=0.4.0 # Apache-2.0 +tosca-parser>=0.5.0 # Apache-2.0 diff --git a/tosca2heat/heat-translator/run_py27-tosca-parser-master.sh b/tosca2heat/heat-translator/run_py27-tosca-parser-master.sh new file mode 100644 index 0000000..19fe15a --- /dev/null +++ b/tosca2heat/heat-translator/run_py27-tosca-parser-master.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +pip install -q -U -e "git+https://git.openstack.org/openstack/tosca-parser.git#egg=tosca_parser" diff --git a/tosca2heat/heat-translator/test-requirements.txt b/tosca2heat/heat-translator/test-requirements.txt index 2b9cae7..70a261b 100644 --- a/tosca2heat/heat-translator/test-requirements.txt +++ b/tosca2heat/heat-translator/test-requirements.txt @@ -4,11 +4,11 @@ hacking<0.11,>=0.10.0 coverage>=3.6 # Apache-2.0 discover # BSD -fixtures>=1.3.1 # Apache-2.0/BSD +fixtures>=3.0.0 # Apache-2.0/BSD oslotest>=1.10.0 # Apache-2.0 oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 python-subunit>=0.0.18 # Apache-2.0/BSD -sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD +sphinx!=1.3b1,<1.3,>=1.2.1 # BSD testrepository>=0.0.18 # Apache-2.0/BSD testscenarios>=0.4 # Apache-2.0/BSD testtools>=1.4.0 # MIT diff --git a/tosca2heat/heat-translator/tox.ini b/tosca2heat/heat-translator/tox.ini index 868e95a..11c9c97 100644 --- a/tosca2heat/heat-translator/tox.ini +++ b/tosca2heat/heat-translator/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = py34,py27,pypy,pep8 +envlist = py34,py27,pep8 skipsdist = True [testenv] @@ -27,6 +27,11 @@ commands = python setup.py build_sphinx [testenv:debug] commands = oslo_debug_helper -t translator/tests {posargs} +[testenv:py27-tosca-parser-master] +commands = + ./run_py27-tosca-parser-master.sh + python setup.py test --slowest --testr-args='{posargs}' + [flake8] # H803 skipped on purpose per list discussion. # E123, E125 skipped as they are invalid PEP-8. diff --git a/tosca2heat/heat-translator/translator/common/utils.py b/tosca2heat/heat-translator/translator/common/utils.py index 459b5ee..8e4b690 100644 --- a/tosca2heat/heat-translator/translator/common/utils.py +++ b/tosca2heat/heat-translator/translator/common/utils.py @@ -18,6 +18,7 @@ import numbers import os import re import requests +import six from six.moves.urllib.parse import urlparse import yaml @@ -262,12 +263,17 @@ class UrlUtils(object): def str_to_num(value): """Convert a string representation of a number into a numeric type.""" - if isinstance(value, numbers.Number): + if isinstance(value, numbers.Number) \ + or isinstance(value, six.integer_types) \ + or isinstance(value, float): return value try: return int(value) except ValueError: - return float(value) + try: + return float(value) + except ValueError: + return None def check_for_env_variables(): diff --git a/tosca2heat/heat-translator/translator/conf/config.py b/tosca2heat/heat-translator/translator/conf/config.py index 4e8fe87..52ac458 100644 --- a/tosca2heat/heat-translator/translator/conf/config.py +++ b/tosca2heat/heat-translator/translator/conf/config.py @@ -12,6 +12,7 @@ # under the License. ''' Provide a global configuration for the TOSCA translator''' +import os from six.moves import configparser @@ -65,3 +66,11 @@ class ConfigProvider(object): raise exception.ConfSectionNotDefined(section=section) return values + + @classmethod + def get_translator_logging_file(cls): + conf_file = '' + CONF_FILENAME = 'heat_translator_logging.conf' + conf_path = os.path.dirname(os.path.abspath(__file__)) + conf_file = os.path.join(conf_path, CONF_FILENAME) + return conf_file diff --git a/tosca2heat/heat-translator/heat_translator_logging.conf b/tosca2heat/heat-translator/translator/conf/heat_translator_logging.conf index e01a889..e01a889 100644 --- a/tosca2heat/heat-translator/heat_translator_logging.conf +++ b/tosca2heat/heat-translator/translator/conf/heat_translator_logging.conf diff --git a/tosca2heat/heat-translator/translator/hot/syntax/hot_output.py b/tosca2heat/heat-translator/translator/hot/syntax/hot_output.py index ad77fb3..a41208a 100644 --- a/tosca2heat/heat-translator/translator/hot/syntax/hot_output.py +++ b/tosca2heat/heat-translator/translator/hot/syntax/hot_output.py @@ -21,5 +21,8 @@ class HotOutput(object): self.description = description def get_dict_output(self): - return {self.name: {'value': self.value, - 'description': self.description}} + if self.description: + return {self.name: {'value': self.value, + 'description': self.description}} + else: + return {self.name: {'value': self.value}} diff --git a/tosca2heat/heat-translator/translator/hot/syntax/hot_resource.py b/tosca2heat/heat-translator/translator/hot/syntax/hot_resource.py index bbbeb46..261d8ee 100644 --- a/tosca2heat/heat-translator/translator/hot/syntax/hot_resource.py +++ b/tosca2heat/heat-translator/translator/hot/syntax/hot_resource.py @@ -1,4 +1,3 @@ -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -13,6 +12,7 @@ from collections import OrderedDict import logging +import os import six from toscaparser.elements.interfaces import InterfacesDef @@ -44,7 +44,26 @@ class HotResource(object): self.properties = properties or {} # special case for HOT softwareconfig if type == 'OS::Heat::SoftwareConfig': - self.properties['group'] = 'script' + config = self.properties.get('config') + if config: + implementation_artifact = config.get('get_file') + if implementation_artifact: + filename, file_extension = os.path.splitext( + implementation_artifact) + file_extension = file_extension.lower() + # artifact_types should be read to find the exact script + # type, unfortunately artifact_types doesn't seem to be + # supported by the parser + if file_extension == '.ansible' \ + or file_extension == '.yaml' \ + or file_extension == '.yml': + self.properties['group'] = 'ansible' + if file_extension == '.pp': + self.properties['group'] = 'puppet' + + if self.properties.get('group') is None: + self.properties['group'] = 'script' + self.metadata = metadata # The difference between depends_on and depends_on_nodes is @@ -84,7 +103,7 @@ class HotResource(object): # scenarios and cannot be fixed or hard coded here operations_deploy_sequence = ['create', 'configure', 'start'] - operations = HotResource._get_all_operations(self.nodetemplate) + operations = HotResource.get_all_operations(self.nodetemplate) # create HotResource for each operation used for deployment: # create, start, configure @@ -121,14 +140,22 @@ class HotResource(object): # hosting_server is None if requirements is None hosting_on_server = (hosting_server.name if hosting_server else None) - if operation.name == reserve_current: + base_type = HotResource.get_base_type( + self.nodetemplate.type_definition).type + # handle interfaces directly defined on a compute + if hosting_on_server is None \ + and base_type == 'tosca.nodes.Compute': + hosting_on_server = self.name + + if operation.name == reserve_current and \ + base_type != 'tosca.nodes.Compute': deploy_resource = self self.name = deploy_name self.type = 'OS::Heat::SoftwareDeployment' self.properties = {'config': {'get_resource': config_name}, 'server': {'get_resource': hosting_on_server}} - deploy_lookup[operation.name] = self + deploy_lookup[operation] = self else: sd_config = {'config': {'get_resource': config_name}, 'server': {'get_resource': @@ -139,7 +166,7 @@ class HotResource(object): 'OS::Heat::SoftwareDeployment', sd_config) hot_resources.append(deploy_resource) - deploy_lookup[operation.name] = deploy_resource + deploy_lookup[operation] = deploy_resource lifecycle_inputs = self._get_lifecycle_inputs(operation) if lifecycle_inputs: deploy_resource.properties['input_values'] = \ @@ -149,24 +176,34 @@ class HotResource(object): # in operations_deploy_sequence # TODO(anyone): find some better way to encode this implicit sequence group = {} + op_index_max = -1 for op, hot in deploy_lookup.items(): # position to determine potential preceding nodes - op_index = operations_deploy_sequence.index(op) - for preceding_op in \ + op_index = operations_deploy_sequence.index(op.name) + if op_index > op_index_max: + op_index_max = op_index + for preceding_op_name in \ reversed(operations_deploy_sequence[:op_index]): - preceding_hot = deploy_lookup.get(preceding_op) + preceding_hot = deploy_lookup.get( + operations.get(preceding_op_name)) if preceding_hot: hot.depends_on.append(preceding_hot) hot.depends_on_nodes.append(preceding_hot) group[preceding_hot] = hot break + if op_index_max >= 0: + last_deploy = deploy_lookup.get(operations.get( + operations_deploy_sequence[op_index_max])) + else: + last_deploy = None + # save this dependency chain in the set of HOT resources self.group_dependencies.update(group) for hot in hot_resources: hot.group_dependencies.update(group) - return hot_resources + return hot_resources, deploy_lookup, last_deploy def handle_connectsto(self, tosca_source, tosca_target, hot_source, hot_target, config_location, operation): @@ -313,14 +350,14 @@ class HotResource(object): return tosca_props @staticmethod - def _get_all_operations(node): + def get_all_operations(node): operations = {} for operation in node.interfaces: operations[operation.name] = operation node_type = node.type_definition if isinstance(node_type, str) or \ - node_type.type == "tosca.policies.Placement"or \ + node_type.type == "tosca.policies.Placement" or \ node_type.type == "tosca.policies.Colocate" or \ node_type.type == "tosca.policies.Antilocate": return operations diff --git a/tosca2heat/heat-translator/translator/hot/syntax/hot_template.py b/tosca2heat/heat-translator/translator/hot/syntax/hot_template.py index 4263c4d..0403562 100644 --- a/tosca2heat/heat-translator/translator/hot/syntax/hot_template.py +++ b/tosca2heat/heat-translator/translator/hot/syntax/hot_template.py @@ -77,6 +77,7 @@ class HotTemplate(object): dict_output.update({self.OUTPUTS: all_outputs}) yaml.add_representer(OrderedDict, self.represent_ordereddict) + yaml.add_representer(dict, self.represent_ordereddict) yaml_string = yaml.dump(dict_output, default_flow_style=False) # get rid of the '' from yaml.dump around numbers yaml_string = yaml_string.replace('\'', '') diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py b/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py index e0cdbb6..d42cdc8 100644 --- a/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py +++ b/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py @@ -16,7 +16,6 @@ from mock import patch from toscaparser.nodetemplate import NodeTemplate from toscaparser.tests.base import TestCase -from toscaparser.utils.gettextutils import _ import toscaparser.utils.yamlparser from translator.hot.tosca.tosca_compute import ToscaCompute @@ -27,22 +26,12 @@ class ToscaComputeTest(TestCase): nodetemplates = (toscaparser.utils.yamlparser. simple_parse(tpl_snippet)['node_templates']) name = list(nodetemplates.keys())[0] - try: - nodetemplate = NodeTemplate(name, nodetemplates) - nodetemplate.validate() - toscacompute = ToscaCompute(nodetemplate) - toscacompute.handle_properties() - if not self._compare_properties(toscacompute.properties, - expectedprops): - raise Exception(_("Hot Properties are not" - " same as expected properties")) - except Exception: - # for time being rethrowing. Will be handled future based - # on new development in Glance and Graffiti - raise + nodetemplate = NodeTemplate(name, nodetemplates) + nodetemplate.validate() + toscacompute = ToscaCompute(nodetemplate) + toscacompute.handle_properties() - def _compare_properties(self, hotprops, expectedprops): - return all(item in hotprops.items() for item in expectedprops.items()) + self.assertDictEqual(expectedprops, toscacompute.properties) def test_node_compute_with_host_and_os_capabilities(self): tpl_snippet = ''' @@ -63,7 +52,8 @@ class ToscaComputeTest(TestCase): version: 18.0 ''' expectedprops = {'flavor': 'm1.large', - 'image': 'fedora-amd64-heat-config'} + 'image': 'fedora-amd64-heat-config', + 'user_data_format': 'SOFTWARE_CONFIG'} self._tosca_compute_test( tpl_snippet, expectedprops) @@ -82,7 +72,8 @@ class ToscaComputeTest(TestCase): #left intentionally ''' expectedprops = {'flavor': 'm1.large', - 'image': None} + 'image': None, + 'user_data_format': 'SOFTWARE_CONFIG'} self._tosca_compute_test( tpl_snippet, expectedprops) @@ -101,7 +92,8 @@ class ToscaComputeTest(TestCase): version: 18.0 ''' expectedprops = {'flavor': None, - 'image': 'fedora-amd64-heat-config'} + 'image': 'fedora-amd64-heat-config', + 'user_data_format': 'SOFTWARE_CONFIG'} self._tosca_compute_test( tpl_snippet, expectedprops) @@ -117,7 +109,8 @@ class ToscaComputeTest(TestCase): #left intentionally ''' expectedprops = {'flavor': None, - 'image': None} + 'image': None, + 'user_data_format': 'SOFTWARE_CONFIG'} self._tosca_compute_test( tpl_snippet, expectedprops) @@ -129,7 +122,8 @@ class ToscaComputeTest(TestCase): type: tosca.nodes.Compute ''' expectedprops = {'flavor': None, - 'image': None} + 'image': None, + 'user_data_format': 'SOFTWARE_CONFIG'} self._tosca_compute_test( tpl_snippet, expectedprops) @@ -144,7 +138,9 @@ class ToscaComputeTest(TestCase): properties: #left intentionally ''' - expectedprops = {'flavor': 'm1.nano'} + expectedprops = {'flavor': None, + 'image': None, + 'user_data_format': 'SOFTWARE_CONFIG'} self._tosca_compute_test( tpl_snippet, expectedprops) @@ -160,7 +156,9 @@ class ToscaComputeTest(TestCase): num_cpus: 4 mem_size: 4 GB ''' - expectedprops = {'flavor': 'm1.large'} + expectedprops = {'flavor': 'm1.large', + 'image': None, + 'user_data_format': 'SOFTWARE_CONFIG'} self._tosca_compute_test( tpl_snippet, expectedprops) @@ -176,7 +174,9 @@ class ToscaComputeTest(TestCase): num_cpus: 4 disk_size: 10 GB ''' - expectedprops = {'flavor': 'm1.large'} + expectedprops = {'flavor': 'm1.large', + 'image': None, + 'user_data_format': 'SOFTWARE_CONFIG'} self._tosca_compute_test( tpl_snippet, expectedprops) @@ -191,7 +191,9 @@ class ToscaComputeTest(TestCase): properties: num_cpus: 4 ''' - expectedprops = {'flavor': 'm1.large'} + expectedprops = {'flavor': 'm1.large', + 'image': None, + 'user_data_format': 'SOFTWARE_CONFIG'} self._tosca_compute_test( tpl_snippet, expectedprops) @@ -249,7 +251,9 @@ class ToscaComputeTest(TestCase): json.dumps(mock_flavor_content) mock_post.return_value = mock_ks_response mock_get.return_value = mock_nova_response - expectedprops = {'flavor': 'm1.mock_flavor'} + expectedprops = {'flavor': 'm1.mock_flavor', + 'image': None, + 'user_data_format': 'SOFTWARE_CONFIG'} self._tosca_compute_test( tpl_snippet, expectedprops) @@ -279,8 +283,8 @@ class ToscaComputeTest(TestCase): mock_ks_content = {} mock_ks_response.content = json.dumps(mock_ks_content) expectedprops = {'flavor': 'm1.small', - 'user_data_format': 'SOFTWARE_CONFIG', - 'image': None} + 'image': None, + 'user_data_format': 'SOFTWARE_CONFIG'} self._tosca_compute_test( tpl_snippet, expectedprops) diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py index d4b2f44..924ff9d 100644 --- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py +++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py @@ -67,5 +67,5 @@ class ToscaBlockStorage(HotResource): # attribute for the matching resource. Unless there is additional # runtime support, this should be a one to one mapping. if attribute == 'volume_id': - attr['get_resource'] = args[0] + attr['get_resource'] = self.name return attr diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py index 715d5b3..2242c2d 100644 --- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py +++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py @@ -44,5 +44,10 @@ class ToscaBlockStorageAttachment(HotResource): if 'location' in self.properties: self.properties['mountpoint'] = self.properties.pop('location') + # TOSCA type can have a device name specified, + # this is unsupported by Heat + if 'device' in self.properties: + self.properties.pop('device') + def handle_life_cycle(self): - pass + return None, None, None diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py index e2ac130..b8ad83c 100644 --- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py +++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py @@ -84,6 +84,14 @@ class ToscaCompute(HotResource): ('architecture', 'distribution', 'type', 'version') toscatype = 'tosca.nodes.Compute' + ALLOWED_NOVA_SERVER_PROPS = \ + ('admin_pass', 'availability_zone', 'block_device_mapping', + 'block_device_mapping_v2', 'config_drive', 'diskConfig', 'flavor', + 'flavor_update_policy', 'image', 'image_update_policy', 'key_name', + 'metadata', 'name', 'networks', 'personality', 'reservation_id', + 'scheduler_hints', 'security_groups', 'software_config_transport', + 'user_data', 'user_data_format', 'user_data_update_policy') + def __init__(self, nodetemplate): super(ToscaCompute, self).__init__(nodetemplate, type='OS::Nova::Server') @@ -98,7 +106,8 @@ class ToscaCompute(HotResource): self.properties['user_data_format'] = 'SOFTWARE_CONFIG' tosca_props = self.get_tosca_props() for key, value in tosca_props.items(): - self.properties[key] = value + if key in self.ALLOWED_NOVA_SERVER_PROPS: + self.properties[key] = value # To be reorganized later based on new development in Glance and Graffiti def translate_compute_flavor_and_image(self, @@ -112,11 +121,17 @@ class ToscaCompute(HotResource): if host_capability: for prop in host_capability.get_properties_objects(): host_cap_props[prop.name] = prop.value - flavor = self._best_flavor(host_cap_props) + # if HOST properties are not specified, we should not attempt to + # find best match of flavor + if host_cap_props: + flavor = self._best_flavor(host_cap_props) if os_capability: for prop in os_capability.get_properties_objects(): os_cap_props[prop.name] = prop.value - image = self._best_image(os_cap_props) + # if OS properties are not specified, we should not attempt to + # find best match of image + if os_cap_props: + image = self._best_image(os_cap_props) hot_properties['flavor'] = flavor hot_properties['image'] = image return hot_properties @@ -153,6 +168,48 @@ class ToscaCompute(HotResource): return None return flavor_dict + def _populate_image_dict(self): + '''Populates and returns the images dict using Glance ReST API''' + images_dict = {} + try: + access_dict = translator.common.utils.get_ks_access_dict() + access_token = translator.common.utils.get_token_id(access_dict) + if access_token is None: + return None + glance_url = translator.common.utils.get_url_for(access_dict, + 'image') + if not glance_url: + return None + glance_response = requests.get(glance_url + '/v2/images', + headers={'X-Auth-Token': + access_token}) + if glance_response.status_code != 200: + return None + images = json.loads(glance_response.content)["images"] + for image in images: + image_resp = requests.get(glance_url + '/v2/images/' + + image["id"], + headers={'X-Auth-Token': + access_token}) + if image_resp.status_code != 200: + continue + metadata = ["architecture", "type", "distribution", "version"] + image_data = json.loads(image_resp.content) + if any(key in image_data.keys() for key in metadata): + images_dict[image_data["name"]] = dict() + for key in metadata: + if key in image_data.keys(): + images_dict[image_data["name"]][key] = \ + image_data[key] + else: + continue + + except Exception as e: + # Handles any exception coming from openstack + log.warn(_('Choosing predefined flavors since received ' + 'Openstack Exception: %s') % str(e)) + return images_dict + def _best_flavor(self, properties): log.info(_('Choosing the best flavor for given attributes.')) # Check whether user exported all required environment variables. @@ -202,26 +259,32 @@ class ToscaCompute(HotResource): return None def _best_image(self, properties): - match_all = IMAGES.keys() + # Check whether user exported all required environment variables. + images = IMAGES + if translator.common.utils.check_for_env_variables(): + resp = self._populate_image_dict() + if len(resp.keys()) > 0: + images = resp + match_all = images.keys() architecture = properties.get(self.ARCHITECTURE) if architecture is None: self._log_compute_msg(self.ARCHITECTURE, 'image') - match_arch = self._match_images(match_all, IMAGES, + match_arch = self._match_images(match_all, images, self.ARCHITECTURE, architecture) type = properties.get(self.TYPE) if type is None: self._log_compute_msg(self.TYPE, 'image') - match_type = self._match_images(match_arch, IMAGES, self.TYPE, type) + match_type = self._match_images(match_arch, images, self.TYPE, type) distribution = properties.get(self.DISTRIBUTION) if distribution is None: self._log_compute_msg(self.DISTRIBUTION, 'image') - match_distribution = self._match_images(match_type, IMAGES, + match_distribution = self._match_images(match_type, images, self.DISTRIBUTION, distribution) version = properties.get(self.VERSION) if version is None: self._log_compute_msg(self.VERSION, 'image') - match_version = self._match_images(match_distribution, IMAGES, + match_version = self._match_images(match_distribution, images, self.VERSION, version) if len(match_version): diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_antilocate.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_antilocate.py index b7aca4a..37bfed6 100644 --- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_antilocate.py +++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_antilocate.py @@ -1,4 +1,3 @@ -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_colocate.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_colocate.py index 1a108bf..778f97e 100644 --- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_colocate.py +++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_colocate.py @@ -1,4 +1,3 @@ -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at diff --git a/tosca2heat/heat-translator/translator/hot/translate_node_templates.py b/tosca2heat/heat-translator/translator/hot/translate_node_templates.py index 46cdd71..d8e7e48 100644 --- a/tosca2heat/heat-translator/translator/hot/translate_node_templates.py +++ b/tosca2heat/heat-translator/translator/hot/translate_node_templates.py @@ -16,6 +16,8 @@ import logging import os import six +from collections import OrderedDict +from toscaparser.functions import Concat from toscaparser.functions import GetAttribute from toscaparser.functions import GetInput from toscaparser.functions import GetProperty @@ -25,6 +27,7 @@ from toscaparser.utils.gettextutils import _ from translator.common.exception import ToscaClassAttributeError from translator.common.exception import ToscaClassImportError from translator.common.exception import ToscaModImportError +from translator.common import utils from translator.conf.config import ConfigProvider as translatorConfig from translator.hot.syntax.hot_resource import HotResource from translator.hot.tosca.tosca_block_storage_attachment import ( @@ -132,6 +135,8 @@ log = logging.getLogger('heat-translator') TOSCA_TO_HOT_TYPE = _generate_type_map() +BASE_TYPES = six.string_types + six.integer_types + (dict, OrderedDict) + class TranslateNodeTemplates(object): '''Translate TOSCA NodeTemplates to Heat Resources.''' @@ -146,6 +151,9 @@ class TranslateNodeTemplates(object): log.debug(_('Mapping between TOSCA nodetemplate and HOT resource.')) self.hot_lookup = {} self.policies = self.tosca.topology_template.policies + # stores the last deploy of generated behavior for a resource + # useful to satisfy underlying dependencies between interfaces + self.last_deploy_map = {} def translate(self): return self._translate_nodetemplates() @@ -219,9 +227,14 @@ class TranslateNodeTemplates(object): # into multiple HOT resources and may change their name lifecycle_resources = [] for resource in self.hot_resources: - expanded = resource.handle_life_cycle() - if expanded: - lifecycle_resources += expanded + expanded_resources, deploy_lookup, last_deploy = resource.\ + handle_life_cycle() + if expanded_resources: + lifecycle_resources += expanded_resources + if deploy_lookup: + self.hot_lookup.update(deploy_lookup) + if last_deploy: + self.last_deploy_map[resource] = last_deploy self.hot_resources += lifecycle_resources # Handle configuration from ConnectsTo relationship in the TOSCA node: @@ -253,7 +266,9 @@ class TranslateNodeTemplates(object): # if the source of dependency is a server and the # relationship type is 'tosca.relationships.HostedOn', # add dependency as properties.server - if node_depend.type == 'tosca.nodes.Compute' and \ + base_type = HotResource.get_base_type( + node_depend.type_definition) + if base_type.type == 'tosca.nodes.Compute' and \ node.related[node_depend].type == \ node.type_definition.HOSTEDON: self.hot_lookup[node].properties['server'] = \ @@ -266,6 +281,13 @@ class TranslateNodeTemplates(object): self.hot_lookup[node].depends_on_nodes.append( self.hot_lookup[node_depend].top_of_chain()) + last_deploy = self.last_deploy_map.get( + self.hot_lookup[node_depend]) + if last_deploy and \ + last_deploy not in self.hot_lookup[node].depends_on: + self.hot_lookup[node].depends_on.append(last_deploy) + self.hot_lookup[node].depends_on_nodes.append(last_deploy) + # handle hosting relationship for resource in self.hot_resources: resource.handle_hosting() @@ -295,53 +317,202 @@ class TranslateNodeTemplates(object): inputs = resource.properties.get('input_values') if inputs: for name, value in six.iteritems(inputs): - inputs[name] = self._translate_input(value, resource) + inputs[name] = self.translate_param_value(value, resource) + + # remove resources without type defined + # for example a SoftwareComponent without interfaces + # would fall in this case + to_remove = [] + for resource in self.hot_resources: + if resource.type is None: + to_remove.append(resource) + + for resource in to_remove: + self.hot_resources.remove(resource) return self.hot_resources - def _translate_input(self, input_value, resource): + def translate_param_value(self, param_value, resource): + tosca_template = None + if resource: + tosca_template = resource.nodetemplate + get_property_args = None - if isinstance(input_value, GetProperty): - get_property_args = input_value.args + if isinstance(param_value, GetProperty): + get_property_args = param_value.args # to remove when the parser is fixed to return GetProperty - if isinstance(input_value, dict) and 'get_property' in input_value: - get_property_args = input_value['get_property'] + elif isinstance(param_value, dict) and 'get_property' in param_value: + get_property_args = param_value['get_property'] if get_property_args is not None: - hot_target = self._find_hot_resource_for_tosca( - get_property_args[0], resource) - if hot_target: - props = hot_target.get_tosca_props() - prop_name = get_property_args[1] - if prop_name in props: - return props[prop_name] - elif isinstance(input_value, GetAttribute): + tosca_target, prop_name, prop_arg = \ + self.decipher_get_operation(get_property_args, + tosca_template) + if tosca_target: + prop_value = tosca_target.get_property_value(prop_name) + if prop_value: + prop_value = self.translate_param_value( + prop_value, resource) + return self._unfold_value(prop_value, prop_arg) + get_attr_args = None + if isinstance(param_value, GetAttribute): + get_attr_args = param_value.result().args + # to remove when the parser is fixed to return GetAttribute + elif isinstance(param_value, dict) and 'get_attribute' in param_value: + get_attr_args = param_value['get_attribute'] + if get_attr_args is not None: # for the attribute # get the proper target type to perform the translation - args = input_value.result() - hot_target = self._find_hot_resource_for_tosca(args[0], resource) - - return hot_target.get_hot_attribute(args[1], args) - # most of artifacts logic should move to the parser - elif isinstance(input_value, dict) and 'get_artifact' in input_value: - get_artifact_args = input_value['get_artifact'] - - hot_target = self._find_hot_resource_for_tosca( - get_artifact_args[0], resource) - artifacts = TranslateNodeTemplates.get_all_artifacts( - hot_target.nodetemplate) - - if get_artifact_args[1] in artifacts: - artifact = artifacts[get_artifact_args[1]] - if artifact.get('type', None) == 'tosca.artifacts.File': - return {'get_file': artifact.get('file')} - elif isinstance(input_value, GetInput): - if isinstance(input_value.args, list) \ - and len(input_value.args) == 1: - return {'get_param': input_value.args[0]} + tosca_target, attr_name, attr_arg = \ + self.decipher_get_operation(get_attr_args, tosca_template) + attr_args = [] + if attr_arg: + attr_args += attr_arg + if tosca_target: + if tosca_target in self.hot_lookup: + attr_value = self.hot_lookup[tosca_target].\ + get_hot_attribute(attr_name, attr_args) + attr_value = self.translate_param_value( + attr_value, resource) + return self._unfold_value(attr_value, attr_arg) + elif isinstance(param_value, dict) and 'get_artifact' in param_value: + get_artifact_args = param_value['get_artifact'] + tosca_target, artifact_name, _ = \ + self.decipher_get_operation(get_artifact_args, + tosca_template) + + if tosca_target: + artifacts = self.get_all_artifacts(tosca_target) + if artifact_name in artifacts: + artifact = artifacts[artifact_name] + if artifact.get('type', None) == 'tosca.artifacts.File': + return {'get_file': artifact.get('file')} + get_input_args = None + if isinstance(param_value, GetInput): + get_input_args = param_value.args + elif isinstance(param_value, dict) and 'get_input' in param_value: + get_input_args = param_value['get_input'] + if get_input_args is not None: + if isinstance(get_input_args, list) \ + and len(get_input_args) == 1: + return {'get_param': self.translate_param_value( + get_input_args[0], resource)} + else: + return {'get_param': self.translate_param_value( + get_input_args, resource)} + elif isinstance(param_value, dict) \ + and 'get_operation_output' in param_value: + res = self._translate_get_operation_output_function( + param_value['get_operation_output'], tosca_template) + if res: + return res + concat_list = None + if isinstance(param_value, Concat): + concat_list = param_value.args + elif isinstance(param_value, dict) and 'concat' in param_value: + concat_list = param_value['concat'] + if concat_list is not None: + res = self._translate_concat_function(concat_list, resource) + if res: + return res + + if isinstance(param_value, list): + translated_list = [] + for elem in param_value: + translated_elem = self.translate_param_value(elem, resource) + if translated_elem: + translated_list.append(translated_elem) + return translated_list + + if isinstance(param_value, BASE_TYPES): + return param_value + + return None + + def _translate_concat_function(self, concat_list, resource): + str_replace_template = '' + str_replace_params = {} + index = 0 + for elem in concat_list: + str_replace_template += '$s' + str(index) + str_replace_params['$s' + str(index)] = \ + self.translate_param_value(elem, resource) + index += 1 + + return {'str_replace': { + 'template': str_replace_template, + 'params': str_replace_params + }} + + def _translate_get_operation_output_function(self, args, tosca_template): + tosca_target = self._find_tosca_node(args[0], + tosca_template) + if tosca_target and len(args) >= 4: + operations = HotResource.get_all_operations(tosca_target) + # ignore Standard interface name, + # it is the only one supported in the translator anyway + op_name = args[2] + output_name = args[3] + if op_name in operations: + operation = operations[op_name] + if operation in self.hot_lookup: + matching_deploy = self.hot_lookup[operation] + matching_config_name = matching_deploy.\ + properties['config']['get_resource'] + matching_config = self.find_hot_resource( + matching_config_name) + if matching_config: + outputs = matching_config.properties.get('outputs') + if outputs is None: + outputs = [] + outputs.append({'name': output_name}) + matching_config.properties['outputs'] = outputs + return {'get_attr': [ + matching_deploy.name, + output_name + ]} + + @staticmethod + def _unfold_value(value, value_arg): + if value_arg is not None: + if isinstance(value, dict): + val = value.get(value_arg) + if val is not None: + return val + + index = utils.str_to_num(value_arg) + if isinstance(value, list) and index is not None: + return value[index] + return value + + def decipher_get_operation(self, args, current_tosca_node): + tosca_target = self._find_tosca_node(args[0], + current_tosca_node) + new_target = None + if tosca_target and len(args) > 2: + cap_or_req_name = args[1] + cap = tosca_target.get_capability(cap_or_req_name) + if cap: + new_target = cap else: - return {'get_param': input_value.args} + for req in tosca_target.requirements: + if cap_or_req_name in req: + new_target = self._find_tosca_node( + req[cap_or_req_name]) + cap = new_target.get_capability(cap_or_req_name) + if cap: + new_target = cap + break + + if new_target: + tosca_target = new_target + + prop_name = args[2] + prop_arg = args[3] if len(args) >= 4 else None + else: + prop_name = args[1] + prop_arg = args[2] if len(args) >= 3 else None - return input_value + return tosca_target, prop_name, prop_arg @staticmethod def get_all_artifacts(nodetemplate): @@ -410,23 +581,29 @@ class TranslateNodeTemplates(object): if resource.name == name: return resource - def _find_tosca_node(self, tosca_name): - for node in self.nodetemplates: - if node.name == tosca_name: - return node - - def _find_hot_resource_for_tosca(self, tosca_name, - current_hot_resource=None): + def _find_tosca_node(self, tosca_name, current_tosca_template=None): + tosca_node = None if tosca_name == 'SELF': - return current_hot_resource - if tosca_name == 'HOST' and current_hot_resource is not None: - for req in current_hot_resource.nodetemplate.requirements: + tosca_node = current_tosca_template + if tosca_name == 'HOST' and current_tosca_template: + for req in current_tosca_template.requirements: if 'host' in req: - return self._find_hot_resource_for_tosca(req['host']) + tosca_node = self._find_tosca_node(req['host']) - for node in self.nodetemplates: - if node.name == tosca_name: - return self.hot_lookup[node] + if tosca_node is None: + for node in self.nodetemplates: + if node.name == tosca_name: + tosca_node = node + break + return tosca_node + + def _find_hot_resource_for_tosca(self, tosca_name, + current_hot_resource=None): + current_tosca_resource = current_hot_resource.nodetemplate \ + if current_hot_resource else None + tosca_node = self._find_tosca_node(tosca_name, current_tosca_resource) + if tosca_node: + return self.hot_lookup[tosca_node] return None diff --git a/tosca2heat/heat-translator/translator/hot/translate_outputs.py b/tosca2heat/heat-translator/translator/hot/translate_outputs.py index 4197cdd..87ec02a 100644 --- a/tosca2heat/heat-translator/translator/hot/translate_outputs.py +++ b/tosca2heat/heat-translator/translator/hot/translate_outputs.py @@ -33,16 +33,8 @@ class TranslateOutputs(object): def _translate_outputs(self): hot_outputs = [] for output in self.outputs: - if output.value.name == 'get_attribute': - get_parameters = output.value.args - hot_target = self.nodes.find_hot_resource(get_parameters[0]) - hot_value = hot_target.get_hot_attribute(get_parameters[1], - get_parameters) - hot_outputs.append(HotOutput(output.name, - hot_value, - output.description)) - else: - hot_outputs.append(HotOutput(output.name, - output.value, + hot_value = self.nodes.translate_param_value(output.value, None) + if hot_value is not None: + hot_outputs.append(HotOutput(output.name, hot_value, output.description)) return hot_outputs diff --git a/tosca2heat/heat-translator/translator/osc/v1/translate.py b/tosca2heat/heat-translator/translator/osc/v1/translate.py index eeaaa18..ef005e2 100644 --- a/tosca2heat/heat-translator/translator/osc/v1/translate.py +++ b/tosca2heat/heat-translator/translator/osc/v1/translate.py @@ -22,11 +22,12 @@ from cliff import command from toscaparser.tosca_template import ToscaTemplate from toscaparser.utils.gettextutils import _ from translator.common.utils import UrlUtils +from translator.conf.config import ConfigProvider from translator.hot.tosca_translator import TOSCATranslator from translator.osc import utils - -logging.config.fileConfig('heat_translator_logging.conf') +conf_file = ConfigProvider.get_translator_logging_file() +logging.config.fileConfig(conf_file) log = logging.getLogger('heat-translator') diff --git a/tosca2heat/heat-translator/translator/shell.py b/tosca2heat/heat-translator/translator/shell.py index 92d92d9..d5333bc 100644 --- a/tosca2heat/heat-translator/translator/shell.py +++ b/tosca2heat/heat-translator/translator/shell.py @@ -11,6 +11,7 @@ # under the License. +import argparse import ast import json import logging @@ -26,6 +27,7 @@ from toscaparser.tosca_template import ToscaTemplate from toscaparser.utils.gettextutils import _ from toscaparser.utils.urlutils import UrlUtils from translator.common import utils +from translator.conf.config import ConfigProvider from translator.hot.tosca_translator import TOSCATranslator """ @@ -44,8 +46,8 @@ without actual translation, pass --validate-only=true along with other required arguments. """ - -logging.config.fileConfig('heat_translator_logging.conf') +conf_file = ConfigProvider.get_translator_logging_file() +logging.config.fileConfig(conf_file) log = logging.getLogger("heat-translator") @@ -53,78 +55,72 @@ class TranslatorShell(object): SUPPORTED_TYPES = ['tosca'] - def _validate(self, args): - if len(args) < 2: - msg = _("The program requires minimum two arguments. " - "Please refer to the usage documentation.") - log.error(msg) - raise ValueError(msg) - if "--template-file=" not in args[0]: - msg = _("The program expects --template-file as first argument. " - "Please refer to the usage documentation.") - log.error(msg) - raise ValueError(msg) - if "--template-type=" not in args[1]: - msg = _("The program expects --template-type as second argument. " - "Please refer to the usage documentation.") - log.error(msg) - raise ValueError(msg) + def get_parser(self): + parser = argparse.ArgumentParser(prog="heat-translator") + + parser.add_argument('--template-file', + metavar='<filename>', + required=True, + help=_('Template file to load.')) + + parser.add_argument('--output-file', + metavar='<filename>', + help=_('Where to store the output file. If not ' + 'passed, it will be printed to stdin.')) + + parser.add_argument('--template-type', + metavar='<input-template-type>', + choices=self.SUPPORTED_TYPES, + default='tosca', + help=(_('Template type to parse. Choose between ' + '%s.') % self.SUPPORTED_TYPES)) + + parser.add_argument('--parameters', + metavar='<param1=val1;param2=val2;...>', + help=_('Optional input parameters.')) + + parser.add_argument('--validate-only', + action='store_true', + default=False, + help=_('Only validate input template, do not ' + 'perform translation.')) + + parser.add_argument('--deploy', + action='store_true', + default=False, + help=_('Whether to deploy the generated template ' + 'or not.')) + + return parser + + def main(self, argv): + + parser = self.get_parser() + (args, args_list) = parser.parse_known_args(argv) + + template_file = args.template_file + template_type = args.template_type + output_file = args.output_file + validate_only = args.validate_only + deploy = args.deploy - def main(self, args): - # TODO(spzala): set self.deploy based on passed args once support for - # --deploy argument is enabled. - self.deploy = False - self._validate(args) - path = args[0].split('--template-file=')[1] - # e.g. --template_file=translator/tests/data/tosca_helloworld.yaml - template_type = args[1].split('--template-type=')[1] - # e.g. --template_type=tosca - if not template_type: - msg = _("Template type is needed. For example, 'tosca'") - log.error(msg) - raise ValueError(msg) - elif template_type not in self.SUPPORTED_TYPES: - msg = _("%(value)s is not a valid template type.") % { - 'value': template_type} - log.error(msg) - raise ValueError(msg) parsed_params = {} - validate_only = None - output_file = None - if len(args) > 2: - parameters = None - for arg in args: - if "--validate-only=" in arg: - validate_only = arg - if "--parameters=" in arg: - parameters = arg - if "--output-file=" in arg: - output = arg - output_file = output.split('--output-file=')[1] - if "--deploy" in arg: - self.deploy = True - if parameters: - parsed_params = self._parse_parameters(parameters) - a_file = os.path.isfile(path) - a_url = UrlUtils.validate_url(path) if not a_file else False + if args.parameters: + parsed_params = self._parse_parameters(args.parameters) + + a_file = os.path.isfile(template_file) + a_url = UrlUtils.validate_url(template_file) if not a_file else False if a_file or a_url: - run_only_validation = False if validate_only: - value = validate_only.split('-validate-only=')[1].lower() - if template_type == 'tosca' and value == 'true': - run_only_validation = True - if run_only_validation: - ToscaTemplate(path, parsed_params, a_file) - msg = (_('The input "%(path)s" successfully passed ' - 'validation.') % {'path': path}) + ToscaTemplate(template_file, parsed_params, a_file) + msg = (_('The input "%(template_file)s" successfully passed ' + 'validation.') % {'template_file': template_file}) print(msg) else: - log.info( - _('Checked whether template path is a file or url path.')) - heat_tpl = self._translate(template_type, path, parsed_params, - a_file) + heat_tpl = self._translate(template_type, template_file, + parsed_params, a_file, deploy) if heat_tpl: - if utils.check_for_env_variables() and self.deploy: + if utils.check_for_env_variables() and deploy: try: heatclient(heat_tpl, parsed_params) except Exception: @@ -132,47 +128,42 @@ class TranslatorShell(object): self._write_output(heat_tpl, output_file) else: - msg = _("The path %(path)s is not a valid file or URL.") % { - 'path': path} + msg = (_('The path %(template_file)s is not a valid ' + 'file or URL.') % {'template_file': template_file}) + log.error(msg) raise ValueError(msg) def _parse_parameters(self, parameter_list): parsed_inputs = {} - if parameter_list.startswith('--parameters'): - # Parameters are semi-colon separated - inputs = parameter_list.split('--parameters=')[1].\ - replace('"', '').split(';') - # Each parameter should be an assignment - for param in inputs: - keyvalue = param.split('=') - # Validate the parameter has both a name and value - msg = _("'%(param)s' is not a well-formed parameter.") % { - 'param': param} - if keyvalue.__len__() is 2: - # Assure parameter name is not zero-length or whitespace - stripped_name = keyvalue[0].strip() - if not stripped_name: - log.error(msg) - raise ValueError(msg) - # Add the valid parameter to the dictionary - parsed_inputs[keyvalue[0]] = keyvalue[1] - else: + + # Parameters are semi-colon separated + inputs = parameter_list.replace('"', '').split(';') + # Each parameter should be an assignment + for param in inputs: + keyvalue = param.split('=') + # Validate the parameter has both a name and value + msg = _("'%(param)s' is not a well-formed parameter.") % { + 'param': param} + if keyvalue.__len__() is 2: + # Assure parameter name is not zero-length or whitespace + stripped_name = keyvalue[0].strip() + if not stripped_name: log.error(msg) raise ValueError(msg) - else: - msg = _("'%(list)s' is not a valid parameter list.") % { - 'list': parameter_list} - log.error(msg) - raise ValueError(msg) + # Add the valid parameter to the dictionary + parsed_inputs[keyvalue[0]] = keyvalue[1] + else: + log.error(msg) + raise ValueError(msg) return parsed_inputs - def _translate(self, sourcetype, path, parsed_params, a_file): + def _translate(self, sourcetype, path, parsed_params, a_file, deploy): output = None if sourcetype == "tosca": log.debug(_('Loading the tosca template.')) tosca = ToscaTemplate(path, parsed_params, a_file) - translator = TOSCATranslator(tosca, parsed_params, self.deploy) + translator = TOSCATranslator(tosca, parsed_params, deploy) log.debug(_('Translating the tosca template.')) output = translator.translate() return output diff --git a/tosca2heat/heat-translator/translator/tests/data/artifacts/ssh/ssh_generate_keys.sh b/tosca2heat/heat-translator/translator/tests/data/artifacts/ssh/ssh_generate_keys.sh new file mode 100644 index 0000000..9dc8e16 --- /dev/null +++ b/tosca2heat/heat-translator/translator/tests/data/artifacts/ssh/ssh_generate_keys.sh @@ -0,0 +1,4 @@ +#!/bin/bash +test -d /root/.ssh || mkdir /root/.ssh +test -f /root/.ssh/id_rsa.pub || ssh-keygen -q -t rsa -N "" -f /root/.ssh/id_rsa +cat /root/.ssh/id_rsa.pub > ${heat_outputs_path}.public_key diff --git a/tosca2heat/heat-translator/translator/tests/data/artifacts/ssh/ssh_import_public_key.sh b/tosca2heat/heat-translator/translator/tests/data/artifacts/ssh/ssh_import_public_key.sh new file mode 100644 index 0000000..1ef94e0 --- /dev/null +++ b/tosca2heat/heat-translator/translator/tests/data/artifacts/ssh/ssh_import_public_key.sh @@ -0,0 +1,3 @@ +#!/bin/bash +test -d /root/.ssh || mkdir /root/.ssh +echo "$public_key" >> /root/.ssh/authorized_keys diff --git a/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_exchange_public_ssh_key.yaml b/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_exchange_public_ssh_key.yaml new file mode 100644 index 0000000..1dfa125 --- /dev/null +++ b/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_exchange_public_ssh_key.yaml @@ -0,0 +1,55 @@ +heat_template_version: 2013-05-23 + +description: > + TOSCA template to test get_operation_output by exchanging ssh public key + +parameters: {} +resources: + generate_ssh_key_create_deploy: + type: OS::Heat::SoftwareDeployment + properties: + config: + get_resource: generate_ssh_key_create_config + server: + get_resource: server1 + import_public_key_create_deploy: + type: OS::Heat::SoftwareDeployment + properties: + config: + get_resource: import_public_key_create_config + input_values: + public_key: + get_attr: + - generate_ssh_key_create_deploy + - public_key + server: + get_resource: server2 + depends_on: + - generate_ssh_key_create_deploy + server1: + type: OS::Nova::Server + properties: + flavor: m1.small + image: ubuntu-12.04-software-config-os-init + user_data_format: SOFTWARE_CONFIG + server2: + type: OS::Nova::Server + properties: + flavor: m1.small + image: ubuntu-12.04-software-config-os-init + user_data_format: SOFTWARE_CONFIG + generate_ssh_key_create_config: + type: OS::Heat::SoftwareConfig + properties: + config: + get_file: artifacts/ssh/ssh_generate_keys.sh + group: script + outputs: + - name: public_key + import_public_key_create_config: + type: OS::Heat::SoftwareConfig + properties: + config: + get_file: artifacts/ssh/ssh_import_public_key.sh + group: script +outputs: {}
\ No newline at end of file diff --git a/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_get_functions_semantic.yaml b/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_get_functions_semantic.yaml new file mode 100644 index 0000000..318a739 --- /dev/null +++ b/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_get_functions_semantic.yaml @@ -0,0 +1,52 @@ +heat_template_version: 2013-05-23 + +description: > + TOSCA template to test get_* functions semantic + +parameters: + map_val: + type: string +resources: + myapp_configure_deploy: + type: OS::Heat::SoftwareDeployment + properties: + input_values: + list_val: list_val_0 + config: + get_resource: myapp_configure_config + server: + get_resource: server + depends_on: + - mysql_database + server: + type: OS::Nova::Server + properties: + flavor: m1.small + image: ubuntu-12.04-software-config-os-init + user_data_format: SOFTWARE_CONFIG + myapp_configure_config: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: + get_file: myapp_configure.sh +outputs: + map_val: + description: map_val + value: + get_input: map_val + concat_map_val: + value: + str_replace: + params: + $s2: :8080 + $s0: http:// + $s1: + get_input: map_val + template: $s0$s1$s2 + static_map_val: + value: static_value + test_list_of_functions: + value: + - get_input: map_val + - static_value diff --git a/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_interface_on_compute.yaml b/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_interface_on_compute.yaml new file mode 100644 index 0000000..0399c06 --- /dev/null +++ b/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_interface_on_compute.yaml @@ -0,0 +1,44 @@ +heat_template_version: 2013-05-23 + +description: > + TOSCA template to test Compute node with interface + +parameters: {} +resources: + softwarecomponent_depending_on_customcompute_install_create_deploy: + type: OS::Heat::SoftwareDeployment + properties: + config: + get_resource: softwarecomponent_depending_on_customcompute_install_create_config + server: + get_resource: server + depends_on: + - server_create_deploy + server: + type: OS::Nova::Server + properties: + flavor: m1.small + image: ubuntu-12.04-software-config-os-init + user_data_format: SOFTWARE_CONFIG + softwarecomponent_depending_on_customcompute_install_create_config: + type: OS::Heat::SoftwareConfig + properties: + config: + get_file: post_install.sh + group: script + server_create_config: + type: OS::Heat::SoftwareConfig + properties: + config: + get_file: install.sh + group: script + server_create_deploy: + type: OS::Heat::SoftwareDeployment + properties: + config: + get_resource: server_create_config + input_values: + install_path: /opt + server: + get_resource: server +outputs: {}
\ No newline at end of file diff --git a/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_script_types.yaml b/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_script_types.yaml new file mode 100644 index 0000000..5f0585d --- /dev/null +++ b/tosca2heat/heat-translator/translator/tests/data/hot_output/hot_script_types.yaml @@ -0,0 +1,101 @@ +heat_template_version: 2013-05-23 + +description: > + TOSCA template to test usage of different script types like Ansible and Puppet + one. + +parameters: {} +resources: + customwebserver2_create_deploy: + type: OS::Heat::SoftwareDeployment + properties: + config: + get_resource: customwebserver2_create_config + server: + get_resource: server + customwebserver_create_deploy: + type: OS::Heat::SoftwareDeployment + properties: + config: + get_resource: customwebserver_create_config + server: + get_resource: server + server: + type: OS::Nova::Server + properties: + flavor: m1.small + image: ubuntu-12.04-software-config-os-init + user_data_format: SOFTWARE_CONFIG + customwebserver2_start_config: + type: OS::Heat::SoftwareConfig + properties: + config: + get_file: start.sh + group: script + customwebserver2_start_deploy: + type: OS::Heat::SoftwareDeployment + properties: + config: + get_resource: customwebserver2_start_config + server: + get_resource: server + depends_on: + - customwebserver2_configure_deploy + customwebserver2_create_config: + type: OS::Heat::SoftwareConfig + properties: + config: + get_file: install.sh + group: script + customwebserver2_configure_config: + type: OS::Heat::SoftwareConfig + properties: + config: + get_file: configure.py + group: script + customwebserver2_configure_deploy: + type: OS::Heat::SoftwareDeployment + properties: + config: + get_resource: customwebserver2_configure_config + server: + get_resource: server + depends_on: + - customwebserver2_create_deploy + customwebserver_start_config: + type: OS::Heat::SoftwareConfig + properties: + config: + get_file: start.pp + group: puppet + customwebserver_start_deploy: + type: OS::Heat::SoftwareDeployment + properties: + config: + get_resource: customwebserver_start_config + server: + get_resource: server + depends_on: + - customwebserver_configure_deploy + customwebserver_create_config: + type: OS::Heat::SoftwareConfig + properties: + config: + get_file: install.yaml + group: ansible + customwebserver_configure_config: + type: OS::Heat::SoftwareConfig + properties: + config: + get_file: configure.yml + group: ansible + customwebserver_configure_deploy: + type: OS::Heat::SoftwareDeployment + properties: + config: + get_resource: customwebserver_configure_config + server: + get_resource: server + depends_on: + - customwebserver_create_deploy +outputs: {} diff --git a/tosca2heat/heat-translator/translator/tests/data/test_tosca_get_functions_semantic.yaml b/tosca2heat/heat-translator/translator/tests/data/test_tosca_get_functions_semantic.yaml new file mode 100644 index 0000000..2a76978 --- /dev/null +++ b/tosca2heat/heat-translator/translator/tests/data/test_tosca_get_functions_semantic.yaml @@ -0,0 +1,84 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: TOSCA template to test get_* functions semantic + +node_types: + tosca.capabilities.MyFeature: + derived_from: tosca.capabilities.Root + properties: + my_list: + type: list + my_map: + type: map + + tosca.nodes.WebApplication.MyApp: + derived_from: tosca.nodes.WebApplication + requirements: + - myfeature: + capability: tosca.capabilities.MyFeature + node: tosca.nodes.MyDatabase + relationship: tosca.relationships.ConnectsTo + + tosca.nodes.MyDatabase: + derived_from: tosca.nodes.Database + capabilities: + myfeature: + type: tosca.capabilities.MyFeature + +topology_template: + inputs: + map_val: + type: string + + node_templates: + server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + num_cpus: 1 + mem_size: 1 GB + os: + properties: + type: Linux + distribution: Ubuntu + version: 12.04 + architecture: x86_64 + + mysql_database: + type: tosca.nodes.MyDatabase + requirements: + - host: server + capabilities: + myfeature: + properties: + my_list: [list_val_0] + my_map: + test_key: { get_input: map_val } + test_key_static: static_value + + myapp: + type: tosca.nodes.WebApplication.MyApp + requirements: + - myfeature: mysql_database + - host: server + interfaces: + Standard: + configure: + implementation: myapp_configure.sh + inputs: + list_val: { get_property: [ SELF, myfeature, my_list, 0 ] } + + outputs: + map_val: + description: map_val + value: { get_property: [ myapp, myfeature, my_map, test_key ] } + + static_map_val: + value: { get_property: [ myapp, myfeature, my_map, test_key_static ] } + + concat_map_val: + value: { concat: [ 'http://', { get_property: [ myapp, myfeature, my_map, test_key ] }, ':8080' ] } + + test_list_of_functions: + value: [ { get_property: [ myapp, myfeature, my_map, test_key ] }, { get_property: [ myapp, myfeature, my_map, test_key_static ] } ] diff --git a/tosca2heat/heat-translator/translator/tests/data/test_tosca_interface_on_compute.yaml b/tosca2heat/heat-translator/translator/tests/data/test_tosca_interface_on_compute.yaml new file mode 100644 index 0000000..e033c3c --- /dev/null +++ b/tosca2heat/heat-translator/translator/tests/data/test_tosca_interface_on_compute.yaml @@ -0,0 +1,48 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: TOSCA template to test Compute node with interface + +node_types: + tosca.nodes.CustomCompute: + derived_from: tosca.nodes.Compute + properties: + install_path: + type: string + default: /opt + interfaces: + Standard: + create: + implementation: install.sh + inputs: + install_path: { get_property: [ SELF, install_path ] } + +topology_template: + node_templates: + + softwarecomponent_without_behavior: + type: tosca.nodes.SoftwareComponent + requirements: + - host: server + + softwarecomponent_depending_on_customcompute_install: + type: tosca.nodes.SoftwareComponent + interfaces: + Standard: + create: + implementation: post_install.sh + requirements: + - host: server + + server: + type: tosca.nodes.CustomCompute + capabilities: + host: + properties: + num_cpus: 1 + mem_size: 1 GB + os: + properties: + type: Linux + distribution: Ubuntu + version: 12.04 + architecture: x86_64 diff --git a/tosca2heat/heat-translator/translator/tests/data/test_tosca_script_types.yaml b/tosca2heat/heat-translator/translator/tests/data/test_tosca_script_types.yaml new file mode 100644 index 0000000..b54cbcb --- /dev/null +++ b/tosca2heat/heat-translator/translator/tests/data/test_tosca_script_types.yaml @@ -0,0 +1,48 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + TOSCA template to test usage of different script types like + Ansible and Puppet one. + +topology_template: + + node_templates: + customwebserver: + type: tosca.nodes.WebServer + requirements: + - host: server + interfaces: + Standard: + create: + implementation: install.yaml + configure: + implementation: configure.yml + start: + implementation: start.pp + + customwebserver2: + type: tosca.nodes.WebServer + requirements: + - host: server + interfaces: + Standard: + create: + implementation: install.sh + configure: + implementation: configure.py + start: + implementation: start.sh + + server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + num_cpus: 1 + mem_size: 1 GB + os: + properties: + type: Linux + distribution: Ubuntu + version: 12.04 + architecture: x86_64 diff --git a/tosca2heat/heat-translator/translator/tests/data/tosca_exchange_public_ssh_key.yaml b/tosca2heat/heat-translator/translator/tests/data/tosca_exchange_public_ssh_key.yaml new file mode 100644 index 0000000..7decb33 --- /dev/null +++ b/tosca2heat/heat-translator/translator/tests/data/tosca_exchange_public_ssh_key.yaml @@ -0,0 +1,54 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: TOSCA template to test get_operation_output by exchanging ssh public key + +topology_template: + + node_templates: + server1: + type: tosca.nodes.Compute + capabilities: + host: + properties: + num_cpus: 1 + mem_size: 1 GB + os: + properties: + type: Linux + distribution: Ubuntu + version: 12.04 + architecture: x86_64 + + server2: + type: tosca.nodes.Compute + capabilities: + host: + properties: + num_cpus: 1 + mem_size: 1 GB + os: + properties: + type: Linux + distribution: Ubuntu + version: 12.04 + architecture: x86_64 + + generate_ssh_key: + type: tosca.nodes.SoftwareComponent + interfaces: + Standard: + create: artifacts/ssh/ssh_generate_keys.sh + requirements: + - host: server1 + + import_public_key: + type: tosca.nodes.SoftwareComponent + interfaces: + Standard: + create: + implementation: artifacts/ssh/ssh_import_public_key.sh + inputs: + public_key: { get_operation_output: [generate_ssh_key, Standard, create, public_key] } + requirements: + - host: server2 + - dependency: generate_ssh_key diff --git a/tosca2heat/heat-translator/translator/tests/test_shell.py b/tosca2heat/heat-translator/translator/tests/test_shell.py index b001c1a..62f3510 100644 --- a/tosca2heat/heat-translator/translator/tests/test_shell.py +++ b/tosca2heat/heat-translator/translator/tests/test_shell.py @@ -29,28 +29,9 @@ class ShellTest(TestCase): "data/tosca_helloworld.yaml") template_file = '--template-file=' + tosca_helloworld template_type = '--template-type=tosca' - template_validation = "--validate-only=true" + template_validation = "--validate-only" failure_msg = _('The program raised an exception unexpectedly.') - def test_missing_arg(self): - error = self.assertRaises(ValueError, shell.main, '') - err_msg = _('The program requires minimum two arguments. ' - 'Please refer to the usage documentation.') - self.assertEqual(err_msg, str(error)) - - def test_invalid_file_arg(self): - error = self.assertRaises(ValueError, shell.main, 'translate me') - err_msg = _('The program expects --template-file as first ' - 'argument. Please refer to the usage documentation.') - self.assertEqual(err_msg, str(error)) - - def test_invalid_type_arg(self): - error = self.assertRaises(ValueError, - shell.main, ('--template-file=', 'xyz')) - err_msg = _('The program expects --template-type as second argument. ' - 'Please refer to the usage documentation.') - self.assertEqual(err_msg, str(error)) - def test_invalid_file_value(self): error = self.assertRaises(ValueError, shell.main, ('--template-file=template.txt', @@ -59,17 +40,13 @@ class ShellTest(TestCase): self.assertEqual(err_msg, str(error)) def test_invalid_type_value(self): - error = self.assertRaises(ValueError, shell.main, - (self.template_file, '--template-type=xyz')) - err_msg = _('xyz is not a valid template type.') - self.assertEqual(err_msg, str(error)) + self.assertRaises(SystemExit, shell.main, + (self.template_file, '--template-type=xyz')) def test_invalid_parameters(self): - error = self.assertRaises(ValueError, shell.main, - (self.template_file, self.template_type, - '--parameters=key')) - err_msg = _("'key' is not a well-formed parameter.") - self.assertEqual(err_msg, str(error)) + self.assertRaises(ValueError, shell.main, + (self.template_file, self.template_type, + '--parameters=key')) def test_valid_template(self): try: @@ -77,6 +54,12 @@ class ShellTest(TestCase): except Exception: self.fail(self.failure_msg) + def test_valid_template_without_type(self): + try: + shell.main([self.template_file]) + except Exception: + self.fail(self.failure_msg) + def test_valid_template_with_parameters(self): tosca_single_instance_wordpress = os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -126,7 +109,10 @@ class ShellTest(TestCase): @patch('os.getenv') @patch('translator.hot.tosca.tosca_compute.' 'ToscaCompute._create_nova_flavor_dict') - def test_template_deploy_with_credentials(self, mock_flavor_dict, + @patch('translator.hot.tosca.tosca_compute.' + 'ToscaCompute._populate_image_dict') + def test_template_deploy_with_credentials(self, mock_populate_image_dict, + mock_flavor_dict, mock_os_getenv, mock_token, mock_url, mock_post, @@ -137,6 +123,14 @@ class ShellTest(TestCase): mock_flavor_dict.return_value = { 'm1.medium': {'mem_size': 4096, 'disk_size': 40, 'num_cpus': 2} } + mock_populate_image_dict.return_value = { + "rhel-6.5-test-image": { + "version": "6.5", + "architecture": "x86_64", + "distribution": "RHEL", + "type": "Linux" + } + } mock_url.return_value = 'http://abc.com' mock_token.return_value = 'mock_token' mock_os_getenv.side_effect = ['demo', 'demo', diff --git a/tosca2heat/heat-translator/translator/tests/test_tosca_hot_translation.py b/tosca2heat/heat-translator/translator/tests/test_tosca_hot_translation.py index e58d842..7a89827 100644 --- a/tosca2heat/heat-translator/translator/tests/test_tosca_hot_translation.py +++ b/tosca2heat/heat-translator/translator/tests/test_tosca_hot_translation.py @@ -23,35 +23,48 @@ from translator.tests.base import TestCase class ToscaHotTranslationTest(TestCase): - def test_hot_translate_single_server(self): - tosca_file = '../tests/data/tosca_single_server.yaml' - hot_file = '../tests/data/hot_output/hot_single_server.yaml' - params = {'cpus': 1} + def _test_successful_translation(self, tosca_file, hot_file, params=None): + if not params: + params = {} diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, hot_file, params) self.assertEqual({}, diff, '<difference> : ' + json.dumps(diff, indent=4, separators=(', ', ': '))) + def _test_failed_translation(self, tosca_file, hot_file, params, msg, + msg_path, error_raise, error_collect): + if msg_path: + path = os.path.normpath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), tosca_file)) + msg = msg % path + self.assertRaises( + error_raise, + TranslationUtils.compare_tosca_translation_with_hot, + tosca_file, hot_file, params) + ExceptionCollector.assertExceptionMessage(error_collect, msg) + + def test_hot_translate_single_server(self): + tosca_file = '../tests/data/tosca_single_server.yaml' + hot_file = '../tests/data/hot_output/hot_single_server.yaml' + params = {'cpus': 1} + self._test_successful_translation(tosca_file, hot_file, params) + def test_hot_translate_single_server_with_defaults(self): tosca_file = \ '../tests/data/tosca_single_server_with_defaults.yaml' + hot_file_with_input = '../tests/data/hot_output/' \ 'hot_single_server_with_defaults_with_input.yaml' - hot_file_without_input = '../tests/data/hot_output/' \ - 'hot_single_server_with_defaults_without_input.yaml' - params1 = {'cpus': '1'} - diff1 = TranslationUtils.compare_tosca_translation_with_hot( - tosca_file, hot_file_with_input, params1) - self.assertEqual({}, diff1, '<difference> : ' + - json.dumps(diff1, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file_with_input, + params1) + hot_file_without_input = '../tests/data/hot_output/' \ + 'hot_single_server_with_defaults_without_input.yaml' params2 = {} - diff2 = TranslationUtils.compare_tosca_translation_with_hot( - tosca_file, hot_file_without_input, params2) - self.assertEqual({}, diff2, '<difference> : ' + - json.dumps(diff2, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file_without_input, + params2) def test_hot_translate_wordpress_single_instance(self): tosca_file = '../tests/data/tosca_single_instance_wordpress.yaml' @@ -63,29 +76,17 @@ class ToscaHotTranslationTest(TestCase): 'db_root_pwd': 'passw0rd', 'db_port': 3366, 'cpus': 8} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_helloworld(self): tosca_file = '../tests/data/tosca_helloworld.yaml' hot_file = '../tests/data/hot_output/hot_hello_world.yaml' - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - {}) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file) def test_hot_translate_host_assignment(self): tosca_file = '../tests/data/test_host_assignment.yaml' hot_file = '../tests/data/hot_output/hot_host_assignment.yaml' - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - {}) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file) def test_hot_translate_elk(self): tosca_file = '../tests/data/tosca_elk.yaml' @@ -93,11 +94,7 @@ class ToscaHotTranslationTest(TestCase): params = {'github_url': 'http://github.com/paypal/rest-api-sample-app-nodejs.git', 'my_cpus': 4} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_nodejs_mongodb_two_instances(self): tosca_file = '../tests/data/tosca_nodejs_mongodb_two_instances.yaml' @@ -106,11 +103,7 @@ class ToscaHotTranslationTest(TestCase): params = {'github_url': 'http://github.com/paypal/rest-api-sample-app-nodejs.git', 'my_cpus': 4} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_blockstorage_with_attachment(self): tosca_file = '../tests/data/storage/' \ @@ -121,11 +114,7 @@ class ToscaHotTranslationTest(TestCase): 'storage_location': '/dev/vdc', 'storage_size': '2000 MB', 'storage_snapshot_id': 'ssid'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_blockstorage_with_custom_relationship_type(self): tosca_file = '../tests/data/storage/' \ @@ -136,11 +125,7 @@ class ToscaHotTranslationTest(TestCase): 'storage_location': '/dev/vdc', 'storage_size': '1 GB', 'storage_snapshot_id': 'ssid'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_blockstorage_with_relationship_template(self): tosca_file = '../tests/data/storage/' \ @@ -150,11 +135,7 @@ class ToscaHotTranslationTest(TestCase): params = {'cpus': 1, 'storage_location': '/dev/vdc', 'storage_size': '1 GB'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_blockstorage_with_attachment_notation1(self): tosca_file = '../tests/data/storage/' \ @@ -167,19 +148,11 @@ class ToscaHotTranslationTest(TestCase): 'storage_location': 'some_folder', 'storage_size': '1 GB', 'storage_snapshot_id': 'ssid'} - diff1 = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file1, - params) + try: - self.assertEqual({}, diff1, '<difference> : ' + - json.dumps(diff1, indent=4, - separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file1, params) except Exception: - diff2 = TranslationUtils.compare_tosca_translation_with_hot( - tosca_file, hot_file2, params) - self.assertEqual({}, diff2, '<difference> : ' + - json.dumps(diff2, indent=4, - separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file2, params) def test_hot_translate_blockstorage_with_attachment_notation2(self): tosca_file = '../tests/data/storage/' \ @@ -192,19 +165,10 @@ class ToscaHotTranslationTest(TestCase): 'storage_location': '/dev/vdc', 'storage_size': '1 GB', 'storage_snapshot_id': 'ssid'} - diff1 = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file1, - params) try: - self.assertEqual({}, diff1, '<difference> : ' + - json.dumps(diff1, indent=4, - separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file1, params) except Exception: - diff2 = TranslationUtils.compare_tosca_translation_with_hot( - tosca_file, hot_file2, params) - self.assertEqual({}, diff2, '<difference> : ' + - json.dumps(diff2, indent=4, - separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file2, params) def test_hot_translate_multiple_blockstorage_with_attachment(self): tosca_file = '../tests/data/storage/' \ @@ -217,40 +181,23 @@ class ToscaHotTranslationTest(TestCase): 'storage_location': '/dev/vdc', 'storage_size': '1 GB', 'storage_snapshot_id': 'ssid'} - diff1 = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file1, - params) try: - self.assertEqual({}, diff1, '<difference> : ' + - json.dumps(diff1, indent=4, - separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file1, params) except Exception: - diff2 = TranslationUtils.compare_tosca_translation_with_hot( - tosca_file, hot_file2, params) - self.assertEqual({}, diff2, '<difference> : ' + - json.dumps(diff2, indent=4, - separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file2, params) def test_hot_translate_single_object_store(self): tosca_file = '../tests/data/storage/tosca_single_object_store.yaml' hot_file = '../tests/data/hot_output/hot_single_object_store.yaml' params = {'objectstore_name': 'myobjstore'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_one_server_one_network(self): tosca_file = '../tests/data/network/tosca_one_server_one_network.yaml' hot_file = '../tests/data/hot_output/network/' \ 'hot_one_server_one_network.yaml' params = {'network_name': 'private_net'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_server_on_existing_network(self): tosca_file = '../tests/data/network/' \ @@ -258,11 +205,7 @@ class ToscaHotTranslationTest(TestCase): hot_file = '../tests/data/hot_output/network/' \ 'hot_server_on_existing_network.yaml' params = {'network_name': 'private_net'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_two_servers_one_network(self): tosca_file = '../tests/data/network/tosca_two_servers_one_network.yaml' @@ -272,11 +215,7 @@ class ToscaHotTranslationTest(TestCase): 'network_cidr': '10.0.0.0/24', 'network_start_ip': '10.0.0.100', 'network_end_ip': '10.0.0.150'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_one_server_three_networks(self): tosca_file = '../tests/data/network/' \ @@ -284,32 +223,20 @@ class ToscaHotTranslationTest(TestCase): hot_file = '../tests/data/hot_output/network/' \ 'hot_one_server_three_networks.yaml' params = {} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_software_component(self): tosca_file = '../tests/data/tosca_software_component.yaml' hot_file = '../tests/data/hot_output/hot_software_component.yaml' params = {'cpus': '1', 'download_url': 'http://www.software.com/download'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_web_application(self): tosca_file = '../tests/data/tosca_web_application.yaml' hot_file = '../tests/data/hot_output/hot_web_application.yaml' params = {'cpus': '2', 'context_root': 'my_web_app'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_template_with_url_import(self): tosca_file = '../tests/data/' \ @@ -322,11 +249,7 @@ class ToscaHotTranslationTest(TestCase): 'db_root_pwd': 'passw0rd', 'db_port': 3366, 'cpus': 8} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_template_by_url_with_local_import(self): tosca_file = 'https://raw.githubusercontent.com/openstack/' \ @@ -340,11 +263,7 @@ class ToscaHotTranslationTest(TestCase): 'db_root_pwd': 'passw0rd', 'db_port': 3366, 'cpus': 8} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_template_by_url_with_local_abspath_import(self): tosca_file = 'https://raw.githubusercontent.com/openstack/' \ @@ -359,17 +278,15 @@ class ToscaHotTranslationTest(TestCase): 'db_root_pwd': 'passw0rd', 'db_port': 3366, 'cpus': 8} - - self.assertRaises( - ValidationError, - TranslationUtils.compare_tosca_translation_with_hot, - tosca_file, hot_file, params) expected_msg = _('Absolute file name "/tmp/wordpress.yaml" cannot be ' 'used in a URL-based input template "https://raw.' 'githubusercontent.com/openstack/heat-translator/' 'master/translator/tests/data/tosca_single_instance_' 'wordpress_with_local_abspath_import.yaml".') - ExceptionCollector.assertExceptionMessage(ImportError, expected_msg) + msg_path = False + self._test_failed_translation(tosca_file, hot_file, params, + expected_msg, msg_path, ValidationError, + ImportError) def test_hot_translate_template_by_url_with_url_import(self): tosca_url = 'https://raw.githubusercontent.com/openstack/' \ @@ -383,20 +300,12 @@ class ToscaHotTranslationTest(TestCase): 'db_root_pwd': 'passw0rd', 'db_port': 3366, 'cpus': 8} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_url, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_url, hot_file, params) def test_translate_hello_world_csar(self): tosca_file = '../tests/data/csar_hello_world.zip' hot_file = '../tests/data/hot_output/hot_hello_world.yaml' - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - {}) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file) def test_translate_single_instance_wordpress_csar(self): tosca_file = '../tests/data/csar_single_instance_wordpress.zip' @@ -408,11 +317,7 @@ class ToscaHotTranslationTest(TestCase): 'db_root_pwd': 'passw0rd', 'db_port': 3366, 'cpus': 8} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_translate_elk_csar_from_url(self): tosca_file = 'https://github.com/openstack/heat-translator/raw/' \ @@ -421,150 +326,103 @@ class ToscaHotTranslationTest(TestCase): params = {'github_url': 'http://github.com/paypal/rest-api-sample-app-nodejs.git', 'my_cpus': 4} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_translate_csar_not_zip(self): tosca_file = '../tests/data/csar_not_zip.zip' hot_file = '' params = {} - - self.assertRaises( - ValidationError, - TranslationUtils.compare_tosca_translation_with_hot, - tosca_file, hot_file, params) - path = os.path.normpath(os.path.join( - os.path.dirname(os.path.realpath(__file__)), tosca_file)) - expected_msg = _('"%s" is not a valid zip file.') % path - ExceptionCollector.assertExceptionMessage(ValidationError, - expected_msg) + expected_msg = _('"%s" is not a valid zip file.') + msg_path = True + self._test_failed_translation(tosca_file, hot_file, params, + expected_msg, msg_path, ValidationError, + ValidationError) def test_translate_csar_metadata_not_yaml(self): tosca_file = '../tests/data/csar_metadata_not_yaml.zip' hot_file = '' params = {} - - self.assertRaises( - ValidationError, - TranslationUtils.compare_tosca_translation_with_hot, - tosca_file, hot_file, params) - path = os.path.normpath(os.path.join( - os.path.dirname(os.path.realpath(__file__)), tosca_file)) expected_msg = _('The file "TOSCA-Metadata/TOSCA.meta" in the CSAR ' - '"%s" does not contain valid YAML content.') % path - ExceptionCollector.assertExceptionMessage(ValidationError, - expected_msg) + '"%s" does not contain valid YAML content.') + msg_path = True + self._test_failed_translation(tosca_file, hot_file, params, + expected_msg, msg_path, ValidationError, + ValidationError) def test_translate_csar_wrong_metadata_file(self): tosca_file = '../tests/data/csar_wrong_metadata_file.zip' hot_file = '' params = {} - - self.assertRaises( - ValidationError, - TranslationUtils.compare_tosca_translation_with_hot, - tosca_file, hot_file, params) - path = os.path.normpath(os.path.join( - os.path.dirname(os.path.realpath(__file__)), tosca_file)) expected_msg = _('"%s" is not a valid CSAR as it does not contain the ' 'required file "TOSCA.meta" in the folder ' - '"TOSCA-Metadata".') % path - ExceptionCollector.assertExceptionMessage(ValidationError, - expected_msg) + '"TOSCA-Metadata".') + msg_path = True + self._test_failed_translation(tosca_file, hot_file, params, + expected_msg, msg_path, ValidationError, + ValidationError) def test_translate_csar_wordpress_invalid_import_path(self): tosca_file = '../tests/data/csar_wordpress_invalid_import_path.zip' hot_file = '' params = {} - - self.assertRaises( - ValidationError, - TranslationUtils.compare_tosca_translation_with_hot, - tosca_file, hot_file, params) expected_msg = _('Import ' '"Invalid_import_path/wordpress.yaml" is not valid.') - ExceptionCollector.assertExceptionMessage(ImportError, expected_msg) + msg_path = False + self._test_failed_translation(tosca_file, hot_file, params, + expected_msg, msg_path, ValidationError, + ImportError) def test_translate_csar_wordpress_invalid_script_url(self): tosca_file = '../tests/data/csar_wordpress_invalid_script_url.zip' hot_file = '' params = {} - - self.assertRaises( - ValidationError, - TranslationUtils.compare_tosca_translation_with_hot, - tosca_file, hot_file, params) expected_msg = _('The resource at ' '"https://raw.githubusercontent.com/openstack/' 'heat-translator/master/translator/tests/data/' 'custom_types/wordpress1.yaml" cannot be accessed.') - ExceptionCollector.assertExceptionMessage(URLException, expected_msg) + msg_path = False + self._test_failed_translation(tosca_file, hot_file, params, + expected_msg, msg_path, ValidationError, + URLException) def test_hot_translate_flavor_image(self): tosca_file = '../tests/data/test_tosca_flavor_and_image.yaml' hot_file = '../tests/data/hot_output/hot_flavor_and_image.yaml' - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - {}) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file) def test_hot_translate_flavor_image_params(self): tosca_file = '../tests/data/test_tosca_flavor_and_image.yaml' hot_file = '../tests/data/hot_output/hot_flavor_and_image_params.yaml' params = {'key_name': 'paramkey'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_custom_type(self): tosca_file = '../tests/data/test_tosca_custom_type.yaml' hot_file = '../tests/data/hot_output/' \ 'hot_custom_type.yaml' params = {} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_custom_type_with_override(self): tosca_file = '../tests/data/test_tosca_custom_type_with_override.yaml' hot_file = '../tests/data/hot_output/' \ 'hot_custom_type_with_override.yaml' params = {} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_custom_type_with_param_override(self): tosca_file = '../tests/data/test_tosca_custom_type_with_override.yaml' hot_file = '../tests/data/hot_output/' \ 'hot_custom_type_with_param_override.yaml' params = {'install_path': '/home/custom/from/cli'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_artifact(self): tosca_file = '../tests/data/test_tosca_artifact.yaml' hot_file = '../tests/data/hot_output/' \ 'hot_artifact.yaml' params = {} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_without_tosca_os_version(self): tosca_file = '../tests/data/' \ @@ -572,21 +430,13 @@ class ToscaHotTranslationTest(TestCase): hot_file = '../tests/data/hot_output/' \ 'hot_single_server_without_tosca_os_version.yaml' params = {} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_helloworld_with_userkey(self): tosca_file = '../tests/data/tosca_helloworld.yaml' hot_file = '../tests/data/hot_output/hot_hello_world_userkey.yaml' params = {'key_name': 'userkey'} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_custom_networks_nodes_inline(self): tosca_file = '../tests/data/network/' \ @@ -594,11 +444,7 @@ class ToscaHotTranslationTest(TestCase): hot_file = '../tests/data/hot_output/network/' \ 'hot_custom_network_nodes.yaml' params = {} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_custom_networks_nodes_imports(self): tosca_file = '../tests/data/network/' \ @@ -606,28 +452,40 @@ class ToscaHotTranslationTest(TestCase): hot_file = '../tests/data/hot_output/network/' \ 'hot_custom_network_nodes.yaml' params = {} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_nfv_sample(self): tosca_file = '../tests/data/test_tosca_nfv_sample.yaml' hot_file = '../tests/data/hot_output/hot_nfv_sample.yaml' params = {} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) def test_hot_translate_policy(self): tosca_file = '../tests/data/tosca_policies.yaml' hot_file = '../tests/data/hot_output/hot_policies.yaml' params = {} - diff = TranslationUtils.compare_tosca_translation_with_hot(tosca_file, - hot_file, - params) - self.assertEqual({}, diff, '<difference> : ' + - json.dumps(diff, indent=4, separators=(', ', ': '))) + self._test_successful_translation(tosca_file, hot_file, params) + + def test_hot_script_types(self): + tosca_file = '../tests/data/test_tosca_script_types.yaml' + hot_file = '../tests/data/hot_output/hot_script_types.yaml' + params = {} + self._test_successful_translation(tosca_file, hot_file, params) + + def test_hot_interface_on_compute(self): + tosca_file = '../tests/data/test_tosca_interface_on_compute.yaml' + hot_file = '../tests/data/hot_output/hot_interface_on_compute.yaml' + params = {} + self._test_successful_translation(tosca_file, hot_file, params) + + def test_hot_get_functions_semantic(self): + tosca_file = '../tests/data/test_tosca_get_functions_semantic.yaml' + hot_file = '../tests/data/hot_output/hot_get_functions_semantic.yaml' + params = {} + self._test_successful_translation(tosca_file, hot_file, params) + + def test_hot_exchange_public_ssh_key(self): + tosca_file = '../tests/data/tosca_exchange_public_ssh_key.yaml' + hot_file = '../tests/data/hot_output/hot_exchange_public_ssh_key.yaml' + params = {} + self._test_successful_translation(tosca_file, hot_file, params) diff --git a/tosca2heat/tosca-parser/.gitignore b/tosca2heat/tosca-parser/.gitignore new file mode 100644 index 0000000..c172104 --- /dev/null +++ b/tosca2heat/tosca-parser/.gitignore @@ -0,0 +1,56 @@ +*.py[cod] + +# C extensions +*.so + +# Packages +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +nosetests.xml +.testrepository + +# Translations +*.mo + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# Complexity +output/*.html +output/*/index.html + +# Sphinx +doc/build + +# pbr generates these +AUTHORS +ChangeLog + +# Editors +*~ +.*.swp +.idea +*.iml + +# OSX +.DS_Store
\ No newline at end of file diff --git a/tosca2heat/tosca-parser/.gitreview b/tosca2heat/tosca-parser/.gitreview new file mode 100644 index 0000000..9105f23 --- /dev/null +++ b/tosca2heat/tosca-parser/.gitreview @@ -0,0 +1,4 @@ +[gerrit] +host=review.openstack.org +port=29418 +project=openstack/tosca-parser.git diff --git a/tosca2heat/tosca-parser/.mailmap b/tosca2heat/tosca-parser/.mailmap new file mode 100644 index 0000000..cc92f17 --- /dev/null +++ b/tosca2heat/tosca-parser/.mailmap @@ -0,0 +1,3 @@ +# Format is: +# <preferred e-mail> <other e-mail 1> +# <preferred e-mail> <other e-mail 2>
\ No newline at end of file diff --git a/tosca2heat/tosca-parser/requirements.txt b/tosca2heat/tosca-parser/requirements.txt index dd27a53..b30cc51 100644 --- a/tosca2heat/tosca-parser/requirements.txt +++ b/tosca2heat/tosca-parser/requirements.txt @@ -2,8 +2,9 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. pbr>=1.6 # Apache-2.0 -Babel>=1.3 # BSD +Babel>=2.3.4 # BSD cliff!=1.16.0,!=1.17.0,>=1.15.0 # Apache-2.0 PyYAML>=3.1.0 # MIT python-dateutil>=2.4.2 # BSD six>=1.9.0 # MIT +requests>=2.10.0 # Apache-2.0 diff --git a/tosca2heat/tosca-parser/test-requirements.txt b/tosca2heat/tosca-parser/test-requirements.txt index 2b9cae7..70a261b 100644 --- a/tosca2heat/tosca-parser/test-requirements.txt +++ b/tosca2heat/tosca-parser/test-requirements.txt @@ -4,11 +4,11 @@ hacking<0.11,>=0.10.0 coverage>=3.6 # Apache-2.0 discover # BSD -fixtures>=1.3.1 # Apache-2.0/BSD +fixtures>=3.0.0 # Apache-2.0/BSD oslotest>=1.10.0 # Apache-2.0 oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 python-subunit>=0.0.18 # Apache-2.0/BSD -sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2 # BSD +sphinx!=1.3b1,<1.3,>=1.2.1 # BSD testrepository>=0.0.18 # Apache-2.0/BSD testscenarios>=0.4 # Apache-2.0/BSD testtools>=1.4.0 # MIT diff --git a/tosca2heat/tosca-parser/toscaparser/common/exception.py b/tosca2heat/tosca-parser/toscaparser/common/exception.py index 4f99dda..bd0ed9c 100644 --- a/tosca2heat/tosca-parser/toscaparser/common/exception.py +++ b/tosca2heat/tosca-parser/toscaparser/common/exception.py @@ -88,6 +88,15 @@ class InvalidTypeError(TOSCAException): msg_fmt = _('Type "%(what)s" is not a valid type.') +class InvalidTypeAdditionalRequirementsError(TOSCAException): + msg_fmt = _('Additional requirements for type "%(type)s" not met.') + + +class RangeValueError(TOSCAException): + msg_fmt = _('The value "%(pvalue)s" of property "%(pname)s" is out of ' + 'range "(min:%(vmin)s, max:%(vmax)s)".') + + class InvalidSchemaError(TOSCAException): msg_fmt = _('%(message)s') diff --git a/tosca2heat/tosca-parser/toscaparser/dataentity.py b/tosca2heat/tosca-parser/toscaparser/dataentity.py index 6e7d59e..507c899 100644 --- a/tosca2heat/tosca-parser/toscaparser/dataentity.py +++ b/tosca2heat/tosca-parser/toscaparser/dataentity.py @@ -16,10 +16,10 @@ from toscaparser.common.exception import TypeMismatchError from toscaparser.common.exception import UnknownFieldError from toscaparser.elements.constraints import Schema from toscaparser.elements.datatype import DataType +from toscaparser.elements.portspectype import PortSpec from toscaparser.elements.scalarunit import ScalarUnit_Frequency from toscaparser.elements.scalarunit import ScalarUnit_Size from toscaparser.elements.scalarunit import ScalarUnit_Time - from toscaparser.utils.gettextutils import _ from toscaparser.utils import validateutils @@ -27,11 +27,13 @@ from toscaparser.utils import validateutils class DataEntity(object): '''A complex data value entity.''' - def __init__(self, datatypename, value_dict, custom_def=None): + def __init__(self, datatypename, value_dict, custom_def=None, + prop_name=None): self.custom_def = custom_def self.datatype = DataType(datatypename, custom_def) self.schema = self.datatype.get_all_properties() self.value = value_dict + self.property_name = prop_name def validate(self): '''Validate the value by the definition of the datatype.''' @@ -43,7 +45,7 @@ class DataEntity(object): self.value, None, self.custom_def) - schema = Schema(None, self.datatype.defs) + schema = Schema(self.property_name, self.datatype.defs) for constraint in schema.constraints: constraint.validate(self.value) # If the datatype has 'properties' definition @@ -89,7 +91,10 @@ class DataEntity(object): # check every field for name, value in list(self.value.items()): - prop_schema = Schema(name, self._find_schema(name)) + schema_name = self._find_schema(name) + if not schema_name: + continue + prop_schema = Schema(name, schema_name) # check if field value meets type defined DataEntity.validate_datatype(prop_schema.type, value, prop_schema.entry_schema, @@ -110,12 +115,16 @@ class DataEntity(object): return self.schema[name].schema @staticmethod - def validate_datatype(type, value, entry_schema=None, custom_def=None): + def validate_datatype(type, value, entry_schema=None, custom_def=None, + prop_name=None): '''Validate value with given type. If type is list or map, validate its entry by entry_schema(if defined) If type is a user-defined complex datatype, custom_def is required. ''' + from toscaparser.functions import is_function + if is_function(value): + return value if type == Schema.STRING: return validateutils.validate_string(value) elif type == Schema.INTEGER: @@ -123,7 +132,7 @@ class DataEntity(object): elif type == Schema.FLOAT: return validateutils.validate_float(value) elif type == Schema.NUMBER: - return validateutils.validate_number(value) + return validateutils.validate_numeric(value) elif type == Schema.BOOLEAN: return validateutils.validate_boolean(value) elif type == Schema.RANGE: @@ -149,6 +158,10 @@ class DataEntity(object): if entry_schema: DataEntity.validate_entry(value, entry_schema, custom_def) return value + elif type == Schema.PORTSPEC: + # TODO(TBD) bug 1567063, validate source & target as PortDef type + # as complex types not just as integers + PortSpec.validate_additional_req(value, prop_name, custom_def) else: data = DataEntity(type, value, custom_def) return data.validate() diff --git a/tosca2heat/tosca-parser/toscaparser/elements/TOSCA_definition_1_0.yaml b/tosca2heat/tosca-parser/toscaparser/elements/TOSCA_definition_1_0.yaml index c1c1cd4..ede5fa5 100644 --- a/tosca2heat/tosca-parser/toscaparser/elements/TOSCA_definition_1_0.yaml +++ b/tosca2heat/tosca-parser/toscaparser/elements/TOSCA_definition_1_0.yaml @@ -390,7 +390,7 @@ tosca.nodes.Container.Application: requirements: - host: capability: tosca.capabilities.Container - node: tosca.nodes.Container + node: tosca.nodes.Container.Runtime relationship: tosca.relationships.HostedOn tosca.nodes.Container.Runtime: @@ -544,7 +544,7 @@ tosca.capabilities.Endpoint: tosca.capabilities.Endpoint.Admin: derived_from: tosca.capabilities.Endpoint properties: - secure: + secure: type: boolean default: true constraints: @@ -815,14 +815,19 @@ tosca.datatypes.Credential: properties: protocol: type: string + required: false token_type: type: string + default: password + required: true token: type: string + required: true keys: type: map entry_schema: type: string + required: false user: type: string required: false diff --git a/tosca2heat/tosca-parser/toscaparser/elements/capabilitytype.py b/tosca2heat/tosca-parser/toscaparser/elements/capabilitytype.py index 0413443..865690e 100644 --- a/tosca2heat/tosca-parser/toscaparser/elements/capabilitytype.py +++ b/tosca2heat/tosca-parser/toscaparser/elements/capabilitytype.py @@ -16,6 +16,7 @@ from toscaparser.elements.statefulentitytype import StatefulEntityType class CapabilityTypeDef(StatefulEntityType): '''TOSCA built-in capabilities type.''' + TOSCA_TYPEURI_CAPABILITY_ROOT = 'tosca.capabilities.Root' def __init__(self, name, ctype, ntype, custom_def=None): self.name = name @@ -23,6 +24,7 @@ class CapabilityTypeDef(StatefulEntityType): custom_def) self.nodetype = ntype self.properties = None + self.custom_def = custom_def if self.PROPERTIES in self.defs: self.properties = self.defs[self.PROPERTIES] self.parent_capabilities = self._get_parent_capabilities(custom_def) @@ -61,7 +63,8 @@ class CapabilityTypeDef(StatefulEntityType): capabilities = {} parent_cap = self.parent_type if parent_cap: - while parent_cap != 'tosca.capabilities.Root': + parent_cap = parent_cap.type + while parent_cap != self.TOSCA_TYPEURI_CAPABILITY_ROOT: if parent_cap in self.TOSCA_DEF.keys(): capabilities[parent_cap] = self.TOSCA_DEF[parent_cap] elif custom_def and parent_cap in custom_def.keys(): @@ -72,4 +75,9 @@ class CapabilityTypeDef(StatefulEntityType): @property def parent_type(self): '''Return a capability this capability is derived from.''' - return self.derived_from(self.defs) + if not hasattr(self, 'defs'): + return None + pnode = self.derived_from(self.defs) + if pnode: + return CapabilityTypeDef(self.name, pnode, + self.nodetype, self.custom_def) diff --git a/tosca2heat/tosca-parser/toscaparser/elements/constraints.py b/tosca2heat/tosca-parser/toscaparser/elements/constraints.py index 9883da3..8594b85 100644 --- a/tosca2heat/tosca-parser/toscaparser/elements/constraints.py +++ b/tosca2heat/tosca-parser/toscaparser/elements/constraints.py @@ -18,6 +18,7 @@ import toscaparser from toscaparser.common.exception import ExceptionCollector from toscaparser.common.exception import InvalidSchemaError from toscaparser.common.exception import ValidationError +from toscaparser.elements.portspectype import PortSpec from toscaparser.elements import scalarunit from toscaparser.utils.gettextutils import _ @@ -36,12 +37,12 @@ class Schema(collections.Mapping): INTEGER, STRING, BOOLEAN, FLOAT, RANGE, NUMBER, TIMESTAMP, LIST, MAP, SCALAR_UNIT_SIZE, SCALAR_UNIT_FREQUENCY, SCALAR_UNIT_TIME, - PORTDEF, VERSION + VERSION, PORTDEF, PORTSPEC ) = ( 'integer', 'string', 'boolean', 'float', 'range', 'number', 'timestamp', 'list', 'map', 'scalar-unit.size', 'scalar-unit.frequency', 'scalar-unit.time', - 'PortDef', 'version' + 'version', 'PortDef', PortSpec.SHORTNAME ) SCALAR_UNIT_SIZE_DEFAULT = 'B' @@ -127,8 +128,6 @@ class Constraint(object): 'less_or_equal', 'in_range', 'valid_values', 'length', 'min_length', 'max_length', 'pattern') - UNBOUNDED = 'UNBOUNDED' - def __new__(cls, property_name, property_type, constraint): if cls is not Constraint: return super(Constraint, cls).__new__(cls) @@ -370,6 +369,7 @@ class InRange(Constraint): Constrains a property or parameter to a value in range of (inclusive) the two values declared. """ + UNBOUNDED = 'UNBOUNDED' constraint_key = Constraint.IN_RANGE diff --git a/tosca2heat/tosca-parser/toscaparser/elements/datatype.py b/tosca2heat/tosca-parser/toscaparser/elements/datatype.py index 7e05a69..93d1b3a 100644 --- a/tosca2heat/tosca-parser/toscaparser/elements/datatype.py +++ b/tosca2heat/tosca-parser/toscaparser/elements/datatype.py @@ -18,7 +18,8 @@ class DataType(StatefulEntityType): '''TOSCA built-in and user defined complex data type.''' def __init__(self, datatypename, custom_def=None): - super(DataType, self).__init__(datatypename, self.DATATYPE_PREFIX, + super(DataType, self).__init__(datatypename, + self.DATATYPE_NETWORK_PREFIX, custom_def) self.custom_def = custom_def diff --git a/tosca2heat/tosca-parser/toscaparser/elements/entity_type.py b/tosca2heat/tosca-parser/toscaparser/elements/entity_type.py index 72e7e3f..5947b1c 100644 --- a/tosca2heat/tosca-parser/toscaparser/elements/entity_type.py +++ b/tosca2heat/tosca-parser/toscaparser/elements/entity_type.py @@ -56,7 +56,8 @@ class EntityType(object): GROUP_PREFIX = 'tosca.groups.' # currently the data types are defined only for network # but may have changes in the future. - DATATYPE_PREFIX = 'tosca.datatypes.network.' + DATATYPE_PREFIX = 'tosca.datatypes.' + DATATYPE_NETWORK_PREFIX = DATATYPE_PREFIX + 'network.' TOSCA = 'tosca' def derived_from(self, defs): diff --git a/tosca2heat/tosca-parser/toscaparser/elements/portspectype.py b/tosca2heat/tosca-parser/toscaparser/elements/portspectype.py new file mode 100644 index 0000000..d32e97e --- /dev/null +++ b/tosca2heat/tosca-parser/toscaparser/elements/portspectype.py @@ -0,0 +1,86 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from toscaparser.common.exception import ExceptionCollector +from toscaparser.common.exception import InvalidTypeAdditionalRequirementsError +from toscaparser.utils.gettextutils import _ +import toscaparser.utils.validateutils as validateutils + +log = logging.getLogger('tosca') + + +class PortSpec(object): + '''Parent class for tosca.datatypes.network.PortSpec type.''' + + SHORTNAME = 'PortSpec' + TYPE_URI = 'tosca.datatypes.network.' + SHORTNAME + + PROPERTY_NAMES = ( + PROTOCOL, SOURCE, SOURCE_RANGE, + TARGET, TARGET_RANGE + ) = ( + 'protocol', 'source', 'source_range', + 'target', 'target_range' + ) + + # TODO(TBD) May want to make this a subclass of DataType + # and change init method to set PortSpec's properties + def __init__(self): + pass + + # The following additional requirements MUST be tested: + # 1) A valid PortSpec MUST have at least one of the following properties: + # target, target_range, source or source_range. + # 2) A valid PortSpec MUST have a value for the source property that + # is within the numeric range specified by the property source_range + # when source_range is specified. + # 3) A valid PortSpec MUST have a value for the target property that is + # within the numeric range specified by the property target_range + # when target_range is specified. + @staticmethod + def validate_additional_req(properties, prop_name, custom_def=None, ): + try: + source = properties.get(PortSpec.SOURCE) + source_range = properties.get(PortSpec.SOURCE_RANGE) + target = properties.get(PortSpec.TARGET) + target_range = properties.get(PortSpec.TARGET_RANGE) + + # verify one of the specified values is set + if source is None and source_range is None and \ + target is None and target_range is None: + ExceptionCollector.appendException( + InvalidTypeAdditionalRequirementsError( + type=PortSpec.TYPE_URI)) + # Validate source value is in specified range + if source and source_range: + validateutils.validate_value_in_range(source, source_range, + PortSpec.SOURCE) + else: + from toscaparser.dataentity import DataEntity + portdef = DataEntity('PortDef', source, None, PortSpec.SOURCE) + portdef.validate() + # Validate target value is in specified range + if target and target_range: + validateutils.validate_value_in_range(target, target_range, + PortSpec.TARGET) + else: + from toscaparser.dataentity import DataEntity + portdef = DataEntity('PortDef', source, None, PortSpec.TARGET) + portdef.validate() + except Exception: + msg = _('"%(value)s" do not meet requirements ' + 'for type "%(type)s".') \ + % {'value': properties, 'type': PortSpec.SHORTNAME} + ExceptionCollector.appendException( + ValueError(msg)) diff --git a/tosca2heat/tosca-parser/toscaparser/entity_template.py b/tosca2heat/tosca-parser/toscaparser/entity_template.py index f416c99..7488c33 100644 --- a/tosca2heat/tosca-parser/toscaparser/entity_template.py +++ b/tosca2heat/tosca-parser/toscaparser/entity_template.py @@ -81,15 +81,11 @@ class EntityTemplate(object): def type(self): if self.type_definition: return self.type_definition.type - else: - return None @property def parent_type(self): if self.type_definition: return self.type_definition.parent_type - else: - return None @property def requirements(self): @@ -189,7 +185,10 @@ class EntityTemplate(object): def _validate_capabilities_properties(self, capabilities): for cap, props in capabilities.items(): - capabilitydef = self.get_capability(cap).definition + capability = self.get_capability(cap) + if not capability: + continue + capabilitydef = capability.definition self._common_validate_properties(capabilitydef, props[self.PROPERTIES]) diff --git a/tosca2heat/tosca-parser/toscaparser/extensions/exttools.py b/tosca2heat/tosca-parser/toscaparser/extensions/exttools.py index 963b958..5310422 100644 --- a/tosca2heat/tosca-parser/toscaparser/extensions/exttools.py +++ b/tosca2heat/tosca-parser/toscaparser/extensions/exttools.py @@ -36,7 +36,7 @@ class ExtTools(object): extdirs = [e for e in os.listdir(abs_path) if not e.startswith('tests') and - not e.endswith('.pyc') and not e.endswith('.py')] + os.path.isdir(os.path.join(abs_path, e))] for e in extdirs: log.info(e) diff --git a/tosca2heat/tosca-parser/toscaparser/extensions/nfv/TOSCA_nfv_definition_1_0.yaml b/tosca2heat/tosca-parser/toscaparser/extensions/nfv/TOSCA_nfv_definition_1_0.yaml index 660cdc0..dc986e5 100644 --- a/tosca2heat/tosca-parser/toscaparser/extensions/nfv/TOSCA_nfv_definition_1_0.yaml +++ b/tosca2heat/tosca-parser/toscaparser/extensions/nfv/TOSCA_nfv_definition_1_0.yaml @@ -235,16 +235,3 @@ tosca.groups.nfv.VNFFG: type: string required: true description: Reference to a list of VNFD used in this VNF Forwarding Graph - - targets: - type: list - entry_schema: - type: string - required: false - description: list of Network Forwarding Path within the VNFFG - - requirements: - - forwarder: - capability: tosca.capabilities.nfv.Forwarder - relationship: tosca.relationships.nfv.ForwardsTo - diff --git a/tosca2heat/tosca-parser/toscaparser/functions.py b/tosca2heat/tosca-parser/toscaparser/functions.py index 011efde..1b64416 100644 --- a/tosca2heat/tosca-parser/toscaparser/functions.py +++ b/tosca2heat/tosca-parser/toscaparser/functions.py @@ -18,6 +18,8 @@ import six from toscaparser.common.exception import ExceptionCollector from toscaparser.common.exception import UnknownInputError from toscaparser.dataentity import DataEntity +from toscaparser.elements.constraints import Schema +from toscaparser.elements.datatype import DataType from toscaparser.elements.entity_type import EntityType from toscaparser.elements.relationshiptype import RelationshipType from toscaparser.utils.gettextutils import _ @@ -27,6 +29,7 @@ GET_PROPERTY = 'get_property' GET_ATTRIBUTE = 'get_attribute' GET_INPUT = 'get_input' CONCAT = 'concat' +TOKEN = 'token' SELF = 'SELF' HOST = 'HOST' @@ -125,30 +128,67 @@ class GetAttribute(Function): * { get_attribute: [ server, private_address ] } * { get_attribute: [ HOST, private_address ] } * { get_attribute: [ HOST, private_address, 0 ] } + * { get_attribute: [ HOST, private_address, 0, some_prop] } """ def validate(self): - if len(self.args) != 2 and len(self.args) != 3: + if len(self.args) < 2: ExceptionCollector.appendException( ValueError(_('Illegal arguments for function "{0}". Expected ' - 'arguments: "node-template-name", ' - '"attribute-name"').format(GET_ATTRIBUTE))) - node_tpl = self._find_node_template_containing_attribute() - if len(self.args) > 2: - # Currently we only check the first level - attrs_def = node_tpl.type_definition.get_attributes_def() - attr_def = attrs_def[self.attribute_name] - if attr_def.schema['type'] == "list": - if not isinstance(self.args[2], int): - ExceptionCollector.appendException( - ValueError(_('Illegal arguments for function "{0}". ' - 'Third argument must be a positive' - ' integer') .format(GET_ATTRIBUTE))) - elif attr_def.schema['type'] != "map": - ExceptionCollector.appendException( - ValueError(_('Illegal arguments for function "{0}". ' - 'Expected arguments: "node-template-name", ' - '"attribute-name"').format(GET_ATTRIBUTE))) + 'arguments: "node-template-name", "req-or-cap"' + '(optional), "property name"' + ).format(GET_ATTRIBUTE))) + return + elif len(self.args) == 2: + self._find_node_template_containing_attribute() + else: + node_tpl = self._find_node_template(self.args[0]) + index = 2 + attrs = node_tpl.type_definition.get_attributes_def() + found = [attrs[self.args[1]]] if self.args[1] in attrs else [] + if found: + attr = found[0] + else: + index = 3 + # then check the req or caps + attr = self._find_req_or_cap_attribute(self.args[1], + self.args[2]) + + value_type = attr.schema['type'] + if len(self.args) > index: + for elem in self.args[index:]: + if value_type == "list": + if not isinstance(elem, int): + ExceptionCollector.appendException( + ValueError(_('Illegal arguments for function' + ' "{0}". "{1}" Expected positive' + ' integer argument' + ).format(GET_ATTRIBUTE, elem))) + value_type = attr.schema['entry_schema']['type'] + elif value_type == "map": + value_type = attr.schema['entry_schema']['type'] + elif value_type in Schema.PROPERTY_TYPES: + ExceptionCollector.appendException( + ValueError(_('Illegal arguments for function' + ' "{0}". Unexpected attribute/' + 'index value "{1}"' + ).format(GET_ATTRIBUTE, elem))) + return + else: # It is a complex type + data_type = DataType(value_type) + props = data_type.get_all_properties() + found = [props[elem]] if elem in props else [] + if found: + prop = found[0] + value_type = prop.schema['type'] + else: + ExceptionCollector.appendException( + KeyError(_('Illegal arguments for function' + ' "{0}". Attribute name "{1}" not' + ' found in "{2}"' + ).format(GET_ATTRIBUTE, + elem, + value_type))) def result(self): return self @@ -163,25 +203,7 @@ class GetAttribute(Function): return self._find_node_template_containing_attribute() def _find_node_template_containing_attribute(self): - if self.node_template_name == HOST: - # Currently this is the only way to tell whether the function - # is used within the outputs section of the TOSCA template. - if isinstance(self.context, list): - ExceptionCollector.appendException( - ValueError(_( - '"get_attribute: [ HOST, ... ]" is not allowed in ' - '"outputs" section of the TOSCA template.'))) - return - node_tpl = self._find_host_containing_attribute() - if not node_tpl: - ExceptionCollector.appendException( - ValueError(_( - '"get_attribute: [ HOST, ... ]" was used in node ' - 'template "{0}" but "{1}" was not found in ' - 'the relationship chain.').format(self.context.name, - HOSTED_ON))) - else: - node_tpl = self._find_node_template(self.args[0]) + node_tpl = self._find_node_template(self.args[0]) if node_tpl and \ not self._attribute_exists_in_type(node_tpl.type_definition): ExceptionCollector.appendException( @@ -214,6 +236,25 @@ class GetAttribute(Function): target_name) def _find_node_template(self, node_template_name): + if node_template_name == HOST: + # Currently this is the only way to tell whether the function + # is used within the outputs section of the TOSCA template. + if isinstance(self.context, list): + ExceptionCollector.appendException( + ValueError(_( + '"get_attribute: [ HOST, ... ]" is not allowed in ' + '"outputs" section of the TOSCA template.'))) + return + node_tpl = self._find_host_containing_attribute() + if not node_tpl: + ExceptionCollector.appendException( + ValueError(_( + '"get_attribute: [ HOST, ... ]" was used in node ' + 'template "{0}" but "{1}" was not found in ' + 'the relationship chain.').format(self.context.name, + HOSTED_ON))) + return + return node_tpl if node_template_name == TARGET: if not isinstance(self.context.type_definition, RelationshipType): ExceptionCollector.appendException( @@ -240,6 +281,51 @@ class GetAttribute(Function): 'Node template "{0}" was not found.' ).format(node_template_name))) + def _find_req_or_cap_attribute(self, req_or_cap, attr_name): + node_tpl = self._find_node_template(self.args[0]) + # Find attribute in node template's requirements + for r in node_tpl.requirements: + for req, node_name in r.items(): + if req == req_or_cap: + node_template = self._find_node_template(node_name) + return self._get_capability_attribute( + node_template, + req, + attr_name) + # If requirement was not found, look in node template's capabilities + return self._get_capability_attribute(node_tpl, + req_or_cap, + attr_name) + + def _get_capability_attribute(self, + node_template, + capability_name, + attr_name): + """Gets a node template capability attribute.""" + caps = node_template.get_capabilities() + if caps and capability_name in caps.keys(): + cap = caps[capability_name] + attribute = None + attrs = cap.definition.get_attributes_def() + if attrs and attr_name in attrs.keys(): + attribute = attrs[attr_name] + if not attribute: + ExceptionCollector.appendException( + KeyError(_('Attribute "%(attr)s" was not found in ' + 'capability "%(cap)s" of node template ' + '"%(ntpl1)s" referenced from node template ' + '"%(ntpl2)s".') % {'attr': attr_name, + 'cap': capability_name, + 'ntpl1': node_template.name, + 'ntpl2': self.context.name})) + return attribute + msg = _('Requirement/Capability "{0}" referenced from node template ' + '"{1}" was not found in node template "{2}".').format( + capability_name, + self.context.name, + node_template.name) + ExceptionCollector.appendException(KeyError(msg)) + @property def node_template_name(self): return self.args[0] @@ -551,11 +637,58 @@ class Concat(Function): def result(self): return self + +class Token(Function): + """Validate the function and provide an instance of the function + + The token function is used within a TOSCA service template on a string to + parse out (tokenize) substrings separated by one or more token characters + within a larger string. + + + Arguments: + + * The composite string that contains one or more substrings separated by + token characters. + * The string that contains one or more token characters that separate + substrings within the composite string. + * The integer indicates the index of the substring to return from the + composite string. Note that the first substring is denoted by using + the '0' (zero) integer value. + + Example: + + [ get_attribute: [ my_server, data_endpoint, ip_address ], ':', 1 ] + + """ + + def validate(self): + if len(self.args) < 3: + ExceptionCollector.appendException( + ValueError(_('Invalid arguments for function "{0}". Expected ' + 'at least three arguments.').format(TOKEN))) + else: + if not isinstance(self.args[1], str) or len(self.args[1]) != 1: + ExceptionCollector.appendException( + ValueError(_('Invalid arguments for function "{0}". ' + 'Expected single char value as second ' + 'argument.').format(TOKEN))) + + if not isinstance(self.args[2], int): + ExceptionCollector.appendException( + ValueError(_('Invalid arguments for function "{0}". ' + 'Expected integer value as third ' + 'argument.').format(TOKEN))) + + def result(self): + return self + function_mappings = { GET_PROPERTY: GetProperty, GET_INPUT: GetInput, GET_ATTRIBUTE: GetAttribute, - CONCAT: Concat + CONCAT: Concat, + TOKEN: Token } diff --git a/tosca2heat/tosca-parser/toscaparser/groups.py b/tosca2heat/tosca-parser/toscaparser/groups.py index 5fd5dec..6a3e5c7 100644 --- a/tosca2heat/tosca-parser/toscaparser/groups.py +++ b/tosca2heat/tosca-parser/toscaparser/groups.py @@ -15,7 +15,7 @@ from toscaparser.common.exception import UnknownFieldError from toscaparser.entity_template import EntityTemplate from toscaparser.utils import validateutils -SECTIONS = (TYPE, METADATA, DESCRIPTION, PROPERTIES, TARGETS, INTERFACES) = \ +SECTIONS = (TYPE, METADATA, DESCRIPTION, PROPERTIES, MEMBERS, INTERFACES) = \ ('type', 'metadata', 'description', 'properties', 'members', 'interfaces') diff --git a/tosca2heat/tosca-parser/toscaparser/imports.py b/tosca2heat/tosca-parser/toscaparser/imports.py index 62748bd..58c9300 100644 --- a/tosca2heat/tosca-parser/toscaparser/imports.py +++ b/tosca2heat/tosca-parser/toscaparser/imports.py @@ -14,6 +14,7 @@ import logging import os from toscaparser.common.exception import ExceptionCollector +from toscaparser.common.exception import InvalidPropertyValueError from toscaparser.common.exception import MissingRequiredFieldError from toscaparser.common.exception import UnknownFieldError from toscaparser.common.exception import ValidationError @@ -161,12 +162,17 @@ class ImportsLoader(object): | URL | URL | OK | +----------+--------+------------------------------+ """ - short_import_notation = False if isinstance(import_uri_def, dict): self._validate_import_keys(import_name, import_uri_def) file_name = import_uri_def.get(self.FILE) repository = import_uri_def.get(self.REPOSITORY) + repos = self.repositories.keys() + if repository is not None: + if repository not in repos: + ExceptionCollector.appendException( + InvalidPropertyValueError( + what=_('Repository is not found in "%s"') % repos)) else: file_name = import_uri_def repository = None diff --git a/tosca2heat/tosca-parser/toscaparser/parameters.py b/tosca2heat/tosca-parser/toscaparser/parameters.py index 983aee3..1d2cb29 100644 --- a/tosca2heat/tosca-parser/toscaparser/parameters.py +++ b/tosca2heat/tosca-parser/toscaparser/parameters.py @@ -27,8 +27,9 @@ log = logging.getLogger('tosca') class Input(object): - INPUTFIELD = (TYPE, DESCRIPTION, DEFAULT, CONSTRAINTS) = \ - ('type', 'description', 'default', 'constraints') + INPUTFIELD = (TYPE, DESCRIPTION, DEFAULT, CONSTRAINTS, REQUIRED, + STATUS) = ('type', 'description', 'default', + 'constraints', 'required', 'status') def __init__(self, name, schema_dict): self.name = name @@ -53,7 +54,7 @@ class Input(object): def validate(self, value=None): self._validate_field() self.validate_type(self.type) - if value: + if value is not None: self._validate_value(value) def _validate_field(self): @@ -68,13 +69,16 @@ class Input(object): ExceptionCollector.appendException( ValueError(_('Invalid type "%s".') % type)) + # TODO(anyone) Need to test for any built-in datatype not just network + # that is, tosca.datatypes.* and not assume tosca.datatypes.network.* + # TODO(anyone) Add support for tosca.datatypes.Credential def _validate_value(self, value): tosca = EntityType.TOSCA_DEF datatype = None if self.type in tosca: datatype = tosca[self.type] - elif EntityType.DATATYPE_PREFIX + self.type in tosca: - datatype = tosca[EntityType.DATATYPE_PREFIX + self.type] + elif EntityType.DATATYPE_NETWORK_PREFIX + self.type in tosca: + datatype = tosca[EntityType.DATATYPE_NETWORK_PREFIX + self.type] DataEntity.validate_datatype(self.type, value, None, datatype) diff --git a/tosca2heat/tosca-parser/toscaparser/properties.py b/tosca2heat/tosca-parser/toscaparser/properties.py index 23c1db2..c69b151 100644 --- a/tosca2heat/tosca-parser/toscaparser/properties.py +++ b/tosca2heat/tosca-parser/toscaparser/properties.py @@ -67,7 +67,8 @@ class Property(object): self.value = str(self.value) self.value = DataEntity.validate_datatype(self.type, self.value, self.entry_schema, - self.custom_def) + self.custom_def, + self.name) self._validate_constraints() def _validate_constraints(self): diff --git a/tosca2heat/tosca-parser/toscaparser/repositories.py b/tosca2heat/tosca-parser/toscaparser/repositories.py new file mode 100644 index 0000000..184eba4 --- /dev/null +++ b/tosca2heat/tosca-parser/toscaparser/repositories.py @@ -0,0 +1,52 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from toscaparser.common.exception import ExceptionCollector +from toscaparser.common.exception import MissingRequiredFieldError +from toscaparser.common.exception import UnknownFieldError +from toscaparser.common.exception import URLException +from toscaparser.utils.gettextutils import _ +import toscaparser.utils.urlutils + +SECTIONS = (DESCRIPTION, URL, CREDENTIAL) = \ + ('description', 'url', 'credential') + + +class Repository(object): + def __init__(self, repositories, values): + self.name = repositories + self.reposit = values + if isinstance(self.reposit, dict): + if 'url' not in self.reposit.keys(): + ExceptionCollector.appendException( + MissingRequiredFieldError(what=_('Repository "%s"') + % self.name, required='url')) + self.url = self.reposit['url'] + self.load_and_validate(self.name, self.reposit) + + def load_and_validate(self, val, reposit_def): + self.keyname = val + if isinstance(reposit_def, dict): + for key in reposit_def.keys(): + if key not in SECTIONS: + ExceptionCollector.appendException( + UnknownFieldError(what=_('repositories "%s"') + % self.keyname, field=key)) + + if URL in reposit_def.keys(): + reposit_url = reposit_def.get(URL) + url_val = toscaparser.utils.urlutils.UrlUtils.\ + validate_url(reposit_url) + if url_val is not True: + ExceptionCollector.appendException( + URLException(what=_('repsositories "%s" Invalid Url') + % self.keyname)) diff --git a/tosca2heat/tosca-parser/toscaparser/shell.py b/tosca2heat/tosca-parser/toscaparser/shell.py index 848726f..b41c024 100644 --- a/tosca2heat/tosca-parser/toscaparser/shell.py +++ b/tosca2heat/tosca-parser/toscaparser/shell.py @@ -67,33 +67,33 @@ class ParserShell(object): version = tosca.version if tosca.version: - print ("\nversion: " + version) + print("\nversion: " + version) if hasattr(tosca, 'description'): description = tosca.description if description: - print ("\ndescription: " + description) + print("\ndescription: " + description) if hasattr(tosca, 'inputs'): inputs = tosca.inputs if inputs: - print ("\ninputs:") + print("\ninputs:") for input in inputs: - print ("\t" + input.name) + print("\t" + input.name) if hasattr(tosca, 'nodetemplates'): nodetemplates = tosca.nodetemplates if nodetemplates: - print ("\nnodetemplates:") + print("\nnodetemplates:") for node in nodetemplates: - print ("\t" + node.name) + print("\t" + node.name) if hasattr(tosca, 'outputs'): outputs = tosca.outputs if outputs: - print ("\noutputs:") + print("\noutputs:") for output in outputs: - print ("\t" + output.name) + print("\t" + output.name) def main(args=None): diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/custom_types/compute_with_nested_atributes.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/custom_types/compute_with_nested_atributes.yaml new file mode 100644 index 0000000..f23a8a1 --- /dev/null +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/custom_types/compute_with_nested_atributes.yaml @@ -0,0 +1,22 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: Compute node type with capability with an atribute of type list + +capability_types: + + tosca.capabilities.indigo.Endpoint: + derived_from: tosca.capabilities.Endpoint + attributes: + credential: + type: list + entry_schema: + type: tosca.datatypes.Credential + +node_types: + + tosca.nodes.ComputeWithCapWithAttr: + derived_from: tosca.nodes.Compute + capabilities: + endpoint: + type: tosca.capabilities.indigo.Endpoint + diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/custom_types/node_with_cap.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/custom_types/node_with_cap.yaml index 11e1b51..b17513f 100644 --- a/tosca2heat/tosca-parser/toscaparser/tests/data/custom_types/node_with_cap.yaml +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/custom_types/node_with_cap.yaml @@ -3,7 +3,8 @@ tosca_definitions_version: tosca_simple_yaml_1_0 description: > Node type that has a requirement of a capability with a defined value -node_types: +capability_types: + tosca.capabilities.SomeCap: derived_from: tosca.capabilities.Root properties: @@ -14,6 +15,8 @@ node_types: constraints: - equal: someval +node_types: + tosca.nodes.SomeNode: derived_from: tosca.nodes.Root requirements: diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/datatypes/test_datatype_portspec_add_req.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/datatypes/test_datatype_portspec_add_req.yaml new file mode 100644 index 0000000..f944927 --- /dev/null +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/datatypes/test_datatype_portspec_add_req.yaml @@ -0,0 +1,41 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: TOSCA test PortSpec Additional Requirement clauses + +node_types: + + MyNodeType: + derived_from: Root + properties: + test_port: + type: PortSpec + +topology_template: + + node_templates: + + # Test invalid source value below (default) specified range constraint + test_node2: + type: MyNodeType + properties: + test_port: + protocol: tcp + source: 0 + + # Test invalid source value over specified range + test_node3: + type: MyNodeType + properties: + test_port: + protocol: tcp + source: 65535 + source_range: [ 2, 65534 ] + + # Test invalid source value under specified range + test_node4: + type: MyNodeType + properties: + test_port: + protocol: tcp + source: 1 + source_range: [ 2, 65534 ] diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_get_attribute_with_index_error.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_get_attribute_with_index_error.yaml index 88a2721..7511999 100644 --- a/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_get_attribute_with_index_error.yaml +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_get_attribute_with_index_error.yaml @@ -1,7 +1,7 @@ tosca_definitions_version: tosca_simple_yaml_1_0 description: > - TOSCA template for testing get_attribute with a list attribute and an index + TOSCA template for testing get_attribute with an incorrect index imports: - ../custom_types/compute_with_attribute_list.yaml diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_get_attribute_with_nested_params.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_get_attribute_with_nested_params.yaml new file mode 100644 index 0000000..79e632c --- /dev/null +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_get_attribute_with_nested_params.yaml @@ -0,0 +1,23 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + TOSCA template for testing get_attribute with nested attributes + +imports: + - ../custom_types/compute_with_nested_atributes.yaml + +topology_template: + node_templates: + server: + type: tosca.nodes.ComputeWithCapWithAttr + capabilities: + endpoint: + properties: + port: 80 + interfaces: + Standard: + configure: + implementation: configure.sh + inputs: + ip_address: { get_attribute: [ SELF, endpoint, credential, 0, token ] } + diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_token.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_token.yaml new file mode 100644 index 0000000..495a930 --- /dev/null +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_token.yaml @@ -0,0 +1,15 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: Template for deploying a single server with token function. + +topology_template: + node_templates: + server: + type: tosca.nodes.Compute + + outputs: + url: + description: Get the first part of the ip + value: { token: [ get_attribute: [ server, public_address ], + '.' , + 0 ] } diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_token_invalid.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_token_invalid.yaml new file mode 100644 index 0000000..35ae2ff --- /dev/null +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/functions/test_token_invalid.yaml @@ -0,0 +1,17 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: Template for deploying a single server with invalid token function. + +topology_template: + outputs: + invalid_token_syntax_1: + description: test token with only two paremeters. + value: { token: ["some_string", "_"]} + + invalid_token_syntax_2: + description: test token with invalid string as third argument. + value: { token: ["some_string", "_", "1"]} + + invalid_token_syntax_3: + description: test token with invalid string as second argument. + value: { token: ["some_string", "aa", "1"]} diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/test_containers.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/test_containers.yaml new file mode 100644 index 0000000..ba1cc16 --- /dev/null +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/test_containers.yaml @@ -0,0 +1,44 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + TOSCA simple profile with mysql docker container. + +# Repositories to retrieve code artifacts from +repositories: + docker_hub: https://registry.hub.docker.com/ + +topology_template: + + inputs: + mysql_root_pwd: + type: string + description: Root password for MySQL. + + node_templates: + # The MYSQL container based on official MySQL image in Docker hub + mysql_container: + type: tosca.nodes.Container.Application + requirements: + - host: mysql_runtime + artifacts: + my_image: + file: mysql + type: tosca.artifacts.Deployment.Image.Container.Docker + repository: docker_hub + interfaces: + Standard: + create: + implementation: my_image + inputs: + MYSQL_ROOT_PASSWORD: { get_input: mysql_root_pwd } + + # The properties of the runtime to host the container + mysql_runtime: + type: tosca.nodes.Container.Runtime + capabilities: + host: + properties: + num_cpus: 1 + disk_size: 10 GB + mem_size: 2 MB + diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/test_credential_datatype.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/test_credential_datatype.yaml new file mode 100644 index 0000000..583ec82 --- /dev/null +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/test_credential_datatype.yaml @@ -0,0 +1,77 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: > + TOSCA simple profile with to demonstrate the usage of the + TOSCA Credential Data Type. + +imports: + - custom_types/wordpress.yaml + +relationship_types: + my.types.WordpressDbConnection: + derived_from: tosca.relationships.ConnectsTo + properties: + credential: + user: db_user + token: db_pwd + +topology_template: + node_templates: + wordpress: + type: tosca.nodes.WebApplication.WordPress + requirements: + - host: webserver + - database_endpoint: + node: mysql_database + relationship: my.types.WordpressDbConnection + + mysql_database: + type: tosca.nodes.Database + properties: + name: db_name + user: db_user + password: db_pwd + capabilities: + database_endpoint: + properties: + port: 3306 + requirements: + - host: + node: mysql_dbms + + mysql_dbms: + type: tosca.nodes.DBMS + properties: + root_password: db_root_pwd + port: 3306 + requirements: + - host: server + + webserver: + type: tosca.nodes.WebServer + properties: + admin_credential: + user: username + token: some_pass + requirements: + - host: server + + server: + type: tosca.nodes.Compute + capabilities: + host: + properties: + disk_size: 10 GB + num_cpus: 1 + mem_size: 4096 MB + os: + properties: + architecture: x86_64 + type: Linux + distribution: Ubuntu + version: 14.04 + + outputs: + website_url: + description: URL for Wordpress wiki. + value: { get_attribute: [server, private_address] } diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/test_invalid_input_defaults.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/test_invalid_input_defaults.yaml new file mode 100644 index 0000000..f8f4ae7 --- /dev/null +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/test_invalid_input_defaults.yaml @@ -0,0 +1,12 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: Test template with default not matching required type. + +topology_template: + inputs: + invalid_default: + type: integer + default: two + valid_default: + type: integer + default: 2 diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/test_repositories_definition.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/test_repositories_definition.yaml index 2145d8f..c2856c8 100644 --- a/tosca2heat/tosca-parser/toscaparser/tests/data/test_repositories_definition.yaml +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/test_repositories_definition.yaml @@ -4,9 +4,9 @@ repositories: some_repository: description: Some repo url: https://raw.githubusercontent.com/openstack/tosca-parser/master/toscaparser/tests/data/custom_types/ - namespace_uri: http://docs.oasis-open.org/tosca/ns/simple/yaml/1.0a - namespace_prefix: oasis_tosca - + credential: #type: Credential + token_type: basic_auth + token: myusername:mypassword imports: - some_import: file: compute_with_prop.yaml diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/topology_template/subsystem.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/topology_template/subsystem.yaml index 99d645b..b27e698 100644 --- a/tosca2heat/tosca-parser/toscaparser/tests/data/topology_template/subsystem.yaml +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/topology_template/subsystem.yaml @@ -12,15 +12,12 @@ topology_template: inputs: mq_server_ip: type: string - required: true description: IP address of the message queuing server to receive messages from. receiver_port: type: string - required: true description: Port to be used for receiving messages. my_cpus: type: integer - default: 2 description: Number of CPUs for the server. constraints: - valid_values: [ 1, 2, 4, 8 ] diff --git a/tosca2heat/tosca-parser/toscaparser/tests/data/tosca_repositories_test_definition.yaml b/tosca2heat/tosca-parser/toscaparser/tests/data/tosca_repositories_test_definition.yaml new file mode 100644 index 0000000..0001d06 --- /dev/null +++ b/tosca2heat/tosca-parser/toscaparser/tests/data/tosca_repositories_test_definition.yaml @@ -0,0 +1,26 @@ +tosca_definitions_version: tosca_simple_yaml_1_0 + +description: TOSCA simple profile with repositories validation and imports. + +repositories: + repo_code0: https://raw.githubusercontent.com/nandinivemula/intern + repo_code1: + description: My project's code Repository in github usercontent. + url: https://raw.githubusercontent.com/nandinivemula/intern/master + credential: #type: Credential + token_type: basic_auth + token: myusername:mypassword + + repo_code2: + description: My Project's code Repository in github. + url: https://github.com/nandinivemula/intern/master + credential: #type: Credential + token_type: basic_auth + token: myusername:mypassword + +imports: + - sample_import: + file: tosca_repository_import.yaml + repository: repo_code1 + namespace_uri: https://github.com/nandinivemula/intern + namespace_prefix: intern diff --git a/tosca2heat/tosca-parser/toscaparser/tests/test_datatypes.py b/tosca2heat/tosca-parser/toscaparser/tests/test_datatypes.py index 0e613b2..0a6cfe0 100644 --- a/tosca2heat/tosca-parser/toscaparser/tests/test_datatypes.py +++ b/tosca2heat/tosca-parser/toscaparser/tests/test_datatypes.py @@ -66,16 +66,21 @@ class DataTypeTest(TestCase): tosca.my.datatypes.TestLab: properties: - temperature: + humidity: + type: range + required: false + constraints: + - in_range: [-256, INFINITY] + temperature1: type: range required: false constraints: - in_range: [-256, UNBOUNDED] - humidity: + temperature2: type: range required: false constraints: - - in_range: [-256, INFINITY] + - in_range: [UNBOUNDED, 256] ''' custom_type_def = yamlparser.simple_parse(custom_type_schema) @@ -84,15 +89,6 @@ class DataTypeTest(TestCase): value = yamlparser.simple_parse(value_snippet) self.assertEqual(value, {}) - # TODO(Matt) - opened as bug 1555300 - # Need a test for PortSpec normative data type - # that tests the spec. requirement: "A valid PortSpec - # must have at least one of the following properties: - # target, target_range, source or source_range." - # TODO(Matt) - opened as bug 1555310 - # test PortSpec value for source and target - # against the source_range and target_range - # when specified. def test_built_in_datatype(self): value_snippet = ''' private_network: @@ -140,6 +136,31 @@ class DataTypeTest(TestCase): data = DataEntity('PortInfo', value.get('ethernet_port')) self.assertIsNotNone(data.validate()) + # Test normative PortSpec datatype's additional requirements + # TODO(Matt) - opened as bug 1555300 + # Need a test for PortSpec normative data type + # that tests the spec. requirement: "A valid PortSpec + # must have at least one of the following properties: + # target, target_range, source or source_range." + # TODO(Matt) - opened as bug 1555310 + # test PortSpec value for source and target + # against the source_range and target_range + # when specified. + def test_port_spec_addl_reqs(self): + value_snippet = ''' + test_port: + protocol: tcp + target: 65535 + target_range: [ 1, 65535 ] + source: 1 + source_range: [ 1, 65535 ] + + ''' + value = yamlparser.simple_parse(value_snippet) + data = DataEntity('tosca.datatypes.network.PortSpec', + value.get('test_port')) + self.assertIsNotNone(data.validate()) + def test_built_in_datatype_without_properties(self): value_snippet = ''' 2 @@ -365,6 +386,7 @@ class DataTypeTest(TestCase): value_snippet = ''' user_port: protocol: tcp + target: 1 target_range: [20000] ''' value = yamlparser.simple_parse(value_snippet) @@ -377,6 +399,7 @@ class DataTypeTest(TestCase): value_snippet = ''' user_port: protocol: tcp + target: 1 target_range: [20000, 3000] ''' value = yamlparser.simple_parse(value_snippet) @@ -400,9 +423,95 @@ class DataTypeTest(TestCase): def test_range_unbounded(self): value_snippet = ''' - temperature: [-100, 999999] + humidity: [-100, 100] + ''' + value = yamlparser.simple_parse(value_snippet) + data = DataEntity('tosca.my.datatypes.TestLab', + value, DataTypeTest.custom_type_def) + err = self.assertRaises(exception.InvalidSchemaError, + lambda: data.validate()) + self.assertEqual(_('The property "in_range" expects comparable values.' + ), + err.__str__()) + + def test_invalid_ranges_against_constraints(self): + # The TestLab range type has min=-256, max=UNBOUNDED + value_snippet = ''' + temperature1: [-257, 999999] + ''' + value = yamlparser.simple_parse(value_snippet) + data = DataEntity('tosca.my.datatypes.TestLab', value, + DataTypeTest.custom_type_def) + err = self.assertRaises(exception.ValidationError, data.validate) + self.assertEqual(_('The value "-257" of property "temperature1" is ' + 'out of range "(min:-256, max:UNBOUNDED)".'), + err.__str__()) + + value_snippet = ''' + temperature2: [-999999, 257] + ''' + value = yamlparser.simple_parse(value_snippet) + data = DataEntity('tosca.my.datatypes.TestLab', value, + DataTypeTest.custom_type_def) + err = self.assertRaises(exception.ValidationError, data.validate) + self.assertEqual(_('The value "257" of property "temperature2" is ' + 'out of range "(min:UNBOUNDED, max:256)".'), + err.__str__()) + + def test_valid_ranges_against_constraints(self): + + # The TestLab range type has max=UNBOUNDED + value_snippet = ''' + temperature1: [-255, 999999] ''' value = yamlparser.simple_parse(value_snippet) data = DataEntity('tosca.my.datatypes.TestLab', value, DataTypeTest.custom_type_def) self.assertIsNotNone(data.validate()) + + # The TestLab range type has min=UNBOUNDED + value_snippet = ''' + temperature2: [-999999, 255] + ''' + value = yamlparser.simple_parse(value_snippet) + data = DataEntity('tosca.my.datatypes.TestLab', value, + DataTypeTest.custom_type_def) + self.assertIsNotNone(data.validate()) + + def test_incorrect_field_in_datatype(self): + tpl_snippet = ''' + tosca_definitions_version: tosca_simple_yaml_1_0 + topology_template: + node_templates: + server: + type: tosca.nodes.Compute + + webserver: + type: tosca.nodes.WebServer + properties: + admin_credential: + user: username + token: some_pass + some_field: value + requirements: + - host: server + ''' + tpl = yamlparser.simple_parse(tpl_snippet) + err = self.assertRaises(exception.ValidationError, ToscaTemplate, + None, None, None, tpl) + self.assertIn(_('The pre-parsed input failed validation with the ' + 'following error(s): \n\n\tUnknownFieldError: Data ' + 'value of type "tosca.datatypes.Credential" contains' + ' unknown field "some_field". Refer to the definition' + ' to verify valid values'), err.__str__()) + + def test_functions_datatype(self): + value_snippet = ''' + admin_credential: + user: username + token: { get_input: password } + ''' + value = yamlparser.simple_parse(value_snippet) + data = DataEntity('tosca.datatypes.Credential', + value.get('admin_credential')) + self.assertIsNotNone(data.validate()) diff --git a/tosca2heat/tosca-parser/toscaparser/tests/test_functions.py b/tosca2heat/tosca-parser/toscaparser/tests/test_functions.py index 2a6225d..4d063e5 100644 --- a/tosca2heat/tosca-parser/toscaparser/tests/test_functions.py +++ b/tosca2heat/tosca-parser/toscaparser/tests/test_functions.py @@ -201,8 +201,9 @@ class GetAttributeTest(TestCase): website_url_output.value.attribute_name) def test_get_attribute_invalid_args(self): - expected_msg = _('Expected arguments: "node-template-name", ' - '"attribute-name"') + expected_msg = _('Illegal arguments for function "get_attribute".' + ' Expected arguments: "node-template-name", ' + '"req-or-cap"(optional), "property name"') err = self.assertRaises(ValueError, functions.get_function, None, None, {'get_attribute': []}) @@ -211,10 +212,6 @@ class GetAttributeTest(TestCase): functions.get_function, None, None, {'get_attribute': ['x']}) self.assertIn(expected_msg, six.text_type(err)) - err = self.assertRaises(ValueError, - functions.get_function, None, None, - {'get_attribute': ['x', 'y', 'z', 'k']}) - self.assertIn(expected_msg, six.text_type(err)) def test_get_attribute_unknown_node_template_name(self): self.assertRaises( @@ -280,7 +277,7 @@ class GetAttributeTest(TestCase): exception.ExceptionCollector.assertExceptionMessage( ValueError, _('Illegal arguments for function "get_attribute". ' - 'Expected arguments: "node-template-name", "attribute-name"')) + 'Unexpected attribute/index value "0"')) def test_get_attribute_source_target_keywords(self): tosca_tpl = os.path.join( @@ -300,6 +297,10 @@ class GetAttributeTest(TestCase): source_port = operation.inputs['source_port'] self.assertTrue(isinstance(source_port, functions.GetAttribute)) + def test_get_attribute_with_nested_params(self): + self._load_template( + 'functions/test_get_attribute_with_nested_params.yaml') + class ConcatTest(TestCase): @@ -322,3 +323,34 @@ class ConcatTest(TestCase): ValueError, _('Invalid arguments for function "concat". Expected at least ' 'one arguments.')) + + +class TokenTest(TestCase): + + def _load_template(self, filename): + return ToscaTemplate(os.path.join( + os.path.dirname(os.path.abspath(__file__)), + filename)) + + def test_validate_token(self): + tosca = self._load_template("data/functions/test_token.yaml") + server_url_output = [ + output for output in tosca.outputs if output.name == 'url'][0] + func = functions.get_function(self, tosca.outputs, + server_url_output.value) + self.assertIsInstance(func, functions.Token) + + self.assertRaises(exception.ValidationError, self._load_template, + 'data/functions/test_token_invalid.yaml') + exception.ExceptionCollector.assertExceptionMessage( + ValueError, + _('Invalid arguments for function "token". Expected at least ' + 'three arguments.')) + exception.ExceptionCollector.assertExceptionMessage( + ValueError, + _('Invalid arguments for function "token". Expected ' + 'integer value as third argument.')) + exception.ExceptionCollector.assertExceptionMessage( + ValueError, + _('Invalid arguments for function "token". Expected ' + 'single char value as second argument.')) diff --git a/tosca2heat/tosca-parser/toscaparser/tests/test_toscatpl.py b/tosca2heat/tosca-parser/toscaparser/tests/test_toscatpl.py index ac55059..2488b65 100644 --- a/tosca2heat/tosca-parser/toscaparser/tests/test_toscatpl.py +++ b/tosca2heat/tosca-parser/toscaparser/tests/test_toscatpl.py @@ -12,10 +12,10 @@ import os import six - from toscaparser.common import exception import toscaparser.elements.interfaces as ifaces from toscaparser.elements.nodetype import NodeType +from toscaparser.elements.portspectype import PortSpec from toscaparser.functions import GetInput from toscaparser.functions import GetProperty from toscaparser.nodetemplate import NodeTemplate @@ -26,16 +26,17 @@ import toscaparser.utils.yamlparser class ToscaTemplateTest(TestCase): - '''TOSCA template.''' tosca_tpl = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data/tosca_single_instance_wordpress.yaml") tosca = ToscaTemplate(tosca_tpl) - tosca_elk_tpl = os.path.join( os.path.dirname(os.path.abspath(__file__)), "data/tosca_elk.yaml") + tosca_repo_tpl = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/tosca_repositories_test_definition.yaml") def test_version(self): self.assertEqual(self.tosca.version, "tosca_simple_yaml_1_0") @@ -251,6 +252,18 @@ class ToscaTemplateTest(TestCase): expected_hosts, sorted([v.type for v in node_tpl.relationships.values()])) + def test_repositories(self): + template = ToscaTemplate(self.tosca_repo_tpl) + self.assertEqual( + ['repo_code0', 'repo_code1', 'repo_code2'], + sorted([input.name for input in template.repositories])) + + input_name = "repo_code2" + expected_url = "https://github.com/nandinivemula/intern/master" + for input in template.repositories: + if input.name == input_name: + self.assertEqual(input.url, expected_url) + def test_template_macro(self): template = ToscaTemplate(self.tosca_elk_tpl) for node_tpl in template.nodetemplates: @@ -747,3 +760,46 @@ class ToscaTemplateTest(TestCase): self.assertTrue(rel.is_derived_from("tosca.relationships.Root")) self.assertEqual(len(rel.interfaces), 1) self.assertEqual(rel.interfaces[0].type, "Configure") + + def test_various_portspec_errors(self): + tosca_tpl = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/datatypes/test_datatype_portspec_add_req.yaml") + self.assertRaises(exception.ValidationError, ToscaTemplate, tosca_tpl, + None) + + # TODO(TBD) find way to reuse error messages from constraints.py + msg = (_('The value "%(pvalue)s" of property "%(pname)s" is out of ' + 'range "(min:%(vmin)s, max:%(vmax)s)".') % + dict(pname=PortSpec.SOURCE, + pvalue='0', + vmin='1', + vmax='65535')) + exception.ExceptionCollector.assertExceptionMessage( + exception.ValidationError, msg) + + # Test value below range min. + msg = (_('The value "%(pvalue)s" of property "%(pname)s" is out of ' + 'range "(min:%(vmin)s, max:%(vmax)s)".') % + dict(pname=PortSpec.SOURCE, + pvalue='1', + vmin='2', + vmax='65534')) + exception.ExceptionCollector.assertExceptionMessage( + exception.RangeValueError, msg) + + # Test value above range max. + msg = (_('The value "%(pvalue)s" of property "%(pname)s" is out of ' + 'range "(min:%(vmin)s, max:%(vmax)s)".') % + dict(pname=PortSpec.SOURCE, + pvalue='65535', + vmin='2', + vmax='65534')) + exception.ExceptionCollector.assertExceptionMessage( + exception.RangeValueError, msg) + + def test_containers(self): + tosca_tpl = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/test_containers.yaml") + ToscaTemplate(tosca_tpl) diff --git a/tosca2heat/tosca-parser/toscaparser/tests/test_toscatplvalidation.py b/tosca2heat/tosca-parser/toscaparser/tests/test_toscatplvalidation.py index 81a1a6c..986d9e4 100644 --- a/tosca2heat/tosca-parser/toscaparser/tests/test_toscatplvalidation.py +++ b/tosca2heat/tosca-parser/toscaparser/tests/test_toscatplvalidation.py @@ -20,12 +20,12 @@ from toscaparser.parameters import Input from toscaparser.parameters import Output from toscaparser.policy import Policy from toscaparser.relationship_template import RelationshipTemplate +from toscaparser.repositories import Repository from toscaparser.tests.base import TestCase from toscaparser.topology_template import TopologyTemplate from toscaparser.tosca_template import ToscaTemplate from toscaparser.triggers import Triggers from toscaparser.utils.gettextutils import _ - import toscaparser.utils.yamlparser @@ -105,7 +105,9 @@ class ToscaTemplateValidationTest(TestCase): type: integer description: Number of CPUs for the server. constraint: - - valid_values: [ 1, 2, 4, 8 ] + - valid_values: [ 1, 2, 4 ] + required: yes + status: supported ''' inputs = (toscaparser.utils.yamlparser. simple_parse(tpl_snippet)['inputs']) @@ -335,6 +337,113 @@ heat-translator/master/translator/tests/data/custom_types/wordpress.yaml 'to verify valid values.'), err.__str__()) + def _repo_content(self, path): + repositories = path['repositories'] + reposit = [] + for name, val in repositories.items(): + reposits = Repository(name, val) + reposit.append(reposits) + return reposit + + def test_repositories(self): + tpl_snippet = ''' + repositories: + repo_code0: https://raw.githubusercontent.com/nandinivemula/intern + repo_code1: + description: My project's code Repository in github usercontent. + url: https://github.com/nandinivemula/intern + credential: + user: nandini + password: tcs@12345 + repo_code2: + description: My Project's code Repository in github. + url: https://github.com/nandinivemula/intern + credential: + user: xyzw + password: xyz@123 + ''' + tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet)) + repoobject = self._repo_content(tpl) + actualrepo_names = [] + for repo in repoobject: + repos = repo.name + actualrepo_names.append(repos) + reposname = list(tpl.values()) + reposnames = reposname[0] + expected_reponames = list(reposnames.keys()) + self.assertEqual(expected_reponames, actualrepo_names) + + def test_repositories_with_missing_required_field(self): + tpl_snippet = ''' + repositories: + repo_code0: https://raw.githubusercontent.com/nandinivemula/intern + repo_code1: + description: My project's code Repository in github usercontent. + credential: + user: nandini + password: tcs@12345 + repo_code2: + description: My Project's code Repository in github. + url: https://github.com/nandinivemula/intern + credential: + user: xyzw + password: xyz@123 + ''' + tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet)) + err = self.assertRaises(exception.MissingRequiredFieldError, + self._repo_content, tpl) + expectedmessage = _('Repository "repo_code1" is missing ' + 'required field "url".') + self.assertEqual(expectedmessage, err.__str__()) + + def test_repositories_with_unknown_field(self): + tpl_snippet = ''' + repositories: + repo_code0: https://raw.githubusercontent.com/nandinivemula/intern + repo_code1: + description: My project's code Repository in github usercontent. + url: https://github.com/nandinivemula/intern + credential: + user: nandini + password: tcs@12345 + repo_code2: + descripton: My Project's code Repository in github. + url: https://github.com/nandinivemula/intern + credential: + user: xyzw + password: xyz@123 + ''' + tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet)) + err = self.assertRaises(exception.UnknownFieldError, + self._repo_content, tpl) + expectedmessage = _('repositories "repo_code2" contains unknown field' + ' "descripton". Refer to the definition to verify' + ' valid values.') + self.assertEqual(expectedmessage, err.__str__()) + + def test_repositories_with_invalid_url(self): + tpl_snippet = ''' + repositories: + repo_code0: https://raw.githubusercontent.com/nandinivemula/intern + repo_code1: + description: My project's code Repository in github usercontent. + url: h + credential: + user: nandini + password: tcs@12345 + repo_code2: + description: My Project's code Repository in github. + url: https://github.com/nandinivemula/intern + credential: + user: xyzw + password: xyz@123 + ''' + tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet)) + err = self.assertRaises(exception.URLException, + self._repo_content, tpl) + expectedmessage = _('repsositories "repo_code1" Invalid Url') + self.assertEqual(expectedmessage, err.__str__()) + def test_groups(self): tpl_snippet = ''' node_templates: @@ -1379,3 +1488,35 @@ heat-translator/master/translator/tests/data/custom_types/wordpress.yaml exception.MissingRequiredFieldError, lambda: Policy(name, policies[name], None, None)) self.assertEqual(expectedmessage, err.__str__()) + + def test_credential_datatype(self): + tosca_tpl = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/test_credential_datatype.yaml") + self.assertIsNotNone(ToscaTemplate(tosca_tpl)) + + def test_invalid_default_value(self): + tpl_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "data/test_invalid_input_defaults.yaml") + self.assertRaises(exception.ValidationError, ToscaTemplate, tpl_path) + exception.ExceptionCollector.assertExceptionMessage( + ValueError, _('"two" is not an integer.')) + + def test_invalid_capability(self): + tpl_snippet = ''' + node_templates: + server: + type: tosca.nodes.Compute + capabilities: + oss: + properties: + architecture: x86_64 + ''' + tpl = (toscaparser.utils.yamlparser.simple_parse(tpl_snippet)) + err = self.assertRaises(exception.UnknownFieldError, + TopologyTemplate, tpl, None) + expectedmessage = _('"capabilities" of template "server" contains ' + 'unknown field "oss". Refer to the definition ' + 'to verify valid values.') + self.assertEqual(expectedmessage, err.__str__()) diff --git a/tosca2heat/tosca-parser/toscaparser/topology_template.py b/tosca2heat/tosca-parser/toscaparser/topology_template.py index d51512a..6cf4f31 100644 --- a/tosca2heat/tosca-parser/toscaparser/topology_template.py +++ b/tosca2heat/tosca-parser/toscaparser/topology_template.py @@ -66,6 +66,10 @@ class TopologyTemplate(object): input = Input(name, attrs) if self.parsed_params and name in self.parsed_params: input.validate(self.parsed_params[name]) + else: + default = input.default + if default: + input.validate(default) inputs.append(input) return inputs diff --git a/tosca2heat/tosca-parser/toscaparser/tosca_template.py b/tosca2heat/tosca-parser/toscaparser/tosca_template.py index 2ab2581..01e6c73 100644 --- a/tosca2heat/tosca-parser/toscaparser/tosca_template.py +++ b/tosca2heat/tosca-parser/toscaparser/tosca_template.py @@ -23,6 +23,7 @@ from toscaparser.elements.entity_type import update_definitions from toscaparser.extensions.exttools import ExtTools import toscaparser.imports from toscaparser.prereq.csar import CSAR +from toscaparser.repositories import Repository from toscaparser.topology_template import TopologyTemplate from toscaparser.tpl_relationship_graph import ToscaGraph from toscaparser.utils.gettextutils import _ @@ -95,6 +96,7 @@ class ToscaTemplate(object): self.relationship_types = self._tpl_relationship_types() self.description = self._tpl_description() self.topology_template = self._topology_template() + self.repositories = self._tpl_repositories() if self.topology_template.tpl: self.inputs = self._inputs() self.relationship_templates = self._relationship_templates() @@ -134,6 +136,15 @@ class ToscaTemplate(object): def _tpl_imports(self): return self.tpl.get(IMPORTS) + def _tpl_repositories(self): + repositories = self.tpl.get(REPOSITORIES) + reposit = [] + if repositories: + for name, val in repositories.items(): + reposits = Repository(name, val) + reposit.append(reposits) + return reposit + def _tpl_relationship_types(self): return self._get_custom_types(RELATIONSHIP_TYPES) diff --git a/tosca2heat/tosca-parser/toscaparser/utils/validateutils.py b/tosca2heat/tosca-parser/toscaparser/utils/validateutils.py index f9b9fc5..43e14d6 100644 --- a/tosca2heat/tosca-parser/toscaparser/utils/validateutils.py +++ b/tosca2heat/tosca-parser/toscaparser/utils/validateutils.py @@ -17,14 +17,20 @@ import numbers import re import six +# from toscaparser.elements import constraints from toscaparser.common.exception import ExceptionCollector from toscaparser.common.exception import InvalidTOSCAVersionPropertyException +from toscaparser.common.exception import RangeValueError from toscaparser.utils.gettextutils import _ + log = logging.getLogger('tosca') +RANGE_UNBOUNDED = 'UNBOUNDED' + def str_to_num(value): '''Convert a string representation of a number into a numeric type.''' + # TODO(TBD) we should not allow numeric values in, input should be str if isinstance(value, numbers.Number): return value try: @@ -33,8 +39,11 @@ def str_to_num(value): return float(value) -def validate_number(value): - return str_to_num(value) +def validate_numeric(value): + if not isinstance(value, numbers.Number): + ExceptionCollector.appendException( + ValueError(_('"%s" is not a numeric.') % value)) + return value def validate_integer(value): @@ -51,7 +60,7 @@ def validate_float(value): if not isinstance(value, float): ExceptionCollector.appendException( ValueError(_('"%s" is not a float.') % value)) - return validate_number(value) + return value def validate_string(value): @@ -68,15 +77,53 @@ def validate_list(value): return value -def validate_range(value): - validate_list(value) - if isinstance(value, list): - if len(value) != 2 or not (value[0] <= value[1]): +def validate_range(range): + # list class check + validate_list(range) + # validate range list has a min and max + if len(range) != 2: + ExceptionCollector.appendException( + ValueError(_('"%s" is not a valid range.') % range)) + # validate min and max are numerics or the keyword UNBOUNDED + min_test = max_test = False + if not range[0] == RANGE_UNBOUNDED: + min = validate_numeric(range[0]) + else: + min_test = True + if not range[1] == RANGE_UNBOUNDED: + max = validate_numeric(range[1]) + else: + max_test = True + # validate the max > min (account for UNBOUNDED) + if not min_test and not max_test: + # Note: min == max is allowed + if min > max: + ExceptionCollector.appendException( + ValueError(_('"%s" is not a valid range.') % range)) + + return range + + +def validate_value_in_range(value, range, prop_name): + validate_numeric(value) + validate_range(range) + + # Note: value is valid if equal to min + if range[0] != RANGE_UNBOUNDED: + if value < range[0]: + ExceptionCollector.appendException( + RangeValueError(pname=prop_name, + pvalue=value, + vmin=range[0], + vmax=range[1])) + # Note: value is valid if equal to max + if range[1] != RANGE_UNBOUNDED: + if value > range[1]: ExceptionCollector.appendException( - ValueError(_('"%s" is not a valid range.') % value)) - validate_integer(value[0]) - if not value[1] == "UNBOUNDED": - validate_integer(value[1]) + RangeValueError(pname=prop_name, + pvalue=value, + vmin=range[0], + vmax=range[1])) return value |