Synchronise the openstack bugs

When run unittests through tox, some test cases are always error,
the errors are already done in openstack community, so it's
necessary to synchronise the fixes.

Change-Id: Ib29078e6cc138a474e89c6a2cc90ad7a1db1bb46
JIRA: PARSER-63
Signed-off-by: shangxdy <shang.xiaodong@zte.com.cn>

6 files changed, 110 insertions, 40 deletions

diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py b/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py
index e0cdbb6..d42cdc8 100644
--- a/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py
+++ b/tosca2heat/heat-translator/translator/hot/tosca/tests/test_tosca_compute.py
@@ -16,7 +16,6 @@ from mock import patch
 
 from toscaparser.nodetemplate import NodeTemplate
 from toscaparser.tests.base import TestCase
-from toscaparser.utils.gettextutils import _
 import toscaparser.utils.yamlparser
 from translator.hot.tosca.tosca_compute import ToscaCompute
 
@@ -27,22 +26,12 @@ class ToscaComputeTest(TestCase):
         nodetemplates = (toscaparser.utils.yamlparser.
                          simple_parse(tpl_snippet)['node_templates'])
         name = list(nodetemplates.keys())[0]
-        try:
-            nodetemplate = NodeTemplate(name, nodetemplates)
-            nodetemplate.validate()
-            toscacompute = ToscaCompute(nodetemplate)
-            toscacompute.handle_properties()
-            if not self._compare_properties(toscacompute.properties,
-                                            expectedprops):
-                raise Exception(_("Hot Properties are not"
-                                  " same as expected properties"))
-        except Exception:
-            # for time being rethrowing. Will be handled future based
-            # on new development in Glance and Graffiti
-            raise
+        nodetemplate = NodeTemplate(name, nodetemplates)
+        nodetemplate.validate()
+        toscacompute = ToscaCompute(nodetemplate)
+        toscacompute.handle_properties()
 
-    def _compare_properties(self, hotprops, expectedprops):
-        return all(item in hotprops.items() for item in expectedprops.items())
+        self.assertDictEqual(expectedprops, toscacompute.properties)
 
     def test_node_compute_with_host_and_os_capabilities(self):
         tpl_snippet = '''
@@ -63,7 +52,8 @@ class ToscaComputeTest(TestCase):
                   version: 18.0
         '''
         expectedprops = {'flavor': 'm1.large',
-                         'image': 'fedora-amd64-heat-config'}
+                         'image': 'fedora-amd64-heat-config',
+                         'user_data_format': 'SOFTWARE_CONFIG'}
         self._tosca_compute_test(
             tpl_snippet,
             expectedprops)
@@ -82,7 +72,8 @@ class ToscaComputeTest(TestCase):
               #left intentionally
         '''
         expectedprops = {'flavor': 'm1.large',
-                         'image': None}
+                         'image': None,
+                         'user_data_format': 'SOFTWARE_CONFIG'}
         self._tosca_compute_test(
             tpl_snippet,
             expectedprops)
@@ -101,7 +92,8 @@ class ToscaComputeTest(TestCase):
                   version: 18.0
         '''
         expectedprops = {'flavor': None,
-                         'image': 'fedora-amd64-heat-config'}
+                         'image': 'fedora-amd64-heat-config',
+                         'user_data_format': 'SOFTWARE_CONFIG'}
         self._tosca_compute_test(
             tpl_snippet,
             expectedprops)
@@ -117,7 +109,8 @@ class ToscaComputeTest(TestCase):
               #left intentionally
         '''
         expectedprops = {'flavor': None,
-                         'image': None}
+                         'image': None,
+                         'user_data_format': 'SOFTWARE_CONFIG'}
         self._tosca_compute_test(
             tpl_snippet,
             expectedprops)
@@ -129,7 +122,8 @@ class ToscaComputeTest(TestCase):
             type: tosca.nodes.Compute
         '''
         expectedprops = {'flavor': None,
-                         'image': None}
+                         'image': None,
+                         'user_data_format': 'SOFTWARE_CONFIG'}
         self._tosca_compute_test(
             tpl_snippet,
             expectedprops)
@@ -144,7 +138,9 @@ class ToscaComputeTest(TestCase):
                 properties:
                 #left intentionally
         '''
-        expectedprops = {'flavor': 'm1.nano'}
+        expectedprops = {'flavor': None,
+                         'image': None,
+                         'user_data_format': 'SOFTWARE_CONFIG'}
         self._tosca_compute_test(
             tpl_snippet,
             expectedprops)
@@ -160,7 +156,9 @@ class ToscaComputeTest(TestCase):
                   num_cpus: 4
                   mem_size: 4 GB
         '''
-        expectedprops = {'flavor': 'm1.large'}
+        expectedprops = {'flavor': 'm1.large',
+                         'image': None,
+                         'user_data_format': 'SOFTWARE_CONFIG'}
         self._tosca_compute_test(
             tpl_snippet,
             expectedprops)
@@ -176,7 +174,9 @@ class ToscaComputeTest(TestCase):
                   num_cpus: 4
                   disk_size: 10 GB
         '''
-        expectedprops = {'flavor': 'm1.large'}
+        expectedprops = {'flavor': 'm1.large',
+                         'image': None,
+                         'user_data_format': 'SOFTWARE_CONFIG'}
         self._tosca_compute_test(
             tpl_snippet,
             expectedprops)
@@ -191,7 +191,9 @@ class ToscaComputeTest(TestCase):
                 properties:
                   num_cpus: 4
         '''
-        expectedprops = {'flavor': 'm1.large'}
+        expectedprops = {'flavor': 'm1.large',
+                         'image': None,
+                         'user_data_format': 'SOFTWARE_CONFIG'}
         self._tosca_compute_test(
             tpl_snippet,
             expectedprops)
@@ -249,7 +251,9 @@ class ToscaComputeTest(TestCase):
                 json.dumps(mock_flavor_content)
             mock_post.return_value = mock_ks_response
             mock_get.return_value = mock_nova_response
-            expectedprops = {'flavor': 'm1.mock_flavor'}
+            expectedprops = {'flavor': 'm1.mock_flavor',
+                             'image': None,
+                             'user_data_format': 'SOFTWARE_CONFIG'}
             self._tosca_compute_test(
                 tpl_snippet,
                 expectedprops)
@@ -279,8 +283,8 @@ class ToscaComputeTest(TestCase):
             mock_ks_content = {}
             mock_ks_response.content = json.dumps(mock_ks_content)
             expectedprops = {'flavor': 'm1.small',
-                             'user_data_format': 'SOFTWARE_CONFIG',
-                             'image': None}
+                             'image': None,
+                             'user_data_format': 'SOFTWARE_CONFIG'}
             self._tosca_compute_test(
                 tpl_snippet,
                 expectedprops)
diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py
index d4b2f44..924ff9d 100644
--- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py
+++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage.py
@@ -67,5 +67,5 @@ class ToscaBlockStorage(HotResource):
         # attribute for the matching resource.  Unless there is additional
         # runtime support, this should be a one to one mapping.
         if attribute == 'volume_id':
-            attr['get_resource'] = args[0]
+            attr['get_resource'] = self.name
         return attr
diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py
index 715d5b3..2242c2d 100644
--- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py
+++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_block_storage_attachment.py
@@ -44,5 +44,10 @@ class ToscaBlockStorageAttachment(HotResource):
         if 'location' in self.properties:
             self.properties['mountpoint'] = self.properties.pop('location')
 
+        # TOSCA type can have a device name specified,
+        # this is unsupported by Heat
+        if 'device' in self.properties:
+            self.properties.pop('device')
+
     def handle_life_cycle(self):
-        pass
+        return None, None, None
diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py
index e2ac130..b8ad83c 100644
--- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py
+++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_compute.py
@@ -84,6 +84,14 @@ class ToscaCompute(HotResource):
                       ('architecture', 'distribution', 'type', 'version')
     toscatype = 'tosca.nodes.Compute'
 
+    ALLOWED_NOVA_SERVER_PROPS = \
+        ('admin_pass', 'availability_zone', 'block_device_mapping',
+         'block_device_mapping_v2', 'config_drive', 'diskConfig', 'flavor',
+         'flavor_update_policy', 'image', 'image_update_policy', 'key_name',
+         'metadata', 'name', 'networks', 'personality', 'reservation_id',
+         'scheduler_hints', 'security_groups', 'software_config_transport',
+         'user_data', 'user_data_format', 'user_data_update_policy')
+
     def __init__(self, nodetemplate):
         super(ToscaCompute, self).__init__(nodetemplate,
                                            type='OS::Nova::Server')
@@ -98,7 +106,8 @@ class ToscaCompute(HotResource):
         self.properties['user_data_format'] = 'SOFTWARE_CONFIG'
         tosca_props = self.get_tosca_props()
         for key, value in tosca_props.items():
-            self.properties[key] = value
+            if key in self.ALLOWED_NOVA_SERVER_PROPS:
+                self.properties[key] = value
 
     # To be reorganized later based on new development in Glance and Graffiti
     def translate_compute_flavor_and_image(self,
@@ -112,11 +121,17 @@ class ToscaCompute(HotResource):
         if host_capability:
             for prop in host_capability.get_properties_objects():
                 host_cap_props[prop.name] = prop.value
-            flavor = self._best_flavor(host_cap_props)
+            # if HOST properties are not specified, we should not attempt to
+            # find best match of flavor
+            if host_cap_props:
+                flavor = self._best_flavor(host_cap_props)
         if os_capability:
             for prop in os_capability.get_properties_objects():
                 os_cap_props[prop.name] = prop.value
-            image = self._best_image(os_cap_props)
+            # if OS properties are not specified, we should not attempt to
+            # find best match of image
+            if os_cap_props:
+                image = self._best_image(os_cap_props)
         hot_properties['flavor'] = flavor
         hot_properties['image'] = image
         return hot_properties
@@ -153,6 +168,48 @@ class ToscaCompute(HotResource):
             return None
         return flavor_dict
 
+    def _populate_image_dict(self):
+        '''Populates and returns the images dict using Glance ReST API'''
+        images_dict = {}
+        try:
+            access_dict = translator.common.utils.get_ks_access_dict()
+            access_token = translator.common.utils.get_token_id(access_dict)
+            if access_token is None:
+                return None
+            glance_url = translator.common.utils.get_url_for(access_dict,
+                                                             'image')
+            if not glance_url:
+                return None
+            glance_response = requests.get(glance_url + '/v2/images',
+                                           headers={'X-Auth-Token':
+                                                    access_token})
+            if glance_response.status_code != 200:
+                return None
+            images = json.loads(glance_response.content)["images"]
+            for image in images:
+                image_resp = requests.get(glance_url + '/v2/images/' +
+                                          image["id"],
+                                          headers={'X-Auth-Token':
+                                                   access_token})
+                if image_resp.status_code != 200:
+                    continue
+                metadata = ["architecture", "type", "distribution", "version"]
+                image_data = json.loads(image_resp.content)
+                if any(key in image_data.keys() for key in metadata):
+                    images_dict[image_data["name"]] = dict()
+                    for key in metadata:
+                        if key in image_data.keys():
+                            images_dict[image_data["name"]][key] = \
+                                image_data[key]
+                else:
+                    continue
+
+        except Exception as e:
+            # Handles any exception coming from openstack
+            log.warn(_('Choosing predefined flavors since received '
+                       'Openstack Exception: %s') % str(e))
+        return images_dict
+
     def _best_flavor(self, properties):
         log.info(_('Choosing the best flavor for given attributes.'))
         # Check whether user exported all required environment variables.
@@ -202,26 +259,32 @@ class ToscaCompute(HotResource):
             return None
 
     def _best_image(self, properties):
-        match_all = IMAGES.keys()
+        # Check whether user exported all required environment variables.
+        images = IMAGES
+        if translator.common.utils.check_for_env_variables():
+            resp = self._populate_image_dict()
+            if len(resp.keys()) > 0:
+                images = resp
+        match_all = images.keys()
         architecture = properties.get(self.ARCHITECTURE)
         if architecture is None:
             self._log_compute_msg(self.ARCHITECTURE, 'image')
-        match_arch = self._match_images(match_all, IMAGES,
+        match_arch = self._match_images(match_all, images,
                                         self.ARCHITECTURE, architecture)
         type = properties.get(self.TYPE)
         if type is None:
             self._log_compute_msg(self.TYPE, 'image')
-        match_type = self._match_images(match_arch, IMAGES, self.TYPE, type)
+        match_type = self._match_images(match_arch, images, self.TYPE, type)
         distribution = properties.get(self.DISTRIBUTION)
         if distribution is None:
             self._log_compute_msg(self.DISTRIBUTION, 'image')
-        match_distribution = self._match_images(match_type, IMAGES,
+        match_distribution = self._match_images(match_type, images,
                                                 self.DISTRIBUTION,
                                                 distribution)
         version = properties.get(self.VERSION)
         if version is None:
             self._log_compute_msg(self.VERSION, 'image')
-        match_version = self._match_images(match_distribution, IMAGES,
+        match_version = self._match_images(match_distribution, images,
                                            self.VERSION, version)
 
         if len(match_version):
diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_antilocate.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_antilocate.py
index b7aca4a..37bfed6 100644
--- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_antilocate.py
+++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_antilocate.py
@@ -1,4 +1,3 @@
-#
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at
diff --git a/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_colocate.py b/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_colocate.py
index 1a108bf..778f97e 100644
--- a/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_colocate.py
+++ b/tosca2heat/heat-translator/translator/hot/tosca/tosca_policies_colocate.py
@@ -1,4 +1,3 @@
-#
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
 # a copy of the License at