Added logging when a heat stack fails.

Added the stack resource reason to the error logs for each
resource who's status is 'CREATE_FAILED'. All resons will be
output to debug.

JIRA: SNAPS-190

Change-Id: Ieb1cdb2089eb6e1c1a7c96c143b82af1b7a9efb7
Signed-off-by: spisarski <s.pisarski@cablelabs.com>

6 files changed, 190 insertions, 51 deletions

diff --git a/docs/how-to-use/IntegrationTests.rst b/docs/how-to-use/IntegrationTests.rst
index 4772357..efa645d 100644
--- a/docs/how-to-use/IntegrationTests.rst
+++ b/docs/how-to-use/IntegrationTests.rst
@@ -362,22 +362,22 @@ create_stack_tests.py - CreateStackSuccessTests
 +---------------------------------------+---------------+-----------------------------------------------------------+
 | Test Name                             |   Heat API    | Description                                               |
 +=======================================+===============+===========================================================+
-| test_create_stack_template_file       | 1             | Ensures that a Heat stack can be created with a file-based|
+| test_create_stack_template_file       | 1-3           | Ensures that a Heat stack can be created with a file-based|
 |                                       |               | Heat template file                                        |
 +---------------------------------------+---------------+-----------------------------------------------------------+
-| test_create_stack_template_dict       | 1             | Ensures that a Heat stack can be created with a dictionary|
+| test_create_stack_template_dict       | 1-3           | Ensures that a Heat stack can be created with a dictionary|
 |                                       |               | Heat template                                             |
 +---------------------------------------+---------------+-----------------------------------------------------------+
-| test_create_delete_stack              | 1             | Ensures that a Heat stack can be created and deleted      |
+| test_create_delete_stack              | 1-3           | Ensures that a Heat stack can be created and deleted      |
 |                                       |               | while having clean() called 2x without an exception       |
 +---------------------------------------+---------------+-----------------------------------------------------------+
-| test_create_same_stack                | 1             | Ensures that a Heat stack with the same name cannot be    |
+| test_create_same_stack                | 1-3           | Ensures that a Heat stack with the same name cannot be    |
 |                                       |               | created 2x                                                |
 +---------------------------------------+---------------+-----------------------------------------------------------+
-| test_retrieve_network_creators        | 1             | Ensures that an OpenStackHeatStack instance can return an |
+| test_retrieve_network_creators        | 1-3           | Ensures that an OpenStackHeatStack instance can return an |
 |                                       |               | OpenStackNetwork instance configured as deployed          |
 +---------------------------------------+---------------+-----------------------------------------------------------+
-| test_retrieve_vm_inst_creators        | 1             | Ensures that an OpenStackHeatStack instance can return an |
+| test_retrieve_vm_inst_creators        | 1-3           | Ensures that an OpenStackHeatStack instance can return an |
 |                                       |               | OpenStackVmInstance instance configured as deployed       |
 +---------------------------------------+---------------+-----------------------------------------------------------+
 
@@ -387,11 +387,11 @@ create_stack_tests.py - CreateStackVolumeTests
 +---------------------------------------+---------------+-----------------------------------------------------------+
 | Test Name                             |   Heat API    | Description                                               |
 +=======================================+===============+===========================================================+
-| test_retrieve_volume_creator          | 1             | Ensures that an OpenStackHeatStack instance can return a  |
+| test_retrieve_volume_creator          | 1-3           | Ensures that an OpenStackHeatStack instance can return a  |
 |                                       |               | OpenStackVolume instance that it was responsible for      |
 |                                       |               | deploying                                                 |
 +---------------------------------------+---------------+-----------------------------------------------------------+
-| test_retrieve_volume_type_creator     | 1             | Ensures that an OpenStackHeatStack instance can return a  |
+| test_retrieve_volume_type_creator     | 1-3           | Ensures that an OpenStackHeatStack instance can return a  |
 |                                       |               | OpenStackVolumeType instance that it was responsible for  |
 |                                       |               | deploying                                                 |
 +---------------------------------------+---------------+-----------------------------------------------------------+
@@ -402,7 +402,7 @@ create_stack_tests.py - CreateStackFlavorTests
 +---------------------------------------+---------------+-----------------------------------------------------------+
 | Test Name                             |   Heat API    | Description                                               |
 +=======================================+===============+===========================================================+
-| test_retrieve_flavor_creator          | 1             | Ensures that an OpenStackHeatStack instance can return a  |
+| test_retrieve_flavor_creator          | 1-3           | Ensures that an OpenStackHeatStack instance can return a  |
 |                                       |               | OpenStackFlavor instance that it was responsible for      |
 |                                       |               | deploying                                                 |
 +---------------------------------------+---------------+-----------------------------------------------------------+
@@ -413,7 +413,7 @@ create_stack_tests.py - CreateStackKeypairTests
 +---------------------------------------+---------------+-----------------------------------------------------------+
 | Test Name                             |   Heat API    | Description                                               |
 +=======================================+===============+===========================================================+
-| test_retrieve_keypair_creator         | 1             | Ensures that an OpenStackHeatStack instance can return a  |
+| test_retrieve_keypair_creator         | 1-3           | Ensures that an OpenStackHeatStack instance can return a  |
 |                                       |               | OpenStackKeypair instance that it was responsible for     |
 |                                       |               | deploying                                                 |
 +---------------------------------------+---------------+-----------------------------------------------------------+
@@ -424,7 +424,7 @@ create_stack_tests.py - CreateComplexStackTests
 +---------------------------------------+---------------+-----------------------------------------------------------+
 | Test Name                             |   Heat API    | Description                                               |
 +=======================================+===============+===========================================================+
-| test_connect_via_ssh_heat_vm          | 1             | Ensures that two OpenStackHeatStack instances can return  |
+| test_connect_via_ssh_heat_vm          | 1-3           | Ensures that two OpenStackHeatStack instances can return  |
 |                                       |               | OpenStackVmInstance instances one configured with a       |
 |                                       |               | floating IP and keypair and can be access via SSH         |
 +---------------------------------------+---------------+-----------------------------------------------------------+
@@ -435,13 +435,23 @@ create_stack_tests.py - CreateStackNegativeTests
 +----------------------------------------+---------------+-----------------------------------------------------------+
 | Test Name                              |   Heat API    | Description                                               |
 +========================================+===============+===========================================================+
-| test_missing_dependencies              | 1             | Ensures that a Heat template fails to deploy when expected|
+| test_missing_dependencies              | 1-3           | Ensures that a Heat template fails to deploy when expected|
 |                                        |               | dependencies are missing                                  |
 +----------------------------------------+---------------+-----------------------------------------------------------+
-| test_bad_stack_file                    | 1             | Ensures that a Heat template fails to deploy when the Heat|
+| test_bad_stack_file                    | 1-3           | Ensures that a Heat template fails to deploy when the Heat|
 |                                        |               | template file does not exist                              |
 +----------------------------------------+---------------+-----------------------------------------------------------+
 
+create_stack_tests.py - CreateStackFailureTests
+-----------------------------------------------
+
++----------------------------------------+---------------+-----------------------------------------------------------+
+| Test Name                              |   Heat API    | Description                                               |
++========================================+===============+===========================================================+
+| test_stack_failure                     | 1-3           | Ensures that a Heat template fails to deploy when expected|
+|                                        |               | dependencies are missing                                  |
++----------------------------------------+---------------+-----------------------------------------------------------+
+
 create_instance_tests.py - CreateInstanceSimpleTests
 ----------------------------------------------------
 
diff --git a/snaps/domain/stack.py b/snaps/domain/stack.py
index 543c78b..080ab17 100644
--- a/snaps/domain/stack.py
+++ b/snaps/domain/stack.py
@@ -37,14 +37,21 @@ class Resource:
     """
     SNAPS domain object for a resource created by a heat template
     """
-    def __init__(self, resource_type, resource_id):
+    def __init__(self, name, resource_type, resource_id, status,
+                 status_reason):
         """
         Constructor
-        :param resource_type: the type
+        :param name: the resource's name
+        :param resource_type: the resource's type
         :param resource_id: the ID attached to the resource of the given type
+        :param status: the resource's status code
+        :param status_reason: the resource's status code reason
         """
+        self.name = name
         self.type = resource_type
         self.id = resource_id
+        self.status = status
+        self.status_reason = status_reason
 
 
 class Output:
diff --git a/snaps/domain/test/stack_tests.py b/snaps/domain/test/stack_tests.py
index f816ef8..21e31d2 100644
--- a/snaps/domain/test/stack_tests.py
+++ b/snaps/domain/test/stack_tests.py
@@ -39,14 +39,22 @@ class ResourceDomainObjectTests(unittest.TestCase):
     """
 
     def test_construction_positional(self):
-        resource = Resource('foo', 'bar')
+        resource = Resource('res_name', 'foo', 'bar', 'status', 'reason')
+        self.assertEqual('res_name', resource.name)
         self.assertEqual('foo', resource.type)
         self.assertEqual('bar', resource.id)
+        self.assertEqual('status', resource.status)
+        self.assertEqual('reason', resource.status_reason)
 
     def test_construction_named(self):
-        resource = Resource(resource_id='bar', resource_type='foo')
+        resource = Resource(
+            status_reason=None, status=None, resource_id='bar',
+            resource_type='foo', name='res_name')
+        self.assertEqual('res_name', resource.name)
         self.assertEqual('foo', resource.type)
         self.assertEqual('bar', resource.id)
+        self.assertIsNone(resource.status)
+        self.assertIsNone(resource.status_reason)
 
 
 class OutputDomainObjectTests(unittest.TestCase):
diff --git a/snaps/openstack/create_stack.py b/snaps/openstack/create_stack.py
index 1820e2a..d383566 100644
--- a/snaps/openstack/create_stack.py
+++ b/snaps/openstack/create_stack.py
@@ -425,6 +425,18 @@ class OpenStackHeatStack(OpenStackCloudObject, object):
             return False
 
         if fail_status and status == fail_status:
+            resources = heat_utils.get_resources(self.__heat_cli, self.__stack)
+            logger.error('Stack %s failed', self.__stack.name)
+            for resource in resources:
+                if resource.status != STATUS_CREATE_COMPLETE:
+                    logger.error(
+                        'Resource: [%s] status: [%s] reason: [%s]',
+                        resource.name, resource.status, resource.status_reason)
+                else:
+                    logger.debug(
+                        'Resource: [%s] status: [%s] reason: [%s]',
+                        resource.name, resource.status, resource.status_reason)
+
             raise StackError('Stack had an error')
         logger.debug('Stack status is - ' + status)
         return status == expected_status_code
diff --git a/snaps/openstack/tests/create_stack_tests.py b/snaps/openstack/tests/create_stack_tests.py
index 94085a0..6d472d0 100644
--- a/snaps/openstack/tests/create_stack_tests.py
+++ b/snaps/openstack/tests/create_stack_tests.py
@@ -12,13 +12,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import os
 import time
 
 import pkg_resources
 from heatclient.exc import HTTPBadRequest
 from snaps import file_utils
 from snaps.openstack.create_flavor import OpenStackFlavor, FlavorSettings
-from snaps.openstack.create_image import OpenStackImage
+from snaps.openstack.create_image import OpenStackImage, ImageSettings
 
 try:
     from urllib.request import URLError
@@ -31,7 +32,7 @@ import uuid
 
 from snaps.openstack import create_stack
 from snaps.openstack.create_stack import (
-    StackSettings, StackSettingsError, StackCreationError)
+    StackSettings, StackSettingsError, StackCreationError, StackError)
 from snaps.openstack.tests import openstack_tests, create_instance_tests
 from snaps.openstack.tests.os_source_file_test import OSIntegrationTestCase
 from snaps.openstack.utils import heat_utils, neutron_utils, nova_utils
@@ -123,14 +124,11 @@ class StackSettingsUnitTests(unittest.TestCase):
 
 class CreateStackSuccessTests(OSIntegrationTestCase):
     """
-    Tests for the CreateStack class defined in create_stack.py
+    Tests for the OpenStackHeatStack class defined in create_stack.py
     """
 
     def setUp(self):
-        """
-        Instantiates the CreateStack object that is responsible for downloading
-        and creating an OS stack file within OpenStack
-        """
+
         super(self.__class__, self).__start__()
 
         self.guid = self.__class__.__name__ + '-' + str(uuid.uuid4())
@@ -396,14 +394,12 @@ class CreateStackSuccessTests(OSIntegrationTestCase):
 
 class CreateStackFloatingIpTests(OSIntegrationTestCase):
     """
-    Tests for the CreateStack class defined in create_stack.py
+    Tests to ensure that floating IPs can be accessed via an
+    OpenStackVmInstance object obtained from the OpenStackHeatStack instance
     """
 
     def setUp(self):
-        """
-        Instantiates the CreateStack object that is responsible for downloading
-        and creating an OS stack file within OpenStack
-        """
+
         super(self.__class__, self).__start__()
 
         self.guid = self.__class__.__name__ + '-' + str(uuid.uuid4())
@@ -495,14 +491,12 @@ class CreateStackFloatingIpTests(OSIntegrationTestCase):
 
 class CreateStackVolumeTests(OSIntegrationTestCase):
     """
-    Tests for the CreateStack class as they pertain to volumes
+    Tests to ensure that floating IPs can be accessed via an
+    OpenStackVolume object obtained from the OpenStackHeatStack instance
     """
 
     def setUp(self):
-        """
-        Instantiates the CreateStack object that is responsible for downloading
-        and creating an OS stack file within OpenStack
-        """
+
         super(self.__class__, self).__start__()
 
         self.guid = self.__class__.__name__ + '-' + str(uuid.uuid4())
@@ -591,14 +585,12 @@ class CreateStackVolumeTests(OSIntegrationTestCase):
 
 class CreateStackFlavorTests(OSIntegrationTestCase):
     """
-    Tests for the CreateStack class defined in create_stack.py
+    Tests to ensure that floating IPs can be accessed via an
+    OpenStackFlavor object obtained from the OpenStackHeatStack instance
     """
 
     def setUp(self):
-        """
-        Instantiates the CreateStack object that is responsible for downloading
-        and creating an OS stack file within OpenStack
-        """
+
         super(self.__class__, self).__start__()
 
         self.guid = self.__class__.__name__ + '-' + str(uuid.uuid4())
@@ -653,14 +645,12 @@ class CreateStackFlavorTests(OSIntegrationTestCase):
 
 class CreateStackKeypairTests(OSIntegrationTestCase):
     """
-    Tests for the CreateStack class as they pertain to keypairs
+    Tests to ensure that floating IPs can be accessed via an
+    OpenStackKeypair object obtained from the OpenStackHeatStack instance
     """
 
     def setUp(self):
-        """
-        Instantiates the CreateStack object that is responsible for downloading
-        and creating an OS stack file within OpenStack
-        """
+
         super(self.__class__, self).__start__()
 
         self.guid = self.__class__.__name__ + '-' + str(uuid.uuid4())
@@ -727,10 +717,12 @@ class CreateStackKeypairTests(OSIntegrationTestCase):
 
 class CreateStackNegativeTests(OSIntegrationTestCase):
     """
-    Negative test cases for the CreateStack class
+    Negative test cases for the OpenStackHeatStack class with poor
+    configuration
     """
 
     def setUp(self):
+
         super(self.__class__, self).__start__()
 
         self.heat_creds = self.admin_os_creds
@@ -767,3 +759,111 @@ class CreateStackNegativeTests(OSIntegrationTestCase):
                                                              stack_settings)
         with self.assertRaises(IOError):
             self.stack_creator.create()
+
+
+class CreateStackFailureTests(OSIntegrationTestCase):
+    """
+    Tests for the OpenStackHeatStack class defined in create_stack.py for
+    when failures occur. Failures are being triggered by allocating 1 million
+    CPUs.
+    """
+
+    def setUp(self):
+
+        super(self.__class__, self).__start__()
+
+        self.guid = self.__class__.__name__ + '-' + str(uuid.uuid4())
+
+        self.heat_creds = self.admin_os_creds
+        self.heat_creds.project_name = self.admin_os_creds.project_name
+
+        self.heat_cli = heat_utils.heat_client(self.heat_creds)
+        self.stack_creator = None
+
+        self.tmp_file = file_utils.save_string_to_file(
+            ' ', str(uuid.uuid4()) + '-bad-image')
+        self.image_creator = OpenStackImage(
+            self.heat_creds, ImageSettings(
+                name=self.guid + 'image', image_file=self.tmp_file.name,
+                image_user='foo', img_format='qcow2'))
+        self.image_creator.create()
+
+        # Create Flavor
+        self.flavor_creator = OpenStackFlavor(
+            self.admin_os_creds,
+            FlavorSettings(name=self.guid + '-flavor-name', ram=256, disk=10,
+                           vcpus=1000000))
+        self.flavor_creator.create()
+
+        self.network_name = self.guid + '-net'
+        self.subnet_name = self.guid + '-subnet'
+        self.vm_inst_name = self.guid + '-inst'
+
+        self.env_values = {
+            'image_name': self.image_creator.image_settings.name,
+            'flavor_name': self.flavor_creator.flavor_settings.name,
+            'net_name': self.network_name,
+            'subnet_name': self.subnet_name,
+            'inst_name': self.vm_inst_name}
+
+        self.heat_tmplt_path = pkg_resources.resource_filename(
+            'snaps.openstack.tests.heat', 'test_heat_template.yaml')
+
+    def tearDown(self):
+        """
+        Cleans the stack and downloaded stack file
+        """
+        if self.stack_creator:
+            try:
+                self.stack_creator.clean()
+            except:
+                pass
+
+        if self.image_creator:
+            try:
+                self.image_creator.clean()
+            except:
+                pass
+
+        if self.flavor_creator:
+            try:
+                self.flavor_creator.clean()
+            except:
+                pass
+
+        if self.tmp_file:
+            try:
+                os.remove(self.tmp_file.name)
+            except:
+                pass
+
+        super(self.__class__, self).__clean__()
+
+    def test_stack_failure(self):
+        """
+        Tests the creation of an OpenStack stack from Heat template file that
+        should always fail due to too many CPU cores
+        """
+        # Create Stack
+        # Set the default stack settings, then set any custom parameters sent
+        # from the app
+        stack_settings = StackSettings(
+            name=self.__class__.__name__ + '-' + str(self.guid) + '-stack',
+            template_path=self.heat_tmplt_path,
+            env_values=self.env_values)
+        self.stack_creator = create_stack.OpenStackHeatStack(self.heat_creds,
+                                                             stack_settings)
+
+        with self.assertRaises(StackError):
+            try:
+                self.stack_creator.create()
+            except StackError:
+                resources = heat_utils.get_resources(
+                    self.heat_cli, self.stack_creator.get_stack())
+
+                found = False
+                for resource in resources:
+                    if resource.status == create_stack.STATUS_CREATE_COMPLETE:
+                        found = True
+                self.assertTrue(found)
+                raise
diff --git a/snaps/openstack/utils/heat_utils.py b/snaps/openstack/utils/heat_utils.py
index ad354e0..15c3533 100644
--- a/snaps/openstack/utils/heat_utils.py
+++ b/snaps/openstack/utils/heat_utils.py
@@ -23,8 +23,8 @@ from oslo_serialization import jsonutils
 from snaps import file_utils
 from snaps.domain.stack import Stack, Resource, Output
 
-from snaps.openstack.utils import keystone_utils, neutron_utils, nova_utils, \
-    cinder_utils
+from snaps.openstack.utils import (
+    keystone_utils, neutron_utils, nova_utils, cinder_utils)
 
 __author__ = 'spisarski'
 
@@ -155,10 +155,13 @@ def get_resources(heat_cli, stack, res_type=None):
         out = list()
         for os_resource in os_resources:
             if ((res_type and os_resource.resource_type == res_type)
-                    or not res_type):
+                or not res_type):
                 out.append(Resource(
+                    name=os_resource.resource_name,
                     resource_type=os_resource.resource_type,
-                    resource_id=os_resource.physical_resource_id))
+                    resource_id=os_resource.physical_resource_id,
+                    status=os_resource.resource_status,
+                    status_reason=os_resource.resource_status_reason))
         return out
 
 
@@ -197,8 +200,7 @@ def get_stack_networks(heat_cli, neutron, stack):
     out = list()
     resources = get_resources(heat_cli, stack, 'OS::Neutron::Net')
     for resource in resources:
-        network = neutron_utils.get_network_by_id(
-            neutron, resource.id)
+        network = neutron_utils.get_network_by_id(neutron, resource.id)
         if network:
             out.append(network)