From 91c308a6248a7412d00c1435c8bdbb21bec88540 Mon Sep 17 00:00:00 2001 From: Ross Brattain Date: Sun, 25 Mar 2018 23:51:28 -0700 Subject: restore Heat stack failure logs for CI We need to see what Heat failures occured so that we can debug CI failures. The only way to do this seems to be to use the private event_utils.get_events function to query for all failures Also since we had a failure go ahead and dump the Heat template that failed. Sample output: 2018-03-25 23:50:08,765 [INFO] yardstick.orchestrator.heat heat.py:629 Creating stack 'yardstick-460ed969' START 2018-03-25 23:50:21,932 [ERROR] yardstick.orchestrator.heat heat.py:644 Resource CREATE failed: BadRequest: resources.yardstick-460ed969-xe1: Invalid input for operation: physical_network 'nosuch' unknown for flat provider network. Neutron server returns request_ids: ['req-6f981f1e-a9e2-4114-af84-1ee528aed51b'] 2018-03-25 23:50:21,933 [ERROR] yardstick.orchestrator.heat heat.py:644 BadRequest: resources.yardstick-460ed969-xe1: Invalid input for operation: physical_network 'nosuch' unknown for flat provider network. Neutron server returns request_ids: ['req-6f981f1e-a9e2-4114-af84-1ee528aed51b'] 2018-03-25 23:50:21,972 [ERROR] yardstick.orchestrator.heat heat.py:645 {'description': '\n' 'All referred generated resources are prefixed with the ' 'template\n' 'name (i.e. yardstick-460ed969).\n', 'heat_template_version': '2013-05-23', 'outputs': {'trafficgen_1.yardstick-460ed969': {'description': 'VM UUID', 'value': {'get_resource': 'trafficgen_1.yardstick-460ed969'}}, 'trafficgen_1.yardstick-460ed969-fip': {'description': 'floating ' 'ip ' JIRA: YARDSTICK-998 Change-Id: Ia8f4e5ba7e280fb9086519680d5ee90a2b442e6b Signed-off-by: Ross Brattain --- yardstick/orchestrator/heat.py | 9 +++++++++ yardstick/tests/unit/orchestrator/test_heat.py | 19 ++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) (limited to 'yardstick') diff --git a/yardstick/orchestrator/heat.py b/yardstick/orchestrator/heat.py index d69f86044..5afa4151e 100644 --- a/yardstick/orchestrator/heat.py +++ b/yardstick/orchestrator/heat.py @@ -15,6 +15,7 @@ import datetime import getpass import logging import pkg_resources +import pprint import socket import tempfile import time @@ -22,6 +23,7 @@ import time from oslo_serialization import jsonutils from oslo_utils import encodeutils import shade +from shade._heat import event_utils import yardstick.common.openstack_utils as op_utils from yardstick.common import exceptions @@ -63,6 +65,10 @@ class HeatStack(object): self._update_stack_tracking() + def get_failures(self): + return event_utils.get_events(self._cloud, self._stack.id, + event_args={'resource_status': 'FAILED'}) + def get(self): """Retrieves an existing stack from the target cloud @@ -625,6 +631,9 @@ name (i.e. %s). return stack if stack.status != self.HEAT_STATUS_COMPLETE: + for event in stack.get_failures(): + log.error("%s", event.resource_status_reason) + log.error(pprint.pformat(self._template)) raise exceptions.HeatTemplateError(stack_name=self.name) log.info("Creating stack '%s' DONE in %d secs", diff --git a/yardstick/tests/unit/orchestrator/test_heat.py b/yardstick/tests/unit/orchestrator/test_heat.py index aae2487aa..9598eeb04 100644 --- a/yardstick/tests/unit/orchestrator/test_heat.py +++ b/yardstick/tests/unit/orchestrator/test_heat.py @@ -354,13 +354,30 @@ class HeatTemplateTestCase(unittest.TestCase): 3600) self.assertEqual(heat_stack, ret) - def test_create_block_status_no_complete(self): heat_stack = mock.Mock() heat_stack.status = 'other status' + heat_stack.get_failures.return_value = [] with mock.patch.object(heat, 'HeatStack', return_value=heat_stack): self.assertRaises(exceptions.HeatTemplateError, self.template.create, block=True) heat_stack.create.assert_called_once_with( self.template._template, self.template.heat_parameters, True, 3600) + + def test_create_block_status_no_complete_with_reasons(self): + heat_stack = mock.Mock() + heat_stack.status = 'other status' + heat_stack.get_failures.return_value = [ + mock.Mock(resource_status_reason="A reason"), + mock.Mock(resource_status_reason="Something else") + ] + with mock.patch.object(heat, 'HeatStack', return_value=heat_stack): + with mock.patch.object(heat, 'log') as mock_log: + self.assertRaises(exceptions.HeatTemplateError, + self.template.create, block=True) + mock_log.error.assert_any_call("%s", "A reason") + mock_log.error.assert_any_call("%s", "Something else") + heat_stack.create.assert_called_once_with( + self.template._template, self.template.heat_parameters, True, + 3600) -- cgit 1.2.3-korg