diff options
Diffstat (limited to 'yardstick/orchestrator/heat.py')
| -rw-r--r-- | yardstick/orchestrator/heat.py | 373 |
1 files changed, 183 insertions, 190 deletions
diff --git a/yardstick/orchestrator/heat.py b/yardstick/orchestrator/heat.py index d58ae5618..9da4948dd 100644 --- a/yardstick/orchestrator/heat.py +++ b/yardstick/orchestrator/heat.py @@ -10,149 +10,132 @@ """Heat template and stack management""" from __future__ import absolute_import -from __future__ import print_function -from six.moves import range - import collections import datetime import getpass import logging - +import pkg_resources +import pprint import socket +import tempfile import time -import heatclient.client -import pkg_resources - +from oslo_serialization import jsonutils from oslo_utils import encodeutils +from shade._heat import event_utils -import yardstick.common.openstack_utils as op_utils +from yardstick.common import constants as consts +from yardstick.common import exceptions from yardstick.common import template_format +from yardstick.common import openstack_utils as op_utils -log = logging.getLogger(__name__) +log = logging.getLogger(__name__) -HEAT_KEY_UUID_LENGTH = 8 PROVIDER_SRIOV = "sriov" +_DEPLOYED_STACKS = {} -def get_short_key_uuid(uuid): - return str(uuid)[:HEAT_KEY_UUID_LENGTH] +class HeatStack(object): + """Represents a Heat stack (deployed template) """ -class HeatObject(object): - """base class for template and stack""" - - def __init__(self): - self._heat_client = None - self.uuid = None + def __init__(self, name, os_cloud_config=None): + self.name = name + self.outputs = {} + os_cloud_config = {} if not os_cloud_config else os_cloud_config + self._cloud = op_utils.get_shade_client(**os_cloud_config) + self._stack = None - @property - def heat_client(self): - """returns a heat client instance""" + def _update_stack_tracking(self): + outputs = self._stack.outputs + self.outputs = {output['output_key']: output['output_value'] for output + in outputs} + if self.uuid: + _DEPLOYED_STACKS[self.uuid] = self._stack - if self._heat_client is None: - sess = op_utils.get_session() - heat_endpoint = op_utils.get_endpoint(service_type='orchestration') - self._heat_client = heatclient.client.Client( - op_utils.get_heat_api_version(), - endpoint=heat_endpoint, session=sess) + def create(self, template, heat_parameters, wait, timeout): + """Creates an OpenStack stack from a template""" + with tempfile.NamedTemporaryFile('wb', delete=False) as template_file: + template_file.write(jsonutils.dump_as_bytes(template)) + template_file.close() + self._stack = self._cloud.create_stack( + self.name, template_file=template_file.name, wait=wait, + timeout=timeout, **heat_parameters) - return self._heat_client + self._update_stack_tracking() - def status(self): - """returns stack state as a string""" - heat_client = self.heat_client - stack = heat_client.stacks.get(self.uuid) - return stack.stack_status + def get_failures(self): + return event_utils.get_events(self._cloud, self._stack.id, + event_args={'resource_status': 'FAILED'}) + def get(self): + """Retrieves an existing stack from the target cloud -class HeatStack(HeatObject): - """Represents a Heat stack (deployed template) """ - stacks = [] + Returns a bool indicating whether the stack exists in the target cloud + If the stack exists, it will be stored as self._stack + """ + self._stack = self._cloud.get_stack(self.name) + if not self._stack: + return False - def __init__(self, name): - super(HeatStack, self).__init__() - self.uuid = None - self.name = name - self.outputs = None - HeatStack.stacks.append(self) + self._update_stack_tracking() + return True @staticmethod def stacks_exist(): - """check if any stack has been deployed""" - return len(HeatStack.stacks) > 0 + """Check if any stack has been deployed""" + return len(_DEPLOYED_STACKS) > 0 - def _delete(self): - """deletes a stack from the target cloud using heat""" + def delete(self, wait=True): + """Deletes a stack in the target cloud""" if self.uuid is None: return - log.info("Deleting stack '%s' START, uuid:%s", self.name, self.uuid) - heat = self.heat_client - template = heat.stacks.get(self.uuid) - start_time = time.time() - template.delete() - - for status in iter(self.status, u'DELETE_COMPLETE'): - log.debug("Deleting stack state: %s", status) - if status == u'DELETE_FAILED': - raise RuntimeError( - heat.stacks.get(self.uuid).stack_status_reason) - - time.sleep(2) + try: + ret = self._cloud.delete_stack(self.uuid, wait=wait) + except TypeError: + # NOTE(ralonsoh): this exception catch solves a bug in Shade, which + # tries to retrieve and read the stack status when it's already + # deleted. + ret = True - end_time = time.time() - log.info("Deleting stack '%s' DONE in %d secs", self.name, - end_time - start_time) - self.uuid = None - - def delete(self, block=True, retries=3): - """deletes a stack in the target cloud using heat (with retry) - Sometimes delete fail with "InternalServerError" and the next attempt - succeeds. So it is worthwhile to test a couple of times. - """ - if self.uuid is None: - return - - if not block: - self._delete() - return - - for _ in range(retries): - try: - self._delete() - break - except RuntimeError as err: - log.warning(err.args) - time.sleep(2) - - # if still not deleted try once more and let it fail everything - if self.uuid is not None: - self._delete() - - HeatStack.stacks.remove(self) + _DEPLOYED_STACKS.pop(self.uuid) + self._stack = None + return ret @staticmethod def delete_all(): - for stack in HeatStack.stacks[:]: + """Delete all deployed stacks""" + for stack in _DEPLOYED_STACKS: stack.delete() - def update(self): - """update a stack""" - raise RuntimeError("not implemented") + @property + def status(self): + """Retrieve the current stack status""" + if self._stack: + return self._stack.status + @property + def uuid(self): + """Retrieve the current stack ID""" + if self._stack: + return self._stack.id -class HeatTemplate(HeatObject): + +class HeatTemplate(object): """Describes a Heat template and a method to deploy template to a stack""" - DESCRIPTION_TEMPLATE = """\ + DESCRIPTION_TEMPLATE = """ Stack built by the yardstick framework for %s on host %s %s. All referred generated resources are prefixed with the template -name (i.e. %s).\ +name (i.e. %s). """ + HEAT_WAIT_LOOP_INTERVAL = 2 + HEAT_STATUS_COMPLETE = 'COMPLETE' + def _init_template(self): timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") self._template = { @@ -170,12 +153,12 @@ name (i.e. %s).\ # short hand for resources part of template self.resources = self._template['resources'] - def __init__(self, name, template_file=None, heat_parameters=None): - super(HeatTemplate, self).__init__() + def __init__(self, name, template_file=None, heat_parameters=None, + os_cloud_config=None): self.name = name - self.state = "NOT_CREATED" self.keystone_client = None self.heat_parameters = {} + self._os_cloud_config = {} if not os_cloud_config else os_cloud_config # heat_parameters is passed to heat in stack create, empty dict when # yardstick creates the template (no get_param in resources part) @@ -184,16 +167,13 @@ name (i.e. %s).\ if template_file: with open(template_file) as stream: - print("Parsing external template:", template_file) + log.info('Parsing external template: %s', template_file) template_str = stream.read() self._template = template_format.parse(template_str) self._parameters = heat_parameters else: self._init_template() - # holds results of requested output after deployment - self.outputs = {} - log.debug("template object '%s' created", name) def add_flavor(self, name, vcpus=1, ram=1024, disk=1, ephemeral=0, @@ -202,9 +182,9 @@ name (i.e. %s).\ """add to the template a Flavor description""" if name is None: name = 'auto' - log.debug("adding Nova::Flavor '%s' vcpus '%d' ram '%d' disk '%d' " + - "ephemeral '%d' is_public '%s' rxtx_factor '%d' " + - "swap '%d' extra_specs '%s' ", + log.debug("adding Nova::Flavor '%s' vcpus '%d' ram '%d' disk '%d' " + "ephemeral '%d' is_public '%s' rxtx_factor '%d' " + "swap '%d' extra_specs '%s'", name, vcpus, ram, disk, ephemeral, is_public, rxtx_factor, swap, str(extra_specs)) @@ -247,14 +227,10 @@ name (i.e. %s).\ def add_volume_attachment(self, server_name, volume_name, mountpoint=None): """add to the template an association of volume to instance""" - log.debug("adding Cinder::VolumeAttachment server '%s' volume '%s' ", server_name, - volume_name) - + log.debug("adding Cinder::VolumeAttachment server '%s' volume '%s' ", + server_name, volume_name) name = "%s-%s" % (server_name, volume_name) - - volume_id = op_utils.get_volume_id(volume_name) - if not volume_id: - volume_id = {'get_resource': volume_name} + volume_id = {'get_resource': volume_name} self.resources[name] = { 'type': 'OS::Cinder::VolumeAttachment', 'properties': {'instance_uuid': {'get_resource': server_name}, @@ -363,21 +339,24 @@ name (i.e. %s).\ } } - def add_port(self, name, network_name, subnet_name, vnic_type, sec_group_id=None, + def add_port(self, name, network, sec_group_id=None, provider=None, allowed_address_pairs=None): """add to the template a named Neutron Port """ - log.debug("adding Neutron::Port '%s', network:'%s', subnet:'%s', vnic_type:'%s', " - "secgroup:%s", name, network_name, subnet_name, vnic_type, sec_group_id) + net_is_existing = network.net_flags.get(consts.IS_EXISTING) + depends_on = [] if net_is_existing else [network.subnet_stack_name] + fixed_ips = [{'subnet': network.subnet}] if net_is_existing else [ + {'subnet': {'get_resource': network.subnet_stack_name}}] + network_ = network.name if net_is_existing else { + 'get_resource': network.stack_name} self.resources[name] = { 'type': 'OS::Neutron::Port', - 'depends_on': [subnet_name], + 'depends_on': depends_on, 'properties': { 'name': name, - 'binding:vnic_type': vnic_type, - 'fixed_ips': [{'subnet': {'get_resource': subnet_name}}], - 'network_id': {'get_resource': network_name}, - 'replacement_policy': 'AUTO', + 'binding:vnic_type': network.vnic_type, + 'fixed_ips': fixed_ips, + 'network': network_, } } @@ -394,6 +373,8 @@ name (i.e. %s).\ self.resources[name]['properties'][ 'allowed_address_pairs'] = allowed_address_pairs + log.debug("adding Neutron::Port %s", self.resources[name]) + self._template['outputs'][name] = { 'description': 'Address for interface %s' % name, 'value': {'get_attr': [name, 'fixed_ips', 0, 'ip_address']} @@ -454,7 +435,7 @@ name (i.e. %s).\ } } - def add_keypair(self, name, key_uuid): + def add_keypair(self, name, key_id): """add to the template a Nova KeyPair""" log.debug("adding Nova::KeyPair '%s'", name) self.resources[name] = { @@ -466,7 +447,7 @@ name (i.e. %s).\ pkg_resources.resource_string( 'yardstick.resources', 'files/yardstick_key-' + - get_short_key_uuid(key_uuid) + '.pub'), + key_id + '.pub'), 'utf-8') } } @@ -490,39 +471,77 @@ name (i.e. %s).\ 'value': {'get_resource': name} } - def add_security_group(self, name): + def add_security_group(self, name, security_group=None): """add to the template a Neutron SecurityGroup""" log.debug("adding Neutron::SecurityGroup '%s'", name) + description = ("Group allowing IPv4 and IPv6 for icmp and upd/tcp on" + "all ports") + rules = [ + {'remote_ip_prefix': '0.0.0.0/0', + 'protocol': 'tcp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '0.0.0.0/0', + 'protocol': 'udp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '0.0.0.0/0', + 'protocol': 'icmp'}, + {'remote_ip_prefix': '::/0', + 'ethertype': 'IPv6', + 'protocol': 'tcp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '::/0', + 'ethertype': 'IPv6', + 'protocol': 'udp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '::/0', + 'ethertype': 'IPv6', + 'protocol': 'ipv6-icmp'}, + {'remote_ip_prefix': '0.0.0.0/0', + 'direction': 'egress', + 'protocol': 'tcp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '0.0.0.0/0', + 'direction': 'egress', + 'protocol': 'udp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '0.0.0.0/0', + 'direction': 'egress', + 'protocol': 'icmp'}, + {'remote_ip_prefix': '::/0', + 'direction': 'egress', + 'ethertype': 'IPv6', + 'protocol': 'tcp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '::/0', + 'direction': 'egress', + 'ethertype': 'IPv6', + 'protocol': 'udp', + 'port_range_min': '1', + 'port_range_max': '65535'}, + {'remote_ip_prefix': '::/0', + 'direction': 'egress', + 'ethertype': 'IPv6', + 'protocol': 'ipv6-icmp'}, + ] + if security_group: + description = "Custom security group rules defined by the user" + rules = security_group.get('rules') + + log.debug("The security group rules is %s", rules) + self.resources[name] = { 'type': 'OS::Neutron::SecurityGroup', 'properties': { 'name': name, - 'description': "Group allowing IPv4 and IPv6 for icmp and upd/tcp on all ports", - 'rules': [ - {'remote_ip_prefix': '0.0.0.0/0', - 'protocol': 'tcp', - 'port_range_min': '1', - 'port_range_max': '65535'}, - {'remote_ip_prefix': '0.0.0.0/0', - 'protocol': 'udp', - 'port_range_min': '1', - 'port_range_max': '65535'}, - {'remote_ip_prefix': '0.0.0.0/0', - 'protocol': 'icmp'}, - {'remote_ip_prefix': '::/0', - 'ethertype': 'IPv6', - 'protocol': 'tcp', - 'port_range_min': '1', - 'port_range_max': '65535'}, - {'remote_ip_prefix': '::/0', - 'ethertype': 'IPv6', - 'protocol': 'udp', - 'port_range_min': '1', - 'port_range_max': '65535'}, - {'remote_ip_prefix': '::/0', - 'ethertype': 'IPv6', - 'protocol': 'ipv6-icmp'} - ] + 'description': description, + 'rules': rules } } @@ -600,57 +619,31 @@ name (i.e. %s).\ 'value': {'get_resource': name} } - HEAT_WAIT_LOOP_INTERVAL = 2 - HEAT_CREATE_COMPLETE_STATUS = u'CREATE_COMPLETE' - def create(self, block=True, timeout=3600): - """ - creates a template in the target cloud using heat - returns a dict with the requested output values from the template + """Creates a stack in the target based on the stored template - :param block: Wait for Heat create to finish - :type block: bool - :param: timeout: timeout in seconds for Heat create, default 3600s - :type timeout: int + :param block: (bool) Wait for Heat create to finish + :param timeout: (int) Timeout in seconds for Heat create, + default 3600s + :return A dict with the requested output values from the template """ log.info("Creating stack '%s' START", self.name) - # create stack early to support cleanup, e.g. ctrl-c while waiting - stack = HeatStack(self.name) - - heat_client = self.heat_client start_time = time.time() - stack.uuid = self.uuid = heat_client.stacks.create( - stack_name=self.name, template=self._template, - parameters=self.heat_parameters)['stack']['id'] + stack = HeatStack(self.name, os_cloud_config=self._os_cloud_config) + stack.create(self._template, self.heat_parameters, block, timeout) if not block: - self.outputs = stack.outputs = {} - end_time = time.time() log.info("Creating stack '%s' DONE in %d secs", - self.name, end_time - start_time) + self.name, time.time() - start_time) return stack - time_limit = start_time + timeout - for status in iter(self.status, self.HEAT_CREATE_COMPLETE_STATUS): - log.debug("Creating stack state: %s", status) - if status == u'CREATE_FAILED': - stack_status_reason = heat_client.stacks.get(self.uuid).stack_status_reason - heat_client.stacks.delete(self.uuid) - raise RuntimeError(stack_status_reason) - if time.time() > time_limit: - raise RuntimeError("Heat stack create timeout") - - time.sleep(self.HEAT_WAIT_LOOP_INTERVAL) + if stack.status != self.HEAT_STATUS_COMPLETE: + for event in stack.get_failures(): + log.error("%s", event.resource_status_reason) + log.error(pprint.pformat(self._template)) + raise exceptions.HeatTemplateError(stack_name=self.name) - end_time = time.time() - outputs = heat_client.stacks.get(self.uuid).outputs log.info("Creating stack '%s' DONE in %d secs", - self.name, end_time - start_time) - - # keep outputs as unicode - self.outputs = {output["output_key"]: output["output_value"] for output - in outputs} - - stack.outputs = self.outputs + self.name, time.time() - start_time) return stack |