bottlenecks openstack Newton support

JIRA: BOTTLENECK-119

Change the file of template.py manager.py and common.py file
This change helps Bottlenecks project support Newton.
template.py support stack operation.
manager.py support nova and glance operation.
common.py support some operations of openstack Newton.

Change-Id: Ibee110a2b7918c80b2651bb86a9fb7160414e842
Signed-off-by: liyin <liyin11@huawei.com>

1 files changed, 171 insertions, 129 deletions

diff --git a/utils/infra_setup/heat/template.py b/utils/infra_setup/heat/template.py
index f71e9166..28c20b72 100755
--- a/utils/infra_setup/heat/template.py
+++ b/utils/infra_setup/heat/template.py
@@ -7,151 +7,193 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 
-"""to create heat templates from the base template
-"""
-import os
-import json
-import shutil
-import common
-import consts.parameters as parameters
+"""Heat template and stack management"""
 
+import time
+import sys
+import logging
 
-class TreeNode:
+from heatclient import client as heatclient
+import common as op_utils
 
-    def __init__(self):
-        self.up = None
-        self.down = []
-        self.variable_name = ''
-        self.variable_value = 0
 
-    def add_child(self, node):
-        node.up = self
-        self.down.append(node)
+log = logging.getLogger(__name__)
 
-    def get_parent(self):
-        return self.up
 
-    def get_children(self):
-        if len(self.down) == 0:
-            return []
-        return self.down
+class HeatObject(object):
+    ''' base class for template and stack'''
+    def __init__(self):
+        self._heat_client = None
+        self.uuid = None
 
-    def get_variable_name(self):
-        return self.variable_name
+    def _get_heat_client(self):
+        '''returns a heat client instance'''
 
-    def get_variable_value(self):
-        return self.variable_value
+        if self._heat_client is None:
+            sess = op_utils.get_session()
+            heat_endpoint = op_utils.get_endpoint(service_type='orchestration')
+            self._heat_client = heatclient.Client(
+                op_utils.get_heat_api_version(),
+                endpoint=heat_endpoint, session=sess)
 
-    def set_variable_name(self, name):
-        self.variable_name = name
+        return self._heat_client
 
-    def set_variable_value(self, value):
-        self.variable_value = value
+    def status(self):
+        '''returns stack state as a string'''
+        heat = self._get_heat_client()
+        stack = heat.stacks.get(self.uuid)
+        return getattr(stack, 'stack_status')
 
-    def get_path(self):
-        ret_val = []
-        if not self.up:
-            ret_val.append(self)
-            return ret_val
-        for node in self.up.get_path():
-            ret_val.append(node)
-        ret_val.append(self)
-        return ret_val
 
-    def __str__(self):
-        return str(self.variable_name) + " --> " + str(self.variable_value)
+class HeatStack(HeatObject):
+    ''' Represents a Heat stack (deployed template) '''
+    stacks = []
 
-    def __repr__(self):
-        return str(self.variable_name) + " = " + str(self.variable_value)
+    def __init__(self, name):
+        super(HeatStack, self).__init__()
+        self.uuid = None
+        self.name = name
+        self.outputs = None
+        HeatStack.stacks.append(self)
 
     @staticmethod
-    def _get_leaves(node, leaves):
-        children = node.get_children()
-        if len(children) == 0:
-            leaves.append(node)
+    def stacks_exist():
+        '''check if any stack has been deployed'''
+        return len(HeatStack.stacks) > 0
+
+    def _delete(self):
+        '''deletes a stack from the target cloud using heat'''
+        if self.uuid is None:
             return
-        for child in children:
-            TreeNode._get_leaves(child, leaves)
+
+        log.info("Deleting stack '%s', uuid:%s", self.name, self.uuid)
+        heat = self._get_heat_client()
+        template = heat.stacks.get(self.uuid)
+        start_time = time.time()
+        template.delete()
+        status = self.status()
+
+        while status != u'DELETE_COMPLETE':
+            log.debug("stack state %s", status)
+            if status == u'DELETE_FAILED':
+                raise RuntimeError(
+                    heat.stacks.get(self.uuid).stack_status_reason)
+
+            time.sleep(2)
+            status = self.status()
+
+        end_time = time.time()
+        log.info("Deleted stack '%s' in %d secs", self.name,
+                 end_time - start_time)
+        self.uuid = None
+
+    def delete(self, block=True, retries=3):
+        '''deletes a stack in the target cloud using heat (with retry)
+        Sometimes delete fail with "InternalServerError" and the next attempt
+        succeeds. So it is worthwhile to test a couple of times.
+        '''
+        if self.uuid is None:
+            return
+
+        if not block:
+            try:
+                self._delete()
+            except RuntimeError as err:
+                log.warn(err.args)
+            HeatStack.stacks.remove(self)
+            return
+
+        i = 0
+        while i < retries:
+            try:
+                self._delete()
+                break
+            except RuntimeError as err:
+                log.warn(err.args)
+                time.sleep(2)
+            i += 1
+
+        if self.uuid is not None:
+           sys.exit("delete stack failed!!!")
+        else:
+           HeatStack.stacks.remove(self)
 
     @staticmethod
-    def get_leaves(node):
-        leaves = list()
-        TreeNode._get_leaves(node, leaves)
-        return leaves
-
-template_name = parameters.TEST_TEMPLATE_NAME
-
-
-def generates_templates(base_heat_template, deployment_configuration):
-    # parameters loaded from file
-    template_dir = common.get_template_dir()
-    template_extension = parameters.TEMPLATE_EXTENSION
-    template_base_name = base_heat_template
-
-    variables = deployment_configuration
-
-    # Delete the templates generated in previous running
-    common.LOG.info("Removing the heat templates already generated")
-    command = "rm {}{}_*".format(template_dir, template_name)
-    os.system(command)
-
-    # Creation of the tree with all the new configurations
-    common.LOG.info("Creation of a tree with all new configurations")
-    tree = TreeNode()
-    for variable in variables:
-        leaves = TreeNode.get_leaves(tree)
-        common.LOG.debug("LEAVES: " + str(leaves))
-        common.LOG.debug("VALUES: " + str(variables[variable]))
-
-        for value in variables[variable]:
-            for leaf in leaves:
-                new_node = TreeNode()
-                new_node.set_variable_name(variable)
-                new_node.set_variable_value(value)
-                leaf.add_child(new_node)
-
-    common.LOG.debug("CONFIGURATION TREE: " + str(tree))
-
-    common.LOG.info("Heat Template and metadata file creation")
-    leaves = TreeNode.get_leaves(tree)
-    counter = 1
-    for leaf in leaves:
-        heat_template_vars = leaf.get_path()
-        if os.path.isabs(template_base_name):
-            base_template = template_base_name
+    def delete_all():
+        for stack in HeatStack.stacks[:]:
+            stack.delete()
+
+    def update(self):
+        '''update a stack'''
+        pass
+
+
+class HeatTemplate(HeatObject):
+    '''Describes a Heat template and a method to deploy template to a stack'''
+
+    def __init__(self, name, template_file=None, heat_parameters=None):
+        super(HeatTemplate, self).__init__()
+        self.name = name
+        self.state = "NOT_CREATED"
+        self.keystone_client = None
+        self.heat_client = None
+        self.heat_parameters = {}
+
+        # heat_parameters is passed to heat in stack create, empty dict when
+        # yardstick creates the template (no get_param in resources part)
+        if heat_parameters:
+            self.heat_parameters = heat_parameters
+
+        if template_file:
+            with open(template_file) as template:
+                print "Parsing external template:", template_file
+                template_str = template.read()
+                self._template = template_str
+            self._parameters = heat_parameters
         else:
-            base_template = template_dir + template_base_name
-        new_template = template_dir + template_name
-        new_template += "_" + str(counter) + template_extension
-        shutil.copy(base_template, new_template)
-
-        metadata = dict()
-        for var in heat_template_vars:
-            if var.get_variable_name():
-                common.replace_in_file(new_template, "#" +
-                                       var.get_variable_name(),
-                                       var.get_variable_value())
-                metadata[var.get_variable_name()] = var.get_variable_value()
-
-        # Save the metadata on a JSON file
-        with open(new_template + ".json", 'w') as outfile:
-            json.dump(metadata, outfile)
-
-        common.LOG.debug("Heat Templates and Metadata file " + str(counter) +
-                         " created")
-        counter += 1
-
-    # Creation of the template files
-    common.LOG.info(str(counter - 1) + " Heat Templates and Metadata files "
-                                       "created")
-
-
-def get_all_heat_templates(template_dir, template_extension):
-    template_files = list()
-    for dirname, dirnames, filenames in os.walk(template_dir):
-        for filename in filenames:
-            if template_extension in filename and filename.endswith(
-                    template_extension) and template_name in filename:
-                template_files.append(filename)
-    template_files.sort()
-    return template_files
+            sys.exit("\nno such template file.")
+
+        # holds results of requested output after deployment
+        self.outputs = {}
+
+        log.debug("template object '%s' created", name)
+
+    def create(self, block=True):
+        '''creates a template in the target cloud using heat
+        returns a dict with the requested output values from the template'''
+        log.info("Creating stack '%s'", self.name)
+
+        # create stack early to support cleanup, e.g. ctrl-c while waiting
+        stack = HeatStack(self.name)
+
+        heat = self._get_heat_client()
+        end_time = start_time = time.time()
+        print(self._template)
+        stack.uuid = self.uuid = heat.stacks.create(
+            stack_name=self.name, template=self._template,
+            parameters=self.heat_parameters)['stack']['id']
+
+        status = self.status()
+
+        if block:
+            while status != u'CREATE_COMPLETE':
+                log.debug("stack state %s", status)
+                if status == u'CREATE_FAILED':
+                    raise RuntimeError(getattr(heat.stacks.get(self.uuid),
+                                               'stack_status_reason'))
+
+                time.sleep(2)
+                status = self.status()
+
+            end_time = time.time()
+            outputs = getattr(heat.stacks.get(self.uuid), 'outputs')
+
+        for output in outputs:
+            self.outputs[output["output_key"].encode("ascii")] = \
+                output["output_value"].encode("ascii")
+
+        log.info("Created stack '%s' in %d secs",
+                 self.name, end_time - start_time)
+
+        stack.outputs = self.outputs
+        return stack