Add heat template generation module with required constants

Adds to ApexLake the support to the generation of the all the possible configurations and related heat templates.
Files containing constants have been added as well to support the module. They will also be useful for other modules of the framework.

JIRA: YARDSTICK-35

Change-Id: I9d33d5b11c586d40dbff199c21e428113bf3cdeb
Signed-off-by: Vincenzo Riccobene <vincenzox.m.riccobene@intel.com>

4 files changed, 371 insertions, 0 deletions

diff --git a/yardstick/vTC/apexlake/experimental_framework/constants/__init__.py b/yardstick/vTC/apexlake/experimental_framework/constants/__init__.py
new file mode 100644
index 000000000..8178e022d
--- /dev/null
+++ b/yardstick/vTC/apexlake/experimental_framework/constants/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (c) 2015 Intel Research and Development Ireland Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Constants
+"""
diff --git a/yardstick/vTC/apexlake/experimental_framework/constants/conf_file_sections.py b/yardstick/vTC/apexlake/experimental_framework/constants/conf_file_sections.py
new file mode 100644
index 000000000..eed00bce0
--- /dev/null
+++ b/yardstick/vTC/apexlake/experimental_framework/constants/conf_file_sections.py
@@ -0,0 +1,86 @@
+# Copyright (c) 2015 Intel Research and Development Ireland Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# ------------------------------------------------------
+# Configuration File Sections
+# ------------------------------------------------------
+CFS_PKTGEN = 'PacketGen'
+CFS_GENERAL = 'General'
+CFS_OPENSTACK = 'OpenStack'
+CFS_EXPERIMENT_VNF = 'Experiment-VNF'
+CFS_EXPERIMENT_GENERIC = 'Experiment-generic'
+CFS_TESTCASE_PARAMETERS = 'Testcase-parameters'
+CFS_DEPLOYMENT_PARAMETERS = 'Deployment-parameters'
+
+
+def get_sections():
+    return [
+        CFS_PKTGEN,
+        CFS_GENERAL,
+        CFS_OPENSTACK,
+        CFS_EXPERIMENT_VNF,
+        CFS_EXPERIMENT_GENERIC,
+        CFS_TESTCASE_PARAMETERS,
+        CFS_DEPLOYMENT_PARAMETERS
+        # Add here eventually new sections in configuration file ...
+    ]
+
+
+def get_sections_api():
+    return [
+        CFS_PKTGEN,
+        CFS_GENERAL,
+        # TODO: TO BE REMOVED AFTER TESTING THE API
+        CFS_OPENSTACK
+        # Add here eventually new sections in configuration file ...
+    ]
+
+# ------------------------------------------------------
+# General section parameters
+# ------------------------------------------------------
+CFSG_ITERATIONS = 'iterations'
+CFSG_TEMPLATE_DIR = 'template_dir'
+CFSG_TEMPLATE_NAME = 'template_base_name'
+CFSG_RESULT_DIRECTORY = 'results_directory'
+CFSG_BENCHMARKS = 'benchmarks'
+
+
+# ------------------------------------------------------
+# Packet generator section parameters
+# ------------------------------------------------------
+CFSP_PACKET_GENERATOR = 'packet_generator'
+CFSP_DPDK_DIRECTORY = 'directory'
+CFSP_DPDK_PROGRAM_NAME = 'program_name'
+CFSP_DPDK_COREMASK = 'coremask'
+CFSP_DPDK_MEMORY_CHANNEL = 'memory_channels'
+CFSP_DPDK_CORE_NICS = 'core_nics'
+
+
+# ------------------------------------------------------
+# Supported Packet generators
+# ------------------------------------------------------
+CFSP_PG_NONE = 'none'
+CFSP_PG_DPDK = 'dpdk_pktgen'
+
+
+# ------------------------------------------------------
+# OpenStack section variables
+# ------------------------------------------------------
+CFSO_IP_CONTROLLER = 'ip_controller'
+CFSO_HEAT_URL = 'heat_url'
+CFSO_USER = 'user'
+CFSO_PASSWORD = 'password'
+CFSO_AUTH_URI = 'auth_uri'
+CFSO_PROJECT = 'project'
diff --git a/yardstick/vTC/apexlake/experimental_framework/constants/framework_parameters.py b/yardstick/vTC/apexlake/experimental_framework/constants/framework_parameters.py
new file mode 100644
index 000000000..4ee3a8aa3
--- /dev/null
+++ b/yardstick/vTC/apexlake/experimental_framework/constants/framework_parameters.py
@@ -0,0 +1,33 @@
+# Copyright (c) 2015 Intel Research and Development Ireland Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from experimental_framework.constants import conf_file_sections as cfs
+
+# ------------------------------------------------------
+# Directories and file locations
+# ------------------------------------------------------
+EXPERIMENTAL_FRAMEWORK_DIR = 'experimental_framework/'
+EXPERIMENT_TEMPLATE_NAME = 'experiment'
+TEMPLATE_FILE_EXTENSION = '.yaml'
+DPDK_PKTGEN_DIR = 'packet_generators/dpdk_pktgen/'
+PCAP_DIR = 'packet_generators/pcap_files/'
+
+
+def get_supported_packet_generators():
+    return [
+        cfs.CFSP_PG_NONE,
+        cfs.CFSP_PG_DPDK
+        # Add here any other supported packet generator
+    ]
diff --git a/yardstick/vTC/apexlake/experimental_framework/heat_template_generation.py b/yardstick/vTC/apexlake/experimental_framework/heat_template_generation.py
new file mode 100644
index 000000000..15c4eff36
--- /dev/null
+++ b/yardstick/vTC/apexlake/experimental_framework/heat_template_generation.py
@@ -0,0 +1,235 @@
+# Copyright (c) 2015 Intel Research and Development Ireland Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+'''
+This file contains the code to Generate the heat templates from the base
+template
+'''
+
+import json
+import os
+import shutil
+from experimental_framework import common
+from experimental_framework.constants import framework_parameters as fp
+
+
+class TreeNode:
+    """
+    This class represent the node of the configuration tree.\
+    Each node represents a single configuration value for a single
+    configuration parameter.
+    """
+
+    def __init__(self):
+        self.up = None
+        self.down = []
+        self.variable_name = ''
+        self.variable_value = 0
+
+    def add_child(self, node):
+        """
+        Adds a node as a child for the current node
+        :param node: node to be added as a child (type: TreeNode)
+        :return: None
+        """
+        node.up = self
+        self.down.append(node)
+
+    def get_parent(self):
+        """
+        Returns the parent node of the current one
+        :return type: TreeNode
+        """
+        return self.up
+
+    def get_children(self):
+        """
+        Returns the children of the current node
+        :return type: list of TreeNode
+        """
+        if len(self.down) == 0:
+            # return [self]
+            return []
+        return self.down
+
+    def get_variable_name(self):
+        """
+        Returns the name of the variable correspondent to the current node
+        :return type: str
+        """
+        return self.variable_name
+
+    def get_variable_value(self):
+        """
+        Returns the value of the variable correspondent to the current node
+        :return type: str or int
+        """
+        return self.variable_value
+
+    def set_variable_name(self, name):
+        """
+        Sets the name of the variable for the current node
+        :param name: Name of the variable (type: str)
+        :return None
+        """
+        self.variable_name = name
+
+    def set_variable_value(self, value):
+        """
+        Sets the value of the variable for the current node
+        :param value: value of the variable (type: str)
+        :return None
+        """
+        self.variable_value = value
+
+    def get_path(self):
+        """
+        Returns all the path from the current node to the root of the tree.
+        :return type: list of TreeNode
+        """
+        ret_val = []
+        if not self.up:
+            ret_val.append(self)
+            return ret_val
+        for node in self.up.get_path():
+            ret_val.append(node)
+        ret_val.append(self)
+        return ret_val
+
+    def __str__(self):
+        return str(self.variable_name) + " --> " + str(self.variable_value)
+
+    def __repr__(self):
+        return str(self.variable_name) + " = " + str(self.variable_value)
+
+    @staticmethod
+    def _get_leaves(node, leaves):
+        """
+        Returns all the leaves of a tree.
+        It changes the "leaves" list.
+        :param node: root of the tree (type: TreeNode)
+        :param leaves: partial list of leaves (type: list of TreeNode)
+        :return type: None
+        """
+        children = node.get_children()
+        if len(children) == 0:
+            leaves.append(node)
+            return
+        for child in children:
+            TreeNode._get_leaves(child, leaves)
+
+    @staticmethod
+    def get_leaves(node):
+        """
+        Returns all the leaves of a tree.
+        :param node: root of the tree (TreeNode)
+        :return type: list
+        """
+        leaves = list()
+        TreeNode._get_leaves(node, leaves)
+        return leaves
+
+
+template_name = fp.EXPERIMENT_TEMPLATE_NAME
+
+
+def generates_templates(base_heat_template, deployment_configuration):
+    """
+    Generates the heat templates for the experiments
+    :return: None
+    """
+    # Load useful parameters from file
+    template_dir = common.get_template_dir()
+    template_file_extension = fp.TEMPLATE_FILE_EXTENSION
+    template_base_name = base_heat_template
+
+    variables = deployment_configuration
+
+    # Delete the templates eventually generated in previous running of the
+    # framework
+    common.LOG.info("Removing the heat templates previously generated")
+    os.system("rm " + template_dir + template_name + "_*")
+
+    # Creation of the tree with all the new configurations
+    common.LOG.info("Creation of the tree with all the new configurations")
+    tree = TreeNode()
+    for variable in variables:
+        leaves = TreeNode.get_leaves(tree)
+        common.LOG.debug("LEAVES: " + str(leaves))
+        common.LOG.debug("VALUES: " + str(variables[variable]))
+
+        for value in variables[variable]:
+            for leaf in leaves:
+                new_node = TreeNode()
+                new_node.set_variable_name(variable)
+                new_node.set_variable_value(value)
+                leaf.add_child(new_node)
+
+    common.LOG.debug("CONFIGURATION TREE: " + str(tree))
+
+    common.LOG.info("Heat Template and metadata file creation")
+    leaves = TreeNode.get_leaves(tree)
+    counter = 1
+    for leaf in leaves:
+        heat_template_vars = leaf.get_path()
+        if os.path.isabs(template_base_name):
+            base_template = template_base_name
+        else:
+            base_template = template_dir + template_base_name
+        if os.path.isabs(template_name):
+            new_template = template_name
+        else:
+            new_template = template_dir + template_name
+        new_template += "_" + str(counter) + template_file_extension
+        shutil.copy(base_template, new_template)
+
+        metadata = dict()
+        for var in heat_template_vars:
+            if var.get_variable_name():
+                common.replace_in_file(new_template, "#" +
+                                       var.get_variable_name(),
+                                       var.get_variable_value())
+                metadata[var.get_variable_name()] = var.get_variable_value()
+
+        # Save the metadata on a JSON file
+        with open(new_template + ".json", 'w') as outfile:
+            json.dump(metadata, outfile)
+
+        common.LOG.debug("Heat Templates and Metadata file " + str(counter) +
+                         " created")
+        counter += 1
+
+    # Creation of the template files
+    common.LOG.info(str(counter - 1) + " Heat Templates and Metadata files "
+                                       "created")
+
+
+def get_all_heat_templates(template_dir, template_file_extension):
+    """
+    Loads and returns all the generated heat templates
+    :param template_dir: directory to search in (type: str)
+    :param template_file_extension: extension of the file for templates
+            (type: str)
+    :return: type: list
+    """
+    template_files = list()
+    for dirname, dirnames, filenames in os.walk(template_dir):
+        for filename in filenames:
+            if template_file_extension in filename and \
+                    filename.endswith(template_file_extension) and \
+                    template_name in filename:
+                template_files.append(filename)
+    template_files.sort()
+    return template_files