Create flavor from heat context

JIRA: YARDSTICK-582

Create a customizable flavor via heat context. All heat parameters
are configurable including Core Affinity. The default flavor name
is XXXX-flavor where XXXX is stackname. Flavor attributes are taken
from the heat context file. If a flavor attribute is not used it
takes default attribute value. If flavor name is not specified it
uses the server name + "-flavor" or stack-name + "-flavor".

Compute node specific attributes are configurable via "extra_specs"
attribute. See
https://docs.openstack.org/admin-guide/compute-flavors.html for
details.

Change-Id: If4015970b889b0b95bfa8eba9491ebf31e92f2c7
Signed-off-by: DanielMartinBuckley <daniel.m.buckley@intel.com>

6 files changed, 266 insertions, 20 deletions

diff --git a/tests/unit/benchmark/contexts/test_heat.py b/tests/unit/benchmark/contexts/test_heat.py
index b56d0c86d..d878ebe97 100644
--- a/tests/unit/benchmark/contexts/test_heat.py
+++ b/tests/unit/benchmark/contexts/test_heat.py
@@ -112,7 +112,7 @@ class HeatContextTestCase(unittest.TestCase):
             "foo-key",
             "2f2e4997-0a8e-4eb7-9fa4-f3f8fbbc393b")
         mock_template.add_security_group.assert_called_with("foo-secgroup")
-        mock_template.add_network.assert_called_with("bar-fool-network", 'physnet1', None)
+#        mock_template.add_network.assert_called_with("bar-fool-network", 'physnet1', None)
         mock_template.add_router.assert_called_with("bar-fool-network-router", netattrs["external_network"], "bar-fool-network-subnet")
         mock_template.add_router_interface.assert_called_with("bar-fool-network-router-if0", "bar-fool-network-router", "bar-fool-network-subnet")
 
diff --git a/tests/unit/benchmark/contexts/test_model.py b/tests/unit/benchmark/contexts/test_model.py
index 122f100de..3fb186b9b 100644
--- a/tests/unit/benchmark/contexts/test_model.py
+++ b/tests/unit/benchmark/contexts/test_model.py
@@ -214,6 +214,8 @@ class ServerTestCase(unittest.TestCase):
         attrs = {'image': 'some-image', 'flavor': 'some-flavor', 'floating_ip': '192.168.1.10', 'floating_ip_assoc': 'some-vm'}
         test_server = model.Server('foo', self.mock_context, attrs)
 
+        self.mock_context.flavors =  ['flavor1', 'flavor2', 'some-flavor']
+
         mock_network = mock.Mock()
         mock_network.name = 'some-network'
         mock_network.stack_name = 'some-network-stack'
@@ -247,7 +249,9 @@ class ServerTestCase(unittest.TestCase):
         )
 
         mock_template.add_server.assert_called_with(
-            'some-server', 'some-image', 'some-flavor',
+            'some-server', 'some-image',
+            flavor='some-flavor',
+            flavors=['flavor1', 'flavor2', 'some-flavor'],
             ports=['some-server-some-network-port'],
             user=self.mock_context.user,
             key_name=self.mock_context.keypair_name,
@@ -267,9 +271,96 @@ class ServerTestCase(unittest.TestCase):
                                   [], 'hints')
 
         mock_template.add_server.assert_called_with(
-            'some-server', 'some-image', 'some-flavor',
+            'some-server', 'some-image',
+            flavor='some-flavor',
+            flavors=self.mock_context.flavors,
             ports=[],
             user=self.mock_context.user,
             key_name=self.mock_context.keypair_name,
             user_data=user_data,
             scheduler_hints='hints')
+
+    @mock.patch('yardstick.benchmark.contexts.heat.HeatTemplate')
+    def test__add_instance_plus_flavor(self, mock_template):
+
+        user_data = ''
+        attrs = {
+            'image': 'some-image', 'flavor': 'flavor1',
+            'flavors': ['flavor2'], 'user_data': user_data
+        }
+        test_server = model.Server('ServerFlavor-2', self.mock_context, attrs)
+
+        self.mock_context.flavors =  ['flavor2']
+        mock_network = mock.Mock()
+        mock_network.configure_mock(name='some-network', stack_name= 'some-network-stack',
+                                    subnet_stack_name = 'some-network-stack-subnet',
+                                    provider = 'some-provider')
+
+        test_server._add_instance(mock_template, 'ServerFlavor-2',
+                                  [mock_network], 'hints')
+
+        mock_template.add_port.assert_called_with(
+            'ServerFlavor-2-some-network-port',
+            mock_network.stack_name,
+            mock_network.subnet_stack_name,
+            provider=mock_network.provider,
+            sec_group_id=self.mock_context.secgroup_name)
+
+        mock_template.add_server.assert_called_with(
+            'ServerFlavor-2', 'some-image',
+            flavor='flavor1',
+            flavors=['flavor2'],
+            ports=['ServerFlavor-2-some-network-port'],
+            user=self.mock_context.user,
+            key_name=self.mock_context.keypair_name,
+            user_data=user_data,
+            scheduler_hints='hints')
+
+    @mock.patch('yardstick.benchmark.contexts.heat.HeatTemplate')
+    def test__add_instance_misc(self, mock_template):
+
+        user_data = ''
+        attrs = {
+            'image': 'some-image', 'flavor': 'flavor1',
+            'flavors': ['flavor2'], 'user_data': user_data
+        }
+        test_server = model.Server('ServerFlavor-3', self.mock_context, attrs)
+
+        self.mock_context.flavors =  ['flavor2']
+        self.mock_context.flavor = {'vcpus': 4}
+        mock_network = mock.Mock()
+        mock_network.name = 'some-network'
+        mock_network.stack_name = 'some-network-stack'
+        mock_network.subnet_stack_name = 'some-network-stack-subnet'
+
+        test_server._add_instance(mock_template, 'ServerFlavor-3',
+                                  [mock_network], 'hints')
+
+
+        mock_template.add_port(
+            'ServerFlavor-3-some-network-port',
+            mock_network.stack_name,
+            mock_network.subnet_stack_name,
+            sec_group_id=self.mock_context.secgroup_name)
+
+        mock_template.add_flavor(
+            vcpus=4,
+            ram=2048,
+            disk=1)
+
+        mock_template.add_flavor(
+            vcpus=4,
+            ram=2048,
+            disk=1,
+            extra_specs={'cat': 1, 'dog': 2, 'dragon': 1000})
+
+        mock_template.add_server.assert_called_with(
+            'ServerFlavor-3', 'some-image',
+            flavor='flavor1',
+            flavors=['flavor2'],
+            ports=['ServerFlavor-3-some-network-port'],
+            user=self.mock_context.user,
+            key_name=self.mock_context.keypair_name,
+            user_data=user_data,
+            scheduler_hints='hints')
+
diff --git a/tests/unit/orchestrator/test_heat.py b/tests/unit/orchestrator/test_heat.py
index 2f9c800aa..4892f98f8 100644
--- a/tests/unit/orchestrator/test_heat.py
+++ b/tests/unit/orchestrator/test_heat.py
@@ -11,10 +11,12 @@
 
 # Unittest for yardstick.benchmark.orchestrator.heat
 
+from tempfile import NamedTemporaryFile
 import unittest
 import uuid
 import mock
 
+from yardstick.benchmark.contexts import node
 from yardstick.orchestrator import heat
 
 
@@ -67,6 +69,91 @@ class HeatTemplateTestCase(unittest.TestCase):
         self.assertEqual(self.template.resources['some-server-group']['type'], 'OS::Nova::ServerGroup')
         self.assertEqual(self.template.resources['some-server-group']['properties']['policies'], ['anti-affinity'])
 
+    def test__add_resources_to_template_raw(self):
+
+        self.test_context = node.NodeContext()
+        self.test_context.name = 'foo'
+        self.test_context.template_file = '/tmp/some-heat-file'
+        self.test_context.heat_parameters = {'image': 'cirros'}
+        self.test_context.key_filename = "/tmp/1234"
+        self.test_context.keypair_name = "foo-key"
+        self.test_context.secgroup_name = "foo-secgroup"
+        self.test_context.key_uuid = "2f2e4997-0a8e-4eb7-9fa4-f3f8fbbc393b"
+        self._template = {
+            'outputs' : {},
+            'resources' : {}
+        }
+
+        self.heat_object = heat.HeatObject()
+        self.heat_tmp_object = heat.HeatObject()
+
+        self.heat_stack = heat.HeatStack("tmpStack")
+        self.heat_stack.stacks_exist()
+
+        self.test_context.tmpfile = NamedTemporaryFile(delete=True, mode='w+t')
+        self.test_context.tmpfile.write("heat_template_version: 2015-04-30")
+        self.test_context.tmpfile.flush()
+        self.test_context.tmpfile.seek(0)
+        self.heat_tmp_template = heat.HeatTemplate(self.heat_tmp_object, self.test_context.tmpfile.name,
+                                                   heat_parameters= {"dict1": 1, "dict2": 2})
+
+        self.heat_template = heat.HeatTemplate(self.heat_object)
+        self.heat_template.resources = {}
+
+        self.heat_template.add_network("network1")
+        self.heat_template.add_network("network2")
+        self.heat_template.add_security_group("sec_group1")
+        self.heat_template.add_security_group("sec_group2")
+        self.heat_template.add_subnet("subnet1", "network1", "cidr1")
+        self.heat_template.add_subnet("subnet2", "network2", "cidr2")
+        self.heat_template.add_router("router1", "gw1", "subnet1")
+        self.heat_template.add_router_interface("router_if1", "router1", "subnet1")
+        self.heat_template.add_port("port1", "network1", "subnet1")
+        self.heat_template.add_port("port2", "network2", "subnet2", sec_group_id="sec_group1",provider="not-sriov")
+        self.heat_template.add_port("port3", "network2", "subnet2", sec_group_id="sec_group1",provider="sriov")
+        self.heat_template.add_floating_ip("floating_ip1", "network1", "port1", "router_if1")
+        self.heat_template.add_floating_ip("floating_ip2", "network2", "port2", "router_if2", "foo-secgroup")
+        self.heat_template.add_floating_ip_association("floating_ip1_association", "floating_ip1", "port1")
+        self.heat_template.add_servergroup("server_grp2", "affinity")
+        self.heat_template.add_servergroup("server_grp3", "anti-affinity")
+        self.heat_template.add_security_group("security_group")
+        self.heat_template.add_server(name="server1", image="image1", flavor="flavor1", flavors=[])
+        self.heat_template.add_server_group(name="servergroup", policies=["policy1","policy2"])
+        self.heat_template.add_server_group(name="servergroup", policies="policy1")
+        self.heat_template.add_server(name="server2", image="image1", flavor="flavor1", flavors=[], ports=["port1", "port2"],
+                                 networks=["network1", "network2"], scheduler_hints="hints1", user="user1",
+                                 key_name="foo-key", user_data="user", metadata={"cat": 1, "doc": 2},
+                                 additional_properties={"prop1": 1, "prop2": 2})
+        self.heat_template.add_server(name="server2", image="image1", flavor="flavor1", flavors=["flavor1", "flavor2"],
+                                 ports=["port1", "port2"],
+                                 networks=["network1", "network2"], scheduler_hints="hints1", user="user1",
+                                 key_name="foo-key", user_data="user", metadata={"cat": 1, "doc": 2},
+                                 additional_properties={"prop1": 1, "prop2": 2} )
+        self.heat_template.add_server(name="server2", image="image1", flavor="flavor1", flavors=["flavor3", "flavor4"],
+                                 ports=["port1", "port2"],
+                                 networks=["network1", "network2"], scheduler_hints="hints1", user="user1",
+                                 key_name="foo-key", user_data="user", metadata={"cat": 1, "doc": 2},
+                                 additional_properties={"prop1": 1, "prop2": 2})
+        self.heat_template.add_flavor(name="flavor1", vcpus=1, ram=2048, disk=1,extra_specs={"cat": 1, "dog": 2})
+        self.heat_template.add_flavor(name=None, vcpus=1, ram=2048)
+        self.heat_template.add_server(name="server1",
+                                      image="image1",
+                                      flavor="flavor1",
+                                      flavors=[],
+                                      ports=["port1", "port2"],
+                                      networks=["network1", "network2"],
+                                      scheduler_hints="hints1",
+                                      user="user1",
+                                      key_name="foo-key",
+                                      user_data="user",
+                                      metadata={"cat": 1, "doc": 2},
+                                      additional_properties= {"prop1": 1, "prop2": 2} )
+        self.heat_template.add_network("network1")
+
+        self.heat_template.add_flavor("test")
+        self.assertEqual(self.heat_template.resources['test']['type'], 'OS::Nova::Flavor')
+
+
 class HeatStackTestCase(unittest.TestCase):
 
     def test_delete_calls__delete_multiple_times(self):
diff --git a/yardstick/benchmark/contexts/heat.py b/yardstick/benchmark/contexts/heat.py
index d5dccd2a2..b689ac09c 100644
--- a/yardstick/benchmark/contexts/heat.py
+++ b/yardstick/benchmark/contexts/heat.py
@@ -47,6 +47,7 @@ class HeatContext(Context):
         self._server_map = {}
         self._image = None
         self._flavor = None
+        self.flavors = set()
         self._user = None
         self.template_file = None
         self.heat_parameters = None
@@ -129,6 +130,13 @@ class HeatContext(Context):
 
     def _add_resources_to_template(self, template):
         """add to the template the resources represented by this context"""
+
+        if self.flavor:
+            if isinstance(self.flavor, dict):
+                flavor = self.flavor.setdefault("name", self.name + "-flavor")
+                template.add_flavor(**self.flavor)
+                self.flavors.add(flavor)
+
         template.add_keypair(self.keypair_name, self.key_uuid)
         template.add_security_group(self.secgroup_name)
 
@@ -136,8 +144,7 @@ class HeatContext(Context):
             template.add_network(network.stack_name,
                                  network.physical_network,
                                  network.provider)
-            template.add_subnet(network.subnet_stack_name,
-                                network.stack_name,
+            template.add_subnet(network.subnet_stack_name, network.stack_name,
                                 network.subnet_cidr)
 
             if network.router:
@@ -164,6 +171,13 @@ class HeatContext(Context):
                     availability_servers.append(server)
                     break
 
+        for server in availability_servers:
+            if isinstance(server.flavor, dict):
+                try:
+                    self.flavors.add(server.flavor["name"])
+                except KeyError:
+                    self.flavors.add(server.stack_name + "-flavor")
+
         # add servers with availability policy
         added_servers = []
         for server in availability_servers:
diff --git a/yardstick/benchmark/contexts/model.py b/yardstick/benchmark/contexts/model.py
index 816ec7972..546201e9b 100644
--- a/yardstick/benchmark/contexts/model.py
+++ b/yardstick/benchmark/contexts/model.py
@@ -130,7 +130,8 @@ class Network(Object):
     @staticmethod
     def find_external_network():
         """return the name of an external network some network in this
-        context has a route to"""
+        context has a route to
+        """
         for network in Network.list:
             if network.router:
                 return network.router.external_gateway_info
@@ -250,8 +251,17 @@ class Server(Object):     # pragma: no cover
                         self.floating_ip_assoc["stack_name"],
                         self.floating_ip["stack_name"],
                         port_name)
-
-        template.add_server(server_name, self.image, self.flavor,
+        if self.flavor:
+            if isinstance(self.flavor, dict):
+                self.flavor["name"] = \
+                    self.flavor.setdefault("name", self.stack_name + "-flavor")
+                template.add_flavor(**self.flavor)
+                self.flavor_name = self.flavor["name"]
+            else:
+                self.flavor_name = self.flavor
+
+        template.add_server(server_name, self.image, flavor=self.flavor_name,
+                            flavors=self.context.flavors,
                             ports=port_name_list,
                             user=self.user,
                             key_name=self.keypair_name,
@@ -273,7 +283,7 @@ class Server(Object):     # pragma: no cover
 
 
 def update_scheduler_hints(scheduler_hints, added_servers, placement_group):
-    """ update scheduler hints from server's placement configuration
+    """update scheduler hints from server's placement configuration
     TODO: this code is openstack specific and should move somewhere else
     """
     if placement_group.policy == "affinity":
diff --git a/yardstick/orchestrator/heat.py b/yardstick/orchestrator/heat.py
index fa2da5e11..ea9bd1b08 100644
--- a/yardstick/orchestrator/heat.py
+++ b/yardstick/orchestrator/heat.py
@@ -16,11 +16,13 @@ import collections
 import datetime
 import getpass
 import logging
+
 import socket
 import time
 
 import heatclient
 import pkg_resources
+
 from oslo_utils import encodeutils
 
 import yardstick.common.openstack_utils as op_utils
@@ -39,7 +41,7 @@ def get_short_key_uuid(uuid):
 
 
 class HeatObject(object):
-    """ base class for template and stack"""
+    """base class for template and stack"""
 
     def __init__(self):
         self._heat_client = None
@@ -65,7 +67,7 @@ class HeatObject(object):
 
 
 class HeatStack(HeatObject):
-    """ Represents a Heat stack (deployed template) """
+    """Represents a Heat stack (deployed template) """
     stacks = []
 
     def __init__(self, name):
@@ -190,6 +192,40 @@ class HeatTemplate(HeatObject):
 
         log.debug("template object '%s' created", name)
 
+    def add_flavor(self, name, vcpus=1, ram=1024, disk=1, ephemeral=0,
+                   is_public=True, rxtx_factor=1.0, swap=0,
+                   extra_specs=None):
+        """add to the template a Flavor description"""
+        if name is None:
+            name = 'auto'
+        log.debug("adding Nova::Flavor '%s' vcpus '%d' ram '%d' disk '%d' " +
+                  "ephemeral '%d' is_public '%s' rxtx_factor '%d' " +
+                  "swap '%d' extra_specs '%s' ",
+                  name, vcpus, ram, disk, ephemeral, is_public,
+                  rxtx_factor, swap, str(extra_specs))
+
+        if extra_specs:
+            assert isinstance(extra_specs, collections.Mapping)
+
+        self.resources[name] = {
+            'type': 'OS::Nova::Flavor',
+            'properties': {'name': name,
+                           'disk': disk,
+                           'vcpus': vcpus,
+                           'swap': swap,
+                           'flavorid': name,
+                           'rxtx_factor': rxtx_factor,
+                           'ram': ram,
+                           'is_public': is_public,
+                           'ephemeral': ephemeral,
+                           'extra_specs': extra_specs}
+        }
+
+        self._template['outputs'][name] = {
+            'description': 'Flavor %s ID' % name,
+            'value': {'get_resource': name}
+        }
+
     def add_network(self, name, physical_network='physnet1', provider=None):
         """add to the template a Neutron Net"""
         log.debug("adding Neutron::Net '%s'", name)
@@ -397,34 +433,41 @@ class HeatTemplate(HeatObject):
             'value': {'get_resource': name}
         }
 
-    def add_server(self, name, image, flavor, ports=None, networks=None,
-                   scheduler_hints=None, user=None, key_name=None,
-                   user_data=None, metadata=None, additional_properties=None):
+    def add_server(self, name, image, flavor, flavors, ports=None,
+                   networks=None, scheduler_hints=None, user=None,
+                   key_name=None, user_data=None, metadata=None,
+                   additional_properties=None):
         """add to the template a Nova Server"""
         log.debug("adding Nova::Server '%s', image '%s', flavor '%s', "
                   "ports %s", name, image, flavor, ports)
 
         self.resources[name] = {
-            'type': 'OS::Nova::Server'
+            'type': 'OS::Nova::Server',
+            'depends_on': []
         }
 
         server_properties = {
             'name': name,
             'image': image,
-            'flavor': flavor,
+            'flavor': {},
             'networks': []  # list of dictionaries
         }
 
+        if flavor in flavors:
+            self.resources[name]['depends_on'].append(flavor)
+            server_properties["flavor"] = {'get_resource': flavor}
+        else:
+            server_properties["flavor"] = flavor
+
         if user:
             server_properties['admin_user'] = user
 
         if key_name:
-            self.resources[name]['depends_on'] = [key_name]
+            self.resources[name]['depends_on'].append(key_name)
             server_properties['key_name'] = {'get_resource': key_name}
 
         if ports:
-            self.resources[name]['depends_on'] = ports
-
+            self.resources[name]['depends_on'].extend(ports)
             for port in ports:
                 server_properties['networks'].append(
                     {'port': {'get_resource': port}}
@@ -460,7 +503,8 @@ class HeatTemplate(HeatObject):
 
     def create(self, block=True):
         """creates a template in the target cloud using heat
-        returns a dict with the requested output values from the template"""
+        returns a dict with the requested output values from the template
+        """
         log.info("Creating stack '%s'", self.name)
 
         # create stack early to support cleanup, e.g. ctrl-c while waiting