Heat: support non-mesh network toplogy

Previsouly we added all servers to every network
in Heat in a full mesh.

To more closely replicate test topology and to limit
then number of ports we need to all each server
to specify which ports should be connected to each network.

This should also allow for some kind of multiport setup.

Add optional network_ports dict to each server with network to port_list
mapping

match inteface based on port name or vld_id

replace vld_id matching with network name matching, since network_name == vld_id

Change-Id: I5de46b8f673949e3c17d8df6fa96f055c43886ce
Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>

7 files changed, 95 insertions, 86 deletions

diff --git a/tests/unit/benchmark/contexts/test_heat.py b/tests/unit/benchmark/contexts/test_heat.py
index cc0c7bc8e..582d9ab99 100644
--- a/tests/unit/benchmark/contexts/test_heat.py
+++ b/tests/unit/benchmark/contexts/test_heat.py
@@ -13,7 +13,6 @@
 
 from __future__ import absolute_import
 
-import ipaddress
 import logging
 import os
 import unittest
@@ -147,30 +146,6 @@ class HeatContextTestCase(unittest.TestCase):
             self.test_context.user = 'foo'
 
     @mock.patch('yardstick.benchmark.contexts.heat.HeatTemplate')
-    @mock.patch('yardstick.benchmark.contexts.heat.get_neutron_client')
-    def test_attrs_get(self, mock_neutron, mock_template):
-        image, flavor, user = expected_tuple = 'foo1', 'foo2', 'foo3'
-        self.assertNotEqual(self.test_context.image, image)
-        self.assertNotEqual(self.test_context.flavor, flavor)
-        self.assertNotEqual(self.test_context.user, user)
-        self.test_context._image = image
-        self.test_context._flavor = flavor
-        self.test_context._user = user
-        attr_tuple = self.test_context.image, self.test_context.flavor, self.test_context.user
-        self.assertEqual(attr_tuple, expected_tuple)
-
-    @mock.patch('yardstick.benchmark.contexts.heat.HeatTemplate')
-    def test_attrs_set_negative(self, mock_template):
-        with self.assertRaises(AttributeError):
-            self.test_context.image = 'foo'
-
-        with self.assertRaises(AttributeError):
-            self.test_context.flavor = 'foo'
-
-        with self.assertRaises(AttributeError):
-            self.test_context.user = 'foo'
-
-    @mock.patch('yardstick.benchmark.contexts.heat.HeatTemplate')
     def test_deploy(self, mock_template):
         self.test_context.name = 'foo'
         self.test_context.template_file = '/bar/baz/some-heat-file'
@@ -210,8 +185,8 @@ class HeatContextTestCase(unittest.TestCase):
         }
         server = mock.MagicMock()
         server.ports = OrderedDict([
-            ('a', {'stack_name': 'b'}),
-            ('c', {'stack_name': 'd'}),
+            ('a', {'stack_name': 'b', 'port': 'port_a'}),
+            ('c', {'stack_name': 'd', 'port': 'port_c'}),
         ])
 
         expected = {
@@ -231,7 +206,7 @@ class HeatContextTestCase(unittest.TestCase):
         self.test_context.add_server_port(server)
         self.assertEqual(server.private_ip, '10.20.30.45')
         self.assertEqual(len(server.interfaces), 2)
-        self.assertDictEqual(server.interfaces['a'], expected)
+        self.assertDictEqual(server.interfaces['port_a'], expected)
 
     @mock.patch('yardstick.benchmark.contexts.heat.HeatTemplate')
     def test_undeploy(self, mock_template):
@@ -472,7 +447,6 @@ class HeatContextTestCase(unittest.TestCase):
 
         network2 = mock.MagicMock()
         network2.name = 'net_2'
-        network2.vld_id = 'vld999'
         network2.segmentation_id = 'seg45'
         network2.network_type = 'type_b'
         network2.physical_network = 'virt'
@@ -488,16 +462,15 @@ class HeatContextTestCase(unittest.TestCase):
         attr_name = {}
         self.assertIsNone(self.test_context._get_network(attr_name))
 
-        attr_name = {'vld_id': 'vld777'}
+        attr_name = {'network_type': 'nosuch'}
         self.assertIsNone(self.test_context._get_network(attr_name))
 
         attr_name = 'vld777'
         self.assertIsNone(self.test_context._get_network(attr_name))
 
-        attr_name = {'vld_id': 'vld999'}
+        attr_name = {'segmentation_id': 'seg45'}
         expected = {
             "name": 'net_2',
-            "vld_id": 'vld999',
             "segmentation_id": 'seg45',
             "network_type": 'type_b',
             "physical_network": 'virt',
@@ -508,7 +481,6 @@ class HeatContextTestCase(unittest.TestCase):
         attr_name = 'a'
         expected = {
             "name": 'net_1',
-            "vld_id": 'vld111',
             "segmentation_id": 'seg54',
             "network_type": 'type_a',
             "physical_network": 'phys',
diff --git a/tests/unit/benchmark/core/test_task.py b/tests/unit/benchmark/core/test_task.py
index 14027e43c..e3917b5b2 100644
--- a/tests/unit/benchmark/core/test_task.py
+++ b/tests/unit/benchmark/core/test_task.py
@@ -66,31 +66,27 @@ class TaskTestCase(unittest.TestCase):
         nodes = {
             'node1': {
                 'interfaces': {
-                    'eth0': {
-                        'name': 'mgmt',
+                    'mgmt': {
+                        'network_name': 'mgmt',
                     },
-                    'eth1': {
-                        'name': 'external',
-                        'vld_id': '23',
+                    'xe0': {
+                        'network_name': 'private_0',
                     },
-                    'eth10': {
-                        'name': 'internal',
-                        'vld_id': '55',
+                    'xe1': {
+                        'network_name': 'public_0',
                     },
                 },
             },
             'node2': {
                 'interfaces': {
-                    'eth4': {
-                        'name': 'mgmt',
+                    'mgmt': {
+                        'network_name': 'mgmt',
                     },
-                    'eth2': {
-                        'name': 'external',
-                        'vld_id': '32',
+                    'private_0': {
+                        'network_name': 'private_0',
                     },
-                    'eth11': {
-                        'name': 'internal',
-                        'vld_id': '55',
+                    'public_0': {
+                        'network_name': 'public_0',
                     },
                 },
             },
@@ -99,30 +95,30 @@ class TaskTestCase(unittest.TestCase):
         mock_context.get_network.side_effect = iter([
             None,
             {
-                'name': 'a',
-                'network_type': 'private',
+                'name': 'mgmt',
+                'network_type': 'flat',
             },
             {},
             {
-                'name': 'b',
-                'vld_id': 'y',
+                'name': 'private_0',
                 'subnet_cidr': '10.20.0.0/16',
             },
             {
-                'name': 'c',
-                'vld_id': 'x',
+                'name': 'public_0',
+                'segmentation_id': '1001',
             },
             {
-                'name': 'd',
-                'vld_id': 'w',
+                'name': 'private_1',
             },
         ])
 
-        # once for each vld_id in the nodes dict
-        expected_get_network_calls = 4
+        # one for each interface
+        expected_get_network_calls = 6
         expected = {
-            'a': {'name': 'a', 'network_type': 'private'},
-            'b': {'name': 'b', 'vld_id': 'y', 'subnet_cidr': '10.20.0.0/16'},
+            'mgmt': {'name': 'mgmt', 'network_type': 'flat'},
+            'private_0': {'name': 'private_0', 'subnet_cidr': '10.20.0.0/16'},
+            'private_1': {'name': 'private_1'},
+            'public_0': {'name': 'public_0', 'segmentation_id': '1001'},
         }
 
         networks = task.get_networks_from_nodes(nodes)
diff --git a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
index 0264facf5..3bb4a9eca 100644
--- a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
+++ b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
@@ -412,6 +412,7 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
                         'virtual-interface': {
                             'dst_mac': '00:00:00:00:00:03',
                             'vpci': '0000:05:00.0',
+                            'dpdk_port_num': '0',
                             'driver': 'i40e',
                             'local_ip': '152.16.100.19',
                             'type': 'PCI-PASSTHROUGH',
@@ -419,7 +420,9 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
                             'dpdk_port_num': '0',
                             'bandwidth': '10 Gbps',
                             'dst_ip': '152.16.100.20',
-                            'local_mac': '00:00:00:00:00:01'
+                            'local_mac': '00:00:00:00:00:01',
+                            'vld_id': 'private_0',
+                            'ifname': 'xe0',
                         },
                         'vnfd-connection-point-ref': 'xe0',
                         'name': 'xe0'
@@ -428,6 +431,7 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
                         'virtual-interface': {
                             'dst_mac': '00:00:00:00:00:04',
                             'vpci': '0000:05:00.1',
+                            'dpdk_port_num': '1',
                             'driver': 'ixgbe',
                             'local_ip': '152.16.40.19',
                             'type': 'PCI-PASSTHROUGH',
@@ -435,7 +439,9 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
                             'dpdk_port_num': '1',
                             'bandwidth': '10 Gbps',
                             'dst_ip': '152.16.40.20',
-                            'local_mac': '00:00:00:00:00:02'
+                            'local_mac': '00:00:00:00:00:02',
+                            'vld_id': 'public_0',
+                            'ifname': 'xe1',
                         },
                         'vnfd-connection-point-ref': 'xe1',
                         'name': 'xe1'
@@ -600,8 +606,8 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
         dpdk_setup_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, scenario_helper)
         dpdk_setup_helper.CFG_CONFIG = 'config'
         dpdk_setup_helper.CFG_SCRIPT = 'script'
-        dpdk_setup_helper.all_ports = [3, 4, 5]
         dpdk_setup_helper.pipeline_kwargs = {}
+        dpdk_setup_helper.all_ports = [0, 1, 2]
 
         expected = {
             'cfg_file': 'config',
diff --git a/yardstick/benchmark/contexts/heat.py b/yardstick/benchmark/contexts/heat.py
index 575467f8f..94a3824a7 100644
--- a/yardstick/benchmark/contexts/heat.py
+++ b/yardstick/benchmark/contexts/heat.py
@@ -329,19 +329,24 @@ class HeatContext(Context):
 
     def add_server_port(self, server):
         # TODO(hafe) can only handle one internal network for now
-        port = next(iter(server.ports.values()))
-        server.private_ip = self.stack.outputs[port["stack_name"]]
+        # use private ip from first port
+        private_port = next(iter(server.ports.values()))
+        server.private_ip = self.stack.outputs[private_port["stack_name"]]
         server.interfaces = {}
         for network_name, port in server.ports.items():
-            server.interfaces[network_name] = self.make_interface_dict(
-                network_name, port['stack_name'], self.stack.outputs)
+            # port['port'] is either port name from mapping or default network_name
+            server.interfaces[port['port']] = self.make_interface_dict(network_name, port['port'],
+                                                                       port['stack_name'],
+                                                                       self.stack.outputs)
 
-    def make_interface_dict(self, network_name, stack_name, outputs):
+    def make_interface_dict(self, network_name, port, stack_name, outputs):
         private_ip = outputs[stack_name]
         mac_address = outputs[h_join(stack_name, "mac_address")]
+        # these are attributes of the network, not the port
         output_subnet_cidr = outputs[h_join(self.name, network_name,
                                             'subnet', 'cidr')]
 
+        # these are attributes of the network, not the port
         output_subnet_gateway = outputs[h_join(self.name, network_name,
                                                'subnet', 'gateway_ip')]
 
@@ -355,6 +360,7 @@ class HeatContext(Context):
             "mac_address": mac_address,
             "device_id": outputs[h_join(stack_name, "device_id")],
             "network_id": outputs[h_join(stack_name, "network_id")],
+            # this should be == vld_id for NSB tests
             "network_name": network_name,
             # to match vnf_generic
             "local_mac": mac_address,
@@ -438,9 +444,11 @@ class HeatContext(Context):
             network = self.networks.get(attr_name, None)
 
         else:
-            # Don't generalize too much  Just support vld_id
-            vld_id = attr_name.get('vld_id', {})
-            network_iter = (n for n in self.networks.values() if n.vld_id == vld_id)
+            # Only take the first key, value
+            key, value = next(iter(attr_name.items()), (None, None))
+            if key is None:
+                return None
+            network_iter = (n for n in self.networks.values() if getattr(n, key) == value)
             network = next(network_iter, None)
 
         if network is None:
@@ -448,7 +456,6 @@ class HeatContext(Context):
 
         result = {
             "name": network.name,
-            "vld_id": network.vld_id,
             "segmentation_id": network.segmentation_id,
             "network_type": network.network_type,
             "physical_network": network.physical_network,
diff --git a/yardstick/benchmark/contexts/model.py b/yardstick/benchmark/contexts/model.py
index 0b8197ce9..da2b74e1c 100644
--- a/yardstick/benchmark/contexts/model.py
+++ b/yardstick/benchmark/contexts/model.py
@@ -208,6 +208,7 @@ class Server(Object):     # pragma: no cover
             self.instances = attrs["instances"]
 
         # dict with key network name, each item is a dict with port name and ip
+        self.network_ports = attrs.get("network_ports", {})
         self.ports = {}
 
         self.floating_ip = None
@@ -253,8 +254,18 @@ class Server(Object):     # pragma: no cover
         """adds to the template one server and corresponding resources"""
         port_name_list = []
         for network in networks:
-            port_name = server_name + "-" + network.name + "-port"
-            self.ports[network.name] = {"stack_name": port_name}
+            # if explicit mapping skip unused networks
+            if self.network_ports:
+                try:
+                    port = self.network_ports[network.name]
+                except KeyError:
+                    # no port for this network
+                    continue
+            # otherwise add a port for every network with port name as network name
+            else:
+                port = network.name
+            port_name = "{0}-{1}-port".format(server_name, port)
+            self.ports[network.name] = {"stack_name": port_name, "port": port}
             # we can't use secgroups if port_security_enabled is False
             if network.port_security_enabled is False:
                 sec_group_id = None
diff --git a/yardstick/benchmark/core/task.py b/yardstick/benchmark/core/task.py
index a49a2cb71..29295537a 100644
--- a/yardstick/benchmark/core/task.py
+++ b/yardstick/benchmark/core/task.py
@@ -377,8 +377,6 @@ class Task(object):     # pragma: no cover
         target_attr: either a name for a server created by yardstick or a dict
         with attribute name mapping when using external heat templates
         """
-        host = None
-        target = None
         for context in self.contexts:
             if context.__context_type__ != "Heat":
                 continue
@@ -628,11 +626,11 @@ def get_networks_from_nodes(nodes):
             continue
         interfaces = node.get('interfaces', {})
         for interface in interfaces.values():
-            vld_id = interface.get('vld_id')
-            # mgmt network doesn't have vld_id
-            if not vld_id:
+            # vld_id is network_name
+            network_name = interface.get('network_name')
+            if not network_name:
                 continue
-            network = Context.get_network({"vld_id": vld_id})
+            network = Context.get_network(network_name)
             if network:
                 networks[network['name']] = network
     return networks
diff --git a/yardstick/benchmark/scenarios/networking/vnf_generic.py b/yardstick/benchmark/scenarios/networking/vnf_generic.py
index f7b2915a2..0e6ceab6e 100644
--- a/yardstick/benchmark/scenarios/networking/vnf_generic.py
+++ b/yardstick/benchmark/scenarios/networking/vnf_generic.py
@@ -216,7 +216,26 @@ class NetworkServiceTestCase(base.Scenario):
 
     @staticmethod
     def get_vld_networks(networks):
-        return {n['vld_id']: n for n in networks.values()}
+        # network name is vld_id
+        vld_map = {}
+        for name, n in networks.items():
+            try:
+                vld_map[n['vld_id']] = n
+            except KeyError:
+                vld_map[name] = n
+        return vld_map
+
+    @staticmethod
+    def find_node_if(nodes, name, if_name, vld_id):
+        try:
+            # check for xe0, xe1
+            intf = nodes[name]["interfaces"][if_name]
+        except KeyError:
+            # if not xe0, then maybe vld_id,  private_0, public_0
+            # pop it and re-insert with the correct name from topology
+            intf = nodes[name]["interfaces"].pop(vld_id)
+            nodes[name]["interfaces"][if_name] = intf
+        return intf
 
     def _resolve_topology(self):
         for vld in self.topology["vld"]:
@@ -234,8 +253,8 @@ class NetworkServiceTestCase(base.Scenario):
 
             try:
                 nodes = self.context_cfg["nodes"]
-                node0_if = nodes[node0_name]["interfaces"][node0_if_name]
-                node1_if = nodes[node1_name]["interfaces"][node1_if_name]
+                node0_if = self.find_node_if(nodes, node0_name, node0_if_name, vld["id"])
+                node1_if = self.find_node_if(nodes, node1_name, node1_if_name, vld["id"])
 
                 # names so we can do reverse lookups
                 node0_if["ifname"] = node0_if_name
@@ -285,8 +304,8 @@ class NetworkServiceTestCase(base.Scenario):
             node1_if_name = node1_data["vnfd-connection-point-ref"]
 
             nodes = self.context_cfg["nodes"]
-            node0_if = nodes[node0_name]["interfaces"][node0_if_name]
-            node1_if = nodes[node1_name]["interfaces"][node1_if_name]
+            node0_if = self.find_node_if(nodes, node0_name, node0_if_name, vld["id"])
+            node1_if = self.find_node_if(nodes, node1_name, node1_if_name, vld["id"])
 
             # add peer interface dict, but remove circular link
             # TODO: don't waste memory