NSB update

Refactored main NSB VNF classes accroding to class diagram

https://wiki.opnfv.org/display/yardstick/NSB+class+diagram

All the SampleVNFs have been separated and placed under
the SampleVNF class.

Added AutoConnectSSH to automatically create SSH conneciton on demand.

Added VnfdHelper class to wrap the VNFD dictionary in prepartion for
class-based modeling.

Extracted DpdkVnfSetupEnvHelper for DPDK based VNF setup.

Extracted Stats and other client config to ResourceHelper

Had to replace dict_key_flatten with deepgetitem due to Python 2.7
Jinja2 infinite recursion.

Change-Id: Ia8840e9c44cdbdf39aab6b02e6d2176b31937dc9
Signed-off-by: Deepak S <deepak.s@linux.intel.com>
Signed-off-by: Edward MacGillivray <edward.s.macgillivray@intel.com>
Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>

4 files changed, 131 insertions, 108 deletions

diff --git a/yardstick/benchmark/contexts/base.py b/yardstick/benchmark/contexts/base.py
index e362c6a3d..c9b5b51c9 100644
--- a/yardstick/benchmark/contexts/base.py
+++ b/yardstick/benchmark/contexts/base.py
@@ -18,6 +18,15 @@ class Context(object):
     """Class that represents a context in the logical model"""
     list = []
 
+    @staticmethod
+    def split_name(name, sep='.'):
+        try:
+            name_iter = iter(name.split(sep))
+        except AttributeError:
+            # name is not a string
+            return None, None
+        return next(name_iter), next(name_iter, None)
+
     def __init__(self):
         Context.list.append(self)
 
@@ -71,7 +80,23 @@ class Context(object):
         try:
             return next(s for s in servers if s)
         except StopIteration:
-            raise ValueError("context not found for server '%r'" %
+            raise ValueError("context not found for server %r" %
+                             attr_name)
+
+    @staticmethod
+    def get_context_from_server(attr_name):
+        """lookup context info by name from node config
+        attr_name: either a name of the node created by yardstick or a dict
+        with attribute name mapping when using external templates
+
+        :returns Context instance
+        """
+        servers = ((context._get_server(attr_name), context)
+                   for context in Context.list)
+        try:
+            return next(con for s, con in servers if s)
+        except StopIteration:
+            raise ValueError("context not found for name %r" %
                              attr_name)
 
     @staticmethod
@@ -85,5 +110,5 @@ class Context(object):
         try:
             return next(n for n in networks if n)
         except StopIteration:
-            raise ValueError("context not found for server '%r'" %
+            raise ValueError("context not found for server %r" %
                              attr_name)
diff --git a/yardstick/benchmark/contexts/heat.py b/yardstick/benchmark/contexts/heat.py
index d5349eab5..c8d53e324 100644
--- a/yardstick/benchmark/contexts/heat.py
+++ b/yardstick/benchmark/contexts/heat.py
@@ -17,7 +17,6 @@ import uuid
 from collections import OrderedDict
 
 import ipaddress
-import paramiko
 import pkg_resources
 
 from yardstick.benchmark.contexts.base import Context
@@ -28,12 +27,21 @@ from yardstick.benchmark.contexts.model import update_scheduler_hints
 from yardstick.common.openstack_utils import get_neutron_client
 from yardstick.orchestrator.heat import HeatTemplate, get_short_key_uuid
 from yardstick.common.constants import YARDSTICK_ROOT_PATH
+from yardstick.ssh import SSH
 
 LOG = logging.getLogger(__name__)
 
 DEFAULT_HEAT_TIMEOUT = 3600
 
 
+def join_args(sep, *args):
+    return sep.join(args)
+
+
+def h_join(*args):
+    return '-'.join(args)
+
+
 class HeatContext(Context):
     """Class that represents a context in the logical model"""
 
@@ -43,12 +51,14 @@ class HeatContext(Context):
         self.name = None
         self.stack = None
         self.networks = OrderedDict()
+        self.heat_timeout = None
         self.servers = []
         self.placement_groups = []
         self.server_groups = []
         self.keypair_name = None
         self.secgroup_name = None
         self._server_map = {}
+        self.attrs = {}
         self._image = None
         self._flavor = None
         self.flavors = set()
@@ -65,7 +75,8 @@ class HeatContext(Context):
              get_short_key_uuid(self.key_uuid)])
         super(HeatContext, self).__init__()
 
-    def assign_external_network(self, networks):
+    @staticmethod
+    def assign_external_network(networks):
         sorted_networks = sorted(networks.items())
         external_network = os.environ.get("EXTERNAL_NETWORK", "net04_ext")
 
@@ -74,8 +85,7 @@ class HeatContext(Context):
             # no external net defined, assign it to first network using os.environ
             sorted_networks[0][1]["external_network"] = external_network
 
-        self.networks = OrderedDict((name, Network(name, self, attrs))
-                                    for name, attrs in sorted_networks)
+        return sorted_networks
 
     def init(self, attrs):
         """initializes itself from the supplied arguments"""
@@ -88,8 +98,8 @@ class HeatContext(Context):
             self.heat_parameters = attrs.get("heat_parameters")
             return
 
-        self.keypair_name = self.name + "-key"
-        self.secgroup_name = self.name + "-secgroup"
+        self.keypair_name = h_join(self.name, "key")
+        self.secgroup_name = h_join(self.name, "secgroup")
 
         self._image = attrs.get("image")
 
@@ -97,29 +107,29 @@ class HeatContext(Context):
 
         self.heat_timeout = attrs.get("timeout", DEFAULT_HEAT_TIMEOUT)
 
-        self.placement_groups = [PlacementGroup(name, self, pgattrs["policy"])
-                                 for name, pgattrs in attrs.get(
+        self.placement_groups = [PlacementGroup(name, self, pg_attrs["policy"])
+                                 for name, pg_attrs in attrs.get(
                                  "placement_groups", {}).items()]
 
-        self.server_groups = [ServerGroup(name, self, sgattrs["policy"])
-                              for name, sgattrs in attrs.get(
+        self.server_groups = [ServerGroup(name, self, sg_attrs["policy"])
+                              for name, sg_attrs in attrs.get(
                               "server_groups", {}).items()]
 
         # we have to do this first, because we are injecting external_network
         # into the dict
-        self.assign_external_network(attrs["networks"])
+        sorted_networks = self.assign_external_network(attrs["networks"])
+
+        self.networks = OrderedDict(
+            (name, Network(name, self, net_attrs)) for name, net_attrs in
+            sorted_networks)
 
-        for name, serverattrs in sorted(attrs["servers"].items()):
-            server = Server(name, self, serverattrs)
+        for name, server_attrs in sorted(attrs["servers"].items()):
+            server = Server(name, self, server_attrs)
             self.servers.append(server)
             self._server_map[server.dn] = server
 
-        rsa_key = paramiko.RSAKey.generate(bits=2048, progress_func=None)
-        rsa_key.write_private_key_file(self.key_filename)
-        print("Writing %s ..." % self.key_filename)
-        with open(self.key_filename + ".pub", "w") as pubkey_file:
-            pubkey_file.write(
-                "%s %s\n" % (rsa_key.get_name(), rsa_key.get_base64()))
+        self.attrs = attrs
+        SSH.gen_keys(self.key_filename)
 
     @property
     def image(self):
@@ -188,7 +198,7 @@ class HeatContext(Context):
                 try:
                     self.flavors.add(server.flavor["name"])
                 except KeyError:
-                    self.flavors.add(server.stack_name + "-flavor")
+                    self.flavors.add(h_join(server.stack_name, "flavor"))
 
         # add servers with availability policy
         added_servers = []
@@ -286,7 +296,7 @@ class HeatContext(Context):
             # let the other failures happen, we want stack trace
             raise
 
-        # TODO: use Neutron to get segementation-id
+        # TODO: use Neutron to get segmentation-id
         self.get_neutron_info()
 
         # copy some vital stack output into server objects
@@ -311,24 +321,26 @@ class HeatContext(Context):
 
     def make_interface_dict(self, network_name, stack_name, outputs):
         private_ip = outputs[stack_name]
-        mac_addr = outputs[stack_name + "-mac_address"]
-        subnet_cidr_key = "-".join([self.name, network_name, 'subnet', 'cidr'])
-        gateway_key = "-".join([self.name, network_name, 'subnet', 'gateway_ip'])
-        subnet_cidr = outputs[subnet_cidr_key]
-        subnet_ip = ipaddress.ip_network(subnet_cidr)
+        mac_address = outputs[h_join(stack_name, "mac_address")]
+        output_subnet_cidr = outputs[h_join(self.name, network_name,
+                                            'subnet', 'cidr')]
+
+        output_subnet_gateway = outputs[h_join(self.name, network_name,
+                                               'subnet', 'gateway_ip')]
+
         return {
             "private_ip": private_ip,
-            "subnet_id": outputs[stack_name + "-subnet_id"],
-            "subnet_cidr": subnet_cidr,
-            "network": str(subnet_ip.network_address),
-            "netmask": str(subnet_ip.netmask),
-            "gateway_ip": outputs[gateway_key],
-            "mac_address": mac_addr,
-            "device_id": outputs[stack_name + "-device_id"],
-            "network_id": outputs[stack_name + "-network_id"],
+            "subnet_id": outputs[h_join(stack_name, "subnet_id")],
+            "subnet_cidr": output_subnet_cidr,
+            "network": str(ipaddress.ip_network(output_subnet_cidr).network_address),
+            "netmask": str(ipaddress.ip_network(output_subnet_cidr).netmask),
+            "gateway_ip": output_subnet_gateway,
+            "mac_address": mac_address,
+            "device_id": outputs[h_join(stack_name, "device_id")],
+            "network_id": outputs[h_join(stack_name, "network_id")],
             "network_name": network_name,
             # to match vnf_generic
-            "local_mac": mac_addr,
+            "local_mac": mac_address,
             "local_ip": private_ip,
             "vld_id": self.networks[network_name].vld_id,
         }
@@ -357,7 +369,8 @@ class HeatContext(Context):
                 "network": intf["network"],
                 "netmask": intf["netmask"],
                 "if": name,
-                "gateway": intf["gateway_ip"],
+                # We have to encode a None gateway as '' for Jinja2 to YAML conversion
+                "gateway": intf["gateway_ip"] if intf["gateway_ip"] else '',
             }
             for name, intf in server.interfaces.items()
         ]
@@ -370,31 +383,24 @@ class HeatContext(Context):
         """
         key_filename = pkg_resources.resource_filename(
             'yardstick.resources',
-            'files/yardstick_key-' + get_short_key_uuid(self.key_uuid))
-
-        if not isinstance(attr_name, collections.Mapping):
-            server = self._server_map.get(attr_name, None)
+            h_join('files/yardstick_key', get_short_key_uuid(self.key_uuid)))
 
-        else:
-            cname = attr_name["name"].split(".")[1]
-            if cname != self.name:
+        if isinstance(attr_name, collections.Mapping):
+            node_name, cname = self.split_name(attr_name['name'])
+            if cname is None or cname != self.name:
                 return None
 
-            public_ip = None
-            private_ip = None
-            if "public_ip_attr" in attr_name:
-                public_ip = self.stack.outputs[attr_name["public_ip_attr"]]
-            if "private_ip_attr" in attr_name:
-                private_ip = self.stack.outputs[
-                    attr_name["private_ip_attr"]]
-
             # Create a dummy server instance for holding the *_ip attributes
-            server = Server(attr_name["name"].split(".")[0], self, {})
-            server.public_ip = public_ip
-            server.private_ip = private_ip
+            server = Server(node_name, self, {})
+            server.public_ip = self.stack.outputs.get(
+                attr_name.get("public_ip_attr", object()), None)
 
-        if server is None:
-            return None
+            server.private_ip = self.stack.outputs.get(
+                attr_name.get("private_ip_attr", object()), None)
+        else:
+            server = self._server_map.get(attr_name, None)
+            if server is None:
+                return None
 
         result = {
             "user": server.context.user,
@@ -417,12 +423,9 @@ class HeatContext(Context):
 
         else:
             # Don't generalize too much  Just support vld_id
-            vld_id = attr_name.get('vld_id')
-            if vld_id is None:
-                return None
-
-            network = next((n for n in self.networks.values() if
-                           getattr(n, "vld_id", None) == vld_id), None)
+            vld_id = attr_name.get('vld_id', {})
+            network_iter = (n for n in self.networks.values() if n.vld_id == vld_id)
+            network = next(network_iter, None)
 
         if network is None:
             return None
diff --git a/yardstick/benchmark/contexts/node.py b/yardstick/benchmark/contexts/node.py
index b3f0aca0e..78a2d1f46 100644
--- a/yardstick/benchmark/contexts/node.py
+++ b/yardstick/benchmark/contexts/node.py
@@ -19,7 +19,7 @@ import pkg_resources
 
 from yardstick import ssh
 from yardstick.benchmark.contexts.base import Context
-from yardstick.common import constants as consts
+from yardstick.common.constants import ANSIBLE_DIR, YARDSTICK_ROOT_PATH
 
 LOG = logging.getLogger(__name__)
 
@@ -38,6 +38,7 @@ class NodeContext(Context):
         self.computes = []
         self.baremetals = []
         self.env = {}
+        self.attrs = {}
         super(NodeContext, self).__init__()
 
     def read_config_file(self):
@@ -45,24 +46,23 @@ class NodeContext(Context):
 
         with open(self.file_path) as stream:
             LOG.info("Parsing pod file: %s", self.file_path)
-            cfg = yaml.load(stream)
+            cfg = yaml.safe_load(stream)
         return cfg
 
     def init(self, attrs):
         """initializes itself from the supplied arguments"""
         self.name = attrs["name"]
-        self.file_path = attrs.get("file", "pod.yaml")
+        self.file_path = file_path = attrs.get("file", "pod.yaml")
 
         try:
             cfg = self.read_config_file()
-        except IOError as ioerror:
-            if ioerror.errno == errno.ENOENT:
-                self.file_path = \
-                    os.path.join(consts.YARDSTICK_ROOT_PATH, self.file_path)
-                cfg = self.read_config_file()
-            else:
+        except IOError as io_error:
+            if io_error.errno != errno.ENOENT:
                 raise
 
+            self.file_path = os.path.join(YARDSTICK_ROOT_PATH, file_path)
+            cfg = self.read_config_file()
+
         self.nodes.extend(cfg["nodes"])
         self.controllers.extend([node for node in cfg["nodes"]
                                  if node["role"] == "Controller"])
@@ -76,6 +76,7 @@ class NodeContext(Context):
         LOG.debug("BareMetals: %r", self.baremetals)
 
         self.env = attrs.get('env', {})
+        self.attrs = attrs
         LOG.debug("Env: %r", self.env)
 
         # add optional static network definition
@@ -112,19 +113,17 @@ class NodeContext(Context):
 
     def _do_ansible_job(self, path):
         cmd = 'ansible-playbook -i inventory.ini %s' % path
-        p = subprocess.Popen(cmd, shell=True, cwd=consts.ANSIBLE_DIR)
+        p = subprocess.Popen(cmd, shell=True, cwd=ANSIBLE_DIR)
         p.communicate()
 
     def _get_server(self, attr_name):
         """lookup server info by name from context
         attr_name: a name for a server listed in nodes config file
         """
-        if isinstance(attr_name, collections.Mapping):
+        node_name, name = self.split_name(attr_name)
+        if name is None or self.name != name:
             return None
 
-        if self.name != attr_name.split(".")[1]:
-            return None
-        node_name = attr_name.split(".")[0]
         matching_nodes = (n for n in self.nodes if n["name"] == node_name)
 
         try:
@@ -140,9 +139,10 @@ class NodeContext(Context):
             pass
         else:
             raise ValueError("Duplicate nodes!!! Nodes: %s %s",
-                             (matching_nodes, duplicate))
+                             (node, duplicate))
 
         node["name"] = attr_name
+        node.setdefault("interfaces", {})
         return node
 
     def _get_network(self, attr_name):
@@ -151,12 +151,10 @@ class NodeContext(Context):
 
         else:
             # Don't generalize too much  Just support vld_id
-            vld_id = attr_name.get('vld_id')
-            if vld_id is None:
-                return None
-
-            network = next((n for n in self.networks.values() if
-                           n.get("vld_id") == vld_id), None)
+            vld_id = attr_name.get('vld_id', {})
+            # for node context networks are dicts
+            iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
+            network = next(iter1, None)
 
         if network is None:
             return None
@@ -193,7 +191,7 @@ class NodeContext(Context):
 
     def _execute_local_script(self, info):
         script, options = self._get_script(info)
-        script = os.path.join(consts.YARDSTICK_ROOT_PATH, script)
+        script = os.path.join(YARDSTICK_ROOT_PATH, script)
         cmd = ['bash', script, options]
 
         p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
diff --git a/yardstick/benchmark/contexts/standalone.py b/yardstick/benchmark/contexts/standalone.py
index 2bc1f3755..ae1046974 100644
--- a/yardstick/benchmark/contexts/standalone.py
+++ b/yardstick/benchmark/contexts/standalone.py
@@ -15,6 +15,7 @@
 
 from __future__ import absolute_import
 import logging
+import os
 import errno
 import collections
 import yaml
@@ -41,14 +42,15 @@ class StandaloneContext(Context):
         self.networks = {}
         self.nfvi_node = []
         self.nfvi_obj = None
-        super(self.__class__, self).__init__()
+        self.attrs = {}
+        super(StandaloneContext, self).__init__()
 
     def read_config_file(self):
         """Read from config file"""
 
         with open(self.file_path) as stream:
             LOG.info("Parsing pod file: %s", self.file_path)
-            cfg = yaml.load(stream)
+            cfg = yaml.safe_load(stream)
         return cfg
 
     def get_nfvi_obj(self):
@@ -63,17 +65,15 @@ class StandaloneContext(Context):
         """initializes itself from the supplied arguments"""
 
         self.name = attrs["name"]
-        self.file_path = attrs.get("file", "pod.yaml")
-        LOG.info("Parsing pod file: %s", self.file_path)
+        self.file_path = file_path = attrs.get("file", "pod.yaml")
 
         try:
             cfg = self.read_config_file()
-        except IOError as ioerror:
-            if ioerror.errno == errno.ENOENT:
-                self.file_path = YARDSTICK_ROOT_PATH + self.file_path
-                cfg = self.read_config_file()
-            else:
+        except IOError as io_error:
+            if io_error.errno != errno.ENOENT:
                 raise
+            self.file_path = os.path.join(YARDSTICK_ROOT_PATH, file_path)
+            cfg = self.read_config_file()
 
         self.vm_deploy = attrs.get("vm_deploy", True)
         self.nodes.extend([node for node in cfg["nodes"]
@@ -90,6 +90,7 @@ class StandaloneContext(Context):
             else:
                 LOG.debug("Node role is other than SRIOV and OVS")
         self.nfvi_obj = self.get_nfvi_obj()
+        self.attrs = attrs
         # add optional static network definition
         self.networks.update(cfg.get("networks", {}))
         self.nfvi_obj = self.get_nfvi_obj()
@@ -146,11 +147,10 @@ class StandaloneContext(Context):
         Keyword arguments:
         attr_name -- A name for a server listed in nodes config file
         """
-        if isinstance(attr_name, collections.Mapping):
-            return None
-        if self.name != attr_name.split(".")[1]:
+        node_name, name = self.split_name(attr_name)
+        if name is None or self.name != name:
             return None
-        node_name = attr_name.split(".")[0]
+
         matching_nodes = (n for n in self.nodes if n["name"] == node_name)
         try:
             # A clone is created in order to avoid affecting the
@@ -165,7 +165,8 @@ class StandaloneContext(Context):
             pass
         else:
             raise ValueError("Duplicate nodes!!! Nodes: %s %s",
-                             (matching_nodes, duplicate))
+                             (node, duplicate))
+
         node["name"] = attr_name
         return node
 
@@ -175,14 +176,10 @@ class StandaloneContext(Context):
 
         else:
             # Don't generalize too much  Just support vld_id
-            vld_id = attr_name.get('vld_id')
-            if vld_id is None:
-                return None
-            try:
-                network = next(n for n in self.networks.values() if
-                               n.get("vld_id") == vld_id)
-            except StopIteration:
-                return None
+            vld_id = attr_name.get('vld_id', {})
+            # for standalone context networks are dicts
+            iter1 = (n for n in self.networks.values() if n.get('vld_id') == vld_id)
+            network = next(iter1, None)
 
         if network is None:
             return None