collectd: write config file from Jinja2 template

We have the collectd.conf inside the python package
so instead of copying it from various places,
write the template directly to the remote system.

collectd: read collect.conf template with pkgresources

read the collectd.conf file as a string directly
and upload without creating temp file

use Jinja2 template, disable failing plugins

use proper Jinja2 template, disable the plugins that
were failing to load and blocking startup

add support for per-testcase collectd.conf config
using YAML

add support for custom interval, default is 25 seconds

Change-Id: Id904f7b7c9f41a9dd7adf5dfa06c064d65c25d2d
Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>

3 files changed, 112 insertions, 81 deletions

diff --git a/yardstick/network_services/nfvi/collectd.conf b/yardstick/network_services/nfvi/collectd.conf
index 3928dcbca..22bd5d49d 100644
--- a/yardstick/network_services/nfvi/collectd.conf
+++ b/yardstick/network_services/nfvi/collectd.conf
@@ -15,7 +15,7 @@
 Hostname "nsb_stats"
 FQDNLookup true
 
-Interval {interval}
+Interval {{ interval }}
 
 ##############################################################################
 # LoadPlugin section                                                         #
@@ -24,7 +24,9 @@ Interval {interval}
 ##############################################################################
 
 #LoadPlugin syslog
-{loadplugin}
+{% for plugin in loadplugins %}
+LoadPlugin {{ plugin }}
+{% endfor %}
 
 ##############################################################################
 # Plugin configuration                                                       #
@@ -38,42 +40,31 @@ Interval {interval}
 #</Plugin>
 
 <Plugin amqp>
-	<Publish "name">
-		Host "0.0.0.0"
-		Port "5672"
-		VHost "/"
-		User "admin"
-		Password "admin"
-		Exchange "amq.fanout"
-		RoutingKey "collectd"
-		Persistent false
-		StoreRates false
-		ConnectionRetryDelay 0
-	</Publish>
+    <Publish "name">
+        Host "0.0.0.0"
+        Port "5672"
+        VHost "/"
+        User "admin"
+        Password "admin"
+        Exchange "amq.fanout"
+        RoutingKey "collectd"
+        Persistent false
+        StoreRates false
+        ConnectionRetryDelay 0
+    </Publish>
 </Plugin>
 
 <Plugin cpu>
-	ReportByCpu true
-	ReportByState true
-	ValuesPercentage true
+    ReportByCpu true
+    ReportByState true
+    ValuesPercentage true
 </Plugin>
 
 <Plugin memory>
-	ValuesAbsolute true
-	ValuesPercentage false
-</Plugin>
-
-<Plugin "intel_rdt">
-  Cores ""
+    ValuesAbsolute true
+    ValuesPercentage false
 </Plugin>
 
-<Plugin intel_pmu>
-     ReportHardwareCacheEvents true
-     ReportKernelPMUEvents true
-     ReportSoftwareEvents true
-     EventList "/root/.cache/pmu-events/GenuineIntel-6-2D-core.json"
-     HardwareEvents "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS" "L2_RQSTS.ALL_CODE_RD"
-</Plugin>
 
 <Plugin hugepages>
     ReportPerNodeHP  true
@@ -83,15 +74,25 @@ Interval {interval}
     ValuesPercentage false
 </Plugin>
 
-<Plugin hugepages>
-    ReportPerNodeHP  true
-    ReportRootHP     true
-    ValuesPages      true
-    ValuesBytes      false
-    ValuesPercentage false
+
+{% if "intel_rdt" in plugins %}
+<Plugin "intel_rdt">
+  Cores ""
+</Plugin>
+{% endif %}
+
+{% if "intel_pmu" in plugins %}
+<Plugin intel_pmu>
+   ReportHardwareCacheEvents true
+   ReportKernelPMUEvents true
+   ReportSoftwareEvents true
+   EventList "/root/.cache/pmu-events/GenuineIntel-6-2D-core.json"
+   HardwareEvents "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS" "L2_RQSTS.ALL_CODE_RD"
 </Plugin>
+{% endif %}
 
-<Plugin dpdkstat>
+{% if "dpdkstat" in plugins %}
+<Plugin "dpdkstat">
   <EAL>
     Coremask "0x1"
     MemoryChannels "4"
@@ -100,20 +101,24 @@ Interval {interval}
   </EAL>
   SharedMemObj "dpdk_collectd_stats_0"
   EnabledPortMask 0xffff
-  {dpdk_interface}
+{% for port_name in port_names %}
+  PortName {{ port_name }}
+{% endfor %}
 </Plugin>
+{% endif %}
 
-<Plugin virt>
-    Domain "samplevnf"
+{% if "virt" in plugins %}
+<Plugin "virt">
+# monitor all domains
 </Plugin>
+{% endif %}
 
-<Plugin ovs_stats>
+{% if "ovs_stats" in plugins %}
+<Plugin "ovs_stats">
   Port "6640"
   Address "127.0.0.1"
   Socket "/usr/local/var/run/openvswitch/db.sock"
-  Bridges "br0" "br_ext"
+# don't specify bridges, monitor all bridges
 </Plugin>
+{% endif %}
 
-<Include "/etc/collectd/collectd.conf.d">
-	Filter "*.conf"
-</Include>
diff --git a/yardstick/network_services/nfvi/collectd.sh b/yardstick/network_services/nfvi/collectd.sh
index 296c4a213..bdc5abd03 100755
--- a/yardstick/network_services/nfvi/collectd.sh
+++ b/yardstick/network_services/nfvi/collectd.sh
@@ -142,7 +142,8 @@ else
 fi
 
 modprobe msr
-cp $INSTALL_NSB_BIN/collectd.conf /opt/collectd/etc/
+# we overwrite the config file during _start_collectd so don't copy it
+#cp $INSTALL_NSB_BIN/collectd.conf /opt/nsb_bin/collectd/etc/
 sudo service rabbitmq-server restart
 echo "Check if admin user already created"
 rabbitmqctl list_users | grep '^admin$' > /dev/null
diff --git a/yardstick/network_services/nfvi/resource.py b/yardstick/network_services/nfvi/resource.py
index fa32a4dcf..d807f5e46 100644
--- a/yardstick/network_services/nfvi/resource.py
+++ b/yardstick/network_services/nfvi/resource.py
@@ -15,16 +15,22 @@
 
 from __future__ import absolute_import
 from __future__ import print_function
-import tempfile
+
 import logging
+from itertools import chain
+
+import jinja2
 import os
 import os.path
 import re
 import multiprocessing
+import pkg_resources
 
 from oslo_config import cfg
+from oslo_utils.encodeutils import safe_decode
 
 from yardstick import ssh
+from yardstick.common.task_template import finalize_for_yaml
 from yardstick.common.utils import validate_non_string_sequence
 from yardstick.network_services.nfvi.collectd import AmqpConsumer
 from yardstick.network_services.utils import get_nsb_option
@@ -34,26 +40,36 @@ LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 ZMQ_OVS_PORT = 5567
 ZMQ_POLLING_TIME = 12000
-LIST_PLUGINS_ENABLED = ["amqp", "cpu", "cpufreq", "intel_rdt", "memory",
-                        "hugepages", "dpdkstat", "virt", "ovs_stats", "intel_pmu"]
+LIST_PLUGINS_ENABLED = ["amqp", "cpu", "cpufreq", "memory",
+                        "hugepages"]
 
 
 class ResourceProfile(object):
     """
     This profile adds a resource at the beginning of the test session
     """
+    COLLECTD_CONF = "collectd.conf"
+    AMPQ_PORT = 5672
+    DEFAULT_INTERVAL = 25
 
-    def __init__(self, mgmt, interfaces=None, cores=None):
+    def __init__(self, mgmt, port_names=None, cores=None, plugins=None, interval=None):
+        if plugins is None:
+            self.plugins = {}
+        else:
+            self.plugins = plugins
+        if interval is None:
+            self.interval = self.DEFAULT_INTERVAL
+        else:
+            self.interval = interval
         self.enable = True
         self.cores = validate_non_string_sequence(cores, default=[])
         self._queue = multiprocessing.Queue()
         self.amqp_client = None
-        self.interfaces = validate_non_string_sequence(interfaces, default=[])
+        self.port_names = validate_non_string_sequence(port_names, default=[])
 
-        # why the host or ip?
-        self.vnfip = mgmt.get("host", mgmt["ip"])
-        self.connection = ssh.SSH.from_node(mgmt, overrides={"ip": self.vnfip})
-        self.connection.wait()
+        # we need to save mgmt so we can connect to port 5672
+        self.mgmt = mgmt
+        self.connection = ssh.AutoConnectSSH.from_node(mgmt)
 
     def check_if_sa_running(self, process):
         """ verify if system agent is running """
@@ -62,7 +78,7 @@ class ResourceProfile(object):
 
     def run_collectd_amqp(self):
         """ run amqp consumer to collect the NFVi data """
-        amqp_url = 'amqp://admin:admin@{}:5672/%2F'.format(self.vnfip)
+        amqp_url = 'amqp://admin:admin@{}:{}/%2F'.format(self.mgmt['ip'], self.AMPQ_PORT)
         amqp = AmqpConsumer(amqp_url, self._queue)
         try:
             amqp.run()
@@ -124,7 +140,9 @@ class ResourceProfile(object):
         }
         testcase = ""
 
-        for key, value in metrics.items():
+        # unicode decode
+        decoded = ((safe_decode(k, 'utf-8'), safe_decode(v, 'utf-8')) for k, v in metrics.items())
+        for key, value in decoded:
             key_split = key.split("/")
             res_key_iter = (key for key in key_split if "nsb_stats" not in key)
             res_key0 = next(res_key_iter)
@@ -176,35 +194,36 @@ class ResourceProfile(object):
         msg = self.parse_collectd_result(metric, self.cores)
         return msg
 
-    def _provide_config_file(self, bin_path, nfvi_cfg, kwargs):
-        with open(os.path.join(bin_path, nfvi_cfg), 'r') as cfg:
-            template = cfg.read()
-        cfg, cfg_content = tempfile.mkstemp()
-        with os.fdopen(cfg, "w+") as cfg:
-            cfg.write(template.format(**kwargs))
-        cfg_file = os.path.join(bin_path, nfvi_cfg)
-        self.connection.put(cfg_content, cfg_file)
-
-    def _prepare_collectd_conf(self, bin_path):
+    def _provide_config_file(self, config_file_path, nfvi_cfg, template_kwargs):
+        template = pkg_resources.resource_string("yardstick.network_services.nfvi",
+                                                 nfvi_cfg).decode('utf-8')
+        cfg_content = jinja2.Template(template, trim_blocks=True, lstrip_blocks=True,
+                                      finalize=finalize_for_yaml).render(
+            **template_kwargs)
+        # cfg_content = io.StringIO(template.format(**template_kwargs))
+        cfg_file = os.path.join(config_file_path, nfvi_cfg)
+        # must write as root, so use sudo
+        self.connection.execute("cat | sudo tee {}".format(cfg_file), stdin=cfg_content)
+
+    def _prepare_collectd_conf(self, config_file_path):
         """ Prepare collectd conf """
-        loadplugin = "\n".join("LoadPlugin {0}".format(plugin)
-                               for plugin in LIST_PLUGINS_ENABLED)
-
-        interfaces = "\n".join("PortName '{0[name]}'".format(interface)
-                               for interface in self.interfaces)
 
         kwargs = {
-            "interval": '25',
-            "loadplugin": loadplugin,
-            "dpdk_interface": interfaces,
+            "interval": self.interval,
+            "loadplugins": set(chain(LIST_PLUGINS_ENABLED, self.plugins.keys())),
+            # Optional fields PortName is descriptive only, use whatever is present
+            "port_names": self.port_names,
+            # "ovs_bridge_interfaces": ["br-int"],
+            "plugins": self.plugins,
         }
-        self._provide_config_file(bin_path, 'collectd.conf', kwargs)
+        self._provide_config_file(config_file_path, self.COLLECTD_CONF, kwargs)
 
     def _start_collectd(self, connection, bin_path):
         LOG.debug("Starting collectd to collect NFVi stats")
-        connection.execute('sudo pkill -9 collectd')
+        connection.execute('sudo pkill -x -9 collectd')
         bin_path = get_nsb_option("bin_path")
-        collectd_path = os.path.join(bin_path, "collectd", "collectd")
+        collectd_path = os.path.join(bin_path, "collectd", "sbin", "collectd")
+        config_file_path = os.path.join(bin_path, "collectd", "etc")
         exit_status = connection.execute("which %s > /dev/null 2>&1" % collectd_path)[0]
         if exit_status != 0:
             LOG.warning("%s is not present disabling", collectd_path)
@@ -217,7 +236,9 @@ class ResourceProfile(object):
             #     collectd_installer, http_proxy, https_proxy))
             return
         LOG.debug("Starting collectd to collect NFVi stats")
-        self._prepare_collectd_conf(bin_path)
+        # ensure collectd.conf.d exists to avoid error/warning
+        connection.execute("sudo mkdir -p /etc/collectd/collectd.conf.d")
+        self._prepare_collectd_conf(config_file_path)
 
         # Reset amqp queue
         LOG.debug("reset and setup amqp to collect data from collectd")
@@ -228,7 +249,7 @@ class ResourceProfile(object):
         connection.execute("sudo rabbitmqctl start_app")
         connection.execute("sudo service rabbitmq-server restart")
 
-        LOG.debug("Creating amdin user for rabbitmq in order to collect data from collectd")
+        LOG.debug("Creating admin user for rabbitmq in order to collect data from collectd")
         connection.execute("sudo rabbitmqctl delete_user guest")
         connection.execute("sudo rabbitmqctl add_user admin admin")
         connection.execute("sudo rabbitmqctl authenticate_user admin admin")
@@ -241,7 +262,11 @@ class ResourceProfile(object):
     def initiate_systemagent(self, bin_path):
         """ Start system agent for NFVi collection on host """
         if self.enable:
-            self._start_collectd(self.connection, bin_path)
+            try:
+                self._start_collectd(self.connection, bin_path)
+            except Exception:
+                LOG.exception("Exception during collectd start")
+                raise
 
     def start(self):
         """ start nfvi collection """