summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRoss Brattain <ross.b.brattain@intel.com>2017-09-05 15:38:52 -0700
committerRoss Brattain <ross.b.brattain@intel.com>2017-09-27 03:15:00 +0000
commitbd11ffdec1f18785696b5c4872e5bd5e1012249c (patch)
treed3f739d4f2f694c6f798bae3babf4341b783fd86
parentc56ca9c1dccaa11a2ff3af588b45d85207fb1e40 (diff)
collectd: write config file from Jinja2 template
We have the collectd.conf inside the python package so instead of copying it from various places, write the template directly to the remote system. collectd: read collect.conf template with pkgresources read the collectd.conf file as a string directly and upload without creating temp file use Jinja2 template, disable failing plugins use proper Jinja2 template, disable the plugins that were failing to load and blocking startup add support for per-testcase collectd.conf config using YAML add support for custom interval, default is 25 seconds Change-Id: Id904f7b7c9f41a9dd7adf5dfa06c064d65c25d2d Signed-off-by: Ross Brattain <ross.b.brattain@intel.com>
-rw-r--r--tests/unit/network_services/nfvi/test_resource.py163
-rw-r--r--tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py10
-rw-r--r--yardstick/network_services/nfvi/collectd.conf95
-rwxr-xr-xyardstick/network_services/nfvi/collectd.sh3
-rw-r--r--yardstick/network_services/nfvi/resource.py95
-rw-r--r--yardstick/network_services/vnf_generic/vnf/base.py9
-rw-r--r--yardstick/network_services/vnf_generic/vnf/sample_vnf.py23
7 files changed, 175 insertions, 223 deletions
diff --git a/tests/unit/network_services/nfvi/test_resource.py b/tests/unit/network_services/nfvi/test_resource.py
index 1c2c1f3e2..eba38c688 100644
--- a/tests/unit/network_services/nfvi/test_resource.py
+++ b/tests/unit/network_services/nfvi/test_resource.py
@@ -14,7 +14,6 @@
from __future__ import absolute_import
import unittest
-import multiprocessing
import mock
from yardstick.network_services.nfvi.resource import ResourceProfile
@@ -86,17 +85,20 @@ class TestResourceProfile(unittest.TestCase):
'id': 'VpeApproxVnf', 'name': 'VPEVnfSsh'}]}}
def setUp(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
+ with mock.patch("yardstick.ssh.AutoConnectSSH") as ssh:
+ self.ssh_mock = mock.Mock(autospec=ssh.SSH)
+ self.ssh_mock.execute = \
mock.Mock(return_value=(0, {}, ""))
- ssh.from_node.return_value = ssh_mock
+ ssh.from_node.return_value = self.ssh_mock
mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
+ # interfaces = \
+ # self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
+ port_names = \
self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
self.resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
+ ResourceProfile(mgmt, port_names, [1, 2, 3])
+ self.resource_profile.connection = self.ssh_mock
def test___init__(self):
self.assertEqual(True, self.resource_profile.enable)
@@ -118,133 +120,33 @@ class TestResourceProfile(unittest.TestCase):
self.assertEqual(val, ('error', 'Invalid', '', ''))
def test__start_collectd(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._prepare_collectd_conf = mock.Mock()
- self.assertIsNone(
- resource_profile._start_collectd(ssh_mock, "/opt/nsb_bin"))
-
- def test__prepare_collectd_conf_BM(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._provide_config_file = mock.Mock()
- self.assertIsNone(
- resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
-
- def test__prepare_collectd_conf_managed_ovs_dpdk(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._provide_config_file = mock.Mock()
- self.assertIsNone(
- resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
-
- def test__prepare_collectd_conf_ovs_dpdk(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._provide_config_file = mock.Mock()
self.assertIsNone(
- resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
+ self.resource_profile._start_collectd(self.ssh_mock, "/opt/nsb_bin"))
- def test__prepare_collectd_conf_managed_sriov(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._provide_config_file = mock.Mock()
+ def test__prepare_collectd_conf(self):
self.assertIsNone(
- resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
+ self.resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
- def test__prepare_collectd_conf_sriov(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._provide_config_file = mock.Mock()
- self.assertIsNone(
- resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
@mock.patch("yardstick.network_services.nfvi.resource.open")
- @mock.patch("yardstick.network_services.nfvi.resource.tempfile")
@mock.patch("yardstick.network_services.nfvi.resource.os")
- def test__provide_config_file(self, mock_open, mock_tempfile, mock_os):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._prepare_collectd_conf = mock.Mock()
- resource_profile.connection = ssh_mock
- resource_profile.connection.put = \
- mock.Mock(return_value=(0, "", ""))
- mock_tempfile.mkstemp = mock.Mock(return_value=["test", ""])
- self.assertIsNone(
- resource_profile._provide_config_file("/opt/nsb_bin",
- "collectd.cfg", {}))
+ def test__provide_config_file(self, mock_open, mock_os):
+ loadplugin = range(5)
+ port_names = range(5)
+ kwargs = {
+ "interval": '25',
+ "loadplugin": loadplugin,
+ "port_names": port_names,
+ }
+ self.resource_profile._provide_config_file("/opt/nsb_bin", "collectd.conf", kwargs)
+ self.ssh_mock.execute.assert_called_once()
+
@mock.patch("yardstick.network_services.nfvi.resource.open")
def test_initiate_systemagent(self, mock_open):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._start_collectd = mock.Mock()
- self.assertIsNone(
- resource_profile.initiate_systemagent("/opt/nsb_bin"))
+ self.resource_profile._start_collectd = mock.Mock()
+ self.assertIsNone(
+ self.resource_profile.initiate_systemagent("/opt/nsb_bin"))
def test__parse_hugepages(self):
reskey = ["cpu", "cpuFreq"]
@@ -301,21 +203,21 @@ class TestResourceProfile(unittest.TestCase):
self.assertDictEqual(res, expected_result)
def test_parse_collectd_result_hugepage(self):
- metric = {"nsb_stats/hugepages/free": "101"}
+ # amqp returns bytes
+ metric = {b"nsb_stats/hugepages/free": b"101"}
self.resource_profile.parse_hugepages = \
mock.Mock(return_value={"free": "101"})
res = self.resource_profile.parse_collectd_result(metric, [0, 1, 2])
- expected_result = {'cpu': {}, 'dpdkstat': {}, 'hugepages': {'free':
- '101'},
+ expected_result = {'cpu': {}, 'dpdkstat': {}, 'hugepages': {'free': '101'},
'memory': {}, 'ovs_stats': {}, 'timestamp': '',
'intel_pmu': {},
'virt': {}}
self.assertDictEqual(res, expected_result)
def test_parse_collectd_result_dpdk_virt_ovs(self):
- metric = {"nsb_stats/dpdkstat/tx": "101",
- "nsb_stats/ovs_stats/tx": "101",
- "nsb_stats/virt/virt/memory": "101"}
+ metric = {b"nsb_stats/dpdkstat/tx": b"101",
+ b"nsb_stats/ovs_stats/tx": b"101",
+ b"nsb_stats/virt/virt/memory": b"101"}
self.resource_profile.parse_dpdkstat = \
mock.Mock(return_value={"tx": "101"})
self.resource_profile.parse_virt = \
@@ -347,7 +249,6 @@ class TestResourceProfile(unittest.TestCase):
self.assertIsNotNone(res)
def test_run_collectd_amqp(self):
- _queue = multiprocessing.Queue()
resource.AmqpConsumer = mock.Mock(autospec=collectd)
self.assertIsNone(self.resource_profile.run_collectd_amqp())
diff --git a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
index 4b9f4172e..d0c4b6f42 100644
--- a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
+++ b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
@@ -723,8 +723,9 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
result = dpdk_setup_helper._validate_cpu_cfg()
self.assertEqual(result, expected)
+ @mock.patch('yardstick.network_services.vnf_generic.vnf.sample_vnf.time')
@mock.patch('yardstick.ssh.SSH')
- def test_setup_vnf_environment(self, _):
+ def test_setup_vnf_environment(self, _, mock_time):
def execute(cmd, *args, **kwargs):
if cmd.startswith('which '):
return exec_failure
@@ -782,6 +783,8 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
dpdk_setup_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, scenario_helper)
dpdk_setup_helper._validate_cpu_cfg = mock.Mock()
+ dpdk_setup_helper.bound_pci = [v['virtual-interface']["vpci"] for v in
+ vnfd_helper.interfaces]
result = dpdk_setup_helper._setup_resources()
self.assertIsInstance(result, ResourceProfile)
self.assertEqual(dpdk_setup_helper.socket, 0)
@@ -796,11 +799,14 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
dpdk_setup_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, scenario_helper)
dpdk_setup_helper._validate_cpu_cfg = mock.Mock()
+ dpdk_setup_helper.bound_pci = [v['virtual-interface']["vpci"] for v in
+ vnfd_helper.interfaces]
result = dpdk_setup_helper._setup_resources()
self.assertIsInstance(result, ResourceProfile)
self.assertEqual(dpdk_setup_helper.socket, 1)
- def test__detect_and_bind_drivers(self):
+ @mock.patch('yardstick.network_services.vnf_generic.vnf.sample_vnf.time')
+ def test__detect_and_bind_drivers(self, mock_time):
vnfd_helper = VnfdHelper(deepcopy(self.VNFD_0))
ssh_helper = mock.Mock()
# ssh_helper.execute = mock.Mock(return_value = (0, 'text', ''))
diff --git a/yardstick/network_services/nfvi/collectd.conf b/yardstick/network_services/nfvi/collectd.conf
index 3928dcbca..22bd5d49d 100644
--- a/yardstick/network_services/nfvi/collectd.conf
+++ b/yardstick/network_services/nfvi/collectd.conf
@@ -15,7 +15,7 @@
Hostname "nsb_stats"
FQDNLookup true
-Interval {interval}
+Interval {{ interval }}
##############################################################################
# LoadPlugin section #
@@ -24,7 +24,9 @@ Interval {interval}
##############################################################################
#LoadPlugin syslog
-{loadplugin}
+{% for plugin in loadplugins %}
+LoadPlugin {{ plugin }}
+{% endfor %}
##############################################################################
# Plugin configuration #
@@ -38,42 +40,31 @@ Interval {interval}
#</Plugin>
<Plugin amqp>
- <Publish "name">
- Host "0.0.0.0"
- Port "5672"
- VHost "/"
- User "admin"
- Password "admin"
- Exchange "amq.fanout"
- RoutingKey "collectd"
- Persistent false
- StoreRates false
- ConnectionRetryDelay 0
- </Publish>
+ <Publish "name">
+ Host "0.0.0.0"
+ Port "5672"
+ VHost "/"
+ User "admin"
+ Password "admin"
+ Exchange "amq.fanout"
+ RoutingKey "collectd"
+ Persistent false
+ StoreRates false
+ ConnectionRetryDelay 0
+ </Publish>
</Plugin>
<Plugin cpu>
- ReportByCpu true
- ReportByState true
- ValuesPercentage true
+ ReportByCpu true
+ ReportByState true
+ ValuesPercentage true
</Plugin>
<Plugin memory>
- ValuesAbsolute true
- ValuesPercentage false
-</Plugin>
-
-<Plugin "intel_rdt">
- Cores ""
+ ValuesAbsolute true
+ ValuesPercentage false
</Plugin>
-<Plugin intel_pmu>
- ReportHardwareCacheEvents true
- ReportKernelPMUEvents true
- ReportSoftwareEvents true
- EventList "/root/.cache/pmu-events/GenuineIntel-6-2D-core.json"
- HardwareEvents "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS" "L2_RQSTS.ALL_CODE_RD"
-</Plugin>
<Plugin hugepages>
ReportPerNodeHP true
@@ -83,15 +74,25 @@ Interval {interval}
ValuesPercentage false
</Plugin>
-<Plugin hugepages>
- ReportPerNodeHP true
- ReportRootHP true
- ValuesPages true
- ValuesBytes false
- ValuesPercentage false
+
+{% if "intel_rdt" in plugins %}
+<Plugin "intel_rdt">
+ Cores ""
+</Plugin>
+{% endif %}
+
+{% if "intel_pmu" in plugins %}
+<Plugin intel_pmu>
+ ReportHardwareCacheEvents true
+ ReportKernelPMUEvents true
+ ReportSoftwareEvents true
+ EventList "/root/.cache/pmu-events/GenuineIntel-6-2D-core.json"
+ HardwareEvents "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS" "L2_RQSTS.ALL_CODE_RD"
</Plugin>
+{% endif %}
-<Plugin dpdkstat>
+{% if "dpdkstat" in plugins %}
+<Plugin "dpdkstat">
<EAL>
Coremask "0x1"
MemoryChannels "4"
@@ -100,20 +101,24 @@ Interval {interval}
</EAL>
SharedMemObj "dpdk_collectd_stats_0"
EnabledPortMask 0xffff
- {dpdk_interface}
+{% for port_name in port_names %}
+ PortName {{ port_name }}
+{% endfor %}
</Plugin>
+{% endif %}
-<Plugin virt>
- Domain "samplevnf"
+{% if "virt" in plugins %}
+<Plugin "virt">
+# monitor all domains
</Plugin>
+{% endif %}
-<Plugin ovs_stats>
+{% if "ovs_stats" in plugins %}
+<Plugin "ovs_stats">
Port "6640"
Address "127.0.0.1"
Socket "/usr/local/var/run/openvswitch/db.sock"
- Bridges "br0" "br_ext"
+# don't specify bridges, monitor all bridges
</Plugin>
+{% endif %}
-<Include "/etc/collectd/collectd.conf.d">
- Filter "*.conf"
-</Include>
diff --git a/yardstick/network_services/nfvi/collectd.sh b/yardstick/network_services/nfvi/collectd.sh
index 296c4a213..bdc5abd03 100755
--- a/yardstick/network_services/nfvi/collectd.sh
+++ b/yardstick/network_services/nfvi/collectd.sh
@@ -142,7 +142,8 @@ else
fi
modprobe msr
-cp $INSTALL_NSB_BIN/collectd.conf /opt/collectd/etc/
+# we overwrite the config file during _start_collectd so don't copy it
+#cp $INSTALL_NSB_BIN/collectd.conf /opt/nsb_bin/collectd/etc/
sudo service rabbitmq-server restart
echo "Check if admin user already created"
rabbitmqctl list_users | grep '^admin$' > /dev/null
diff --git a/yardstick/network_services/nfvi/resource.py b/yardstick/network_services/nfvi/resource.py
index fa32a4dcf..d807f5e46 100644
--- a/yardstick/network_services/nfvi/resource.py
+++ b/yardstick/network_services/nfvi/resource.py
@@ -15,16 +15,22 @@
from __future__ import absolute_import
from __future__ import print_function
-import tempfile
+
import logging
+from itertools import chain
+
+import jinja2
import os
import os.path
import re
import multiprocessing
+import pkg_resources
from oslo_config import cfg
+from oslo_utils.encodeutils import safe_decode
from yardstick import ssh
+from yardstick.common.task_template import finalize_for_yaml
from yardstick.common.utils import validate_non_string_sequence
from yardstick.network_services.nfvi.collectd import AmqpConsumer
from yardstick.network_services.utils import get_nsb_option
@@ -34,26 +40,36 @@ LOG = logging.getLogger(__name__)
CONF = cfg.CONF
ZMQ_OVS_PORT = 5567
ZMQ_POLLING_TIME = 12000
-LIST_PLUGINS_ENABLED = ["amqp", "cpu", "cpufreq", "intel_rdt", "memory",
- "hugepages", "dpdkstat", "virt", "ovs_stats", "intel_pmu"]
+LIST_PLUGINS_ENABLED = ["amqp", "cpu", "cpufreq", "memory",
+ "hugepages"]
class ResourceProfile(object):
"""
This profile adds a resource at the beginning of the test session
"""
+ COLLECTD_CONF = "collectd.conf"
+ AMPQ_PORT = 5672
+ DEFAULT_INTERVAL = 25
- def __init__(self, mgmt, interfaces=None, cores=None):
+ def __init__(self, mgmt, port_names=None, cores=None, plugins=None, interval=None):
+ if plugins is None:
+ self.plugins = {}
+ else:
+ self.plugins = plugins
+ if interval is None:
+ self.interval = self.DEFAULT_INTERVAL
+ else:
+ self.interval = interval
self.enable = True
self.cores = validate_non_string_sequence(cores, default=[])
self._queue = multiprocessing.Queue()
self.amqp_client = None
- self.interfaces = validate_non_string_sequence(interfaces, default=[])
+ self.port_names = validate_non_string_sequence(port_names, default=[])
- # why the host or ip?
- self.vnfip = mgmt.get("host", mgmt["ip"])
- self.connection = ssh.SSH.from_node(mgmt, overrides={"ip": self.vnfip})
- self.connection.wait()
+ # we need to save mgmt so we can connect to port 5672
+ self.mgmt = mgmt
+ self.connection = ssh.AutoConnectSSH.from_node(mgmt)
def check_if_sa_running(self, process):
""" verify if system agent is running """
@@ -62,7 +78,7 @@ class ResourceProfile(object):
def run_collectd_amqp(self):
""" run amqp consumer to collect the NFVi data """
- amqp_url = 'amqp://admin:admin@{}:5672/%2F'.format(self.vnfip)
+ amqp_url = 'amqp://admin:admin@{}:{}/%2F'.format(self.mgmt['ip'], self.AMPQ_PORT)
amqp = AmqpConsumer(amqp_url, self._queue)
try:
amqp.run()
@@ -124,7 +140,9 @@ class ResourceProfile(object):
}
testcase = ""
- for key, value in metrics.items():
+ # unicode decode
+ decoded = ((safe_decode(k, 'utf-8'), safe_decode(v, 'utf-8')) for k, v in metrics.items())
+ for key, value in decoded:
key_split = key.split("/")
res_key_iter = (key for key in key_split if "nsb_stats" not in key)
res_key0 = next(res_key_iter)
@@ -176,35 +194,36 @@ class ResourceProfile(object):
msg = self.parse_collectd_result(metric, self.cores)
return msg
- def _provide_config_file(self, bin_path, nfvi_cfg, kwargs):
- with open(os.path.join(bin_path, nfvi_cfg), 'r') as cfg:
- template = cfg.read()
- cfg, cfg_content = tempfile.mkstemp()
- with os.fdopen(cfg, "w+") as cfg:
- cfg.write(template.format(**kwargs))
- cfg_file = os.path.join(bin_path, nfvi_cfg)
- self.connection.put(cfg_content, cfg_file)
-
- def _prepare_collectd_conf(self, bin_path):
+ def _provide_config_file(self, config_file_path, nfvi_cfg, template_kwargs):
+ template = pkg_resources.resource_string("yardstick.network_services.nfvi",
+ nfvi_cfg).decode('utf-8')
+ cfg_content = jinja2.Template(template, trim_blocks=True, lstrip_blocks=True,
+ finalize=finalize_for_yaml).render(
+ **template_kwargs)
+ # cfg_content = io.StringIO(template.format(**template_kwargs))
+ cfg_file = os.path.join(config_file_path, nfvi_cfg)
+ # must write as root, so use sudo
+ self.connection.execute("cat | sudo tee {}".format(cfg_file), stdin=cfg_content)
+
+ def _prepare_collectd_conf(self, config_file_path):
""" Prepare collectd conf """
- loadplugin = "\n".join("LoadPlugin {0}".format(plugin)
- for plugin in LIST_PLUGINS_ENABLED)
-
- interfaces = "\n".join("PortName '{0[name]}'".format(interface)
- for interface in self.interfaces)
kwargs = {
- "interval": '25',
- "loadplugin": loadplugin,
- "dpdk_interface": interfaces,
+ "interval": self.interval,
+ "loadplugins": set(chain(LIST_PLUGINS_ENABLED, self.plugins.keys())),
+ # Optional fields PortName is descriptive only, use whatever is present
+ "port_names": self.port_names,
+ # "ovs_bridge_interfaces": ["br-int"],
+ "plugins": self.plugins,
}
- self._provide_config_file(bin_path, 'collectd.conf', kwargs)
+ self._provide_config_file(config_file_path, self.COLLECTD_CONF, kwargs)
def _start_collectd(self, connection, bin_path):
LOG.debug("Starting collectd to collect NFVi stats")
- connection.execute('sudo pkill -9 collectd')
+ connection.execute('sudo pkill -x -9 collectd')
bin_path = get_nsb_option("bin_path")
- collectd_path = os.path.join(bin_path, "collectd", "collectd")
+ collectd_path = os.path.join(bin_path, "collectd", "sbin", "collectd")
+ config_file_path = os.path.join(bin_path, "collectd", "etc")
exit_status = connection.execute("which %s > /dev/null 2>&1" % collectd_path)[0]
if exit_status != 0:
LOG.warning("%s is not present disabling", collectd_path)
@@ -217,7 +236,9 @@ class ResourceProfile(object):
# collectd_installer, http_proxy, https_proxy))
return
LOG.debug("Starting collectd to collect NFVi stats")
- self._prepare_collectd_conf(bin_path)
+ # ensure collectd.conf.d exists to avoid error/warning
+ connection.execute("sudo mkdir -p /etc/collectd/collectd.conf.d")
+ self._prepare_collectd_conf(config_file_path)
# Reset amqp queue
LOG.debug("reset and setup amqp to collect data from collectd")
@@ -228,7 +249,7 @@ class ResourceProfile(object):
connection.execute("sudo rabbitmqctl start_app")
connection.execute("sudo service rabbitmq-server restart")
- LOG.debug("Creating amdin user for rabbitmq in order to collect data from collectd")
+ LOG.debug("Creating admin user for rabbitmq in order to collect data from collectd")
connection.execute("sudo rabbitmqctl delete_user guest")
connection.execute("sudo rabbitmqctl add_user admin admin")
connection.execute("sudo rabbitmqctl authenticate_user admin admin")
@@ -241,7 +262,11 @@ class ResourceProfile(object):
def initiate_systemagent(self, bin_path):
""" Start system agent for NFVi collection on host """
if self.enable:
- self._start_collectd(self.connection, bin_path)
+ try:
+ self._start_collectd(self.connection, bin_path)
+ except Exception:
+ LOG.exception("Exception during collectd start")
+ raise
def start(self):
""" start nfvi collection """
diff --git a/yardstick/network_services/vnf_generic/vnf/base.py b/yardstick/network_services/vnf_generic/vnf/base.py
index 42e3d2a48..56c57a94b 100644
--- a/yardstick/network_services/vnf_generic/vnf/base.py
+++ b/yardstick/network_services/vnf_generic/vnf/base.py
@@ -106,15 +106,18 @@ class VnfdHelper(dict):
if int(virtual_intf['dpdk_port_num']) == port:
return interface
- def port_num(self, name):
+ def port_num(self, port):
# we need interface name -> DPDK port num (PMD ID) -> LINK ID
# LINK ID -> PMD ID is governed by the port mask
"""
:rtype: int
- :type name: str
+ :type port: str
"""
- intf = self.find_interface(name=name)
+ if isinstance(port, dict):
+ intf = port
+ else:
+ intf = self.find_interface(name=port)
return int(intf["virtual-interface"]["dpdk_port_num"])
def port_nums(self, intfs):
diff --git a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
index 557009d30..91530860e 100644
--- a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
+++ b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
@@ -282,9 +282,11 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper):
def setup_vnf_environment(self):
self._setup_dpdk()
- resource = self._setup_resources()
+ self.bound_pci = [v['virtual-interface']["vpci"] for v in self.vnfd_helper.interfaces]
self.kill_vnf()
+ # bind before _setup_resources so we can use dpdk_port_num
self._detect_and_bind_drivers()
+ resource = self._setup_resources()
return resource
def kill_vnf(self):
@@ -307,10 +309,13 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper):
if exit_status != 0:
self.ssh_helper.execute("bash %s dpdk >/dev/null 2>&1" % dpdk_setup)
- def _setup_resources(self):
- interfaces = self.vnfd_helper.interfaces
- self.bound_pci = [v['virtual-interface']["vpci"] for v in interfaces]
+ def get_collectd_options(self):
+ options = self.scenario_helper.all_options.get("collectd", {})
+ # override with specific node settings
+ options.update(self.scenario_helper.options.get("collectd", {}))
+ return options
+ def _setup_resources(self):
# what is this magic? how do we know which socket is for which port?
# what about quad-socket?
if any(v[5] == "0" for v in self.bound_pci):
@@ -319,8 +324,14 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper):
self.socket = 1
cores = self._validate_cpu_cfg()
- return ResourceProfile(self.vnfd_helper.mgmt_interface,
- interfaces=self.vnfd_helper.interfaces, cores=cores)
+ # implicit ordering, presumably by DPDK port num, so pre-sort by port_num
+ # this won't work because we don't have DPDK port numbers yet
+ ports = sorted(self.vnfd_helper.interfaces, key=self.vnfd_helper.port_num)
+ port_names = (intf["name"] for intf in ports)
+ collectd_options = self.get_collectd_options()
+ plugins = collectd_options.get("plugins", {})
+ return ResourceProfile(self.vnfd_helper.mgmt_interface, port_names=port_names, cores=cores,
+ plugins=plugins, interval=collectd_options.get("interval"))
def _detect_and_bind_drivers(self):
interfaces = self.vnfd_helper.interfaces