aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRoss Brattain <ross.b.brattain@intel.com>2017-09-27 20:59:34 +0000
committerGerrit Code Review <gerrit@opnfv.org>2017-09-27 20:59:34 +0000
commit804a541be394f794047a55091c9378c605e9e7ec (patch)
treed44218e7ee5a07fb52caa290b6aae0298710764c
parentbd33c3fee8d285dba5f190de0af2eb62ba7f04b2 (diff)
parentbd11ffdec1f18785696b5c4872e5bd5e1012249c (diff)
Merge "collectd: write config file from Jinja2 template"
-rw-r--r--tests/unit/network_services/nfvi/test_resource.py163
-rw-r--r--tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py10
-rw-r--r--yardstick/network_services/nfvi/collectd.conf95
-rwxr-xr-xyardstick/network_services/nfvi/collectd.sh3
-rw-r--r--yardstick/network_services/nfvi/resource.py95
-rw-r--r--yardstick/network_services/vnf_generic/vnf/base.py9
-rw-r--r--yardstick/network_services/vnf_generic/vnf/sample_vnf.py23
7 files changed, 175 insertions, 223 deletions
diff --git a/tests/unit/network_services/nfvi/test_resource.py b/tests/unit/network_services/nfvi/test_resource.py
index 1c2c1f3e2..eba38c688 100644
--- a/tests/unit/network_services/nfvi/test_resource.py
+++ b/tests/unit/network_services/nfvi/test_resource.py
@@ -14,7 +14,6 @@
from __future__ import absolute_import
import unittest
-import multiprocessing
import mock
from yardstick.network_services.nfvi.resource import ResourceProfile
@@ -86,17 +85,20 @@ class TestResourceProfile(unittest.TestCase):
'id': 'VpeApproxVnf', 'name': 'VPEVnfSsh'}]}}
def setUp(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
+ with mock.patch("yardstick.ssh.AutoConnectSSH") as ssh:
+ self.ssh_mock = mock.Mock(autospec=ssh.SSH)
+ self.ssh_mock.execute = \
mock.Mock(return_value=(0, {}, ""))
- ssh.from_node.return_value = ssh_mock
+ ssh.from_node.return_value = self.ssh_mock
mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
+ # interfaces = \
+ # self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
+ port_names = \
self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
self.resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
+ ResourceProfile(mgmt, port_names, [1, 2, 3])
+ self.resource_profile.connection = self.ssh_mock
def test___init__(self):
self.assertEqual(True, self.resource_profile.enable)
@@ -118,133 +120,33 @@ class TestResourceProfile(unittest.TestCase):
self.assertEqual(val, ('error', 'Invalid', '', ''))
def test__start_collectd(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._prepare_collectd_conf = mock.Mock()
- self.assertIsNone(
- resource_profile._start_collectd(ssh_mock, "/opt/nsb_bin"))
-
- def test__prepare_collectd_conf_BM(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._provide_config_file = mock.Mock()
- self.assertIsNone(
- resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
-
- def test__prepare_collectd_conf_managed_ovs_dpdk(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._provide_config_file = mock.Mock()
- self.assertIsNone(
- resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
-
- def test__prepare_collectd_conf_ovs_dpdk(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._provide_config_file = mock.Mock()
self.assertIsNone(
- resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
+ self.resource_profile._start_collectd(self.ssh_mock, "/opt/nsb_bin"))
- def test__prepare_collectd_conf_managed_sriov(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._provide_config_file = mock.Mock()
+ def test__prepare_collectd_conf(self):
self.assertIsNone(
- resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
+ self.resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
- def test__prepare_collectd_conf_sriov(self):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._provide_config_file = mock.Mock()
- self.assertIsNone(
- resource_profile._prepare_collectd_conf("/opt/nsb_bin"))
@mock.patch("yardstick.network_services.nfvi.resource.open")
- @mock.patch("yardstick.network_services.nfvi.resource.tempfile")
@mock.patch("yardstick.network_services.nfvi.resource.os")
- def test__provide_config_file(self, mock_open, mock_tempfile, mock_os):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._prepare_collectd_conf = mock.Mock()
- resource_profile.connection = ssh_mock
- resource_profile.connection.put = \
- mock.Mock(return_value=(0, "", ""))
- mock_tempfile.mkstemp = mock.Mock(return_value=["test", ""])
- self.assertIsNone(
- resource_profile._provide_config_file("/opt/nsb_bin",
- "collectd.cfg", {}))
+ def test__provide_config_file(self, mock_open, mock_os):
+ loadplugin = range(5)
+ port_names = range(5)
+ kwargs = {
+ "interval": '25',
+ "loadplugin": loadplugin,
+ "port_names": port_names,
+ }
+ self.resource_profile._provide_config_file("/opt/nsb_bin", "collectd.conf", kwargs)
+ self.ssh_mock.execute.assert_called_once()
+
@mock.patch("yardstick.network_services.nfvi.resource.open")
def test_initiate_systemagent(self, mock_open):
- with mock.patch("yardstick.ssh.SSH") as ssh:
- ssh_mock = mock.Mock(autospec=ssh.SSH)
- ssh_mock.execute = \
- mock.Mock(return_value=(0, "", ""))
- ssh.from_node.return_value = ssh_mock
- mgmt = self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['mgmt-interface']
- interfaces = \
- self.VNFD['vnfd:vnfd-catalog']['vnfd'][0]['vdu'][0]['external-interface']
- resource_profile = \
- ResourceProfile(mgmt, interfaces, [1, 2, 3])
- resource_profile._start_collectd = mock.Mock()
- self.assertIsNone(
- resource_profile.initiate_systemagent("/opt/nsb_bin"))
+ self.resource_profile._start_collectd = mock.Mock()
+ self.assertIsNone(
+ self.resource_profile.initiate_systemagent("/opt/nsb_bin"))
def test__parse_hugepages(self):
reskey = ["cpu", "cpuFreq"]
@@ -301,21 +203,21 @@ class TestResourceProfile(unittest.TestCase):
self.assertDictEqual(res, expected_result)
def test_parse_collectd_result_hugepage(self):
- metric = {"nsb_stats/hugepages/free": "101"}
+ # amqp returns bytes
+ metric = {b"nsb_stats/hugepages/free": b"101"}
self.resource_profile.parse_hugepages = \
mock.Mock(return_value={"free": "101"})
res = self.resource_profile.parse_collectd_result(metric, [0, 1, 2])
- expected_result = {'cpu': {}, 'dpdkstat': {}, 'hugepages': {'free':
- '101'},
+ expected_result = {'cpu': {}, 'dpdkstat': {}, 'hugepages': {'free': '101'},
'memory': {}, 'ovs_stats': {}, 'timestamp': '',
'intel_pmu': {},
'virt': {}}
self.assertDictEqual(res, expected_result)
def test_parse_collectd_result_dpdk_virt_ovs(self):
- metric = {"nsb_stats/dpdkstat/tx": "101",
- "nsb_stats/ovs_stats/tx": "101",
- "nsb_stats/virt/virt/memory": "101"}
+ metric = {b"nsb_stats/dpdkstat/tx": b"101",
+ b"nsb_stats/ovs_stats/tx": b"101",
+ b"nsb_stats/virt/virt/memory": b"101"}
self.resource_profile.parse_dpdkstat = \
mock.Mock(return_value={"tx": "101"})
self.resource_profile.parse_virt = \
@@ -347,7 +249,6 @@ class TestResourceProfile(unittest.TestCase):
self.assertIsNotNone(res)
def test_run_collectd_amqp(self):
- _queue = multiprocessing.Queue()
resource.AmqpConsumer = mock.Mock(autospec=collectd)
self.assertIsNone(self.resource_profile.run_collectd_amqp())
diff --git a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
index 4b9f4172e..d0c4b6f42 100644
--- a/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
+++ b/tests/unit/network_services/vnf_generic/vnf/test_sample_vnf.py
@@ -723,8 +723,9 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
result = dpdk_setup_helper._validate_cpu_cfg()
self.assertEqual(result, expected)
+ @mock.patch('yardstick.network_services.vnf_generic.vnf.sample_vnf.time')
@mock.patch('yardstick.ssh.SSH')
- def test_setup_vnf_environment(self, _):
+ def test_setup_vnf_environment(self, _, mock_time):
def execute(cmd, *args, **kwargs):
if cmd.startswith('which '):
return exec_failure
@@ -782,6 +783,8 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
dpdk_setup_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, scenario_helper)
dpdk_setup_helper._validate_cpu_cfg = mock.Mock()
+ dpdk_setup_helper.bound_pci = [v['virtual-interface']["vpci"] for v in
+ vnfd_helper.interfaces]
result = dpdk_setup_helper._setup_resources()
self.assertIsInstance(result, ResourceProfile)
self.assertEqual(dpdk_setup_helper.socket, 0)
@@ -796,11 +799,14 @@ class TestDpdkVnfSetupEnvHelper(unittest.TestCase):
dpdk_setup_helper = DpdkVnfSetupEnvHelper(vnfd_helper, ssh_helper, scenario_helper)
dpdk_setup_helper._validate_cpu_cfg = mock.Mock()
+ dpdk_setup_helper.bound_pci = [v['virtual-interface']["vpci"] for v in
+ vnfd_helper.interfaces]
result = dpdk_setup_helper._setup_resources()
self.assertIsInstance(result, ResourceProfile)
self.assertEqual(dpdk_setup_helper.socket, 1)
- def test__detect_and_bind_drivers(self):
+ @mock.patch('yardstick.network_services.vnf_generic.vnf.sample_vnf.time')
+ def test__detect_and_bind_drivers(self, mock_time):
vnfd_helper = VnfdHelper(deepcopy(self.VNFD_0))
ssh_helper = mock.Mock()
# ssh_helper.execute = mock.Mock(return_value = (0, 'text', ''))
diff --git a/yardstick/network_services/nfvi/collectd.conf b/yardstick/network_services/nfvi/collectd.conf
index 3928dcbca..22bd5d49d 100644
--- a/yardstick/network_services/nfvi/collectd.conf
+++ b/yardstick/network_services/nfvi/collectd.conf
@@ -15,7 +15,7 @@
Hostname "nsb_stats"
FQDNLookup true
-Interval {interval}
+Interval {{ interval }}
##############################################################################
# LoadPlugin section #
@@ -24,7 +24,9 @@ Interval {interval}
##############################################################################
#LoadPlugin syslog
-{loadplugin}
+{% for plugin in loadplugins %}
+LoadPlugin {{ plugin }}
+{% endfor %}
##############################################################################
# Plugin configuration #
@@ -38,42 +40,31 @@ Interval {interval}
#</Plugin>
<Plugin amqp>
- <Publish "name">
- Host "0.0.0.0"
- Port "5672"
- VHost "/"
- User "admin"
- Password "admin"
- Exchange "amq.fanout"
- RoutingKey "collectd"
- Persistent false
- StoreRates false
- ConnectionRetryDelay 0
- </Publish>
+ <Publish "name">
+ Host "0.0.0.0"
+ Port "5672"
+ VHost "/"
+ User "admin"
+ Password "admin"
+ Exchange "amq.fanout"
+ RoutingKey "collectd"
+ Persistent false
+ StoreRates false
+ ConnectionRetryDelay 0
+ </Publish>
</Plugin>
<Plugin cpu>
- ReportByCpu true
- ReportByState true
- ValuesPercentage true
+ ReportByCpu true
+ ReportByState true
+ ValuesPercentage true
</Plugin>
<Plugin memory>
- ValuesAbsolute true
- ValuesPercentage false
-</Plugin>
-
-<Plugin "intel_rdt">
- Cores ""
+ ValuesAbsolute true
+ ValuesPercentage false
</Plugin>
-<Plugin intel_pmu>
- ReportHardwareCacheEvents true
- ReportKernelPMUEvents true
- ReportSoftwareEvents true
- EventList "/root/.cache/pmu-events/GenuineIntel-6-2D-core.json"
- HardwareEvents "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS" "L2_RQSTS.ALL_CODE_RD"
-</Plugin>
<Plugin hugepages>
ReportPerNodeHP true
@@ -83,15 +74,25 @@ Interval {interval}
ValuesPercentage false
</Plugin>
-<Plugin hugepages>
- ReportPerNodeHP true
- ReportRootHP true
- ValuesPages true
- ValuesBytes false
- ValuesPercentage false
+
+{% if "intel_rdt" in plugins %}
+<Plugin "intel_rdt">
+ Cores ""
+</Plugin>
+{% endif %}
+
+{% if "intel_pmu" in plugins %}
+<Plugin intel_pmu>
+ ReportHardwareCacheEvents true
+ ReportKernelPMUEvents true
+ ReportSoftwareEvents true
+ EventList "/root/.cache/pmu-events/GenuineIntel-6-2D-core.json"
+ HardwareEvents "L2_RQSTS.CODE_RD_HIT,L2_RQSTS.CODE_RD_MISS" "L2_RQSTS.ALL_CODE_RD"
</Plugin>
+{% endif %}
-<Plugin dpdkstat>
+{% if "dpdkstat" in plugins %}
+<Plugin "dpdkstat">
<EAL>
Coremask "0x1"
MemoryChannels "4"
@@ -100,20 +101,24 @@ Interval {interval}
</EAL>
SharedMemObj "dpdk_collectd_stats_0"
EnabledPortMask 0xffff
- {dpdk_interface}
+{% for port_name in port_names %}
+ PortName {{ port_name }}
+{% endfor %}
</Plugin>
+{% endif %}
-<Plugin virt>
- Domain "samplevnf"
+{% if "virt" in plugins %}
+<Plugin "virt">
+# monitor all domains
</Plugin>
+{% endif %}
-<Plugin ovs_stats>
+{% if "ovs_stats" in plugins %}
+<Plugin "ovs_stats">
Port "6640"
Address "127.0.0.1"
Socket "/usr/local/var/run/openvswitch/db.sock"
- Bridges "br0" "br_ext"
+# don't specify bridges, monitor all bridges
</Plugin>
+{% endif %}
-<Include "/etc/collectd/collectd.conf.d">
- Filter "*.conf"
-</Include>
diff --git a/yardstick/network_services/nfvi/collectd.sh b/yardstick/network_services/nfvi/collectd.sh
index 296c4a213..bdc5abd03 100755
--- a/yardstick/network_services/nfvi/collectd.sh
+++ b/yardstick/network_services/nfvi/collectd.sh
@@ -142,7 +142,8 @@ else
fi
modprobe msr
-cp $INSTALL_NSB_BIN/collectd.conf /opt/collectd/etc/
+# we overwrite the config file during _start_collectd so don't copy it
+#cp $INSTALL_NSB_BIN/collectd.conf /opt/nsb_bin/collectd/etc/
sudo service rabbitmq-server restart
echo "Check if admin user already created"
rabbitmqctl list_users | grep '^admin$' > /dev/null
diff --git a/yardstick/network_services/nfvi/resource.py b/yardstick/network_services/nfvi/resource.py
index fa32a4dcf..d807f5e46 100644
--- a/yardstick/network_services/nfvi/resource.py
+++ b/yardstick/network_services/nfvi/resource.py
@@ -15,16 +15,22 @@
from __future__ import absolute_import
from __future__ import print_function
-import tempfile
+
import logging
+from itertools import chain
+
+import jinja2
import os
import os.path
import re
import multiprocessing
+import pkg_resources
from oslo_config import cfg
+from oslo_utils.encodeutils import safe_decode
from yardstick import ssh
+from yardstick.common.task_template import finalize_for_yaml
from yardstick.common.utils import validate_non_string_sequence
from yardstick.network_services.nfvi.collectd import AmqpConsumer
from yardstick.network_services.utils import get_nsb_option
@@ -34,26 +40,36 @@ LOG = logging.getLogger(__name__)
CONF = cfg.CONF
ZMQ_OVS_PORT = 5567
ZMQ_POLLING_TIME = 12000
-LIST_PLUGINS_ENABLED = ["amqp", "cpu", "cpufreq", "intel_rdt", "memory",
- "hugepages", "dpdkstat", "virt", "ovs_stats", "intel_pmu"]
+LIST_PLUGINS_ENABLED = ["amqp", "cpu", "cpufreq", "memory",
+ "hugepages"]
class ResourceProfile(object):
"""
This profile adds a resource at the beginning of the test session
"""
+ COLLECTD_CONF = "collectd.conf"
+ AMPQ_PORT = 5672
+ DEFAULT_INTERVAL = 25
- def __init__(self, mgmt, interfaces=None, cores=None):
+ def __init__(self, mgmt, port_names=None, cores=None, plugins=None, interval=None):
+ if plugins is None:
+ self.plugins = {}
+ else:
+ self.plugins = plugins
+ if interval is None:
+ self.interval = self.DEFAULT_INTERVAL
+ else:
+ self.interval = interval
self.enable = True
self.cores = validate_non_string_sequence(cores, default=[])
self._queue = multiprocessing.Queue()
self.amqp_client = None
- self.interfaces = validate_non_string_sequence(interfaces, default=[])
+ self.port_names = validate_non_string_sequence(port_names, default=[])
- # why the host or ip?
- self.vnfip = mgmt.get("host", mgmt["ip"])
- self.connection = ssh.SSH.from_node(mgmt, overrides={"ip": self.vnfip})
- self.connection.wait()
+ # we need to save mgmt so we can connect to port 5672
+ self.mgmt = mgmt
+ self.connection = ssh.AutoConnectSSH.from_node(mgmt)
def check_if_sa_running(self, process):
""" verify if system agent is running """
@@ -62,7 +78,7 @@ class ResourceProfile(object):
def run_collectd_amqp(self):
""" run amqp consumer to collect the NFVi data """
- amqp_url = 'amqp://admin:admin@{}:5672/%2F'.format(self.vnfip)
+ amqp_url = 'amqp://admin:admin@{}:{}/%2F'.format(self.mgmt['ip'], self.AMPQ_PORT)
amqp = AmqpConsumer(amqp_url, self._queue)
try:
amqp.run()
@@ -124,7 +140,9 @@ class ResourceProfile(object):
}
testcase = ""
- for key, value in metrics.items():
+ # unicode decode
+ decoded = ((safe_decode(k, 'utf-8'), safe_decode(v, 'utf-8')) for k, v in metrics.items())
+ for key, value in decoded:
key_split = key.split("/")
res_key_iter = (key for key in key_split if "nsb_stats" not in key)
res_key0 = next(res_key_iter)
@@ -176,35 +194,36 @@ class ResourceProfile(object):
msg = self.parse_collectd_result(metric, self.cores)
return msg
- def _provide_config_file(self, bin_path, nfvi_cfg, kwargs):
- with open(os.path.join(bin_path, nfvi_cfg), 'r') as cfg:
- template = cfg.read()
- cfg, cfg_content = tempfile.mkstemp()
- with os.fdopen(cfg, "w+") as cfg:
- cfg.write(template.format(**kwargs))
- cfg_file = os.path.join(bin_path, nfvi_cfg)
- self.connection.put(cfg_content, cfg_file)
-
- def _prepare_collectd_conf(self, bin_path):
+ def _provide_config_file(self, config_file_path, nfvi_cfg, template_kwargs):
+ template = pkg_resources.resource_string("yardstick.network_services.nfvi",
+ nfvi_cfg).decode('utf-8')
+ cfg_content = jinja2.Template(template, trim_blocks=True, lstrip_blocks=True,
+ finalize=finalize_for_yaml).render(
+ **template_kwargs)
+ # cfg_content = io.StringIO(template.format(**template_kwargs))
+ cfg_file = os.path.join(config_file_path, nfvi_cfg)
+ # must write as root, so use sudo
+ self.connection.execute("cat | sudo tee {}".format(cfg_file), stdin=cfg_content)
+
+ def _prepare_collectd_conf(self, config_file_path):
""" Prepare collectd conf """
- loadplugin = "\n".join("LoadPlugin {0}".format(plugin)
- for plugin in LIST_PLUGINS_ENABLED)
-
- interfaces = "\n".join("PortName '{0[name]}'".format(interface)
- for interface in self.interfaces)
kwargs = {
- "interval": '25',
- "loadplugin": loadplugin,
- "dpdk_interface": interfaces,
+ "interval": self.interval,
+ "loadplugins": set(chain(LIST_PLUGINS_ENABLED, self.plugins.keys())),
+ # Optional fields PortName is descriptive only, use whatever is present
+ "port_names": self.port_names,
+ # "ovs_bridge_interfaces": ["br-int"],
+ "plugins": self.plugins,
}
- self._provide_config_file(bin_path, 'collectd.conf', kwargs)
+ self._provide_config_file(config_file_path, self.COLLECTD_CONF, kwargs)
def _start_collectd(self, connection, bin_path):
LOG.debug("Starting collectd to collect NFVi stats")
- connection.execute('sudo pkill -9 collectd')
+ connection.execute('sudo pkill -x -9 collectd')
bin_path = get_nsb_option("bin_path")
- collectd_path = os.path.join(bin_path, "collectd", "collectd")
+ collectd_path = os.path.join(bin_path, "collectd", "sbin", "collectd")
+ config_file_path = os.path.join(bin_path, "collectd", "etc")
exit_status = connection.execute("which %s > /dev/null 2>&1" % collectd_path)[0]
if exit_status != 0:
LOG.warning("%s is not present disabling", collectd_path)
@@ -217,7 +236,9 @@ class ResourceProfile(object):
# collectd_installer, http_proxy, https_proxy))
return
LOG.debug("Starting collectd to collect NFVi stats")
- self._prepare_collectd_conf(bin_path)
+ # ensure collectd.conf.d exists to avoid error/warning
+ connection.execute("sudo mkdir -p /etc/collectd/collectd.conf.d")
+ self._prepare_collectd_conf(config_file_path)
# Reset amqp queue
LOG.debug("reset and setup amqp to collect data from collectd")
@@ -228,7 +249,7 @@ class ResourceProfile(object):
connection.execute("sudo rabbitmqctl start_app")
connection.execute("sudo service rabbitmq-server restart")
- LOG.debug("Creating amdin user for rabbitmq in order to collect data from collectd")
+ LOG.debug("Creating admin user for rabbitmq in order to collect data from collectd")
connection.execute("sudo rabbitmqctl delete_user guest")
connection.execute("sudo rabbitmqctl add_user admin admin")
connection.execute("sudo rabbitmqctl authenticate_user admin admin")
@@ -241,7 +262,11 @@ class ResourceProfile(object):
def initiate_systemagent(self, bin_path):
""" Start system agent for NFVi collection on host """
if self.enable:
- self._start_collectd(self.connection, bin_path)
+ try:
+ self._start_collectd(self.connection, bin_path)
+ except Exception:
+ LOG.exception("Exception during collectd start")
+ raise
def start(self):
""" start nfvi collection """
diff --git a/yardstick/network_services/vnf_generic/vnf/base.py b/yardstick/network_services/vnf_generic/vnf/base.py
index 42e3d2a48..56c57a94b 100644
--- a/yardstick/network_services/vnf_generic/vnf/base.py
+++ b/yardstick/network_services/vnf_generic/vnf/base.py
@@ -106,15 +106,18 @@ class VnfdHelper(dict):
if int(virtual_intf['dpdk_port_num']) == port:
return interface
- def port_num(self, name):
+ def port_num(self, port):
# we need interface name -> DPDK port num (PMD ID) -> LINK ID
# LINK ID -> PMD ID is governed by the port mask
"""
:rtype: int
- :type name: str
+ :type port: str
"""
- intf = self.find_interface(name=name)
+ if isinstance(port, dict):
+ intf = port
+ else:
+ intf = self.find_interface(name=port)
return int(intf["virtual-interface"]["dpdk_port_num"])
def port_nums(self, intfs):
diff --git a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
index 557009d30..91530860e 100644
--- a/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
+++ b/yardstick/network_services/vnf_generic/vnf/sample_vnf.py
@@ -282,9 +282,11 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper):
def setup_vnf_environment(self):
self._setup_dpdk()
- resource = self._setup_resources()
+ self.bound_pci = [v['virtual-interface']["vpci"] for v in self.vnfd_helper.interfaces]
self.kill_vnf()
+ # bind before _setup_resources so we can use dpdk_port_num
self._detect_and_bind_drivers()
+ resource = self._setup_resources()
return resource
def kill_vnf(self):
@@ -307,10 +309,13 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper):
if exit_status != 0:
self.ssh_helper.execute("bash %s dpdk >/dev/null 2>&1" % dpdk_setup)
- def _setup_resources(self):
- interfaces = self.vnfd_helper.interfaces
- self.bound_pci = [v['virtual-interface']["vpci"] for v in interfaces]
+ def get_collectd_options(self):
+ options = self.scenario_helper.all_options.get("collectd", {})
+ # override with specific node settings
+ options.update(self.scenario_helper.options.get("collectd", {}))
+ return options
+ def _setup_resources(self):
# what is this magic? how do we know which socket is for which port?
# what about quad-socket?
if any(v[5] == "0" for v in self.bound_pci):
@@ -319,8 +324,14 @@ class DpdkVnfSetupEnvHelper(SetupEnvHelper):
self.socket = 1
cores = self._validate_cpu_cfg()
- return ResourceProfile(self.vnfd_helper.mgmt_interface,
- interfaces=self.vnfd_helper.interfaces, cores=cores)
+ # implicit ordering, presumably by DPDK port num, so pre-sort by port_num
+ # this won't work because we don't have DPDK port numbers yet
+ ports = sorted(self.vnfd_helper.interfaces, key=self.vnfd_helper.port_num)
+ port_names = (intf["name"] for intf in ports)
+ collectd_options = self.get_collectd_options()
+ plugins = collectd_options.get("plugins", {})
+ return ResourceProfile(self.vnfd_helper.mgmt_interface, port_names=port_names, cores=cores,
+ plugins=plugins, interval=collectd_options.get("interval"))
def _detect_and_bind_drivers(self):
interfaces = self.vnfd_helper.interfaces