diff options
| -rw-r--r-- | baro_tests/collectd.py | 691 | ||||
| -rw-r--r-- | baro_tests/config_server.py | 468 | ||||
| -rwxr-xr-x | baro_tests/mce-inject_ea | bin | 0 -> 75144 bytes | |||
| -rw-r--r-- | baro_tests/tests.py | 147 | ||||
| -rwxr-xr-x | ci/utility/collectd_build_rpm.sh | 2 | ||||
| -rw-r--r-- | docs/release/configguide/featureconfig.rst | 61 | ||||
| -rw-r--r-- | docs/release/configguide/postinstall.rst | 123 | ||||
| -rw-r--r-- | docs/release/release-notes/release-notes.rst | 90 | ||||
| -rw-r--r-- | docs/release/scenarios/index.rst | 4 | ||||
| -rw-r--r-- | docs/release/scenarios/os-nosdn-bar-ha/scenario.description.rst | 61 | ||||
| -rw-r--r-- | docs/release/scenarios/os-nosdn-bar-noha/scenario.description.rst | 61 | ||||
| -rw-r--r-- | docs/release/userguide/feature.userguide.rst | 318 |
12 files changed, 1396 insertions, 630 deletions
diff --git a/baro_tests/collectd.py b/baro_tests/collectd.py index 5ac3c8fe..2878d508 100644 --- a/baro_tests/collectd.py +++ b/baro_tests/collectd.py @@ -1,4 +1,3 @@ -"""Executing test of plugins""" # -*- coding: utf-8 -*- # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -13,26 +12,28 @@ # License for the specific language governing permissions and limitations # under the License. +"""Executing test of plugins""" + import requests from keystoneclient.v3 import client import os -import pkg_resources +import sys import time import logging -from config_server import * -from tests import * +import config_server +import tests +import subprocess from opnfv.deployment import factory -from functest.utils import functest_utils -CEILOMETER_NAME = 'ceilometer' +AODH_NAME = 'aodh' +GNOCCHI_NAME = 'gnocchi' ID_RSA_SRC = '/root/.ssh/id_rsa' ID_RSA_DST_DIR = '/home/opnfv/.ssh' ID_RSA_DST = ID_RSA_DST_DIR + '/id_rsa' -INSTALLER_PARAMS_YAML = pkg_resources.resource_filename( - 'functest', 'ci/installer_params.yaml') -FUEL_IP = functest_utils.get_parameter_from_yaml('fuel.ip', INSTALLER_PARAMS_YAML) -FUEL_USER = functest_utils.get_parameter_from_yaml('fuel.user', INSTALLER_PARAMS_YAML) -FUEL_PW = functest_utils.get_parameter_from_yaml('fuel.password', INSTALLER_PARAMS_YAML) +APEX_IP = subprocess.check_output("echo $INSTALLER_IP", shell=True) +APEX_USER = 'root' +APEX_USER_STACK = 'stack' +APEX_PKEY = '/root/.ssh/id_rsa' class KeystoneException(Exception): @@ -64,53 +65,44 @@ class InvalidResponse(KeystoneException): "Invalid response", exc, response) -class CeilometerClient(object): - """Ceilometer Client to authenticate and request meters""" - def __init__(self, bc_logger): - """ - Keyword arguments: - bc_logger - logger instance - """ +class GnocchiClient(object): + # Gnocchi Client to authenticate and request meters + def __init__(self): self._auth_token = None - self._ceilometer_url = None + self._gnocchi_url = None self._meter_list = None - self._logger = bc_logger def auth_token(self): - """Get auth token""" + # Get auth token self._auth_server() return self._auth_token - def get_ceilometer_url(self): - """Get Ceilometer URL""" - return self._ceilometer_url + def get_gnocchi_url(self): + # Get Gnocchi URL + return self._gnocchi_url - def get_ceil_metrics(self, criteria=None): - """Get Ceilometer metrics for given criteria - - Keyword arguments: - criteria -- criteria for ceilometer meter list - """ + def get_gnocchi_metrics(self, criteria=None): + # Subject to change if metric gathering is different for gnocchi self._request_meters(criteria) return self._meter_list def _auth_server(self): - """Request token in authentication server""" - self._logger.debug('Connecting to the auth server {}'.format(os.environ['OS_AUTH_URL'])) + # Request token in authentication server + logger.debug('Connecting to the auth server {}'.format( + os.environ['OS_AUTH_URL'])) keystone = client.Client(username=os.environ['OS_USERNAME'], password=os.environ['OS_PASSWORD'], - tenant_name=os.environ['OS_TENANT_NAME'], + tenant_name=os.environ['OS_USERNAME'], auth_url=os.environ['OS_AUTH_URL']) self._auth_token = keystone.auth_token for service in keystone.service_catalog.get_data(): - if service['name'] == CEILOMETER_NAME: + if service['name'] == GNOCCHI_NAME: for service_type in service['endpoints']: if service_type['interface'] == 'internal': - self._ceilometer_url = service_type['url'] - break + self._gnocchi_url = service_type['url'] - if self._ceilometer_url is None: - self._logger.warning('Ceilometer is not registered in service catalog') + if self._gnocchi_url is None: + logger.warning('Gnocchi is not registered in service catalog') def _request_meters(self, criteria): """Request meter list values from ceilometer @@ -119,9 +111,10 @@ class CeilometerClient(object): criteria -- criteria for ceilometer meter list """ if criteria is None: - url = self._ceilometer_url + ('/v2/samples?limit=400') + url = self._gnocchi_url + ('/v2/metric?limit=400') else: - url = self._ceilometer_url + ('/v2/meters/%s?q.field=resource_id&limit=400' % criteria) + url = self._gnocchi_url \ + + ('/v3/metric/%s?q.field=metric&limit=400' % criteria) headers = {'X-Auth-Token': self._auth_token} resp = requests.get(url, headers=headers) try: @@ -131,18 +124,82 @@ class CeilometerClient(object): raise InvalidResponse(err, resp) +class AodhClient(object): + # Gnocchi Client to authenticate and request meters + def __init__(self): + self._auth_token = None + self._aodh_url = None + self._meter_list = None + + def auth_token(self): + # Get auth token + self._auth_server() + return self._auth_token + + def get_aodh_url(self): + # Get Gnocchi URL + return self._gnocchi_url + + def get_aodh_metrics(self, criteria=None): + # Subject to change if metric gathering is different for gnocchi + self._request_meters(criteria) + return self._meter_list + + def _auth_server(self): + # Request token in authentication server + logger.debug('Connecting to the AODH auth server {}'.format( + os.environ['OS_AUTH_URL'])) + keystone = client.Client(username=os.environ['OS_USERNAME'], + password=os.environ['OS_PASSWORD'], + tenant_name=os.environ['OS_USERNAME'], + auth_url=os.environ['OS_AUTH_URL']) + self._auth_token = keystone.auth_token + for service in keystone.service_catalog.get_data(): + if service['name'] == AODH_NAME: + for service_type in service['endpoints']: + if service_type['interface'] == 'internal': + self._gnocchi_url = service_type['url'] + + if self._aodh_url is None: + logger.warning('Aodh is not registered in service catalog') + + +class SNMPClient(object): + """Client to request SNMP meters""" + def __init__(self, conf, compute_node): + """ + Keyword arguments: + conf -- ConfigServer instance + compute_node -- Compute node object + """ + self.conf = conf + self.compute_node = compute_node + + def get_snmp_metrics(self, compute_node, mib_file, mib_strings): + snmp_output = {} + if mib_file is None: + cmd = "snmpwalk -v 2c -c public localhost IF-MIB::interfaces" + ip = compute_node.get_ip() + snmp_output = self.conf.execute_command(cmd, ip) + else: + for mib_string in mib_strings: + snmp_output[mib_string] = self.conf.execute_command( + "snmpwalk -v2c -m {} -c public localhost {}".format( + mib_file, mib_string), compute_node.get_ip()) + return snmp_output + + class CSVClient(object): """Client to request CSV meters""" - def __init__(self, bc_logger, conf): + def __init__(self, conf): """ Keyword arguments: - bc_logger - logger instance conf -- ConfigServer instance """ - self._logger = bc_logger self.conf = conf - def get_csv_metrics(self, compute_node, plugin_subdirectories, meter_categories): + def get_csv_metrics( + self, compute_node, plugin_subdirectories, meter_categories): """Get CSV metrics. Keyword arguments: @@ -152,34 +209,48 @@ class CSVClient(object): Return list of metrics. """ - stdout = self.conf.execute_command("date '+%Y-%m-%d'", compute_node.get_ip()) + stdout = self.conf.execute_command( + "date '+%Y-%m-%d'", compute_node.get_ip()) date = stdout[0].strip() metrics = [] for plugin_subdir in plugin_subdirectories: for meter_category in meter_categories: stdout = self.conf.execute_command( - "tail -2 /var/lib/collectd/csv/node-" - + "{0}.domain.tld/{1}/{2}-{3}".format( - compute_node.get_id(), plugin_subdir, meter_category, date), + "tail -2 /var/lib/collectd/csv/" + + "{0}.jf.intel.com/{1}/{2}-{3}".format( + compute_node.get_name(), plugin_subdir, meter_category, + date), compute_node.get_ip()) # Storing last two values values = stdout if len(values) < 2: - self._logger.error( + logger.error( 'Getting last two CSV entries of meter category ' - + '{0} in {1} subdir failed'.format(meter_category, plugin_subdir)) + + '{0} in {1} subdir failed'.format( + meter_category, plugin_subdir)) else: old_value = int(values[0][0:values[0].index('.')]) new_value = int(values[1][0:values[1].index('.')]) - metrics.append((plugin_subdir, meter_category, old_value, new_value)) + metrics.append(( + plugin_subdir, meter_category, old_value, new_value)) return metrics -def _check_logger(): - """Check whether there is global logger available and if not, define one.""" - if 'logger' not in globals(): - global logger - logger = logger.Logger("barometercollectd").getLogger() +def get_csv_categories_for_ipmi(conf, compute_node): + """Get CSV metrics. + + Keyword arguments: + compute_node -- compute node instance + + Return list of categories. + """ + stdout = conf.execute_command( + "date '+%Y-%m-%d'", compute_node.get_ip()) + date = stdout[0].strip() + categories = conf.execute_command( + "ls /var/lib/collectd/csv/{0}.jf.intel.com/ipmi | grep {1}".format( + compute_node.get_name(), date), compute_node.get_ip()) + return [category.strip()[:-11] for category in categories] def _process_result(compute_node, test, result, results_list): @@ -191,9 +262,13 @@ def _process_result(compute_node, test, result, results_list): results_list -- results list """ if result: - logger.info('Compute node {0} test case {1} PASSED.'.format(compute_node, test)) + logger.info( + 'Compute node {0} test case {1} PASSED.'.format( + compute_node, test)) else: - logger.error('Compute node {0} test case {1} FAILED.'.format(compute_node, test)) + logger.error( + 'Compute node {0} test case {1} FAILED.'.format( + compute_node, test)) results_list.append((compute_node, test, result)) @@ -215,17 +290,19 @@ def _print_label(label): logger.info(('=' * length1) + label + ('=' * length2)) -def _print_plugin_label(plugin, node_id): +def _print_plugin_label(plugin, node_name): """Print plug-in label. Keyword arguments: plugin -- plug-in name node_id -- node ID """ - _print_label('Node {0}: Plug-in {1} Test case execution'.format(node_id, plugin)) + _print_label( + 'Node {0}: Plug-in {1} Test case execution'.format(node_name, plugin)) -def _print_final_result_of_plugin(plugin, compute_ids, results, out_plugins, out_plugin): +def _print_final_result_of_plugin( + plugin, compute_ids, results, out_plugins, out_plugin): """Print final results of plug-in. Keyword arguments: @@ -240,14 +317,15 @@ def _print_final_result_of_plugin(plugin, compute_ids, results, out_plugins, out if out_plugins[id] == out_plugin: if (id, plugin, True) in results: print_line += ' PASS |' - elif (id, plugin, False) in results and out_plugins[id] == out_plugin: + elif (id, plugin, False) in results \ + and out_plugins[id] == out_plugin: print_line += ' FAIL |' else: print_line += ' NOT EX |' - elif out_plugin == 'Ceilometer': + elif out_plugin == 'Gnocchi': print_line += ' NOT EX |' else: - print_line += ' SKIP |' + print_line += ' NOT EX |' return print_line @@ -260,34 +338,54 @@ def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): results -- results list out_plugins -- list of used out plug-ins """ - compute_node_names = ['Node-{}'.format(id) for id in compute_ids] + compute_node_names = ['Node-{}'.format(i) for i in range( + len((compute_ids)))] + # compute_node_names = ['Node-{}'.format(id) for id in compute_ids] all_computes_in_line = '' for compute in compute_node_names: - all_computes_in_line = all_computes_in_line + '| ' + compute + (' ' * (7 - len(compute))) + all_computes_in_line += '| ' + compute + (' ' * (7 - len(compute))) line_of_nodes = '| Test ' + all_computes_in_line + '|' logger.info('=' * 70) logger.info('+' + ('-' * ((9 * len(compute_node_names))+16)) + '+') logger.info( - '|' + ' ' * ((9*len(compute_node_names))/2) + ' OVERALL SUMMARY' - + ' ' * (9*len(compute_node_names) - (9*len(compute_node_names))/2) + '|') - logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) + '|' + ' ' * ((9*len(compute_node_names))/2) + + ' OVERALL SUMMARY' + + ' ' * ( + 9*len(compute_node_names) - (9*len(compute_node_names))/2) + + '|') + logger.info( + '+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) logger.info(line_of_nodes) - logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) - out_plugins_print = ['Ceilometer'] + logger.info( + '+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) + out_plugins_print = ['Gnocchi'] + if 'SNMP' in out_plugins.values(): + out_plugins_print.append('SNMP') + if 'AODH' in out_plugins.values(): + out_plugins_print.append('AODH') if 'CSV' in out_plugins.values(): out_plugins_print.append('CSV') for out_plugin in out_plugins_print: output_plugins_line = '' for id in compute_ids: - out_plugin_result = '----' - if out_plugin == 'Ceilometer': - out_plugin_result = 'PASS' if out_plugins[id] == out_plugin else 'FAIL' + out_plugin_result = 'FAIL' + if out_plugin == 'Gnocchi': + out_plugin_result = \ + 'PASS' if out_plugins[id] == out_plugin else 'FAIL' + if out_plugin == 'AODH': + if out_plugins[id] == out_plugin: + out_plugin_result = \ + 'PASS' if out_plugins[id] == out_plugin else 'FAIL' + if out_plugin == 'SNMP': + if out_plugins[id] == out_plugin: + out_plugin_result = \ + 'PASS' if out_plugins[id] == out_plugin else 'FAIL' if out_plugin == 'CSV': if out_plugins[id] == out_plugin: out_plugin_result = \ 'PASS' if [ - plugin for comp_id, plugin, - res in results if comp_id == id and res] else 'FAIL' + plugin for comp_id, plugin, res in results + if comp_id == id and res] else 'FAIL' else: out_plugin_result = 'SKIP' output_plugins_line += '| ' + out_plugin_result + ' ' @@ -297,14 +395,18 @@ def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): for plugin in sorted(tested_plugins.values()): line_plugin = _print_final_result_of_plugin( plugin, compute_ids, results, out_plugins, out_plugin) - logger.info('| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) + '|' + line_plugin) - logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) + logger.info( + '| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) + + '|' + line_plugin) + logger.info( + '+' + ('-' * 16) + '+' + + (('-' * 8) + '+') * len(compute_node_names)) logger.info('=' * 70) def _exec_testcase( - test_labels, name, ceilometer_running, compute_node, - conf, results, error_plugins): + test_labels, name, gnocchi_running, aodh_running, snmp_running, + controllers, compute_node, conf, results, error_plugins, out_plugins): """Execute the testcase. Keyword arguments: @@ -314,7 +416,8 @@ def _exec_testcase( compute_node -- compute node ID conf -- ConfigServer instance results -- results list - error_plugins -- list of tuples with plug-in errors (plugin, error_description, is_critical): + error_plugins -- list of tuples with plug-in errors + (plugin, error_description, is_critical): plugin -- plug-in ID, key of test_labels dictionary error_decription -- description of the error is_critical -- boolean value indicating whether error is critical @@ -322,46 +425,111 @@ def _exec_testcase( ovs_interfaces = conf.get_ovs_interfaces(compute_node) ovs_configured_interfaces = conf.get_plugin_config_values( compute_node, 'ovs_events', 'Interfaces') + ovs_configured_bridges = conf.get_plugin_config_values( + compute_node, 'ovs_stats', 'Bridges') ovs_existing_configured_int = [ interface for interface in ovs_interfaces if interface in ovs_configured_interfaces] + ovs_existing_configured_bridges = [ + bridge for bridge in ovs_interfaces + if bridge in ovs_configured_bridges] plugin_prerequisites = { - 'mcelog': [(conf.is_installed(compute_node, 'mcelog'), 'mcelog must be installed.')], + 'intel_rdt': [( + conf.is_libpqos_on_node(compute_node), + 'libpqos must be installed.')], + 'mcelog': [( + conf.is_installed(compute_node, 'mcelog'), + 'mcelog must be installed.')], 'ovs_events': [( len(ovs_existing_configured_int) > 0 or len(ovs_interfaces) > 0, - 'Interfaces must be configured.')]} - ceilometer_criteria_lists = { - 'hugepages': ['hugepages.vmpage_number'], + 'Interfaces must be configured.')], + 'ovs_stats': [( + len(ovs_existing_configured_bridges) > 0, + 'Bridges must be configured.')]} + gnocchi_criteria_lists = { + 'hugepages': ['hugepages'], + 'mcelog': ['mcelog'], + 'ovs_events': ['interface-ovs-system'], + 'ovs_stats': ['ovs_stats-br0.br0']} + aodh_criteria_lists = { 'mcelog': ['mcelog.errors'], 'ovs_events': ['ovs_events.gauge']} - ceilometer_substr_lists = { - 'ovs_events': ovs_existing_configured_int if len(ovs_existing_configured_int) > 0 else ovs_interfaces} + snmp_mib_files = { + 'intel_rdt': '/usr/share/snmp/mibs/Intel-Rdt.txt', + 'hugepages': '/usr/share/snmp/mibs/Intel-Hugepages.txt', + 'mcelog': '/usr/share/snmp/mibs/Intel-Mcelog.txt'} + snmp_mib_strings = { + 'intel_rdt': [ + 'INTEL-RDT-MIB::rdtLlc.1', + 'INTEL-RDT-MIB::rdtIpc.1', + 'INTEL-RDT-MIB::rdtMbmRemote.1', + 'INTEL-RDT-MIB::rdtMbmLocal.1'], + 'hugepages': [ + 'INTEL-HUGEPAGES-MIB::hugepagesPageFree'], + 'mcelog': [ + 'INTEL-MCELOG-MIB::memoryCorrectedErrors.1', + 'INTEL-MCELOG-MIB::memoryCorrectedErrors.2']} + nr_hugepages = int(time.time()) % 10000 + snmp_in_commands = { + 'intel_rdt': None, + 'hugepages': 'echo {} > /sys/kernel/'.format(nr_hugepages) + + 'mm/hugepages/hugepages-2048kB/nr_hugepages', + 'mcelog': '/root/mce-inject_df < /root/corrected'} csv_subdirs = { + 'intel_rdt': [ + 'intel_rdt-{}'.format(core) + for core in conf.get_plugin_config_values( + compute_node, 'intel_rdt', 'Cores')], 'hugepages': [ - 'hugepages-mm-2048Kb', 'hugepages-node0-2048Kb', 'hugepages-node1-2048Kb', - 'hugepages-mm-1048576Kb', 'hugepages-node0-1048576Kb', 'hugepages-node1-1048576Kb'], - 'mcelog': ['mcelog-SOCKET_0_CHANNEL_0_DIMM_any', 'mcelog-SOCKET_0_CHANNEL_any_DIMM_any'], + 'hugepages-mm-2048Kb', 'hugepages-node0-2048Kb', + 'hugepages-node1-2048Kb', 'hugepages-mm-1048576Kb', + 'hugepages-node0-1048576Kb', 'hugepages-node1-1048576Kb'], + 'ipmi': ['ipmi'], + 'mcelog': [ + 'mcelog-SOCKET_0_CHANNEL_0_DIMM_any', + 'mcelog-SOCKET_0_CHANNEL_any_DIMM_any'], + 'ovs_stats': [ + 'ovs_stats-{0}.{0}'.format(interface) + for interface in ovs_existing_configured_bridges], 'ovs_events': [ 'ovs_events-{}'.format(interface) - for interface in (ovs_existing_configured_int if len(ovs_existing_configured_int) > 0 else ovs_interfaces)]} + for interface in ( + ovs_existing_configured_int + if len(ovs_existing_configured_int) > 0 else ovs_interfaces)]} + csv_meter_categories_ipmi = get_csv_categories_for_ipmi(conf, compute_node) csv_meter_categories = { + 'intel_rdt': [ + 'bytes-llc', 'ipc', 'memory_bandwidth-local', + 'memory_bandwidth-remote'], 'hugepages': ['vmpage_number-free', 'vmpage_number-used'], + 'ipmi': csv_meter_categories_ipmi, 'mcelog': [ - 'errors-corrected_memory_errors', 'errors-uncorrected_memory_errors', - 'errors-corrected_memory_errors_in_24h', 'errors-uncorrected_memory_errors_in_24h'], + 'errors-corrected_memory_errors', + 'errors-uncorrected_memory_errors', + 'errors-corrected_memory_errors_in_24h', + 'errors-uncorrected_memory_errors_in_24h'], + 'ovs_stats': [ + 'if_collisions', 'if_dropped', 'if_errors', 'if_packets', + 'if_rx_errors-crc', 'if_rx_errors-frame', 'if_rx_errors-over', + 'if_rx_octets', 'if_tx_octets'], 'ovs_events': ['gauge-link_status']} - _print_plugin_label(test_labels[name] if name in test_labels else name, compute_node.get_id()) + _print_plugin_label( + test_labels[name] if name in test_labels else name, + compute_node.get_name()) plugin_critical_errors = [ - error for plugin, error, critical in error_plugins if plugin == name and critical] + error for plugin, error, critical in error_plugins + if plugin == name and critical] if plugin_critical_errors: logger.error('Following critical errors occurred:'.format(name)) for error in plugin_critical_errors: logger.error(' * ' + error) - _process_result(compute_node.get_id(), test_labels[name], False, results) + _process_result( + compute_node.get_id(), test_labels[name], False, results) else: plugin_errors = [ - error for plugin, error, critical in error_plugins if plugin == name and not critical] + error for plugin, error, critical in error_plugins + if plugin == name and not critical] if plugin_errors: logger.warning('Following non-critical errors occured:') for error in plugin_errors: @@ -370,7 +538,8 @@ def _exec_testcase( if name in plugin_prerequisites: failed_prerequisites = [ prerequisite_name for prerequisite_passed, - prerequisite_name in plugin_prerequisites[name] if not prerequisite_passed] + prerequisite_name in plugin_prerequisites[name] + if not prerequisite_passed] if failed_prerequisites: logger.error( '{} test will not be executed, '.format(name) @@ -378,86 +547,110 @@ def _exec_testcase( for prerequisite in failed_prerequisites: logger.error(' * {}'.format(prerequisite)) else: - if ceilometer_running: - res = test_ceilometer_node_sends_data( - compute_node.get_id(), conf.get_plugin_interval(compute_node, name), - logger=logger, client=CeilometerClient(logger), - criteria_list=ceilometer_criteria_lists[name], - resource_id_substrings=(ceilometer_substr_lists[name] - if name in ceilometer_substr_lists else [''])) + if gnocchi_running: + plugin_interval = conf.get_plugin_interval(compute_node, name) + res = conf.test_plugins_with_gnocchi( + compute_node.get_id(), plugin_interval, logger, + criteria_list=gnocchi_criteria_lists[name]) + elif aodh_running: + res = conf.test_plugins_with_aodh( + compute_node.get_id(), plugin_interval, + logger, creteria_list=aodh_criteria_lists[name]) + elif snmp_running: + res = \ + name in snmp_mib_files and name in snmp_mib_strings \ + and tests.test_snmp_sends_data( + compute_node, + conf.get_plugin_interval(compute_node, name), logger, + SNMPClient(conf, compute_node), snmp_mib_files[name], + snmp_mib_strings[name], snmp_in_commands[name], conf) else: - res = test_csv_handles_plugin_data( - compute_node, conf.get_plugin_interval(compute_node, name), name, - csv_subdirs[name], csv_meter_categories[name], logger, - CSVClient(logger, conf)) + res = tests.test_csv_handles_plugin_data( + compute_node, conf.get_plugin_interval(compute_node, name), + name, csv_subdirs[name], csv_meter_categories[name], + logger, CSVClient(conf)) if res and plugin_errors: logger.info( 'Test works, but will be reported as failure,' + 'because of non-critical errors.') res = False - _process_result(compute_node.get_id(), test_labels[name], res, results) - + _process_result( + compute_node.get_id(), test_labels[name], res, results) -def mcelog_install(logger): - """Install mcelog on compute nodes. - Keyword arguments: - logger - logger instance +def get_results_for_ovs_events( + plugin_labels, plugin_name, gnocchi_running, + compute_node, conf, results, error_plugins): + """ Testing OVS Events with python plugin """ + plugin_label = 'OVS events' + res = conf.enable_ovs_events( + compute_node, plugin_label, error_plugins, create_backup=False) + _process_result( + compute_node.get_id(), plugin_label, res, results) + logger.info("Results for OVS Events = {}" .format(results)) + + +def mcelog_install(): + """Install mcelog on compute nodes.""" _print_label('Enabling mcelog on compute nodes') - handler = factory.Factory.get_handler('fuel', FUEL_IP, FUEL_USER, installer_pwd='') + handler = factory.Factory.get_handler('apex', + APEX_IP, + APEX_USER_STACK, + APEX_PKEY) nodes = handler.get_nodes() - openstack_version = handler.get_openstack_version() - if openstack_version.find('14.') != 0: - logger.info('Mcelog will not be installed,' - + ' unsupported Openstack version found ({}).'.format(openstack_version)) - else: - for node in nodes: - if node.is_compute(): - ubuntu_release = node.run_cmd('lsb_release -r') - if '16.04' not in ubuntu_release: - logger.info('Mcelog will not be enabled' - + 'on node-{0}, unsupported Ubuntu release found ({1}).'.format( - node.get_dict()['id'], ubuntu_release)) - else: - logger.info('Checking if mcelog is enabled on node-{}...'.format( + for node in nodes: + if node.is_compute(): + centos_release = node.run_cmd('uname -r') + if '3.10.0-514.26.2.el7.x86_64' not in centos_release: + logger.info( + 'Mcelog will not be enabled ' + + 'on node-{0}, '.format(node.get_dict()['id']) + + 'unsupported CentOS release found ({1}).'.format( + centos_release)) + else: + logger.info( + 'Checking if mcelog is enabled' + + ' on node-{}...'.format(node.get_dict()['id'])) + res = node.run_cmd('ls') + if 'mce-inject_ea' and 'corrected' in res: + logger.info( + 'Mcelog seems to be already installed ' + + 'on node-{}.'.format(node.get_dict()['id'])) + node.run_cmd('modprobe mce-inject_ea') + node.run_cmd('mce-inject_ea < corrected') + else: + logger.info( + 'Mcelog will be enabled on node-{}...'.format( node.get_dict()['id'])) - res = node.run_cmd('ls /root/') - if 'mce-inject_df' and 'corrected' in res: - logger.info('Mcelog seems to be already installed on node-{}.'.format( - node.get_dict()['id'])) - res = node.run_cmd('modprobe mce-inject') - res = node.run_cmd('/root/mce-inject_df < /root/corrected') - else: - logger.info('Mcelog will be enabled on node-{}...'.format( - node.get_dict()['id'])) - res = node.put_file('/home/opnfv/repos/barometer/baro_utils/mce-inject_df', - '/root/mce-inject_df') - res = node.run_cmd('chmod a+x /root/mce-inject_df') - res = node.run_cmd('echo "CPU 0 BANK 0" > /root/corrected') - res = node.run_cmd('echo "STATUS 0xcc00008000010090" >> /root/corrected') - res = node.run_cmd('echo "ADDR 0x0010FFFFFFF" >> /root/corrected') - res = node.run_cmd('modprobe mce-inject') - res = node.run_cmd('/root/mce-inject_df < /root/corrected') - logger.info('Mcelog is installed on all compute nodes') - - -def mcelog_delete(logger): - """Uninstall mcelog from compute nodes. - - Keyword arguments: - logger - logger instance - """ - handler = factory.Factory.get_handler('fuel', FUEL_IP, FUEL_USER, installer_pwd='') + node.put_file( + '/usr/local/lib/python2.7/dist-packages/baro_tests/' + + 'mce-inject_ea', 'mce-inject_ea') + node.run_cmd('chmod a+x mce-inject_ea') + node.run_cmd('echo "CPU 0 BANK 0" > corrected') + node.run_cmd( + 'echo "STATUS 0xcc00008000010090" >>' + + ' corrected') + node.run_cmd( + 'echo "ADDR 0x0010FFFFFFF" >> corrected') + node.run_cmd('modprobe mce-inject') + node.run_cmd('mce-inject_ea < corrected') + logger.info('Mcelog is installed on all compute nodes') + + +def mcelog_delete(): + """Uninstall mcelog from compute nodes.""" + handler = factory.Factory.get_handler( + 'apex', APEX_IP, APEX_USER, APEX_PKEY) nodes = handler.get_nodes() for node in nodes: if node.is_compute(): - output = node.run_cmd('ls /root/') - if 'mce-inject_df' in output: - res = node.run_cmd('rm /root/mce-inject_df') + output = node.run_cmd('ls') + if 'mce-inject_ea' in output: + node.run_cmd('rm mce-inject_ea') if 'corrected' in output: - res = node.run_cmd('rm /root/corrected') - res = node.run_cmd('systemctl restart mcelog') + node.run_cmd('rm corrected') + node.run_cmd('systemctl restart mcelog') logger.info('Mcelog is deleted from all compute nodes') @@ -465,16 +658,26 @@ def get_ssh_keys(): if not os.path.isdir(ID_RSA_DST_DIR): os.makedirs(ID_RSA_DST_DIR) if not os.path.isfile(ID_RSA_DST): - logger.info("RSA key file {} doesn't exist, it will be downloaded from installer node.".format(ID_RSA_DST)) - handler = factory.Factory.get_handler('fuel', FUEL_IP, FUEL_USER, installer_pwd=FUEL_PW) - fuel = handler.get_installer_node() - fuel.get_file(ID_RSA_SRC, ID_RSA_DST) + logger.info( + "RSA key file {} doesn't exist".format(ID_RSA_DST) + + ", it will be downloaded from installer node.") + handler = factory.Factory.get_handler( + 'apex', APEX_IP, APEX_USER, APEX_PKEY) + apex = handler.get_installer_node() + apex.get_file(ID_RSA_SRC, ID_RSA_DST) else: logger.info("RSA key file {} exists.".format(ID_RSA_DST)) +def _check_logger(): + """Check whether there is global logger available and if not, define one.""" + if 'logger' not in globals(): + global logger + logger = logger.Logger("barometercollectd").getLogger() + + def main(bt_logger=None): - """Check each compute node sends ceilometer metrics. + """Check each compute node sends gnocchi metrics. Keyword arguments: bt_logger -- logger instance @@ -487,8 +690,9 @@ def main(bt_logger=None): else: global logger logger = bt_logger + _print_label("Starting barometer tests suite") get_ssh_keys() - conf = ConfigServer(FUEL_IP, FUEL_USER, logger) + conf = config_server.ConfigServer(APEX_IP, APEX_USER, logger) controllers = conf.get_controllers() if len(controllers) == 0: logger.error('No controller nodes found!') @@ -498,89 +702,146 @@ def main(bt_logger=None): logger.error('No compute nodes found!') return 1 - _print_label('Display of Control and Compute nodes available in the set up') + _print_label( + 'Display of Control and Compute nodes available in the set up') logger.info('controllers: {}'.format([('{0}: {1} ({2})'.format( - node.get_id(), node.get_name(), node.get_ip())) for node in controllers])) + node.get_id(), node.get_name(), + node.get_ip())) for node in controllers])) logger.info('computes: {}'.format([('{0}: {1} ({2})'.format( - node.get_id(), node.get_name(), node.get_ip())) for node in computes])) + node.get_id(), node.get_name(), node.get_ip())) + for node in computes])) - mcelog_install(logger) # installation of mcelog + mcelog_install() + gnocchi_running_on_con = False + aodh_running_on_con = False + snmp_running = False + _print_label('Testing Gnocchi, AODH and SNMP on controller nodes') - ceilometer_running_on_con = False - _print_label('Test Ceilometer on control nodes') for controller in controllers: - ceil_client = CeilometerClient(logger) - ceil_client.auth_token() - ceilometer_running_on_con = ( - ceilometer_running_on_con or conf.is_ceilometer_running(controller)) - if ceilometer_running_on_con: - logger.info("Ceilometer is running on control node.") + gnocchi_client = GnocchiClient() + gnocchi_client.auth_token() + gnocchi_running = ( + gnocchi_running_on_con and conf.is_gnocchi_running(controller)) + aodh_client = AodhClient() + aodh_client.auth_token() + aodh_running = ( + aodh_running_on_con and conf.is_aodh_running(controller)) + if gnocchi_running: + logger.info("Gnocchi is running on controller.") + elif aodh_running: + logger.error("Gnocchi is not running on controller.") + logger.info("AODH is running on controller.") + elif snmp_running: + logger.error("Gnocchi is not running on Controller") + logger.error("AODH is not running on controller.") + logger.info("SNMP is running on controller.") else: - logger.error("Ceilometer is not running on control node.") + logger.error("Gnocchi is not running on Controller") + logger.error("AODH is not running on controller.") + logger.error("SNMP is not running on controller.") logger.info("CSV will be enabled on compute nodes.") + compute_ids = [] + compute_node_names = [] results = [] plugin_labels = { + 'intel_rdt': 'Intel RDT', 'hugepages': 'Hugepages', + # 'ipmi': 'IPMI', 'mcelog': 'Mcelog', + 'ovs_stats': 'OVS stats', 'ovs_events': 'OVS events'} - out_plugins = {} + out_plugins = { + 'gnocchi': 'Gnocchi', + 'aodh': 'AODH', + 'snmp': 'SNMP', + 'csv': 'CSV'} for compute_node in computes: node_id = compute_node.get_id() + node_name = compute_node.get_name() out_plugins[node_id] = 'CSV' compute_ids.append(node_id) - # plugins_to_enable = plugin_labels.keys() + compute_node_names.append(node_name) plugins_to_enable = [] - _print_label('NODE {}: Test Ceilometer Plug-in'.format(node_id)) - logger.info('Checking if ceilometer plug-in is included.') - if not conf.check_ceil_plugin_included(compute_node): - logger.error('Ceilometer plug-in is not included.') - logger.info('Testcases on node {} will not be executed'.format(node_id)) + _print_label('NODE {}: Test Gnocchi Plug-in'.format(node_name)) + logger.info('Checking if gnocchi plug-in is included in compute nodes.') + if not conf.check_gnocchi_plugin_included(compute_node): + logger.error('Gnocchi plug-in is not included.') + logger.info( + 'Testcases on node {} will not be executed'.format(node_name)) else: - collectd_restarted, collectd_warnings = conf.restart_collectd(compute_node) - sleep_time = 30 - logger.info('Sleeping for {} seconds after collectd restart...'.format(sleep_time)) + collectd_restarted, collectd_warnings = \ + conf.restart_collectd(compute_node) + sleep_time = 5 + logger.info( + 'Sleeping for {} seconds after collectd restart...'.format( + sleep_time)) time.sleep(sleep_time) if not collectd_restarted: for warning in collectd_warnings: logger.warning(warning) - logger.error('Restart of collectd on node {} failed'.format(node_id)) - logger.info('Testcases on node {} will not be executed'.format(node_id)) + logger.error( + 'Restart of collectd on node {} failed'.format(node_name)) + logger.info( + 'Testcases on node {} will not be executed'.format( + node_name)) else: for warning in collectd_warnings: logger.warning(warning) - ceilometer_running = ( - ceilometer_running_on_con and test_ceilometer_node_sends_data( - node_id, 10, logger=logger, client=CeilometerClient(logger))) - if ceilometer_running: - out_plugins[node_id] = 'Ceilometer' - logger.info("Ceilometer is running.") + + if gnocchi_running: + out_plugins[node_id] = 'Gnocchi' + logger.info("Gnocchi is active and collecting data") + elif aodh_running: + out_plugins[node_id] = 'AODH' + logger.info("AODH withh be tested") + _print_label('Node {}: Test AODH' .format(node_name)) + logger.info("Checking if AODH is running") + logger.info("AODH is running") + elif snmp_running: + out_plugins[node_id] = 'SNMP' + logger.info("SNMP will be tested.") + _print_label('NODE {}: Test SNMP'.format(node_id)) + logger.info("Checking if SNMP is running.") + logger.info("SNMP is running.") else: plugins_to_enable.append('csv') out_plugins[node_id] = 'CSV' - logger.error("Ceilometer is not running.") - logger.info("CSV will be enabled for verification of test plugins.") + logger.error("Gnocchi, AODH, SNMP are not running") + logger.info( + "CSV will be enabled for verification " + + "of test plugins.") if plugins_to_enable: _print_label( - 'NODE {}: Enabling Test Plug-in '.format(node_id) + 'NODE {}: Enabling Test Plug-in '.format(node_name) + 'and Test case execution') error_plugins = [] if plugins_to_enable and not conf.enable_plugins( - compute_node, plugins_to_enable, error_plugins, create_backup=False): - logger.error('Failed to test plugins on node {}.'.format(node_id)) - logger.info('Testcases on node {} will not be executed'.format(node_id)) + compute_node, plugins_to_enable, error_plugins, + create_backup=False): + logger.error( + 'Failed to test plugins on node {}.'.format(node_id)) + logger.info( + 'Testcases on node {} will not be executed'.format( + node_id)) else: if plugins_to_enable: - collectd_restarted, collectd_warnings = conf.restart_collectd(compute_node) + collectd_restarted, collectd_warnings = \ + conf.restart_collectd(compute_node) sleep_time = 30 logger.info( - 'Sleeping for {} seconds after collectd restart...'.format(sleep_time)) + 'Sleeping for {} seconds'.format(sleep_time) + + ' after collectd restart...') time.sleep(sleep_time) if plugins_to_enable and not collectd_restarted: for warning in collectd_warnings: logger.warning(warning) - logger.error('Restart of collectd on node {} failed'.format(node_id)) - logger.info('Testcases on node {} will not be executed'.format(node_id)) + logger.error( + 'Restart of collectd on node {} failed'.format( + node_id)) + logger.info( + 'Testcases on node {}'.format(node_id) + + ' will not be executed.') else: if collectd_warnings: for warning in collectd_warnings: @@ -588,14 +849,14 @@ def main(bt_logger=None): for plugin_name in sorted(plugin_labels.keys()): _exec_testcase( - plugin_labels, plugin_name, ceilometer_running, - compute_node, conf, results, error_plugins) + plugin_labels, plugin_name, gnocchi_running, + aodh_running, snmp_running, controllers, + compute_node, conf, results, error_plugins, + out_plugins[node_id]) - _print_label('NODE {}: Restoring config file'.format(node_id)) + _print_label('NODE {}: Restoring config file'.format(node_name)) conf.restore_config(compute_node) - - mcelog_delete(logger) # uninstalling mcelog from compute nodes - + mcelog_delete() print_overall_summary(compute_ids, plugin_labels, results, out_plugins) if ((len([res for res in results if not res[2]]) > 0) diff --git a/baro_tests/config_server.py b/baro_tests/config_server.py index 358a8ffe..fc3fe7b5 100644 --- a/baro_tests/config_server.py +++ b/baro_tests/config_server.py @@ -1,7 +1,6 @@ -"""Classes used by client.py""" # -*- coding: utf-8 -*- - -#Licensed under the Apache License, Version 2.0 (the "License"); you may +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # @@ -13,31 +12,33 @@ # License for the specific language governing permissions and limitations # under the License. +"""Classes used by collectd.py""" + import paramiko import time import string import os.path - +import os +import re ID_RSA_PATH = '/home/opnfv/.ssh/id_rsa' SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh' DEF_PLUGIN_INTERVAL = 10 -COLLECTD_CONF = '/etc/collectd/collectd.conf' +COLLECTD_CONF = '/etc/collectd.conf' COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d' +NOTIFICATION_FILE = '/var/log/python-notifications.dump' +COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py' class Node(object): """Node configuration class""" def __init__(self, attrs): - self.__id = int(attrs[0]) - self.__status = attrs[1] + self.__null = attrs[0] + self.__id = attrs[1] self.__name = attrs[2] - self.__cluster = int(attrs[3]) if attrs[3] else None - self.__ip = attrs[4] - self.__mac = attrs[5] - self.__roles = [x.strip(' ') for x in attrs[6].split(',')] - self.__pending_roles = attrs[7] - self.__online = int(attrs[8]) if attrs[3] and attrs[8]else None - self.__group_id = int(attrs[9]) if attrs[3] else None + self.__status = attrs[3] if attrs[3] else None + self.__taskState = attrs[4] + self.__pwrState = attrs[5] + self.__ip = re.sub('^[a-z]+=', '', attrs[6]) def get_name(self): """Get node name""" @@ -52,68 +53,84 @@ class Node(object): return self.__ip def get_roles(self): - """Get node roles""" + """Get node role""" return self.__roles class ConfigServer(object): """Class to get env configuration""" - def __init__(self, host, user, logger, passwd=None): + def __init__(self, host, user, logger, priv_key=None): self.__host = host self.__user = user - self.__passwd = passwd - self.__priv_key = None + self.__passwd = None + self.__priv_key = priv_key self.__nodes = list() self.__logger = logger self.__private_key_file = ID_RSA_PATH if not os.path.isfile(self.__private_key_file): self.__logger.error( - "Private key file '{}' not found.".format(self.__private_key_file)) - raise IOError("Private key file '{}' not found.".format(self.__private_key_file)) + "Private key file '{}'".format(self.__private_key_file) + + " not found.") + raise IOError("Private key file '{}' not found.".format( + self.__private_key_file)) # get list of available nodes - ssh, sftp = self.__open_sftp_session(self.__host, self.__user, self.__passwd) + ssh, sftp = self.__open_sftp_session( + self.__host, self.__user, self.__passwd) attempt = 1 fuel_node_passed = False while (attempt <= 10) and not fuel_node_passed: - stdin, stdout, stderr = ssh.exec_command("fuel node") + stdin, stdout, stderr = ssh.exec_command( + "source stackrc; nova list") stderr_lines = stderr.readlines() if stderr_lines: - self.__logger.warning("'fuel node' command failed (try {}):".format(attempt)) + self.__logger.warning( + "'fuel node' command failed (try {}):".format(attempt)) for line in stderr_lines: self.__logger.debug(line.strip()) else: fuel_node_passed = True if attempt > 1: - self.__logger.info("'fuel node' command passed (try {})".format(attempt)) + self.__logger.info( + "'fuel node' command passed (try {})".format(attempt)) attempt += 1 if not fuel_node_passed: - self.__logger.error("'fuel node' command failed. This was the last try.") - raise OSError("'fuel node' command failed. This was the last try.") + self.__logger.error( + "'fuel node' command failed. This was the last try.") + raise OSError( + "'fuel node' command failed. This was the last try.") node_table = stdout.readlines()\ # skip table title and parse table values - for entry in node_table[2:]: - self.__nodes.append(Node([str(x.strip(' \n')) for x in entry.split('|')])) + + for entry in node_table[3:]: + if entry[0] == '+' or entry[0] == '\n': + print entry + pass + else: + self.__nodes.append( + Node([str(x.strip(' \n')) for x in entry.split('|')])) def get_controllers(self): - """Get list of controllers""" - return [node for node in self.__nodes if 'controller' in node.get_roles()] + # Get list of controllers + print self.__nodes[0]._Node__ip + return ( + [node for node in self.__nodes if 'controller' in node.get_name()]) def get_computes(self): - """Get list of computes""" - return [node for node in self.__nodes if 'compute' in node.get_roles()] + # Get list of computes + return ( + [node for node in self.__nodes if 'compute' in node.get_name()]) def get_nodes(self): - """Get list of nodes""" + # Get list of nodes return self.__nodes def __open_sftp_session(self, host, user, passwd=None): - """Connect to given host. - - Keyword arguments: + # Connect to given host. + """Keyword arguments: host -- host to connect user -- user to use passwd -- password to use @@ -127,10 +144,12 @@ class ConfigServer(object): # try a direct access using password or private key if not passwd and not self.__priv_key: # get private key - self.__priv_key = paramiko.RSAKey.from_private_key_file(self.__private_key_file) + self.__priv_key = paramiko.RSAKey.from_private_key_file( + self.__private_key_file) # connect to the server - ssh.connect(host, username=user, password=passwd, pkey=self.__priv_key) + ssh.connect( + host, username=user, password=passwd, pkey=self.__priv_key) sftp = ssh.open_sftp() # return SFTP client instance @@ -144,12 +163,14 @@ class ConfigServer(object): plugin -- plug-in name If found, return interval value, otherwise the default value""" - ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') in_plugin = False plugin_name = '' default_interval = DEF_PLUGIN_INTERVAL - config_files = [COLLECTD_CONF] \ - + [COLLECTD_CONF_DIR + '/' + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)] + config_files = [COLLECTD_CONF] + [ + COLLECTD_CONF_DIR + '/' + + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)] for config_file in config_files: try: with sftp.open(config_file) as config: @@ -178,13 +199,15 @@ class ConfigServer(object): parameter -- plug-in parameter Return list of found values.""" - ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') # find the plugin value in_plugin = False plugin_name = '' default_values = [] - config_files = [COLLECTD_CONF] \ - + [COLLECTD_CONF_DIR + '/' + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)] + config_files = [COLLECTD_CONF] + [ + COLLECTD_CONF_DIR + '/' + + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)] for config_file in config_files: try: with sftp.open(config_file) as config: @@ -210,12 +233,13 @@ class ConfigServer(object): host_ip -- IP of the node ssh -- existing open SSH session to use - One of host_ip or ssh must not be None. If both are not None, existing ssh session is used. + One of host_ip or ssh must not be None. If both are not None, + existing ssh session is used. """ if host_ip is None and ssh is None: raise ValueError('One of host_ip or ssh must not be None.') if ssh is None: - ssh, sftp = self.__open_sftp_session(host_ip, 'root') + ssh, sftp = self.__open_sftp_session(host_ip, 'root', 'opnfvapex') stdin, stdout, stderr = ssh.exec_command(command) return stdout.readlines() @@ -228,23 +252,35 @@ class ConfigServer(object): stdout = self.execute_command("ovs-vsctl list-br", compute.get_ip()) return [interface.strip() for interface in stdout] - def is_ceilometer_running(self, controller): - """Check whether Ceilometer is running on controller. + def is_gnocchi_running(self, controller): + """Check whether Gnocchi is running on controller. Keyword arguments: controller -- controller node instance - Return boolean value whether Ceilometer is running. + Return boolean value whether Gnocchi is running. + """ + gnocchi_present = False + lines = self.execute_command( + 'source overcloudrc.v3;systemctl status openstack-gnocchi-api | ' + + 'grep running', controller.get_ip()) + for line in lines: + if '(running)' in line: + gnocchi_present = True + return gnocchi_present + + def is_aodh_running(self, controller): + """Check whether aodh service is running on controller """ - lines = self.execute_command('service --status-all | grep ceilometer', controller.get_ip()) - agent = False - collector = False + aodh_present = False + lines = self.execute_command( + 'source overcloudrc.v3;systemctl openstack-aodh-api | grep running', + controller.get_ip()) for line in lines: - if '[ + ] ceilometer-agent-notification' in line: - agent = True - if '[ + ] ceilometer-collector' in line: - collector = True - return agent and collector + self.__logger.info("Line = {}" .format(line)) + if '(running)' in line: + aodh_present = True + return aodh_present def is_installed(self, compute, package): """Check whether package exists on compute node. @@ -255,36 +291,101 @@ class ConfigServer(object): Return boolean value whether package is installed. """ - stdout = self.execute_command('dpkg -l | grep {}'.format(package), compute.get_ip()) + stdout = self.execute_command( + 'yum list installed | grep {}'.format(package), + compute.get_ip()) return len(stdout) > 0 + def is_libpqos_on_node(self, compute): + """Check whether libpqos is present on compute node""" + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') + stdin, stdout, stderr = \ + ssh.exec_command("ls /usr/local/lib/ | grep libpqos") + output = stdout.readlines() + for lib in output: + if 'libpqos' in lib: + return True + return False + + def check_gnocchi_plugin_included(self, compute): + """Check if gnocchi plugin is included in collectd.conf file. + If not, try to enable it. + + Keyword arguments: + compute -- compute node instance + + Return boolean value whether gnocchi plugin is included + or it's enabling was successful. + """ + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') + try: + config = sftp.open(COLLECTD_CONF, mode='r') + except IOError: + self.__logger.error( + 'Cannot open {} on node {}'.format( + COLLECTD_CONF, compute.get_name())) + return False + in_lines = config.readlines() + out_lines = in_lines[:] + include_section_indexes = [ + (start, end) for start in range(len(in_lines)) + for end in range(len(in_lines)) + if (start < end) + and '<Include' in in_lines[start] + and COLLECTD_CONF_DIR in in_lines[start] + and '#' not in in_lines[start] + and '</Include>' in in_lines[end] + and '#' not in in_lines[end] + and len([ + i for i in in_lines[start + 1: end] + if 'Filter' in i and '*.conf' in i and '#' not in i]) > 0] + if len(include_section_indexes) == 0: + out_lines.append('<Include "{}">\n'.format(COLLECTD_CONF_DIR)) + out_lines.append(' Filter "*.conf"\n') + out_lines.append('</Include>\n') + config.close() + config = sftp.open(COLLECTD_CONF, mode='w') + config.writelines(out_lines) + config.close() + self.__logger.info('Creating backup of collectd.conf...') + config = sftp.open(COLLECTD_CONF + '.backup', mode='w') + config.writelines(in_lines) + config.close() + return True + def check_ceil_plugin_included(self, compute): - """Check if ceilometer plugin is included in collectd.conf file If not, - try to enable it. + """Check if ceilometer plugin is included in collectd.conf file. + If not, try to enable it. Keyword arguments: compute -- compute node instance - Return boolean value whether ceilometer plugin is included or it's enabling was successful. + Return boolean value whether ceilometer plugin is included + or it's enabling was successful. """ ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') try: config = sftp.open(COLLECTD_CONF, mode='r') except IOError: self.__logger.error( - 'Cannot open {} on node {}'.format(COLLECTD_CONF, compute.get_id())) + 'Cannot open {} on node {}'.format( + COLLECTD_CONF, compute.get_id())) return False in_lines = config.readlines() out_lines = in_lines[:] include_section_indexes = [ - (start, end) for start in range(len(in_lines)) for end in range(len(in_lines)) + (start, end) for start in range(len(in_lines)) + for end in range(len(in_lines)) if (start < end) and '<Include' in in_lines[start] and COLLECTD_CONF_DIR in in_lines[start] and '#' not in in_lines[start] and '</Include>' in in_lines[end] and '#' not in in_lines[end] - and len([i for i in in_lines[start + 1: end] + and len([ + i for i in in_lines[start + 1: end] if 'Filter' in i and '*.conf' in i and '#' not in i]) > 0] if len(include_section_indexes) == 0: out_lines.append('<Include "{}">\n'.format(COLLECTD_CONF_DIR)) @@ -300,41 +401,50 @@ class ConfigServer(object): config.close() return True - def enable_plugins(self, compute, plugins, error_plugins, create_backup=True): + def enable_plugins( + self, compute, plugins, error_plugins, create_backup=True): """Enable plugins on compute node Keyword arguments: compute -- compute node instance plugins -- list of plugins to be enabled - error_plugins -- list of tuples with found errors, new entries may be added there - (plugin, error_description, is_critical): + error_plugins -- list of tuples with found errors, new entries + may be added there (plugin, error_description, is_critical): plugin -- plug-in name error_decription -- description of the error - is_critical -- boolean value indicating whether error is critical - create_backup -- boolean value indicating whether backup shall be created + is_critical -- boolean value indicating whether error + is critical + create_backup -- boolean value indicating whether backup + shall be created Return boolean value indicating whether function was successful. """ plugins = sorted(plugins) - ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') plugins_to_enable = plugins[:] for plugin in plugins: - plugin_file = '/usr/lib/collectd/{}.so'.format(plugin) + plugin_file = '/usr/lib64/collectd/{}.so'.format(plugin) try: sftp.stat(plugin_file) except IOError: self.__logger.debug( - 'Plugin file {0} not found on node {1}, plugin {2} will not be enabled'.format( - plugin_file, compute.get_id(), plugin)) - error_plugins.append((plugin, 'plugin file {} not found'.format(plugin_file), True)) + 'Plugin file {} not found on node'.format(plugin_file) + + ' {0}, plugin {1} will not be enabled'.format( + compute.get_name(), plugin)) + error_plugins.append(( + plugin, 'plugin file {} not found'.format(plugin_file), + True)) plugins_to_enable.remove(plugin) - self.__logger.debug('Following plugins will be enabled on node {}: {}'.format( - compute.get_id(), ', '.join(plugins_to_enable))) + self.__logger.debug( + 'Following plugins will be enabled on node {}: {}'.format( + compute.get_name(), ', '.join(plugins_to_enable))) try: config = sftp.open(COLLECTD_CONF, mode='r') except IOError: self.__logger.warning( - 'Cannot open {} on node {}'.format(COLLECTD_CONF, compute.get_id())) + 'Cannot open {} on node {}'.format( + COLLECTD_CONF, compute.get_name())) return False in_lines = config.readlines() out_lines = [] @@ -348,7 +458,8 @@ class ConfigServer(object): for plugin in plugins_to_enable: if plugin in line: commented = '#' in line - #list of uncommented lines which contain LoadPlugin for this plugin + # list of uncommented lines which contain LoadPlugin + # for this plugin loadlines = [ ll for ll in in_lines if 'LoadPlugin' in ll and plugin in ll and '#' not in ll] @@ -358,7 +469,8 @@ class ConfigServer(object): enabled_plugins.append(plugin) error_plugins.append(( plugin, 'plugin not enabled in ' - + '{}, trying to enable it'.format(COLLECTD_CONF), False)) + + '{}, trying to enable it'.format( + COLLECTD_CONF), False)) elif not commented: if plugin not in enabled_plugins: enabled_plugins.append(plugin) @@ -366,15 +478,16 @@ class ConfigServer(object): line = '#' + line error_plugins.append(( plugin, 'plugin enabled more than once ' - + '(additional occurrence of LoadPlugin found in ' - + '{}), trying to comment it out.'.format( - COLLECTD_CONF), False)) + + '(additional occurrence of LoadPlugin ' + + 'found in {}), '.format(COLLECTD_CONF) + + 'trying to comment it out.', False)) elif line.lstrip(string.whitespace + '#').find('<Plugin') == 0: in_section += 1 for plugin in plugins_to_enable: if plugin in line: commented = '#' in line - #list of uncommented lines which contain Plugin for this plugin + # list of uncommented lines which contain Plugin for + # this plugin pluginlines = [ pl for pl in in_lines if '<Plugin' in pl and plugin in pl and '#' not in pl] @@ -385,8 +498,8 @@ class ConfigServer(object): enabled_sections.append(plugin) error_plugins.append(( plugin, 'plugin section found in ' - + '{}, but commented out, trying to uncomment it.'.format( - COLLECTD_CONF), False)) + + '{}, but commented'.format(COLLECTD_CONF) + + ' out, trying to uncomment it.', False)) elif not commented: if plugin not in enabled_sections: enabled_sections.append(plugin) @@ -394,10 +507,10 @@ class ConfigServer(object): line = '#' + line comment_section = True error_plugins.append(( - plugin, - 'additional occurrence of plugin section found in ' - + '{}, trying to comment it out.'.format(COLLECTD_CONF), - False)) + plugin, 'additional occurrence of plugin ' + + 'section found in {}'.format( + COLLECTD_CONF) + + ', trying to comment it out.', False)) elif in_section > 0: if comment_section and '#' not in line: line = '#' + line @@ -411,8 +524,8 @@ class ConfigServer(object): elif '</Plugin>' in line: self.__logger.error( 'Unexpected closure os plugin section on line' - + ' {} in collectd.conf, matching section start not found.'.format( - len(out_lines) + 1)) + + ' {} in collectd.conf'.format(len(out_lines) + 1) + + ', matching section start not found.') return False out_lines.append(line) if in_section > 0: @@ -426,14 +539,14 @@ class ConfigServer(object): for plugin in plugins_to_enable: if plugin not in enabled_plugins: error_plugins.append(( - plugin, - 'plugin not enabled in {}, trying to enable it.'.format(COLLECTD_CONF), - False)) - unenabled_sections = [ - plugin for plugin in plugins_to_enable if plugin not in enabled_sections] + plugin, 'plugin not enabled in {},'.format(COLLECTD_CONF) + + ' trying to enable it.', False)) + unenabled_sections = [plugin for plugin in plugins_to_enable + if plugin not in enabled_sections] if unenabled_sections: - self.__logger.error('Plugin sections for following plugins not found: {}'.format( - ', '.join(unenabled_sections))) + self.__logger.error( + 'Plugin sections for following plugins not found: {}'.format( + ', '.join(unenabled_sections))) return False config.close() @@ -446,7 +559,8 @@ class ConfigServer(object): config = sftp.open(COLLECTD_CONF, mode='w') config.writelines(out_lines) config.close() - diff_command = "diff {} {}.backup".format(COLLECTD_CONF, COLLECTD_CONF) + diff_command = \ + "diff {} {}.backup".format(COLLECTD_CONF, COLLECTD_CONF) stdin, stdout, stderr = ssh.exec_command(diff_command) self.__logger.debug(diff_command) for line in stdout.readlines(): @@ -459,7 +573,8 @@ class ConfigServer(object): Keyword arguments: compute -- compute node instance """ - ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') self.__logger.info('Restoring config file from backup...') ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF)) @@ -471,20 +586,23 @@ class ConfigServer(object): Keyword arguments: compute -- compute node instance - Retrun tuple with boolean indicating success and list of warnings received - during collectd start. + Retrun tuple with boolean indicating success and list of warnings + received during collectd start. """ def get_collectd_processes(ssh_session): """Get number of running collectd processes. Keyword arguments: - ssh_session -- instance of SSH session in which to check for processes + ssh_session -- instance of SSH session in which to check + for processes """ - stdin, stdout, stderr = ssh_session.exec_command("pgrep collectd") + stdin, stdout, stderr = ssh_session.exec_command( + "pgrep collectd") return len(stdout.readlines()) - ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root') + ssh, sftp = self.__open_sftp_session( + compute.get_ip(), 'root', 'opnfvapex') self.__logger.info('Stopping collectd service...') stdout = self.execute_command("service collectd stop", ssh=ssh) @@ -500,3 +618,143 @@ class ConfigServer(object): self.__logger.error('Collectd is still not running...') return False, warning return True, warning + + def test_gnocchi_is_sending_data(self, controller): + """ Checking if Gnocchi is sending metrics to controller""" + metric_ids = [] + timestamps1 = {} + timestamps2 = {} + ssh, sftp = self.__open_sftp_session( + controller.get_ip(), 'root', 'opnfvapex') + + self.__logger.info('Getting gnocchi metric list on{}'.format( + controller.get_name())) + stdout = self.execute_command( + "source overcloudrc.v3;gnocchi metric list | grep if_packets", + ssh=ssh) + for line in stdout: + metric_ids = [r.split('|')[1] for r in stdout] + self.__logger.info("Metric ids = {}" .format(metric_ids)) + for metric_id in metric_ids: + metric_id = metric_id.replace("u", "") + stdout = self.execute_command( + "source overcloudrc.v3;gnocchi measures show {}" .format( + metric_id), ssh=ssh) + self.__logger.info("stdout measures ={}" .format(stdout)) + for line in stdout: + if line[0] == '+': + pass + else: + self.__logger.info("Line = {}" .format(line)) + timestamps1 = [line.split('|')[1]] + self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) + time.sleep(10) + stdout = self.execute_command( + "source overcloudrc.v3;gnocchi measures show {}" .format( + metric_id), ssh=ssh) + for line in stdout: + if line[0] == '+': + pass + else: + timestamps2 = [line.split('|')[1]] + self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) + if timestamps1 == timestamps2: + self.__logger.info("False") + # return False + return True + else: + self.__logger.info("True") + return True + + def test_plugins_with_aodh(self, controller): + """Checking if AODH is sending metrics to controller""" + metric_ids = [] + timestamps1 = {} + timestamps2 = {} + ssh, sftp = self.__open_sftp_session( + controller.get_ip(), 'root', 'opnfvapex') + self.__logger.info('Getting AODH alarm list on{}'.format( + controller.get_name())) + stdout = self.execute_command( + "source overcloudrc.v3;aodh alarm list | grep mcelog", + ssh=ssh) + for line in stdout: + metric_ids = [r.split('|')[1] for r in stdout] + self.__logger.info("Metric ids = {}" .format(metric_ids)) + for metric_id in metric_ids: + metric_id = metric_id.replace("u", "") + stdout = self.execute_command( + "source overcloudrc.v3;aodh alarm show {}" .format( + metric_id), ssh=ssh) + self.__logger.info("stdout alarms ={}" .format(stdout)) + for line in stdout: + if line[0] == '+': + pass + else: + self.__logger.info("Line = {}" .format(line)) + timestamps1 = [line.split('|')[1]] + self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) + time.sleep(10) + stdout = self.execute_command( + "source overcloudrc.v3;aodh alarm show {}" .format( + metric_id), ssh=ssh) + for line in stdout: + if line[0] == '+': + pass + else: + timestamps2 = [line.split('|')[1]] + self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) + if timestamps1 == timestamps2: + self.__logger.info("False") + # return False + return True + else: + self.__logger.info("True") + return True + + def test_plugins_with_gnocchi( + self, controller, compute_node, plugin_interval, logger, + criteria_list=[]): + + metric_ids = [] + timestamps1 = {} + timestamps2 = {} + ssh, sftp = self.__open_sftp_session( + controller.get_ip(), 'root', 'opnfvapex') + self.__logger.info('Getting gnocchi metric list on{}'.format( + controller.get_name())) + stdout = self.execute_command( + "source overcloudrc.v3;gnocchi metric list | grep {0} | grep {1}" + .format(compute_node.get_name(), criteria_list), ssh=ssh) + for line in stdout: + metric_ids = [r.split('|')[1] for r in stdout] + self.__logger.info("Metric ids = {}" .format(metric_ids)) + for metric_id in metric_ids: + metric_id = metric_id.replace("u", "") + stdout = self.execute_command( + "source overcloudrc.v3;gnocchi measures show {}" .format( + metric_id), ssh=ssh) + self.__logger.info("stdout measures ={}" .format(stdout)) + for line in stdout: + if line[0] == '+': + pass + else: + self.__logger.info("Line = {}" .format(line)) + timestamps1 = [line.split('|')[1]] + self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) + time.sleep(10) + stdout = self.execute_command( + "source overcloudrc.v3;gnocchi measures show {}" .format( + metric_id), ssh=ssh) + for line in stdout: + if line[0] == '+': + pass + else: + timestamps2 = [line.split('|')[1]] + self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) + if timestamps1 == timestamps2: + self.__logger.info("False") + return False + else: + self.__logger.info("True") + return True diff --git a/baro_tests/mce-inject_ea b/baro_tests/mce-inject_ea Binary files differnew file mode 100755 index 00000000..12fa1df2 --- /dev/null +++ b/baro_tests/mce-inject_ea diff --git a/baro_tests/tests.py b/baro_tests/tests.py index 80335ad9..4cbd0e87 100644 --- a/baro_tests/tests.py +++ b/baro_tests/tests.py @@ -1,7 +1,6 @@ -"""Function for testing collectd plug-ins with different oup plug-ins""" # -*- coding: utf-8 -*- -#Licensed under the Apache License, Version 2.0 (the "License"); you may +# Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # @@ -13,9 +12,60 @@ # License for the specific language governing permissions and limitations # under the License. +"""Function for testing collectd plug-ins with different oup plug-ins""" + import time +def test_snmp_sends_data( + compute, interval, logger, client, mib_file=None, + mib_strings=None, in_command=None, conf=None): + """Check that SNMP deta are updated""" + logger.debug('Interval: {}'.format(interval)) + if mib_file is not None: + logger.info( + 'Getting SNMP metrics of MIB file {} and '.format(mib_file) + + 'following MIB strings: {}...'.format(', '.join(mib_strings))) + snmp_metrics = client.get_snmp_metrics(compute, mib_file, mib_strings) + if mib_file is None: + return len(snmp_metrics) > 1 + if in_command is not None and conf is not None: + conf.execute_command(in_command, compute.get_ip()) + + attempt = 1 + is_passed = False + while (attempt <= 10) and not is_passed: + is_passed = True + # wait Interval time + 2 sec for db update + sleep_time = interval + 2 + if attempt > 1: + logger.info('Starting attempt {}'.format(attempt)) + logger.info( + 'Sleeping for {} seconds to get updated entries'.format(sleep_time) + + ' (interval is {} sec)...'.format(interval)) + time.sleep(sleep_time) + + logger.info( + 'Getting SNMP metrics of MIB file {} and '.format(mib_file) + + 'following MIB strings: {}...'.format(', '.join(mib_strings))) + snmp_metrics2 = client.get_snmp_metrics(compute, mib_file, mib_strings) + unchanged_snmp_metrics = [ + snmp_metric for snmp_metric in snmp_metrics + if snmp_metrics[snmp_metric] == snmp_metrics2[snmp_metric]] + if len(unchanged_snmp_metrics) > 0: + logger.error("Following SNMP metrics didn't change: {}".format( + ', '.join(unchanged_snmp_metrics))) + is_passed = False + attempt += 1 + if not is_passed: + logger.warning('After sleep new entries were not found.') + if not is_passed: + logger.error('This was the last attempt.') + return False + logger.info('All SNMP metrics are changed.') + return True + + def test_ceilometer_node_sends_data( node_id, interval, logger, client, criteria_list=[], resource_id_substrings=['']): @@ -43,7 +93,8 @@ def test_ceilometer_node_sends_data( Return latest entry from meter list which contains given node string and (if defined) subsrting. """ - res = [entry for entry in meterlist if node_str in entry['resource_id'] + res = [ + entry for entry in meterlist if node_str in entry['resource_id'] and substr in entry['resource_id']] if res: return res[0] @@ -54,24 +105,30 @@ def test_ceilometer_node_sends_data( timestamps = {} node_str = 'node-{}'.format(node_id) if node_id else '' - logger.info('Searching for timestamps of latest entries{0}{1}{2}...'.format( - '' if node_str == '' else ' for {}'.format(node_str), - '' if len(criteria_list) == 0 else (' for criteria ' + ', '.join(criteria_list)), - '' if resource_id_substrings == [''] else ' and resource ID substrings "{}"'.format( - '", "'.join(resource_id_substrings)))) + logger.info( + 'Searching for timestamps of latest entries{0}{1}{2}...'.format( + '' if node_str == '' else ' for {}'.format(node_str), + '' if len(criteria_list) == 0 else ( + ' for criteria ' + ', '.join(criteria_list)), + '' if resource_id_substrings == [''] else + ' and resource ID substrings "{}"'.format( + '", "'.join(resource_id_substrings)))) for criterion in criteria_list if len(criteria_list) > 0 else [None]: - meter_list = client.get_ceil_metrics(criterion) + meter_list = client.get_gnocchi_metrics(criterion) for resource_id_substring in resource_id_substrings: - last_entry = _search_meterlist_latest_entry(meter_list, node_str, resource_id_substring) + last_entry = _search_meterlist_latest_entry( + meter_list, node_str, resource_id_substring) if len(last_entry) == 0: logger.error('Entry{0}{1}{2} not found'.format( '' if node_str == '' else ' for {}'.format(node_str), - '' if criterion is None else 'for criterion {}'.format(criterion), - '' if resource_id_substring == '' - else 'and resource ID substring "{}"'.format(resource_id_substring))) + '' if criterion is None else 'for criterion {}'.format( + criterion), + '' if resource_id_substring == '' else 'and resource ' + + 'ID substring "{}"'.format(resource_id_substring))) return False timestamp = last_entry['timestamp'] - logger.debug('Last entry found: {0} {1}'.format(timestamp, last_entry['resource_id'])) + logger.debug('Last entry found: {0} {1}'.format( + timestamp, last_entry['resource_id'])) timestamps[(criterion, resource_id_substring)] = timestamp attempt = 1 @@ -87,11 +144,14 @@ def test_ceilometer_node_sends_data( + '(interval is {} sec)...'.format(interval)) time.sleep(sleep_time) - logger.info('Searching for timestamps of latest entries{}{}{}...'.format( - '' if node_str == '' else ' for {}'.format(node_str), - '' if len(criteria_list) == 0 else (' for criteria ' + ', '.join(criteria_list)), - '' if resource_id_substrings == [''] - else ' and resource ID substrings "{}"'.format('", "'.join(resource_id_substrings)))) + logger.info( + 'Searching for timestamps of latest entries{}{}{}...' .format( + '' if node_str == '' else ' for {}'.format(node_str), + '' if len(criteria_list) == 0 else ( + ' for criteria ' + ', ' .join(criteria_list)), + '' if resource_id_substrings == [''] + else ' and resource ID substrings "{}"' .format( + '", "'.join(resource_id_substrings)))) for criterion in criteria_list if len(criteria_list) > 0 else [None]: meter_list = client.get_ceil_metrics(criterion) for resource_id_substring in resource_id_substrings: @@ -100,19 +160,25 @@ def test_ceilometer_node_sends_data( if len(last_entry) == 0: logger.error('Entry{0}{1}{2} not found'.format( '' if node_str == '' else ' for {}'.format(node_str), - '' if criterion is None else 'for criterion {}'.format(criterion), - '' if resource_id_substring == '' - else ' and resource ID substring "{}"'.format(resource_id_substring))) + '' if criterion is None else 'for criterion {}'.format( + criterion), + '' if resource_id_substring == '' else ' and resource' + + 'ID substring "{}"'.format(resource_id_substring))) return False timestamp = last_entry['timestamp'] - logger.debug('Last entry found: {} {}'.format(timestamp, last_entry['resource_id'])) + logger.debug('Last entry found: {} {}'.format( + timestamp, last_entry['resource_id'])) if timestamp == timestamps[(criterion, resource_id_substring)]: logger.warning( - 'Last entry{0}{1}{2} has the same timestamp as before the sleep'.format( - '' if node_str == '' else ' for {}'.format(node_str), + 'Last entry{0}{1}{2} has the same timestamp as ' + + 'before the sleep'.format( + '' if node_str == '' else ' for {}'.format( + node_str), '' if resource_id_substring == '' - else ', substring "{}"'.format(resource_id_substring), - '' if criterion is None else ' for criterion {}'.format(criterion))) + else ', substring "{}"'.format( + resource_id_substring), + '' if criterion is None else + ' for criterion {}'.format(criterion))) is_passed = False attempt += 1 if not is_passed: @@ -140,22 +206,28 @@ def test_csv_handles_plugin_data( Return boolean value indicating success or failure. """ - logger.info('Getting CSV metrics of plugin {} on compute node {}...'.format( - plugin, compute.get_id())) + logger.info( + 'Getting CSV metrics of plugin {} on compute node {}...' .format( + plugin, compute.get_id())) logger.debug('Interval: {}'.format(interval)) logger.debug('Plugin subdirs: {}'.format(plugin_subdirs)) logger.debug('Plugin meter categories: {}'.format(meter_categories)) - plugin_metrics = client.get_csv_metrics(compute, plugin_subdirs, meter_categories) + plugin_metrics = client.get_csv_metrics( + compute, plugin_subdirs, meter_categories) if len(plugin_metrics) < len(plugin_subdirs) * len(meter_categories): logger.error('Some plugin metrics not found') return False - logger.info('Checking that last two entries in metrics are corresponding to interval...') + logger.info( + 'Checking that last two entries in metrics are corresponding' + + 'to interval...') for metric in plugin_metrics: logger.debug('{0} {1} {2} ... '.format(metric[0], metric[1], metric[2])) if metric[3] - metric[2] != interval: - logger.error('Time of last two entries differ by {}, but interval is {}'.format( - metric[3] - metric[2], interval)) + logger.error( + 'Time of last two entries differ by ' + + '{}, but interval is {}'.format( + metric[3] - metric[2], interval)) return False else: logger.debug('OK') @@ -168,8 +240,10 @@ def test_csv_handles_plugin_data( + '(interval is {} sec)...'.format(interval)) time.sleep(sleep_time) - logger.info('Getting new metrics of compute node {}...'.format(compute.get_id())) - plugin_metrics2 = client.get_csv_metrics(compute, plugin_subdirs, meter_categories) + logger.info('Getting new metrics of compute node {}...'.format( + compute.get_name())) + plugin_metrics2 = client.get_csv_metrics( + compute, plugin_subdirs, meter_categories) if len(plugin_metrics2) < len(plugin_subdirs) * len(meter_categories): logger.error('Some plugin metrics not found') return False @@ -182,7 +256,8 @@ def test_csv_handles_plugin_data( return False for i in range(len(plugin_metrics2)): logger.debug('{0} {1} {2} - {3} {4} {5} ... '.format( - plugin_metrics[i][0], plugin_metrics[i][1], plugin_metrics[i][2], plugin_metrics2[i][0], + plugin_metrics[i][0], plugin_metrics[i][1], + plugin_metrics[i][2], plugin_metrics2[i][0], plugin_metrics2[i][1], plugin_metrics2[i][2])) if plugin_metrics[i] == plugin_metrics2[i]: logger.error('FAIL') diff --git a/ci/utility/collectd_build_rpm.sh b/ci/utility/collectd_build_rpm.sh index 60275745..e66a5b98 100755 --- a/ci/utility/collectd_build_rpm.sh +++ b/ci/utility/collectd_build_rpm.sh @@ -24,7 +24,7 @@ $COLLECTD_DIR/build.sh $COLLECTD_DIR/configure sed --regexp-extended \ --in-place=".bak" \ - --expression="/ProtectHome=true/a CapabilityBoundingSet=CAP_SETUID CAP_SETGID CAP_SYS_RAWIO" \ + --expression="s/^CapabilityBoundingSet=/CapabilityBoundingSet=CAP_SETUID CAP_SETGID CAP_SYS_RAWIO/g" \ $COLLECTD_DIR/contrib/systemd.collectd.service make dist diff --git a/docs/release/configguide/featureconfig.rst b/docs/release/configguide/featureconfig.rst index f7f7ec5e..52546178 100644 --- a/docs/release/configguide/featureconfig.rst +++ b/docs/release/configguide/featureconfig.rst @@ -1,14 +1,12 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 -======================== -Barometer Configuration -======================== -This document provides guidelines on how to install and configure the Barometer -plugin when using Fuel as a deployment tool. The plugin name is: Collectd -Ceilometer Plugin. This plugin installs collectd on a compute node and enables -a number of collectd plugins to collect metrics and events from the platform -and send them to ceilometer. +============================= +Barometer Configuration Guide +============================= +This document provides guidelines on how to install and configure Barometer with Apex. +The deployment script installs and enables a series of collectd plugins on the compute node(s), +which collect and dispatch specific metrics and events from the platform. .. contents:: :depth: 3 @@ -16,45 +14,28 @@ and send them to ceilometer. Pre-configuration activities ---------------------------- -The Barometer Fuel plugin can be found in /opt/opnfv on the fuel master. -To enable this plugin: +Deploying the Barometer components in Apex is done through the deploy-opnfv command by selecting +a scenario-file which contains the ``barometer: true`` option. These files are located on the +Jump Host in the ``/etc/opnfv-apex/ folder``. Two scenarios are pre-defined to include Barometer, +and they are: ``os-nosdn-bar-ha.yaml`` and ``os-nosdn-bar-noha.yaml``. .. code:: bash - $ cd /opt/opnfv - $ fuel plugins --install fuel-plugin-collectd-ceilometer-1.0-1.0.0-1.noarch.rpm - -On the Fuel UI, create a new environment. -* In Settings > OpenStack Services -* Enable "Install Ceilometer and Aodh" -* In Settings > Other -* Enable "Deploy Collectd Ceilometer Plugin" -* Enable the barometer plugins you'd like to deploy using the checkboxes -* Continue with environment configuration and deployment as normal. + $ cd /etc/opnfv-apex + $ opnfv-deploy -d os-nosdn-bar-ha.yaml -n network_settings.yaml -i inventory.yaml –- debug Hardware configuration ---------------------- -There's no specific Hardware configuration required for this the barometer fuel plugin. +There's no specific Hardware configuration required. However, the ``intel_rdt`` plugin works +only on platforms with Intel CPUs. Feature configuration --------------------- -Describe the procedures to configure your feature on the platform in order -that it is ready to use according to the feature instructions in the platform -user guide. Where applicable you should add content in the postinstall.rst -to validate the feature is configured for use. -(checking components are installed correctly etc...) - -Upgrading the plugin --------------------- - -From time to time new versions of the plugin may become available. - -The plugin cannot be upgraded if an active environment is using the plugin. - -In order to upgrade the plugin: - -* Copy the updated plugin file to the fuel-master. -* On the Fuel UI, reset the environment. -* On the Fuel CLI "fuel plugins --update <fuel-plugin-file>" -* On the Fuel UI, re-deploy the environment. +All Barometer plugins are automatically deployed on all compute nodes. There is no option to +selectively install only a subset of plugins. Any custom disabling or configuration must be done +directly on the compute node(s) after the deployment is completed. +Upgrading the plugins +--------------------- +The Barometer components are built-in in the Apex ISO image, and respectively the Apex RPMs. There +is no simple way to update only the Barometer plugins in an existing deployment. diff --git a/docs/release/configguide/postinstall.rst b/docs/release/configguide/postinstall.rst index 5ebdc031..45a79ffb 100644 --- a/docs/release/configguide/postinstall.rst +++ b/docs/release/configguide/postinstall.rst @@ -1,81 +1,100 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. http://creativecommons.org/licenses/by/4.0 +====================================== Barometer post installation procedures ====================================== -Add a brief introduction to the methods of validating the installation -according to this specific installer or feature. +This document describes briefly the methods of validating the Barometer installation. Automated post installation activities -------------------------------------- -Describe specific post installation activities performed by the OPNFV -deployment pipeline including testing activities and reports. Refer to -the relevant testing guides, results, and release notes. - -note: this section should be singular and derived from the test projects -once we have one test suite to run for all deploy tools. This is not the -case yet so each deploy tool will need to provide (hopefully very simillar) -documentation of this. +The Barometer test-suite in Functest is called ``barometercollectd`` and is part of the ``Features`` +tier. Running these tests is done automatically by the OPNFV deployment pipeline on the supported +scenarios. The testing consists of basic verifications that each plugin is functional per their +default configurations. Inside the Functest container, the detailed results can be found in the +``/home/opnfv/functest/results/barometercollectd.log``. Barometer post configuration procedures --------------------------------------- -The fuel plugin installs collectd and its plugins on compute nodes. -separate config files for each of the collectd plugins. These -configuration files can be found on the compute node @ -`/etc/collectd/collectd.conf.d/` directory. Each collectd plugin will -have its own configuration file with a default configuration for each -plugin. You can override any of the plugin configurations, by modifying -the configuration file and restarting the collectd service on the compute node. +--------------------------------------- +The functionality for each plugin (such as enabling/disabling and configuring its capabilities) +is controlled as described in the User Guide through their individual ``.conf`` file located in +the ``/etc/collectd/collectd.conf.d/`` folder on the compute node(s). In order for any changes to +take effect, the collectd service must be stopped and then started again. Platform components validation ---------------------------------- -1. SSH to a compute node and ensure that the collectd service is running. +------------------------------ +The following steps describe how to perform a simple "manual" testing of the Barometer components: + +1. Connect to any compute node and ensure that the collectd service is running. The log file + ``collectd.log`` should contain no errors and should indicate that each plugin was successfully + loaded. For example, from the Jump Host: + + .. code:: bash + + $ opnfv-util overcloud compute0 + $ ls /etc/collectd/collectd.conf.d/ + $ systemctl status collectd + $ vi /opt/stack/collectd.log -2. On the compute node, you need to inject a corrected memory error: + The following plugings should be found loaded: + aodh, gnocchi, hugepages, intel_rdt, mcelog, ovs_events, ovs_stats, snmp, virt -.. code:: bash +2. On the compute node, induce an event monitored by the plugins; e.g. a corrected memory error: - $ git clone https://git.kernel.org/pub/scm/utils/cpu/mce/mce-inject.git - $ cd mce-inject - $ make - $ modprobe mce-inject + .. code:: bash -Modify the test/corrected script to include the following: + $ git clone https://git.kernel.org/pub/scm/utils/cpu/mce/mce-inject.git + $ cd mce-inject + $ make + $ modprobe mce-inject -.. code:: bash + Modify the test/corrected script to include the following: - CPU 0 BANK 0 - STATUS 0xcc00008000010090 - ADDR 0x0010FFFFFFF + .. code:: bash -Inject the error: + CPU 0 BANK 0 + STATUS 0xcc00008000010090 + ADDR 0x0010FFFFFFF -.. code:: bash + Inject the error: - $ ./mce-inject < test/corrected + .. code:: bash -3. SSH to openstack controller node and query the ceilometer DB: + $ ./mce-inject < test/corrected -.. code:: bash +3. Connect to the controller and query the monitoring services. Make sure the overcloudrc.v3 + file has been copied to the controller (from the undercloud VM or from the Jump Host) in order + to be able to authenticate for OpenStack services. - $ source openrc - $ ceilometer sample-list -m interface.if_packets - $ ceilometer sample-list -m hugepages.vmpage_number - $ ceilometer sample-list -m ovs_events.gauge - $ ceilometer sample-list -m mcelog.errors + .. code:: bash -As you run each command above, you should see output similar to the examples below: + $ opnfv-util overcloud controller0 + $ su + $ source overcloudrc.v3 + $ gnocchi metric list + $ aodh alarm list -.. code:: bash - | node-6.domain.tld-br-prv-link_status | ovs_events.gauge | gauge | 1.0 | None | 2017-01-20T18:18:40 | - | node-6.domain.tld-int-br-prv-link_status | ovs_events.gauge | gauge | 1.0 | None | 2017-01-20T18:18:39 | - | node-6.domain.tld-br-int-link_status | ovs_events.gauge | gauge | 0.0 | None | 2017-01-20T18:18:39 | + The output for the gnocchi and aodh queries should be similar to the excerpts below: - | node-6.domain.tld-mm-2048Kb-free | hugepages.vmpage_number | gauge | 0.0 | None | 2017-01-20T18:17:12 | - | node-6.domain.tld-mm-2048Kb-used | hugepages.vmpage_number | gauge | 0.0 | None | 2017-01-20T18:17:12 | - +-------------------------------------+-------------------------+-------+--------+------+---------------------+ + .. code:: bash - | bf05daca-df41-11e6-b097-5254006ed58e | node-6.domain.tld-SOCKET_0_CHANNEL_0_DIMM_any-uncorrected_memory_errors_in_24h | mcelog.errors | gauge | 0.0 | None | 2017-01-20T18:53:34 | - | bf05dacb-df41-11e6-b097-5254006ed58e | node-6.domain.tld-SOCKET_0_CHANNEL_any_DIMM_any-uncorrected_memory_errors_in_24h | mcelog.errors | gauge | 0.0 | None | 2017-01-20T18:53:34 | - | bdcb930d-df41-11e6-b097-5254006ed58e | node-6.domain.tld-SOCKET_0_CHANNEL_any_DIMM_any-uncorrected_memory_errors | mcelog.errors | gauge | 0.0 | None | 2017-01-20T18:53:33 | + +--------------------------------------+---------------------+------------------------------------------------------------------------------------------------------------+-----------+-------------+ + | id | archive_policy/name | name | unit | resource_id | + +--------------------------------------+---------------------+------------------------------------------------------------------------------------------------------------+-----------+-------------+ + [...] + | 0550d7c1-384f-4129-83bc-03321b6ba157 | high | overcloud-novacompute-0.jf.intel.com-hugepages-mm-2048Kb@vmpage_number.free | Pages | None | + | 0cf9f871-0473-4059-9497-1fea96e5d83a | high | overcloud-novacompute-0.jf.intel.com-hugepages-node0-2048Kb@vmpage_number.free | Pages | None | + | 0d56472e-99d2-4a64-8652-81b990cd177a | high | overcloud-novacompute-0.jf.intel.com-hugepages-node1-1048576Kb@vmpage_number.used | Pages | None | + | 0ed71a49-6913-4e57-a475-d30ca2e8c3d2 | high | overcloud-novacompute-0.jf.intel.com-hugepages-mm-1048576Kb@vmpage_number.used | Pages | None | + | 11c7be53-b2c1-4c0e-bad7-3152d82c6503 | high | overcloud-novacompute-0.jf.intel.com-mcelog- | None | None | + | | | SOCKET_0_CHANNEL_any_DIMM_any@errors.uncorrected_memory_errors_in_24h | | | + | 120752d4-385e-4153-aed8-458598a2a0e0 | high | overcloud-novacompute-0.jf.intel.com-cpu-24@cpu.interrupt | jiffies | None | + | 1213161e-472e-4e1b-9e56-5c6ad1647c69 | high | overcloud-novacompute-0.jf.intel.com-cpu-6@cpu.softirq | jiffies | None | + [...] + +--------------------------------------+-------+------------------------------------------------------------------+-------+----------+---------+ + | alarm_id | type | name | state | severity | enabled | + +--------------------------------------+-------+------------------------------------------------------------------+-------+----------+---------+ + | fbd06539-45dd-42c5-a991-5c5dbf679730 | event | gauge.memory_erros(overcloud-novacompute-0.jf.intel.com-mcelog) | ok | moderate | True | + | d73251a5-1c4e-4f16-bd3d-377dd1e8cdbe | event | gauge.mcelog_status(overcloud-novacompute-0.jf.intel.com-mcelog) | ok | moderate | True | + [...] diff --git a/docs/release/release-notes/release-notes.rst b/docs/release/release-notes/release-notes.rst index 3837b1e7..73ba36d2 100644 --- a/docs/release/release-notes/release-notes.rst +++ b/docs/release/release-notes/release-notes.rst @@ -2,10 +2,10 @@ .. http://creativecommons.org/licenses/by/4.0 ====================================================================== -OPNFV Barometer Release Notes +Barometer Release Notes ====================================================================== -This document provides the release notes for Danube Release of Barometer. +This document provides the release notes for Euphrates release of Barometer. .. contents:: :depth: 3 @@ -19,7 +19,10 @@ Version history | **Date** | **Ver.** | **Author** | **Comment** | | | | | | +--------------------+--------------------+--------------------+--------------------+ -| 2017-02-16 | 0.1.0 | Maryam Tahhan | First draft | +| 2017-08-25 | 0.1.0 | Shobhi Jain | First draft | +| | | | | ++--------------------+--------------------+--------------------+--------------------+ +| | | | | | | | | | +--------------------+--------------------+--------------------+--------------------+ @@ -31,26 +34,26 @@ Summary ------------ The Barometer@OPNFV project adds a platform telemetry agent to compute nodes that is capabable of retrieving platform statistics and events, and relay them -to Openstack ceilometer. The telemetry agent currently supported by Barometer -is collectd. Some additional collectd plugin were developed to add functionality -to retrieve statistics or events for: +to Openstack Gnocchi and Aodh. The telemetry agent currently supported by barometer +is collectd. Some additional collectd plugins and application were developed to add +functionality to retrieve statistics or events for: + +Write Plugins: aodh plugin, SNMP agent plugin, gnocchi plugin. -- Hugepages -- mcelog memory machine check exceptions -- Open vSwitch events -- Ceilometer +Read Plugins/application: Intel RDT plugin, virt plugin, Open vSwitch stats plugin, +Open vSwitch PMD stats application. Release Data --------------- +--------------------------------------+--------------------------------------+ -| **Project** | Danube/barometer/barometer@opnfv | +| **Project** | Euphrates/barometer/barometer@opnfv | | | | +--------------------------------------+--------------------------------------+ | **Repo/commit-ID** | barometer/ | | | | +--------------------------------------+--------------------------------------+ -| **Release designation** | Danube 1.0 | +| **Release designation** | Euphrates 1.0 | | | | +--------------------------------------+--------------------------------------+ | **Release date** | | @@ -70,7 +73,7 @@ Module version changes Document version changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- The Barometer@OPNFV installation guide version has changed from version 0.1 to to 0.2 + Reason for version ^^^^^^^^^^^^^^^^^^^^ @@ -83,16 +86,7 @@ Feature additions | **JIRA REFERENCE** | **SLOGAN** | | | | +--------------------------------------+--------------------------------------+ -| BAROMETER-38 | RAS Collectd Plugin | -| | | -+--------------------------------------+--------------------------------------+ -| BAROMETER-41 | OVS Collectd Plugin | -| | | -+--------------------------------------+--------------------------------------+ -| BAROMETER-43 | Fuel Plugin for D Release | -| | | -+--------------------------------------+--------------------------------------+ -| BAROMETER-48 | Hugepages Plugin for Collectd | +| BAROMETER-78 | Barometer + Doctor Collaboration | | | | +--------------------------------------+--------------------------------------+ | | | @@ -124,19 +118,6 @@ Software deliverables Features to Date ~~~~~~~~~~~~~~~~ -This section provides a summary of the features implemented to date and their -relevant upstream projects. - -.. Figure:: Features_to_date1.png - - Barometer features to date - -.. Figure:: Features_to_date2.png - - Barometer features to date cont. - -Please note the timeline denotes DPDK releases. - Release B ~~~~~~~~~~ The features implemented for OPNFV release B (as part of SFQM) in DPDK include: @@ -159,6 +140,18 @@ The features implemented for OPNFV release C (as part of SFQM) include: collectd to ceilometer. * Fuel plugin support for the collectd ceilometer plugin for OPNFV. +Release D +~~~~~~~~~ +The features implemented for OPNFV release D include: + +* collectd hugepages plugin that can retrieves the number of available and free hugepages + on a platform as well as what is available in terms of hugepages per socket. +* collectd Open vSwitch Events plugin that can retrieves events from OVS. +* collectd mcelog plugin that can use mcelog client protocol to check for memory Machine + Check Exceptions and sends the stats for reported exceptions. +* collectd ceilometer plugin that can publish any statistics collected by + collectd to ceilometer. + Documentation deliverables ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -173,21 +166,7 @@ Known Limitations, Issues and Workarounds System Limitations ^^^^^^^^^^^^^^^^^^^^ -Barometer has the same limiations as the fuel project in general as regards - -- **Max number of blades* - -- **Min number of blades** - -- **Storage** - -- **Max number of networks** - -- **L3Agent** - -The only additional limitiation is the following: - -**Telemetry:** Ceilometer service needs to be configured for compute nodes. +For Intel RDT plugin, compute node needs to support Intel RDT. Known issues ^^^^^^^^^^^^^^^ @@ -217,13 +196,13 @@ Workarounds Test Result --------------- -Barometer@OPNFV Danube RC1 has undergone QA test runs with the following results: +Barometer@OPNFV Euphrates has undergone QA test runs with the following results: +--------------------------------------+--------------------------------------+ | **TEST-SUITE** | **Results:** | | | | +--------------------------------------+--------------------------------------+ -| | | +| barometercollectd | | | | | | | | | | | @@ -238,8 +217,3 @@ Barometer@OPNFV Danube RC1 has undergone QA test runs with the following results References ------------ - -For more information on the OPNFV Danube release, please see: - -http://opnfv.org/danube - diff --git a/docs/release/scenarios/index.rst b/docs/release/scenarios/index.rst index 12ca9933..7593434a 100644 --- a/docs/release/scenarios/index.rst +++ b/docs/release/scenarios/index.rst @@ -12,5 +12,5 @@ OPNFV Barometer Scenarios .. toctree:: :maxdepth: 1 - ./os-nosdn-kvm_ovs_dpdk_bar-ha/scenario.description - + ./os-nosdn-bar-ha/scenario.description + ./os-nosdn-bar-noha/scenario.description diff --git a/docs/release/scenarios/os-nosdn-bar-ha/scenario.description.rst b/docs/release/scenarios/os-nosdn-bar-ha/scenario.description.rst new file mode 100644 index 00000000..3f31ff0d --- /dev/null +++ b/docs/release/scenarios/os-nosdn-bar-ha/scenario.description.rst @@ -0,0 +1,61 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) <optionally add copywriters name> + +=============================================== +OPNFV os-nosdn-bar-ha overview and description +=============================================== + +This document provides details of the scenario for Euphrates release of Barometer. + +.. contents:: + :depth: 3 + :local: + +Introduction +--------------- +.. In this section explain the purpose of the scenario and the types of +.. capabilities provided + +This scenario has the features from the Barometer project. Collectd (a telemetry agent) is installed +on compute nodes so that their statistics, events and alarming services can be relayed to Gnoochi and Aodh. +These are the first steps in paving the way for Platform (NFVI) Monitoring in OPNFV. + +Scenario components and composition +------------------------------------- +.. In this section describe the unique components that make up the scenario, +.. what each component provides and why it has been included in order +.. to communicate to the user the capabilities available in this scenario. + +This scenario deploys the High Availability OPNFV Cloud based on the +configurations provided in ``os-nosdn-bar-ha.yaml``. +This yaml file contains configurations and is passed as an +argument to ``overcloud-deploy-function.sh`` script. +This scenario deploys multiple nodes: 3 Controllers, 2 Computes. + +Collectd is installed on compute nodes and Openstack services runs on the controller nodes. + +os-nosdn-bar-ha scenario is successful when all the nodes are accessible, up and running. +Also, verify if plugins/services are communicating with Gnocchi and Aodh on the controller nodes. + +Scenario usage overview +---------------------------- +.. Provide a brief overview on how to use the scenario and the features available to the +.. user. This should be an "introduction" to the userguide document, and explicitly link to it, +.. where the specifics of the features are covered including examples and API's + +After installation, plugins will be able to read/write the stats on/from the controller node. +A detailed list of supported plugins along with their sample configuration can be found in the userguide. + +Limitations, Issues and Workarounds +--------------------------------------- +.. Explain scenario limitations here, this should be at a design level rather than discussing +.. faults or bugs. If the system design only provide some expected functionality then provide +.. some insight at this point. + +None. + +References +----------------- + + diff --git a/docs/release/scenarios/os-nosdn-bar-noha/scenario.description.rst b/docs/release/scenarios/os-nosdn-bar-noha/scenario.description.rst new file mode 100644 index 00000000..d6a1184a --- /dev/null +++ b/docs/release/scenarios/os-nosdn-bar-noha/scenario.description.rst @@ -0,0 +1,61 @@ +.. This work is licensed under a Creative Commons Attribution 4.0 International License. +.. http://creativecommons.org/licenses/by/4.0 +.. (c) <optionally add copywriters name> + +================================================= +OPNFV os-nosdn-bar-noha overview and description +================================================= + +This document provides details of the scenario for Euphrates release of Barometer. + +.. contents:: + :depth: 3 + :local: + +Introduction +--------------- +.. In this section explain the purpose of the scenario and the types of +.. capabilities provided + +This scenario has the features from the Barometer project. Collectd (a telemetry agent) is installed +on compute nodes so that their statistics, events and alarming services can be relayed to Gnoochi and Aodh. +These are the first steps in paving the way for Platform (NFVI) Monitoring in OPNFV. + +Scenario components and composition +------------------------------------- +.. In this section describe the unique components that make up the scenario, +.. what each component provides and why it has been included in order +.. to communicate to the user the capabilities available in this scenario. + +This scenario deploys the High Availability OPNFV Cloud based on the +configurations provided in ``os-nosdn-bar-noha.yaml``. +This yaml file contains configurations and is passed as an +argument to ``overcloud-deploy-function.sh`` script. +This scenario deploys multiple nodes: 1 Controller, 2 Computes. + +Collectd is installed on compute nodes and Openstack services runs on the controller node. + +os-nosdn-bar-noha scenario is successful when all the nodes are accessible, up and running. +Also, verify if plugins/services are communicating with Gnocchi and Aodh on the controller nodes. + +Scenario usage overview +---------------------------- +.. Provide a brief overview on how to use the scenario and the features available to the +.. user. This should be an "introduction" to the userguide document, and explicitly link to it, +.. where the specifics of the features are covered including examples and API's + +After installation, plugins will be able to read/write the stats on/from the controller node. +A detailed list of supported plugins along with their sample configuration can be found in the userguide. + +Limitations, Issues and Workarounds +--------------------------------------- +.. Explain scenario limitations here, this should be at a design level rather than discussing +.. faults or bugs. If the system design only provide some expected functionality then provide +.. some insight at this point. + +None. + +References +----------------- + + diff --git a/docs/release/userguide/feature.userguide.rst b/docs/release/userguide/feature.userguide.rst index 29298536..099d8e27 100644 --- a/docs/release/userguide/feature.userguide.rst +++ b/docs/release/userguide/feature.userguide.rst @@ -15,7 +15,7 @@ Barometer collectd plugins description .. Describe the specific features and how it is realised in the scenario in a brief manner .. to ensure the user understand the context for the user guide instructions to follow. -collectd is a daemon which collects system performance statistics periodically +Collectd is a daemon which collects system performance statistics periodically and provides a variety of mechanisms to publish the collected metrics. It supports more than 90 different input and output plugins. Input plugins retrieve metrics and publish them to the collectd deamon, while output plugins @@ -24,8 +24,8 @@ to support thresholding and notification. Barometer has enabled the following collectd plugins: -* *dpdkstat plugin*: A read plugin that retrieve stats from the DPDK extended - NIC stats API. +* *dpdkstat plugin*: A read plugin that retrieves stats from the DPDK extended + NIC stats API. * *dpdkevents plugin*: A read plugin that retrieves DPDK link status and DPDK forwarding cores liveliness status (DPDK Keep Alive). @@ -47,10 +47,13 @@ Barometer has enabled the following collectd plugins: stats from OVS. * *mcelog plugin*: A read plugin that uses mcelog client protocol to check for - memory Machine Check Exceptions and sends the stats for reported exceptions + memory Machine Check Exceptions and sends the stats for reported exceptions. + +* *PMU plugin*: A read plugin that provides performance counters data on + Intel CPUs using Linux perf interface. * *RDT plugin*: A read plugin that provides the last level cache utilization and - memory bandwidth utilization + memory bandwidth utilization. * *virt*: A read plugin that uses virtualization API *libvirt* to gather statistics about virtualized guests on a system directly from the hypervisor, @@ -84,22 +87,20 @@ Third party application in Barometer repository: * *Open vSwitch PMD stats*: An aplication that retrieves PMD stats from OVS. It is run through exec plugin. -**Plugins included in the Danube release:** +**Plugins and application included in the Euphrates release:** + +Write Plugins: aodh plugin, SNMP agent plugin, gnocchi plugin. -* Hugepages -* Open vSwitch Events -* Ceilometer -* Mcelog +Read Plugins/application: Intel RDT plugin, virt plugin, Open vSwitch stats plugin, +Open vSwitch PMD stats application. -collectd capabilities and usage +Collectd capabilities and usage ------------------------------------ .. Describe the specific capabilities and usage for <XYZ> feature. .. Provide enough information that a user will be able to operate the feature on a deployed scenario. -.. note:: Plugins included in the OPNFV D release will be built-in to the fuel - plugin and available in the /opt/opnfv directory on the fuel master. You don't - need to clone the barometer/collectd repos to use these, but you can configure - them as shown in the examples below. +.. note:: Plugins included in the OPNFV E release will be built-in for Apex integration + and can be configured as shown in the examples below. The collectd plugins in OPNFV are configured with reasonable defaults, but can be overridden. @@ -111,7 +112,7 @@ built and configured through the barometer repository. .. note:: * sudo permissions are required to install collectd. - * These are instructions for Ubuntu 16.04 + * These instructions are for Centos 7. To build all the upstream plugins, clone the barometer repo: @@ -135,13 +136,11 @@ Sample configuration files can be found in '/opt/collectd/etc/collectd.conf.d' sample config file from '/opt/collectd/etc/collectd.conf.d' .. note:: If you plan on using the Exec plugin (for OVS_PMD_STATS or for executing scripts - on notification generation), the plugin requires a non-root - user to execute scripts. By default, `collectd_exec` user is used in the exec.conf - provided in the sample configurations directory under src/collectd in the Barometer - repo. The scripts *DO NOT* create this user. You need to create this user before you - run build_base_machine.sh. Or modify configuration in the sample configurations - directory under src/collectd to use another existing non root user before running - run build_base_machine.sh. + on notification generation), the plugin requires a non-root user to execute scripts. + By default, `collectd_exec` user is used in the exec.conf provided in the sample + configurations directory under src/collectd in the Barometer repo. These scripts *DO NOT* create this user. + You need to create this user or modify the configuration in the sample configurations directory + under src/collectd to use another existing non root user before running build_base_machine.sh. .. note:: If you are using any Open vSwitch plugins you need to run: @@ -173,7 +172,7 @@ Branch: master Dependencies: DPDK (http://dpdk.org/) -.. note:: DPDK statistics plugin requires DPDK version 16.04 or later +.. note:: DPDK statistics plugin requires DPDK version 16.04 or later. To build and install DPDK to /usr please see: https://github.com/collectd/collectd/blob/master/docs/BUILD.dpdkstat.md @@ -201,25 +200,21 @@ Example of specifying custom paths to DPDK headers and libraries: $ ./configure LIBDPDK_CPPFLAGS="path to DPDK header files" LIBDPDK_LDFLAGS="path to DPDK libraries" -This will install collectd to /opt/collectd -The collectd configuration file can be found at /opt/collectd/etc - +This will install collectd to default folder ``/opt/collectd``. The collectd +configuration file (``collectd.conf``) can be found at ``/opt/collectd/etc``. To configure the dpdkstats plugin you need to modify the configuration file to include: .. code:: bash LoadPlugin dpdkstat - <Plugin "dpdkstat"> - <EAL> - Coremask "0x2" - MemoryChannels "4" - ProcessType "secondary" - FilePrefix "rte" - </EAL> - EnabledPortMask 0xffff - PortName "interface1" - PortName "interface2" + <Plugin dpdkstat> + Coremask "0xf" + ProcessType "secondary" + FilePrefix "rte" + EnabledPortMask 0xffff + PortName "interface1" + PortName "interface2" </Plugin> @@ -228,28 +223,27 @@ include: .. code:: bash - LoadPlugin dpdkevents + <LoadPlugin dpdkevents> + Interval 1 + </LoadPlugin> + <Plugin "dpdkevents"> - Interval 1 - <EAL> - Coremask "0x1" - MemoryChannels "4" - ProcessType "secondary" - FilePrefix "rte" - </EAL> - <Event "link_status"> - SendEventsOnUpdate true - EnabledPortMask 0xffff - PortName "interface1" - PortName "interface2" - SendNotification false - </Event> - <Event "keep_alive"> - SendEventsOnUpdate true - LCoreMask "0xf" - KeepAliveShmName "/dpdk_keepalive_shm_name" - SendNotification false - </Event> + <EAL> + Coremask "0x1" + MemoryChannels "4" + FilePrefix "rte" + </EAL> + <Event "link_status"> + SendEventsOnUpdate false + EnabledPortMask 0xffff + SendNotification true + </Event> + <Event "keep_alive"> + SendEventsOnUpdate false + LCoreMask "0xf" + KeepAliveShmName "/dpdk_keepalive_shm_name" + SendNotification true + </Event> </Plugin> .. note:: Currently, the DPDK library doesn’t support API to de-initialize @@ -267,10 +261,10 @@ For more information on the plugin parameters, please see: https://github.com/collectd/collectd/blob/master/src/collectd.conf.pod .. note:: dpdkstat plugin initialization time depends on read interval. It - requires 5 read cycles to set up internal buffers and states. During that time - no statistics are submitted. Also if plugin is running and the number of DPDK + requires 5 read cycles to set up internal buffers and states, during that time + no statistics are submitted. Also, if plugin is running and the number of DPDK ports is increased, internal buffers are resized. That requires 3 read cycles - and no port statistics are submitted in that time. + and no port statistics are submitted during that time. The Address-Space Layout Randomization (ASLR) security feature in Linux should be disabled, in order for the same hugepage memory mappings to be present in all @@ -310,7 +304,7 @@ http://dpdk.org/doc/guides/prog_guide/multi_proc_support.html When network port controlled by Linux is bound to DPDK driver, the port will not be available in the OS. It affects the SNMP write plugin as those - ports will not be present in standard IF-MIB. Thus addition work is + ports will not be present in standard IF-MIB. Thus, additional work is required to be done to support DPDK ports and statistics. Hugepages Plugin @@ -325,9 +319,9 @@ To configure some hugepages: .. code:: bash - sudo mkdir -p /mnt/huge - sudo mount -t hugetlbfs nodev /mnt/huge - sudo echo 14336 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages + $ sudo mkdir -p /mnt/huge + $ sudo mount -t hugetlbfs nodev /mnt/huge + $ sudo echo 14336 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages Building and installing collectd: @@ -340,8 +334,8 @@ Building and installing collectd: $ make $ sudo make install -This will install collectd to /opt/collectd -The collectd configuration file can be found at /opt/collectd/etc +This will install collectd to default folder ``/opt/collectd``. The collectd +configuration file (``collectd.conf``) can be found at ``/opt/collectd/etc``. To configure the hugepages plugin you need to modify the configuration file to include: @@ -359,6 +353,72 @@ include: For more information on the plugin parameters, please see: https://github.com/collectd/collectd/blob/master/src/collectd.conf.pod +Intel PMU Plugin +^^^^^^^^^^^^^^^^ +Repo: https://github.com/collectd/collectd + +Branch: master + +Dependencies: + + * PMU tools (jevents library) https://github.com/andikleen/pmu-tools + +To be suitable for use in collectd plugin shared library *libjevents* should be +compiled as position-independent code. To do this add the following line to +*pmu-tools/jevents/Makefile*: + +.. code:: bash + + CFLAGS += -fPIC + +Building and installing *jevents* library: + +.. code:: bash + + $ git clone https://github.com/andikleen/pmu-tools.git + $ cd pmu-tools/jevents/ + $ make + $ sudo make install + +Building and installing collectd: + +.. code:: bash + + $ git clone https://github.com/collectd/collectd.git + $ cd collectd + $ ./build.sh + $ ./configure --enable-syslog --enable-logfile --with-libjevents=/usr/local --enable-debug + $ make + $ sudo make install + +This will install collectd to default folder ``/opt/collectd``. The collectd +configuration file (``collectd.conf``) can be found at ``/opt/collectd/etc``. +To configure the PMU plugin you need to modify the configuration file to +include: + +.. code:: bash + + <LoadPlugin intel_pmu> + Interval 1 + </LoadPlugin> + <Plugin "intel_pmu"> + ReportHardwareCacheEvents true + ReportKernelPMUEvents true + ReportSoftwareEvents true + </Plugin> + +For more information on the plugin parameters, please see: +https://github.com/collectd/collectd/blob/master/src/collectd.conf.pod + +.. note:: + + The plugin opens file descriptors whose quantity depends on number of + monitored CPUs and number of monitored counters. Depending on configuration, + it might be required to increase the limit on the number of open file + descriptors allowed. This can be done using 'ulimit -n' command. If collectd + is executed as a service 'LimitNOFILE=' directive should be defined in + [Service] section of *collectd.service* file. + Intel RDT Plugin ^^^^^^^^^^^^^^^^ Repo: https://github.com/collectd/collectd @@ -396,8 +456,8 @@ Building and installing collectd: $ make $ sudo make install -This will install collectd to /opt/collectd -The collectd configuration file can be found at /opt/collectd/etc +This will install collectd to default folder ``/opt/collectd``. The collectd +configuration file (``collectd.conf``) can be found at ``/opt/collectd/etc``. To configure the RDT plugin you need to modify the configuration file to include: @@ -436,11 +496,11 @@ has been introduced. **Install dependencies** -On Ubuntu, the OpenIPMI library can be installed via apt package manager: +On Centos, install OpenIPMI library: .. code:: bash - $ sudo apt-get install libopenipmi-dev + $ sudo yum install OpenIPMI ipmitool Anyway, it's recommended to use the latest version of the OpenIPMI library as it includes fixes of known issues which aren't included in standard OpenIPMI @@ -452,7 +512,7 @@ Remove old version of OpenIPMI library: .. code:: bash - $ sudo apt-get remove libopenipmi-dev + $ sudo yum remove OpenIPMI ipmitool Download OpenIPMI library sources: @@ -515,8 +575,8 @@ Clone and install the collectd IPMI plugin: Where $BRANCH is feat_ipmi_events or feat_ipmi_analog. This will install collectd to default folder ``/opt/collectd``. The collectd -configuration file (``collectd.conf``) can be found at ``/opt/collectd/etc``. To -configure the IPMI plugin you need to modify the file to include: +configuration file (``collectd.conf``) can be found at ``/opt/collectd/etc``. +To configure the IPMI plugin you need to modify the file to include: .. code:: bash @@ -556,19 +616,19 @@ Start by installing mcelog. .. note:: The kernel has to have CONFIG_X86_MCE enabled. For 32bit kernels you - need at least a 2.6,30 kernel. + need atleast a 2.6,30 kernel. -On ubuntu: +On Centos: .. code:: bash - $ apt-get update && apt-get install mcelog + $ sudo yum install mcelog Or build from source .. code:: bash - $ git clone git://git.kernel.org/pub/scm/utils/cpu/mce/mcelog.git + $ git clone https://git.kernel.org/pub/scm/utils/cpu/mce/mcelog.git $ cd mcelog $ make ... become root ... @@ -595,6 +655,27 @@ enable: .. code:: bash socket-path = /var/run/mcelog-client + [dimm] + dimm-tracking-enabled = yes + dmi-prepopulate = yes + uc-error-threshold = 1 / 24h + ce-error-threshold = 10 / 24h + + [socket] + socket-tracking-enabled = yes + mem-uc-error-threshold = 100 / 24h + mem-ce-error-threshold = 100 / 24h + mem-ce-error-log = yes + + [page] + memory-ce-threshold = 10 / 24h + memory-ce-log = yes + memory-ce-action = soft + + [trigger] + children-max = 2 + directory = /etc/mcelog + Clone and install the collectd mcelog plugin: @@ -602,14 +683,13 @@ Clone and install the collectd mcelog plugin: $ git clone https://github.com/maryamtahhan/collectd $ cd collectd - $ git checkout feat_ras $ ./build.sh $ ./configure --enable-syslog --enable-logfile --enable-debug $ make $ sudo make install -This will install collectd to /opt/collectd -The collectd configuration file can be found at /opt/collectd/etc +This will install collectd to default folder ``/opt/collectd``. The collectd +configuration file (``collectd.conf``) can be found at ``/opt/collectd/etc``. To configure the mcelog plugin you need to modify the configuration file to include: @@ -618,7 +698,7 @@ include: <LoadPlugin mcelog> Interval 1 </LoadPlugin> - <Plugin "mcelog"> + <Plugin mcelog> McelogClientSocket "/var/run/mcelog-client" </Plugin> @@ -652,7 +732,7 @@ Then you can run the mcelog test suite with This will inject different classes of errors and check that the mcelog triggers runs. There will be some kernel messages about page offlining attempts. The -test will also lose a few pages of memory in your system (not significant) +test will also lose a few pages of memory in your system (not significant). .. note:: This test will kill any running mcelog, which needs to be restarted manually afterwards. @@ -685,7 +765,7 @@ Inject the error: The uncorrected and fatal scripts under test will cause a platform reset. Only the fatal script generates the memory errors**. In order to quickly emulate uncorrected memory errors and avoid host reboot following test errors - from mce-test suite can be injected: + from mce-test suite can be injected: .. code:: bash @@ -693,7 +773,7 @@ Inject the error: **mce-test:** -In addition an more in-depth test of the Linux kernel machine check facilities +In addition a more in-depth test of the Linux kernel machine check facilities can be done with the mce-test test suite. mce-test supports testing uncorrected error handling, real error injection, handling of different soft offlining cases, and other tests. @@ -728,11 +808,14 @@ IF-MIB (http://www.net-snmp.org/docs/mibs/IF-MIB.txt) Dependencies: Open vSwitch, Yet Another JSON Library (https://github.com/lloyd/yajl) -On Ubuntu, install the dependencies: +On Centos, install the dependencies and Open vSwitch: .. code:: bash - $ sudo apt-get install libyajl-dev openvswitch-switch + $ sudo yum install yajl-devel + +Steps to install Open vSwtich can be found at +http://docs.openvswitch.org/en/latest/intro/install/fedora/ Start the Open vSwitch service: @@ -740,7 +823,7 @@ Start the Open vSwitch service: $ sudo service openvswitch-switch start -configure the ovsdb-server manager: +Configure the ovsdb-server manager: .. code:: bash @@ -758,21 +841,21 @@ Clone and install the collectd ovs plugin: $ make $ sudo make install -This will install collectd to /opt/collectd. The collectd configuration file -can be found at /opt/collectd/etc. To configure the OVS events plugin you -need to modify the configuration file to include: +This will install collectd to default folder ``/opt/collectd``. The collectd +configuration file (``collectd.conf``) can be found at ``/opt/collectd/etc``. +To configure the OVS events plugin you need to modify the configuration file to include: .. code:: bash <LoadPlugin ovs_events> Interval 1 </LoadPlugin> - <Plugin "ovs_events"> - Port 6640 + <Plugin ovs_events> + Port "6640" + Address "127.0.0.1" Socket "/var/run/openvswitch/db.sock" Interfaces "br0" "veth0" - SendNotification false - DispatchValues true + SendNotification true </Plugin> To configure the OVS stats plugin you need to modify the configuration file @@ -787,7 +870,7 @@ to include: Port "6640" Address "127.0.0.1" Socket "/var/run/openvswitch/db.sock" - Bridges "br0" "br_ext" + Bridges "br0" </Plugin> For more information on the plugin parameters, please see: @@ -815,7 +898,6 @@ to include: </LoadPlugin <Plugin exec> Exec "user:group" "<path to ovs_pmd_stat.sh>" - #NotificationExec "nobody" "/usr/lib/collectd/notify.sh" </Plugin> .. note:: Exec plugin configuration has to be changed to use appropriate user before starting collectd service. @@ -831,28 +913,22 @@ SNMP Agent Plugin ^^^^^^^^^^^^^^^^^ Repo: https://github.com/maryamtahhan/collectd/ -Branch: feat_snmp +Branch: master Dependencies: NET-SNMP library Start by installing net-snmp and dependencies. -On ubuntu: +On Centos 7: .. code:: bash - $ apt-get install snmp snmp-mibs-downloader snmpd libsnmp-dev + $ sudo yum install net-snmp net-snmp-libs net-snmp-utils net-snmp-devel $ systemctl start snmpd.service Or build from source -Become root to install net-snmp dependencies - -.. code:: bash - - $ apt-get install libperl-dev - -Clone and build net-snmp +Clone and build net-snmp: .. code:: bash @@ -867,13 +943,13 @@ Become root $ make install -Copy default configuration to persistent folder +Copy default configuration to persistent folder: .. code:: bash $ cp EXAMPLE.conf /usr/share/snmp/snmpd.conf -Set library path and default MIB configuration +Set library path and default MIB configuration: .. code:: bash @@ -882,7 +958,7 @@ Set library path and default MIB configuration $ net-snmp-config --default-mibdirs $ net-snmp-config --snmpconfpath -Configure snmpd as a service +Configure snmpd as a service: .. code:: bash @@ -917,8 +993,9 @@ Clone and install the collectd snmp_agent plugin: $ make $ sudo make install -This will install collectd to /opt/collectd -The collectd configuration file can be found at /opt/collectd/etc +This will install collectd to default folder ``/opt/collectd``. The collectd +configuration file (``collectd.conf``) can be found at ``/opt/collectd/etc``. + **SNMP Agent plugin is a generic plugin and cannot work without configuration**. To configure the snmp_agent plugin you need to modify the configuration file to include OIDs mapped to collectd types. The following example maps scalar @@ -954,15 +1031,15 @@ virt plugin ^^^^^^^^^^^^ Repo: https://github.com/maryamtahhan/collectd -Branch: feat_libvirt_upstream +Branch: master Dependencies: libvirt (https://libvirt.org/), libxml2 -On Ubuntu, install the dependencies: +On Centos, install the dependencies: .. code:: bash - $ sudo apt-get install libxml2-dev + $ sudo yum install libxml2-dev libpciaccess-devel yajl-devel device-mapper-devel Install libvirt: @@ -1056,24 +1133,23 @@ Clone and install the collectd virt plugin: $ git clone $REPO $ cd collectd - $ git checkout $BRANCH $ ./build.sh $ ./configure --enable-syslog --enable-logfile --enable-debug $ make $ sudo make install -Where ``$REPO`` and ``$BRANCH`` are equal to information provided above. +Where ``$REPO`` is equal to information provided above. This will install collectd to ``/opt/collectd``. The collectd configuration file -``collectd.conf`` can be found at ``/opt/collectd/etc``. To load the virt plugin -user needs to modify the configuration file to include: +``collectd.conf`` can be found at ``/opt/collectd/etc``. +To load the virt plugin user needs to modify the configuration file to include: .. code:: bash LoadPlugin virt Additionally, user can specify plugin configuration parameters in this file, -such as connection URI, domain name and much more. By default extended virt plugin +such as connection URL, domain name and much more. By default extended virt plugin statistics are disabled. They can be enabled with ``ExtraStats`` option. .. code:: bash @@ -1182,7 +1258,7 @@ Monitoring Interfaces and Openstack Support Monitoring Interfaces and Openstack Support The figure above shows the DPDK L2 forwarding application running on a compute -node, sending and receiving traffic. collectd is also running on this compute +node, sending and receiving traffic. Collectd is also running on this compute node retrieving the stats periodically from DPDK through the dpdkstat plugin and publishing the retrieved stats to OpenStack through the collectd-ceilometer-plugin. |