diff options
Diffstat (limited to 'baro_tests/collectd.py')
| -rw-r--r-- | baro_tests/collectd.py | 519 |
1 files changed, 510 insertions, 9 deletions
diff --git a/baro_tests/collectd.py b/baro_tests/collectd.py index 5631cf55..3f2067af 100644 --- a/baro_tests/collectd.py +++ b/baro_tests/collectd.py @@ -1,15 +1,516 @@ -#!/usr/bin/python +"""Executing test of plugins""" +# -*- coding: utf-8 -*- -import sys +#Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import requests +from keystoneclient.v3 import client +import os +import time +import logging +from config_server import * +from tests import * -def main(logger): - logger.info("Running Baromtercollectd test suite...") - # - # TODO: implementation - # - logger.info("Test suite successfully completed.") +CEILOMETER_NAME = 'ceilometer' + + +class KeystoneException(Exception): + """Keystone exception class""" + def __init__(self, message, exc=None, response=None): + """ + Keyword arguments: + message -- error message + exc -- exception + response -- response + """ + if exc: + message += "\nReason: %s" % exc + super(KeystoneException, self).__init__(message) + + self.response = response + self.exception = exc + + +class InvalidResponse(KeystoneException): + """Invalid Keystone exception class""" + def __init__(self, exc, response): + """ + Keyword arguments: + exc -- exception + response -- response + """ + super(InvalidResponse, self).__init__( + "Invalid response", exc, response) + + +class CeilometerClient(object): + """Ceilometer Client to authenticate and request meters""" + def __init__(self, bc_logger): + """ + Keyword arguments: + bc_logger - logger instance + """ + self._auth_token = None + self._ceilometer_url = None + self._meter_list = None + self._logger = bc_logger + + def auth_token(self): + """Get auth token""" + self._auth_server() + return self._auth_token + + def get_ceilometer_url(self): + """Get Ceilometer URL""" + return self._ceilometer_url + + def get_ceil_metrics(self, criteria=None): + """Get Ceilometer metrics for given criteria + + Keyword arguments: + criteria -- criteria for ceilometer meter list + """ + self._request_meters(criteria) + return self._meter_list + + def _auth_server(self): + """Request token in authentication server""" + self._logger.debug('Connecting to the auth server {}'.format(os.environ['OS_AUTH_URL'])) + keystone = client.Client(username=os.environ['OS_USERNAME'], + password=os.environ['OS_PASSWORD'], + tenant_name=os.environ['OS_TENANT_NAME'], + auth_url=os.environ['OS_AUTH_URL']) + self._auth_token = keystone.auth_token + for service in keystone.service_catalog.get_data(): + if service['name'] == CEILOMETER_NAME: + for service_type in service['endpoints']: + if service_type['interface'] == 'internal': + self._ceilometer_url = service_type['url'] + break + + if self._ceilometer_url is None: + self._logger.warning('Ceilometer is not registered in service catalog') + + def _request_meters(self, criteria): + """Request meter list values from ceilometer + + Keyword arguments: + criteria -- criteria for ceilometer meter list + """ + if criteria is None: + url = self._ceilometer_url + ('/v2/samples?limit=400') + else: + url = self._ceilometer_url + ('/v2/meters/%s?q.field=resource_id&limit=400' % criteria) + headers = {'X-Auth-Token': self._auth_token} + resp = requests.get(url, headers=headers) + try: + resp.raise_for_status() + self._meter_list = resp.json() + except (KeyError, ValueError, requests.exceptions.HTTPError) as err: + raise InvalidResponse(err, resp) + + +class CSVClient(object): + """Client to request CSV meters""" + def __init__(self, bc_logger, conf): + """ + Keyword arguments: + bc_logger - logger instance + conf -- ConfigServer instance + """ + self._logger = bc_logger + self.conf = conf + + def get_csv_metrics(self, compute_node, plugin_subdirectories, meter_categories): + """Get CSV metrics. + + Keyword arguments: + compute_node -- compute node instance + plugin_subdirectories -- list of subdirectories of plug-in + meter_categories -- categories which will be tested + + Return list of metrics. + """ + stdout = self.conf.execute_command("date '+%Y-%m-%d'", compute_node.get_ip()) + date = stdout[0].strip() + metrics = [] + for plugin_subdir in plugin_subdirectories: + for meter_category in meter_categories: + stdout = self.conf.execute_command( + "tail -2 /var/lib/collectd/csv/node-" + + "{0}.domain.tld/{1}/{2}-{3}".format( + compute_node.get_id(), plugin_subdir, meter_category, date), + compute_node.get_ip()) + #Storing last two values + values = stdout + if len(values) < 2: + self._logger.error( + 'Getting last two CSV entries of meter category ' + + '{0} in {1} subdir failed'.format(meter_category, plugin_subdir)) + else: + old_value = int(values[0][0:values[0].index('.')]) + new_value = int(values[1][0:values[1].index('.')]) + metrics.append((plugin_subdir, meter_category, old_value, new_value)) + return metrics + + +def _check_logger(): + """Check whether there is global logger available and if not, define one.""" + if 'logger' not in globals(): + global logger + logger = logger.Logger("barometercollectd").getLogger() + + +def _process_result(compute_node, test, result, results_list): + """Print test result and append it to results list. + + Keyword arguments: + test -- testcase name + result -- boolean test result + results_list -- results list + """ + if result: + logger.info('Compute node {0} test case {1} PASSED.'.format(compute_node, test)) + else: + logger.error('Compute node {0} test case {1} FAILED.'.format(compute_node, test)) + results_list.append((compute_node, test, result)) + + +def _print_label(label): + """Print label on the screen + + Keyword arguments: + label -- label string + """ + label = label.strip() + length = 70 + if label != '': + label = ' ' + label + ' ' + length_label = len(label) + length1 = (length - length_label) / 2 + length2 = length - length_label - length1 + length1 = max(3, length1) + length2 = max(3, length2) + logger.info(('=' * length1) + label + ('=' * length2)) + + +def _print_plugin_label(plugin, node_id): + """Print plug-in label. + + Keyword arguments: + plugin -- plug-in name + node_id -- node ID + """ + _print_label('Node {0}: Plug-in {1} Test case execution'.format(node_id, plugin)) + + +def _print_final_result_of_plugin(plugin, compute_ids, results, out_plugins, out_plugin): + """Print final results of plug-in. + + Keyword arguments: + plugin -- plug-in name + compute_ids -- list of compute node IDs + results -- results list + out_plugins -- list of out plug-ins + out_plugin -- used out plug-in + """ + print_line = '' + for id in compute_ids: + if out_plugins[id] == out_plugin: + if (id, plugin, True) in results: + print_line += ' PASS |' + elif (id, plugin, False) in results and out_plugins[id] == out_plugin: + print_line += ' FAIL |' + else: + print_line += ' NOT EX |' + elif out_plugin == 'Ceilometer': + print_line += ' NOT EX |' + else: + print_line += ' SKIP |' + return print_line + + +def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): + """Print overall summary table. + + Keyword arguments: + compute_ids -- list of compute IDs + tested_plugins -- list of plug-ins + results -- results list + out_plugins -- list of used out plug-ins + """ + compute_node_names = ['Node-{}'.format(id) for id in compute_ids] + all_computes_in_line = '' + for compute in compute_node_names: + all_computes_in_line = all_computes_in_line + '| ' + compute + (' ' * (7 - len(compute))) + line_of_nodes = '| Test ' + all_computes_in_line + '|' + logger.info('=' * 70) + logger.info('+' + ('-' * ((9 * len(compute_node_names))+16)) + '+') + logger.info( + '|' + ' ' * ((9*len(compute_node_names))/2) + ' OVERALL SUMMARY' + + ' ' * (9*len(compute_node_names) - (9*len(compute_node_names))/2) + '|') + logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) + logger.info(line_of_nodes) + logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) + out_plugins_print = ['Ceilometer'] + if 'CSV' in out_plugins.values(): + out_plugins_print.append('CSV') + for out_plugin in out_plugins_print: + output_plugins_line = '' + for id in compute_ids: + out_plugin_result = '----' + if out_plugin == 'Ceilometer': + out_plugin_result = 'PASS' if out_plugins[id] == out_plugin else 'FAIL' + if out_plugin == 'CSV': + if out_plugins[id] == out_plugin: + out_plugin_result = \ + 'PASS' if [ + plugin for comp_id, plugin, + res in results if comp_id == id and res] else 'FAIL' + else: + out_plugin_result = 'SKIP' + output_plugins_line += '| ' + out_plugin_result + ' ' + logger.info( + '| OUT:{}'.format(out_plugin) + (' ' * (11 - len(out_plugin))) + + output_plugins_line + '|') + for plugin in sorted(tested_plugins.values()): + line_plugin = _print_final_result_of_plugin( + plugin, compute_ids, results, out_plugins, out_plugin) + logger.info('| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) + '|' + line_plugin) + logger.info('+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) + logger.info('=' * 70) + + +def _exec_testcase( + test_labels, name, ceilometer_running, compute_node, + conf, results, error_plugins): + """Execute the testcase. + + Keyword arguments: + test_labels -- dictionary of plug-in IDs and their display names + name -- plug-in ID, key of test_labels dictionary + ceilometer_running -- boolean indicating whether Ceilometer is running + compute_node -- compute node ID + conf -- ConfigServer instance + results -- results list + error_plugins -- list of tuples with plug-in errors (plugin, error_description, is_critical): + plugin -- plug-in ID, key of test_labels dictionary + error_decription -- description of the error + is_critical -- boolean value indicating whether error is critical + """ + ovs_interfaces = conf.get_ovs_interfaces(compute_node) + ovs_configured_interfaces = conf.get_plugin_config_values( + compute_node, 'ovs_events', 'Interfaces') + ovs_existing_configured_int = [ + interface for interface in ovs_interfaces + if interface in ovs_configured_interfaces] + plugin_prerequisites = { + 'mcelog': [(conf.is_installed(compute_node, 'mcelog'), 'mcelog must be installed.')], + 'ovs_events': [( + len(ovs_existing_configured_int) > 0 or len(ovs_interfaces) > 0, + 'Interfaces must be configured.')]} + ceilometer_criteria_lists = { + 'hugepages': ['hugepages.vmpage_number'], + 'mcelog': ['mcelog.errors'], + 'ovs_events': ['ovs_events.gauge']} + ceilometer_substr_lists = { + 'ovs_events': ovs_existing_configured_int if len(ovs_existing_configured_int) > 0 else ovs_interfaces} + csv_subdirs = { + 'hugepages': [ + 'hugepages-mm-2048Kb', 'hugepages-node0-2048Kb', 'hugepages-node1-2048Kb', + 'hugepages-mm-1048576Kb', 'hugepages-node0-1048576Kb', 'hugepages-node1-1048576Kb'], + 'mcelog': ['mcelog-SOCKET_0_CHANNEL_0_DIMM_any', 'mcelog-SOCKET_0_CHANNEL_any_DIMM_any'], + 'ovs_events': [ + 'ovs_events-{}'.format(interface) + for interface in (ovs_existing_configured_int if len(ovs_existing_configured_int) > 0 else ovs_interfaces)]} + csv_meter_categories = { + 'hugepages': ['vmpage_number-free', 'vmpage_number-used'], + 'mcelog': [ + 'errors-corrected_memory_errors', 'errors-uncorrected_memory_errors', + 'errors-corrected_memory_errors_in_24h', 'errors-uncorrected_memory_errors_in_24h'], + 'ovs_events': ['gauge-link_status']} + + _print_plugin_label(test_labels[name] if name in test_labels else name, compute_node.get_id()) + plugin_critical_errors = [ + error for plugin, error, critical in error_plugins if plugin == name and critical] + if plugin_critical_errors: + logger.error('Following critical errors occurred:'.format(name)) + for error in plugin_critical_errors: + logger.error(' * ' + error) + _process_result(compute_node.get_id(), test_labels[name], False, results) + else: + plugin_errors = [ + error for plugin, error, critical in error_plugins if plugin == name and not critical] + if plugin_errors: + logger.warning('Following non-critical errors occured:') + for error in plugin_errors: + logger.warning(' * ' + error) + failed_prerequisites = [] + if name in plugin_prerequisites: + failed_prerequisites = [ + prerequisite_name for prerequisite_passed, + prerequisite_name in plugin_prerequisites[name] if not prerequisite_passed] + if failed_prerequisites: + logger.error( + '{} test will not be executed, '.format(name) + + 'following prerequisites failed:') + for prerequisite in failed_prerequisites: + logger.error(' * {}'.format(prerequisite)) + else: + if ceilometer_running: + res = test_ceilometer_node_sends_data( + compute_node.get_id(), conf.get_plugin_interval(compute_node, name), + logger=logger, client=CeilometerClient(logger), + criteria_list=ceilometer_criteria_lists[name], + resource_id_substrings = (ceilometer_substr_lists[name] + if name in ceilometer_substr_lists else [''])) + else: + res = test_csv_handles_plugin_data( + compute_node, conf.get_plugin_interval(compute_node, name), name, + csv_subdirs[name], csv_meter_categories[name], logger, + CSVClient(logger, conf)) + if res and plugin_errors: + logger.info( + 'Test works, but will be reported as failure,' + + 'because of non-critical errors.') + res = False + _process_result(compute_node.get_id(), test_labels[name], res, results) + + +def main(bt_logger=None): + """Check each compute node sends ceilometer metrics. + + Keyword arguments: + bt_logger -- logger instance + """ + logging.getLogger("paramiko").setLevel(logging.WARNING) + logging.getLogger("stevedore").setLevel(logging.WARNING) + if bt_logger is None: + _check_logger() + else: + global logger + logger = bt_logger + conf = ConfigServer('10.20.0.2', 'root', logger) + controllers = conf.get_controllers() + if len(controllers) == 0: + logger.error('No controller nodes found!') + return 1 + computes = conf.get_computes() + if len(computes) == 0: + logger.error('No compute nodes found!') + return 1 + + _print_label('Display of Control and Compute nodes available in the set up') + logger.info('controllers: {}'.format([('{0}: {1} ({2})'.format( + node.get_id(), node.get_name(), node.get_ip())) for node in controllers])) + logger.info('computes: {}'.format([('{0}: {1} ({2})'.format( + node.get_id(), node.get_name(), node.get_ip())) for node in computes])) + + ceilometer_running_on_con = False + _print_label('Test Ceilometer on control nodes') + for controller in controllers: + ceil_client = CeilometerClient(logger) + ceil_client.auth_token() + ceilometer_running_on_con = ( + ceilometer_running_on_con or conf.is_ceilometer_running(controller)) + if ceilometer_running_on_con: + logger.info("Ceilometer is running on control node.") + else: + logger.error("Ceilometer is not running on control node.") + logger.info("CSV will be enabled on compute nodes.") + compute_ids = [] + results = [] + plugin_labels = { + 'hugepages': 'Hugepages', + 'mcelog': 'Mcelog', + 'ovs_events': 'OVS events'} + out_plugins = {} + for compute_node in computes: + node_id = compute_node.get_id() + out_plugins[node_id] = 'CSV' + compute_ids.append(node_id) + #plugins_to_enable = plugin_labels.keys() + plugins_to_enable = [] + _print_label('NODE {}: Test Ceilometer Plug-in'.format(node_id)) + logger.info('Checking if ceilometer plug-in is included.') + if not conf.check_ceil_plugin_included(compute_node): + logger.error('Ceilometer plug-in is not included.') + logger.info('Testcases on node {} will not be executed'.format(node_id)) + else: + collectd_restarted, collectd_warnings = conf.restart_collectd(compute_node) + sleep_time = 30 + logger.info('Sleeping for {} seconds after collectd restart...'.format(sleep_time)) + time.sleep(sleep_time) + if not collectd_restarted: + for warning in collectd_warnings: + logger.warning(warning) + logger.error('Restart of collectd on node {} failed'.format(node_id)) + logger.info('Testcases on node {} will not be executed'.format(node_id)) + else: + for warning in collectd_warnings: + logger.warning(warning) + ceilometer_running = ( + ceilometer_running_on_con and test_ceilometer_node_sends_data( + node_id, 10, logger=logger, client=CeilometerClient(logger))) + if ceilometer_running: + out_plugins[node_id] = 'Ceilometer' + logger.info("Ceilometer is running.") + else: + plugins_to_enable.append('csv') + out_plugins[node_id] = 'CSV' + logger.error("Ceilometer is not running.") + logger.info("CSV will be enabled for verification of test plugins.") + if plugins_to_enable: + _print_label( + 'NODE {}: Enabling Test Plug-in '.format(node_id) + + 'and Test case execution') + error_plugins = [] + if plugins_to_enable and not conf.enable_plugins( + compute_node, plugins_to_enable, error_plugins, create_backup=False): + logger.error('Failed to test plugins on node {}.'.format(node_id)) + logger.info('Testcases on node {} will not be executed'.format(node_id)) + else: + if plugins_to_enable: + collectd_restarted, collectd_warnings = conf.restart_collectd(compute_node) + sleep_time = 30 + logger.info( + 'Sleeping for {} seconds after collectd restart...'.format(sleep_time)) + time.sleep(sleep_time) + if plugins_to_enable and not collectd_restarted: + for warning in collectd_warnings: + logger.warning(warning) + logger.error('Restart of collectd on node {} failed'.format(node_id)) + logger.info('Testcases on node {} will not be executed'.format(node_id)) + else: + if collectd_warnings: + for warning in collectd_warnings: + logger.warning(warning) + + for plugin_name in sorted(plugin_labels.keys()): + _exec_testcase( + plugin_labels, plugin_name, ceilometer_running, + compute_node, conf, results, error_plugins) + + _print_label('NODE {}: Restoring config file'.format(node_id)) + conf.restore_config(compute_node) + + print_overall_summary(compute_ids, plugin_labels, results, out_plugins) + + if ((len([res for res in results if not res[2]]) > 0) + or (len(results) < len(computes) * len(plugin_labels))): + logger.error('Some tests have failed or have not been executed') + return 1 return 0 if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) |