diff options
| -rw-r--r-- | baro_tests/collectd.py | 370 | ||||
| -rw-r--r-- | baro_tests/config_server.py | 314 |
2 files changed, 321 insertions, 363 deletions
diff --git a/baro_tests/collectd.py b/baro_tests/collectd.py index a002314e..cd492d83 100644 --- a/baro_tests/collectd.py +++ b/baro_tests/collectd.py @@ -224,13 +224,15 @@ class CSVClient(object): if compute_name == node.get_dict()['name']: date = node.run_cmd( "date '+%Y-%m-%d'") + hostname = node.run_cmd('hostname -A') + hostname = hostname.split()[0] metrics = [] for plugin_subdir in plugin_subdirectories: for meter_category in meter_categories: stdout1 = node.run_cmd( "tail -2 /var/lib/collectd/csv/" - + "{0}.jf.intel.com/{1}/{2}-{3}".format( - compute_node.get_name(), plugin_subdir, + + "{0}/{1}/{2}-{3}".format( + hostname, plugin_subdir, meter_category, date)) stdout2 = node.run_cmd( "tail -1 /var/lib/collectd/csv/" @@ -272,7 +274,7 @@ def get_csv_categories_for_ipmi(conf, compute_node): return [category.strip()[:-11] for category in categories] -def _process_result(compute_node, test, result, results_list): +def _process_result(compute_node, out_plugin, test, result, results_list): """Print test result and append it to results list. Keyword arguments: @@ -282,13 +284,13 @@ def _process_result(compute_node, test, result, results_list): """ if result: logger.info( - 'Compute node {0} test case {1} PASSED.'.format( - compute_node, test)) + 'Test case {0} PASSED with {1}.'.format( + test, out_plugin)) else: logger.error( - 'Compute node {0} test case {1} FAILED.'.format( - compute_node, test)) - results_list.append((compute_node, test, result)) + 'Test case {0} FAILED with {1}.'.format( + test, out_plugin)) + results_list.append((compute_node, out_plugin, test, result)) def _print_label(label): @@ -333,22 +335,34 @@ def _print_final_result_of_plugin( """ print_line = '' for id in compute_ids: - if out_plugins[id] == out_plugin: - if (id, plugin, True) in results: + if out_plugin == 'Gnocchi': + if (id, out_plugin, plugin, True) in results: + print_line += ' PASS |' + elif (id, out_plugin, plugin, False) in results: + print_line += ' FAIL |' + else: + print_line += ' NOT EX |' + elif out_plugin == 'AODH': + if (id, out_plugin, plugin, True) in results: + print_line += ' PASS |' + elif (id, out_plugin, plugin, False) in results: + print_line += ' FAIL |' + else: + print_line += ' NOT EX |' + elif out_plugin == 'CSV': + if (id, out_plugin, plugin, True) in results: print_line += ' PASS |' - elif (id, plugin, False) in results \ - and out_plugins[id] == out_plugin: + elif (id, out_plugin, plugin, False) in results: print_line += ' FAIL |' else: print_line += ' NOT EX |' - elif out_plugin == 'Gnocchi': - print_line += ' NOT EX |' else: - print_line += ' NOT EX |' + print_line += ' SKIP |' return print_line -def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): +def print_overall_summary( + compute_ids, tested_plugins, aodh_plugins, results, out_plugins): """Print overall summary table. Keyword arguments: @@ -359,7 +373,6 @@ def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): """ compute_node_names = ['Node-{}'.format(i) for i in range( len((compute_ids)))] - # compute_node_names = ['Node-{}'.format(id) for id in compute_ids] all_computes_in_line = '' for compute in compute_node_names: all_computes_in_line += '| ' + compute + (' ' * (7 - len(compute))) @@ -377,46 +390,61 @@ def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): logger.info(line_of_nodes) logger.info( '+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) - out_plugins_print = ['Gnocchi'] - if 'SNMP' in out_plugins.values(): - out_plugins_print.append('SNMP') - if 'AODH' in out_plugins.values(): - out_plugins_print.append('AODH') - if 'CSV' in out_plugins.values(): - out_plugins_print.append('CSV') + out_plugins_print = [] + out_plugins_print1 = [] + for key in out_plugins.keys(): + if 'Gnocchi' in out_plugins[key]: + out_plugins_print1.append('Gnocchi') + if 'AODH' in out_plugins[key]: + out_plugins_print1.append('AODH') + if 'SNMP' in out_plugins[key]: + out_plugins_print1.append('SNMP') + if 'CSV' in out_plugins[key]: + out_plugins_print1.append('CSV') + for i in out_plugins_print1: + if i not in out_plugins_print: + out_plugins_print.append(i) for out_plugin in out_plugins_print: output_plugins_line = '' for id in compute_ids: - out_plugin_result = 'FAIL' + out_plugin_result = '----' if out_plugin == 'Gnocchi': out_plugin_result = \ - 'PASS' if out_plugins[id] == out_plugin else 'FAIL' + 'PASS' if 'Gnocchi' in out_plugins_print else 'FAIL' if out_plugin == 'AODH': - if out_plugins[id] == out_plugin: - out_plugin_result = \ - 'PASS' if out_plugins[id] == out_plugin else 'FAIL' + out_plugin_result = \ + 'PASS' if out_plugin in out_plugins_print else 'FAIL' if out_plugin == 'SNMP': - if out_plugins[id] == out_plugin: - out_plugin_result = \ - 'PASS' if out_plugins[id] == out_plugin else 'FAIL' + out_plugin_result = \ + 'PASS' if [ + plugin for comp_id, out_pl, plugin, res in results + if comp_id == id and res] else 'FAIL' if out_plugin == 'CSV': - if out_plugins[id] == out_plugin: - out_plugin_result = \ - 'PASS' if [ - plugin for comp_id, plugin, res in results - if comp_id == id and res] else 'FAIL' - else: - out_plugin_result = 'SKIP' + out_plugin_result = \ + 'PASS' if [ + plugin for comp_id, out_pl, plugin, res in results + if comp_id == id and res] else 'FAIL' + else: + out_plugin_result = 'FAIL' output_plugins_line += '| ' + out_plugin_result + ' ' logger.info( '| OUT:{}'.format(out_plugin) + (' ' * (11 - len(out_plugin))) + output_plugins_line + '|') - for plugin in sorted(tested_plugins.values()): - line_plugin = _print_final_result_of_plugin( - plugin, compute_ids, results, out_plugins, out_plugin) - logger.info( - '| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) - + '|' + line_plugin) + + if out_plugin == 'AODH': + for plugin in sorted(aodh_plugins.values()): + line_plugin = _print_final_result_of_plugin( + plugin, compute_ids, results, out_plugins, out_plugin) + logger.info( + '| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) + + '|' + line_plugin) + else: + for plugin in sorted(tested_plugins.values()): + line_plugin = _print_final_result_of_plugin( + plugin, compute_ids, results, out_plugins, out_plugin) + logger.info( + '| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) + + '|' + line_plugin) logger.info( '+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) @@ -424,8 +452,8 @@ def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): def _exec_testcase( - test_labels, name, gnocchi_running, aodh_running, snmp_running, - controllers, compute_node, conf, results, error_plugins, out_plugins): + test_labels, name, out_plugin, controllers, compute_node, + conf, results, error_plugins, out_plugins): """Execute the testcase. Keyword arguments: @@ -457,7 +485,7 @@ def _exec_testcase( conf.is_libpqos_on_node(compute_node), 'libpqos must be installed.')], 'mcelog': [( - conf.is_installed(compute_node, 'mcelog'), + conf.is_mcelog_installed(compute_node, 'mcelog'), 'mcelog must be installed.')], 'ovs_events': [( len(ovs_existing_configured_int) > 0 or len(ovs_interfaces) > 0, @@ -466,13 +494,13 @@ def _exec_testcase( len(ovs_existing_configured_bridges) > 0, 'Bridges must be configured.')]} gnocchi_criteria_lists = { - 'hugepages': ['hugepages'], - 'mcelog': ['mcelog'], - 'ovs_events': ['interface-ovs-system'], - 'ovs_stats': ['ovs_stats-br0.br0']} + 'hugepages': 'hugepages', + 'mcelog': 'mcelog', + 'ovs_events': 'interface-ovs-system', + 'ovs_stats': 'ovs_stats-br0.br0'} aodh_criteria_lists = { - 'mcelog': ['mcelog.errors'], - 'ovs_events': ['ovs_events.gauge']} + 'mcelog': 'mcelog', + 'ovs_events': 'ovs_events'} snmp_mib_files = { 'intel_rdt': '/usr/share/snmp/mibs/Intel-Rdt.txt', 'hugepages': '/usr/share/snmp/mibs/Intel-Hugepages.txt', @@ -501,8 +529,7 @@ def _exec_testcase( compute_node, 'intel_rdt', 'Cores')], 'hugepages': [ 'hugepages-mm-2048Kb', 'hugepages-node0-2048Kb', - 'hugepages-node1-2048Kb', 'hugepages-mm-1048576Kb', - 'hugepages-node0-1048576Kb', 'hugepages-node1-1048576Kb'], + 'hugepages-node1-2048Kb'], # 'ipmi': ['ipmi'], 'mcelog': [ 'mcelog-SOCKET_0_CHANNEL_0_DIMM_any', @@ -521,13 +548,9 @@ def _exec_testcase( # 'ipmi': csv_meter_categories_ipmi, 'mcelog': [ 'errors-corrected_memory_errors', - 'errors-uncorrected_memory_errors', - 'errors-corrected_memory_errors_in_24h', - 'errors-uncorrected_memory_errors_in_24h'], + 'errors-uncorrected_memory_errors'], 'ovs_stats': [ - 'if_collisions', 'if_dropped', 'if_errors', 'if_packets', - 'if_rx_errors-crc', 'if_rx_errors-frame', 'if_rx_errors-over', - 'if_rx_octets', 'if_tx_octets'], + 'if_dropped', 'if_errors', 'if_packets'], 'ovs_events': ['gauge-link_status']} _print_plugin_label( @@ -541,7 +564,8 @@ def _exec_testcase( for error in plugin_critical_errors: logger.error(' * ' + error) _process_result( - compute_node.get_id(), test_labels[name], False, results) + compute_node.get_id(), out_plugin, test_labels[name], False, + results) else: plugin_errors = [ error for plugin, error, critical in error_plugins @@ -563,35 +587,37 @@ def _exec_testcase( for prerequisite in failed_prerequisites: logger.error(' * {}'.format(prerequisite)) else: - if gnocchi_running: - plugin_interval = conf.get_plugin_interval(compute_node, name) + plugin_interval = conf.get_plugin_interval(compute_node, name) + if out_plugin == 'Gnocchi': res = conf.test_plugins_with_gnocchi( - compute_node.get_id(), plugin_interval, logger, - criteria_list=gnocchi_criteria_lists[name]) - elif aodh_running: + compute_node.get_name(), plugin_interval, + logger, criteria_list=gnocchi_criteria_lists[name]) + if out_plugin == 'AODH': res = conf.test_plugins_with_aodh( - compute_node.get_id(), plugin_interval, - logger, creteria_list=aodh_criteria_lists[name]) - elif snmp_running: + compute_node.get_name(), plugin_interval, + logger, criteria_list=aodh_criteria_lists[name]) + if out_plugin == 'SNMP': res = \ name in snmp_mib_files and name in snmp_mib_strings \ and tests.test_snmp_sends_data( compute_node, - conf.get_plugin_interval(compute_node, name), logger, + plugin_interval, logger, SNMPClient(conf, compute_node), snmp_mib_files[name], snmp_mib_strings[name], snmp_in_commands[name], conf) - else: + if out_plugin == 'CSV': res = tests.test_csv_handles_plugin_data( compute_node, conf.get_plugin_interval(compute_node, name), name, csv_subdirs[name], csv_meter_categories[name], logger, CSVClient(conf)) + if res and plugin_errors: logger.info( 'Test works, but will be reported as failure,' + 'because of non-critical errors.') res = False _process_result( - compute_node.get_id(), test_labels[name], res, results) + compute_node.get_id(), out_plugin, test_labels[name], + res, results) def get_results_for_ovs_events( @@ -618,7 +644,7 @@ def create_ovs_bridge(): if node.is_compute(): node.run_cmd('sudo ovs-vsctl add-br br0') node.run_cmd('sudo ovs-vsctl set-manager ptcp:6640') - logger.info('OVS Bridges created on compute nodes') + logger.info('OVS Bridges created on compute nodes') def mcelog_install(): @@ -635,18 +661,18 @@ def mcelog_install(): if '3.10.0-514.26.2.el7.x86_64' not in centos_release: logger.info( 'Mcelog will not be enabled ' - + 'on node-{0}, '.format(node.get_dict()['id']) + + 'on node-{0}, '.format(node.get_dict()['name']) + 'unsupported CentOS release found ({1}).'.format( centos_release)) else: logger.info( 'Checking if mcelog is enabled' - + ' on node-{}...'.format(node.get_dict()['id'])) + + ' on node-{}...'.format(node.get_dict()['name'])) res = node.run_cmd('ls') if 'mce-inject_ea' and 'corrected' in res: logger.info( 'Mcelog seems to be already installed ' - + 'on node-{}.'.format(node.get_dict()['id'])) + + 'on node-{}.'.format(node.get_dict()['name'])) node.run_cmd('sudo modprobe mce-inject') node.run_cmd('sudo ./mce-inject_ea < corrected') else: @@ -734,43 +760,23 @@ def main(bt_logger=None): _print_label( 'Display of Control and Compute nodes available in the set up') - logger.info('controllers: {}'.format([('{0}: {1} ({2})'.format( - node.get_id(), node.get_name(), - node.get_ip())) for node in controllers])) - logger.info('computes: {}'.format([('{0}: {1} ({2})'.format( - node.get_id(), node.get_name(), node.get_ip())) - for node in computes])) + logger.info('controllers: {}'.format([('{0}: {1}'.format( + node.get_name(), node.get_ip())) for node in controllers])) + logger.info('computes: {}'.format([('{0}: {1}'.format( + node.get_name(), node.get_ip())) for node in computes])) mcelog_install() create_ovs_bridge() gnocchi_running_on_con = False aodh_running_on_con = False snmp_running = False - _print_label('Testing Gnocchi, AODH and SNMP on controller nodes') + _print_label('Testing Gnocchi, AODH and SNMP on nodes') for controller in controllers: - gnocchi_client = GnocchiClient() - gnocchi_client.auth_token() gnocchi_running = ( gnocchi_running_on_con and conf.is_gnocchi_running(controller)) - aodh_client = AodhClient() - aodh_client.auth_token() aodh_running = ( - aodh_running_on_con and conf.is_aodh_running(controller)) - if gnocchi_running: - logger.info("Gnocchi is running on controller.") - elif aodh_running: - logger.error("Gnocchi is not running on controller.") - logger.info("AODH is running on controller.") - elif snmp_running: - logger.error("Gnocchi is not running on Controller") - logger.error("AODH is not running on controller.") - logger.info("SNMP is running on controller.") - else: - logger.error("Gnocchi is not running on Controller") - logger.error("AODH is not running on controller.") - logger.error("SNMP is not running on controller.") - logger.info("CSV will be enabled on compute nodes.") + aodh_running_on_con or conf.is_aodh_running(controller)) compute_ids = [] compute_node_names = [] @@ -782,113 +788,87 @@ def main(bt_logger=None): 'mcelog': 'Mcelog', 'ovs_stats': 'OVS stats', 'ovs_events': 'OVS events'} - out_plugins = { - 'gnocchi': 'Gnocchi', - 'aodh': 'AODH', - 'snmp': 'SNMP', - 'csv': 'CSV'} + aodh_plugin_labels = { + 'mcelog': 'Mcelog', + 'ovs_events': 'OVS events'} + out_plugins = {} + out_plugins_to_test = [] for compute_node in computes: node_id = compute_node.get_id() node_name = compute_node.get_name() - out_plugins[node_id] = 'CSV' + out_plugins[node_id] = [] compute_ids.append(node_id) compute_node_names.append(node_name) plugins_to_enable = [] - _print_label('NODE {}: Test Gnocchi Plug-in'.format(node_name)) - logger.info('Checking if gnocchi plug-in is included in compute nodes.') - if not conf.check_gnocchi_plugin_included(compute_node): - logger.error('Gnocchi plug-in is not included.') - logger.info( - 'Testcases on node {} will not be executed'.format(node_name)) - else: - collectd_restarted, collectd_warnings = \ - conf.restart_collectd(compute_node) - sleep_time = 30 - logger.info( - 'Sleeping for {} seconds after collectd restart...'.format( - sleep_time)) - time.sleep(sleep_time) - if not collectd_restarted: - for warning in collectd_warnings: - logger.warning(warning) + error_plugins = [] + gnocchi_running = ( + gnocchi_running or conf.check_gnocchi_plugin_included( + compute_node)) + aodh_running = ( + aodh_running and conf.check_aodh_plugin_included(compute_node)) + if gnocchi_running: + out_plugins[node_id].append("Gnocchi") + if aodh_running: + out_plugins[node_id].append("AODH") + if snmp_running: + out_plugins_to_test.append("SNMP") + + if 'gnocchi' not in out_plugins[node_id]: + logger.info("CSV will be enabled for verification") + plugins_to_enable.append('csv') + out_plugins[node_id].append("CSV") + if plugins_to_enable: + _print_label( + 'NODE {}: Enabling Test Plug-in '.format(node_name) + + 'and Test case execution') + if plugins_to_enable and not conf.enable_plugins( + compute_node, plugins_to_enable, error_plugins, + create_backup=False): logger.error( - 'Restart of collectd on node {} failed'.format(node_name)) + 'Failed to test plugins on node {}.'.format(node_id)) logger.info( 'Testcases on node {} will not be executed'.format( - node_name)) + node_id)) else: - for warning in collectd_warnings: - logger.warning(warning) - - if gnocchi_running: - out_plugins[node_id] = 'Gnocchi' - logger.info("Gnocchi is active and collecting data") - elif aodh_running: - out_plugins[node_id] = 'AODH' - logger.info("AODH withh be tested") - _print_label('Node {}: Test AODH' .format(node_name)) - logger.info("Checking if AODH is running") - logger.info("AODH is running") - elif snmp_running: - out_plugins[node_id] = 'SNMP' - logger.info("SNMP will be tested.") - _print_label('NODE {}: Test SNMP'.format(node_id)) - logger.info("Checking if SNMP is running.") - logger.info("SNMP is running.") - else: - plugins_to_enable.append('csv') - out_plugins[node_id] = 'CSV' - logger.error("Gnocchi, AODH, SNMP are not running") - logger.info( - "CSV will be enabled for verification " - + "of test plugins.") if plugins_to_enable: - _print_label( - 'NODE {}: Enabling Test Plug-in '.format(node_name) - + 'and Test case execution') - error_plugins = [] - if plugins_to_enable and not conf.enable_plugins( - compute_node, plugins_to_enable, error_plugins, - create_backup=False): - logger.error( - 'Failed to test plugins on node {}.'.format(node_id)) + collectd_restarted, collectd_warnings = \ + conf.restart_collectd(compute_node) + sleep_time = 10 logger.info( - 'Testcases on node {} will not be executed'.format( + 'Sleeping for {} seconds'.format(sleep_time) + + ' after collectd restart...') + time.sleep(sleep_time) + if plugins_to_enable and not collectd_restarted: + for warning in collectd_warnings: + logger.warning(warning) + logger.error( + 'Restart of collectd on node {} failed'.format( node_id)) + logger.info( + 'Testcases on node {}'.format(node_id) + + ' will not be executed.') else: - if plugins_to_enable: - collectd_restarted, collectd_warnings = \ - conf.restart_collectd(compute_node) - sleep_time = 30 - logger.info( - 'Sleeping for {} seconds'.format(sleep_time) - + ' after collectd restart...') - time.sleep(sleep_time) - if plugins_to_enable and not collectd_restarted: + if collectd_warnings: for warning in collectd_warnings: logger.warning(warning) - logger.error( - 'Restart of collectd on node {} failed'.format( - node_id)) - logger.info( - 'Testcases on node {}'.format(node_id) - + ' will not be executed.') - else: - if collectd_warnings: - for warning in collectd_warnings: - logger.warning(warning) - - for plugin_name in sorted(plugin_labels.keys()): - _exec_testcase( - plugin_labels, plugin_name, gnocchi_running, - aodh_running, snmp_running, controllers, - compute_node, conf, results, error_plugins, - out_plugins[node_id]) - - # _print_label('NODE {}: Restoring config file'.format(node_name)) - # conf.restore_config(compute_node) - mcelog_delete() - print_overall_summary(compute_ids, plugin_labels, results, out_plugins) + + for i in out_plugins[node_id]: + if i == 'AODH': + for plugin_name in sorted(aodh_plugin_labels.keys()): + _exec_testcase( + aodh_plugin_labels, plugin_name, i, + controllers, compute_node, conf, results, + error_plugins, out_plugins[node_id]) + else: + for plugin_name in sorted(plugin_labels.keys()): + _exec_testcase( + plugin_labels, plugin_name, i, + controllers, compute_node, conf, results, + error_plugins, out_plugins[node_id]) + + mcelog_delete() + print_overall_summary( + compute_ids, plugin_labels, aodh_plugin_labels, results, out_plugins) if ((len([res for res in results if not res[2]]) > 0) or (len(results) < len(computes) * len(plugin_labels))): diff --git a/baro_tests/config_server.py b/baro_tests/config_server.py index f156fcf7..0b16ea6e 100644 --- a/baro_tests/config_server.py +++ b/baro_tests/config_server.py @@ -257,28 +257,39 @@ class ConfigServer(object): Return boolean value whether Gnocchi is running. """ gnocchi_present = False - lines = self.execute_command( - 'source overcloudrc.v3;systemctl status openstack-gnocchi-api | ' - + 'grep running', controller.get_ip()) - for line in lines: - if '(running)' in line: - gnocchi_present = True + controller_name = controller.get_name() + nodes = get_apex_nodes() + for node in nodes: + if controller_name == node.get_dict()['name']: + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "openstack catalog list | grep gnocchi") + if 'gnocchi' in stdout: + gnocchi_present = True return gnocchi_present def is_aodh_running(self, controller): """Check whether aodh service is running on controller """ aodh_present = False - lines = self.execute_command( - 'source overcloudrc.v3;systemctl openstack-aodh-api | grep running', - controller.get_ip()) - for line in lines: - self.__logger.info("Line = {}" .format(line)) - if '(running)' in line: - aodh_present = True + controller_name = controller.get_name() + nodes = get_apex_nodes() + for node in nodes: + if controller_name == node.get_dict()['name']: + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "openstack catalog list | grep aodh") + if 'aodh' in stdout: + aodh_present = True return aodh_present - def is_installed(self, compute, package): + def is_mcelog_installed(self, compute, package): """Check whether package exists on compute node. Keyword arguments: @@ -310,6 +321,32 @@ class ConfigServer(object): return True return False + def check_aodh_plugin_included(self, compute): + """Check if aodh plugin is included in collectd.conf file. + If not, try to enable it. + + Keyword arguments: + compute -- compute node instance + + Return boolean value whether AODH plugin is included + or it's enabling was successful. + """ + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + aodh_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d') + if 'aodh.conf' not in aodh_conf: + self.__logger.info( + "AODH Plugin not included in compute node") + return False + else: + self.__logger.info( + "AODH plugin present in compute node {}" .format( + compute_name)) + return True + return True + def check_gnocchi_plugin_included(self, compute): """Check if gnocchi plugin is included in collectd.conf file. If not, try to enable it. @@ -324,13 +361,14 @@ class ConfigServer(object): nodes = get_apex_nodes() for node in nodes: if compute_name == node.get_dict()['name']: - # node.run_cmd('su; "opnfvapex"') gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d') if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf: self.__logger.info("Gnocchi Plugin not included") - return True + return False else: - self.__logger.info("Gnochi plugin present") + self.__logger.info( + "Gnochi plugin available in compute node {}" .format( + compute_name)) return True return True @@ -341,14 +379,6 @@ class ConfigServer(object): Keyword arguments: compute -- compute node instance plugins -- list of plugins to be enabled - error_plugins -- list of tuples with found errors, new entries - may be added there (plugin, error_description, is_critical): - plugin -- plug-in name - error_decription -- description of the error - is_critical -- boolean value indicating whether error - is critical - create_backup -- boolean value indicating whether backup - shall be created Return boolean value indicating whether function was successful. """ @@ -365,19 +395,6 @@ class ConfigServer(object): + '/etc/collectd/collectd.conf.d/csv.conf') return True - def restore_config(self, compute): - """Restore collectd config file from backup on compute node. - - Keyword arguments: - compute -- compute node instance - """ - ssh, sftp = self.__open_sftp_session( - compute.get_ip(), 'root', 'opnfvapex') - - self.__logger.info('Restoring config file from backup...') - ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF)) - ssh.exec_command("cp {0}.backup {0}".format(COLLECTD_CONF)) - def restart_collectd(self, compute): """Restart collectd on compute node. @@ -419,142 +436,103 @@ class ConfigServer(object): return False, warning return True, warning - def test_gnocchi_is_sending_data(self, controller): - """ Checking if Gnocchi is sending metrics to controller""" - metric_ids = [] - timestamps1 = {} - timestamps2 = {} - ssh, sftp = self.__open_sftp_session( - controller.get_ip(), 'root', 'opnfvapex') - - self.__logger.info('Getting gnocchi metric list on{}'.format( - controller.get_name())) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi metric list | grep if_packets", - ssh=ssh) - for line in stdout: - metric_ids = [r.split('|')[1] for r in stdout] - self.__logger.info("Metric ids = {}" .format(metric_ids)) - for metric_id in metric_ids: - metric_id = metric_id.replace("u", "") - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - self.__logger.info("stdout measures ={}" .format(stdout)) - for line in stdout: - if line[0] == '+': - pass - else: - self.__logger.info("Line = {}" .format(line)) - timestamps1 = [line.split('|')[1]] - self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) - time.sleep(10) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - for line in stdout: - if line[0] == '+': - pass - else: - timestamps2 = [line.split('|')[1]] - self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) - if timestamps1 == timestamps2: - self.__logger.info("False") - # return False - return True - else: - self.__logger.info("True") - return True + def test_plugins_with_aodh( + self, compute, plugin_interval, logger, + criteria_list=[]): - def test_plugins_with_aodh(self, controller): - """Checking if AODH is sending metrics to controller""" - metric_ids = [] + metric_id = {} timestamps1 = {} timestamps2 = {} - ssh, sftp = self.__open_sftp_session( - controller.get_ip(), 'root', 'opnfvapex') - self.__logger.info('Getting AODH alarm list on{}'.format( - controller.get_name())) - stdout = self.execute_command( - "source overcloudrc.v3;aodh alarm list | grep mcelog", - ssh=ssh) - for line in stdout: - metric_ids = [r.split('|')[1] for r in stdout] - self.__logger.info("Metric ids = {}" .format(metric_ids)) - for metric_id in metric_ids: - metric_id = metric_id.replace("u", "") - stdout = self.execute_command( - "source overcloudrc.v3;aodh alarm show {}" .format( - metric_id), ssh=ssh) - self.__logger.info("stdout alarms ={}" .format(stdout)) - for line in stdout: - if line[0] == '+': - pass - else: - self.__logger.info("Line = {}" .format(line)) - timestamps1 = [line.split('|')[1]] - self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) - time.sleep(10) - stdout = self.execute_command( - "source overcloudrc.v3;aodh alarm show {}" .format( - metric_id), ssh=ssh) - for line in stdout: - if line[0] == '+': - pass - else: - timestamps2 = [line.split('|')[1]] - self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) - if timestamps1 == timestamps2: - self.__logger.info("False") - # return False - return True - else: - self.__logger.info("True") - return True + nodes = get_apex_nodes() + for node in nodes: + if node.is_controller(): + self.__logger.info('Getting AODH Alarm list on {}' .format( + (node.get_dict()['name']))) + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "aodh alarm list | grep {0} | grep {1}" + .format(criteria_list, compute)) + for line in stdout.splitlines(): + line = line.replace('|', "") + metric_id = line.split()[0] + stdout = node.run_cmd( + 'source overcloudrc.v3; aodh alarm show {}' .format( + metric_id)) + for line in stdout.splitlines()[3: -1]: + line = line.replace('|', "") + if line.split()[0] == 'timestamp': + timestamps1 = line.split()[1] + self.__logger.info("timestamp_before = {}" .format( + timestamps1)) + else: + pass + time.sleep(12) + stdout = node.run_cmd( + "source overcloudrc.v3; aodh alarm show {}" .format( + metric_id)) + for line in stdout.splitlines()[3:-1]: + line = line.replace('|', "") + if line.split()[0] == 'timestamp': + timestamps2 = line.split()[1] + self.__logger.info("timestamp_after = {}" .format( + timestamps2)) + else: + pass + if timestamps1 == timestamps2: + self.__logger.info( + "Data not updated after interval of 12 seconds") + return False + else: + self.__logger.info("PASS") + return True def test_plugins_with_gnocchi( - self, controller, compute_node, plugin_interval, logger, + self, compute, plugin_interval, logger, criteria_list=[]): - metric_ids = [] + metric_id = {} timestamps1 = {} timestamps2 = {} - ssh, sftp = self.__open_sftp_session( - controller.get_ip(), 'root', 'opnfvapex') - self.__logger.info('Getting gnocchi metric list on{}'.format( - controller.get_name())) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi metric list | grep {0} | grep {1}" - .format(compute_node.get_name(), criteria_list), ssh=ssh) - for line in stdout: - metric_ids = [r.split('|')[1] for r in stdout] - self.__logger.info("Metric ids = {}" .format(metric_ids)) - for metric_id in metric_ids: - metric_id = metric_id.replace("u", "") - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - self.__logger.info("stdout measures ={}" .format(stdout)) - for line in stdout: - if line[0] == '+': - pass - else: - self.__logger.info("Line = {}" .format(line)) - timestamps1 = [line.split('|')[1]] - self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) - time.sleep(10) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - for line in stdout: - if line[0] == '+': - pass - else: - timestamps2 = [line.split('|')[1]] - self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) - if timestamps1 == timestamps2: - self.__logger.info("False") - return False - else: - self.__logger.info("True") - return True + nodes = get_apex_nodes() + for node in nodes: + if node.is_controller(): + self.__logger.info('Getting gnocchi metric list on {}' .format( + (node.get_dict()['name']))) + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "gnocchi metric list | grep {0} | grep {1}" + .format(criteria_list, compute)) + for line in stdout.splitlines(): + line = line.replace('|', "") + metric_id = line.split()[0] + stdout = node.run_cmd( + 'source overcloudrc.v3;gnocchi measures show {}'.format( + metric_id)) + for line in stdout.splitlines()[3: -1]: + if line[0] == '+': + pass + else: + timestamps1 = line.replace('|', "") + timestamps1 = timestamps1.split()[0] + time.sleep(10) + stdout = node.run_cmd( + "source overcloudrc.v3;gnocchi measures show {}".format( + metric_id)) + for line in stdout.splitlines()[3:-1]: + if line[0] == '+': + pass + else: + timestamps2 = line.replace('|', "") + timestamps2 = timestamps2.split()[0] + if timestamps1 == timestamps2: + self.__logger.info("Data not updated after 12 seconds") + return False + else: + self.__logger.info("PASS") + return True |