diff options
Diffstat (limited to 'baro_tests')
| -rw-r--r-- | baro_tests/collectd.py | 495 | ||||
| -rw-r--r-- | baro_tests/config_server.py | 445 |
2 files changed, 492 insertions, 448 deletions
diff --git a/baro_tests/collectd.py b/baro_tests/collectd.py index a002314e..304b87b8 100644 --- a/baro_tests/collectd.py +++ b/baro_tests/collectd.py @@ -11,6 +11,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +# Patch on October 10 2017 """Executing test of plugins""" @@ -22,7 +23,6 @@ import time import logging import config_server import tests -import subprocess from opnfv.deployment import factory AODH_NAME = 'aodh' @@ -30,7 +30,7 @@ GNOCCHI_NAME = 'gnocchi' ID_RSA_SRC = '/root/.ssh/id_rsa' ID_RSA_DST_DIR = '/root/.ssh' ID_RSA_DST = ID_RSA_DST_DIR + '/id_rsa' -APEX_IP = subprocess.check_output("echo $INSTALLER_IP", shell=True) +APEX_IP = os.getenv("INSTALLER_IP").rstrip('\n') APEX_USER = 'root' APEX_USER_STACK = 'stack' APEX_PKEY = '/root/.ssh/id_rsa' @@ -173,31 +173,6 @@ class AodhClient(object): logger.warning('Aodh is not registered in service catalog') -class SNMPClient(object): - """Client to request SNMP meters""" - def __init__(self, conf, compute_node): - """ - Keyword arguments: - conf -- ConfigServer instance - compute_node -- Compute node object - """ - self.conf = conf - self.compute_node = compute_node - - def get_snmp_metrics(self, compute_node, mib_file, mib_strings): - snmp_output = {} - if mib_file is None: - cmd = "snmpwalk -v 2c -c public localhost IF-MIB::interfaces" - ip = compute_node.get_ip() - snmp_output = self.conf.execute_command(cmd, ip) - else: - for mib_string in mib_strings: - snmp_output[mib_string] = self.conf.execute_command( - "snmpwalk -v2c -m {} -c public localhost {}".format( - mib_file, mib_string), compute_node.get_ip()) - return snmp_output - - class CSVClient(object): """Client to request CSV meters""" def __init__(self, conf): @@ -224,31 +199,39 @@ class CSVClient(object): if compute_name == node.get_dict()['name']: date = node.run_cmd( "date '+%Y-%m-%d'") + hostname = node.run_cmd('hostname -A') + hostname = hostname.split()[0] metrics = [] for plugin_subdir in plugin_subdirectories: for meter_category in meter_categories: stdout1 = node.run_cmd( "tail -2 /var/lib/collectd/csv/" - + "{0}.jf.intel.com/{1}/{2}-{3}".format( - compute_node.get_name(), plugin_subdir, + + "{0}/{1}/{2}-{3}".format( + hostname, plugin_subdir, meter_category, date)) stdout2 = node.run_cmd( "tail -1 /var/lib/collectd/csv/" - + "{0}.jf.intel.com/{1}/{2}-{3}".format( - compute_node.get_name(), plugin_subdir, + + "{0}/{1}/{2}-{3}".format( + hostname, plugin_subdir, meter_category, date)) - # Storing last two values + # Storing last two values values = stdout1 + values2 = stdout2 if values is None: logger.error( 'Getting last two CSV entries of meter category' + ' {0} in {1} subdir failed'.format( meter_category, plugin_subdir)) + elif values2 is None: + logger.error( + 'Getting last CSV entries of meter category' + + ' {0} in {1} subdir failed'.format( + meter_category, plugin_subdir)) else: values = values.split(',') old_value = float(values[0]) - stdout2 = stdout2.split(',') - new_value = float(stdout2[0]) + values2 = values2.split(',') + new_value = float(values2[0]) metrics.append(( plugin_subdir, meter_category, old_value, new_value)) @@ -272,7 +255,7 @@ def get_csv_categories_for_ipmi(conf, compute_node): return [category.strip()[:-11] for category in categories] -def _process_result(compute_node, test, result, results_list): +def _process_result(compute_node, out_plugin, test, result, results_list, node): """Print test result and append it to results list. Keyword arguments: @@ -282,13 +265,13 @@ def _process_result(compute_node, test, result, results_list): """ if result: logger.info( - 'Compute node {0} test case {1} PASSED.'.format( - compute_node, test)) + 'Test case for {0} with {1} PASSED on {2}.'.format( + node, out_plugin, test)) else: logger.error( - 'Compute node {0} test case {1} FAILED.'.format( - compute_node, test)) - results_list.append((compute_node, test, result)) + 'Test case for {0} with {1} FAILED on {2}.'.format( + node, out_plugin, test)) + results_list.append((compute_node, out_plugin, test, result)) def _print_label(label): @@ -333,22 +316,41 @@ def _print_final_result_of_plugin( """ print_line = '' for id in compute_ids: - if out_plugins[id] == out_plugin: - if (id, plugin, True) in results: + if out_plugin == 'Gnocchi': + if (id, out_plugin, plugin, True) in results: print_line += ' PASS |' - elif (id, plugin, False) in results \ - and out_plugins[id] == out_plugin: + elif (id, out_plugin, plugin, False) in results: + print_line += ' FAIL |' + else: + print_line += ' NOT EX |' + elif out_plugin == 'AODH': + if (id, out_plugin, plugin, True) in results: + print_line += ' PASS |' + elif (id, out_plugin, plugin, False) in results: + print_line += ' FAIL |' + else: + print_line += ' FAIL |' + elif out_plugin == 'SNMP': + if (id, out_plugin, plugin, True) in results: + print_line += ' PASS |' + elif (id, out_plugin, plugin, False) in results: + print_line += ' FAIL |' + else: + print_line += ' FAIL |' + elif out_plugin == 'CSV': + if (id, out_plugin, plugin, True) in results: + print_line += ' PASS |' + elif (id, out_plugin, plugin, False) in results: print_line += ' FAIL |' else: print_line += ' NOT EX |' - elif out_plugin == 'Gnocchi': - print_line += ' NOT EX |' else: - print_line += ' NOT EX |' + print_line += ' SKIP |' return print_line -def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): +def print_overall_summary( + compute_ids, tested_plugins, aodh_plugins, results, out_plugins): """Print overall summary table. Keyword arguments: @@ -359,7 +361,6 @@ def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): """ compute_node_names = ['Node-{}'.format(i) for i in range( len((compute_ids)))] - # compute_node_names = ['Node-{}'.format(id) for id in compute_ids] all_computes_in_line = '' for compute in compute_node_names: all_computes_in_line += '| ' + compute + (' ' * (7 - len(compute))) @@ -377,46 +378,60 @@ def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): logger.info(line_of_nodes) logger.info( '+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) - out_plugins_print = ['Gnocchi'] - if 'SNMP' in out_plugins.values(): - out_plugins_print.append('SNMP') - if 'AODH' in out_plugins.values(): - out_plugins_print.append('AODH') - if 'CSV' in out_plugins.values(): - out_plugins_print.append('CSV') + out_plugins_print = [] + out_plugins_print1 = [] + for key in out_plugins.keys(): + if 'Gnocchi' in out_plugins[key]: + out_plugins_print1.append('Gnocchi') + if 'AODH' in out_plugins[key]: + out_plugins_print1.append('AODH') + if 'SNMP' in out_plugins[key]: + out_plugins_print1.append('SNMP') + if 'CSV' in out_plugins[key]: + out_plugins_print1.append('CSV') + for i in out_plugins_print1: + if i not in out_plugins_print: + out_plugins_print.append(i) for out_plugin in out_plugins_print: output_plugins_line = '' for id in compute_ids: - out_plugin_result = 'FAIL' + out_plugin_result = '----' if out_plugin == 'Gnocchi': out_plugin_result = \ - 'PASS' if out_plugins[id] == out_plugin else 'FAIL' - if out_plugin == 'AODH': - if out_plugins[id] == out_plugin: - out_plugin_result = \ - 'PASS' if out_plugins[id] == out_plugin else 'FAIL' - if out_plugin == 'SNMP': - if out_plugins[id] == out_plugin: - out_plugin_result = \ - 'PASS' if out_plugins[id] == out_plugin else 'FAIL' - if out_plugin == 'CSV': - if out_plugins[id] == out_plugin: - out_plugin_result = \ - 'PASS' if [ - plugin for comp_id, plugin, res in results - if comp_id == id and res] else 'FAIL' - else: - out_plugin_result = 'SKIP' + 'PASS' + elif out_plugin == 'AODH': + out_plugin_result = \ + 'PASS' + elif out_plugin == 'SNMP': + out_plugin_result = \ + 'PASS' + elif out_plugin == 'CSV': + out_plugin_result = \ + 'PASS' if [ + plugin for comp_id, out_pl, plugin, res in results + if comp_id == id and res] else 'FAIL' + else: + out_plugin_result = \ + 'FAIL' output_plugins_line += '| ' + out_plugin_result + ' ' logger.info( '| OUT:{}'.format(out_plugin) + (' ' * (11 - len(out_plugin))) + output_plugins_line + '|') - for plugin in sorted(tested_plugins.values()): - line_plugin = _print_final_result_of_plugin( - plugin, compute_ids, results, out_plugins, out_plugin) - logger.info( - '| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) - + '|' + line_plugin) + + if out_plugin == 'AODH': + for plugin in sorted(aodh_plugins.values()): + line_plugin = _print_final_result_of_plugin( + plugin, compute_ids, results, out_plugins, out_plugin) + logger.info( + '| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) + + '|' + line_plugin) + else: + for plugin in sorted(tested_plugins.values()): + line_plugin = _print_final_result_of_plugin( + plugin, compute_ids, results, out_plugins, out_plugin) + logger.info( + '| IN:{}'.format(plugin) + (' ' * (11-len(plugin))) + + '|' + line_plugin) logger.info( '+' + ('-' * 16) + '+' + (('-' * 8) + '+') * len(compute_node_names)) @@ -424,8 +439,8 @@ def print_overall_summary(compute_ids, tested_plugins, results, out_plugins): def _exec_testcase( - test_labels, name, gnocchi_running, aodh_running, snmp_running, - controllers, compute_node, conf, results, error_plugins, out_plugins): + test_labels, name, out_plugin, controllers, compute_node, + conf, results, error_plugins, out_plugins): """Execute the testcase. Keyword arguments: @@ -453,11 +468,8 @@ def _exec_testcase( bridge for bridge in ovs_interfaces if bridge in ovs_configured_bridges] plugin_prerequisites = { - 'intel_rdt': [( - conf.is_libpqos_on_node(compute_node), - 'libpqos must be installed.')], 'mcelog': [( - conf.is_installed(compute_node, 'mcelog'), + conf.is_mcelog_installed(compute_node, 'mcelog'), 'mcelog must be installed.')], 'ovs_events': [( len(ovs_existing_configured_int) > 0 or len(ovs_interfaces) > 0, @@ -466,28 +478,22 @@ def _exec_testcase( len(ovs_existing_configured_bridges) > 0, 'Bridges must be configured.')]} gnocchi_criteria_lists = { - 'hugepages': ['hugepages'], - 'mcelog': ['mcelog'], - 'ovs_events': ['interface-ovs-system'], - 'ovs_stats': ['ovs_stats-br0.br0']} + 'hugepages': 'hugepages', + 'intel_rdt': 'rdt', + 'mcelog': 'mcelog', + 'ovs_events': 'interface-ovs-system', + 'ovs_stats': 'ovs_stats-br0.br0'} aodh_criteria_lists = { - 'mcelog': ['mcelog.errors'], - 'ovs_events': ['ovs_events.gauge']} + 'mcelog': 'mcelog', + 'ovs_events': 'ovs_events'} snmp_mib_files = { 'intel_rdt': '/usr/share/snmp/mibs/Intel-Rdt.txt', 'hugepages': '/usr/share/snmp/mibs/Intel-Hugepages.txt', 'mcelog': '/usr/share/snmp/mibs/Intel-Mcelog.txt'} snmp_mib_strings = { - 'intel_rdt': [ - 'INTEL-RDT-MIB::rdtLlc.1', - 'INTEL-RDT-MIB::rdtIpc.1', - 'INTEL-RDT-MIB::rdtMbmRemote.1', - 'INTEL-RDT-MIB::rdtMbmLocal.1'], - 'hugepages': [ - 'INTEL-HUGEPAGES-MIB::hugepagesPageFree'], - 'mcelog': [ - 'INTEL-MCELOG-MIB::memoryCorrectedErrors.1', - 'INTEL-MCELOG-MIB::memoryCorrectedErrors.2']} + 'intel_rdt': 'INTEL-RDT-MIB::rdtLlc.1', + 'hugepages': 'INTEL-HUGEPAGES-MIB::hugepagesPageFree', + 'mcelog': 'INTEL-MCELOG-MIB::memoryCorrectedErrors.1'} nr_hugepages = int(time.time()) % 10000 snmp_in_commands = { 'intel_rdt': None, @@ -496,13 +502,10 @@ def _exec_testcase( 'mcelog': '/root/mce-inject_df < /root/corrected'} csv_subdirs = { 'intel_rdt': [ - 'intel_rdt-{}'.format(core) - for core in conf.get_plugin_config_values( - compute_node, 'intel_rdt', 'Cores')], + 'intel_rdt-0-2'], 'hugepages': [ 'hugepages-mm-2048Kb', 'hugepages-node0-2048Kb', - 'hugepages-node1-2048Kb', 'hugepages-mm-1048576Kb', - 'hugepages-node0-1048576Kb', 'hugepages-node1-1048576Kb'], + 'hugepages-node1-2048Kb'], # 'ipmi': ['ipmi'], 'mcelog': [ 'mcelog-SOCKET_0_CHANNEL_0_DIMM_any', @@ -515,19 +518,14 @@ def _exec_testcase( # compute_node) csv_meter_categories = { 'intel_rdt': [ - 'bytes-llc', 'ipc', 'memory_bandwidth-local', - 'memory_bandwidth-remote'], + 'bytes-llc', 'ipc'], 'hugepages': ['vmpage_number-free', 'vmpage_number-used'], # 'ipmi': csv_meter_categories_ipmi, 'mcelog': [ 'errors-corrected_memory_errors', - 'errors-uncorrected_memory_errors', - 'errors-corrected_memory_errors_in_24h', - 'errors-uncorrected_memory_errors_in_24h'], + 'errors-uncorrected_memory_errors'], 'ovs_stats': [ - 'if_collisions', 'if_dropped', 'if_errors', 'if_packets', - 'if_rx_errors-crc', 'if_rx_errors-frame', 'if_rx_errors-over', - 'if_rx_octets', 'if_tx_octets'], + 'if_dropped', 'if_errors', 'if_packets'], 'ovs_events': ['gauge-link_status']} _print_plugin_label( @@ -541,7 +539,8 @@ def _exec_testcase( for error in plugin_critical_errors: logger.error(' * ' + error) _process_result( - compute_node.get_id(), test_labels[name], False, results) + compute_node.get_id(), out_plugin, test_labels[name], False, + results, compute_node.get_name()) else: plugin_errors = [ error for plugin, error, critical in error_plugins @@ -563,35 +562,36 @@ def _exec_testcase( for prerequisite in failed_prerequisites: logger.error(' * {}'.format(prerequisite)) else: - if gnocchi_running: - plugin_interval = conf.get_plugin_interval(compute_node, name) + plugin_interval = conf.get_plugin_interval(compute_node, name) + if out_plugin == 'Gnocchi': res = conf.test_plugins_with_gnocchi( - compute_node.get_id(), plugin_interval, logger, - criteria_list=gnocchi_criteria_lists[name]) - elif aodh_running: + compute_node.get_name(), plugin_interval, + logger, criteria_list=gnocchi_criteria_lists[name]) + if out_plugin == 'AODH': res = conf.test_plugins_with_aodh( - compute_node.get_id(), plugin_interval, - logger, creteria_list=aodh_criteria_lists[name]) - elif snmp_running: + compute_node.get_name(), plugin_interval, + logger, criteria_list=aodh_criteria_lists[name]) + if out_plugin == 'SNMP': res = \ name in snmp_mib_files and name in snmp_mib_strings \ - and tests.test_snmp_sends_data( - compute_node, - conf.get_plugin_interval(compute_node, name), logger, - SNMPClient(conf, compute_node), snmp_mib_files[name], - snmp_mib_strings[name], snmp_in_commands[name], conf) - else: + and conf.test_plugins_with_snmp( + compute_node.get_name(), plugin_interval, logger, name, + snmp_mib_files[name], snmp_mib_strings[name], + snmp_in_commands[name]) + if out_plugin == 'CSV': res = tests.test_csv_handles_plugin_data( compute_node, conf.get_plugin_interval(compute_node, name), name, csv_subdirs[name], csv_meter_categories[name], logger, CSVClient(conf)) + if res and plugin_errors: logger.info( 'Test works, but will be reported as failure,' + 'because of non-critical errors.') res = False _process_result( - compute_node.get_id(), test_labels[name], res, results) + compute_node.get_id(), out_plugin, test_labels[name], + res, results, compute_node.get_name()) def get_results_for_ovs_events( @@ -614,48 +614,48 @@ def create_ovs_bridge(): APEX_USER_STACK, APEX_PKEY) nodes = handler.get_nodes() + logger.info("Creating OVS bridges on computes nodes") for node in nodes: if node.is_compute(): node.run_cmd('sudo ovs-vsctl add-br br0') node.run_cmd('sudo ovs-vsctl set-manager ptcp:6640') - logger.info('OVS Bridges created on compute nodes') + logger.info('OVS Bridges created on compute nodes') def mcelog_install(): """Install mcelog on compute nodes.""" - _print_label('Enabling mcelog on compute nodes') + _print_label('Enabling mcelog and OVS bridges on compute nodes') handler = factory.Factory.get_handler('apex', APEX_IP, APEX_USER_STACK, APEX_PKEY) nodes = handler.get_nodes() + mce_bin = os.path.dirname(os.path.realpath(__file__)) + '/mce-inject_ea' for node in nodes: if node.is_compute(): centos_release = node.run_cmd('uname -r') if '3.10.0-514.26.2.el7.x86_64' not in centos_release: logger.info( 'Mcelog will not be enabled ' - + 'on node-{0}, '.format(node.get_dict()['id']) + + 'on node-{0}, '.format(node.get_dict()['name']) + 'unsupported CentOS release found ({1}).'.format( centos_release)) else: logger.info( 'Checking if mcelog is enabled' - + ' on node-{}...'.format(node.get_dict()['id'])) + + ' on node-{}...'.format(node.get_dict()['name'])) res = node.run_cmd('ls') if 'mce-inject_ea' and 'corrected' in res: logger.info( 'Mcelog seems to be already installed ' - + 'on node-{}.'.format(node.get_dict()['id'])) + + 'on node-{}.'.format(node.get_dict()['name'])) node.run_cmd('sudo modprobe mce-inject') node.run_cmd('sudo ./mce-inject_ea < corrected') else: logger.info( 'Mcelog will be enabled on node-{}...'.format( node.get_dict()['id'])) - node.put_file( - '/usr/local/lib/python2.7/dist-packages/baro_tests/' - + 'mce-inject_ea', 'mce-inject_ea') + node.put_file(mce_bin, 'mce-inject_ea') node.run_cmd('chmod a+x mce-inject_ea') node.run_cmd('echo "CPU 0 BANK 0" > corrected') node.run_cmd( @@ -734,43 +734,24 @@ def main(bt_logger=None): _print_label( 'Display of Control and Compute nodes available in the set up') - logger.info('controllers: {}'.format([('{0}: {1} ({2})'.format( - node.get_id(), node.get_name(), - node.get_ip())) for node in controllers])) - logger.info('computes: {}'.format([('{0}: {1} ({2})'.format( - node.get_id(), node.get_name(), node.get_ip())) - for node in computes])) + logger.info('controllers: {}'.format([('{0}: {1}'.format( + node.get_name(), node.get_ip())) for node in controllers])) + logger.info('computes: {}'.format([('{0}: {1}'.format( + node.get_name(), node.get_ip())) for node in computes])) mcelog_install() create_ovs_bridge() gnocchi_running_on_con = False aodh_running_on_con = False + # Disabling SNMP write plug-in snmp_running = False - _print_label('Testing Gnocchi, AODH and SNMP on controller nodes') + _print_label('Testing Gnocchi and AODH plugins on nodes') for controller in controllers: - gnocchi_client = GnocchiClient() - gnocchi_client.auth_token() gnocchi_running = ( - gnocchi_running_on_con and conf.is_gnocchi_running(controller)) - aodh_client = AodhClient() - aodh_client.auth_token() + gnocchi_running_on_con or conf.is_gnocchi_running(controller)) aodh_running = ( - aodh_running_on_con and conf.is_aodh_running(controller)) - if gnocchi_running: - logger.info("Gnocchi is running on controller.") - elif aodh_running: - logger.error("Gnocchi is not running on controller.") - logger.info("AODH is running on controller.") - elif snmp_running: - logger.error("Gnocchi is not running on Controller") - logger.error("AODH is not running on controller.") - logger.info("SNMP is running on controller.") - else: - logger.error("Gnocchi is not running on Controller") - logger.error("AODH is not running on controller.") - logger.error("SNMP is not running on controller.") - logger.info("CSV will be enabled on compute nodes.") + aodh_running_on_con or conf.is_aodh_running(controller)) compute_ids = [] compute_node_names = [] @@ -782,118 +763,98 @@ def main(bt_logger=None): 'mcelog': 'Mcelog', 'ovs_stats': 'OVS stats', 'ovs_events': 'OVS events'} - out_plugins = { - 'gnocchi': 'Gnocchi', - 'aodh': 'AODH', - 'snmp': 'SNMP', - 'csv': 'CSV'} + aodh_plugin_labels = { + 'mcelog': 'Mcelog', + 'ovs_events': 'OVS events'} + out_plugins = {} for compute_node in computes: node_id = compute_node.get_id() node_name = compute_node.get_name() - out_plugins[node_id] = 'CSV' + out_plugins[node_id] = [] compute_ids.append(node_id) compute_node_names.append(node_name) plugins_to_enable = [] - _print_label('NODE {}: Test Gnocchi Plug-in'.format(node_name)) - logger.info('Checking if gnocchi plug-in is included in compute nodes.') - if not conf.check_gnocchi_plugin_included(compute_node): - logger.error('Gnocchi plug-in is not included.') - logger.info( - 'Testcases on node {} will not be executed'.format(node_name)) - else: - collectd_restarted, collectd_warnings = \ - conf.restart_collectd(compute_node) - sleep_time = 30 - logger.info( - 'Sleeping for {} seconds after collectd restart...'.format( - sleep_time)) - time.sleep(sleep_time) - if not collectd_restarted: - for warning in collectd_warnings: - logger.warning(warning) + error_plugins = [] + gnocchi_running = ( + gnocchi_running and conf.check_gnocchi_plugin_included( + compute_node)) + aodh_running = ( + aodh_running and conf.check_aodh_plugin_included(compute_node)) + # logger.info("SNMP enabled on {}" .format(node_name)) + if gnocchi_running: + out_plugins[node_id].append("Gnocchi") + if aodh_running: + out_plugins[node_id].append("AODH") + if snmp_running: + out_plugins[node_id].append("SNMP") + + if 'Gnocchi' in out_plugins[node_id]: + plugins_to_enable.append('csv') + out_plugins[node_id].append("CSV") + if plugins_to_enable: + _print_label( + 'NODE {}: Enabling Test Plug-in '.format(node_name) + + 'and Test case execution') + if plugins_to_enable and not conf.enable_plugins( + compute_node, plugins_to_enable, error_plugins, + create_backup=False): logger.error( - 'Restart of collectd on node {} failed'.format(node_name)) + 'Failed to test plugins on node {}.'.format(node_id)) logger.info( 'Testcases on node {} will not be executed'.format( - node_name)) - else: - for warning in collectd_warnings: - logger.warning(warning) - - if gnocchi_running: - out_plugins[node_id] = 'Gnocchi' - logger.info("Gnocchi is active and collecting data") - elif aodh_running: - out_plugins[node_id] = 'AODH' - logger.info("AODH withh be tested") - _print_label('Node {}: Test AODH' .format(node_name)) - logger.info("Checking if AODH is running") - logger.info("AODH is running") - elif snmp_running: - out_plugins[node_id] = 'SNMP' - logger.info("SNMP will be tested.") - _print_label('NODE {}: Test SNMP'.format(node_id)) - logger.info("Checking if SNMP is running.") - logger.info("SNMP is running.") - else: - plugins_to_enable.append('csv') - out_plugins[node_id] = 'CSV' - logger.error("Gnocchi, AODH, SNMP are not running") - logger.info( - "CSV will be enabled for verification " - + "of test plugins.") - if plugins_to_enable: - _print_label( - 'NODE {}: Enabling Test Plug-in '.format(node_name) - + 'and Test case execution') - error_plugins = [] - if plugins_to_enable and not conf.enable_plugins( - compute_node, plugins_to_enable, error_plugins, - create_backup=False): + node_id)) + + for i in out_plugins[node_id]: + if i == 'AODH': + for plugin_name in sorted(aodh_plugin_labels.keys()): + _exec_testcase( + aodh_plugin_labels, plugin_name, i, + controllers, compute_node, conf, results, + error_plugins, out_plugins[node_id]) + elif i == 'CSV': + _print_label("Node {}: Executing CSV Testcases".format( + node_name)) + logger.info("Restarting collectd for CSV tests") + collectd_restarted, collectd_warnings = \ + conf.restart_collectd(compute_node) + sleep_time = 10 + logger.info( + 'Sleeping for {} seconds'.format(sleep_time) + + ' after collectd restart...') + time.sleep(sleep_time) + if not collectd_restarted: + for warning in collectd_warnings: + logger.warning(warning) logger.error( - 'Failed to test plugins on node {}.'.format(node_id)) + 'Restart of collectd on node {} failed'.format( + compute_node)) logger.info( - 'Testcases on node {} will not be executed'.format( - node_id)) - else: - if plugins_to_enable: - collectd_restarted, collectd_warnings = \ - conf.restart_collectd(compute_node) - sleep_time = 30 - logger.info( - 'Sleeping for {} seconds'.format(sleep_time) - + ' after collectd restart...') - time.sleep(sleep_time) - if plugins_to_enable and not collectd_restarted: - for warning in collectd_warnings: - logger.warning(warning) - logger.error( - 'Restart of collectd on node {} failed'.format( - node_id)) - logger.info( - 'Testcases on node {}'.format(node_id) - + ' will not be executed.') - else: - if collectd_warnings: - for warning in collectd_warnings: - logger.warning(warning) - - for plugin_name in sorted(plugin_labels.keys()): - _exec_testcase( - plugin_labels, plugin_name, gnocchi_running, - aodh_running, snmp_running, controllers, - compute_node, conf, results, error_plugins, - out_plugins[node_id]) - - # _print_label('NODE {}: Restoring config file'.format(node_name)) - # conf.restore_config(compute_node) - mcelog_delete() - print_overall_summary(compute_ids, plugin_labels, results, out_plugins) - - if ((len([res for res in results if not res[2]]) > 0) - or (len(results) < len(computes) * len(plugin_labels))): - logger.error('Some tests have failed or have not been executed') - return 1 + 'CSV Testcases on node {}'.format(compute_node) + + ' will not be executed.') + for plugin_name in sorted(plugin_labels.keys()): + _exec_testcase( + plugin_labels, plugin_name, i, + controllers, compute_node, conf, results, + error_plugins, out_plugins[node_id]) + + else: + for plugin_name in sorted(plugin_labels.keys()): + _exec_testcase( + plugin_labels, plugin_name, i, + controllers, compute_node, conf, results, + error_plugins, out_plugins[node_id]) + + mcelog_delete() + print_overall_summary( + compute_ids, plugin_labels, aodh_plugin_labels, results, out_plugins) + + for res in results: + if res[3] is 'False' or 'None': + logger.error('Some tests have failed or have not been executed') + logger.error('Overall Result is Fail') + return 1 + else: + pass return 0 diff --git a/baro_tests/config_server.py b/baro_tests/config_server.py index f156fcf7..f35f7882 100644 --- a/baro_tests/config_server.py +++ b/baro_tests/config_server.py @@ -19,7 +19,6 @@ import time import os.path import os import re -import subprocess from opnfv.deployment import factory ID_RSA_PATH = '/root/.ssh/id_rsa' SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh' @@ -28,7 +27,7 @@ COLLECTD_CONF = '/etc/collectd.conf' COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d' NOTIFICATION_FILE = '/var/log/python-notifications.dump' COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py' -APEX_IP = subprocess.check_output("echo $INSTALLER_IP", shell=True) +APEX_IP = os.getenv("INSTALLER_IP").rstrip('\n') APEX_USER = 'root' APEX_USER_STACK = 'stack' APEX_PKEY = '/root/.ssh/id_rsa' @@ -101,20 +100,20 @@ class ConfigServer(object): stderr_lines = stderr.readlines() if stderr_lines: self.__logger.warning( - "'fuel node' command failed (try {}):".format(attempt)) + "'Apex node' command failed (try {}):".format(attempt)) for line in stderr_lines: self.__logger.debug(line.strip()) else: fuel_node_passed = True if attempt > 1: self.__logger.info( - "'fuel node' command passed (try {})".format(attempt)) + "'Apex node' command passed (try {})".format(attempt)) attempt += 1 if not fuel_node_passed: self.__logger.error( - "'fuel node' command failed. This was the last try.") + "'Apex node' command failed. This was the last try.") raise OSError( - "'fuel node' command failed. This was the last try.") + "'Apex node' command failed. This was the last try.") node_table = stdout.readlines()\ # skip table title and parse table values @@ -184,9 +183,10 @@ class ConfigServer(object): if compute_name == node.get_dict()['name']: stdout = node.run_cmd( 'cat /etc/collectd/collectd.conf.d/{}.conf'.format(plugin)) + if stdout is None: + return default_interval for line in stdout.split('\n'): if 'Interval' in line: - # line = line.strip('Interval') return 1 return default_interval @@ -206,6 +206,8 @@ class ConfigServer(object): if compute_name == node.get_dict()['name']: stdout = node.run_cmd( 'cat /etc/collectd/collectd.conf.d/{}.conf' .format(plugin)) + if stdout is None: + return default_values for line in stdout.split('\n'): if 'Interfaces' in line: return line.split(' ', 1)[1] @@ -257,28 +259,49 @@ class ConfigServer(object): Return boolean value whether Gnocchi is running. """ gnocchi_present = False - lines = self.execute_command( - 'source overcloudrc.v3;systemctl status openstack-gnocchi-api | ' - + 'grep running', controller.get_ip()) - for line in lines: - if '(running)' in line: - gnocchi_present = True + controller_name = controller.get_name() + nodes = get_apex_nodes() + for node in nodes: + if controller_name == node.get_dict()['name']: + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "openstack catalog list | grep gnocchi") + if stdout is None: + return False + elif 'gnocchi' in stdout: + gnocchi_present = True + return gnocchi_present + else: + return False return gnocchi_present def is_aodh_running(self, controller): """Check whether aodh service is running on controller """ aodh_present = False - lines = self.execute_command( - 'source overcloudrc.v3;systemctl openstack-aodh-api | grep running', - controller.get_ip()) - for line in lines: - self.__logger.info("Line = {}" .format(line)) - if '(running)' in line: - aodh_present = True + controller_name = controller.get_name() + nodes = get_apex_nodes() + for node in nodes: + if controller_name == node.get_dict()['name']: + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "openstack catalog list | grep aodh") + if stdout is None: + return False + elif 'aodh' in stdout: + aodh_present = True + return aodh_present + else: + return False return aodh_present - def is_installed(self, compute, package): + def is_mcelog_installed(self, compute, package): """Check whether package exists on compute node. Keyword arguments: @@ -292,8 +315,10 @@ class ConfigServer(object): for node in nodes: if compute_name == node.get_dict()['name']: stdout = node.run_cmd( - 'yum list installed | grep mcelog') - if 'mcelog' in stdout: + 'rpm -qa | grep mcelog') + if stdout is None: + return 0 + elif 'mcelog' in stdout: return 1 else: return 0 @@ -310,6 +335,32 @@ class ConfigServer(object): return True return False + def check_aodh_plugin_included(self, compute): + """Check if aodh plugin is included in collectd.conf file. + If not, try to enable it. + + Keyword arguments: + compute -- compute node instance + + Return boolean value whether AODH plugin is included + or it's enabling was successful. + """ + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + aodh_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d') + if 'aodh.conf' not in aodh_conf: + self.__logger.info( + "AODH Plugin not included in {}".format(compute_name)) + return False + else: + self.__logger.info( + "AODH plugin present in compute node {}" .format( + compute_name)) + return True + return True + def check_gnocchi_plugin_included(self, compute): """Check if gnocchi plugin is included in collectd.conf file. If not, try to enable it. @@ -324,16 +375,37 @@ class ConfigServer(object): nodes = get_apex_nodes() for node in nodes: if compute_name == node.get_dict()['name']: - # node.run_cmd('su; "opnfvapex"') gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d') if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf: - self.__logger.info("Gnocchi Plugin not included") - return True + self.__logger.info( + "Gnocchi Plugin not included in node {}".format( + compute_name)) + return False else: - self.__logger.info("Gnochi plugin present") + self.__logger.info( + "Gnocchi plugin available in compute node {}" .format( + compute_name)) return True return True + def check_snmp_plugin_included(self, compute): + """Check if SNMP plugin is active in compute node. + """ + snmp_mib = '/usr/share/snmp/mibs/Intel-Rdt.txt' + snmp_string = 'INTEL-RDT-MIB::intelRdt' + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + stdout = node.run_cmd( + 'snmpwalk -v2c -m {0} -c public localhost {1}' .format( + snmp_mib, snmp_string)) + self.__logger.info("snmp output = {}" .format(stdout)) + if 'OID' in stdout: + return False + else: + return True + def enable_plugins( self, compute, plugins, error_plugins, create_backup=True): """Enable plugins on compute node @@ -341,43 +413,21 @@ class ConfigServer(object): Keyword arguments: compute -- compute node instance plugins -- list of plugins to be enabled - error_plugins -- list of tuples with found errors, new entries - may be added there (plugin, error_description, is_critical): - plugin -- plug-in name - error_decription -- description of the error - is_critical -- boolean value indicating whether error - is critical - create_backup -- boolean value indicating whether backup - shall be created Return boolean value indicating whether function was successful. """ + csv_file = os.path.dirname(os.path.realpath(__file__)) + '/csv.conf' plugins = sorted(plugins) compute_name = compute.get_name() nodes = get_apex_nodes() for node in nodes: if compute_name == node.get_dict()['name']: - node.put_file( - '/usr/local/lib/python2.7/dist-packages/baro_tests/' - + 'csv.conf', 'csv.conf') + node.put_file(csv_file, 'csv.conf') node.run_cmd( 'sudo cp csv.conf ' + '/etc/collectd/collectd.conf.d/csv.conf') return True - def restore_config(self, compute): - """Restore collectd config file from backup on compute node. - - Keyword arguments: - compute -- compute node instance - """ - ssh, sftp = self.__open_sftp_session( - compute.get_ip(), 'root', 'opnfvapex') - - self.__logger.info('Restoring config file from backup...') - ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF)) - ssh.exec_command("cp {0}.backup {0}".format(COLLECTD_CONF)) - def restart_collectd(self, compute): """Restart collectd on compute node. @@ -419,142 +469,175 @@ class ConfigServer(object): return False, warning return True, warning - def test_gnocchi_is_sending_data(self, controller): - """ Checking if Gnocchi is sending metrics to controller""" - metric_ids = [] - timestamps1 = {} - timestamps2 = {} - ssh, sftp = self.__open_sftp_session( - controller.get_ip(), 'root', 'opnfvapex') - - self.__logger.info('Getting gnocchi metric list on{}'.format( - controller.get_name())) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi metric list | grep if_packets", - ssh=ssh) - for line in stdout: - metric_ids = [r.split('|')[1] for r in stdout] - self.__logger.info("Metric ids = {}" .format(metric_ids)) - for metric_id in metric_ids: - metric_id = metric_id.replace("u", "") - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - self.__logger.info("stdout measures ={}" .format(stdout)) - for line in stdout: - if line[0] == '+': - pass - else: - self.__logger.info("Line = {}" .format(line)) - timestamps1 = [line.split('|')[1]] - self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) - time.sleep(10) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - for line in stdout: - if line[0] == '+': - pass - else: - timestamps2 = [line.split('|')[1]] - self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) - if timestamps1 == timestamps2: - self.__logger.info("False") - # return False - return True - else: - self.__logger.info("True") - return True + def test_plugins_with_aodh( + self, compute, plugin_interval, logger, + criteria_list=[]): - def test_plugins_with_aodh(self, controller): - """Checking if AODH is sending metrics to controller""" - metric_ids = [] + metric_id = {} timestamps1 = {} timestamps2 = {} - ssh, sftp = self.__open_sftp_session( - controller.get_ip(), 'root', 'opnfvapex') - self.__logger.info('Getting AODH alarm list on{}'.format( - controller.get_name())) - stdout = self.execute_command( - "source overcloudrc.v3;aodh alarm list | grep mcelog", - ssh=ssh) - for line in stdout: - metric_ids = [r.split('|')[1] for r in stdout] - self.__logger.info("Metric ids = {}" .format(metric_ids)) - for metric_id in metric_ids: - metric_id = metric_id.replace("u", "") - stdout = self.execute_command( - "source overcloudrc.v3;aodh alarm show {}" .format( - metric_id), ssh=ssh) - self.__logger.info("stdout alarms ={}" .format(stdout)) - for line in stdout: - if line[0] == '+': - pass - else: - self.__logger.info("Line = {}" .format(line)) - timestamps1 = [line.split('|')[1]] - self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) - time.sleep(10) - stdout = self.execute_command( - "source overcloudrc.v3;aodh alarm show {}" .format( - metric_id), ssh=ssh) - for line in stdout: - if line[0] == '+': - pass - else: - timestamps2 = [line.split('|')[1]] - self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) - if timestamps1 == timestamps2: - self.__logger.info("False") - # return False - return True - else: - self.__logger.info("True") - return True + nodes = get_apex_nodes() + for node in nodes: + if node.is_controller(): + self.__logger.info('Getting AODH Alarm list on {}' .format( + (node.get_dict()['name']))) + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "aodh alarm list | grep {0} | grep {1}" + .format(criteria_list, compute)) + if stdout is None: + self.__logger.info("aodh alarm list was empty") + return False + for line in stdout.splitlines(): + line = line.replace('|', "") + metric_id = line.split()[0] + stdout = node.run_cmd( + 'source overcloudrc.v3; aodh alarm show {}' .format( + metric_id)) + if stdout is None: + self.__logger.info("aodh alarm list was empty") + return False + for line in stdout.splitlines()[3: -1]: + line = line.replace('|', "") + if line.split()[0] == 'timestamp': + timestamps1 = line.split()[1] + else: + pass + time.sleep(12) + stdout = node.run_cmd( + "source overcloudrc.v3; aodh alarm show {}" .format( + metric_id)) + if stdout is None: + self.__logger.info("aodh alarm list was empty") + return False + for line in stdout.splitlines()[3:-1]: + line = line.replace('|', "") + if line.split()[0] == 'timestamp': + timestamps2 = line.split()[1] + else: + pass + if timestamps1 == timestamps2: + self.__logger.info( + "Data not updated after interval of 12 seconds") + return False + else: + self.__logger.info("PASS") + return True def test_plugins_with_gnocchi( - self, controller, compute_node, plugin_interval, logger, + self, compute, plugin_interval, logger, criteria_list=[]): - metric_ids = [] + metric_id = {} timestamps1 = {} timestamps2 = {} - ssh, sftp = self.__open_sftp_session( - controller.get_ip(), 'root', 'opnfvapex') - self.__logger.info('Getting gnocchi metric list on{}'.format( - controller.get_name())) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi metric list | grep {0} | grep {1}" - .format(compute_node.get_name(), criteria_list), ssh=ssh) - for line in stdout: - metric_ids = [r.split('|')[1] for r in stdout] - self.__logger.info("Metric ids = {}" .format(metric_ids)) - for metric_id in metric_ids: - metric_id = metric_id.replace("u", "") - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - self.__logger.info("stdout measures ={}" .format(stdout)) - for line in stdout: - if line[0] == '+': - pass - else: - self.__logger.info("Line = {}" .format(line)) - timestamps1 = [line.split('|')[1]] - self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) - time.sleep(10) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - for line in stdout: - if line[0] == '+': - pass - else: - timestamps2 = [line.split('|')[1]] - self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) - if timestamps1 == timestamps2: - self.__logger.info("False") - return False - else: - self.__logger.info("True") - return True + nodes = get_apex_nodes() + sleep_time = plugin_interval + 2 + for node in nodes: + if node.is_controller(): + self.__logger.info('Getting gnocchi metric list on {}' .format( + (node.get_dict()['name']))) + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "gnocchi metric list | grep {0} | grep {1}" + .format(criteria_list, compute)) + if stdout is None: + self.__logger.info("gnocchi list was empty") + return False + for line in stdout.splitlines(): + line = line.replace('|', "") + metric_id = line.split()[0] + stdout = node.run_cmd( + 'source overcloudrc.v3;gnocchi measures show {}'.format( + metric_id)) + if stdout is None: + self.__logger.info("gnocchi list was empty") + return False + for line in stdout.splitlines()[3: -1]: + if line[0] == '+': + pass + else: + timestamps1 = line.replace('|', "") + timestamps1 = timestamps1.split()[0] + time.sleep(sleep_time) + stdout = node.run_cmd( + "source overcloudrc.v3;gnocchi measures show {}".format( + metric_id)) + if stdout is None: + self.__logger.info("gnocchi measures was empty") + return False + for line in stdout.splitlines()[3:-1]: + if line[0] == '+': + pass + else: + timestamps2 = line.replace('|', "") + timestamps2 = timestamps2.split()[0] + if timestamps1 == timestamps2: + self.__logger.info( + "Plugin Interval is {}" .format(plugin_interval)) + self.__logger.info( + "Data not updated after {} seconds".format( + sleep_time)) + return False + else: + self.__logger.info("PASS") + return True + return False + + def test_plugins_with_snmp( + self, compute, plugin_interval, logger, plugin, snmp_mib_files=[], + snmp_mib_strings=[], snmp_in_commands=[]): + + if plugin == 'hugepages' or 'intel_rdt' or 'mcelog': + nodes = get_apex_nodes() + for node in nodes: + if compute == node.get_dict()['name']: + stdout = node.run_cmd( + 'snmpwalk -v2c -m {0} -c public localhost {1}' .format( + snmp_mib_files, snmp_mib_strings)) + self.__logger.info("{}" .format(stdout)) + if stdout is None: + self.__logger.info("No output from snmpwalk") + return False + elif 'OID' in stdout: + self.__logger.info("SNMP query failed") + return False + else: + counter1 = stdout.split()[3] + time.sleep(10) + stdout = node.run_cmd( + 'snmpwalk -v2c -m {0} -c public localhost {1}' .format( + snmp_mib_files, snmp_mib_strings)) + self.__logger.info("{}" .format(stdout)) + if stdout is None: + self.__logger.info("No output from snmpwalk") + elif 'OID' in stdout: + self.__logger.info( + "SNMP query failed during second check") + self.__logger.info("waiting for 10 sec") + time.sleep(10) + stdout = node.run_cmd( + 'snmpwalk -v2c -m {0} -c public localhost {1}' .format( + snmp_mib_files, snmp_mib_strings)) + self.__logger.info("{}" .format(stdout)) + if stdout is None: + self.__logger.info("No output from snmpwalk") + elif 'OID' in stdout: + self.__logger.info("SNMP query failed again") + self.__logger.info("Failing this test case") + return False + else: + counter2 = stdout.split()[3] + + if counter1 == counter2: + return False + else: + return True + else: + return False |