diff options
Diffstat (limited to 'baro_tests/config_server.py')
| -rw-r--r-- | baro_tests/config_server.py | 445 |
1 files changed, 264 insertions, 181 deletions
diff --git a/baro_tests/config_server.py b/baro_tests/config_server.py index f156fcf7..f35f7882 100644 --- a/baro_tests/config_server.py +++ b/baro_tests/config_server.py @@ -19,7 +19,6 @@ import time import os.path import os import re -import subprocess from opnfv.deployment import factory ID_RSA_PATH = '/root/.ssh/id_rsa' SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh' @@ -28,7 +27,7 @@ COLLECTD_CONF = '/etc/collectd.conf' COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d' NOTIFICATION_FILE = '/var/log/python-notifications.dump' COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py' -APEX_IP = subprocess.check_output("echo $INSTALLER_IP", shell=True) +APEX_IP = os.getenv("INSTALLER_IP").rstrip('\n') APEX_USER = 'root' APEX_USER_STACK = 'stack' APEX_PKEY = '/root/.ssh/id_rsa' @@ -101,20 +100,20 @@ class ConfigServer(object): stderr_lines = stderr.readlines() if stderr_lines: self.__logger.warning( - "'fuel node' command failed (try {}):".format(attempt)) + "'Apex node' command failed (try {}):".format(attempt)) for line in stderr_lines: self.__logger.debug(line.strip()) else: fuel_node_passed = True if attempt > 1: self.__logger.info( - "'fuel node' command passed (try {})".format(attempt)) + "'Apex node' command passed (try {})".format(attempt)) attempt += 1 if not fuel_node_passed: self.__logger.error( - "'fuel node' command failed. This was the last try.") + "'Apex node' command failed. This was the last try.") raise OSError( - "'fuel node' command failed. This was the last try.") + "'Apex node' command failed. This was the last try.") node_table = stdout.readlines()\ # skip table title and parse table values @@ -184,9 +183,10 @@ class ConfigServer(object): if compute_name == node.get_dict()['name']: stdout = node.run_cmd( 'cat /etc/collectd/collectd.conf.d/{}.conf'.format(plugin)) + if stdout is None: + return default_interval for line in stdout.split('\n'): if 'Interval' in line: - # line = line.strip('Interval') return 1 return default_interval @@ -206,6 +206,8 @@ class ConfigServer(object): if compute_name == node.get_dict()['name']: stdout = node.run_cmd( 'cat /etc/collectd/collectd.conf.d/{}.conf' .format(plugin)) + if stdout is None: + return default_values for line in stdout.split('\n'): if 'Interfaces' in line: return line.split(' ', 1)[1] @@ -257,28 +259,49 @@ class ConfigServer(object): Return boolean value whether Gnocchi is running. """ gnocchi_present = False - lines = self.execute_command( - 'source overcloudrc.v3;systemctl status openstack-gnocchi-api | ' - + 'grep running', controller.get_ip()) - for line in lines: - if '(running)' in line: - gnocchi_present = True + controller_name = controller.get_name() + nodes = get_apex_nodes() + for node in nodes: + if controller_name == node.get_dict()['name']: + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "openstack catalog list | grep gnocchi") + if stdout is None: + return False + elif 'gnocchi' in stdout: + gnocchi_present = True + return gnocchi_present + else: + return False return gnocchi_present def is_aodh_running(self, controller): """Check whether aodh service is running on controller """ aodh_present = False - lines = self.execute_command( - 'source overcloudrc.v3;systemctl openstack-aodh-api | grep running', - controller.get_ip()) - for line in lines: - self.__logger.info("Line = {}" .format(line)) - if '(running)' in line: - aodh_present = True + controller_name = controller.get_name() + nodes = get_apex_nodes() + for node in nodes: + if controller_name == node.get_dict()['name']: + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "openstack catalog list | grep aodh") + if stdout is None: + return False + elif 'aodh' in stdout: + aodh_present = True + return aodh_present + else: + return False return aodh_present - def is_installed(self, compute, package): + def is_mcelog_installed(self, compute, package): """Check whether package exists on compute node. Keyword arguments: @@ -292,8 +315,10 @@ class ConfigServer(object): for node in nodes: if compute_name == node.get_dict()['name']: stdout = node.run_cmd( - 'yum list installed | grep mcelog') - if 'mcelog' in stdout: + 'rpm -qa | grep mcelog') + if stdout is None: + return 0 + elif 'mcelog' in stdout: return 1 else: return 0 @@ -310,6 +335,32 @@ class ConfigServer(object): return True return False + def check_aodh_plugin_included(self, compute): + """Check if aodh plugin is included in collectd.conf file. + If not, try to enable it. + + Keyword arguments: + compute -- compute node instance + + Return boolean value whether AODH plugin is included + or it's enabling was successful. + """ + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + aodh_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d') + if 'aodh.conf' not in aodh_conf: + self.__logger.info( + "AODH Plugin not included in {}".format(compute_name)) + return False + else: + self.__logger.info( + "AODH plugin present in compute node {}" .format( + compute_name)) + return True + return True + def check_gnocchi_plugin_included(self, compute): """Check if gnocchi plugin is included in collectd.conf file. If not, try to enable it. @@ -324,16 +375,37 @@ class ConfigServer(object): nodes = get_apex_nodes() for node in nodes: if compute_name == node.get_dict()['name']: - # node.run_cmd('su; "opnfvapex"') gnocchi_conf = node.run_cmd('ls /etc/collectd/collectd.conf.d') if 'collectd-ceilometer-plugin.conf' not in gnocchi_conf: - self.__logger.info("Gnocchi Plugin not included") - return True + self.__logger.info( + "Gnocchi Plugin not included in node {}".format( + compute_name)) + return False else: - self.__logger.info("Gnochi plugin present") + self.__logger.info( + "Gnocchi plugin available in compute node {}" .format( + compute_name)) return True return True + def check_snmp_plugin_included(self, compute): + """Check if SNMP plugin is active in compute node. + """ + snmp_mib = '/usr/share/snmp/mibs/Intel-Rdt.txt' + snmp_string = 'INTEL-RDT-MIB::intelRdt' + compute_name = compute.get_name() + nodes = get_apex_nodes() + for node in nodes: + if compute_name == node.get_dict()['name']: + stdout = node.run_cmd( + 'snmpwalk -v2c -m {0} -c public localhost {1}' .format( + snmp_mib, snmp_string)) + self.__logger.info("snmp output = {}" .format(stdout)) + if 'OID' in stdout: + return False + else: + return True + def enable_plugins( self, compute, plugins, error_plugins, create_backup=True): """Enable plugins on compute node @@ -341,43 +413,21 @@ class ConfigServer(object): Keyword arguments: compute -- compute node instance plugins -- list of plugins to be enabled - error_plugins -- list of tuples with found errors, new entries - may be added there (plugin, error_description, is_critical): - plugin -- plug-in name - error_decription -- description of the error - is_critical -- boolean value indicating whether error - is critical - create_backup -- boolean value indicating whether backup - shall be created Return boolean value indicating whether function was successful. """ + csv_file = os.path.dirname(os.path.realpath(__file__)) + '/csv.conf' plugins = sorted(plugins) compute_name = compute.get_name() nodes = get_apex_nodes() for node in nodes: if compute_name == node.get_dict()['name']: - node.put_file( - '/usr/local/lib/python2.7/dist-packages/baro_tests/' - + 'csv.conf', 'csv.conf') + node.put_file(csv_file, 'csv.conf') node.run_cmd( 'sudo cp csv.conf ' + '/etc/collectd/collectd.conf.d/csv.conf') return True - def restore_config(self, compute): - """Restore collectd config file from backup on compute node. - - Keyword arguments: - compute -- compute node instance - """ - ssh, sftp = self.__open_sftp_session( - compute.get_ip(), 'root', 'opnfvapex') - - self.__logger.info('Restoring config file from backup...') - ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF)) - ssh.exec_command("cp {0}.backup {0}".format(COLLECTD_CONF)) - def restart_collectd(self, compute): """Restart collectd on compute node. @@ -419,142 +469,175 @@ class ConfigServer(object): return False, warning return True, warning - def test_gnocchi_is_sending_data(self, controller): - """ Checking if Gnocchi is sending metrics to controller""" - metric_ids = [] - timestamps1 = {} - timestamps2 = {} - ssh, sftp = self.__open_sftp_session( - controller.get_ip(), 'root', 'opnfvapex') - - self.__logger.info('Getting gnocchi metric list on{}'.format( - controller.get_name())) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi metric list | grep if_packets", - ssh=ssh) - for line in stdout: - metric_ids = [r.split('|')[1] for r in stdout] - self.__logger.info("Metric ids = {}" .format(metric_ids)) - for metric_id in metric_ids: - metric_id = metric_id.replace("u", "") - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - self.__logger.info("stdout measures ={}" .format(stdout)) - for line in stdout: - if line[0] == '+': - pass - else: - self.__logger.info("Line = {}" .format(line)) - timestamps1 = [line.split('|')[1]] - self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) - time.sleep(10) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - for line in stdout: - if line[0] == '+': - pass - else: - timestamps2 = [line.split('|')[1]] - self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) - if timestamps1 == timestamps2: - self.__logger.info("False") - # return False - return True - else: - self.__logger.info("True") - return True + def test_plugins_with_aodh( + self, compute, plugin_interval, logger, + criteria_list=[]): - def test_plugins_with_aodh(self, controller): - """Checking if AODH is sending metrics to controller""" - metric_ids = [] + metric_id = {} timestamps1 = {} timestamps2 = {} - ssh, sftp = self.__open_sftp_session( - controller.get_ip(), 'root', 'opnfvapex') - self.__logger.info('Getting AODH alarm list on{}'.format( - controller.get_name())) - stdout = self.execute_command( - "source overcloudrc.v3;aodh alarm list | grep mcelog", - ssh=ssh) - for line in stdout: - metric_ids = [r.split('|')[1] for r in stdout] - self.__logger.info("Metric ids = {}" .format(metric_ids)) - for metric_id in metric_ids: - metric_id = metric_id.replace("u", "") - stdout = self.execute_command( - "source overcloudrc.v3;aodh alarm show {}" .format( - metric_id), ssh=ssh) - self.__logger.info("stdout alarms ={}" .format(stdout)) - for line in stdout: - if line[0] == '+': - pass - else: - self.__logger.info("Line = {}" .format(line)) - timestamps1 = [line.split('|')[1]] - self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) - time.sleep(10) - stdout = self.execute_command( - "source overcloudrc.v3;aodh alarm show {}" .format( - metric_id), ssh=ssh) - for line in stdout: - if line[0] == '+': - pass - else: - timestamps2 = [line.split('|')[1]] - self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) - if timestamps1 == timestamps2: - self.__logger.info("False") - # return False - return True - else: - self.__logger.info("True") - return True + nodes = get_apex_nodes() + for node in nodes: + if node.is_controller(): + self.__logger.info('Getting AODH Alarm list on {}' .format( + (node.get_dict()['name']))) + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "aodh alarm list | grep {0} | grep {1}" + .format(criteria_list, compute)) + if stdout is None: + self.__logger.info("aodh alarm list was empty") + return False + for line in stdout.splitlines(): + line = line.replace('|', "") + metric_id = line.split()[0] + stdout = node.run_cmd( + 'source overcloudrc.v3; aodh alarm show {}' .format( + metric_id)) + if stdout is None: + self.__logger.info("aodh alarm list was empty") + return False + for line in stdout.splitlines()[3: -1]: + line = line.replace('|', "") + if line.split()[0] == 'timestamp': + timestamps1 = line.split()[1] + else: + pass + time.sleep(12) + stdout = node.run_cmd( + "source overcloudrc.v3; aodh alarm show {}" .format( + metric_id)) + if stdout is None: + self.__logger.info("aodh alarm list was empty") + return False + for line in stdout.splitlines()[3:-1]: + line = line.replace('|', "") + if line.split()[0] == 'timestamp': + timestamps2 = line.split()[1] + else: + pass + if timestamps1 == timestamps2: + self.__logger.info( + "Data not updated after interval of 12 seconds") + return False + else: + self.__logger.info("PASS") + return True def test_plugins_with_gnocchi( - self, controller, compute_node, plugin_interval, logger, + self, compute, plugin_interval, logger, criteria_list=[]): - metric_ids = [] + metric_id = {} timestamps1 = {} timestamps2 = {} - ssh, sftp = self.__open_sftp_session( - controller.get_ip(), 'root', 'opnfvapex') - self.__logger.info('Getting gnocchi metric list on{}'.format( - controller.get_name())) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi metric list | grep {0} | grep {1}" - .format(compute_node.get_name(), criteria_list), ssh=ssh) - for line in stdout: - metric_ids = [r.split('|')[1] for r in stdout] - self.__logger.info("Metric ids = {}" .format(metric_ids)) - for metric_id in metric_ids: - metric_id = metric_id.replace("u", "") - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - self.__logger.info("stdout measures ={}" .format(stdout)) - for line in stdout: - if line[0] == '+': - pass - else: - self.__logger.info("Line = {}" .format(line)) - timestamps1 = [line.split('|')[1]] - self.__logger.info("Last line timetamp1 = {}" .format(timestamps1)) - time.sleep(10) - stdout = self.execute_command( - "source overcloudrc.v3;gnocchi measures show {}" .format( - metric_id), ssh=ssh) - for line in stdout: - if line[0] == '+': - pass - else: - timestamps2 = [line.split('|')[1]] - self.__logger.info("Last line timetamp2 = {}" .format(timestamps2)) - if timestamps1 == timestamps2: - self.__logger.info("False") - return False - else: - self.__logger.info("True") - return True + nodes = get_apex_nodes() + sleep_time = plugin_interval + 2 + for node in nodes: + if node.is_controller(): + self.__logger.info('Getting gnocchi metric list on {}' .format( + (node.get_dict()['name']))) + node.put_file( + '/home/opnfv/functest/conf/openstack.creds', + 'overcloudrc.v3') + stdout = node.run_cmd( + "source overcloudrc.v3;" + + "gnocchi metric list | grep {0} | grep {1}" + .format(criteria_list, compute)) + if stdout is None: + self.__logger.info("gnocchi list was empty") + return False + for line in stdout.splitlines(): + line = line.replace('|', "") + metric_id = line.split()[0] + stdout = node.run_cmd( + 'source overcloudrc.v3;gnocchi measures show {}'.format( + metric_id)) + if stdout is None: + self.__logger.info("gnocchi list was empty") + return False + for line in stdout.splitlines()[3: -1]: + if line[0] == '+': + pass + else: + timestamps1 = line.replace('|', "") + timestamps1 = timestamps1.split()[0] + time.sleep(sleep_time) + stdout = node.run_cmd( + "source overcloudrc.v3;gnocchi measures show {}".format( + metric_id)) + if stdout is None: + self.__logger.info("gnocchi measures was empty") + return False + for line in stdout.splitlines()[3:-1]: + if line[0] == '+': + pass + else: + timestamps2 = line.replace('|', "") + timestamps2 = timestamps2.split()[0] + if timestamps1 == timestamps2: + self.__logger.info( + "Plugin Interval is {}" .format(plugin_interval)) + self.__logger.info( + "Data not updated after {} seconds".format( + sleep_time)) + return False + else: + self.__logger.info("PASS") + return True + return False + + def test_plugins_with_snmp( + self, compute, plugin_interval, logger, plugin, snmp_mib_files=[], + snmp_mib_strings=[], snmp_in_commands=[]): + + if plugin == 'hugepages' or 'intel_rdt' or 'mcelog': + nodes = get_apex_nodes() + for node in nodes: + if compute == node.get_dict()['name']: + stdout = node.run_cmd( + 'snmpwalk -v2c -m {0} -c public localhost {1}' .format( + snmp_mib_files, snmp_mib_strings)) + self.__logger.info("{}" .format(stdout)) + if stdout is None: + self.__logger.info("No output from snmpwalk") + return False + elif 'OID' in stdout: + self.__logger.info("SNMP query failed") + return False + else: + counter1 = stdout.split()[3] + time.sleep(10) + stdout = node.run_cmd( + 'snmpwalk -v2c -m {0} -c public localhost {1}' .format( + snmp_mib_files, snmp_mib_strings)) + self.__logger.info("{}" .format(stdout)) + if stdout is None: + self.__logger.info("No output from snmpwalk") + elif 'OID' in stdout: + self.__logger.info( + "SNMP query failed during second check") + self.__logger.info("waiting for 10 sec") + time.sleep(10) + stdout = node.run_cmd( + 'snmpwalk -v2c -m {0} -c public localhost {1}' .format( + snmp_mib_files, snmp_mib_strings)) + self.__logger.info("{}" .format(stdout)) + if stdout is None: + self.__logger.info("No output from snmpwalk") + elif 'OID' in stdout: + self.__logger.info("SNMP query failed again") + self.__logger.info("Failing this test case") + return False + else: + counter2 = stdout.split()[3] + + if counter1 == counter2: + return False + else: + return True + else: + return False |