aboutsummaryrefslogtreecommitdiffstats
path: root/baro_tests/config_server.py
diff options
context:
space:
mode:
Diffstat (limited to 'baro_tests/config_server.py')
-rw-r--r--baro_tests/config_server.py360
1 files changed, 256 insertions, 104 deletions
diff --git a/baro_tests/config_server.py b/baro_tests/config_server.py
index 358a8ffe..efe2691a 100644
--- a/baro_tests/config_server.py
+++ b/baro_tests/config_server.py
@@ -1,7 +1,6 @@
-"""Classes used by client.py"""
# -*- coding: utf-8 -*-
-#Licensed under the Apache License, Version 2.0 (the "License"); you may
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
@@ -13,31 +12,33 @@
# License for the specific language governing permissions and limitations
# under the License.
+"""Classes used by collectd.py"""
+
import paramiko
import time
import string
import os.path
-
+import os
+import re
ID_RSA_PATH = '/home/opnfv/.ssh/id_rsa'
SSH_KEYS_SCRIPT = '/home/opnfv/barometer/baro_utils/get_ssh_keys.sh'
DEF_PLUGIN_INTERVAL = 10
-COLLECTD_CONF = '/etc/collectd/collectd.conf'
+COLLECTD_CONF = '/etc/collectd.conf'
COLLECTD_CONF_DIR = '/etc/collectd/collectd.conf.d'
+NOTIFICATION_FILE = '/var/log/python-notifications.dump'
+COLLECTD_NOTIFICATION = '/etc/collectd_notification_dump.py'
class Node(object):
"""Node configuration class"""
def __init__(self, attrs):
- self.__id = int(attrs[0])
- self.__status = attrs[1]
+ self.__null = attrs[0]
+ self.__id = attrs[1]
self.__name = attrs[2]
- self.__cluster = int(attrs[3]) if attrs[3] else None
- self.__ip = attrs[4]
- self.__mac = attrs[5]
- self.__roles = [x.strip(' ') for x in attrs[6].split(',')]
- self.__pending_roles = attrs[7]
- self.__online = int(attrs[8]) if attrs[3] and attrs[8]else None
- self.__group_id = int(attrs[9]) if attrs[3] else None
+ self.__status = attrs[3] if attrs[3] else None
+ self.__taskState = attrs[4]
+ self.__pwrState = attrs[5]
+ self.__ip = re.sub('^[a-z]+=', '', attrs[6])
def get_name(self):
"""Get node name"""
@@ -52,68 +53,84 @@ class Node(object):
return self.__ip
def get_roles(self):
- """Get node roles"""
+ """Get node role"""
return self.__roles
class ConfigServer(object):
"""Class to get env configuration"""
- def __init__(self, host, user, logger, passwd=None):
+ def __init__(self, host, user, logger, priv_key=None):
self.__host = host
self.__user = user
- self.__passwd = passwd
- self.__priv_key = None
+ self.__passwd = None
+ self.__priv_key = priv_key
self.__nodes = list()
self.__logger = logger
self.__private_key_file = ID_RSA_PATH
if not os.path.isfile(self.__private_key_file):
self.__logger.error(
- "Private key file '{}' not found.".format(self.__private_key_file))
- raise IOError("Private key file '{}' not found.".format(self.__private_key_file))
+ "Private key file '{}'".format(self.__private_key_file)
+ + " not found.")
+ raise IOError("Private key file '{}' not found.".format(
+ self.__private_key_file))
# get list of available nodes
- ssh, sftp = self.__open_sftp_session(self.__host, self.__user, self.__passwd)
+ ssh, sftp = self.__open_sftp_session(
+ self.__host, self.__user, self.__passwd)
attempt = 1
fuel_node_passed = False
while (attempt <= 10) and not fuel_node_passed:
- stdin, stdout, stderr = ssh.exec_command("fuel node")
+ stdin, stdout, stderr = ssh.exec_command(
+ "source stackrc; nova list")
stderr_lines = stderr.readlines()
if stderr_lines:
- self.__logger.warning("'fuel node' command failed (try {}):".format(attempt))
+ self.__logger.warning(
+ "'fuel node' command failed (try {}):".format(attempt))
for line in stderr_lines:
self.__logger.debug(line.strip())
else:
fuel_node_passed = True
if attempt > 1:
- self.__logger.info("'fuel node' command passed (try {})".format(attempt))
+ self.__logger.info(
+ "'fuel node' command passed (try {})".format(attempt))
attempt += 1
if not fuel_node_passed:
- self.__logger.error("'fuel node' command failed. This was the last try.")
- raise OSError("'fuel node' command failed. This was the last try.")
+ self.__logger.error(
+ "'fuel node' command failed. This was the last try.")
+ raise OSError(
+ "'fuel node' command failed. This was the last try.")
node_table = stdout.readlines()\
# skip table title and parse table values
- for entry in node_table[2:]:
- self.__nodes.append(Node([str(x.strip(' \n')) for x in entry.split('|')]))
+
+ for entry in node_table[3:]:
+ if entry[0] == '+' or entry[0] == '\n':
+ print entry
+ pass
+ else:
+ self.__nodes.append(
+ Node([str(x.strip(' \n')) for x in entry.split('|')]))
def get_controllers(self):
- """Get list of controllers"""
- return [node for node in self.__nodes if 'controller' in node.get_roles()]
+ # Get list of controllers
+ print self.__nodes[0]._Node__ip
+ return (
+ [node for node in self.__nodes if 'controller' in node.get_name()])
def get_computes(self):
- """Get list of computes"""
- return [node for node in self.__nodes if 'compute' in node.get_roles()]
+ # Get list of computes
+ return (
+ [node for node in self.__nodes if 'compute' in node.get_name()])
def get_nodes(self):
- """Get list of nodes"""
+ # Get list of nodes
return self.__nodes
def __open_sftp_session(self, host, user, passwd=None):
- """Connect to given host.
-
- Keyword arguments:
+ # Connect to given host.
+ """Keyword arguments:
host -- host to connect
user -- user to use
passwd -- password to use
@@ -127,10 +144,12 @@ class ConfigServer(object):
# try a direct access using password or private key
if not passwd and not self.__priv_key:
# get private key
- self.__priv_key = paramiko.RSAKey.from_private_key_file(self.__private_key_file)
+ self.__priv_key = paramiko.RSAKey.from_private_key_file(
+ self.__private_key_file)
# connect to the server
- ssh.connect(host, username=user, password=passwd, pkey=self.__priv_key)
+ ssh.connect(
+ host, username=user, password=passwd, pkey=self.__priv_key)
sftp = ssh.open_sftp()
# return SFTP client instance
@@ -144,12 +163,14 @@ class ConfigServer(object):
plugin -- plug-in name
If found, return interval value, otherwise the default value"""
- ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
+ ssh, sftp = self.__open_sftp_session(
+ compute.get_ip(), 'root', 'opnfvapex')
in_plugin = False
plugin_name = ''
default_interval = DEF_PLUGIN_INTERVAL
- config_files = [COLLECTD_CONF] \
- + [COLLECTD_CONF_DIR + '/' + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)]
+ config_files = [COLLECTD_CONF] + [
+ COLLECTD_CONF_DIR + '/'
+ + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)]
for config_file in config_files:
try:
with sftp.open(config_file) as config:
@@ -178,13 +199,15 @@ class ConfigServer(object):
parameter -- plug-in parameter
Return list of found values."""
- ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
+ ssh, sftp = self.__open_sftp_session(
+ compute.get_ip(), 'root', 'opnfvapex')
# find the plugin value
in_plugin = False
plugin_name = ''
default_values = []
- config_files = [COLLECTD_CONF] \
- + [COLLECTD_CONF_DIR + '/' + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)]
+ config_files = [COLLECTD_CONF] + [
+ COLLECTD_CONF_DIR + '/'
+ + conf_file for conf_file in sftp.listdir(COLLECTD_CONF_DIR)]
for config_file in config_files:
try:
with sftp.open(config_file) as config:
@@ -210,12 +233,13 @@ class ConfigServer(object):
host_ip -- IP of the node
ssh -- existing open SSH session to use
- One of host_ip or ssh must not be None. If both are not None, existing ssh session is used.
+ One of host_ip or ssh must not be None. If both are not None,
+ existing ssh session is used.
"""
if host_ip is None and ssh is None:
raise ValueError('One of host_ip or ssh must not be None.')
if ssh is None:
- ssh, sftp = self.__open_sftp_session(host_ip, 'root')
+ ssh, sftp = self.__open_sftp_session(host_ip, 'root', 'opnfvapex')
stdin, stdout, stderr = ssh.exec_command(command)
return stdout.readlines()
@@ -228,23 +252,22 @@ class ConfigServer(object):
stdout = self.execute_command("ovs-vsctl list-br", compute.get_ip())
return [interface.strip() for interface in stdout]
- def is_ceilometer_running(self, controller):
- """Check whether Ceilometer is running on controller.
+ def is_gnocchi_running(self, controller):
+ """Check whether Gnocchi is running on controller.
Keyword arguments:
controller -- controller node instance
- Return boolean value whether Ceilometer is running.
+ Return boolean value whether Gnocchi is running.
"""
- lines = self.execute_command('service --status-all | grep ceilometer', controller.get_ip())
- agent = False
- collector = False
+ gnocchi_present = False
+ lines = self.execute_command(
+ 'source overcloudrc.v3;openstack service list | grep gnocchi',
+ controller.get_ip())
for line in lines:
- if '[ + ] ceilometer-agent-notification' in line:
- agent = True
- if '[ + ] ceilometer-collector' in line:
- collector = True
- return agent and collector
+ if 'gnocchi' in line:
+ gnocchi_present = True
+ return not gnocchi_present
def is_installed(self, compute, package):
"""Check whether package exists on compute node.
@@ -255,36 +278,101 @@ class ConfigServer(object):
Return boolean value whether package is installed.
"""
- stdout = self.execute_command('dpkg -l | grep {}'.format(package), compute.get_ip())
+ stdout = self.execute_command(
+ 'yum list installed | grep {}'.format(package),
+ compute.get_ip())
return len(stdout) > 0
+ def is_libpqos_on_node(self, compute):
+ """Check whether libpqos is present on compute node"""
+ ssh, sftp = self.__open_sftp_session(
+ compute.get_ip(), 'root', 'opnfvapex')
+ stdin, stdout, stderr = \
+ ssh.exec_command("ls /usr/local/lib/ | grep libpqos")
+ output = stdout.readlines()
+ for lib in output:
+ if 'libpqos' in lib:
+ return True
+ return False
+
+ def check_gnocchi_plugin_included(self, compute):
+ """Check if gnocchi plugin is included in collectd.conf file.
+ If not, try to enable it.
+
+ Keyword arguments:
+ compute -- compute node instance
+
+ Return boolean value whether gnocchi plugin is included
+ or it's enabling was successful.
+ """
+ ssh, sftp = self.__open_sftp_session(
+ compute.get_ip(), 'root', 'opnfvapex')
+ try:
+ config = sftp.open(COLLECTD_CONF, mode='r')
+ except IOError:
+ self.__logger.error(
+ 'Cannot open {} on node {}'.format(
+ COLLECTD_CONF, compute.get_name()))
+ return False
+ in_lines = config.readlines()
+ out_lines = in_lines[:]
+ include_section_indexes = [
+ (start, end) for start in range(len(in_lines))
+ for end in range(len(in_lines))
+ if (start < end)
+ and '<Include' in in_lines[start]
+ and COLLECTD_CONF_DIR in in_lines[start]
+ and '#' not in in_lines[start]
+ and '</Include>' in in_lines[end]
+ and '#' not in in_lines[end]
+ and len([
+ i for i in in_lines[start + 1: end]
+ if 'Filter' in i and '*.conf' in i and '#' not in i]) > 0]
+ if len(include_section_indexes) == 0:
+ out_lines.append('<Include "{}">\n'.format(COLLECTD_CONF_DIR))
+ out_lines.append(' Filter "*.conf"\n')
+ out_lines.append('</Include>\n')
+ config.close()
+ config = sftp.open(COLLECTD_CONF, mode='w')
+ config.writelines(out_lines)
+ config.close()
+ self.__logger.info('Creating backup of collectd.conf...')
+ config = sftp.open(COLLECTD_CONF + '.backup', mode='w')
+ config.writelines(in_lines)
+ config.close()
+ return True
+
def check_ceil_plugin_included(self, compute):
- """Check if ceilometer plugin is included in collectd.conf file If not,
- try to enable it.
+ """Check if ceilometer plugin is included in collectd.conf file.
+ If not, try to enable it.
Keyword arguments:
compute -- compute node instance
- Return boolean value whether ceilometer plugin is included or it's enabling was successful.
+ Return boolean value whether ceilometer plugin is included
+ or it's enabling was successful.
"""
ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
try:
config = sftp.open(COLLECTD_CONF, mode='r')
except IOError:
self.__logger.error(
- 'Cannot open {} on node {}'.format(COLLECTD_CONF, compute.get_id()))
+ 'Cannot open {} on node {}'.format(
+ COLLECTD_CONF, compute.get_id()))
return False
in_lines = config.readlines()
out_lines = in_lines[:]
include_section_indexes = [
- (start, end) for start in range(len(in_lines)) for end in range(len(in_lines))
+ (start, end) for start in range(len(in_lines))
+ for end in range(len(in_lines))
if (start < end)
and '<Include' in in_lines[start]
and COLLECTD_CONF_DIR in in_lines[start]
and '#' not in in_lines[start]
and '</Include>' in in_lines[end]
and '#' not in in_lines[end]
- and len([i for i in in_lines[start + 1: end]
+ and len([
+ i for i in in_lines[start + 1: end]
if 'Filter' in i and '*.conf' in i and '#' not in i]) > 0]
if len(include_section_indexes) == 0:
out_lines.append('<Include "{}">\n'.format(COLLECTD_CONF_DIR))
@@ -300,41 +388,50 @@ class ConfigServer(object):
config.close()
return True
- def enable_plugins(self, compute, plugins, error_plugins, create_backup=True):
+ def enable_plugins(
+ self, compute, plugins, error_plugins, create_backup=True):
"""Enable plugins on compute node
Keyword arguments:
compute -- compute node instance
plugins -- list of plugins to be enabled
- error_plugins -- list of tuples with found errors, new entries may be added there
- (plugin, error_description, is_critical):
+ error_plugins -- list of tuples with found errors, new entries
+ may be added there (plugin, error_description, is_critical):
plugin -- plug-in name
error_decription -- description of the error
- is_critical -- boolean value indicating whether error is critical
- create_backup -- boolean value indicating whether backup shall be created
+ is_critical -- boolean value indicating whether error
+ is critical
+ create_backup -- boolean value indicating whether backup
+ shall be created
Return boolean value indicating whether function was successful.
"""
plugins = sorted(plugins)
- ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
+ ssh, sftp = self.__open_sftp_session(
+ compute.get_ip(), 'root', 'opnfvapex')
plugins_to_enable = plugins[:]
for plugin in plugins:
- plugin_file = '/usr/lib/collectd/{}.so'.format(plugin)
+ plugin_file = '/usr/lib64/collectd/{}.so'.format(plugin)
try:
sftp.stat(plugin_file)
except IOError:
self.__logger.debug(
- 'Plugin file {0} not found on node {1}, plugin {2} will not be enabled'.format(
- plugin_file, compute.get_id(), plugin))
- error_plugins.append((plugin, 'plugin file {} not found'.format(plugin_file), True))
+ 'Plugin file {} not found on node'.format(plugin_file)
+ + ' {0}, plugin {1} will not be enabled'.format(
+ compute.get_name(), plugin))
+ error_plugins.append((
+ plugin, 'plugin file {} not found'.format(plugin_file),
+ True))
plugins_to_enable.remove(plugin)
- self.__logger.debug('Following plugins will be enabled on node {}: {}'.format(
- compute.get_id(), ', '.join(plugins_to_enable)))
+ self.__logger.debug(
+ 'Following plugins will be enabled on node {}: {}'.format(
+ compute.get_name(), ', '.join(plugins_to_enable)))
try:
config = sftp.open(COLLECTD_CONF, mode='r')
except IOError:
self.__logger.warning(
- 'Cannot open {} on node {}'.format(COLLECTD_CONF, compute.get_id()))
+ 'Cannot open {} on node {}'.format(
+ COLLECTD_CONF, compute.get_name()))
return False
in_lines = config.readlines()
out_lines = []
@@ -348,7 +445,8 @@ class ConfigServer(object):
for plugin in plugins_to_enable:
if plugin in line:
commented = '#' in line
- #list of uncommented lines which contain LoadPlugin for this plugin
+ # list of uncommented lines which contain LoadPlugin
+ # for this plugin
loadlines = [
ll for ll in in_lines if 'LoadPlugin' in ll
and plugin in ll and '#' not in ll]
@@ -358,7 +456,8 @@ class ConfigServer(object):
enabled_plugins.append(plugin)
error_plugins.append((
plugin, 'plugin not enabled in '
- + '{}, trying to enable it'.format(COLLECTD_CONF), False))
+ + '{}, trying to enable it'.format(
+ COLLECTD_CONF), False))
elif not commented:
if plugin not in enabled_plugins:
enabled_plugins.append(plugin)
@@ -366,15 +465,16 @@ class ConfigServer(object):
line = '#' + line
error_plugins.append((
plugin, 'plugin enabled more than once '
- + '(additional occurrence of LoadPlugin found in '
- + '{}), trying to comment it out.'.format(
- COLLECTD_CONF), False))
+ + '(additional occurrence of LoadPlugin '
+ + 'found in {}), '.format(COLLECTD_CONF)
+ + 'trying to comment it out.', False))
elif line.lstrip(string.whitespace + '#').find('<Plugin') == 0:
in_section += 1
for plugin in plugins_to_enable:
if plugin in line:
commented = '#' in line
- #list of uncommented lines which contain Plugin for this plugin
+ # list of uncommented lines which contain Plugin for
+ # this plugin
pluginlines = [
pl for pl in in_lines if '<Plugin' in pl
and plugin in pl and '#' not in pl]
@@ -385,8 +485,8 @@ class ConfigServer(object):
enabled_sections.append(plugin)
error_plugins.append((
plugin, 'plugin section found in '
- + '{}, but commented out, trying to uncomment it.'.format(
- COLLECTD_CONF), False))
+ + '{}, but commented'.format(COLLECTD_CONF)
+ + ' out, trying to uncomment it.', False))
elif not commented:
if plugin not in enabled_sections:
enabled_sections.append(plugin)
@@ -394,10 +494,10 @@ class ConfigServer(object):
line = '#' + line
comment_section = True
error_plugins.append((
- plugin,
- 'additional occurrence of plugin section found in '
- + '{}, trying to comment it out.'.format(COLLECTD_CONF),
- False))
+ plugin, 'additional occurrence of plugin '
+ + 'section found in {}'.format(
+ COLLECTD_CONF)
+ + ', trying to comment it out.', False))
elif in_section > 0:
if comment_section and '#' not in line:
line = '#' + line
@@ -411,8 +511,8 @@ class ConfigServer(object):
elif '</Plugin>' in line:
self.__logger.error(
'Unexpected closure os plugin section on line'
- + ' {} in collectd.conf, matching section start not found.'.format(
- len(out_lines) + 1))
+ + ' {} in collectd.conf'.format(len(out_lines) + 1)
+ + ', matching section start not found.')
return False
out_lines.append(line)
if in_section > 0:
@@ -426,14 +526,14 @@ class ConfigServer(object):
for plugin in plugins_to_enable:
if plugin not in enabled_plugins:
error_plugins.append((
- plugin,
- 'plugin not enabled in {}, trying to enable it.'.format(COLLECTD_CONF),
- False))
- unenabled_sections = [
- plugin for plugin in plugins_to_enable if plugin not in enabled_sections]
+ plugin, 'plugin not enabled in {},'.format(COLLECTD_CONF)
+ + ' trying to enable it.', False))
+ unenabled_sections = [plugin for plugin in plugins_to_enable
+ if plugin not in enabled_sections]
if unenabled_sections:
- self.__logger.error('Plugin sections for following plugins not found: {}'.format(
- ', '.join(unenabled_sections)))
+ self.__logger.error(
+ 'Plugin sections for following plugins not found: {}'.format(
+ ', '.join(unenabled_sections)))
return False
config.close()
@@ -446,7 +546,8 @@ class ConfigServer(object):
config = sftp.open(COLLECTD_CONF, mode='w')
config.writelines(out_lines)
config.close()
- diff_command = "diff {} {}.backup".format(COLLECTD_CONF, COLLECTD_CONF)
+ diff_command = \
+ "diff {} {}.backup".format(COLLECTD_CONF, COLLECTD_CONF)
stdin, stdout, stderr = ssh.exec_command(diff_command)
self.__logger.debug(diff_command)
for line in stdout.readlines():
@@ -459,7 +560,8 @@ class ConfigServer(object):
Keyword arguments:
compute -- compute node instance
"""
- ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
+ ssh, sftp = self.__open_sftp_session(
+ compute.get_ip(), 'root', 'opnfvapex')
self.__logger.info('Restoring config file from backup...')
ssh.exec_command("cp {0} {0}.used".format(COLLECTD_CONF))
@@ -471,20 +573,23 @@ class ConfigServer(object):
Keyword arguments:
compute -- compute node instance
- Retrun tuple with boolean indicating success and list of warnings received
- during collectd start.
+ Retrun tuple with boolean indicating success and list of warnings
+ received during collectd start.
"""
def get_collectd_processes(ssh_session):
"""Get number of running collectd processes.
Keyword arguments:
- ssh_session -- instance of SSH session in which to check for processes
+ ssh_session -- instance of SSH session in which to check
+ for processes
"""
- stdin, stdout, stderr = ssh_session.exec_command("pgrep collectd")
+ stdin, stdout, stderr = ssh_session.exec_command(
+ "pgrep collectd")
return len(stdout.readlines())
- ssh, sftp = self.__open_sftp_session(compute.get_ip(), 'root')
+ ssh, sftp = self.__open_sftp_session(
+ compute.get_ip(), 'root', 'opnfvapex')
self.__logger.info('Stopping collectd service...')
stdout = self.execute_command("service collectd stop", ssh=ssh)
@@ -500,3 +605,50 @@ class ConfigServer(object):
self.__logger.error('Collectd is still not running...')
return False, warning
return True, warning
+
+ def test_gnocchi_is_sending_data(self, controller):
+ """ Checking if Gnocchi is sending metrics to controller"""
+ metric_ids = []
+ timestamps1 = {}
+ timestamps2 = {}
+ ssh, sftp = self.__open_sftp_session(
+ controller.get_ip(), 'root', 'opnfvapex')
+
+ self.__logger.info('Getting gnocchi metric list on{}'.format(
+ controller.get_name()))
+ stdout = self.execute_command(
+ "source overcloudrc.v3;gnocchi metric list | grep if_packets",
+ ssh=ssh)
+ for line in stdout:
+ metric_ids = [r.split('|')[1] for r in stdout]
+ self.__logger.info("Metric ids = {}" .format(metric_ids))
+ for metric_id in metric_ids:
+ metric_id = metric_id.replace("u", "")
+ stdout = self.execute_command(
+ "source overcloudrc.v3;gnocchi measures show {}" .format(
+ metric_id), ssh=ssh)
+ self.__logger.info("stdout measures ={}" .format(stdout))
+ for line in stdout:
+ if line[0] == '+':
+ pass
+ else:
+ self.__logger.info("Line = {}" .format(line))
+ timestamps1 = [line.split('|')[1]]
+ self.__logger.info("Last line timetamp1 = {}" .format(timestamps1))
+ time.sleep(10)
+ stdout = self.execute_command(
+ "source overcloudrc.v3;gnocchi measures show {}" .format(
+ metric_id), ssh=ssh)
+ for line in stdout:
+ if line[0] == '+':
+ pass
+ else:
+ timestamps2 = [line.split('|')[1]]
+ self.__logger.info("Last line timetamp2 = {}" .format(timestamps2))
+ if timestamps1 == timestamps2:
+ self.__logger.info("False")
+ # return False
+ return True
+ else:
+ self.__logger.info("True")
+ return True