aboutsummaryrefslogtreecommitdiffstats
path: root/charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet
diff options
context:
space:
mode:
Diffstat (limited to 'charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet')
-rw-r--r--charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/__init__.py15
-rw-r--r--charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/deployment.py95
-rw-r--r--charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/utils.py829
3 files changed, 939 insertions, 0 deletions
diff --git a/charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/__init__.py b/charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/__init__.py
new file mode 100644
index 0000000..d1400a0
--- /dev/null
+++ b/charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/__init__.py
@@ -0,0 +1,15 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
diff --git a/charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/deployment.py b/charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/deployment.py
new file mode 100644
index 0000000..d451698
--- /dev/null
+++ b/charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/deployment.py
@@ -0,0 +1,95 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
+
+import amulet
+import os
+import six
+
+
+class AmuletDeployment(object):
+ """Amulet deployment.
+
+ This class provides generic Amulet deployment and test runner
+ methods.
+ """
+
+ def __init__(self, series=None):
+ """Initialize the deployment environment."""
+ self.series = None
+
+ if series:
+ self.series = series
+ self.d = amulet.Deployment(series=self.series)
+ else:
+ self.d = amulet.Deployment()
+
+ def _add_services(self, this_service, other_services):
+ """Add services.
+
+ Add services to the deployment where this_service is the local charm
+ that we're testing and other_services are the other services that
+ are being used in the local amulet tests.
+ """
+ if this_service['name'] != os.path.basename(os.getcwd()):
+ s = this_service['name']
+ msg = "The charm's root directory name needs to be {}".format(s)
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ if 'units' not in this_service:
+ this_service['units'] = 1
+
+ self.d.add(this_service['name'], units=this_service['units'],
+ constraints=this_service.get('constraints'))
+
+ for svc in other_services:
+ if 'location' in svc:
+ branch_location = svc['location']
+ elif self.series:
+ branch_location = 'cs:{}/{}'.format(self.series, svc['name']),
+ else:
+ branch_location = None
+
+ if 'units' not in svc:
+ svc['units'] = 1
+
+ self.d.add(svc['name'], charm=branch_location, units=svc['units'],
+ constraints=svc.get('constraints'))
+
+ def _add_relations(self, relations):
+ """Add all of the relations for the services."""
+ for k, v in six.iteritems(relations):
+ self.d.relate(k, v)
+
+ def _configure_services(self, configs):
+ """Configure all of the services."""
+ for service, config in six.iteritems(configs):
+ self.d.configure(service, config)
+
+ def _deploy(self):
+ """Deploy environment and wait for all hooks to finish executing."""
+ try:
+ self.d.setup(timeout=900)
+ self.d.sentry.wait(timeout=900)
+ except amulet.helpers.TimeoutError:
+ amulet.raise_status(amulet.FAIL, msg="Deployment timed out")
+ except Exception:
+ raise
+
+ def run_tests(self):
+ """Run all of the methods that are prefixed with 'test_'."""
+ for test in dir(self):
+ if test.startswith('test_'):
+ getattr(self, test)()
diff --git a/charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/utils.py b/charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/utils.py
new file mode 100644
index 0000000..7e5c25a
--- /dev/null
+++ b/charms/trusty/ceilometer/tests/charmhelpers/contrib/amulet/utils.py
@@ -0,0 +1,829 @@
+# Copyright 2014-2015 Canonical Limited.
+#
+# This file is part of charm-helpers.
+#
+# charm-helpers is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# charm-helpers is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
+
+import io
+import json
+import logging
+import os
+import re
+import socket
+import subprocess
+import sys
+import time
+import uuid
+
+import amulet
+import distro_info
+import six
+from six.moves import configparser
+if six.PY3:
+ from urllib import parse as urlparse
+else:
+ import urlparse
+
+
+class AmuletUtils(object):
+ """Amulet utilities.
+
+ This class provides common utility functions that are used by Amulet
+ tests.
+ """
+
+ def __init__(self, log_level=logging.ERROR):
+ self.log = self.get_logger(level=log_level)
+ self.ubuntu_releases = self.get_ubuntu_releases()
+
+ def get_logger(self, name="amulet-logger", level=logging.DEBUG):
+ """Get a logger object that will log to stdout."""
+ log = logging
+ logger = log.getLogger(name)
+ fmt = log.Formatter("%(asctime)s %(funcName)s "
+ "%(levelname)s: %(message)s")
+
+ handler = log.StreamHandler(stream=sys.stdout)
+ handler.setLevel(level)
+ handler.setFormatter(fmt)
+
+ logger.addHandler(handler)
+ logger.setLevel(level)
+
+ return logger
+
+ def valid_ip(self, ip):
+ if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", ip):
+ return True
+ else:
+ return False
+
+ def valid_url(self, url):
+ p = re.compile(
+ r'^(?:http|ftp)s?://'
+ r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' # noqa
+ r'localhost|'
+ r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})'
+ r'(?::\d+)?'
+ r'(?:/?|[/?]\S+)$',
+ re.IGNORECASE)
+ if p.match(url):
+ return True
+ else:
+ return False
+
+ def get_ubuntu_release_from_sentry(self, sentry_unit):
+ """Get Ubuntu release codename from sentry unit.
+
+ :param sentry_unit: amulet sentry/service unit pointer
+ :returns: list of strings - release codename, failure message
+ """
+ msg = None
+ cmd = 'lsb_release -cs'
+ release, code = sentry_unit.run(cmd)
+ if code == 0:
+ self.log.debug('{} lsb_release: {}'.format(
+ sentry_unit.info['unit_name'], release))
+ else:
+ msg = ('{} `{}` returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, release, code))
+ if release not in self.ubuntu_releases:
+ msg = ("Release ({}) not found in Ubuntu releases "
+ "({})".format(release, self.ubuntu_releases))
+ return release, msg
+
+ def validate_services(self, commands):
+ """Validate that lists of commands succeed on service units. Can be
+ used to verify system services are running on the corresponding
+ service units.
+
+ :param commands: dict with sentry keys and arbitrary command list vals
+ :returns: None if successful, Failure string message otherwise
+ """
+ self.log.debug('Checking status of system services...')
+
+ # /!\ DEPRECATION WARNING (beisner):
+ # New and existing tests should be rewritten to use
+ # validate_services_by_name() as it is aware of init systems.
+ self.log.warn('DEPRECATION WARNING: use '
+ 'validate_services_by_name instead of validate_services '
+ 'due to init system differences.')
+
+ for k, v in six.iteritems(commands):
+ for cmd in v:
+ output, code = k.run(cmd)
+ self.log.debug('{} `{}` returned '
+ '{}'.format(k.info['unit_name'],
+ cmd, code))
+ if code != 0:
+ return "command `{}` returned {}".format(cmd, str(code))
+ return None
+
+ def validate_services_by_name(self, sentry_services):
+ """Validate system service status by service name, automatically
+ detecting init system based on Ubuntu release codename.
+
+ :param sentry_services: dict with sentry keys and svc list values
+ :returns: None if successful, Failure string message otherwise
+ """
+ self.log.debug('Checking status of system services...')
+
+ # Point at which systemd became a thing
+ systemd_switch = self.ubuntu_releases.index('vivid')
+
+ for sentry_unit, services_list in six.iteritems(sentry_services):
+ # Get lsb_release codename from unit
+ release, ret = self.get_ubuntu_release_from_sentry(sentry_unit)
+ if ret:
+ return ret
+
+ for service_name in services_list:
+ if (self.ubuntu_releases.index(release) >= systemd_switch or
+ service_name in ['rabbitmq-server', 'apache2']):
+ # init is systemd (or regular sysv)
+ cmd = 'sudo service {} status'.format(service_name)
+ output, code = sentry_unit.run(cmd)
+ service_running = code == 0
+ elif self.ubuntu_releases.index(release) < systemd_switch:
+ # init is upstart
+ cmd = 'sudo status {}'.format(service_name)
+ output, code = sentry_unit.run(cmd)
+ service_running = code == 0 and "start/running" in output
+
+ self.log.debug('{} `{}` returned '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code))
+ if not service_running:
+ return u"command `{}` returned {} {}".format(
+ cmd, output, str(code))
+ return None
+
+ def _get_config(self, unit, filename):
+ """Get a ConfigParser object for parsing a unit's config file."""
+ file_contents = unit.file_contents(filename)
+
+ # NOTE(beisner): by default, ConfigParser does not handle options
+ # with no value, such as the flags used in the mysql my.cnf file.
+ # https://bugs.python.org/issue7005
+ config = configparser.ConfigParser(allow_no_value=True)
+ config.readfp(io.StringIO(file_contents))
+ return config
+
+ def validate_config_data(self, sentry_unit, config_file, section,
+ expected):
+ """Validate config file data.
+
+ Verify that the specified section of the config file contains
+ the expected option key:value pairs.
+
+ Compare expected dictionary data vs actual dictionary data.
+ The values in the 'expected' dictionary can be strings, bools, ints,
+ longs, or can be a function that evaluates a variable and returns a
+ bool.
+ """
+ self.log.debug('Validating config file data ({} in {} on {})'
+ '...'.format(section, config_file,
+ sentry_unit.info['unit_name']))
+ config = self._get_config(sentry_unit, config_file)
+
+ if section != 'DEFAULT' and not config.has_section(section):
+ return "section [{}] does not exist".format(section)
+
+ for k in expected.keys():
+ if not config.has_option(section, k):
+ return "section [{}] is missing option {}".format(section, k)
+
+ actual = config.get(section, k)
+ v = expected[k]
+ if (isinstance(v, six.string_types) or
+ isinstance(v, bool) or
+ isinstance(v, six.integer_types)):
+ # handle explicit values
+ if actual != v:
+ return "section [{}] {}:{} != expected {}:{}".format(
+ section, k, actual, k, expected[k])
+ # handle function pointers, such as not_null or valid_ip
+ elif not v(actual):
+ return "section [{}] {}:{} != expected {}:{}".format(
+ section, k, actual, k, expected[k])
+ return None
+
+ def _validate_dict_data(self, expected, actual):
+ """Validate dictionary data.
+
+ Compare expected dictionary data vs actual dictionary data.
+ The values in the 'expected' dictionary can be strings, bools, ints,
+ longs, or can be a function that evaluates a variable and returns a
+ bool.
+ """
+ self.log.debug('actual: {}'.format(repr(actual)))
+ self.log.debug('expected: {}'.format(repr(expected)))
+
+ for k, v in six.iteritems(expected):
+ if k in actual:
+ if (isinstance(v, six.string_types) or
+ isinstance(v, bool) or
+ isinstance(v, six.integer_types)):
+ # handle explicit values
+ if v != actual[k]:
+ return "{}:{}".format(k, actual[k])
+ # handle function pointers, such as not_null or valid_ip
+ elif not v(actual[k]):
+ return "{}:{}".format(k, actual[k])
+ else:
+ return "key '{}' does not exist".format(k)
+ return None
+
+ def validate_relation_data(self, sentry_unit, relation, expected):
+ """Validate actual relation data based on expected relation data."""
+ actual = sentry_unit.relation(relation[0], relation[1])
+ return self._validate_dict_data(expected, actual)
+
+ def _validate_list_data(self, expected, actual):
+ """Compare expected list vs actual list data."""
+ for e in expected:
+ if e not in actual:
+ return "expected item {} not found in actual list".format(e)
+ return None
+
+ def not_null(self, string):
+ if string is not None:
+ return True
+ else:
+ return False
+
+ def _get_file_mtime(self, sentry_unit, filename):
+ """Get last modification time of file."""
+ return sentry_unit.file_stat(filename)['mtime']
+
+ def _get_dir_mtime(self, sentry_unit, directory):
+ """Get last modification time of directory."""
+ return sentry_unit.directory_stat(directory)['mtime']
+
+ def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None):
+ """Get start time of a process based on the last modification time
+ of the /proc/pid directory.
+
+ :sentry_unit: The sentry unit to check for the service on
+ :service: service name to look for in process table
+ :pgrep_full: [Deprecated] Use full command line search mode with pgrep
+ :returns: epoch time of service process start
+ :param commands: list of bash commands
+ :param sentry_units: list of sentry unit pointers
+ :returns: None if successful; Failure message otherwise
+ """
+ if pgrep_full is not None:
+ # /!\ DEPRECATION WARNING (beisner):
+ # No longer implemented, as pidof is now used instead of pgrep.
+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+ self.log.warn('DEPRECATION WARNING: pgrep_full bool is no '
+ 'longer implemented re: lp 1474030.')
+
+ pid_list = self.get_process_id_list(sentry_unit, service)
+ pid = pid_list[0]
+ proc_dir = '/proc/{}'.format(pid)
+ self.log.debug('Pid for {} on {}: {}'.format(
+ service, sentry_unit.info['unit_name'], pid))
+
+ return self._get_dir_mtime(sentry_unit, proc_dir)
+
+ def service_restarted(self, sentry_unit, service, filename,
+ pgrep_full=None, sleep_time=20):
+ """Check if service was restarted.
+
+ Compare a service's start time vs a file's last modification time
+ (such as a config file for that service) to determine if the service
+ has been restarted.
+ """
+ # /!\ DEPRECATION WARNING (beisner):
+ # This method is prone to races in that no before-time is known.
+ # Use validate_service_config_changed instead.
+
+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+ # used instead of pgrep. pgrep_full is still passed through to ensure
+ # deprecation WARNS. lp1474030
+ self.log.warn('DEPRECATION WARNING: use '
+ 'validate_service_config_changed instead of '
+ 'service_restarted due to known races.')
+
+ time.sleep(sleep_time)
+ if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
+ self._get_file_mtime(sentry_unit, filename)):
+ return True
+ else:
+ return False
+
+ def service_restarted_since(self, sentry_unit, mtime, service,
+ pgrep_full=None, sleep_time=20,
+ retry_count=30, retry_sleep_time=10):
+ """Check if service was been started after a given time.
+
+ Args:
+ sentry_unit (sentry): The sentry unit to check for the service on
+ mtime (float): The epoch time to check against
+ service (string): service name to look for in process table
+ pgrep_full: [Deprecated] Use full command line search mode with pgrep
+ sleep_time (int): Initial sleep time (s) before looking for file
+ retry_sleep_time (int): Time (s) to sleep between retries
+ retry_count (int): If file is not found, how many times to retry
+
+ Returns:
+ bool: True if service found and its start time it newer than mtime,
+ False if service is older than mtime or if service was
+ not found.
+ """
+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+ # used instead of pgrep. pgrep_full is still passed through to ensure
+ # deprecation WARNS. lp1474030
+
+ unit_name = sentry_unit.info['unit_name']
+ self.log.debug('Checking that %s service restarted since %s on '
+ '%s' % (service, mtime, unit_name))
+ time.sleep(sleep_time)
+ proc_start_time = None
+ tries = 0
+ while tries <= retry_count and not proc_start_time:
+ try:
+ proc_start_time = self._get_proc_start_time(sentry_unit,
+ service,
+ pgrep_full)
+ self.log.debug('Attempt {} to get {} proc start time on {} '
+ 'OK'.format(tries, service, unit_name))
+ except IOError as e:
+ # NOTE(beisner) - race avoidance, proc may not exist yet.
+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+ self.log.debug('Attempt {} to get {} proc start time on {} '
+ 'failed\n{}'.format(tries, service,
+ unit_name, e))
+ time.sleep(retry_sleep_time)
+ tries += 1
+
+ if not proc_start_time:
+ self.log.warn('No proc start time found, assuming service did '
+ 'not start')
+ return False
+ if proc_start_time >= mtime:
+ self.log.debug('Proc start time is newer than provided mtime'
+ '(%s >= %s) on %s (OK)' % (proc_start_time,
+ mtime, unit_name))
+ return True
+ else:
+ self.log.warn('Proc start time (%s) is older than provided mtime '
+ '(%s) on %s, service did not '
+ 'restart' % (proc_start_time, mtime, unit_name))
+ return False
+
+ def config_updated_since(self, sentry_unit, filename, mtime,
+ sleep_time=20, retry_count=30,
+ retry_sleep_time=10):
+ """Check if file was modified after a given time.
+
+ Args:
+ sentry_unit (sentry): The sentry unit to check the file mtime on
+ filename (string): The file to check mtime of
+ mtime (float): The epoch time to check against
+ sleep_time (int): Initial sleep time (s) before looking for file
+ retry_sleep_time (int): Time (s) to sleep between retries
+ retry_count (int): If file is not found, how many times to retry
+
+ Returns:
+ bool: True if file was modified more recently than mtime, False if
+ file was modified before mtime, or if file not found.
+ """
+ unit_name = sentry_unit.info['unit_name']
+ self.log.debug('Checking that %s updated since %s on '
+ '%s' % (filename, mtime, unit_name))
+ time.sleep(sleep_time)
+ file_mtime = None
+ tries = 0
+ while tries <= retry_count and not file_mtime:
+ try:
+ file_mtime = self._get_file_mtime(sentry_unit, filename)
+ self.log.debug('Attempt {} to get {} file mtime on {} '
+ 'OK'.format(tries, filename, unit_name))
+ except IOError as e:
+ # NOTE(beisner) - race avoidance, file may not exist yet.
+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
+ self.log.debug('Attempt {} to get {} file mtime on {} '
+ 'failed\n{}'.format(tries, filename,
+ unit_name, e))
+ time.sleep(retry_sleep_time)
+ tries += 1
+
+ if not file_mtime:
+ self.log.warn('Could not determine file mtime, assuming '
+ 'file does not exist')
+ return False
+
+ if file_mtime >= mtime:
+ self.log.debug('File mtime is newer than provided mtime '
+ '(%s >= %s) on %s (OK)' % (file_mtime,
+ mtime, unit_name))
+ return True
+ else:
+ self.log.warn('File mtime is older than provided mtime'
+ '(%s < on %s) on %s' % (file_mtime,
+ mtime, unit_name))
+ return False
+
+ def validate_service_config_changed(self, sentry_unit, mtime, service,
+ filename, pgrep_full=None,
+ sleep_time=20, retry_count=30,
+ retry_sleep_time=10):
+ """Check service and file were updated after mtime
+
+ Args:
+ sentry_unit (sentry): The sentry unit to check for the service on
+ mtime (float): The epoch time to check against
+ service (string): service name to look for in process table
+ filename (string): The file to check mtime of
+ pgrep_full: [Deprecated] Use full command line search mode with pgrep
+ sleep_time (int): Initial sleep in seconds to pass to test helpers
+ retry_count (int): If service is not found, how many times to retry
+ retry_sleep_time (int): Time in seconds to wait between retries
+
+ Typical Usage:
+ u = OpenStackAmuletUtils(ERROR)
+ ...
+ mtime = u.get_sentry_time(self.cinder_sentry)
+ self.d.configure('cinder', {'verbose': 'True', 'debug': 'True'})
+ if not u.validate_service_config_changed(self.cinder_sentry,
+ mtime,
+ 'cinder-api',
+ '/etc/cinder/cinder.conf')
+ amulet.raise_status(amulet.FAIL, msg='update failed')
+ Returns:
+ bool: True if both service and file where updated/restarted after
+ mtime, False if service is older than mtime or if service was
+ not found or if filename was modified before mtime.
+ """
+
+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
+ # used instead of pgrep. pgrep_full is still passed through to ensure
+ # deprecation WARNS. lp1474030
+
+ service_restart = self.service_restarted_since(
+ sentry_unit, mtime,
+ service,
+ pgrep_full=pgrep_full,
+ sleep_time=sleep_time,
+ retry_count=retry_count,
+ retry_sleep_time=retry_sleep_time)
+
+ config_update = self.config_updated_since(
+ sentry_unit,
+ filename,
+ mtime,
+ sleep_time=sleep_time,
+ retry_count=retry_count,
+ retry_sleep_time=retry_sleep_time)
+
+ return service_restart and config_update
+
+ def get_sentry_time(self, sentry_unit):
+ """Return current epoch time on a sentry"""
+ cmd = "date +'%s'"
+ return float(sentry_unit.run(cmd)[0])
+
+ def relation_error(self, name, data):
+ return 'unexpected relation data in {} - {}'.format(name, data)
+
+ def endpoint_error(self, name, data):
+ return 'unexpected endpoint data in {} - {}'.format(name, data)
+
+ def get_ubuntu_releases(self):
+ """Return a list of all Ubuntu releases in order of release."""
+ _d = distro_info.UbuntuDistroInfo()
+ _release_list = _d.all
+ return _release_list
+
+ def file_to_url(self, file_rel_path):
+ """Convert a relative file path to a file URL."""
+ _abs_path = os.path.abspath(file_rel_path)
+ return urlparse.urlparse(_abs_path, scheme='file').geturl()
+
+ def check_commands_on_units(self, commands, sentry_units):
+ """Check that all commands in a list exit zero on all
+ sentry units in a list.
+
+ :param commands: list of bash commands
+ :param sentry_units: list of sentry unit pointers
+ :returns: None if successful; Failure message otherwise
+ """
+ self.log.debug('Checking exit codes for {} commands on {} '
+ 'sentry units...'.format(len(commands),
+ len(sentry_units)))
+ for sentry_unit in sentry_units:
+ for cmd in commands:
+ output, code = sentry_unit.run(cmd)
+ if code == 0:
+ self.log.debug('{} `{}` returned {} '
+ '(OK)'.format(sentry_unit.info['unit_name'],
+ cmd, code))
+ else:
+ return ('{} `{}` returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code, output))
+ return None
+
+ def get_process_id_list(self, sentry_unit, process_name,
+ expect_success=True):
+ """Get a list of process ID(s) from a single sentry juju unit
+ for a single process name.
+
+ :param sentry_unit: Amulet sentry instance (juju unit)
+ :param process_name: Process name
+ :param expect_success: If False, expect the PID to be missing,
+ raise if it is present.
+ :returns: List of process IDs
+ """
+ cmd = 'pidof -x {}'.format(process_name)
+ if not expect_success:
+ cmd += " || exit 0 && exit 1"
+ output, code = sentry_unit.run(cmd)
+ if code != 0:
+ msg = ('{} `{}` returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code, output))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+ return str(output).split()
+
+ def get_unit_process_ids(self, unit_processes, expect_success=True):
+ """Construct a dict containing unit sentries, process names, and
+ process IDs.
+
+ :param unit_processes: A dictionary of Amulet sentry instance
+ to list of process names.
+ :param expect_success: if False expect the processes to not be
+ running, raise if they are.
+ :returns: Dictionary of Amulet sentry instance to dictionary
+ of process names to PIDs.
+ """
+ pid_dict = {}
+ for sentry_unit, process_list in six.iteritems(unit_processes):
+ pid_dict[sentry_unit] = {}
+ for process in process_list:
+ pids = self.get_process_id_list(
+ sentry_unit, process, expect_success=expect_success)
+ pid_dict[sentry_unit].update({process: pids})
+ return pid_dict
+
+ def validate_unit_process_ids(self, expected, actual):
+ """Validate process id quantities for services on units."""
+ self.log.debug('Checking units for running processes...')
+ self.log.debug('Expected PIDs: {}'.format(expected))
+ self.log.debug('Actual PIDs: {}'.format(actual))
+
+ if len(actual) != len(expected):
+ return ('Unit count mismatch. expected, actual: {}, '
+ '{} '.format(len(expected), len(actual)))
+
+ for (e_sentry, e_proc_names) in six.iteritems(expected):
+ e_sentry_name = e_sentry.info['unit_name']
+ if e_sentry in actual.keys():
+ a_proc_names = actual[e_sentry]
+ else:
+ return ('Expected sentry ({}) not found in actual dict data.'
+ '{}'.format(e_sentry_name, e_sentry))
+
+ if len(e_proc_names.keys()) != len(a_proc_names.keys()):
+ return ('Process name count mismatch. expected, actual: {}, '
+ '{}'.format(len(expected), len(actual)))
+
+ for (e_proc_name, e_pids), (a_proc_name, a_pids) in \
+ zip(e_proc_names.items(), a_proc_names.items()):
+ if e_proc_name != a_proc_name:
+ return ('Process name mismatch. expected, actual: {}, '
+ '{}'.format(e_proc_name, a_proc_name))
+
+ a_pids_length = len(a_pids)
+ fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
+ '{}, {} ({})'.format(e_sentry_name, e_proc_name,
+ e_pids, a_pids_length,
+ a_pids))
+
+ # If expected is a list, ensure at least one PID quantity match
+ if isinstance(e_pids, list) and \
+ a_pids_length not in e_pids:
+ return fail_msg
+ # If expected is not bool and not list,
+ # ensure PID quantities match
+ elif not isinstance(e_pids, bool) and \
+ not isinstance(e_pids, list) and \
+ a_pids_length != e_pids:
+ return fail_msg
+ # If expected is bool True, ensure 1 or more PIDs exist
+ elif isinstance(e_pids, bool) and \
+ e_pids is True and a_pids_length < 1:
+ return fail_msg
+ # If expected is bool False, ensure 0 PIDs exist
+ elif isinstance(e_pids, bool) and \
+ e_pids is False and a_pids_length != 0:
+ return fail_msg
+ else:
+ self.log.debug('PID check OK: {} {} {}: '
+ '{}'.format(e_sentry_name, e_proc_name,
+ e_pids, a_pids))
+ return None
+
+ def validate_list_of_identical_dicts(self, list_of_dicts):
+ """Check that all dicts within a list are identical."""
+ hashes = []
+ for _dict in list_of_dicts:
+ hashes.append(hash(frozenset(_dict.items())))
+
+ self.log.debug('Hashes: {}'.format(hashes))
+ if len(set(hashes)) == 1:
+ self.log.debug('Dicts within list are identical')
+ else:
+ return 'Dicts within list are not identical'
+
+ return None
+
+ def validate_sectionless_conf(self, file_contents, expected):
+ """A crude conf parser. Useful to inspect configuration files which
+ do not have section headers (as would be necessary in order to use
+ the configparser). Such as openstack-dashboard or rabbitmq confs."""
+ for line in file_contents.split('\n'):
+ if '=' in line:
+ args = line.split('=')
+ if len(args) <= 1:
+ continue
+ key = args[0].strip()
+ value = args[1].strip()
+ if key in expected.keys():
+ if expected[key] != value:
+ msg = ('Config mismatch. Expected, actual: {}, '
+ '{}'.format(expected[key], value))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ def get_unit_hostnames(self, units):
+ """Return a dict of juju unit names to hostnames."""
+ host_names = {}
+ for unit in units:
+ host_names[unit.info['unit_name']] = \
+ str(unit.file_contents('/etc/hostname').strip())
+ self.log.debug('Unit host names: {}'.format(host_names))
+ return host_names
+
+ def run_cmd_unit(self, sentry_unit, cmd):
+ """Run a command on a unit, return the output and exit code."""
+ output, code = sentry_unit.run(cmd)
+ if code == 0:
+ self.log.debug('{} `{}` command returned {} '
+ '(OK)'.format(sentry_unit.info['unit_name'],
+ cmd, code))
+ else:
+ msg = ('{} `{}` command returned {} '
+ '{}'.format(sentry_unit.info['unit_name'],
+ cmd, code, output))
+ amulet.raise_status(amulet.FAIL, msg=msg)
+ return str(output), code
+
+ def file_exists_on_unit(self, sentry_unit, file_name):
+ """Check if a file exists on a unit."""
+ try:
+ sentry_unit.file_stat(file_name)
+ return True
+ except IOError:
+ return False
+ except Exception as e:
+ msg = 'Error checking file {}: {}'.format(file_name, e)
+ amulet.raise_status(amulet.FAIL, msg=msg)
+
+ def file_contents_safe(self, sentry_unit, file_name,
+ max_wait=60, fatal=False):
+ """Get file contents from a sentry unit. Wrap amulet file_contents
+ with retry logic to address races where a file checks as existing,
+ but no longer exists by the time file_contents is called.
+ Return None if file not found. Optionally raise if fatal is True."""
+ unit_name = sentry_unit.info['unit_name']
+ file_contents = False
+ tries = 0
+ while not file_contents and tries < (max_wait / 4):
+ try:
+ file_contents = sentry_unit.file_contents(file_name)
+ except IOError:
+ self.log.debug('Attempt {} to open file {} from {} '
+ 'failed'.format(tries, file_name,
+ unit_name))
+ time.sleep(4)
+ tries += 1
+
+ if file_contents:
+ return file_contents
+ elif not fatal:
+ return None
+ elif fatal:
+ msg = 'Failed to get file contents from unit.'
+ amulet.raise_status(amulet.FAIL, msg)
+
+ def port_knock_tcp(self, host="localhost", port=22, timeout=15):
+ """Open a TCP socket to check for a listening sevice on a host.
+
+ :param host: host name or IP address, default to localhost
+ :param port: TCP port number, default to 22
+ :param timeout: Connect timeout, default to 15 seconds
+ :returns: True if successful, False if connect failed
+ """
+
+ # Resolve host name if possible
+ try:
+ connect_host = socket.gethostbyname(host)
+ host_human = "{} ({})".format(connect_host, host)
+ except socket.error as e:
+ self.log.warn('Unable to resolve address: '
+ '{} ({}) Trying anyway!'.format(host, e))
+ connect_host = host
+ host_human = connect_host
+
+ # Attempt socket connection
+ try:
+ knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ knock.settimeout(timeout)
+ knock.connect((connect_host, port))
+ knock.close()
+ self.log.debug('Socket connect OK for host '
+ '{} on port {}.'.format(host_human, port))
+ return True
+ except socket.error as e:
+ self.log.debug('Socket connect FAIL for'
+ ' {} port {} ({})'.format(host_human, port, e))
+ return False
+
+ def port_knock_units(self, sentry_units, port=22,
+ timeout=15, expect_success=True):
+ """Open a TCP socket to check for a listening sevice on each
+ listed juju unit.
+
+ :param sentry_units: list of sentry unit pointers
+ :param port: TCP port number, default to 22
+ :param timeout: Connect timeout, default to 15 seconds
+ :expect_success: True by default, set False to invert logic
+ :returns: None if successful, Failure message otherwise
+ """
+ for unit in sentry_units:
+ host = unit.info['public-address']
+ connected = self.port_knock_tcp(host, port, timeout)
+ if not connected and expect_success:
+ return 'Socket connect failed.'
+ elif connected and not expect_success:
+ return 'Socket connected unexpectedly.'
+
+ def get_uuid_epoch_stamp(self):
+ """Returns a stamp string based on uuid4 and epoch time. Useful in
+ generating test messages which need to be unique-ish."""
+ return '[{}-{}]'.format(uuid.uuid4(), time.time())
+
+# amulet juju action helpers:
+ def run_action(self, unit_sentry, action,
+ _check_output=subprocess.check_output,
+ params=None):
+ """Run the named action on a given unit sentry.
+
+ params a dict of parameters to use
+ _check_output parameter is used for dependency injection.
+
+ @return action_id.
+ """
+ unit_id = unit_sentry.info["unit_name"]
+ command = ["juju", "action", "do", "--format=json", unit_id, action]
+ if params is not None:
+ for key, value in params.iteritems():
+ command.append("{}={}".format(key, value))
+ self.log.info("Running command: %s\n" % " ".join(command))
+ output = _check_output(command, universal_newlines=True)
+ data = json.loads(output)
+ action_id = data[u'Action queued with id']
+ return action_id
+
+ def wait_on_action(self, action_id, _check_output=subprocess.check_output):
+ """Wait for a given action, returning if it completed or not.
+
+ _check_output parameter is used for dependency injection.
+ """
+ command = ["juju", "action", "fetch", "--format=json", "--wait=0",
+ action_id]
+ output = _check_output(command, universal_newlines=True)
+ data = json.loads(output)
+ return data.get(u"status") == "completed"
+
+ def status_get(self, unit):
+ """Return the current service status of this unit."""
+ raw_status, return_code = unit.run(
+ "status-get --format=json --include-data")
+ if return_code != 0:
+ return ("unknown", "")
+ status = json.loads(raw_status)
+ return (status["status"], status["message"])