diff options
| -rw-r--r-- | all-nodes-validation.yaml | 6 | ||||
| -rwxr-xr-x | docker/docker-puppet.py | 86 | ||||
| -rwxr-xr-x | extraconfig/tasks/yum_update.sh | 18 | ||||
| -rw-r--r-- | validation-scripts/all-nodes.sh | 30 |
4 files changed, 86 insertions, 54 deletions
diff --git a/all-nodes-validation.yaml b/all-nodes-validation.yaml index eea3e40a..11a5b371 100644 --- a/all-nodes-validation.yaml +++ b/all-nodes-validation.yaml @@ -14,6 +14,10 @@ parameters: default: false description: Optional validation to ensure FQDN as set by Nova matches the name set in /etc/hosts. type: boolean + ValidateNtp: + default: true + description: Validation to ensure at least one time source is accessible. + type: boolean resources: AllNodesValidationsImpl: @@ -25,6 +29,8 @@ resources: default: {get_param: PingTestIps} - name: validate_fqdn default: {get_param: ValidateFqdn} + - name: validate_ntp + default: {get_param: ValidateNtp} config: {get_file: ./validation-scripts/all-nodes.sh} outputs: diff --git a/docker/docker-puppet.py b/docker/docker-puppet.py index 909a2c8a..111005ac 100755 --- a/docker/docker-puppet.py +++ b/docker/docker-puppet.py @@ -19,12 +19,20 @@ # inside of a container. import json +import logging import os import subprocess import sys import tempfile import multiprocessing +log = logging.getLogger() +log.setLevel(logging.DEBUG) +ch = logging.StreamHandler(sys.stdout) +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s %(levelname)s: %(message)s') +ch.setFormatter(formatter) +log.addHandler(ch) # this is to match what we do in deployed-server def short_hostname(): @@ -36,42 +44,47 @@ def short_hostname(): def pull_image(name): - print('Pulling image: %s' % name) + log.info('Pulling image: %s' % name) subproc = subprocess.Popen(['/usr/bin/docker', 'pull', name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) cmd_stdout, cmd_stderr = subproc.communicate() - print(cmd_stdout) - print(cmd_stderr) + if cmd_stdout: + log.debug(cmd_stdout) + if cmd_stderr: + log.debug(cmd_stderr) def rm_container(name): if os.environ.get('SHOW_DIFF', None): - print('Diffing container: %s' % name) + log.info('Diffing container: %s' % name) subproc = subprocess.Popen(['/usr/bin/docker', 'diff', name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) cmd_stdout, cmd_stderr = subproc.communicate() - print(cmd_stdout) - print(cmd_stderr) + if cmd_stdout: + log.debug(cmd_stdout) + if cmd_stderr: + log.debug(cmd_stderr) - print('Removing container: %s' % name) + log.info('Removing container: %s' % name) subproc = subprocess.Popen(['/usr/bin/docker', 'rm', name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) cmd_stdout, cmd_stderr = subproc.communicate() - print(cmd_stdout) + if cmd_stdout: + log.debug(cmd_stdout) if cmd_stderr and \ - cmd_stderr != 'Error response from daemon: ' \ - 'No such container: {}\n'.format(name): - print(cmd_stderr) + cmd_stderr != 'Error response from daemon: ' \ + 'No such container: {}\n'.format(name): + log.debug(cmd_stderr) process_count = int(os.environ.get('PROCESS_COUNT', multiprocessing.cpu_count())) +log.info('Running docker-puppet') config_file = os.environ.get('CONFIG', '/var/lib/docker-puppet/docker-puppet.json') -print('docker-puppet') -print('CONFIG: %s' % config_file) +log.debug('CONFIG: %s' % config_file) with open(config_file) as f: json_data = json.load(f) @@ -108,16 +121,15 @@ for service in (json_data or []): if not manifest or not config_image: continue - print('---------') - print('config_volume %s' % config_volume) - print('puppet_tags %s' % puppet_tags) - print('manifest %s' % manifest) - print('config_image %s' % config_image) - print('volumes %s' % volumes) + log.debug('config_volume %s' % config_volume) + log.debug('puppet_tags %s' % puppet_tags) + log.debug('manifest %s' % manifest) + log.debug('config_image %s' % config_image) + log.debug('volumes %s' % volumes) # We key off of config volume for all configs. if config_volume in configs: # Append puppet tags and manifest. - print("Existing service, appending puppet tags and manifest\n") + log.info("Existing service, appending puppet tags and manifest") if puppet_tags: configs[config_volume][1] = '%s,%s' % (configs[config_volume][1], puppet_tags) @@ -125,22 +137,21 @@ for service in (json_data or []): configs[config_volume][2] = '%s\n%s' % (configs[config_volume][2], manifest) if configs[config_volume][3] != config_image: - print("WARNING: Config containers do not match even though" - " shared volumes are the same!\n") + log.warn("Config containers do not match even though" + " shared volumes are the same!") else: - print("Adding new service\n") + log.info("Adding new service") configs[config_volume] = service -print('Service compilation completed.\n') +log.info('Service compilation completed.') def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volumes)): - print('---------') - print('config_volume %s' % config_volume) - print('puppet_tags %s' % puppet_tags) - print('manifest %s' % manifest) - print('config_image %s' % config_image) - print('volumes %s' % volumes) + log.debug('config_volume %s' % config_volume) + log.debug('puppet_tags %s' % puppet_tags) + log.debug('manifest %s' % manifest) + log.debug('config_image %s' % config_image) + log.debug('volumes %s' % volumes) hostname = short_hostname() sh_script = '/var/lib/docker-puppet/docker-puppet-%s.sh' % config_volume @@ -226,18 +237,21 @@ def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volume env[k] = os.environ.get(k) if os.environ.get('NET_HOST', 'false') == 'true': - print('NET_HOST enabled') + log.debug('NET_HOST enabled') dcmd.extend(['--net', 'host', '--volume', '/etc/hosts:/etc/hosts:ro']) dcmd.append(config_image) + log.debug('Running docker command: %s' % ' '.join(dcmd)) subproc = subprocess.Popen(dcmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) cmd_stdout, cmd_stderr = subproc.communicate() - print(cmd_stdout) - print(cmd_stderr) + if cmd_stdout: + log.debug(cmd_stdout) + if cmd_stderr: + log.debug(cmd_stderr) if subproc.returncode != 0: - print('Failed running docker-puppet.py for %s' % config_volume) + log.error('Failed running docker-puppet.py for %s' % config_volume) rm_container('docker-puppet-%s' % config_volume) return subproc.returncode @@ -263,7 +277,7 @@ for config_volume in configs: process_map.append([config_volume, puppet_tags, manifest, config_image, volumes]) for p in process_map: - print '--\n%s' % p + log.debug('- %s' % p) # Fire off processes to perform each configuration. Defaults # to the number of CPUs on the system. @@ -273,7 +287,7 @@ config_volumes = [pm[0] for pm in process_map] success = True for returncode, config_volume in zip(returncodes, config_volumes): if returncode != 0: - print('ERROR configuring %s' % config_volume) + log.error('ERROR configuring %s' % config_volume) success = False if not success: diff --git a/extraconfig/tasks/yum_update.sh b/extraconfig/tasks/yum_update.sh index 20a5b658..018c9b74 100755 --- a/extraconfig/tasks/yum_update.sh +++ b/extraconfig/tasks/yum_update.sh @@ -60,24 +60,6 @@ if hiera -c /etc/puppet/hiera.yaml service_names | grep -q pacemaker; then pacemaker_status=$(systemctl is-active pacemaker) fi -# Fix the redis/rabbit resource start/stop timeouts. See https://bugs.launchpad.net/tripleo/+bug/1633455 -# and https://bugs.launchpad.net/tripleo/+bug/1634851 -if [[ "$pacemaker_status" == "active" && \ - "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]] ; then - if pcs resource show rabbitmq | grep -E "start.*timeout=100"; then - pcs resource update rabbitmq op start timeout=200s - fi - if pcs resource show rabbitmq | grep -E "stop.*timeout=90"; then - pcs resource update rabbitmq op stop timeout=200s - fi - if pcs resource show redis | grep -E "start.*timeout=120"; then - pcs resource update redis op start timeout=200s - fi - if pcs resource show redis | grep -E "stop.*timeout=120"; then - pcs resource update redis op stop timeout=200s - fi -fi - # special case https://bugs.launchpad.net/tripleo/+bug/1635205 +bug/1669714 special_case_ovs_upgrade_if_needed diff --git a/validation-scripts/all-nodes.sh b/validation-scripts/all-nodes.sh index f1f4cc11..ed7fefb7 100644 --- a/validation-scripts/all-nodes.sh +++ b/validation-scripts/all-nodes.sh @@ -82,8 +82,38 @@ function fqdn_check() { echo "SUCCESS" } +# Verify at least one time source is available. +function ntp_check() { + NTP_SERVERS=$(hiera ntp::servers nil |tr -d '[],"') + if [[ "$NTP_SERVERS" != "nil" ]];then + echo -n "Testing NTP..." + NTP_SUCCESS=0 + for NTP_SERVER in $NTP_SERVERS; do + set +e + NTPDATE_OUT=$(ntpdate -qud $NTP_SERVER 2>&1) + NTPDATE_EXIT=$? + set -e + if [[ "$NTPDATE_EXIT" == "0" ]];then + NTP_SUCCESS=1 + break + else + NTPDATE_OUT_FULL="$NTPDATE_OUT_FULL $NTPDATE_OUT" + fi + done + if [[ "$NTP_SUCCESS" == "0" ]];then + echo "FAILURE" + echo "$NTPDATE_OUT_FULL" + exit 1 + fi + echo "SUCCESS" + fi +} + ping_controller_ips "$ping_test_ips" ping_default_gateways if [[ $validate_fqdn == "True" ]];then fqdn_check fi +if [[ $validate_ntp == "True" ]];then + ntp_check +fi |