diff options
| -rwxr-xr-x | docker/docker-puppet.py | 58 |
1 files changed, 40 insertions, 18 deletions
diff --git a/docker/docker-puppet.py b/docker/docker-puppet.py index fe87ce7a..a642e848 100755 --- a/docker/docker-puppet.py +++ b/docker/docker-puppet.py @@ -23,6 +23,7 @@ import os import subprocess import sys import tempfile +import multiprocessing # this is to match what we do in deployed-server @@ -53,6 +54,8 @@ def rm_container(name): print(cmd_stdout) print(cmd_stderr) +process_count = int(os.environ.get('PROCESS_COUNT', + multiprocessing.cpu_count())) config_file = os.environ.get('CONFIG', '/var/lib/docker-puppet/docker-puppet.json') print('docker-puppet') @@ -106,27 +109,18 @@ for service in (json_data or []): print('Service compilation completed.\n') -for config_volume in configs: - - service = configs[config_volume] - puppet_tags = service[1] or '' - manifest = service[2] or '' - config_image = service[3] or '' - volumes = service[4] if len(service) > 4 else [] - - if puppet_tags: - puppet_tags = "file,file_line,concat,%s" % puppet_tags - else: - puppet_tags = "file,file_line,concat" +def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volumes)): print('---------') print('config_volume %s' % config_volume) print('puppet_tags %s' % puppet_tags) print('manifest %s' % manifest) print('config_image %s' % config_image) + print('volumes %s' % volumes) hostname = short_hostname() + sh_script = '/var/lib/docker-puppet/docker-puppet-%s.sh' % config_volume - with open('/var/lib/docker-puppet/docker-puppet.sh', 'w') as script_file: + with open(sh_script, 'w') as script_file: os.chmod(script_file.name, 0755) script_file.write("""#!/bin/bash set -ex @@ -186,12 +180,12 @@ for config_volume in configs: '--volume', '/usr/share/openstack-puppet/modules/:/usr/share/openstack-puppet/modules/:ro', '--volume', '/var/lib/config-data/:/var/lib/config-data/:rw', '--volume', 'tripleo_logs:/var/log/tripleo/', - '--volume', '/var/lib/docker-puppet/docker-puppet.sh:/var/lib/docker-puppet/docker-puppet.sh:ro'] + '--volume', '%s:%s:rw' % (sh_script, sh_script) ] for volume in volumes: dcmd.extend(['--volume', volume]) - dcmd.extend(['--entrypoint', '/var/lib/docker-puppet/docker-puppet.sh']) + dcmd.extend(['--entrypoint', sh_script]) env = {} if os.environ.get('NET_HOST', 'false') == 'true': @@ -207,6 +201,34 @@ for config_volume in configs: print(cmd_stderr) if subproc.returncode != 0: print('Failed running docker-puppet.py for %s' % config_volume) - sys.exit(subproc.returncode) - else: - rm_container('docker-puppet-%s' % config_volume) + rm_container('docker-puppet-%s' % config_volume) + return subproc.returncode + +# Holds all the information for each process to consume. +# Instead of starting them all linearly we run them using a process +# pool. This creates a list of arguments for the above function +# to consume. +process_map = [] + +for config_volume in configs: + + service = configs[config_volume] + puppet_tags = service[1] or '' + manifest = service[2] or '' + config_image = service[3] or '' + volumes = service[4] if len(service) > 4 else [] + + if puppet_tags: + puppet_tags = "file,file_line,concat,%s" % puppet_tags + else: + puppet_tags = "file,file_line,concat" + + process_map.append([config_volume, puppet_tags, manifest, config_image, volumes]) + +for p in process_map: + print '--\n%s' % p + +# Fire off processes to perform each configuration. Defaults +# to the number of CPUs on the system. +p = multiprocessing.Pool(process_count) +p.map(mp_puppet_config, process_map) |