aboutsummaryrefslogtreecommitdiffstats
path: root/docker
diff options
context:
space:
mode:
authorIan Main <imain@redhat.com>2017-02-07 17:08:14 -0500
committerIan Main <imain@redhat.com>2017-02-22 14:56:42 -0500
commit5a74923bc547b8af856a30ccd7be0dc919baa044 (patch)
treee8de6e192e64586264d8ce37f9a6a395d16f40d0 /docker
parent5be125e1171671960f0593d2f14ecf7f3d4abba5 (diff)
Parallelize docker-puppet.py
Use a pool of worker processes to run the puppet modules so they can all be done in parallel. Defaults to cpu count processes. Change-Id: I083d302b8cf6538569e4d165221c21df152266bc
Diffstat (limited to 'docker')
-rwxr-xr-xdocker/docker-puppet.py58
1 files changed, 40 insertions, 18 deletions
diff --git a/docker/docker-puppet.py b/docker/docker-puppet.py
index fe87ce7a..a642e848 100755
--- a/docker/docker-puppet.py
+++ b/docker/docker-puppet.py
@@ -23,6 +23,7 @@ import os
import subprocess
import sys
import tempfile
+import multiprocessing
# this is to match what we do in deployed-server
@@ -53,6 +54,8 @@ def rm_container(name):
print(cmd_stdout)
print(cmd_stderr)
+process_count = int(os.environ.get('PROCESS_COUNT',
+ multiprocessing.cpu_count()))
config_file = os.environ.get('CONFIG', '/var/lib/docker-puppet/docker-puppet.json')
print('docker-puppet')
@@ -106,27 +109,18 @@ for service in (json_data or []):
print('Service compilation completed.\n')
-for config_volume in configs:
-
- service = configs[config_volume]
- puppet_tags = service[1] or ''
- manifest = service[2] or ''
- config_image = service[3] or ''
- volumes = service[4] if len(service) > 4 else []
-
- if puppet_tags:
- puppet_tags = "file,file_line,concat,%s" % puppet_tags
- else:
- puppet_tags = "file,file_line,concat"
+def mp_puppet_config((config_volume, puppet_tags, manifest, config_image, volumes)):
print('---------')
print('config_volume %s' % config_volume)
print('puppet_tags %s' % puppet_tags)
print('manifest %s' % manifest)
print('config_image %s' % config_image)
+ print('volumes %s' % volumes)
hostname = short_hostname()
+ sh_script = '/var/lib/docker-puppet/docker-puppet-%s.sh' % config_volume
- with open('/var/lib/docker-puppet/docker-puppet.sh', 'w') as script_file:
+ with open(sh_script, 'w') as script_file:
os.chmod(script_file.name, 0755)
script_file.write("""#!/bin/bash
set -ex
@@ -186,12 +180,12 @@ for config_volume in configs:
'--volume', '/usr/share/openstack-puppet/modules/:/usr/share/openstack-puppet/modules/:ro',
'--volume', '/var/lib/config-data/:/var/lib/config-data/:rw',
'--volume', 'tripleo_logs:/var/log/tripleo/',
- '--volume', '/var/lib/docker-puppet/docker-puppet.sh:/var/lib/docker-puppet/docker-puppet.sh:ro']
+ '--volume', '%s:%s:rw' % (sh_script, sh_script) ]
for volume in volumes:
dcmd.extend(['--volume', volume])
- dcmd.extend(['--entrypoint', '/var/lib/docker-puppet/docker-puppet.sh'])
+ dcmd.extend(['--entrypoint', sh_script])
env = {}
if os.environ.get('NET_HOST', 'false') == 'true':
@@ -207,6 +201,34 @@ for config_volume in configs:
print(cmd_stderr)
if subproc.returncode != 0:
print('Failed running docker-puppet.py for %s' % config_volume)
- sys.exit(subproc.returncode)
- else:
- rm_container('docker-puppet-%s' % config_volume)
+ rm_container('docker-puppet-%s' % config_volume)
+ return subproc.returncode
+
+# Holds all the information for each process to consume.
+# Instead of starting them all linearly we run them using a process
+# pool. This creates a list of arguments for the above function
+# to consume.
+process_map = []
+
+for config_volume in configs:
+
+ service = configs[config_volume]
+ puppet_tags = service[1] or ''
+ manifest = service[2] or ''
+ config_image = service[3] or ''
+ volumes = service[4] if len(service) > 4 else []
+
+ if puppet_tags:
+ puppet_tags = "file,file_line,concat,%s" % puppet_tags
+ else:
+ puppet_tags = "file,file_line,concat"
+
+ process_map.append([config_volume, puppet_tags, manifest, config_image, volumes])
+
+for p in process_map:
+ print '--\n%s' % p
+
+# Fire off processes to perform each configuration. Defaults
+# to the number of CPUs on the system.
+p = multiprocessing.Pool(process_count)
+p.map(mp_puppet_config, process_map)