docker bootstrap service commands

This patch guards db syncs and initialization code from executing
on multiple nodes at the same time by using the new
bootstrap_host_exec script. This helper script checks to make
sure the container is executing on the "bootstrap host" for the
specified service (arg 0) and then if it matches runs the
specified command.

Depends-On: If25f217bbb592edab4e1dde53ca99ed93c0e146c
Depends-On: Ic1585bae27c318bd6bafc287e905f2ed250cce0f

Change-Id: I0c864ca093ea476248b619d8c88477ef0b64e2eb
Closes-Bug: 1688380

11 files changed, 36 insertions, 28 deletions

diff --git a/docker/services/aodh-api.yaml b/docker/services/aodh-api.yaml
index 8a02d8fd..f802e4e6 100644
--- a/docker/services/aodh-api.yaml
+++ b/docker/services/aodh-api.yaml
@@ -100,13 +100,14 @@ outputs:
             net: host
             privileged: false
             detach: false
+            user: root
             volumes:
               list_concat:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/aodh/etc/aodh/:/etc/aodh/:ro
                   - /var/log/containers/aodh:/var/log/aodh
-            command: /usr/bin/aodh-dbsync
+            command: "/usr/bin/bootstrap_host_exec aodh_api su aodh -s /bin/bash -c /usr/bin/aodh-dbsync"
         step_4:
           aodh_api:
             image: *aodh_image
diff --git a/docker/services/containers-common.yaml b/docker/services/containers-common.yaml
index fad1456c..45a462fd 100644
--- a/docker/services/containers-common.yaml
+++ b/docker/services/containers-common.yaml
@@ -9,6 +9,8 @@ outputs:
     value:
       - /etc/hosts:/etc/hosts:ro
       - /etc/localtime:/etc/localtime:ro
+      # required for bootstrap_host_exec
+      - /etc/puppet:/etc/puppet:ro
       # OpenSSL trusted CAs
       - /etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro
       - /etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro
diff --git a/docker/services/glance-api.yaml b/docker/services/glance-api.yaml
index ef1e00ed..b2dc8ce7 100644
--- a/docker/services/glance-api.yaml
+++ b/docker/services/glance-api.yaml
@@ -100,6 +100,7 @@ outputs:
             net: host
             privileged: false
             detach: false
+            user: root
             volumes: &glance_volumes
               list_concat:
                 - {get_attr: [ContainersCommon, volumes]}
@@ -110,6 +111,7 @@ outputs:
             environment:
               - KOLLA_BOOTSTRAP=True
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+            command: "/usr/bin/bootstrap_host_exec glance_api su glance -s /bin/bash -c '/usr/local/bin/kolla_start'"
         step_4:
           map_merge:
             - glance_api:
diff --git a/docker/services/gnocchi-api.yaml b/docker/services/gnocchi-api.yaml
index 9b474731..e59d6095 100644
--- a/docker/services/gnocchi-api.yaml
+++ b/docker/services/gnocchi-api.yaml
@@ -100,13 +100,14 @@ outputs:
             net: host
             detach: false
             privileged: false
+            user: root
             volumes:
               list_concat:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/gnocchi/etc/gnocchi/:/etc/gnocchi/:ro
                   - /var/log/containers/gnocchi:/var/log/gnocchi
-            command: ["/usr/bin/gnocchi-upgrade", "--skip-storage"]
+            command: "/usr/bin/bootstrap_host_exec gnocchi_api su gnocchi -s /bin/bash -c '/usr/bin/gnocchi-upgrade --skip-storage'"
         step_4:
           gnocchi_api:
             image: *gnocchi_image
diff --git a/docker/services/heat-engine.yaml b/docker/services/heat-engine.yaml
index 8c554a50..0adad538 100644
--- a/docker/services/heat-engine.yaml
+++ b/docker/services/heat-engine.yaml
@@ -94,13 +94,14 @@ outputs:
             net: host
             privileged: false
             detach: false
+            user: root
             volumes:
               list_concat:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/heat/etc/heat/:/etc/heat/:ro
                   - /var/log/containers/heat:/var/log/heat
-            command: ['heat-manage', 'db_sync']
+            command: "/usr/bin/bootstrap_host_exec heat_engine su heat -s /bin/bash -c 'heat-manage db_sync'"
         step_4:
           heat_engine:
             image: *heat_engine_image
diff --git a/docker/services/ironic-api.yaml b/docker/services/ironic-api.yaml
index 300aa0bd..c8978aa2 100644
--- a/docker/services/ironic-api.yaml
+++ b/docker/services/ironic-api.yaml
@@ -100,13 +100,14 @@ outputs:
             net: host
             privileged: false
             detach: false
+            user: root
             volumes:
               list_concat:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/ironic/etc/:/etc/:ro
                   - /var/log/containers/ironic:/var/log/ironic
-            command: ['ironic-dbsync', '--config-file', '/etc/ironic/ironic.conf']
+            command: "/usr/bin/bootstrap_host_exec ironic_api su ironic -s /bin/bash -c 'ironic-dbsync --config-file /etc/ironic/ironic.conf'"
         step_4:
           ironic_api:
             start_order: 10
diff --git a/docker/services/keystone.yaml b/docker/services/keystone.yaml
index ca643749..772859ee 100644
--- a/docker/services/keystone.yaml
+++ b/docker/services/keystone.yaml
@@ -131,8 +131,9 @@ outputs:
             environment:
               - KOLLA_BOOTSTRAP=True
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+            command: ['/usr/bin/bootstrap_host_exec', 'keystone', '/usr/local/bin/kolla_start']
           keystone:
-            start_order: 1
+            start_order: 2
             image: *keystone_image
             net: host
             privileged: false
@@ -141,10 +142,10 @@ outputs:
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
           keystone_bootstrap:
-            start_order: 2
+            start_order: 3
             action: exec
             command:
-              [ 'keystone', 'keystone-manage', 'bootstrap', '--bootstrap-password', {get_param: AdminPassword} ]
+              [ 'keystone', '/usr/bin/bootstrap_host_exec', 'keystone' ,'keystone-manage', 'bootstrap', '--bootstrap-password', {get_param: AdminPassword} ]
       docker_puppet_tasks:
         # Keystone endpoint creation occurs only on single node
         step_3:
diff --git a/docker/services/mistral-api.yaml b/docker/services/mistral-api.yaml
index 3b256fdd..5586d41b 100644
--- a/docker/services/mistral-api.yaml
+++ b/docker/services/mistral-api.yaml
@@ -100,19 +100,21 @@ outputs:
             net: host
             privileged: false
             detach: false
+            user: root
             volumes:
               list_concat:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/mistral/etc/:/etc/:ro
                   - /var/log/containers/mistral:/var/log/mistral
-            command: ['mistral-db-manage', '--config-file', '/etc/mistral/mistral.conf', 'upgrade', 'head']
+            command: "/usr/bin/bootstrap_host_exec mistral_api su mistral -s /bin/bash -c 'mistral-db-manage --config-file /etc/mistral/mistral.conf upgrade head'"
           mistral_db_populate:
             start_order: 2
             image: *mistral_image
             net: host
             privileged: false
             detach: false
+            user: root
             volumes:
               list_concat:
                 - {get_attr: [ContainersCommon, volumes]}
@@ -121,7 +123,7 @@ outputs:
                   - /var/log/containers/mistral:/var/log/mistral
             # NOTE: dprince this requires that we install openstack-tripleo-common into
             # the Mistral API image so that we get tripleo* actions
-            command: ['mistral-db-manage', '--config-file', '/etc/mistral/mistral.conf', 'populate']
+            command: "/usr/bin/bootstrap_host_exec mistral_api su mistral -s /bin/bash -c 'mistral-db-manage --config-file /etc/mistral/mistral.conf populate'"
         step_4:
           mistral_api:
             start_order: 15
diff --git a/docker/services/neutron-api.yaml b/docker/services/neutron-api.yaml
index 748371d5..5f9c9567 100644
--- a/docker/services/neutron-api.yaml
+++ b/docker/services/neutron-api.yaml
@@ -110,8 +110,6 @@ outputs:
             net: host
             privileged: false
             detach: false
-            # FIXME: we should make config file permissions right
-            # and run as neutron user
             user: root
             volumes:
               list_concat:
@@ -120,7 +118,10 @@ outputs:
                   - /var/lib/config-data/neutron/etc/neutron:/etc/neutron:ro
                   - /var/lib/config-data/neutron/usr/share/neutron:/usr/share/neutron:ro
                   - /var/log/containers/neutron:/var/log/neutron
-            command: ['neutron-db-manage', 'upgrade', 'heads']
+            command: ['/usr/bin/bootstrap_host_exec', 'neutron_api', 'neutron-db-manage', 'upgrade', 'heads']
+            # FIXME: we should make config file permissions right
+            # and run as neutron user
+            #command: "/usr/bin/bootstrap_host_exec neutron_api su neutron -s /bin/bash -c 'neutron-db-manage upgrade heads'"
         step_4:
           map_merge:
             - neutron_api:
diff --git a/docker/services/nova-api.yaml b/docker/services/nova-api.yaml
index d571b21b..2375dada 100644
--- a/docker/services/nova-api.yaml
+++ b/docker/services/nova-api.yaml
@@ -103,6 +103,7 @@ outputs:
             image: *nova_api_image
             net: host
             detach: false
+            user: root
             volumes: &nova_api_volumes
               list_concat:
                 - {get_attr: [ContainersCommon, volumes]}
@@ -110,7 +111,7 @@ outputs:
                   - /var/lib/kolla/config_files/nova_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/nova/etc/nova/:/etc/nova/:ro
                   - /var/log/containers/nova:/var/log/nova
-            command: ['/usr/bin/nova-manage', 'api_db', 'sync']
+            command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage api_db sync'"
           # FIXME: we probably want to wait on the 'cell_v2 update' in order for this
           # to be capable of upgrading a baremetal setup. This is to ensure the name
           # of the cell is 'default'
@@ -119,11 +120,9 @@ outputs:
             image: *nova_api_image
             net: host
             detach: false
+            user: root
             volumes: *nova_api_volumes
-            command:
-              - '/usr/bin/nova-manage'
-              - 'cell_v2'
-              - 'map_cell0'
+            command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage cell_v2 map_cell0'"
           nova_api_create_default_cell:
             start_order: 3
             image: *nova_api_image
@@ -134,18 +133,16 @@ outputs:
             # this idempotent (if the resource already exists a conflict
             # is raised)
             exit_codes: [0,2]
-            command:
-              - '/usr/bin/nova-manage'
-              - 'cell_v2'
-              - 'create_cell'
-              - '--name="default"'
+            user: root
+            command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage cell_v2 create_cell --name=default'"
           nova_db_sync:
             start_order: 4
             image: *nova_api_image
             net: host
             detach: false
             volumes: *nova_api_volumes
-            command: ['/usr/bin/nova-manage', 'db', 'sync']
+            user: root
+            command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage db sync'"
         step_4:
           nova_api:
             start_order: 2
@@ -164,10 +161,8 @@ outputs:
             net: host
             detach: false
             volumes: *nova_api_volumes
-            command:
-              - '/usr/bin/nova-manage'
-              - 'cell_v2'
-              - 'discover_hosts'
+            user: root
+            command: "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage cell_v2 discover_hosts'"
       host_prep_tasks:
         - name: create persistent logs directory
           file:
diff --git a/docker/services/panko-api.yaml b/docker/services/panko-api.yaml
index 46cfa5ab..b9e6e93a 100644
--- a/docker/services/panko-api.yaml
+++ b/docker/services/panko-api.yaml
@@ -101,13 +101,14 @@ outputs:
             net: host
             detach: false
             privileged: false
+            user: root
             volumes:
               list_concat:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/panko/etc/panko:/etc/panko:ro
                   - /var/log/containers/panko:/var/log/panko
-            command: /usr/bin/panko-dbsync
+            command: "/usr/bin/bootstrap_host_exec panko_api su panko -s /bin/bash -c '/usr/bin/panko-dbsync'"
         step_4:
           panko_api:
             start_order: 2