Mount hostpath logs on /var/log

Some containers are using the logs named volume for collecting logs
written to `/var/log`. We should make this consistent for all the
containers.

This patch also cleans up some mounts that weren't needed for some
services. For example, glance-api doesn't need `/run` to be mounted.

Other changes:
* Rework log volumes to hostpath mounts to omit slow COW writes.
* Add kolla_config's permission and host_prep_tasks create and
  manage hostpath mounted log dirs permissions.
* Rework data owning init containers to kolla_config permissions
* When a step wants KOLLA_BOOTSTRAP or DB sync, use logs data owning
  init containers to set permissions for logs. This is required
  because kolla bootsrap and DB sync runs before the kolla config
  stage and there is yet permissions set for logs.
* In order to address hybrid cases for host services vs containerized
  ones to access logs having different UIDs, persist containerized
  services' logs into separate directories (an upgrade impact)
* Ensure host prep tasks to create /var/log/containers/ and /var/lib/
  sub-directories for services
* Fix missing /etc/httpd, /var/www config-data mounts for zaqar/ironic
* Fix YAML indentation and drop strings quotation.

Co-authored-by: Bogdan Dobrelya <bdobreli@redhat.com>
Partial blueprint containerized-services-logs

Change-Id: I53e737120bf0121bd28667f355b6f29f1b2a6b82

37 files changed, 501 insertions, 60 deletions

diff --git a/docker/services/aodh-api.yaml b/docker/services/aodh-api.yaml
index b93a92e1..0bf15e12 100644
--- a/docker/services/aodh-api.yaml
+++ b/docker/services/aodh-api.yaml
@@ -70,15 +70,20 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/aodh-api.json:
           command: /usr/sbin/httpd -DFOREGROUND
+          permissions:
+            - path: /var/log/aodh
+              owner: aodh:aodh
+              recurse: true
       docker_config:
+        # db sync runs before permissions set by kolla_config
         step_3:
           aodh_init_log:
             start_order: 0
             image: *aodh_image
             user: root
-            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && mkdir -p /var/log/aodh && chown aodh:aodh /var/log/aodh']
             volumes:
-              - logs:/var/log
+              - /var/log/containers/aodh:/var/log/aodh
+            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R aodh:aodh /var/log/aodh']
           aodh_db_sync:
             start_order: 1
             image: *aodh_image
@@ -90,7 +95,7 @@ outputs:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/aodh/etc/aodh/:/etc/aodh/:ro
-                  - logs:/var/log
+                  - /var/log/containers/aodh:/var/log/aodh
             command: /usr/bin/aodh-dbsync
         step_4:
           aodh_api:
@@ -106,7 +111,7 @@ outputs:
                   - /var/lib/config-data/aodh/etc/aodh/:/etc/aodh/:ro
                   - /var/lib/config-data/aodh/etc/httpd/:/etc/httpd/:ro
                   - /var/lib/config-data/aodh/var/www/:/var/www/:ro
-                  - logs:/var/log
+                  - /var/log/containers/aodh:/var/log/aodh
                   -
                     if:
                       - internal_tls_enabled
@@ -119,6 +124,11 @@ outputs:
                       - ''
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+      - name: create persistent logs directory
+        file:
+          path: /var/log/containers/aodh
+          state: directory
       upgrade_tasks:
         - name: Stop and disable aodh service (running under httpd)
           tags: step2
diff --git a/docker/services/aodh-evaluator.yaml b/docker/services/aodh-evaluator.yaml
index c8e7d691..065f03e8 100644
--- a/docker/services/aodh-evaluator.yaml
+++ b/docker/services/aodh-evaluator.yaml
@@ -62,6 +62,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/aodh-evaluator.json:
           command: /usr/bin/aodh-evaluator
+          permissions:
+            - path: /var/log/aodh
+              owner: aodh:aodh
+              recurse: true
       docker_config:
         step_4:
           aodh_evaluator:
@@ -75,8 +79,14 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/aodh-evaluator.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/aodh/etc/aodh/:/etc/aodh/:ro
+                  - /var/log/containers/aodh:/var/log/aodh
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/aodh
+            state: directory
       upgrade_tasks:
         - name: Stop and disable openstack-aodh-evaluator service
           tags: step2
diff --git a/docker/services/aodh-listener.yaml b/docker/services/aodh-listener.yaml
index 9e65c1c4..eb77e506 100644
--- a/docker/services/aodh-listener.yaml
+++ b/docker/services/aodh-listener.yaml
@@ -62,6 +62,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/aodh-listener.json:
           command: /usr/bin/aodh-listener
+          permissions:
+            - path: /var/log/aodh
+              owner: aodh:aodh
+              recurse: true
       docker_config:
         step_4:
           aodh_listener:
@@ -75,8 +79,14 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/aodh-listener.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/aodh/etc/aodh/:/etc/aodh/:ro
+                  - /var/log/containers/aodh:/var/log/aodh
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/aodh
+            state: directory
       upgrade_tasks:
         - name: Stop and disable openstack-aodh-listener service
           tags: step2
diff --git a/docker/services/aodh-notifier.yaml b/docker/services/aodh-notifier.yaml
index 402b8abf..fb6913b8 100644
--- a/docker/services/aodh-notifier.yaml
+++ b/docker/services/aodh-notifier.yaml
@@ -62,6 +62,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/aodh-notifier.json:
           command: /usr/bin/aodh-notifier
+          permissions:
+            - path: /var/log/aodh
+              owner: aodh:aodh
+              recurse: true
       docker_config:
         step_4:
           aodh_notifier:
@@ -75,8 +79,14 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/aodh-notifier.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/aodh/etc/aodh/:/etc/aodh/:ro
+                  - /var/log/containers/aodh:/var/log/aodh
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/aodh
+            state: directory
       upgrade_tasks:
         - name: Stop and disable openstack-aodh-notifier service
           tags: step2
diff --git a/docker/services/database/mongodb.yaml b/docker/services/database/mongodb.yaml
index 7d2d1a15..4a620a4a 100644
--- a/docker/services/database/mongodb.yaml
+++ b/docker/services/database/mongodb.yaml
@@ -66,6 +66,9 @@ outputs:
             - path: /var/lib/mongodb
               owner: mongodb:mongodb
               recurse: true
+            - path: /var/log/mongodb
+              owner: mongodb:mongodb
+              recurse: true
       docker_config:
         step_2:
           mongodb:
@@ -76,7 +79,7 @@ outputs:
               - /var/lib/kolla/config_files/mongodb.json:/var/lib/kolla/config_files/config.json
               - /var/lib/config-data/mongodb/etc/:/etc/:ro
               - /etc/localtime:/etc/localtime:ro
-              - logs:/var/log/kolla
+              - /var/log/containers/mongodb:/var/log/mongodb
               - /var/lib/mongodb:/var/lib/mongodb
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
@@ -88,13 +91,16 @@ outputs:
           step_config: 'include ::tripleo::profile::base::database::mongodb'
           config_image: *mongodb_image
           volumes:
-          - /var/lib/mongodb:/var/lib/mongodb
-          - logs:/var/log/kolla:ro
+            - /var/lib/mongodb:/var/lib/mongodb
+            - /var/log/containers/mongodb:/var/log/mongodb
       host_prep_tasks:
-        - name: create /var/lib/mongodb
+        - name: create persistent directories
           file:
-            path: /var/lib/mongodb
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/mongodb
+            - /var/lib/mongodb
       upgrade_tasks:
         - name: Stop and disable mongodb service
           tags: step2
diff --git a/docker/services/database/mysql.yaml b/docker/services/database/mysql.yaml
index cba2070d..e065e20e 100644
--- a/docker/services/database/mysql.yaml
+++ b/docker/services/database/mysql.yaml
@@ -76,9 +76,18 @@ outputs:
               owner: mysql:mysql
               recurse: true
       docker_config:
+        # Kolla_bootstrap runs before permissions set by kolla_config
         step_2:
-          mysql_bootstrap:
+          mysql_init_logs:
             start_order: 0
+            image: *mysql_image
+            privileged: false
+            user: root
+            volumes:
+              - /var/log/containers/mysql:/var/log/mariadb
+            command: ['/bin/bash', '-c', 'chown -R mysql:mysql /var/log/mariadb']
+          mysql_bootstrap:
+            start_order: 1
             detach: false
             image: *mysql_image
             net: host
@@ -90,12 +99,13 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
               - /etc/hosts:/etc/hosts:ro
               - /var/lib/mysql:/var/lib/mysql
+              - /var/log/containers/mysql:/var/log/mariadb
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
               - KOLLA_BOOTSTRAP=True
               # NOTE(mandre) skip wsrep cluster status check
               - KOLLA_KUBERNETES=True
-              - 
+              -
                 list_join:
                   - '='
                   - - 'DB_ROOT_PASSWORD'
@@ -107,7 +117,7 @@ outputs:
                             - {get_param: MysqlRootPassword}
                             - {get_param: [DefaultPasswords, mysql_root_password]}
           mysql:
-            start_order: 1
+            start_order: 2
             image: *mysql_image
             restart: always
             net: host
@@ -123,12 +133,16 @@ outputs:
           config_image: *mysql_image
           volumes:
             - /var/lib/mysql:/var/lib/mysql/:ro
+            - /var/log/containers/mysql:/var/log/mariadb
             - /var/lib/config-data/mysql/root:/root:ro #provides .my.cnf
       host_prep_tasks:
-        - name: create /var/lib/mysql
+        - name: create persistent directories
           file:
-            path: /var/lib/mysql
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/mysql
+            - /var/lib/mysql
       upgrade_tasks:
         - name: Stop and disable mysql service
           tags: step2
diff --git a/docker/services/glance-api.yaml b/docker/services/glance-api.yaml
index 7f4ee434..9fa90082 100644
--- a/docker/services/glance-api.yaml
+++ b/docker/services/glance-api.yaml
@@ -64,8 +64,18 @@ outputs:
         /var/lib/kolla/config_files/glance-api.json:
           command: /usr/bin/glance-api --config-file /usr/share/glance/glance-api-dist.conf --config-file /etc/glance/glance-api.conf
       docker_config:
+        # Kolla_bootstrap/db_sync runs before permissions set by kolla_config
         step_3:
+          glance_init_logs:
+            start_order: 0
+            image: *glance_image
+            privileged: false
+            user: root
+            volumes:
+              - /var/log/containers/glance:/var/log/glance
+            command: ['/bin/bash', '-c', 'chown -R glance:glance /var/log/glance']
           glance_api_db_sync:
+            start_order: 1
             image: *glance_image
             net: host
             privileged: false
@@ -76,14 +86,13 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/glance-api.json:/var/lib/kolla/config_files/config.json
                   - /var/lib/config-data/glance_api/etc/glance/:/etc/glance/:ro
-                  - /lib/modules:/lib/modules:ro
-                  - /run:/run
-                  - /dev:/dev
+                  - /var/log/containers/glance:/var/log/glance
             environment:
               - KOLLA_BOOTSTRAP=True
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
         step_4:
           glance_api:
+            start_order: 2
             image: *glance_image
             net: host
             privileged: false
@@ -91,6 +100,11 @@ outputs:
             volumes: *glance_volumes
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/glance
+            state: directory
       upgrade_tasks:
         - name: Stop and disable glance_api service
           tags: step2
diff --git a/docker/services/gnocchi-api.yaml b/docker/services/gnocchi-api.yaml
index 3fbdac4e..aab351fc 100644
--- a/docker/services/gnocchi-api.yaml
+++ b/docker/services/gnocchi-api.yaml
@@ -70,15 +70,20 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/gnocchi-api.json:
           command: /usr/sbin/httpd -DFOREGROUND
+          permissions:
+            - path: /var/log/gnocchi
+              owner: gnocchi:gnocchi
+              recurse: true
       docker_config:
+        # db sync runs before permissions set by kolla_config
         step_3:
           gnocchi_init_log:
             start_order: 0
             image: *gnocchi_image
             user: root
-            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && mkdir -p /var/log/gnocchi && chown gnocchi:gnocchi /var/log/gnocchi']
             volumes:
-              - logs:/var/log
+              - /var/log/containers/gnocchi:/var/log/gnocchi
+            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R gnocchi:gnocchi /var/log/gnocchi']
           gnocchi_db_sync:
             start_order: 1
             image: *gnocchi_image
@@ -90,7 +95,7 @@ outputs:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/gnocchi/etc/gnocchi/:/etc/gnocchi/:ro
-                  - logs:/var/log
+                  - /var/log/containers/gnocchi:/var/log/gnocchi
             command: ["/usr/bin/gnocchi-upgrade", "--skip-storage"]
         step_4:
           gnocchi_api:
@@ -106,6 +111,7 @@ outputs:
                   - /var/lib/config-data/gnocchi/etc/gnocchi/:/etc/gnocchi/:ro
                   - /var/lib/config-data/gnocchi/etc/httpd/:/etc/httpd/:ro
                   - /var/lib/config-data/gnocchi/var/www/:/var/www/:ro
+                  - /var/log/containers/gnocchi:/var/log/gnocchi
                   -
                     if:
                       - internal_tls_enabled
@@ -118,6 +124,11 @@ outputs:
                       - ''
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/gnocchi
+            state: directory
       upgrade_tasks:
         - name: Stop and disable httpd service
           tags: step2
diff --git a/docker/services/gnocchi-metricd.yaml b/docker/services/gnocchi-metricd.yaml
index 9739735b..b0faa51c 100644
--- a/docker/services/gnocchi-metricd.yaml
+++ b/docker/services/gnocchi-metricd.yaml
@@ -60,6 +60,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/gnocchi-metricd.json:
           command: /usr/bin/gnocchi-metricd
+          permissions:
+            - path: /var/log/gnocchi
+              owner: gnocchi:gnocchi
+              recurse: true
       docker_config:
         step_4:
           gnocchi_metricd:
@@ -73,8 +77,14 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/gnocchi-metricd.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/gnocchi/etc/gnocchi/:/etc/gnocchi/:ro
+                  - /var/log/containers/gnocchi:/var/log/gnocchi
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/gnocchi
+            state: directory
       upgrade_tasks:
         - name: Stop and disable openstack-gnocchi-metricd service
           tags: step2
diff --git a/docker/services/gnocchi-statsd.yaml b/docker/services/gnocchi-statsd.yaml
index 8b3071a3..3b9667d7 100644
--- a/docker/services/gnocchi-statsd.yaml
+++ b/docker/services/gnocchi-statsd.yaml
@@ -60,6 +60,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/gnocchi-statsd.json:
           command: /usr/bin/gnocchi-statsd
+          permissions:
+            - path: /var/log/gnocchi
+              owner: gnocchi:gnocchi
+              recurse: true
       docker_config:
         step_4:
           gnocchi_statsd:
@@ -73,8 +77,14 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/gnocchi-statsd.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/gnocchi/etc/gnocchi/:/etc/gnocchi/:ro
+                  - /var/log/containers/gnocchi:/var/log/gnocchi
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/gnocchi
+            state: directory
       upgrade_tasks:
         - name: Stop and disable openstack-gnocchi-statsd service
           tags: step2
diff --git a/docker/services/heat-api-cfn.yaml b/docker/services/heat-api-cfn.yaml
index 5a1c6057..fc228155 100644
--- a/docker/services/heat-api-cfn.yaml
+++ b/docker/services/heat-api-cfn.yaml
@@ -69,6 +69,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/heat_api_cfn.json:
           command: /usr/sbin/httpd -DFOREGROUND
+          permissions:
+            - path: /var/log/heat
+              owner: heat:heat
+              recurse: true
       docker_config:
         step_4:
           heat_api_cfn:
@@ -90,8 +94,14 @@ outputs:
                   - /var/lib/config-data/heat_api_cfn/etc/heat/:/etc/heat/:ro
                   - /var/lib/config-data/heat_api_cfn/etc/httpd/:/etc/httpd/:ro
                   - /var/lib/config-data/heat_api_cfn/var/www/:/var/www/:ro
+                  - /var/log/containers/heat:/var/log/heat
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/heat
+            state: directory
       upgrade_tasks:
         - name: Stop and disable heat_api_cfn service
           tags: step2
diff --git a/docker/services/heat-api.yaml b/docker/services/heat-api.yaml
index 784794b0..fe565411 100644
--- a/docker/services/heat-api.yaml
+++ b/docker/services/heat-api.yaml
@@ -69,6 +69,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/heat_api.json:
           command: /usr/sbin/httpd -DFOREGROUND
+          permissions:
+            - path: /var/log/heat
+              owner: heat:heat
+              recurse: true
       docker_config:
         step_4:
           heat_api:
@@ -90,8 +94,14 @@ outputs:
                   - /var/lib/config-data/heat_api/etc/heat/:/etc/heat/:ro
                   - /var/lib/config-data/heat_api/etc/httpd/:/etc/httpd/:ro
                   - /var/lib/config-data/heat_api/var/www/:/var/www/:ro
+                  - /var/log/containers/heat:/var/log/heat
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/heat
+            state: directory
       upgrade_tasks:
         - name: Stop and disable heat_api service
           tags: step2
diff --git a/docker/services/heat-engine.yaml b/docker/services/heat-engine.yaml
index ced81561..da0552af 100644
--- a/docker/services/heat-engine.yaml
+++ b/docker/services/heat-engine.yaml
@@ -64,9 +64,22 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/heat_engine.json:
           command: /usr/bin/heat-engine --config-file /usr/share/heat/heat-dist.conf --config-file /etc/heat/heat.conf
+          permissions:
+            - path: /var/log/heat
+              owner: heat:heat
+              recurse: true
       docker_config:
+        # db sync runs before permissions set by kolla_config
         step_3:
+          heat_init_log:
+            start_order: 0
+            image: *heat_engine_image
+            user: root
+            volumes:
+              - /var/log/containers/heat:/var/log/heat
+            command: ['/bin/bash', '-c', 'chown -R heat:heat /var/log/heat']
           heat_engine_db_sync:
+            start_order: 1
             image: *heat_engine_image
             net: host
             privileged: false
@@ -76,6 +89,7 @@ outputs:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/heat/etc/heat/:/etc/heat/:ro
+                  - /var/log/containers/heat:/var/log/heat
             command: ['heat-manage', 'db_sync']
         step_4:
           heat_engine:
@@ -89,9 +103,14 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/heat_engine.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/heat/etc/heat/:/etc/heat/:ro
-                  - /run:/run
+                  - /var/log/containers/heat:/var/log/heat
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/heat
+            state: directory
       upgrade_tasks:
         - name: Stop and disable heat_engine service
           tags: step2
diff --git a/docker/services/ironic-api.yaml b/docker/services/ironic-api.yaml
index a15e74d0..5e7565cc 100644
--- a/docker/services/ironic-api.yaml
+++ b/docker/services/ironic-api.yaml
@@ -66,13 +66,27 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/ironic_api.json:
           command: /usr/bin/ironic-api
+          permissions:
+            - path: /var/log/ironic
+              owner: ironic:ironic
+              recurse: true
       docker_config:
+        # db sync runs before permissions set by kolla_config
         step_3:
-          ironic_db_sync:
+          ironic_init_logs:
+            start_order: 0
             image: &ironic_image
               list_join:
                 - '/'
                 - [ {get_param: DockerNamespace}, {get_param: DockerIronicApiImage} ]
+            privileged: false
+            user: root
+            volumes:
+              - /var/log/containers/ironic:/var/log/ironic
+            command: ['/bin/bash', '-c', 'chown -R ironic:ironic /var/log/ironic']
+          ironic_db_sync:
+            start_order: 1
+            image: *ironic_image
             net: host
             privileged: false
             detach: false
@@ -81,6 +95,7 @@ outputs:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/ironic/etc/:/etc/:ro
+                  - /var/log/containers/ironic:/var/log/ironic
             command: ['ironic-dbsync', '--config-file', '/etc/ironic/ironic.conf']
         step_4:
           ironic_api:
@@ -95,8 +110,14 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/ironic_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/ironic/etc/:/etc/:ro
+                  - /var/log/containers/ironic:/var/log/ironic
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/ironic
+            state: directory
       upgrade_tasks:
         - name: Stop and disable ironic_api service
           tags: step2
diff --git a/docker/services/ironic-conductor.yaml b/docker/services/ironic-conductor.yaml
index 99d67e04..0b1d448a 100644
--- a/docker/services/ironic-conductor.yaml
+++ b/docker/services/ironic-conductor.yaml
@@ -78,6 +78,9 @@ outputs:
             - path: /var/lib/ironic
               owner: ironic:ironic
               recurse: true
+            - path: /var/log/ironic
+              owner: ironic:ironic
+              recurse: true
       docker_config:
         step_4:
           ironic_conductor:
@@ -100,13 +103,17 @@ outputs:
                   - /dev:/dev
                   - /run:/run #shared?
                   - /var/lib/ironic:/var/lib/ironic
+                  - /var/log/containers/ironic:/var/log/ironic
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
-        - name: create ironic persistent data directory
+        - name: create persistent directories
           file:
-            path: /var/lib/ironic
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/ironic
+            - /var/lib/ironic
         - name: stat /httpboot
           stat: path=/httpboot
           register: stat_httpboot
diff --git a/docker/services/ironic-pxe.yaml b/docker/services/ironic-pxe.yaml
index 7b72db20..94db8490 100644
--- a/docker/services/ironic-pxe.yaml
+++ b/docker/services/ironic-pxe.yaml
@@ -58,6 +58,10 @@ outputs:
           command: /usr/sbin/httpd -DFOREGROUND
         /var/lib/kolla/config_files/ironic_pxe_tftp.json:
           command: /usr/sbin/in.tftpd --foreground --user root --address 0.0.0.0:69 --map-file /var/lib/ironic/tftpboot/map-file /var/lib/ironic/tftpboot
+          permissions:
+            - path: /var/log/ironic
+              owner: ironic:ironic
+              recurse: true
       docker_config:
         step_4:
           ironic_pxe_tftp:
@@ -86,6 +90,7 @@ outputs:
                   - /var/lib/config-data/ironic/var/lib/ironic/tftpboot/undionly.kpxe:/var/lib/ironic/tftpboot/undionly.kpxe
                   - /var/lib/config-data/ironic/var/lib/ironic/tftpboot/map-file:/var/lib/ironic/tftpboot/map-file
                   - /dev/log:/dev/log
+                  - /var/log/containers/ironic:/var/log/ironic
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
           ironic_pxe_http:
@@ -101,11 +106,16 @@ outputs:
                   - /var/lib/kolla/config_files/ironic_pxe_http.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/ironic/etc/ironic/:/etc/ironic/:ro
                   - /var/lib/config-data/ironic/etc/httpd/:/etc/httpd/:ro
+                  - /var/lib/config-data/ironic/var/www/:/var/www/:ro
                   - /var/lib/ironic:/var/lib/ironic/
+                  - /var/log/containers/ironic:/var/log/ironic
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
-        - name: create ironic persistent data directory
+        - name: create persistent directories
           file:
-            path: /var/lib/ironic
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/lib/ironic
+            - /var/log/containers/ironic
diff --git a/docker/services/keystone.yaml b/docker/services/keystone.yaml
index a751c054..ff1b4477 100644
--- a/docker/services/keystone.yaml
+++ b/docker/services/keystone.yaml
@@ -84,14 +84,15 @@ outputs:
         /var/lib/kolla/config_files/keystone.json:
           command: /usr/sbin/httpd -DFOREGROUND
       docker_config:
+        # Kolla_bootstrap/db sync runs before permissions set by kolla_config
         step_3:
-          keystone-init-log:
+          keystone_init_log:
             start_order: 0
             image: *keystone_image
             user: root
-            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && mkdir -p /var/log/keystone && chown keystone:keystone /var/log/keystone']
+            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R keystone:keystone /var/log/keystone']
             volumes:
-              - logs:/var/log
+              - /var/log/containers/keystone:/var/log/keystone
           keystone_db_sync:
             start_order: 1
             image: *keystone_image
@@ -106,7 +107,7 @@ outputs:
                   - /var/lib/config-data/keystone/var/www/:/var/www/:ro
                   - /var/lib/config-data/keystone/etc/keystone/:/etc/keystone/:ro
                   - /var/lib/config-data/keystone/etc/httpd/:/etc/httpd/:ro
-                  - logs:/var/log
+                  - /var/log/containers/keystone:/var/log/keystone
                   -
                     if:
                       - internal_tls_enabled
@@ -141,6 +142,11 @@ outputs:
           puppet_tags: 'keystone_config,keystone_domain_config,keystone_endpoint,keystone_identity_provider,keystone_paste_ini,keystone_role,keystone_service,keystone_tenant,keystone_user,keystone_user_role,keystone_domain'
           step_config: 'include ::tripleo::profile::base::keystone'
           config_image: *keystone_image
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/keystone
+            state: directory
       upgrade_tasks:
         - name: Stop and disable keystone service (running under httpd)
           tags: step2
diff --git a/docker/services/memcached.yaml b/docker/services/memcached.yaml
index f9d73f4d..c689662f 100644
--- a/docker/services/memcached.yaml
+++ b/docker/services/memcached.yaml
@@ -60,7 +60,17 @@ outputs:
       kolla_config: {}
       docker_config:
         step_1:
+          memcached_init_logs:
+            start_order: 0
+            image: *memcached_image
+            privileged: false
+            user: root
+            volumes:
+               - /var/lib/config-data/memcached/etc/sysconfig/memcached:/etc/sysconfig/memcached:ro
+               - /var/log/memcached.log:/var/log/memcached.log
+            command: ['/bin/bash', '-c', 'source /etc/sysconfig/memcached; chown ${USER} /var/log/memcached.log']
           memcached:
+            start_order: 1
             image: *memcached_image
             net: host
             privileged: false
@@ -70,6 +80,7 @@ outputs:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/memcached/etc/sysconfig/memcached:/etc/sysconfig/memcached:ro
+            # TODO(bogdando) capture memcached syslog logs from a container
             command: ['/bin/bash', '-c', 'source /etc/sysconfig/memcached; /usr/bin/memcached -p ${PORT} -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS']
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
diff --git a/docker/services/mistral-api.yaml b/docker/services/mistral-api.yaml
index 652656ef..0684ee0c 100644
--- a/docker/services/mistral-api.yaml
+++ b/docker/services/mistral-api.yaml
@@ -66,14 +66,27 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/mistral_api.json:
           command: /usr/bin/mistral-server --config-file=/etc/mistral/mistral.conf --log-file=/var/log/mistral/api.log --server=api
+          permissions:
+            - path: /var/log/mistral
+              owner: mistral:mistral
+              recurse: true
       docker_config:
+        # db sync runs before permissions set by kolla_config
         step_3:
-          mistral_db_sync:
-            start_order: 1
+          mistral_init_logs:
+            start_order: 0
             image: &mistral_image
               list_join:
                 - '/'
                 - [ {get_param: DockerNamespace}, {get_param: DockerMistralApiImage} ]
+            privileged: false
+            user: root
+            volumes:
+              - /var/log/containers/mistral:/var/log/mistral
+            command: ['/bin/bash', '-c', 'chown -R mistral:mistral /var/log/mistral']
+          mistral_db_sync:
+            start_order: 1
+            image: *mistral_image
             net: host
             privileged: false
             detach: false
@@ -82,6 +95,7 @@ outputs:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/mistral/etc/:/etc/:ro
+                  - /var/log/containers/mistral:/var/log/mistral
             command: ['mistral-db-manage', '--config-file', '/etc/mistral/mistral.conf', 'upgrade', 'head']
           mistral_db_populate:
             start_order: 2
@@ -94,6 +108,7 @@ outputs:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/mistral/etc/:/etc/:ro
+                  - /var/log/containers/mistral:/var/log/mistral
             # NOTE: dprince this requires that we install openstack-tripleo-common into
             # the Mistral API image so that we get tripleo* actions
             command: ['mistral-db-manage', '--config-file', '/etc/mistral/mistral.conf', 'populate']
@@ -110,8 +125,14 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/mistral_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/mistral/etc/mistral/:/etc/mistral/:ro
+                  - /var/log/containers/mistral:/var/log/mistral
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/mistral
+            state: directory
       upgrade_tasks:
         - name: Stop and disable mistral_api service
           tags: step2
diff --git a/docker/services/mistral-engine.yaml b/docker/services/mistral-engine.yaml
index 9d543da9..39ab5a87 100644
--- a/docker/services/mistral-engine.yaml
+++ b/docker/services/mistral-engine.yaml
@@ -67,6 +67,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/mistral_engine.json:
           command: /usr/bin/mistral-server --config-file=/etc/mistral/mistral.conf --log-file=/var/log/mistral/engine.log --server=engine
+          permissions:
+            - path: /var/log/mistral
+              owner: mistral:mistral
+              recurse: true
       docker_config:
         step_4:
           mistral_engine:
@@ -84,8 +88,14 @@ outputs:
                   - /run:/run
                   - /var/lib/kolla/config_files/mistral_engine.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/mistral/etc/mistral/:/etc/mistral/:ro
+                  - /var/log/containers/mistral:/var/log/mistral
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/mistral
+            state: directory
       upgrade_tasks:
         - name: Stop and disable mistral_engine service
           tags: step2
diff --git a/docker/services/mistral-executor.yaml b/docker/services/mistral-executor.yaml
index 9c3bfb33..d878bb53 100644
--- a/docker/services/mistral-executor.yaml
+++ b/docker/services/mistral-executor.yaml
@@ -67,6 +67,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/mistral_executor.json:
           command: /usr/bin/mistral-server --config-file=/etc/mistral/mistral.conf --log-file=/var/log/mistral/executor.log --server=executor
+          permissions:
+            - path: /var/log/mistral
+              owner: mistral:mistral
+              recurse: true
       docker_config:
         step_4:
           mistral_executor:
@@ -88,8 +92,14 @@ outputs:
                   # initialization workflows on the Undercloud. Need to
                   # exclude this on the overcloud for security reasons.
                   - /var/lib/config-data/nova/etc/nova:/etc/nova:ro
+                  - /var/log/containers/mistral:/var/log/mistral
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/mistral
+            state: directory
       upgrade_tasks:
         - name: Stop and disable mistral_executor service
           tags: step2
diff --git a/docker/services/neutron-api.yaml b/docker/services/neutron-api.yaml
index 06675089..7a422768 100644
--- a/docker/services/neutron-api.yaml
+++ b/docker/services/neutron-api.yaml
@@ -67,13 +67,27 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/neutron_api.json:
           command: /usr/bin/neutron-server --config-file /usr/share/neutron/neutron-dist.conf --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugin.ini
+          permissions:
+            - path: /var/log/neutron
+              owner: neutron:neutron
+              recurse: true
       docker_config:
+        # db sync runs before permissions set by kolla_config
         step_3:
-          neutron_db_sync:
+          neutron_init_logs:
+            start_order: 0
             image: &neutron_api_image
               list_join:
                 - '/'
                 - [ {get_param: DockerNamespace}, {get_param: DockerNeutronApiImage} ]
+            privileged: false
+            user: root
+            volumes:
+              - /var/log/containers/neutron:/var/log/neutron
+            command: ['/bin/bash', '-c', 'chown -R neutron:neutron /var/log/neutron']
+          neutron_db_sync:
+            start_order: 1
+            image: *neutron_api_image
             net: host
             privileged: false
             detach: false
@@ -86,6 +100,7 @@ outputs:
                 -
                   - /var/lib/config-data/neutron/etc/neutron:/etc/neutron:ro
                   - /var/lib/config-data/neutron/usr/share/neutron:/usr/share/neutron:ro
+                  - /var/log/containers/neutron:/var/log/neutron
             command: ['neutron-db-manage', 'upgrade', 'heads']
         step_4:
           neutron_api:
@@ -99,8 +114,14 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/neutron_api.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/neutron/etc/neutron/:/etc/neutron/:ro
+                  - /var/log/containers/neutron:/var/log/neutron
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/neutron
+            state: directory
       upgrade_tasks:
         - name: Stop and disable neutron_api service
           tags: step2
diff --git a/docker/services/neutron-dhcp.yaml b/docker/services/neutron-dhcp.yaml
index b17e97b1..985b2727 100644
--- a/docker/services/neutron-dhcp.yaml
+++ b/docker/services/neutron-dhcp.yaml
@@ -67,6 +67,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/neutron_dhcp.json:
           command: /usr/bin/neutron-dhcp-agent --config-file /usr/share/neutron/neutron-dist.conf --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/dhcp_agent.ini --log-file /var/log/neutron/dhcp-agent.log
+          permissions:
+            - path: /var/log/neutron
+              owner: neutron:neutron
+              recurse: true
       docker_config:
         step_4:
           neutron_dhcp:
@@ -86,8 +90,14 @@ outputs:
                   - /var/lib/config-data/neutron/etc/neutron/:/etc/neutron/:ro
                   - /lib/modules:/lib/modules:ro
                   - /run/:/run
+                  - /var/log/containers/neutron:/var/log/neutron
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/neutron
+            state: directory
       upgrade_tasks:
         - name: Stop and disable neutron_dhcp service
           tags: step2
diff --git a/docker/services/neutron-l3.yaml b/docker/services/neutron-l3.yaml
index c9441b11..77784ef8 100644
--- a/docker/services/neutron-l3.yaml
+++ b/docker/services/neutron-l3.yaml
@@ -63,6 +63,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/neutron-l3-agent.json:
           command: /usr/bin/neutron-l3-agent --config-file /usr/share/neutron/neutron-dist.conf --config-dir /usr/share/neutron/l3_agent --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/l3_agent.ini
+          permissions:
+            - path: /var/log/neutron
+              owner: neutron:neutron
+              recurse: true
       docker_config:
         step_4:
           neutronl3agent:
@@ -82,5 +86,11 @@ outputs:
                   - /var/lib/config-data/neutron/etc/neutron/:/etc/neutron/:ro
                   - /lib/modules:/lib/modules:ro
                   - /run:/run
+                  - /var/log/containers/neutron:/var/log/neutron
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/neutron
+            state: directory
diff --git a/docker/services/neutron-ovs-agent.yaml b/docker/services/neutron-ovs-agent.yaml
index 70851f7d..48b67abe 100644
--- a/docker/services/neutron-ovs-agent.yaml
+++ b/docker/services/neutron-ovs-agent.yaml
@@ -57,7 +57,11 @@ outputs:
           - [ {get_param: DockerNamespace}, {get_param: DockerOpenvswitchImage} ]
       kolla_config:
         /var/lib/kolla/config_files/neutron-openvswitch-agent.json:
-           command: /usr/bin/neutron-openvswitch-agent --config-file /usr/share/neutron/neutron-dist.conf --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/openvswitch_agent.ini --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
+          command: /usr/bin/neutron-openvswitch-agent --config-file /usr/share/neutron/neutron-dist.conf --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/openvswitch_agent.ini --config-file /etc/neutron/plugins/ml2/ml2_conf.ini
+          permissions:
+            - path: /var/log/neutron
+              owner: neutron:neutron
+              recurse: true
       docker_config:
         step_4:
           neutronovsagent:
@@ -74,8 +78,14 @@ outputs:
                   - /var/lib/config-data/neutron/etc/neutron/:/etc/neutron/:ro
                   - /lib/modules:/lib/modules:ro
                   - /run:/run
+                  - /var/log/containers/neutron:/var/log/neutron
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/neutron
+            state: directory
       upgrade_tasks:
         - name: Stop and disable neutron_ovs_agent service
           tags: step2
diff --git a/docker/services/nova-api.yaml b/docker/services/nova-api.yaml
index 6817fc7f..f7904a71 100644
--- a/docker/services/nova-api.yaml
+++ b/docker/services/nova-api.yaml
@@ -70,20 +70,36 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/nova_api.json:
           command: /usr/bin/nova-api
+          permissions:
+            - path: /var/log/nova
+              owner: nova:nova
+              recurse: true
       docker_config:
+        # db sync runs before permissions set by kolla_config
         step_3:
-          nova_api_db_sync:
-            start_order: 1
+          nova_init_logs:
+            start_order: 0
             image: &nova_api_image
               list_join:
                 - '/'
                 - [ {get_param: DockerNamespace}, {get_param: DockerNovaApiImage} ]
+            privileged: false
+            user: root
+            volumes:
+              - /var/log/containers/nova:/var/log/nova
+            command: ['/bin/bash', '-c', 'chown -R nova:nova /var/log/nova']
+          nova_api_db_sync:
+            start_order: 1
+            image: *nova_api_image
             net: host
             detach: false
             volumes: &nova_api_volumes
-              - /var/lib/config-data/nova/etc/:/etc/:ro
-              - /etc/hosts:/etc/hosts:ro
-              - /etc/localtime:/etc/localtime:ro
+              list_concat:
+                - {get_attr: [ContainersCommon, volumes]}
+                -
+                  - /var/lib/kolla/config_files/nova_api.json:/var/lib/kolla/config_files/config.json:ro
+                  - /var/lib/config-data/nova/etc/nova/:/etc/nova/:ro
+                  - /var/log/containers/nova:/var/log/nova
             command: ['/usr/bin/nova-manage', 'api_db', 'sync']
           # FIXME: we probably want to wait on the 'cell_v2 update' in order for this
           # to be capable of upgrading a baremetal setup. This is to ensure the name
@@ -128,12 +144,7 @@ outputs:
             user: nova
             privileged: true
             restart: always
-            volumes:
-              list_concat:
-                - {get_attr: [ContainersCommon, volumes]}
-                -
-                  - /var/lib/kolla/config_files/nova_api.json:/var/lib/kolla/config_files/config.json:ro
-                  - /var/lib/config-data/nova/etc/nova/:/etc/nova/:ro
+            volumes: *nova_api_volumes
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
         step_5:
@@ -147,6 +158,11 @@ outputs:
               - '/usr/bin/nova-manage'
               - 'cell_v2'
               - 'discover_hosts'
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/nova
+            state: directory
       upgrade_tasks:
         - name: Stop and disable nova_api service
           tags: step2
diff --git a/docker/services/nova-compute.yaml b/docker/services/nova-compute.yaml
index 624596ec..3e146740 100644
--- a/docker/services/nova-compute.yaml
+++ b/docker/services/nova-compute.yaml
@@ -66,6 +66,13 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/nova-compute.json:
           command: /usr/bin/nova-compute --config-file /etc/nova/nova.conf --config-file /etc/nova/rootwrap.conf
+          permissions:
+            - path: /var/log/nova
+              owner: nova:nova
+              recurse: true
+            - path: /var/lib/nova
+              owner: nova:nova
+              recurse: true
       docker_config:
         # FIXME: run discover hosts here
         step_4:
@@ -87,13 +94,18 @@ outputs:
                   - /run:/run
                   - /var/lib/nova:/var/lib/nova
                   - /var/lib/libvirt:/var/lib/libvirt
+                  - /var/log/containers/nova:/var/log/nova
             environment:
              - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
-        - name: create /var/lib/libvirt
+        - name: create persistent directories
           file:
-            path: /var/lib/libvirt
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/nova
+            - /var/lib/nova
+            - /var/lib/libvirt
       upgrade_tasks:
         - name: Stop and disable nova-compute service
           tags: step2
diff --git a/docker/services/nova-conductor.yaml b/docker/services/nova-conductor.yaml
index fc20422d..063e0167 100644
--- a/docker/services/nova-conductor.yaml
+++ b/docker/services/nova-conductor.yaml
@@ -65,6 +65,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/nova_conductor.json:
           command: /usr/bin/nova-conductor
+          permissions:
+            - path: /var/log/nova
+              owner: nova:nova
+              recurse: true
       docker_config:
         step_4:
           nova_conductor:
@@ -81,9 +85,14 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/nova_conductor.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/nova/etc/nova/:/etc/nova/:ro
-                  - /run:/run
+                  - /var/log/containers/nova:/var/log/nova
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/nova
+            state: directory
       upgrade_tasks:
         - name: Stop and disable nova_conductor service
           tags: step2
diff --git a/docker/services/nova-ironic.yaml b/docker/services/nova-ironic.yaml
index 3fd71d88..b0952b1d 100644
--- a/docker/services/nova-ironic.yaml
+++ b/docker/services/nova-ironic.yaml
@@ -59,7 +59,14 @@ outputs:
           - [ {get_param: DockerNamespace}, {get_param: DockerNovaConfigImage} ]
       kolla_config:
         /var/lib/kolla/config_files/nova_ironic.json:
-           command: /usr/bin/nova-compute --config-file /etc/nova/nova.conf --config-file /etc/nova/rootwrap.conf
+          command: /usr/bin/nova-compute --config-file /etc/nova/nova.conf --config-file /etc/nova/rootwrap.conf
+          permissions:
+            - path: /var/log/nova
+              owner: nova:nova
+              recurse: true
+            - path: /var/lib/nova
+              owner: nova:nova
+              recurse: true
       docker_config:
         step_5:
           novacompute:
@@ -81,8 +88,17 @@ outputs:
                   - /dev:/dev
                   - /etc/iscsi:/etc/iscsi
                   - /var/lib/nova/:/var/lib/nova
+                  - /var/log/containers/nova:/var/log/nova
             environment:
              - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent directories
+          file:
+            path: "{{ item }}"
+            state: directory
+          with_items:
+            - /var/log/containers/nova
+            - /var/lib/nova
       upgrade_tasks:
         - name: Stop and disable nova-compute service
           tags: step2
diff --git a/docker/services/nova-libvirt.yaml b/docker/services/nova-libvirt.yaml
index 1b103df4..775a9daa 100644
--- a/docker/services/nova-libvirt.yaml
+++ b/docker/services/nova-libvirt.yaml
@@ -72,6 +72,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/nova-libvirt.json:
           command: /usr/sbin/libvirtd --config /etc/libvirt/libvirtd.conf
+          permissions:
+            - path: /var/log/nova
+              owner: nova:nova
+              recurse: true
       docker_config:
         step_3:
           nova_libvirt:
@@ -98,6 +102,7 @@ outputs:
                   - /var/run/libvirt:/var/run/libvirt
                   - /var/lib/libvirt:/var/lib/libvirt
                   - /etc/libvirt/qemu:/etc/libvirt/qemu
+                  - /var/log/containers/nova:/var/log/nova
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
@@ -108,6 +113,7 @@ outputs:
           with_items:
             - /etc/libvirt/qemu
             - /var/lib/libvirt
+            - /var/log/containers/nova
       upgrade_tasks:
         - name: Stop and disable libvirtd service
           tags: step2
diff --git a/docker/services/nova-placement.yaml b/docker/services/nova-placement.yaml
index 7202ca42..944646e5 100644
--- a/docker/services/nova-placement.yaml
+++ b/docker/services/nova-placement.yaml
@@ -63,6 +63,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/nova_placement.json:
           command: /usr/sbin/httpd -DFOREGROUND
+          permissions:
+            - path: /var/log/nova
+              owner: nova:nova
+              recurse: true
       docker_config:
         # start this early so it is up before computes start reporting
         step_3:
@@ -80,8 +84,14 @@ outputs:
                   - /var/lib/config-data/nova_placement/etc/nova/:/etc/nova/:ro
                   - /var/lib/config-data/nova_placement/etc/httpd/:/etc/httpd/:ro
                   - /var/lib/config-data/nova_placement/var/www/:/var/www/:ro
+                  - /var/log/containers/nova:/var/log/nova
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/nova
+            state: directory
       upgrade_tasks:
         - name: Stop and disable nova_placement service (running under httpd)
           tags: step2
diff --git a/docker/services/nova-scheduler.yaml b/docker/services/nova-scheduler.yaml
index 9be24137..c6bf5c0a 100644
--- a/docker/services/nova-scheduler.yaml
+++ b/docker/services/nova-scheduler.yaml
@@ -64,6 +64,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/nova_scheduler.json:
           command: /usr/bin/nova-scheduler
+          permissions:
+            - path: /var/log/nova
+              owner: nova:nova
+              recurse: true
       docker_config:
         step_4:
           nova_scheduler:
@@ -81,8 +85,14 @@ outputs:
                   - /var/lib/kolla/config_files/nova_scheduler.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/nova/etc/nova/:/etc/nova/:ro
                   - /run:/run
+                  - /var/log/containers/nova:/var/log/nova
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/nova
+            state: directory
       upgrade_tasks:
         - name: Stop and disable nova_scheduler service
           tags: step2
diff --git a/docker/services/panko-api.yaml b/docker/services/panko-api.yaml
index cf0e1718..2eb50b35 100644
--- a/docker/services/panko-api.yaml
+++ b/docker/services/panko-api.yaml
@@ -72,15 +72,19 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/panko-api.json:
           command: /usr/sbin/httpd -DFOREGROUND
+          permissions:
+            - path: /var/log/panko
+              owner: panko:panko
+              recurse: true
       docker_config:
         step_3:
           panko-init-log:
             start_order: 0
             image: *panko_image
             user: root
-            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd && mkdir -p /var/log/panko && chown panko:panko /var/log/panko']
             volumes:
-              - logs:/var/log
+              - /var/log/containers/panko:/var/log/panko
+            command: ['/bin/bash', '-c', 'mkdir -p /var/log/httpd; chown -R panko:panko /var/log/panko']
           panko_db_sync:
             start_order: 1
             image: *panko_image
@@ -92,7 +96,7 @@ outputs:
                 - {get_attr: [ContainersCommon, volumes]}
                 -
                   - /var/lib/config-data/panko/etc/panko:/etc/panko:ro
-                  - logs:/var/log
+                  - /var/log/containers/panko:/var/log/panko
             command: /usr/bin/panko-dbsync
         step_4:
           panko_api:
@@ -109,6 +113,7 @@ outputs:
                   - /var/lib/config-data/panko/etc/panko/:/etc/panko/:ro
                   - /var/lib/config-data/panko/etc/httpd/:/etc/httpd/:ro
                   - /var/lib/config-data/panko/var/www/:/var/www/:ro
+                  - /var/log/containers/panko:/var/log/panko
                   -
                     if:
                       - internal_tls_enabled
@@ -121,5 +126,10 @@ outputs:
                       - ''
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/panko
+            state: directory
       metadata_settings:
         get_attr: [PankoApiPuppetBase, role_data, metadata_settings]
diff --git a/docker/services/rabbitmq.yaml b/docker/services/rabbitmq.yaml
index 3d647d5e..b6428fce 100644
--- a/docker/services/rabbitmq.yaml
+++ b/docker/services/rabbitmq.yaml
@@ -70,11 +70,24 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/rabbitmq.json:
           command: /usr/lib/rabbitmq/bin/rabbitmq-server
+          permissions:
+            - path: /var/lib/rabbitmq
+              owner: rabbitmq:rabbitmq
+              recurse: true
       docker_config:
+        # Kolla_bootstrap runs before permissions set by kolla_config
         step_1:
-          rabbitmq_bootstrap:
+          rabbitmq_init_logs:
             start_order: 0
             image: *rabbitmq_image
+            privileged: false
+            user: root
+            volumes:
+              - /var/log/containers/rabbitmq:/var/log/rabbitmq
+            command: ['/bin/bash', '-c', 'chown -R rabbitmq:rabbitmq /var/log/rabbitmq']
+          rabbitmq_bootstrap:
+            start_order: 1
+            image: *rabbitmq_image
             net: host
             privileged: false
             volumes:
@@ -84,6 +97,7 @@ outputs:
                   - /var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/rabbitmq/etc/rabbitmq/:/etc/rabbitmq/:ro
                   - /var/lib/rabbitmq:/var/lib/rabbitmq
+                  - /var/log/containers/rabbitmq:/var/log/rabbitmq
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
               - KOLLA_BOOTSTRAP=True
@@ -99,7 +113,7 @@ outputs:
                             - {get_param: RabbitCookie}
                             - {get_param: [DefaultPasswords, rabbit_cookie]}
           rabbitmq:
-            start_order: 1
+            start_order: 2
             image: *rabbitmq_image
             net: host
             privileged: false
@@ -111,6 +125,7 @@ outputs:
                   - /var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/rabbitmq/etc/rabbitmq/:/etc/rabbitmq/:ro
                   - /var/lib/rabbitmq:/var/lib/rabbitmq
+                  - /var/log/containers/rabbitmq:/var/log/rabbitmq
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       docker_puppet_tasks:
@@ -124,10 +139,13 @@ outputs:
             - /var/lib/config-data/rabbitmq/etc/:/etc/
             - /var/lib/rabbitmq:/var/lib/rabbitmq:ro
       host_prep_tasks:
-        - name: create /var/lib/rabbitmq
+        - name: create persistent directories
           file:
-            path: /var/lib/rabbitmq
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/rabbitmq
+            - /var/lib/rabbitmq
       upgrade_tasks:
         - name: Stop and disable rabbitmq service
           tags: step2
diff --git a/docker/services/swift-proxy.yaml b/docker/services/swift-proxy.yaml
index 8ea42222..bcf24c33 100644
--- a/docker/services/swift-proxy.yaml
+++ b/docker/services/swift-proxy.yaml
@@ -60,6 +60,10 @@ outputs:
       kolla_config:
         /var/lib/kolla/config_files/swift_proxy.json:
           command: /usr/bin/swift-proxy-server /etc/swift/proxy-server.conf
+          permissions:
+            - path: /var/log/swift
+              owner: swift:swift
+              recurse: true
       docker_config:
         step_4:
           swift_proxy:
@@ -78,13 +82,17 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
-        - name: create /srv/node
+        - name: create persistent directories
           file:
-            path: /srv/node
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/swift
+            - /srv/node
       upgrade_tasks:
         - name: Stop and disable swift_proxy service
           tags: step2
diff --git a/docker/services/swift-storage.yaml b/docker/services/swift-storage.yaml
index b4a6a940..6d60dde6 100644
--- a/docker/services/swift-storage.yaml
+++ b/docker/services/swift-storage.yaml
@@ -96,6 +96,10 @@ outputs:
           command: /usr/bin/swift-object-updater /etc/swift/object-server.conf
         /var/lib/kolla/config_files/swift_object_server.json:
           command: /usr/bin/swift-object-server /etc/swift/object-server.conf
+          permissions:
+            - path: /var/log/swift
+              owner: swift:swift
+              recurse: true
       docker_config:
         step_3:
           # The puppet config sets this up but we don't have a way to mount the named
@@ -125,6 +129,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: &kolla_env
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
           swift_account_reaper:
@@ -141,6 +146,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
           swift_account_replicator:
             image: *swift_account_image
@@ -156,6 +162,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
           swift_account_server:
             image: *swift_account_image
@@ -171,6 +178,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
           swift_container_auditor:
             image: &swift_container_image
@@ -189,6 +197,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
           swift_container_replicator:
             image: *swift_container_image
@@ -204,6 +213,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
           swift_container_updater:
             image: *swift_container_image
@@ -219,6 +229,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
           swift_container_server:
             image: *swift_container_image
@@ -234,6 +245,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
           swift_object_auditor:
             image: &swift_object_image
@@ -252,6 +264,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
           swift_object_expirer:
             image: *swift_proxy_image
@@ -267,6 +280,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
           swift_object_replicator:
             image: *swift_object_image
@@ -282,6 +296,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
           swift_object_updater:
             image: *swift_object_image
@@ -297,6 +312,7 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
           swift_object_server:
             image: *swift_object_image
@@ -312,12 +328,16 @@ outputs:
                   - /run:/run
                   - /srv/node:/srv/node
                   - /dev:/dev
+                  - /var/log/containers/swift:/var/log/swift
             environment: *kolla_env
       host_prep_tasks:
-        - name: create /srv/node
+        - name: create persistent directories
           file:
-            path: /srv/node
+            path: "{{ item }}"
             state: directory
+          with_items:
+            - /var/log/containers/swift
+            - /srv/node
       upgrade_tasks:
         - name: Stop and disable swift storage services
           tags: step2
diff --git a/docker/services/zaqar.yaml b/docker/services/zaqar.yaml
index c450fe2f..5ba044ea 100644
--- a/docker/services/zaqar.yaml
+++ b/docker/services/zaqar.yaml
@@ -62,6 +62,10 @@ outputs:
           command: /usr/bin/zaqar-server --config-file /etc/zaqar/zaqar.conf
         /var/lib/kolla/config_files/zaqar_websocket.json:
           command: /usr/bin/zaqar-server --config-file /etc/zaqar/zaqar.conf --config-file /etc/zaqar/1.conf
+          permissions:
+            - path: /var/log/zaqar
+              owner: zaqar:zaqar
+              recurse: true
       docker_config:
         step_4:
           zaqar:
@@ -75,6 +79,9 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/zaqar.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/zaqar/etc/zaqar/:/etc/zaqar/:ro
+                  - /var/lib/config-data/zaqar/var/www/:/var/www/:ro
+                  - /var/lib/config-data/zaqar/etc/httpd/:/etc/httpd/:ro
+                  - /var/log/containers/zaqar:/var/log/zaqar
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
           zaqar_websocket:
@@ -88,8 +95,16 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/zaqar_websocket.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/zaqar/etc/zaqar/:/etc/zaqar/:ro
+                  - /var/lib/config-data/zaqar/var/www/:/var/www/:ro
+                  - /var/lib/config-data/zaqar/etc/httpd/:/etc/httpd/:ro
+                  - /var/log/containers/zaqar:/var/log/zaqar
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      host_prep_tasks:
+        - name: create persistent logs directory
+          file:
+            path: /var/log/containers/zaqar
+            state: directory
       upgrade_tasks:
         - name: Stop and disable zaqar service
           tags: step2