Support configurable Zaqar backends

This patch adds parameters to configure alternative version
of the Zaqar messaging and management backends.

The intent is to make use of these settings in the
containers undercloud to use swift/mysql backends as a default
thus avoiding the dependency on MongoDB.

Change-Id: Ifd6a561737184c9322192ffc9a412c77d6eac3e9
Depends-On: Ie6a56b9163950cee2c0341afa0c0ddce665f3704
Depends-On: I3598e39c0a3cdf80b96e728d9aa8a7e6505e0690

4 files changed, 141 insertions, 49 deletions

diff --git a/docker/services/zaqar.yaml b/docker/services/zaqar.yaml
index 85a84550..061a4a70 100644
--- a/docker/services/zaqar.yaml
+++ b/docker/services/zaqar.yaml
@@ -10,6 +10,10 @@ parameters:
   DockerZaqarConfigImage:
     description: The container image to use for the zaqar config_volume
     type: string
+  ZaqarManagementStore:
+    type: string
+    description: The management store for Zaqar
+    default: mongodb
   EndpointMap:
     default: {}
     description: Mapping of service endpoint -> protocol. Typically set
@@ -37,6 +41,9 @@ parameters:
     description: Parameters specific to the role
     type: json
 
+conditions:
+  zaqar_management_store_sqlalchemy: {equals : [{get_param: ZaqarManagementStore}, 'sqlalchemy']}
+
 resources:
 
   ContainersCommon:
@@ -87,38 +94,65 @@ outputs:
               owner: zaqar:zaqar
               recurse: true
       docker_config:
-        step_4:
-          zaqar:
-            image: &zaqar_image {get_param: DockerZaqarImage}
-            net: host
-            privileged: false
-            restart: always
-            # NOTE(mandre) kolla image changes the user to 'zaqar', we need it
-            # to be root to run httpd
-            user: root
-            volumes:
-              list_concat:
-                - {get_attr: [ContainersCommon, volumes]}
-                -
-                  - /var/lib/kolla/config_files/zaqar.json:/var/lib/kolla/config_files/config.json:ro
-                  - /var/lib/config-data/puppet-generated/zaqar/:/var/lib/kolla/config_files/src:ro
-                  - /var/log/containers/zaqar:/var/log/zaqar
-            environment:
-              - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
-          zaqar_websocket:
-            image: *zaqar_image
-            net: host
-            privileged: false
-            restart: always
-            volumes:
-              list_concat:
-                - {get_attr: [ContainersCommon, volumes]}
-                -
-                  - /var/lib/kolla/config_files/zaqar_websocket.json:/var/lib/kolla/config_files/config.json:ro
-                  - /var/lib/config-data/puppet-generated/zaqar/:/var/lib/kolla/config_files/src:ro
-                  - /var/log/containers/zaqar:/var/log/zaqar
-            environment:
-              - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+        map_merge:
+          -
+            if:
+            - zaqar_management_store_sqlalchemy
+            -
+              step_2:
+                zaqar_init_log:
+                  image: &zaqar_image {get_param: DockerZaqarImage}
+                  user: root
+                  volumes:
+                    - /var/log/containers/zaqar:/var/log/zaqar
+                  command: ['/bin/bash', '-c', 'chown -R zaqar:zaqar /var/log/zaqar']
+              step_3:
+                zaqar_db_sync:
+                  image: *zaqar_image
+                  net: host
+                  privileged: false
+                  detach: false
+                  user: root
+                  volumes:
+                    list_concat:
+                      - {get_attr: [ContainersCommon, volumes]}
+                      -
+                        - /var/lib/config-data/zaqar/etc/zaqar/:/etc/zaqar/:ro
+                        - /var/log/containers/zaqar:/var/log/zaqar
+                  command: "/usr/bin/bootstrap_host_exec zaqar su zaqar -s /bin/bash -c 'zaqar-sql-db-manage upgrade head'"
+            - {}
+          - step_4:
+              zaqar:
+                image: *zaqar_image
+                net: host
+                privileged: false
+                restart: always
+                # NOTE(mandre) kolla image changes the user to 'zaqar', we need it
+                # to be root to run httpd
+                user: root
+                volumes:
+                  list_concat:
+                    - {get_attr: [ContainersCommon, volumes]}
+                    -
+                      - /var/lib/kolla/config_files/zaqar.json:/var/lib/kolla/config_files/config.json:ro
+                      - /var/lib/config-data/puppet-generated/zaqar/:/var/lib/kolla/config_files/src:ro
+                      - /var/log/containers/zaqar:/var/log/zaqar
+                environment:
+                  - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+              zaqar_websocket:
+                image: *zaqar_image
+                net: host
+                privileged: false
+                restart: always
+                volumes:
+                  list_concat:
+                    - {get_attr: [ContainersCommon, volumes]}
+                    -
+                      - /var/lib/kolla/config_files/zaqar_websocket.json:/var/lib/kolla/config_files/config.json:ro
+                      - /var/lib/config-data/puppet-generated/zaqar/:/var/lib/kolla/config_files/src:ro
+                      - /var/log/containers/zaqar:/var/log/zaqar
+                environment:
+                  - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
       host_prep_tasks:
         - name: create persistent logs directory
           file:
diff --git a/puppet/services/zaqar.yaml b/puppet/services/zaqar.yaml
index debdc742..21857423 100644
--- a/puppet/services/zaqar.yaml
+++ b/puppet/services/zaqar.yaml
@@ -56,6 +56,14 @@ parameters:
     type: string
     description: Set the number of workers for zaqar::wsgi::apache
     default: '%{::os_workers}'
+  ZaqarMessageStore:
+    type: string
+    description: The messaging store for Zaqar
+    default: mongodb
+  ZaqarManagementStore:
+    type: string
+    description: The management store for Zaqar
+    default: mongodb
   EnableInternalTLS:
     type: boolean
     default: false
@@ -63,6 +71,8 @@ parameters:
 conditions:
   zaqar_workers_zero: {equals : [{get_param: ZaqarWorkers}, 0]}
   service_debug_unset: {equals : [{get_param: ZaqarDebug}, '']}
+  zaqar_messaging_store_swift: {equals : [{get_param: ZaqarMessageStore}, 'swift']}
+  zaqar_management_store_sqlalchemy: {equals : [{get_param: ZaqarManagementStore}, 'sqlalchemy']}
 
 resources:
 
@@ -105,26 +115,67 @@ outputs:
                   "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, ZaqarApiNetwork]}
+            zaqar::message_store: {get_param: ZaqarMessageStore}
+            zaqar::management_store: {get_param: ZaqarManagementStore}
+          -
+            if:
+            - zaqar_messaging_store_swift
+            -
+              zaqar::messaging::swift::uri:
+                list_join:
+                  - ''
+                  - ['swift://zaqar:', {get_param: ZaqarPassword}, '@/service']
+              zaqar::messaging::swift::auth_url: {get_param: [EndpointMap, KeystoneInternal, uri_no_suffix] }
+              tripleo::profile::base::zaqar::messaging_store: 'swift'
+            - {}
+          -
+            if:
+            - zaqar_management_store_sqlalchemy
+            -
+              tripleo::profile::base::zaqar::management_store: 'sqlalchemy'
+              zaqar::management::sqlalchemy::uri:
+                make_url:
+                  scheme: {get_param: [EndpointMap, MysqlInternal, protocol]}
+                  username: zaqar
+                  password: {get_param: ZaqarPassword}
+                  host: {get_param: [EndpointMap, MysqlInternal, host]}
+                  path: /zaqar
+                  query:
+                    read_default_file: /etc/my.cnf.d/tripleo.cnf
+                    read_default_group: tripleo
+            - {}
           -
             if:
             - zaqar_workers_zero
             - {}
             - zaqar::wsgi::apache::workers: {get_param: ZaqarWorkers}
       service_config_settings:
-        keystone:
-          zaqar::keystone::auth::password: {get_param: ZaqarPassword}
-          zaqar::keystone::auth::public_url: {get_param: [EndpointMap, ZaqarPublic, uri]}
-          zaqar::keystone::auth::admin_url: {get_param: [EndpointMap, ZaqarAdmin, uri]}
-          zaqar::keystone::auth::internal_url: {get_param: [EndpointMap, ZaqarInternal, uri]}
-          zaqar::keystone::auth::region: {get_param: KeystoneRegion}
-          zaqar::keystone::auth::tenant: 'service'
-          zaqar::keystone::auth_websocket::password: {get_param: ZaqarPassword}
-          zaqar::keystone::auth_websocket::public_url: {get_param: [EndpointMap, ZaqarWebSocketPublic, uri]}
-          zaqar::keystone::auth_websocket::admin_url: {get_param: [EndpointMap, ZaqarWebSocketAdmin, uri]}
-          zaqar::keystone::auth_websocket::internal_url: {get_param: [EndpointMap, ZaqarWebSocketInternal, uri]}
-          zaqar::keystone::auth_websocket::region: {get_param: KeystoneRegion}
-          zaqar::keystone::auth_websocket::tenant: 'service'
-
+        map_merge:
+          - keystone:
+              zaqar::keystone::auth::password: {get_param: ZaqarPassword}
+              zaqar::keystone::auth::public_url: {get_param: [EndpointMap, ZaqarPublic, uri]}
+              zaqar::keystone::auth::admin_url: {get_param: [EndpointMap, ZaqarAdmin, uri]}
+              zaqar::keystone::auth::internal_url: {get_param: [EndpointMap, ZaqarInternal, uri]}
+              zaqar::keystone::auth::region: {get_param: KeystoneRegion}
+              zaqar::keystone::auth::tenant: 'service'
+              zaqar::keystone::auth_websocket::password: {get_param: ZaqarPassword}
+              zaqar::keystone::auth_websocket::public_url: {get_param: [EndpointMap, ZaqarWebSocketPublic, uri]}
+              zaqar::keystone::auth_websocket::admin_url: {get_param: [EndpointMap, ZaqarWebSocketAdmin, uri]}
+              zaqar::keystone::auth_websocket::internal_url: {get_param: [EndpointMap, ZaqarWebSocketInternal, uri]}
+              zaqar::keystone::auth_websocket::region: {get_param: KeystoneRegion}
+              zaqar::keystone::auth_websocket::tenant: 'service'
+          -
+            if:
+            - zaqar_management_store_sqlalchemy
+            - mysql:
+                zaqar::db::mysql::user: zaqar
+                zaqar::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
+                zaqar::db::mysql::dbname: zaqar
+                zaqar::db::mysql::password: {get_param: ZaqarPassword}
+                zaqar::db::mysql::allowed_hosts:
+                  - '%'
+                  - "%{hiera('mysql_bind_host')}"
+            - {}
       step_config: |
         include ::tripleo::profile::base::zaqar
       upgrade_tasks:
diff --git a/releasenotes/notes/zaqar_backends-ccf8adfd24a17bf5.yaml b/releasenotes/notes/zaqar_backends-ccf8adfd24a17bf5.yaml
new file mode 100644
index 00000000..a72da829
--- /dev/null
+++ b/releasenotes/notes/zaqar_backends-ccf8adfd24a17bf5.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    Add Heat parameters which allow the end user to configure custom
+    management and messaging backends for MySQL and Swift.
diff --git a/tools/yaml-validate.py b/tools/yaml-validate.py
index 3828766f..d35cba63 100755
--- a/tools/yaml-validate.py
+++ b/tools/yaml-validate.py
@@ -219,11 +219,13 @@ def validate_docker_service(filename, tpl):
         if 'docker_config' in role_data:
             docker_config = role_data['docker_config']
             for _, step in docker_config.items():
+                if not isinstance(step, dict):
+                    # NOTE(mandre) this skips everything that is not a dict
+                    # so we may ignore some containers definitions if they
+                    # are in a map_merge for example
+                    continue
                 for _, container in step.items():
                     if not isinstance(container, dict):
-                        # NOTE(mandre) this skips everything that is not a dict
-                        # so we may ignore some containers definitions if they
-                        # are in a map_merge for example
                         continue
                     command = container.get('command', '')
                     if isinstance(command, list):