Composable Mistral services

Adds new puppet specific services for Mistral
API and Mistral Engine.

This submission enables the mistral service by default in the
overcloud, a following submission will disable it and make it
optional by enabling it on demand based in an environment file.

Depends-On: Iae42ffa37c4c9b1e070b7c3753e04c45bb97703f
Depends-On: I942d419be951651e305d01460f394870c30a9878
Depends-On: I6cb2cbf4a2abf494668d24b8c36b0d525643f0af
Implements: blueprint composable-services-within-roles
Co-Authored-By: Carlos Camacho <ccamacho@redhat.com>

Change-Id: Id5ff9cb498b5a47af38413d211ff0ed6ccd0015b

11 files changed, 498 insertions, 0 deletions

diff --git a/environments/tls-endpoints-public-dns.yaml b/environments/tls-endpoints-public-dns.yaml
index 0a0996d3..f94a7726 100644
--- a/environments/tls-endpoints-public-dns.yaml
+++ b/environments/tls-endpoints-public-dns.yaml
@@ -37,6 +37,9 @@ parameter_defaults:
     ManilaAdmin: {protocol: 'http', port: '8786', host: 'IP_ADDRESS'}
     ManilaInternal: {protocol: 'http', port: '8786', host: 'IP_ADDRESS'}
     ManilaPublic: {protocol: 'https', port: '13786', host: 'CLOUDNAME'}
+    MistralAdmin: {protocol: 'http', port: '8989', host: 'IP_ADDRESS'}
+    MistralInternal: {protocol: 'http', port: '8989', host: 'IP_ADDRESS'}
+    MistralPublic: {protocol: 'https', port: '13989', host: 'CLOUDNAME'}
     MysqlInternal: {protocol: 'mysql+pymysql', port: '3306', host: 'IP_ADDRESS'}
     NeutronAdmin: {protocol: 'http', port: '9696', host: 'IP_ADDRESS'}
     NeutronInternal: {protocol: 'http', port: '9696', host: 'IP_ADDRESS'}
diff --git a/environments/tls-endpoints-public-ip.yaml b/environments/tls-endpoints-public-ip.yaml
index 5a2b8839..eb2a23b4 100644
--- a/environments/tls-endpoints-public-ip.yaml
+++ b/environments/tls-endpoints-public-ip.yaml
@@ -37,6 +37,9 @@ parameter_defaults:
     ManilaAdmin: {protocol: 'http', port: '8786', host: 'IP_ADDRESS'}
     ManilaInternal: {protocol: 'http', port: '8786', host: 'IP_ADDRESS'}
     ManilaPublic: {protocol: 'https', port: '13786', host: 'IP_ADDRESS'}
+    MistralAdmin: {protocol: 'http', port: '8989', host: 'IP_ADDRESS'}
+    MistralInternal: {protocol: 'http', port: '8989', host: 'IP_ADDRESS'}
+    MistralPublic: {protocol: 'https', port: '13989', host: 'IP_ADDRESS'}
     MysqlInternal: {protocol: 'mysql+pymysql', port: '3306', host: 'IP_ADDRESS'}
     NeutronAdmin: {protocol: 'http', port: '9696', host: 'IP_ADDRESS'}
     NeutronInternal: {protocol: 'http', port: '9696', host: 'IP_ADDRESS'}
diff --git a/environments/tls-everywhere-endpoints-dns.yaml b/environments/tls-everywhere-endpoints-dns.yaml
index 88a108a6..b6c36df1 100644
--- a/environments/tls-everywhere-endpoints-dns.yaml
+++ b/environments/tls-everywhere-endpoints-dns.yaml
@@ -34,6 +34,9 @@ parameter_defaults:
     ManilaAdmin: {protocol: 'https', port: '8786', host: 'CLOUDNAME'}
     ManilaInternal: {protocol: 'https', port: '8786', host: 'CLOUDNAME'}
     ManilaPublic: {protocol: 'https', port: '13786', host: 'CLOUDNAME'}
+    MistralAdmin: {protocol: 'https', port: '8989', host: 'CLOUDNAME'}
+    MistralInternal: {protocol: 'https', port: '8989', host: 'CLOUDNAME'}
+    MistralPublic: {protocol: 'https', port: '13989', host: 'CLOUDNAME'}
     MysqlInternal: {protocol: 'mysql+pymysql', port: '3306', host: 'CLOUDNAME'}
     NeutronAdmin: {protocol: 'https', port: '9696', host: 'CLOUDNAME'}
     NeutronInternal: {protocol: 'https', port: '9696', host: 'CLOUDNAME'}
diff --git a/network/endpoints/endpoint_data.yaml b/network/endpoints/endpoint_data.yaml
index fb01925b..011dea7d 100644
--- a/network/endpoints/endpoint_data.yaml
+++ b/network/endpoints/endpoint_data.yaml
@@ -148,6 +148,21 @@ Manila:
             V1: /v1/%(tenant_id)s
     port: 8786
 
+Mistral:
+    Internal:
+        net_param: MistralApi
+        uri_suffixes:
+            '': /v2
+    Public:
+        net_param: Public
+        uri_suffixes:
+            '': /v2
+    Admin:
+        net_param: MistralApi
+        uri_suffixes:
+            '': /v2
+    port: 8989
+
 Neutron:
     Internal:
         net_param: NeutronApi
diff --git a/network/endpoints/endpoint_map.yaml b/network/endpoints/endpoint_map.yaml
index 734b6431..ac519a5f 100644
--- a/network/endpoints/endpoint_map.yaml
+++ b/network/endpoints/endpoint_map.yaml
@@ -54,6 +54,9 @@ parameters:
       ManilaAdmin: {protocol: http, port: '8786', host: IP_ADDRESS}
       ManilaInternal: {protocol: http, port: '8786', host: IP_ADDRESS}
       ManilaPublic: {protocol: http, port: '8786', host: IP_ADDRESS}
+      MistralAdmin: {protocol: http, port: '8989', host: IP_ADDRESS}
+      MistralInternal: {protocol: http, port: '8989', host: IP_ADDRESS}
+      MistralPublic: {protocol: http, port: '8989', host: IP_ADDRESS}
       MysqlInternal: {protocol: mysql+pymysql, port: '3306', host: IP_ADDRESS}
       NeutronAdmin: {protocol: http, port: '9696', host: IP_ADDRESS}
       NeutronInternal: {protocol: http, port: '9696', host: IP_ADDRESS}
@@ -4003,6 +4006,252 @@ outputs:
                         template: NETWORK_uri
             - ':'
             - get_param: [EndpointMap, ManilaPublic, port]
+      MistralAdmin:
+        host:
+          str_replace:
+            template:
+              get_param: [EndpointMap, MistralAdmin, host]
+            params:
+              CLOUDNAME:
+                get_param:
+                - CloudEndpoints
+                - get_param: [ServiceNetMap, MistralApiNetwork]
+              IP_ADDRESS:
+                get_param:
+                - NetIpMap
+                - str_replace:
+                    params:
+                      NETWORK:
+                        get_param: [ServiceNetMap, MistralApiNetwork]
+                    template: NETWORK_uri
+        host_nobrackets:
+          str_replace:
+            template:
+              get_param: [EndpointMap, MistralAdmin, host]
+            params:
+              CLOUDNAME:
+                get_param:
+                - CloudEndpoints
+                - get_param: [ServiceNetMap, MistralApiNetwork]
+              IP_ADDRESS:
+                get_param:
+                - NetIpMap
+                - get_param: [ServiceNetMap, MistralApiNetwork]
+        port:
+          get_param: [EndpointMap, MistralAdmin, port]
+        protocol:
+          get_param: [EndpointMap, MistralAdmin, protocol]
+        uri:
+          list_join:
+          - ''
+          - - get_param: [EndpointMap, MistralAdmin, protocol]
+            - ://
+            - str_replace:
+                template:
+                  get_param: [EndpointMap, MistralAdmin, host]
+                params:
+                  CLOUDNAME:
+                    get_param:
+                    - CloudEndpoints
+                    - get_param: [ServiceNetMap, MistralApiNetwork]
+                  IP_ADDRESS:
+                    get_param:
+                    - NetIpMap
+                    - str_replace:
+                        params:
+                          NETWORK:
+                            get_param: [ServiceNetMap, MistralApiNetwork]
+                        template: NETWORK_uri
+            - ':'
+            - get_param: [EndpointMap, MistralAdmin, port]
+            - /v2
+        uri_no_suffix:
+          list_join:
+          - ''
+          - - get_param: [EndpointMap, MistralAdmin, protocol]
+            - ://
+            - str_replace:
+                template:
+                  get_param: [EndpointMap, MistralAdmin, host]
+                params:
+                  CLOUDNAME:
+                    get_param:
+                    - CloudEndpoints
+                    - get_param: [ServiceNetMap, MistralApiNetwork]
+                  IP_ADDRESS:
+                    get_param:
+                    - NetIpMap
+                    - str_replace:
+                        params:
+                          NETWORK:
+                            get_param: [ServiceNetMap, MistralApiNetwork]
+                        template: NETWORK_uri
+            - ':'
+            - get_param: [EndpointMap, MistralAdmin, port]
+      MistralInternal:
+        host:
+          str_replace:
+            template:
+              get_param: [EndpointMap, MistralInternal, host]
+            params:
+              CLOUDNAME:
+                get_param:
+                - CloudEndpoints
+                - get_param: [ServiceNetMap, MistralApiNetwork]
+              IP_ADDRESS:
+                get_param:
+                - NetIpMap
+                - str_replace:
+                    params:
+                      NETWORK:
+                        get_param: [ServiceNetMap, MistralApiNetwork]
+                    template: NETWORK_uri
+        host_nobrackets:
+          str_replace:
+            template:
+              get_param: [EndpointMap, MistralInternal, host]
+            params:
+              CLOUDNAME:
+                get_param:
+                - CloudEndpoints
+                - get_param: [ServiceNetMap, MistralApiNetwork]
+              IP_ADDRESS:
+                get_param:
+                - NetIpMap
+                - get_param: [ServiceNetMap, MistralApiNetwork]
+        port:
+          get_param: [EndpointMap, MistralInternal, port]
+        protocol:
+          get_param: [EndpointMap, MistralInternal, protocol]
+        uri:
+          list_join:
+          - ''
+          - - get_param: [EndpointMap, MistralInternal, protocol]
+            - ://
+            - str_replace:
+                template:
+                  get_param: [EndpointMap, MistralInternal, host]
+                params:
+                  CLOUDNAME:
+                    get_param:
+                    - CloudEndpoints
+                    - get_param: [ServiceNetMap, MistralApiNetwork]
+                  IP_ADDRESS:
+                    get_param:
+                    - NetIpMap
+                    - str_replace:
+                        params:
+                          NETWORK:
+                            get_param: [ServiceNetMap, MistralApiNetwork]
+                        template: NETWORK_uri
+            - ':'
+            - get_param: [EndpointMap, MistralInternal, port]
+            - /v2
+        uri_no_suffix:
+          list_join:
+          - ''
+          - - get_param: [EndpointMap, MistralInternal, protocol]
+            - ://
+            - str_replace:
+                template:
+                  get_param: [EndpointMap, MistralInternal, host]
+                params:
+                  CLOUDNAME:
+                    get_param:
+                    - CloudEndpoints
+                    - get_param: [ServiceNetMap, MistralApiNetwork]
+                  IP_ADDRESS:
+                    get_param:
+                    - NetIpMap
+                    - str_replace:
+                        params:
+                          NETWORK:
+                            get_param: [ServiceNetMap, MistralApiNetwork]
+                        template: NETWORK_uri
+            - ':'
+            - get_param: [EndpointMap, MistralInternal, port]
+      MistralPublic:
+        host:
+          str_replace:
+            template:
+              get_param: [EndpointMap, MistralPublic, host]
+            params:
+              CLOUDNAME:
+                get_param:
+                - CloudEndpoints
+                - get_param: [ServiceNetMap, PublicNetwork]
+              IP_ADDRESS:
+                get_param:
+                - NetIpMap
+                - str_replace:
+                    params:
+                      NETWORK:
+                        get_param: [ServiceNetMap, PublicNetwork]
+                    template: NETWORK_uri
+        host_nobrackets:
+          str_replace:
+            template:
+              get_param: [EndpointMap, MistralPublic, host]
+            params:
+              CLOUDNAME:
+                get_param:
+                - CloudEndpoints
+                - get_param: [ServiceNetMap, PublicNetwork]
+              IP_ADDRESS:
+                get_param:
+                - NetIpMap
+                - get_param: [ServiceNetMap, PublicNetwork]
+        port:
+          get_param: [EndpointMap, MistralPublic, port]
+        protocol:
+          get_param: [EndpointMap, MistralPublic, protocol]
+        uri:
+          list_join:
+          - ''
+          - - get_param: [EndpointMap, MistralPublic, protocol]
+            - ://
+            - str_replace:
+                template:
+                  get_param: [EndpointMap, MistralPublic, host]
+                params:
+                  CLOUDNAME:
+                    get_param:
+                    - CloudEndpoints
+                    - get_param: [ServiceNetMap, PublicNetwork]
+                  IP_ADDRESS:
+                    get_param:
+                    - NetIpMap
+                    - str_replace:
+                        params:
+                          NETWORK:
+                            get_param: [ServiceNetMap, PublicNetwork]
+                        template: NETWORK_uri
+            - ':'
+            - get_param: [EndpointMap, MistralPublic, port]
+            - /v2
+        uri_no_suffix:
+          list_join:
+          - ''
+          - - get_param: [EndpointMap, MistralPublic, protocol]
+            - ://
+            - str_replace:
+                template:
+                  get_param: [EndpointMap, MistralPublic, host]
+                params:
+                  CLOUDNAME:
+                    get_param:
+                    - CloudEndpoints
+                    - get_param: [ServiceNetMap, PublicNetwork]
+                  IP_ADDRESS:
+                    get_param:
+                    - NetIpMap
+                    - str_replace:
+                        params:
+                          NETWORK:
+                            get_param: [ServiceNetMap, PublicNetwork]
+                        template: NETWORK_uri
+            - ':'
+            - get_param: [EndpointMap, MistralPublic, port]
       MysqlInternal:
         host:
           str_replace:
diff --git a/network/service_net_map.j2.yaml b/network/service_net_map.j2.yaml
index c4d86fb9..ac05fc73 100644
--- a/network/service_net_map.j2.yaml
+++ b/network/service_net_map.j2.yaml
@@ -56,6 +56,7 @@ parameters:
       CephRgwNetwork: storage
       PublicNetwork: external
       OpendaylightApiNetwork: internal_api
+      MistralApiNetwork: internal_api
       # We special-case the default ResolveNetwork for the CephStorage role
       # for backwards compatibility, all other roles default to internal_api
       CephStorageHostnameResolveNetwork: storage
diff --git a/overcloud.j2.yaml b/overcloud.j2.yaml
index 2e6412db..d8d38c2a 100644
--- a/overcloud.j2.yaml
+++ b/overcloud.j2.yaml
@@ -577,6 +577,9 @@ outputs:
   GnocchiInternalVip:
     description: VIP for Gnocchi API internal endpoint
     value: {get_attr: [VipMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, GnocchiApiNetwork]}]}
+  MistralInternalVip:
+    description: VIP for Mistral API internal endpoint
+    value: {get_attr: [VipMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, MistralApiNetwork]}]}
   HeatInternalVip:
     description: VIP for Heat API internal endpoint
     value: {get_attr: [VipMap, net_ip_map, {get_attr: [ServiceNetMap, service_net_map, HeatApiNetwork]}]}
diff --git a/puppet/services/mistral-api.yaml b/puppet/services/mistral-api.yaml
new file mode 100644
index 00000000..44d30358
--- /dev/null
+++ b/puppet/services/mistral-api.yaml
@@ -0,0 +1,52 @@
+heat_template_version: 2016-04-08
+
+description: >
+  Openstack Mistral API service configured with Puppet
+
+parameters:
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+  MistralWorkers:
+    default: 1
+    description: The number of workers for the mistral-api.
+    type: number
+
+resources:
+  MistralBase:
+    type: ./mistral-base.yaml
+    properties:
+      ServiceNetMap: {get_param: ServiceNetMap}
+      DefaultPasswords: {get_param: DefaultPasswords}
+      EndpointMap: {get_param: EndpointMap}
+
+outputs:
+  role_data:
+    description: Role data for the Mistral API role.
+    value:
+      service_name: mistral_api
+      config_settings:
+        map_merge:
+          - get_attr: [MistralBase, role_data, config_settings]
+          - mistral::api::api_workers: {get_param: MistralWorkers}
+            mistral::api::bind_host: {get_param: [ServiceNetMap, MistralApiNetwork]}
+            tripleo.mistral_api.firewall_rules:
+              '133 mistral':
+                dport:
+                  - 8989
+                  - 13989
+      service_config_settings:
+        get_attr: [MistralBase, role_data, service_config_settings]
+      step_config: |
+        include ::tripleo::profile::base::mistral::api
diff --git a/puppet/services/mistral-base.yaml b/puppet/services/mistral-base.yaml
new file mode 100644
index 00000000..a11624c0
--- /dev/null
+++ b/puppet/services/mistral-base.yaml
@@ -0,0 +1,93 @@
+heat_template_version: 2016-04-08
+
+description: >
+  Openstack Mistral base service. Shared for all Mistral services.
+
+parameters:
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+  Debug:
+    default: ''
+    description: Set to True to enable debugging on all services.
+    type: string
+  RabbitPassword:
+    description: The password for RabbitMQ
+    type: string
+    hidden: true
+  RabbitUserName:
+    default: guest
+    description: The username for RabbitMQ
+    type: string
+  RabbitClientUseSSL:
+    default: false
+    description: >
+        Rabbit client subscriber parameter to specify
+        an SSL connection to the RabbitMQ host.
+    type: string
+  RabbitClientPort:
+    default: 5672
+    description: Set rabbit subscriber port, change this if using SSL
+    type: number
+  MistralPassword:
+    description: The password for the Mistral service and db account, used by the Mistral services.
+    type: string
+    hidden: true
+  KeystoneRegion:
+    type: string
+    default: 'regionOne'
+    description: Keystone region for endpoint
+
+outputs:
+  role_data:
+    description: Shared role data for the Mistral services.
+    value:
+      service_name: mistral_base
+      config_settings:
+        mistral::database_connection:
+          list_join:
+            - ''
+            - - {get_param: [EndpointMap, MysqlInternal, protocol]}
+              - '://mistral:'
+              - {get_param: MistralPassword}
+              - '@'
+              - {get_param: [EndpointMap, MysqlInternal, host]}
+              - '/mistral'
+        mistral::rabbit_userid: {get_param: RabbitUserName}
+        mistral::rabbit_password: {get_param: RabbitPassword}
+        mistral::rabbit_use_ssl: {get_param: RabbitClientUseSSL}
+        mistral::rabbit_port: {get_param: RabbitClientPort}
+        mistral::debug: {get_param: Debug}
+        mistral::keystone_password: {get_param: MistralPassword}
+        mistral::keystone_tenant: 'service'
+        mistral::auth_uri: {get_param: [EndpointMap, KeystoneInternal, uri]}
+        mistral::keystone_ec2_uri: {get_param: [EndpointMap, KeystoneEC2, uri]}
+        mistral::identity_uri: {get_param: [EndpointMap, KeystoneAdmin, uri_no_suffix]}
+      service_config_settings:
+        keystone:
+          mistral::keystone::auth::tenant: 'service'
+          mistral::keystone::auth::public_url: {get_param: [EndpointMap, MistralPublic, uri]}
+          mistral::keystone::auth::internal_url: {get_param: [EndpointMap, MistralInternal, uri]}
+          mistral::keystone::auth::admin_url: {get_param: [EndpointMap, MistralAdmin, uri]}
+          mistral::keystone::auth::password: {get_param: MistralPassword}
+          mistral::keystone::auth::region: {get_param: KeystoneRegion}
+        mysql:
+          mistral::db::mysql::user: mistral
+          mistral::db::mysql::host: {get_param: [EndpointMap, MysqlInternal, host_nobrackets]}
+          mistral::db::mysql::dbname: mistral
+          mistral::db::mysql::password: {get_param: MistralPassword}
+          mistral::db::mysql::allowed_hosts:
+            - '%'
+            - "%{hiera('mysql_bind_host')}"
diff --git a/puppet/services/mistral-engine.yaml b/puppet/services/mistral-engine.yaml
new file mode 100644
index 00000000..10af670d
--- /dev/null
+++ b/puppet/services/mistral-engine.yaml
@@ -0,0 +1,38 @@
+heat_template_version: 2016-04-08
+
+description: >
+  Openstack Mistral Engine service configured with Puppet
+
+parameters:
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+  MistralBase:
+    type: ./mistral-base.yaml
+    properties:
+      ServiceNetMap: {get_param: ServiceNetMap}
+      DefaultPasswords: {get_param: DefaultPasswords}
+      EndpointMap: {get_param: EndpointMap}
+
+outputs:
+  role_data:
+    description: Role data for the Mistral Engine role.
+    value:
+      service_name: mistral_engine
+      config_settings:
+        get_attr: [MistralBase, role_data, config_settings]
+      step_config: |
+        include ::tripleo::profile::base::mistral::engine
diff --git a/puppet/services/mistral-executor.yaml b/puppet/services/mistral-executor.yaml
new file mode 100644
index 00000000..7afaf0db
--- /dev/null
+++ b/puppet/services/mistral-executor.yaml
@@ -0,0 +1,38 @@
+heat_template_version: 2016-04-08
+
+description: >
+  Openstack Mistral API service configured with Puppet
+
+parameters:
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+
+resources:
+  MistralBase:
+    type: ./mistral-base.yaml
+    properties:
+      ServiceNetMap: {get_param: ServiceNetMap}
+      DefaultPasswords: {get_param: DefaultPasswords}
+      EndpointMap: {get_param: EndpointMap}
+
+outputs:
+  role_data:
+    description: Role data for the Mistral Executor role.
+    value:
+      service_name: mistral_executor
+      config_settings:
+        get_attr: [MistralBase, role_data, config_settings]
+      step_config: |
+        include ::tripleo::profile::base::mistral::executor