19 files changed, 174 insertions, 25 deletions
diff --git a/puppet/services/README.rst b/puppet/services/README.rst
index 3accff36..856b306e 100644
--- a/puppet/services/README.rst
+++ b/puppet/services/README.rst
@@ -22,8 +22,8 @@ Config Settings
 Each service may define a config_settings output variable which returns
 Hiera settings to be configured.
 
-Steps
------
+Deployment Steps
+----------------
 
 Each service may define an output variable which returns a puppet manifest
 snippet that will run at each of the following steps. Earlier manifests
@@ -48,3 +48,29 @@ are re-asserted when applying latter ones.
    4) General OpenStack Services
 
    5) Service activation (Pacemaker)
+
+Upgrade Steps
+-------------
+
+Each service template may optionally define a `upgrade_tasks` key, which is a
+list of ansible tasks to be performed during the upgrade process.
+
+Similar to the step_config, we allow a series of steps for the per-service
+upgrade sequence, defined as ansible tasks with a tag e.g "step1" for the first
+step, "step2" for the second, etc.
+
+   Steps/tages correlate to the following:
+
+   1) Quiesce the control-plane, e.g disable LoadBalancer, stop pacemaker cluster
+
+   2) Stop all control-plane services, ready for upgrade
+
+   3) Perform a package update, (either specific packages or the whole system)
+
+   4) Start services needed for migration tasks (e.g DB)
+
+   5) Perform any migration tasks, e.g DB sync commands
+
+   6) Start control-plane services
+
+   7) Any additional online migration tasks (e.g data migrations)
diff --git a/puppet/services/aodh-api.yaml b/puppet/services/aodh-api.yaml
index 347a8c13..0cc7ad8b 100644
--- a/puppet/services/aodh-api.yaml
+++ b/puppet/services/aodh-api.yaml
@@ -55,7 +55,7 @@ outputs:
             aodh::wsgi::apache::servername:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, AodhApiNetwork]}
             aodh::api::service_name: 'httpd'
@@ -68,7 +68,7 @@ outputs:
             aodh::api::host:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, AodhApiNetwork]}
             # NOTE: bind IP is found in Heat replacing the network name with the
diff --git a/puppet/services/apache-internal-tls-certmonger.yaml b/puppet/services/apache-internal-tls-certmonger.yaml
index 87e53f13..1d76b9a3 100644
--- a/puppet/services/apache-internal-tls-certmonger.yaml
+++ b/puppet/services/apache-internal-tls-certmonger.yaml
@@ -35,8 +35,8 @@ outputs:
                 httpd-NETWORK:
                   service_certificate: '/etc/pki/tls/certs/httpd-NETWORK.crt'
                   service_key: '/etc/pki/tls/private/httpd-NETWORK.key'
-                  hostname: "%{::fqdn_NETWORK}"
-                  principal: "HTTP/%{::fqdn_NETWORK}"
+                  hostname: "%{hiera('fqdn_NETWORK')}"
+                  principal: "HTTP/%{hiera('fqdn_NETWORK')}"
               for_each:
                 NETWORK:
                   # NOTE(jaosorior) Get unique network names to create
diff --git a/puppet/services/barbican-api.yaml b/puppet/services/barbican-api.yaml
index 1a5e9134..24687d03 100644
--- a/puppet/services/barbican-api.yaml
+++ b/puppet/services/barbican-api.yaml
@@ -93,7 +93,7 @@ outputs:
             barbican::wsgi::apache::servername:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, BarbicanApiNetwork]}
             barbican::db::database_connection:
diff --git a/puppet/services/ceilometer-api.yaml b/puppet/services/ceilometer-api.yaml
index 2f34f248..2e2d3f2d 100644
--- a/puppet/services/ceilometer-api.yaml
+++ b/puppet/services/ceilometer-api.yaml
@@ -75,7 +75,7 @@ outputs:
             ceilometer::api::host:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, CeilometerApiNetwork]}
             ceilometer::wsgi::apache::bind_host: {get_param: [ServiceNetMap, CeilometerApiNetwork]}
@@ -83,7 +83,7 @@ outputs:
             ceilometer::wsgi::apache::servername:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, CeilometerApiNetwork]}
       service_config_settings:
diff --git a/puppet/services/ceph-rgw.yaml b/puppet/services/ceph-rgw.yaml
index 89c1a5ee..4b85d28f 100644
--- a/puppet/services/ceph-rgw.yaml
+++ b/puppet/services/ceph-rgw.yaml
@@ -68,6 +68,7 @@ outputs:
           ceph::rgw::keystone::auth::public_url: {get_param: [EndpointMap, CephRgwPublic, uri]}
           ceph::rgw::keystone::auth::internal_url: {get_param: [EndpointMap, CephRgwInternal, uri]}
           ceph::rgw::keystone::auth::admin_url: {get_param: [EndpointMap, CephRgwAdmin, uri]}
+          ceph::rgw::keystone::auth::user: 'swift'
           ceph::rgw::keystone::auth::password: {get_param: SwiftPassword}
           ceph::rgw::keystone::auth::region: {get_param: KeystoneRegion}
           ceph::rgw::keystone::auth::tenant: 'service'
diff --git a/puppet/services/cinder-api.yaml b/puppet/services/cinder-api.yaml
index 8d57418e..6cb2b194 100644
--- a/puppet/services/cinder-api.yaml
+++ b/puppet/services/cinder-api.yaml
@@ -101,7 +101,7 @@ outputs:
             cinder::api::bind_host:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, CinderApiNetwork]}
             cinder::wsgi::apache::ssl: {get_param: EnableInternalTLS}
@@ -115,7 +115,7 @@ outputs:
             cinder::wsgi::apache::servername:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, CinderApiNetwork]}
           -
diff --git a/puppet/services/database/mysql.yaml b/puppet/services/database/mysql.yaml
index bed8f7d3..abe752e2 100644
--- a/puppet/services/database/mysql.yaml
+++ b/puppet/services/database/mysql.yaml
@@ -87,8 +87,16 @@ outputs:
             tripleo::profile::base::database::mysql::bind_address:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, MysqlNetwork]}
       step_config: |
         include ::tripleo::profile::base::database::mysql
+      upgrade_tasks:
+        - name: Stop service
+          tags: step2
+          service: name=mariadb state=stopped
+        - name: Start service
+          tags: step4
+          service: name=mariadb state=started
+
diff --git a/puppet/services/gnocchi-api.yaml b/puppet/services/gnocchi-api.yaml
index d5f8e62d..b3d39e0f 100644
--- a/puppet/services/gnocchi-api.yaml
+++ b/puppet/services/gnocchi-api.yaml
@@ -91,7 +91,7 @@ outputs:
             gnocchi::wsgi::apache::servername:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, GnocchiApiNetwork]}
             tripleo::profile::base::gnocchi::api::gnocchi_backend: {get_param: GnocchiBackend}
@@ -105,7 +105,7 @@ outputs:
             gnocchi::api::host:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, GnocchiApiNetwork]}
 
diff --git a/puppet/services/haproxy.yaml b/puppet/services/haproxy.yaml
index c8edade5..675a79ec 100644
--- a/puppet/services/haproxy.yaml
+++ b/puppet/services/haproxy.yaml
@@ -77,3 +77,10 @@ outputs:
                 - get_attr: [HAProxyInternalTLS, role_data, certificates_specs]
       step_config: |
         include ::tripleo::profile::base::haproxy
+      upgrade_tasks:
+        - name: Stop haproxy service
+          tags: step1
+          service: name=haproxy state=stopped
+        - name: Start haproxy service
+          tags: step4 # Needed at step 4 for mysql
+          service: name=haproxy state=started
diff --git a/puppet/services/keepalived.yaml b/puppet/services/keepalived.yaml
index 6f2c44ec..b4f1a100 100644
--- a/puppet/services/keepalived.yaml
+++ b/puppet/services/keepalived.yaml
@@ -64,3 +64,11 @@ outputs:
           - tripleo::keepalived::public_virtual_interface: {get_param: PublicVirtualInterface}
       step_config: |
         include ::tripleo::profile::base::keepalived
+      upgrade_tasks:
+        - name: Stop keepalived service
+          tags: step1
+          service: name=keepalived state=stopped
+        - name: Start keepalived service
+          tags: step4 # Needed at step 4 for mysql
+          service: name=keepalived state=started
+
diff --git a/puppet/services/keystone.yaml b/puppet/services/keystone.yaml
index c2a282d4..e48d7037 100644
--- a/puppet/services/keystone.yaml
+++ b/puppet/services/keystone.yaml
@@ -195,13 +195,13 @@ outputs:
             keystone::wsgi::apache::servername:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, KeystonePublicApiNetwork]}
             keystone::wsgi::apache::servername_admin:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, KeystoneAdminApiNetwork]}
             keystone::wsgi::apache::workers: {get_param: KeystoneWorkers}
@@ -219,13 +219,13 @@ outputs:
             keystone::admin_bind_host:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, KeystoneAdminApiNetwork]}
             keystone::public_bind_host:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, KeystonePublicApiNetwork]}
             # NOTE: bind IP is found in Heat replacing the network name with the
@@ -248,3 +248,14 @@ outputs:
           keystone::db::mysql::allowed_hosts:
             - '%'
             - "%{hiera('mysql_bind_host')}"
+      # Ansible tasks to handle upgrade
+      upgrade_tasks:
+        - name: Stop keystone service (running under httpd)
+          tags: step2
+          service: name=httpd state=stopped
+        - name: Sync keystone DB
+          tags: step5
+          command: keystone-manage db_sync
+        - name: Start keystone service (running under httpd)
+          tags: step6
+          service: name=httpd state=started
diff --git a/puppet/services/nova-api.yaml b/puppet/services/nova-api.yaml
index c4d5c6bb..50e4c996 100644
--- a/puppet/services/nova-api.yaml
+++ b/puppet/services/nova-api.yaml
@@ -108,7 +108,7 @@ outputs:
           nova::api::api_bind_address:
             str_replace:
               template:
-                '%{::fqdn_$NETWORK}'
+                "%{hiera('fqdn_$NETWORK')}"
               params:
                 $NETWORK: {get_param: [ServiceNetMap, NovaApiNetwork]}
           nova::api::service_name: 'httpd'
@@ -122,7 +122,7 @@ outputs:
           nova::wsgi::apache::servername:
             str_replace:
               template:
-                '%{::fqdn_$NETWORK}'
+                "%{hiera('fqdn_$NETWORK')}"
               params:
                 $NETWORK: {get_param: [ServiceNetMap, NovaApiNetwork]}
           nova::api::neutron_metadata_proxy_shared_secret: {get_param: NeutronMetadataProxySharedSecret}
diff --git a/puppet/services/pacemaker/database/mysql.yaml b/puppet/services/pacemaker/database/mysql.yaml
index ea3d8abd..f2905903 100644
--- a/puppet/services/pacemaker/database/mysql.yaml
+++ b/puppet/services/pacemaker/database/mysql.yaml
@@ -40,7 +40,7 @@ outputs:
           - tripleo::profile::pacemaker::database::mysql::bind_address:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, MysqlNetwork]}
             # NOTE: bind IP is found in Heat replacing the network name with the
diff --git a/puppet/services/panko-api.yaml b/puppet/services/panko-api.yaml
index 700edc7f..a8bd5e8a 100644
--- a/puppet/services/panko-api.yaml
+++ b/puppet/services/panko-api.yaml
@@ -55,7 +55,7 @@ outputs:
             panko::wsgi::apache::servername:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, PankoApiNetwork]}
             panko::api::service_name: 'httpd'
@@ -68,7 +68,7 @@ outputs:
             panko::api::host:
               str_replace:
                 template:
-                  '%{::fqdn_$NETWORK}'
+                  "%{hiera('fqdn_$NETWORK')}"
                 params:
                   $NETWORK: {get_param: [ServiceNetMap, PankoApiNetwork]}
             # NOTE: bind IP is found in Heat replacing the network name with the
diff --git a/puppet/services/rabbitmq.yaml b/puppet/services/rabbitmq.yaml
index 44a09a42..190487e0 100644
--- a/puppet/services/rabbitmq.yaml
+++ b/puppet/services/rabbitmq.yaml
@@ -73,7 +73,13 @@ outputs:
         rabbitmq::repos_ensure: false
         rabbitmq::tcp_keepalive: true
         rabbitmq_environment:
-          RABBITMQ_NODENAME: "rabbit@%{::hostname}"
+          RABBITMQ_NODENAME:
+            str_replace:
+              template:
+                "rabbit@%{hiera('fqdn_$NETWORK')}"
+              params:
+                $NETWORK: {get_param: [ServiceNetMap, RabbitmqNetwork]}
+          RABBITMQ_USE_LONGNAME: true
           RABBITMQ_SERVER_ERL_ARGS: '"+K true +P 1048576 -kernel inet_default_connect_options [{nodelay,true},{raw,6,18,<<5000:64/native>>}] -kernel inet_default_listen_options [{raw,6,18,<<5000:64/native>>}]"'
         rabbitmq_kernel_variables:
           inet_dist_listen_min: '25672'
@@ -97,6 +103,13 @@ outputs:
         # internal_api_subnet - > IP/CIDR
         rabbitmq::node_ip_address: {get_param: [ServiceNetMap, RabbitmqNetwork]}
         rabbitmq::nr_ha_queues: {get_param: RabbitHAQueues}
-
       step_config: |
         include ::tripleo::profile::base::rabbitmq
+      upgrade_tasks:
+        - name: Stop rabbitmq service
+          tags: step2
+          service: name=rabbitmq-server state=stopped
+        - name: Start rabbitmq service
+          tags: step6
+          service: name=rabbitmq-server state=started
+
diff --git a/puppet/services/services.yaml b/puppet/services/services.yaml
index ffe2d2d4..13df5bbe 100644
--- a/puppet/services/services.yaml
+++ b/puppet/services/services.yaml
@@ -108,3 +108,8 @@ outputs:
           expression: $.data.role_data.where($ != null).select($.get('service_config_settings')).where($ != null).reduce($1.mergeWith($2), {})
           data: {role_data: {get_attr: [ServiceChain, role_data]}}
       step_config: {list_join: ["\n", {get_attr: [ServiceChain, role_data, step_config]}]}
+      upgrade_tasks:
+        yaql:
+          # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
+          expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct()
+          data: {get_attr: [ServiceChain, role_data]}
diff --git a/puppet/services/tripleo-packages.yaml b/puppet/services/tripleo-packages.yaml
index 124f5fe8..69912fa5 100644
--- a/puppet/services/tripleo-packages.yaml
+++ b/puppet/services/tripleo-packages.yaml
@@ -32,3 +32,7 @@ outputs:
         tripleo::packages::enable_install: {get_param: EnablePackageInstall}
       step_config: |
         include ::tripleo::packages
+      upgrade_tasks:
+        - name: Update all packages
+          tags: step3
+          yum: name=* state=latest
diff --git a/puppet/services/zaqar.yaml b/puppet/services/zaqar.yaml
new file mode 100644
index 00000000..77240c3c
--- /dev/null
+++ b/puppet/services/zaqar.yaml
@@ -0,0 +1,66 @@
+heat_template_version: 2016-04-08
+
+description: >
+  Openstack Zaqar service. Shared for all Heat services.
+
+parameters:
+  ServiceNetMap:
+    default: {}
+    description: Mapping of service_name -> network name. Typically set
+                 via parameter_defaults in the resource registry.  This
+                 mapping overrides those in ServiceNetMapDefaults.
+    type: json
+  EndpointMap:
+    default: {}
+    description: Mapping of service endpoint -> protocol. Typically set
+                 via parameter_defaults in the resource registry.
+    type: json
+  DefaultPasswords:
+    default: {}
+    type: json
+  Debug:
+    default: ''
+    description: Set to True to enable debugging on all services.
+    type: string
+  ZaqarPassword:
+    description: The password for Zaqar
+    type: string
+    hidden: true
+  KeystoneRegion:
+    type: string
+    default: 'regionOne'
+    description: Keystone region for endpoint
+
+
+outputs:
+  role_data:
+    description: Shared role data for the Heat services.
+    value:
+      service_name: zaqar
+      config_settings:
+        zaqar::keystone::authtoken::password: {get_param: ZaqarPassword}
+        zaqar::keystone::authtoken::project_name: 'service'
+        zaqar::keystone::authtoken::auth_url: {get_param: [EndpointMap, KeystoneAdmin, uri_no_suffix]}
+        zaqar::keystone::authtoken::auth_uri: {get_param: [EndpointMap, KeystoneInternal, uri]}
+        zaqar::debug: {get_param: Debug}
+        zaqar::transport::websocket::bind: {get_param: [EndpointMap, ZaqarInternal, host]}
+        zaqar::transport::wsgi::bind: {get_param: [ServiceNetMap, ZaqarApiNetwork]}
+        zaqar::message_pipeline: 'zaqar.notification.notifier'
+        zaqar::unreliable: true
+      service_config_settings:
+        keystone:
+          zaqar::keystone::auth::password: {get_param: ZaqarPassword}
+          zaqar::keystone::auth::public_url: {get_param: [EndpointMap, ZaqarPublic, uri]}
+          zaqar::keystone::auth::admin_url: {get_param: [EndpointMap, ZaqarAdmin, uri]}
+          zaqar::keystone::auth::internal_url: {get_param: [EndpointMap, ZaqarInternal, uri]}
+          zaqar::keystone::auth::region: {get_param: KeystoneRegion}
+          zaqar::keystone::auth::tenant: 'service'
+          zaqar::keystone::auth_websocket::password: {get_param: ZaqarPassword}
+          zaqar::keystone::auth_websocket::public_url: {get_param: [EndpointMap, ZaqarWebSocketPublic, uri]}
+          zaqar::keystone::auth_websocket::admin_url: {get_param: [EndpointMap, ZaqarWebSocketAdmin, uri]}
+          zaqar::keystone::auth_websocket::internal_url: {get_param: [EndpointMap, ZaqarWebSocketInternal, uri]}
+          zaqar::keystone::auth_websocket::region: {get_param: KeystoneRegion}
+          zaqar::keystone::auth_websocket::tenant: 'service'
+
+      step_config: |
+        include ::tripleo::profile::base::zaqar