Enable composable upgrades for docker service templates

This aligns the docker based services with the new composable upgrades
architecture we landed for ocata, and does a first-pass adding upgrade_tasks
for the services (these may change, atm we only disable the service on
the host).

To run the upgrade workflow you basically do two steps:

openstack overcloud deploy --templates \
  -e environments/major-upgrade-composable-steps-docker.yaml

This will run the ansible upgrade steps we define via upgrade_tasks
then run the normal docker PostDeploySteps to bring up the containers.

For the puppet workflow there's then an operator driven step where
compute nodes (and potentially storage nodes) are upgrades in batches
and finally you do:

openstack overcloud deploy --templates \
  -e environments/major-upgrade-converge-docker.yaml

In the puppet case this re-applies puppet to unpin the nova RPC API
so I guess it'll restart the nova containers this affects but otherwise
will be a no-op (we also disable the ansible steps at this point.

Depends-On: I9057d47eea15c8ba92ca34717b6b5965d4425ab1
Change-Id: Ia50169819cb959025866348b11337728f8ed5c9e

25 files changed, 121 insertions, 2 deletions

diff --git a/docker/services/database/mongodb.yaml b/docker/services/database/mongodb.yaml
index 3c4146fd..e83f4f19 100644
--- a/docker/services/database/mongodb.yaml
+++ b/docker/services/database/mongodb.yaml
@@ -50,7 +50,6 @@ outputs:
           - "\n"
           - - "['Mongodb_database', 'Mongodb_user', 'Mongodb_replset'].each |String $val| { noop_resource($val) }"
             - {get_attr: [MongodbPuppetBase, role_data, step_config]}
-      upgrade_tasks: {get_attr: [MongodbPuppetBase, role_data, upgrade_tasks]}
       # BEGIN DOCKER SETTINGS #
       docker_image: &mongodb_image
         list_join:
@@ -100,3 +99,7 @@ outputs:
           volumes:
           - "mongodb:/var/lib/mongodb/"
           - "logs:/var/log/kolla:ro"
+      upgrade_tasks:
+        - name: Stop and disable mongodb service
+          tags: step2
+          service: name=mongod state=stopped enabled=no
diff --git a/docker/services/database/mysql.yaml b/docker/services/database/mysql.yaml
index fd92e99c..c34ebe93 100644
--- a/docker/services/database/mysql.yaml
+++ b/docker/services/database/mysql.yaml
@@ -59,7 +59,6 @@ outputs:
           - "\n"
           - - "['Mysql_datadir', 'Mysql_user', 'Mysql_database', 'Mysql_grant', 'Mysql_plugin'].each |String $val| { noop_resource($val) }"
             - {get_attr: [MysqlPuppetBase, role_data, step_config]}
-      upgrade_tasks: {get_attr: [MysqlPuppetBase, role_data, upgrade_tasks]}
       # BEGIN DOCKER SETTINGS #
       docker_image: &mysql_image
         list_join:
@@ -132,3 +131,7 @@ outputs:
           volumes:
             - "mariadb:/var/lib/mysql/:ro"
             - "/var/lib/config-data/mysql/root:/root:ro" #provides .my.cnf
+      upgrade_tasks:
+        - name: Stop and disable mysql service
+          tags: step2
+          service: name=mariadb state=stopped enabled=no
diff --git a/docker/services/glance-api.yaml b/docker/services/glance-api.yaml
index ab62f7e3..73d76ad5 100644
--- a/docker/services/glance-api.yaml
+++ b/docker/services/glance-api.yaml
@@ -97,3 +97,7 @@ outputs:
             volumes: *glance_volumes
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable glance_api service
+          tags: step2
+          service: name=openstack-glance-api state=stopped enabled=no
diff --git a/docker/services/heat-api-cfn.yaml b/docker/services/heat-api-cfn.yaml
index fbaacbe6..2f54c0f1 100644
--- a/docker/services/heat-api-cfn.yaml
+++ b/docker/services/heat-api-cfn.yaml
@@ -91,3 +91,7 @@ outputs:
               - /dev:/dev
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable heat_api_cfn service
+          tags: step2
+          service: name=openstack-heat-api-cfn state=stopped enabled=no
diff --git a/docker/services/heat-api.yaml b/docker/services/heat-api.yaml
index df9160f4..a212d254 100644
--- a/docker/services/heat-api.yaml
+++ b/docker/services/heat-api.yaml
@@ -91,3 +91,7 @@ outputs:
               - /dev:/dev
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable heat_api service
+          tags: step2
+          service: name=openstack-heat-api state=stopped enabled=no
diff --git a/docker/services/heat-engine.yaml b/docker/services/heat-engine.yaml
index c8259f93..c60a3840 100644
--- a/docker/services/heat-engine.yaml
+++ b/docker/services/heat-engine.yaml
@@ -93,3 +93,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable heat_engine service
+          tags: step2
+          service: name=openstack-heat-engine state=stopped enabled=no
diff --git a/docker/services/ironic-api.yaml b/docker/services/ironic-api.yaml
index 97ba9706..ca42c9ec 100644
--- a/docker/services/ironic-api.yaml
+++ b/docker/services/ironic-api.yaml
@@ -100,3 +100,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable ironic_api service
+          tags: step2
+          service: name=openstack-ironic-api state=stopped enabled=no
diff --git a/docker/services/ironic-conductor.yaml b/docker/services/ironic-conductor.yaml
index 2cc29054..ff470008 100644
--- a/docker/services/ironic-conductor.yaml
+++ b/docker/services/ironic-conductor.yaml
@@ -112,3 +112,7 @@ outputs:
               - ironic:/var/lib/ironic
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable ironic_conductor service
+          tags: step2
+          service: name=openstack-ironic-conductor state=stopped enabled=no
diff --git a/docker/services/keystone.yaml b/docker/services/keystone.yaml
index f9b94a74..358277a5 100644
--- a/docker/services/keystone.yaml
+++ b/docker/services/keystone.yaml
@@ -154,3 +154,7 @@ outputs:
             list_join:
             - '/'
             - [ {get_param: DockerNamespace}, {get_param: DockerKeystoneImage} ]
+      upgrade_tasks:
+        - name: Stop and disable keystone service (running under httpd)
+          tags: step2
+          service: name=httpd state=stopped enabled=no
diff --git a/docker/services/memcached.yaml b/docker/services/memcached.yaml
index 3e51f5b4..9467567f 100644
--- a/docker/services/memcached.yaml
+++ b/docker/services/memcached.yaml
@@ -70,3 +70,7 @@ outputs:
             command: ['/bin/bash', '-c', 'source /etc/sysconfig/memcached; /usr/bin/memcached -p ${PORT} -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS']
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable memcached service
+          tags: step2
+          service: name=memcached state=stopped enabled=no
diff --git a/docker/services/mistral-api.yaml b/docker/services/mistral-api.yaml
index 843d5b23..7680bc62 100644
--- a/docker/services/mistral-api.yaml
+++ b/docker/services/mistral-api.yaml
@@ -116,3 +116,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable mistral_api service
+          tags: step2
+          service: name=openstack-mistral-api state=stopped enabled=no
diff --git a/docker/services/mistral-engine.yaml b/docker/services/mistral-engine.yaml
index e50fc0a6..d61ab1c2 100644
--- a/docker/services/mistral-engine.yaml
+++ b/docker/services/mistral-engine.yaml
@@ -88,3 +88,8 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable mistral_engine service
+          tags: step2
+          service: name=openstack-mistral-engine state=stopped enabled=no
+
diff --git a/docker/services/mistral-executor.yaml b/docker/services/mistral-executor.yaml
index 8833154e..42286426 100644
--- a/docker/services/mistral-executor.yaml
+++ b/docker/services/mistral-executor.yaml
@@ -92,3 +92,7 @@ outputs:
               - /var/lib/config-data/nova/etc/nova:/etc/nova:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable mistral_executor service
+          tags: step2
+          service: name=openstack-mistral-executor state=stopped enabled=no
diff --git a/docker/services/neutron-api.yaml b/docker/services/neutron-api.yaml
index bf19586c..71389046 100644
--- a/docker/services/neutron-api.yaml
+++ b/docker/services/neutron-api.yaml
@@ -106,3 +106,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable neutron_api service
+          tags: step2
+          service: name=neutron-server state=stopped enabled=no
diff --git a/docker/services/neutron-dhcp.yaml b/docker/services/neutron-dhcp.yaml
index 15f3055e..ccde63f2 100644
--- a/docker/services/neutron-dhcp.yaml
+++ b/docker/services/neutron-dhcp.yaml
@@ -94,3 +94,7 @@ outputs:
               - /run/:/run
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable neutron_dhcp service
+          tags: step2
+          service: name=neutron-dhcp-agent state=stopped enabled=no
diff --git a/docker/services/neutron-ovs-agent.yaml b/docker/services/neutron-ovs-agent.yaml
index f6b29292..6dcf91d9 100644
--- a/docker/services/neutron-ovs-agent.yaml
+++ b/docker/services/neutron-ovs-agent.yaml
@@ -85,3 +85,7 @@ outputs:
               - /run:/run
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable neutron_ovs_agent service
+          tags: step2
+          service: name=neutron-openvswitch-agent state=stopped enabled=no
diff --git a/docker/services/nova-api.yaml b/docker/services/nova-api.yaml
index 28b3a9d0..8a892325 100644
--- a/docker/services/nova-api.yaml
+++ b/docker/services/nova-api.yaml
@@ -145,3 +145,7 @@ outputs:
               - '/usr/bin/nova-manage'
               - 'cell_v2'
               - 'discover_hosts'
+      upgrade_tasks:
+        - name: Stop and disable nova_api service
+          tags: step2
+          service: name=openstack-nova-api state=stopped enabled=no
diff --git a/docker/services/nova-conductor.yaml b/docker/services/nova-conductor.yaml
index 73acd0ac..8bc81e32 100644
--- a/docker/services/nova-conductor.yaml
+++ b/docker/services/nova-conductor.yaml
@@ -86,3 +86,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable nova_conductor service
+          tags: step2
+          service: name=openstack-nova-conductor state=stopped enabled=no
diff --git a/docker/services/nova-placement.yaml b/docker/services/nova-placement.yaml
index 54469795..1df209bd 100644
--- a/docker/services/nova-placement.yaml
+++ b/docker/services/nova-placement.yaml
@@ -102,3 +102,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable nova_placement service (running under httpd)
+          tags: step2
+          service: name=httpd state=stopped enabled=no
diff --git a/docker/services/nova-scheduler.yaml b/docker/services/nova-scheduler.yaml
index e3955a2b..c24d5b26 100644
--- a/docker/services/nova-scheduler.yaml
+++ b/docker/services/nova-scheduler.yaml
@@ -85,3 +85,7 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable nova_scheduler service
+          tags: step2
+          service: name=openstack-nova-scheduler state=stopped enabled=no
diff --git a/docker/services/rabbitmq.yaml b/docker/services/rabbitmq.yaml
index aa604439..ed440718 100644
--- a/docker/services/rabbitmq.yaml
+++ b/docker/services/rabbitmq.yaml
@@ -120,3 +120,7 @@ outputs:
               - rabbitmq:/var/lib/rabbitmq/
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable rabbitmq service
+          tags: step2
+          service: name=rabbitmq-server state=stopped enabled=no
diff --git a/docker/services/services.yaml b/docker/services/services.yaml
index 125c4468..3f094ff8 100644
--- a/docker/services/services.yaml
+++ b/docker/services/services.yaml
@@ -75,3 +75,13 @@ outputs:
         {get_attr: [ServiceChain, role_data, docker_config]}
       docker_puppet_tasks:
         {get_attr: [ServiceChain, role_data, docker_puppet_tasks]}
+      upgrade_tasks:
+        yaql:
+          # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
+          expression: $.data.where($ != null).select($.get('upgrade_tasks')).where($ != null).flatten().distinct()
+          data: {get_attr: [ServiceChain, role_data]}
+      upgrade_batch_tasks:
+        yaql:
+          # Note we use distinct() here to filter any identical tasks, e.g yum update for all services
+          expression: $.data.where($ != null).select($.get('upgrade_batch_tasks')).where($ != null).flatten().distinct()
+          data: {get_attr: [ServiceChain, role_data]}
diff --git a/docker/services/swift-proxy.yaml b/docker/services/swift-proxy.yaml
index cdfefe2f..66118412 100644
--- a/docker/services/swift-proxy.yaml
+++ b/docker/services/swift-proxy.yaml
@@ -77,3 +77,7 @@ outputs:
               - /dev:/dev
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable swift_proxy service
+          tags: step2
+          service: name=openstack-swift-proxy state=stopped enabled=no
diff --git a/docker/services/swift-storage.yaml b/docker/services/swift-storage.yaml
index ab9946f3..2eb55632 100644
--- a/docker/services/swift-storage.yaml
+++ b/docker/services/swift-storage.yaml
@@ -344,3 +344,20 @@ outputs:
               - swift-srv:/srv
               - /dev:/dev
             environment: *kolla_env
+      upgrade_tasks:
+        - name: Stop and disable swift storage services
+          tags: step2
+          service: name={{ item }} state=stopped enabled=no
+          with_items:
+            - openstack-swift-account-auditor
+            - openstack-swift-account-reaper
+            - openstack-swift-account-replicator
+            - openstack-swift-account
+            - openstack-swift-container-auditor
+            - openstack-swift-container-replicator
+            - openstack-swift-container-updater
+            - openstack-swift-container
+            - openstack-swift-object-auditor
+            - openstack-swift-object-replicator
+            - openstack-swift-object-updater
+            - openstack-swift-object
diff --git a/docker/services/zaqar.yaml b/docker/services/zaqar.yaml
index 1704278e..30905ffe 100644
--- a/docker/services/zaqar.yaml
+++ b/docker/services/zaqar.yaml
@@ -100,3 +100,8 @@ outputs:
               - /etc/localtime:/etc/localtime:ro
             environment:
               - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS
+      upgrade_tasks:
+        - name: Stop and disable zaqar service
+          tags: step2
+          service: name=openstack-zaqar.service state=stopped enabled=no
+