diff options
| -rw-r--r-- | docker/services/README.rst | 21 | ||||
| -rw-r--r-- | docker/services/pacemaker/cinder-backup.yaml | 30 | ||||
| -rw-r--r-- | docker/services/pacemaker/cinder-volume.yaml | 33 | ||||
| -rw-r--r-- | docker/services/pacemaker/clustercheck.yaml | 2 | ||||
| -rw-r--r-- | docker/services/pacemaker/database/mysql.yaml | 30 | ||||
| -rw-r--r-- | docker/services/pacemaker/database/redis.yaml | 30 | ||||
| -rw-r--r-- | docker/services/pacemaker/haproxy.yaml | 30 | ||||
| -rw-r--r-- | docker/services/pacemaker/manila-share.yaml | 30 | ||||
| -rw-r--r-- | docker/services/pacemaker/rabbitmq.yaml | 30 | ||||
| -rw-r--r-- | docker/services/rabbitmq.yaml | 6 | ||||
| -rw-r--r-- | puppet/services/pacemaker.yaml | 12 | ||||
| -rw-r--r-- | puppet/services/tripleo-packages.yaml | 9 | ||||
| -rwxr-xr-x | tools/yaml-validate.py | 3 |
13 files changed, 261 insertions, 5 deletions
diff --git a/docker/services/README.rst b/docker/services/README.rst index ce255ba8..a843efc3 100644 --- a/docker/services/README.rst +++ b/docker/services/README.rst @@ -124,3 +124,24 @@ Steps correlate to the following: 5) Service activation (Pacemaker) a) step 5 baremetal b) step 5 containers + +Update steps: +------------- + +All services have an associated update_tasks output that is an ansible +snippet that will be run during update in an rolling update that is +expected to run in a rolling update fashion (one node at a time) + +For Controller (where pacemaker is running) we have the following states: + 1. Step=1: stop the cluster on the updated node; + 2. Step=2: Pull the latest image and retag the it pcmklatest + 3. Step=3: yum upgrade happens on the host. + 4. Step=4: Restart the cluster on the node + 5. Step=5: Verification: + Currently we test that the pacemaker services are running. + +Then the usual deploy steps are run which pull in the latest image for +all containerized services and the updated configuration if any. + +Note: as pacemaker is not containerized, the points 1 and 4 happen in +puppet/services/pacemaker.yaml. diff --git a/docker/services/pacemaker/cinder-backup.yaml b/docker/services/pacemaker/cinder-backup.yaml index 46b9932a..4a99184f 100644 --- a/docker/services/pacemaker/cinder-backup.yaml +++ b/docker/services/pacemaker/cinder-backup.yaml @@ -207,3 +207,33 @@ outputs: - name: Disable cinder_backup service tags: step2 service: name=openstack-cinder-backup enabled=no + update_tasks: + - name: Get docker Cinder-Backup image + set_fact: + docker_image: {get_param: DockerCinderBackupImage} + docker_image_latest: *cinder_backup_image_pcmklatest + when: step == '2' + - name: Pull latest Cinder-Backup images + command: "docker pull {{docker_image}}" + when: step == "2" + - name: Get previous Cinder-Backup image id + shell: "docker images | awk '/cinder-backup.* pcmklatest/{print $3}'" + register: cinder_backup_image_id + - block: + - name: Get a list of container using Cinder-Backup image + shell: "docker ps -q -f 'ancestor={{cinder_backup_image_id.stdout}}'" + register: cinder_backup_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same Cinder-Backup image + shell: "docker rm -fv {{item}}" + with_items: "{{ cinder_backup_containers_to_destroy.stdout_lines }}" + - name: Remove previous Cinder-Backup images + shell: "docker rmi -f {{cinder_backup_image_id.stdout}}" + when: + - step == '2' + - cinder_backup_image_id.stdout != '' + - name: Retag pcmklatest to latest Cinder-Backup image + shell: "docker tag {{docker_image}} {{docker_image_latest}}" + when: step == "2" + # Got to check that pacemaker_is_active is working fine with bundle. + # TODO: pacemaker_is_active resource doesn't support bundle. diff --git a/docker/services/pacemaker/cinder-volume.yaml b/docker/services/pacemaker/cinder-volume.yaml index 2f68b126..ce93e5f1 100644 --- a/docker/services/pacemaker/cinder-volume.yaml +++ b/docker/services/pacemaker/cinder-volume.yaml @@ -225,6 +225,33 @@ outputs: - name: Disable cinder_volume service from boot tags: step2 service: name=openstack-cinder-volume enabled=no - - - + update_tasks: + - name: Get docker Cinder-Volume image + set_fact: + docker_image: {get_param: DockerCinderVolumeImage} + docker_image_latest: *cinder_volume_image_pcmklatest + when: step == '2' + - name: Get previous Cinder-Volume image id + shell: "docker images | awk '/cinder-volume.* pcmklatest/{print $3}'" + register: cinder_volume_image_id + - block: + - name: Get a list of container using Cinder-Volume image + shell: "docker ps -q -f 'ancestor={{cinder_volume_image_id.stdout}}'" + register: cinder_volume_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same Cinder-Volume image + shell: "docker rm -fv {{item}}" + with_items: "{{ cinder_volume_containers_to_destroy.stdout_lines }}" + - name: Remove previous Cinder-Volume images + shell: "docker rmi -f {{cinder_volume_image_id.stdout}}" + when: + - step == '2' + - cinder_volume_image_id.stdout != '' + - name: Pull latest Cinder-Volume images + command: "docker pull {{docker_image}}" + when: step == "2" + - name: Retag pcmklatest to latest Cinder-Volume image + shell: "docker tag {{docker_image}} {{docker_image_latest}}" + when: step == "2" + # Got to check that pacemaker_is_active is working fine with bundle. + # TODO: pacemaker_is_active resource doesn't support bundle. diff --git a/docker/services/pacemaker/clustercheck.yaml b/docker/services/pacemaker/clustercheck.yaml index 69f306b1..b5d128d4 100644 --- a/docker/services/pacemaker/clustercheck.yaml +++ b/docker/services/pacemaker/clustercheck.yaml @@ -95,3 +95,5 @@ outputs: - KOLLA_CONFIG_STRATEGY=COPY_ALWAYS host_prep_tasks: upgrade_tasks: + update_tasks: + # Nothing: It's not managed by pacemaker, so let paunch do it. diff --git a/docker/services/pacemaker/database/mysql.yaml b/docker/services/pacemaker/database/mysql.yaml index 80c1985a..a65117fc 100644 --- a/docker/services/pacemaker/database/mysql.yaml +++ b/docker/services/pacemaker/database/mysql.yaml @@ -313,3 +313,33 @@ outputs: - name: Restart xinetd service after clustercheck removal tags: step2 service: name=xinetd state=restarted + update_tasks: + - name: Get docker Mariadb image + set_fact: + docker_image: {get_param: DockerMysqlImage} + docker_image_latest: *mysql_image_pcmklatest + when: step == '2' + - name: Get previous Mariadb image id + shell: "docker images | awk '/mariadb.* pcmklatest/{print $3}'" + register: mariadb_image_id + - block: + - name: Get a list of container using Mariadb image + shell: "docker ps -q -f 'ancestor={{mariadb_image_id.stdout}}'" + register: mariadb_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same Mariadb image + shell: "docker rm -fv {{item}}" + with_items: "{{ mariadb_containers_to_destroy.stdout_lines }}" + - name: Remove previous Mariadb images + shell: "docker rmi -f {{mariadb_image_id.stdout}}" + when: + - step == '2' + - mariadb_image_id.stdout != '' + - name: Pull latest Mariadb images + command: "docker pull {{docker_image}}" + when: step == "2" + - name: Retag pcmklatest to latest Mariadb image + shell: "docker tag {{docker_image}} {{docker_image_latest}}" + when: step == "2" + # Got to check that pacemaker_is_active is working fine with bundle. + # TODO: pacemaker_is_active resource doesn't support bundle. diff --git a/docker/services/pacemaker/database/redis.yaml b/docker/services/pacemaker/database/redis.yaml index bcda4efc..fe69bb00 100644 --- a/docker/services/pacemaker/database/redis.yaml +++ b/docker/services/pacemaker/database/redis.yaml @@ -255,3 +255,33 @@ outputs: - name: Disable redis service tags: step2 service: name=redis enabled=no + update_tasks: + - name: Get docker Redis image + set_fact: + docker_image: {get_param: DockerRedisImage} + docker_image_latest: *redis_image_pcmklatest + when: step == '2' + - name: Get previous Redis image id + shell: "docker images | awk '/redis.* pcmklatest/{print $3}'" + register: redis_image_id + - block: + - name: Get a list of container using Redis image + shell: "docker ps -q -f 'ancestor={{redis_image_id.stdout}}'" + register: redis_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same Redis image + shell: "docker rm -fv {{item}}" + with_items: "{{ redis_containers_to_destroy.stdout_lines }}" + - name: Remove previous Redis images + shell: "docker rmi -f {{redis_image_id.stdout}}" + when: + - step == '2' + - redis_image_id.stdout != '' + - name: Pull latest Redis images + command: "docker pull {{docker_image}}" + when: step == "2" + - name: Retag pcmklatest to latest Redis image + shell: "docker tag {{docker_image}} {{docker_image_latest}}" + when: step == "2" + # Got to check that pacemaker_is_active is working fine with bundle. + # TODO: pacemaker_is_active resource doesn't support bundle. diff --git a/docker/services/pacemaker/haproxy.yaml b/docker/services/pacemaker/haproxy.yaml index ba0f0efc..f1c8f905 100644 --- a/docker/services/pacemaker/haproxy.yaml +++ b/docker/services/pacemaker/haproxy.yaml @@ -253,3 +253,33 @@ outputs: retries: 5 until: output.rc == 0 when: is_bootstrap_node and haproxy_res|succeeded + update_tasks: + - name: Get docker Haproxy image + set_fact: + docker_image: {get_param: DockerHAProxyImage} + docker_image_latest: *haproxy_image_pcmklatest + when: step == '2' + - name: Get previous Haproxy image id + shell: "docker images | awk '/haproxy.* pcmklatest/{print $3}'" + register: haproxy_image_id + - block: + - name: Get a list of container using Haproxy image + shell: "docker ps -q -f 'ancestor={{haproxy_image_id.stdout}}'" + register: haproxy_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same Haproxy image + shell: "docker rm -fv {{item}}" + with_items: "{{ haproxy_containers_to_destroy.stdout_lines }}" + - name: Remove previous Haproxy images + shell: "docker rmi -f {{haproxy_image_id.stdout}}" + when: + - step == '2' + - haproxy_image_id.stdout != '' + - name: Pull latest Haproxy images + command: "docker pull {{docker_image}}" + when: step == "2" + - name: Retag pcmklatest to latest Haproxy image + shell: "docker tag {{docker_image}} {{docker_image_latest}}" + when: step == "2" + # Got to check that pacemaker_is_active is working fine with bundle. + # TODO: pacemaker_is_active resource doesn't support bundle. diff --git a/docker/services/pacemaker/manila-share.yaml b/docker/services/pacemaker/manila-share.yaml index a31c1c74..55f66b9b 100644 --- a/docker/services/pacemaker/manila-share.yaml +++ b/docker/services/pacemaker/manila-share.yaml @@ -166,3 +166,33 @@ outputs: - name: Stop and disable manila_share service tags: step2 service: name=openstack-manila-share state=stopped enabled=no + update_tasks: + - name: Get docker Manila-Share image + set_fact: + docker_image: {get_param: DockerManilaShareImage} + docker_image_latest: *manila_share_image_pcmklatest + when: step == '2' + - name: Get previous Manila-Share image id + shell: "docker images | awk '/manila-share.* pcmklatest/{print $3}'" + register: manila_share_image_id + - block: + - name: Get a list of container using Manila-Share image + shell: "docker ps -q -f 'ancestor={{manila_share_image_id.stdout}}'" + register: manila-share_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same Manila-Share image + shell: "docker rm -fv {{item}}" + with_items: "{{ manila_share_containers_to_destroy.stdout_lines }}" + - name: Remove previous Manila-Share images + shell: "docker rmi -f {{manila_share_image_id.stdout}}" + when: + - step == '2' + - manila_share_image_id.stdout != '' + - name: Pull latest Manila-Share images + command: "docker pull {{docker_image}}" + when: step == "2" + - name: Retag pcmklatest to latest Manila-Share image + shell: "docker tag {{docker_image}} {{docker_image_latest}}" + when: step == "2" + # Got to check that pacemaker_is_active is working fine with bundle. + # TODO: pacemaker_is_active resource doesn't support bundle. diff --git a/docker/services/pacemaker/rabbitmq.yaml b/docker/services/pacemaker/rabbitmq.yaml index 5660856a..f17747c0 100644 --- a/docker/services/pacemaker/rabbitmq.yaml +++ b/docker/services/pacemaker/rabbitmq.yaml @@ -248,3 +248,33 @@ outputs: - name: Disable rabbitmq service tags: step2 service: name=rabbitmq-server enabled=no + update_tasks: + - name: Get docker Rabbitmq image + set_fact: + docker_image: {get_param: DockerRabbitmqImage} + docker_image_latest: *rabbitmq_image_pcmklatest + when: step == '2' + - name: Get previous Rabbitmq image id + shell: "docker images | awk '/rabbitmq.* pcmklatest/{print $3}'" + register: rabbitmq_image_id + - block: + - name: Get a list of container using Rabbitmq image + shell: "docker ps -q -f 'ancestor={{rabbitmq_image_id.stdout}}'" + register: rabbitmq_containers_to_destroy + # It will be recreated with the delpoy step. + - name: Remove any container using the same Rabbitmq image + shell: "docker rm -fv {{item}}" + with_items: "{{ rabbitmq_containers_to_destroy.stdout_lines }}" + - name: Remove previous Rabbitmq images + shell: "docker rmi -f {{rabbitmq_image_id.stdout}}" + when: + - step == '2' + - rabbitmq_image_id.stdout != '' + - name: Pull latest Rabbitmq images + command: "docker pull {{docker_image}}" + when: step == "2" + - name: Retag pcmklatest to latest Rabbitmq image + shell: "docker tag {{docker_image}} {{docker_image_latest}}" + when: step == "2" + # Got to check that pacemaker_is_active is working fine with bundle. + # TODO: pacemaker_is_active resource doesn't support bundle. diff --git a/docker/services/rabbitmq.yaml b/docker/services/rabbitmq.yaml index 632bdc29..237995b9 100644 --- a/docker/services/rabbitmq.yaml +++ b/docker/services/rabbitmq.yaml @@ -220,3 +220,9 @@ outputs: - name: Stop and disable rabbitmq service tags: step2 service: name=rabbitmq-server state=stopped enabled=no + update_tasks: + # TODO: Are we sure we want to support this. Rolling update + # without pacemaker may fail. Do we test this ? In any case, + # this is under paunch control so the latest image should be + # pulled in by the deploy steps. Same question for other + # usually managed by pacemaker container. diff --git a/puppet/services/pacemaker.yaml b/puppet/services/pacemaker.yaml index 158d04bd..badb1a4e 100644 --- a/puppet/services/pacemaker.yaml +++ b/puppet/services/pacemaker.yaml @@ -156,3 +156,15 @@ outputs: - name: Start pacemaker cluster tags: step4 pacemaker_cluster: state=online + update_tasks: + - name: Check pacemaker cluster running before the minor update + when: step == "0" # TODO(marios) disabling validations? + pacemaker_cluster: state=online check_and_fail=true + async: 30 + poll: 4 + - name: Stop pacemaker cluster + when: step == "1" + pacemaker_cluster: state=offline + - name: Start pacemaker cluster + when: step == "4" + pacemaker_cluster: state=online diff --git a/puppet/services/tripleo-packages.yaml b/puppet/services/tripleo-packages.yaml index 0dbdbf89..bfdac3e3 100644 --- a/puppet/services/tripleo-packages.yaml +++ b/puppet/services/tripleo-packages.yaml @@ -66,6 +66,13 @@ outputs: tags: step3 yum: name=* state=latest update_tasks: + - name: Check for existing yum.pid + stat: path=/var/run/yum.pid + register: yum_pid_file + when: step == "0" or step == "3" + - name: Exit if existing yum process + fail: msg="ERROR existing yum.pid detected - can't continue! Please ensure there is no other package update process for the duration of the minor update worfklow. Exiting." + when: (step == "0" or step == "3") and yum_pid_file.stat.exists - name: Update all packages - yum: name=* state=latest + yum: name=* state=latest update_cache=yes # cache for tripleo/+bug/1703830 when: step == "3" diff --git a/tools/yaml-validate.py b/tools/yaml-validate.py index de8ba806..c3229621 100755 --- a/tools/yaml-validate.py +++ b/tools/yaml-validate.py @@ -35,7 +35,8 @@ OPTIONAL_SECTIONS = ['workflow_tasks'] REQUIRED_DOCKER_SECTIONS = ['service_name', 'docker_config', 'puppet_config', 'config_settings', 'step_config'] OPTIONAL_DOCKER_SECTIONS = ['docker_puppet_tasks', 'upgrade_tasks', - 'post_upgrade_tasks', 'service_config_settings', + 'post_upgrade_tasks', 'update_tasks', + 'service_config_settings', 'host_prep_tasks', 'metadata_settings', 'kolla_config', 'logging_source', 'logging_groups'] REQUIRED_DOCKER_PUPPET_CONFIG_SECTIONS = ['config_volume', 'step_config', |