diff options
Diffstat (limited to 'extraconfig')
| -rw-r--r-- | extraconfig/all_nodes/mac_hostname.yaml | 6 | ||||
| -rw-r--r-- | extraconfig/all_nodes/random_string.yaml | 2 | ||||
| -rw-r--r-- | extraconfig/post_deploy/example.yaml | 1 | ||||
| -rw-r--r-- | extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml | 2 | ||||
| -rw-r--r-- | extraconfig/tasks/noop.yaml | 10 | ||||
| -rwxr-xr-x | extraconfig/tasks/pacemaker_resource_restart.sh | 70 | ||||
| -rw-r--r-- | extraconfig/tasks/post_puppet_pacemaker.yaml | 44 | ||||
| -rw-r--r-- | extraconfig/tasks/pre_puppet_pacemaker.yaml | 30 | ||||
| -rwxr-xr-x | extraconfig/tasks/yum_update.sh | 29 |
9 files changed, 183 insertions, 11 deletions
diff --git a/extraconfig/all_nodes/mac_hostname.yaml b/extraconfig/all_nodes/mac_hostname.yaml index 739cbf0a..5883e06a 100644 --- a/extraconfig/all_nodes/mac_hostname.yaml +++ b/extraconfig/all_nodes/mac_hostname.yaml @@ -40,6 +40,7 @@ resources: CollectMacDeploymentsController: type: OS::Heat::SoftwareDeployments properties: + name: CollectMacDeploymentsController servers: {get_param: controller_servers} config: {get_resource: CollectMacConfig} actions: ['CREATE'] # Only do this on CREATE @@ -47,6 +48,7 @@ resources: CollectMacDeploymentsCompute: type: OS::Heat::SoftwareDeployments properties: + name: CollectMacDeploymentsCompute servers: {get_param: compute_servers} config: {get_resource: CollectMacConfig} actions: ['CREATE'] # Only do this on CREATE @@ -54,6 +56,7 @@ resources: CollectMacDeploymentsBlockStorage: type: OS::Heat::SoftwareDeployments properties: + name: CollectMacDeploymentsBlockStorage servers: {get_param: blockstorage_servers} config: {get_resource: CollectMacConfig} actions: ['CREATE'] # Only do this on CREATE @@ -61,6 +64,7 @@ resources: CollectMacDeploymentsObjectStorage: type: OS::Heat::SoftwareDeployments properties: + name: CollectMacDeploymentsObjectStorage servers: {get_param: objectstorage_servers} config: {get_resource: CollectMacConfig} actions: ['CREATE'] # Only do this on CREATE @@ -68,6 +72,7 @@ resources: CollectMacDeploymentsCephStorage: type: OS::Heat::SoftwareDeployments properties: + name: CollectMacDeploymentsCephStorage servers: {get_param: cephstorage_servers} config: {get_resource: CollectMacConfig} actions: ['CREATE'] # Only do this on CREATE @@ -95,6 +100,7 @@ resources: DistributeMacDeploymentsController: type: OS::Heat::SoftwareDeployments properties: + name: DistributeMacDeploymentsController servers: {get_param: controller_servers} config: {get_resource: DistributeMacConfig} input_values: diff --git a/extraconfig/all_nodes/random_string.yaml b/extraconfig/all_nodes/random_string.yaml index b4b30274..49d2d8b6 100644 --- a/extraconfig/all_nodes/random_string.yaml +++ b/extraconfig/all_nodes/random_string.yaml @@ -41,6 +41,7 @@ resources: RandomDeploymentsController: type: OS::Heat::SoftwareDeployments properties: + name: RandomDeploymentsController servers: {get_param: controller_servers} config: {get_resource: RandomConfig} actions: ['CREATE'] # Only do this on CREATE @@ -50,6 +51,7 @@ resources: RandomDeploymentsCompute: type: OS::Heat::SoftwareDeployments properties: + name: RandomDeploymentsCompute servers: {get_param: compute_servers} config: {get_resource: RandomConfig} actions: ['CREATE'] # Only do this on CREATE diff --git a/extraconfig/post_deploy/example.yaml b/extraconfig/post_deploy/example.yaml index 1d3dca25..f83dff76 100644 --- a/extraconfig/post_deploy/example.yaml +++ b/extraconfig/post_deploy/example.yaml @@ -22,6 +22,7 @@ resources: ExtraDeployments: type: OS::Heat::SoftwareDeployments properties: + name: ExtraDeployments servers: {get_param: servers} config: {get_resource: ExtraConfig} actions: ['CREATE'] # Only do this on CREATE diff --git a/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml b/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml index d5160915..a884bdae 100644 --- a/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml +++ b/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml @@ -73,6 +73,7 @@ resources: RHELRegistrationDeployment: type: OS::Heat::SoftwareDeployment properties: + name: RHELRegistrationDeployment server: {get_param: server} config: {get_resource: RHELRegistration} actions: ['CREATE'] # Only do this on CREATE @@ -106,6 +107,7 @@ resources: RHELUnregistrationDeployment: type: OS::Heat::SoftwareDeployment properties: + name: RHELUnregistrationDeployment server: {get_param: server} config: {get_resource: RHELUnregistration} actions: ['DELETE'] # Only do this on DELETE diff --git a/extraconfig/tasks/noop.yaml b/extraconfig/tasks/noop.yaml new file mode 100644 index 00000000..0cff7469 --- /dev/null +++ b/extraconfig/tasks/noop.yaml @@ -0,0 +1,10 @@ +heat_template_version: 2014-10-16 +description: 'No-op task' + +parameters: + servers: + type: json + input_values: + type: json + default: {} + description: input values for the software deployments diff --git a/extraconfig/tasks/pacemaker_resource_restart.sh b/extraconfig/tasks/pacemaker_resource_restart.sh new file mode 100755 index 00000000..12201097 --- /dev/null +++ b/extraconfig/tasks/pacemaker_resource_restart.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +set -eux + +pacemaker_status=$(systemctl is-active pacemaker) +check_interval=3 + +function check_resource { + + service=$1 + state=$2 + timeout=$3 + tstart=$(date +%s) + tend=$(( $tstart + $timeout )) + + if [ "$state" = "stopped" ]; then + match_for_incomplete='Started' + else # started + match_for_incomplete='Stopped' + fi + + while (( $(date +%s) < $tend )); do + node_states=$(pcs status --full | grep "$service" | grep -v Clone) + if echo "$node_states" | grep -q "$match_for_incomplete"; then + echo "$service not yet $state, sleeping $check_interval seconds." + sleep $check_interval + else + echo "$service has $state" + timeout -k 10 $timeout crm_resource --wait + return + fi + done + + echo "$service never $state after $timeout seconds" | tee /dev/fd/2 + exit 1 + +} + +# Run if pacemaker is running, we're the bootstrap node, +# and we're updating the deployment (not creating). +if [ "$pacemaker_status" = "active" -a \ + "$(hiera bootstrap_nodeid)" = "$(facter hostname)" -a \ + "$(hiera update_identifier)" != "nil" ]; then + + #ensure neutron constraints like + #https://review.openstack.org/#/c/245093/ + if pcs constraint order show | grep "start neutron-server-clone then start neutron-ovs-cleanup-clone"; then + pcs constraint remove order-neutron-server-clone-neutron-ovs-cleanup-clone-mandatory + fi + + pcs resource disable httpd + check_resource httpd stopped 300 + pcs resource disable openstack-keystone + check_resource openstack-keystone stopped 1800 + + if pcs status | grep haproxy-clone; then + pcs resource restart haproxy-clone + fi + pcs resource restart redis-master + pcs resource restart mongod-clone + pcs resource restart rabbitmq-clone + pcs resource restart memcached-clone + pcs resource restart galera-master + + pcs resource enable openstack-keystone + check_resource openstack-keystone started 1800 + pcs resource enable httpd + check_resource httpd started 800 + +fi diff --git a/extraconfig/tasks/post_puppet_pacemaker.yaml b/extraconfig/tasks/post_puppet_pacemaker.yaml new file mode 100644 index 00000000..7de41d94 --- /dev/null +++ b/extraconfig/tasks/post_puppet_pacemaker.yaml @@ -0,0 +1,44 @@ +heat_template_version: 2014-10-16 +description: 'Post-Puppet Config for Pacemaker deployments' + +parameters: + servers: + type: json + input_values: + type: json + description: input values for the software deployments + +resources: + + ControllerPostPuppetMaintenanceModeConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: | + #!/bin/bash + pacemaker_status=$(systemctl is-active pacemaker) + + if [ "$pacemaker_status" = "active" ]; then + pcs property set maintenance-mode=false + fi + + ControllerPostPuppetMaintenanceModeDeployment: + type: OS::Heat::SoftwareDeployments + properties: + servers: {get_param: servers} + config: {get_resource: ControllerPostPuppetMaintenanceModeConfig} + input_values: {get_param: input_values} + + ControllerPostPuppetRestartConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: {get_file: pacemaker_resource_restart.sh} + + ControllerPostPuppetRestartDeployment: + type: OS::Heat::SoftwareDeployments + depends_on: ControllerPostPuppetMaintenanceModeDeployment + properties: + servers: {get_param: servers} + config: {get_resource: ControllerPostPuppetRestartConfig} + input_values: {get_param: input_values} diff --git a/extraconfig/tasks/pre_puppet_pacemaker.yaml b/extraconfig/tasks/pre_puppet_pacemaker.yaml new file mode 100644 index 00000000..2cfe92a7 --- /dev/null +++ b/extraconfig/tasks/pre_puppet_pacemaker.yaml @@ -0,0 +1,30 @@ +heat_template_version: 2014-10-16 +description: 'Pre-Puppet Config for Pacemaker deployments' + +parameters: + servers: + type: json + input_values: + type: json + description: input values for the software deployments + +resources: + + ControllerPrePuppetMaintenanceModeConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: | + #!/bin/bash + pacemaker_status=$(systemctl is-active pacemaker) + + if [ "$pacemaker_status" = "active" ]; then + pcs property set maintenance-mode=true + fi + + ControllerPrePuppetMaintenanceModeDeployment: + type: OS::Heat::SoftwareDeployments + properties: + servers: {get_param: servers} + config: {get_resource: ControllerPrePuppetMaintenanceModeConfig} + input_values: {get_param: input_values} diff --git a/extraconfig/tasks/yum_update.sh b/extraconfig/tasks/yum_update.sh index e32369e1..39179024 100755 --- a/extraconfig/tasks/yum_update.sh +++ b/extraconfig/tasks/yum_update.sh @@ -24,6 +24,7 @@ update_identifier=${update_identifier//[^a-zA-Z0-9-_]/} # seconds to wait for this node to rejoin the cluster after update cluster_start_timeout=600 galera_sync_timeout=360 +cluster_settle_timeout=1800 timestamp_file="$timestamp_dir/$update_identifier" if [[ -a "$timestamp_file" ]]; then @@ -122,13 +123,16 @@ openstack-nova-scheduler" echo "Setting resource start/stop timeouts" for service in $SERVICES; do - pcs -f $pacemaker_dumpfile resource update $service op start timeout=100s op stop timeout=100s + pcs -f $pacemaker_dumpfile resource update $service op start timeout=200s op stop timeout=200s done # mongod start timeout is higher, setting only stop timeout - pcs -f $pacemaker_dumpfile resource update mongod op stop timeout=100s + pcs -f $pacemaker_dumpfile resource update mongod op start timeout=370s op stop timeout=200s echo "Applying new Pacemaker config" - pcs cluster cib-push $pacemaker_dumpfile + if ! pcs cluster cib-push $pacemaker_dumpfile; then + echo "ERROR failed to apply new pacemaker config" + exit 1 + fi echo "Pacemaker running, stopping cluster node and doing full package update" node_count=$(pcs status xml | grep -o "<nodes_configured.*/>" | grep -o 'number="[0-9]*"' | grep -o "[0-9]*") @@ -146,11 +150,11 @@ openstack-nova-scheduler" kill $(ps ax | grep -e "keepalived.*\.pid-vrrp" | awk '{print $1}') 2>/dev/null || : kill $(ps ax | grep -e "radvd.*\.pid\.radvd" | awk '{print $1}') 2>/dev/null || : else - echo "Excluding upgrading packages that are handled by config management tooling" - command_arguments="$command_arguments --skip-broken" - for exclude in $(cat /var/lib/tripleo/installed-packages/* | sort -u); do - command_arguments="$command_arguments --exclude $exclude" - done + echo "Upgrading openstack-puppet-modules" + yum -y update openstack-puppet-modules + echo "Upgrading other packages is handled by config management tooling" + echo -n "true" > $heat_outputs_path.update_managed_packages + exit 0 fi command=${command:-update} @@ -188,10 +192,13 @@ if [[ "$pacemaker_status" == "active" ]] ; then fi done - pcs status + echo "Waiting for pacemaker cluster to settle" + if ! timeout -k 10 $cluster_settle_timeout crm_resource --wait; then + echo "ERROR timed out while waiting for the cluster to settle" + exit 1 + fi -else - echo -n "true" > $heat_outputs_path.update_managed_packages + pcs status fi echo "Finished yum_update.sh on server $deploy_server_id at `date`" |