summaryrefslogtreecommitdiffstats
path: root/extraconfig/tasks/yum_update.sh
blob: fa523e8353927289aff48c796d995a3207989c4f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
#!/bin/bash

# A heat-config-script which runs yum update during a stack-update.
# Inputs:
#   deploy_action - yum will only be run if this is UPDATE
#   update_identifier - yum will only run for previously unused values of update_identifier
#   command - yum sub-command to run, defaults to "update"
#   command_arguments - yum command arguments, defaults to ""

echo "Started yum_update.sh on server $deploy_server_id at `date`"
echo -n "false" > $heat_outputs_path.update_managed_packages

if [[ -z "$update_identifier" ]]; then
    echo "Not running due to unset update_identifier"
    exit 0
fi

timestamp_dir=/var/lib/overcloud-yum-update
mkdir -p $timestamp_dir

# sanitise to remove unusual characters
update_identifier=${update_identifier//[^a-zA-Z0-9-_]/}

# seconds to wait for this node to rejoin the cluster after update
cluster_start_timeout=360
galera_sync_timeout=360

timestamp_file="$timestamp_dir/$update_identifier"
if [[ -a "$timestamp_file" ]]; then
    echo "Not running for already-run timestamp \"$update_identifier\""
    exit 0
fi
touch "$timestamp_file"

command_arguments=${command_arguments:-}

list_updates=$(yum list updates)

if [[ "$list_updates" == "" ]]; then
    echo "No packages require updating"
    exit 0
fi

pacemaker_status=$(systemctl is-active pacemaker)

if [[ "$pacemaker_status" == "active" ]] ; then
    echo "Checking for and adding missing constraints"

    if ! pcs constraint order show | grep "start openstack-nova-novncproxy-clone then start openstack-nova-api-clone"; then
        pcs constraint order start openstack-nova-novncproxy-clone then openstack-nova-api-clone
    fi

    if ! pcs constraint order show | grep "start rabbitmq-clone then start openstack-keystone-clone"; then
        pcs constraint order start rabbitmq-clone then openstack-keystone-clone
    fi

    if ! pcs constraint order show | grep "promote galera-master then start openstack-keystone-clone"; then
        pcs constraint order promote galera-master then openstack-keystone-clone
    fi

    if ! pcs constraint order show | grep "start haproxy-clone then start openstack-keystone-clone"; then
        pcs constraint order start haproxy-clone then openstack-keystone-clone
    fi

    if ! pcs constraint order show | grep "start memcached-clone then start openstack-keystone-clone"; then
        pcs constraint order start memcached-clone then openstack-keystone-clone
    fi

    if ! pcs constraint order show | grep "promote redis-master then start openstack-ceilometer-central-clone"; then
        pcs constraint order promote redis-master then start openstack-ceilometer-central-clone require-all=false
    fi

    if ! pcs resource defaults | grep "resource-stickiness: INFINITY"; then
        pcs resource defaults resource-stickiness=INFINITY
    fi

    echo "Setting resource start/stop timeouts"

    # timeouts for non-openstack services and special cases
    pcs resource update haproxy op start timeout=100s
    pcs resource update haproxy op stop timeout=100s
    # mongod start timeout is also higher, setting only stop timeout
    pcs resource update mongod op stop timeout=100s
    # rabbit start timeout is already 100s
    pcs resource update rabbitmq op stop timeout=100s
    pcs resource update memcached op start timeout=100s
    pcs resource update memcached op stop timeout=100s
    pcs resource update httpd op start timeout=100s
    pcs resource update httpd op stop timeout=100s
    # neutron-netns-cleanup stop timeout is 300s, setting only start timeout
    pcs resource update neutron-netns-cleanup op start timeout=100s
    # neutron-ovs-cleanup stop timeout is 300s, setting only start timeout
    pcs resource update neutron-ovs-cleanup op start timeout=100s

    # timeouts for openstack services
    pcs resource update neutron-dhcp-agent op start timeout=100s
    pcs resource update neutron-dhcp-agent op stop timeout=100s
    pcs resource update neutron-l3-agent op start timeout=100s
    pcs resource update neutron-l3-agent op stop timeout=100s
    pcs resource update neutron-metadata-agent op start timeout=100s
    pcs resource update neutron-metadata-agent op stop timeout=100s
    pcs resource update neutron-openvswitch-agent op start timeout=100s
    pcs resource update neutron-openvswitch-agent op stop timeout=100s
    pcs resource update neutron-server op start timeout=100s
    pcs resource update neutron-server op stop timeout=100s
    pcs resource update openstack-ceilometer-alarm-evaluator op start timeout=100s
    pcs resource update openstack-ceilometer-alarm-evaluator op stop timeout=100s
    pcs resource update openstack-ceilometer-alarm-notifier op start timeout=100s
    pcs resource update openstack-ceilometer-alarm-notifier op stop timeout=100s
    pcs resource update openstack-ceilometer-api op start timeout=100s
    pcs resource update openstack-ceilometer-api op stop timeout=100s
    pcs resource update openstack-ceilometer-central op start timeout=100s
    pcs resource update openstack-ceilometer-central op stop timeout=100s
    pcs resource update openstack-ceilometer-collector op start timeout=100s
    pcs resource update openstack-ceilometer-collector op stop timeout=100s
    pcs resource update openstack-ceilometer-notification op start timeout=100s
    pcs resource update openstack-ceilometer-notification op stop timeout=100s
    pcs resource update openstack-cinder-api op start timeout=100s
    pcs resource update openstack-cinder-api op stop timeout=100s
    pcs resource update openstack-cinder-scheduler op start timeout=100s
    pcs resource update openstack-cinder-scheduler op stop timeout=100s
    pcs resource update openstack-cinder-volume op start timeout=100s
    pcs resource update openstack-cinder-volume op stop timeout=100s
    pcs resource update openstack-glance-api op start timeout=100s
    pcs resource update openstack-glance-api op stop timeout=100s
    pcs resource update openstack-glance-registry op start timeout=100s
    pcs resource update openstack-glance-registry op stop timeout=100s
    pcs resource update openstack-heat-api op start timeout=100s
    pcs resource update openstack-heat-api op stop timeout=100s
    pcs resource update openstack-heat-api-cfn op start timeout=100s
    pcs resource update openstack-heat-api-cfn op stop timeout=100s
    pcs resource update openstack-heat-api-cloudwatch op start timeout=100s
    pcs resource update openstack-heat-api-cloudwatch op stop timeout=100s
    pcs resource update openstack-heat-engine op start timeout=100s
    pcs resource update openstack-heat-engine op stop timeout=100s
    pcs resource update openstack-keystone op start timeout=100s
    pcs resource update openstack-keystone op stop timeout=100s
    pcs resource update openstack-nova-api op start timeout=100s
    pcs resource update openstack-nova-api op stop timeout=100s
    pcs resource update openstack-nova-conductor op start timeout=100s
    pcs resource update openstack-nova-conductor op stop timeout=100s
    pcs resource update openstack-nova-consoleauth op start timeout=100s
    pcs resource update openstack-nova-consoleauth op stop timeout=100s
    pcs resource update openstack-nova-novncproxy op start timeout=100s
    pcs resource update openstack-nova-novncproxy op stop timeout=100s
    pcs resource update openstack-nova-scheduler op start timeout=100s
    pcs resource update openstack-nova-scheduler op stop timeout=100s

    echo "Pacemaker running, stopping cluster node and doing full package update"
    node_count=$(pcs status xml | grep -o "<nodes_configured.*/>" | grep -o 'number="[0-9]*"' | grep -o "[0-9]*")
    if [[ "$node_count" == "1" ]] ; then
        echo "Active node count is 1, stopping node with --force"
        pcs cluster stop --force
    else
        pcs cluster stop
    fi
else
    echo "Excluding upgrading packages that are handled by config management tooling"
    command_arguments="$command_arguments --skip-broken"
    for exclude in $(cat /var/lib/tripleo/installed-packages/* | sort -u); do
        command_arguments="$command_arguments --exclude $exclude"
    done
fi

command=${command:-update}
full_command="yum -y $command $command_arguments"
echo "Running: $full_command"

result=$($full_command)
return_code=$?
echo "$result"
echo "yum return code: $return_code"

if [[ "$pacemaker_status" == "active" ]] ; then
    echo "Starting cluster node"
    pcs cluster start

    hostname=$(hostname -s)
    tstart=$(date +%s)
    while [[ "$(pcs status | grep "^Online" | grep -F -o $hostname)" == "" ]]; do
        sleep 5
        tnow=$(date +%s)
        if (( tnow-tstart > cluster_start_timeout )) ; then
            echo "ERROR $hostname failed to join cluster in $cluster_start_timeout seconds"
            pcs status
            exit 1
        fi
    done

    tstart=$(date +%s)
    while ! clustercheck; do
        sleep 5
        tnow=$(date +%s)
        if (( tnow-tstart > galera_sync_timeout )) ; then
            echo "ERROR galera sync timed out"
            exit 1
        fi
    done

    pcs status

else
    echo -n "true" > $heat_outputs_path.update_managed_packages
fi

echo "Finished yum_update.sh on server $deploy_server_id at `date`"

exit $return_code