diff options
Diffstat (limited to 'extraconfig')
16 files changed, 544 insertions, 459 deletions
diff --git a/extraconfig/all_nodes/mac_hostname.yaml b/extraconfig/all_nodes/mac_hostname.j2.yaml index 7d8704e3..af6aa7f7 100644 --- a/extraconfig/all_nodes/mac_hostname.yaml +++ b/extraconfig/all_nodes/mac_hostname.j2.yaml @@ -9,15 +9,7 @@ description: > # out-of-tree templates they may require additional parameters if the # in-tree templates add a new role. parameters: - controller_servers: - type: json - compute_servers: - type: json - blockstorage_servers: - type: json - objectstorage_servers: - type: json - cephstorage_servers: + servers: type: json # Note extra parameters can be defined, then passed data via the # environment parameter_defaults, without modifying the parent template @@ -37,47 +29,17 @@ resources: # FIXME(shardy): Long term it'd be better if Heat SoftwareDeployments accepted # list instead of a map, then we could join the lists of servers into one # deployment instead of requiring one deployment per-role. - CollectMacDeploymentsController: +{% for role in roles %} + CollectMacDeployments{{role.name}}: type: OS::Heat::SoftwareDeployments properties: name: CollectMacDeploymentsController - servers: {get_param: controller_servers} - config: {get_resource: CollectMacConfig} - actions: ['CREATE'] # Only do this on CREATE - - CollectMacDeploymentsCompute: - type: OS::Heat::SoftwareDeployments - properties: - name: CollectMacDeploymentsCompute - servers: {get_param: compute_servers} - config: {get_resource: CollectMacConfig} - actions: ['CREATE'] # Only do this on CREATE - - CollectMacDeploymentsBlockStorage: - type: OS::Heat::SoftwareDeployments - properties: - name: CollectMacDeploymentsBlockStorage - servers: {get_param: blockstorage_servers} - config: {get_resource: CollectMacConfig} - actions: ['CREATE'] # Only do this on CREATE - - CollectMacDeploymentsObjectStorage: - type: OS::Heat::SoftwareDeployments - properties: - name: CollectMacDeploymentsObjectStorage - servers: {get_param: objectstorage_servers} - config: {get_resource: CollectMacConfig} - actions: ['CREATE'] # Only do this on CREATE - - CollectMacDeploymentsCephStorage: - type: OS::Heat::SoftwareDeployments - properties: - name: CollectMacDeploymentsCephStorage - servers: {get_param: cephstorage_servers} + servers: {get_param: servers, {{role.name}}} config: {get_resource: CollectMacConfig} actions: ['CREATE'] # Only do this on CREATE +{% endfor %} - # Now we distribute all-the-macs to all nodes + # Now we distribute all-the-macs to all Controller nodes DistributeMacConfig: type: OS::Heat::SoftwareConfig properties: @@ -101,7 +63,7 @@ resources: type: OS::Heat::SoftwareDeployments properties: name: DistributeMacDeploymentsController - servers: {get_param: controller_servers} + servers: {get_param: servers, Controller} config: {get_resource: DistributeMacConfig} input_values: # FIXME(shardy): It'd be more convenient if we could join these diff --git a/extraconfig/all_nodes/random_string.yaml b/extraconfig/all_nodes/random_string.j2.yaml index d38701e2..1c42cb85 100644 --- a/extraconfig/all_nodes/random_string.yaml +++ b/extraconfig/all_nodes/random_string.j2.yaml @@ -10,15 +10,7 @@ description: > # out-of-tree templates they may require additional parameters if the # in-tree templates add a new role. parameters: - controller_servers: - type: json - compute_servers: - type: json - blockstorage_servers: - type: json - objectstorage_servers: - type: json - cephstorage_servers: + servers: type: json # Note extra parameters can be defined, then passed data via the # environment parameter_defaults, without modifying the parent template @@ -42,7 +34,7 @@ resources: type: OS::Heat::SoftwareDeployments properties: name: RandomDeploymentsController - servers: {get_param: controller_servers} + servers: {get_param: servers, Controller} config: {get_resource: RandomConfig} actions: ['CREATE'] # Only do this on CREATE input_values: @@ -52,7 +44,7 @@ resources: type: OS::Heat::SoftwareDeployments properties: name: RandomDeploymentsCompute - servers: {get_param: compute_servers} + servers: {get_param: servers, Compute} config: {get_resource: RandomConfig} actions: ['CREATE'] # Only do this on CREATE input_values: diff --git a/extraconfig/all_nodes/swap-partition.j2.yaml b/extraconfig/all_nodes/swap-partition.j2.yaml new file mode 100644 index 00000000..014a96a1 --- /dev/null +++ b/extraconfig/all_nodes/swap-partition.j2.yaml @@ -0,0 +1,44 @@ +heat_template_version: 2014-10-16 + +description: > + Extra config to add swap space to nodes. + +# Parameters passed from the parent template - note if you maintain +# out-of-tree templates they may require additional parameters if the +# in-tree templates add a new role. +parameters: + servers: + type: json + swap_partition_label: + type: string + description: Swap partition label + default: 'swap1' + + +resources: + + SwapConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: | + #!/bin/bash + set -eux + swap_partition=$(realpath /dev/disk/by-label/$swap_partition_label) + swapon $swap_partition + echo "$swap_partition swap swap defaults 0 0" >> /etc/fstab + inputs: + - name: swap_partition_label + description: Swap partition label + default: 'swap1' + +{% for role in roles %} + {{role.name}}SwapDeployment: + type: OS::Heat::SoftwareDeploymentGroup + properties: + config: {get_resource: SwapConfig} + servers: {get_param: servers, {{role.name}}} + input_values: + swap_partition_label: {get_param: swap_partition_label} + actions: ["CREATE"] +{% endfor %} diff --git a/extraconfig/all_nodes/swap-partition.yaml b/extraconfig/all_nodes/swap-partition.yaml deleted file mode 100644 index e6fa9eca..00000000 --- a/extraconfig/all_nodes/swap-partition.yaml +++ /dev/null @@ -1,86 +0,0 @@ -heat_template_version: 2014-10-16 - -description: > - Extra config to add swap space to nodes. - -# Parameters passed from the parent template - note if you maintain -# out-of-tree templates they may require additional parameters if the -# in-tree templates add a new role. -parameters: - controller_servers: - type: json - compute_servers: - type: json - blockstorage_servers: - type: json - objectstorage_servers: - type: json - cephstorage_servers: - type: json - swap_partition_label: - type: string - description: Swap partition label - default: 'swap1' - - -resources: - - SwapConfig: - type: OS::Heat::SoftwareConfig - properties: - group: script - config: | - #!/bin/bash - set -eux - swap_partition=$(realpath /dev/disk/by-label/$swap_partition_label) - swapon $swap_partition - echo "$swap_partition swap swap defaults 0 0" >> /etc/fstab - inputs: - - name: swap_partition_label - description: Swap partition label - default: 'swap1' - - ControllerSwapDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - config: {get_resource: SwapConfig} - servers: {get_param: controller_servers} - input_values: - swap_partition_label: {get_param: swap_partition_label} - actions: ["CREATE"] - - ComputeSwapDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - config: {get_resource: SwapConfig} - servers: {get_param: compute_servers} - input_values: - swap_partition_label: {get_param: swap_partition_label} - actions: ["CREATE"] - - BlockStorageSwapDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - config: {get_resource: SwapConfig} - servers: {get_param: blockstorage_servers} - input_values: - swap_partition_label: {get_param: swap_partition_label} - actions: ["CREATE"] - - ObjectStorageSwapDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - config: {get_resource: SwapConfig} - servers: {get_param: objectstorage_servers} - input_values: - swap_partition_label: {get_param: swap_partition_label} - actions: ["CREATE"] - - CephStorageSwapDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - config: {get_resource: SwapConfig} - servers: {get_param: cephstorage_servers} - input_values: - swap_partition_label: {get_param: swap_partition_label} - actions: ["CREATE"] diff --git a/extraconfig/all_nodes/swap.j2.yaml b/extraconfig/all_nodes/swap.j2.yaml new file mode 100644 index 00000000..97149080 --- /dev/null +++ b/extraconfig/all_nodes/swap.j2.yaml @@ -0,0 +1,58 @@ +heat_template_version: 2014-10-16 + +description: > + Extra config to add swap space to nodes. + +# Parameters passed from the parent template - note if you maintain +# out-of-tree templates they may require additional parameters if the +# in-tree templates add a new role. +parameters: + servers: + type: json + swap_size_megabytes: + type: string + description: Amount of swap space to allocate in megabytes + default: '4096' + swap_path: + type: string + description: Full path to location of swap file + default: '/swap' + + +resources: + + SwapConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: | + #!/bin/bash + set -eux + if [ ! -f $swap_path ]; then + dd if=/dev/zero of=$swap_path count=$swap_size_megabytes bs=1M + chmod 0600 $swap_path + mkswap $swap_path + swapon $swap_path + else + echo "$swap_path already exists" + fi + echo "$swap_path swap swap defaults 0 0" >> /etc/fstab + inputs: + - name: swap_size_megabytes + description: Amount of swap space to allocate in megabytes + default: '4096' + - name: swap_path + description: Full path to location of swap file + default: '/swap' + +{% for role in roles %} + {{role.name}}SwapDeployment: + type: OS::Heat::SoftwareDeploymentGroup + properties: + config: {get_resource: SwapConfig} + servers: {get_param: servers, {{role.name}}} + input_values: + swap_size_megabytes: {get_param: swap_size_megabytes} + swap_path: {get_param: swap_path} + actions: ["CREATE"] +{% endfor %} diff --git a/extraconfig/all_nodes/swap.yaml b/extraconfig/all_nodes/swap.yaml deleted file mode 100644 index 5383ffc9..00000000 --- a/extraconfig/all_nodes/swap.yaml +++ /dev/null @@ -1,104 +0,0 @@ -heat_template_version: 2014-10-16 - -description: > - Extra config to add swap space to nodes. - -# Parameters passed from the parent template - note if you maintain -# out-of-tree templates they may require additional parameters if the -# in-tree templates add a new role. -parameters: - controller_servers: - type: json - compute_servers: - type: json - blockstorage_servers: - type: json - objectstorage_servers: - type: json - cephstorage_servers: - type: json - swap_size_megabytes: - type: string - description: Amount of swap space to allocate in megabytes - default: '4096' - swap_path: - type: string - description: Full path to location of swap file - default: '/swap' - - -resources: - - SwapConfig: - type: OS::Heat::SoftwareConfig - properties: - group: script - config: | - #!/bin/bash - set -eux - if [ ! -f $swap_path ]; then - dd if=/dev/zero of=$swap_path count=$swap_size_megabytes bs=1M - chmod 0600 $swap_path - mkswap $swap_path - swapon $swap_path - else - echo "$swap_path already exists" - fi - echo "$swap_path swap swap defaults 0 0" >> /etc/fstab - inputs: - - name: swap_size_megabytes - description: Amount of swap space to allocate in megabytes - default: '4096' - - name: swap_path - description: Full path to location of swap file - default: '/swap' - - ControllerSwapDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - config: {get_resource: SwapConfig} - servers: {get_param: controller_servers} - input_values: - swap_size_megabytes: {get_param: swap_size_megabytes} - swap_path: {get_param: swap_path} - actions: ["CREATE"] - - ComputeSwapDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - config: {get_resource: SwapConfig} - servers: {get_param: compute_servers} - input_values: - swap_size_megabytes: {get_param: swap_size_megabytes} - swap_path: {get_param: swap_path} - actions: ["CREATE"] - - BlockStorageSwapDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - config: {get_resource: SwapConfig} - servers: {get_param: blockstorage_servers} - input_values: - swap_size_megabytes: {get_param: swap_size_megabytes} - swap_path: {get_param: swap_path} - actions: ["CREATE"] - - ObjectStorageSwapDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - config: {get_resource: SwapConfig} - servers: {get_param: objectstorage_servers} - input_values: - swap_size_megabytes: {get_param: swap_size_megabytes} - swap_path: {get_param: swap_path} - actions: ["CREATE"] - - CephStorageSwapDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - config: {get_resource: SwapConfig} - servers: {get_param: cephstorage_servers} - input_values: - swap_size_megabytes: {get_param: swap_size_megabytes} - swap_path: {get_param: swap_path} - actions: ["CREATE"] diff --git a/extraconfig/tasks/major_upgrade_ceph_mon.sh b/extraconfig/tasks/major_upgrade_ceph_mon.sh index b76dd7c3..21a2b5bc 100755 --- a/extraconfig/tasks/major_upgrade_ceph_mon.sh +++ b/extraconfig/tasks/major_upgrade_ceph_mon.sh @@ -18,13 +18,13 @@ if ! [[ "$INSTALLED_VERSION" =~ ^0\.94.* ]]; then fi CEPH_STATUS=$(ceph health | awk '{print $1}') -if [ ${CEPH_STATUS} = HEALTH_ERR ]; do +if [ ${CEPH_STATUS} = HEALTH_ERR ]; then echo ERROR: Ceph cluster status is HEALTH_ERR, cannot be upgraded exit 1 fi # Useful when upgrading with OSDs num < replica size -if [ $ignore_ceph_upgrade_warnings != "true" ]; then +if [ ${ignore_ceph_upgrade_warnings:-false} != "true" ]; then timeout 300 bash -c "while [ ${CEPH_STATUS} != HEALTH_OK ]; do echo WARNING: Waiting for Ceph cluster status to go HEALTH_OK; sleep 30; @@ -44,7 +44,7 @@ timeout 60 bash -c "while kill -0 ${MON_PID} 2> /dev/null; do done" # Update to Jewel -yum -y -q update ceph-mon +yum -y -q update ceph-mon ceph # Restart/Exit if not on Jewel, only in that case we need the changes UPDATED_VERSION=$(ceph --version | awk '{print $3}') @@ -54,7 +54,7 @@ if [[ "$UPDATED_VERSION" =~ ^0\.94.* ]]; then elif [[ "$UPDATED_VERSION" =~ ^10\.2.* ]]; then # RPM could own some of these but we can't take risks on the pre-existing files for d in /var/lib/ceph/mon /var/log/ceph /var/run/ceph /etc/ceph; do - chown -R ceph:ceph $d + chown -R ceph:ceph $d || echo WARNING: chown of $d failed done # Replay udev events with newer rules diff --git a/extraconfig/tasks/major_upgrade_ceph_storage.sh b/extraconfig/tasks/major_upgrade_ceph_storage.sh index 03a1c1c2..dc80a724 100644 --- a/extraconfig/tasks/major_upgrade_ceph_storage.sh +++ b/extraconfig/tasks/major_upgrade_ceph_storage.sh @@ -63,7 +63,7 @@ if [[ "$UPDATED_VERSION" =~ ^0\.94.* ]]; then elif [[ "$UPDATED_VERSION" =~ ^10\.2.* ]]; then # RPM could own some of these but we can't take risks on the pre-existing files for d in /var/lib/ceph/osd /var/log/ceph /var/run/ceph /etc/ceph; do - chown -R ceph:ceph $d + chown -R ceph:ceph $d || echo WARNING: chown of $d failed done # Replay udev events with newer rules diff --git a/extraconfig/tasks/major_upgrade_check.sh b/extraconfig/tasks/major_upgrade_check.sh new file mode 100755 index 00000000..dc7ec71a --- /dev/null +++ b/extraconfig/tasks/major_upgrade_check.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +set -eu + +check_cluster() +{ + if pcs status 2>&1 | grep -E '(cluster is not currently running)|(OFFLINE:)'; then + echo_error "ERROR: upgrade cannot start with some cluster nodes being offline" + exit 1 + fi +} + +check_pcsd() +{ + if pcs status 2>&1 | grep -E 'Offline'; then + echo_error "ERROR: upgrade cannot start with some pcsd daemon offline" + exit 1 + fi +} + +check_disk_for_mysql_dump() +{ + # Where to backup current database if mysql need to be upgraded + MYSQL_BACKUP_DIR=/var/tmp/mysql_upgrade_osp + MYSQL_TEMP_UPGRADE_BACKUP_DIR=/var/lib/mysql-temp-upgrade-backup + # Spare disk ratio for extra safety + MYSQL_BACKUP_SIZE_RATIO=1.2 + + # Shall we upgrade mysql data directory during the stack upgrade? + if [ "$mariadb_do_major_upgrade" = "auto" ]; then + ret=$(is_mysql_upgrade_needed) + if [ $ret = "1" ]; then + DO_MYSQL_UPGRADE=1 + else + DO_MYSQL_UPGRADE=0 + fi + echo "mysql upgrade required: $DO_MYSQL_UPGRADE" + elif [ "$mariadb_do_major_upgrade" = "no" ]; then + DO_MYSQL_UPGRADE=0 + else + DO_MYSQL_UPGRADE=1 + fi + + if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then + if [ $DO_MYSQL_UPGRADE -eq 1 ]; then + + if [ -d "$MYSQL_BACKUP_DIR" ]; then + echo_error "Error: $MYSQL_BACKUP_DIR exists already. Likely an upgrade failed previously" + exit 1 + fi + mkdir "$MYSQL_BACKUP_DIR" + if [ $? -ne 0 ]; then + echo_error "Error: could not create temporary backup directory $MYSQL_BACKUP_DIR" + exit 1 + fi + + # the /root/.my.cnf is needed because we set the mysql root + # password from liberty onwards + backup_flags="--defaults-extra-file=/root/.my.cnf -u root --flush-privileges --all-databases --single-transaction" + # While not ideal, this step allows us to calculate exactly how much space the dump + # will need. Our main goal here is avoiding any chance of corruption due to disk space + # exhaustion + backup_size=$(mysqldump $backup_flags 2>/dev/null | wc -c) + database_size=$(du -cb /var/lib/mysql | tail -1 | awk '{ print $1 }') + free_space=$(df -B1 --output=avail "$MYSQL_BACKUP_DIR" | tail -1) + + # we need at least space for a new mysql database + dump of the existing one, + # times a small factor for additional safety room + # note: bash doesn't do floating point math or floats in if statements, + # so use python to apply the ratio and cast it back to integer + required_space=$(python -c "from __future__ import print_function; print(\"%d\" % int((($database_size + $backup_size) * $MYSQL_BACKUP_SIZE_RATIO)))") + if [ $required_space -ge $free_space ]; then + echo_error "Error: not enough free space in $MYSQL_BACKUP_DIR ($required_space bytes required)" + exit 1 + fi + fi + fi +} + +check_python_rpm() +{ + # If for some reason rpm-python are missing we want to error out early enough + if ! rpm -q rpm-python &> /dev/null; then + echo_error "ERROR: upgrade cannot start without rpm-python installed" + exit 1 + fi +} + +check_clean_cluster() +{ + if crm_mon -1 | grep -A3 Failed; then + echo_error "ERROR: upgrade cannot start with failed resources on the cluster. Clean them up before starting: pcs resource cleanup." + exit 1 + fi +} + +check_galera_root_password() +{ + # BZ: 1357112 + if [ ! -e /root/.my.cnf ]; then + echo_error "ERROR: upgrade cannot be started, the galera password is missing. The overcloud needs update." + exit 1 + fi +} diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh index 0b702630..08c49c2c 100755 --- a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh +++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh @@ -4,11 +4,12 @@ set -eu cluster_sync_timeout=1800 -if pcs status 2>&1 | grep -E '(cluster is not currently running)|(OFFLINE:)'; then - echo_error "ERROR: upgrade cannot start with some cluster nodes being offline" - exit 1 -fi - +check_cluster +check_pcsd +check_clean_cluster +check_python_rpm +check_galera_root_password +check_disk_for_mysql_dump # We want to disable fencing during the cluster --stop as it might fence # nodes where a service fails to stop, which could be fatal during an upgrade @@ -17,12 +18,6 @@ fi STONITH_STATE=$(pcs property show stonith-enabled | grep "stonith-enabled" | awk '{ print $2 }') pcs property set stonith-enabled=false -# If for some reason rpm-python are missing we want to error out early enough -if ! rpm -q rpm-python &> /dev/null; then - echo_error "ERROR: upgrade cannot start without rpm-python installed" - exit 1 -fi - # In case the mysql package is updated, the database on disk must be # upgraded as well. This typically needs to happen during major # version upgrades (e.g. 5.5 -> 5.6, 5.5 -> 10.1...) @@ -35,75 +30,18 @@ fi # on mysql package versionning, but this can be overriden manually # to support specific upgrade scenario -# Where to backup current database if mysql need to be upgraded -MYSQL_BACKUP_DIR=/var/tmp/mysql_upgrade_osp -MYSQL_TEMP_UPGRADE_BACKUP_DIR=/var/lib/mysql-temp-upgrade-backup -# Spare disk ratio for extra safety -MYSQL_BACKUP_SIZE_RATIO=1.2 - -# Shall we upgrade mysql data directory during the stack upgrade? -if [ "$mariadb_do_major_upgrade" = "auto" ]; then - ret=$(is_mysql_upgrade_needed) - if [ $ret = "1" ]; then - DO_MYSQL_UPGRADE=1 - else - DO_MYSQL_UPGRADE=0 - fi - echo "mysql upgrade required: $DO_MYSQL_UPGRADE" -elif [ "$mariadb_do_major_upgrade" = "no" ]; then - DO_MYSQL_UPGRADE=0 -else - DO_MYSQL_UPGRADE=1 -fi - if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then if [ $DO_MYSQL_UPGRADE -eq 1 ]; then - if [ -d "$MYSQL_BACKUP_DIR" ]; then - echo_error "Error: $MYSQL_BACKUP_DIR exists already. Likely an upgrade failed previously" - exit 1 - fi - mkdir "$MYSQL_BACKUP_DIR" - if [ $? -ne 0 ]; then - echo_error "Error: could not create temporary backup directory $MYSQL_BACKUP_DIR" - exit 1 - fi - - # the /root/.my.cnf is needed because we set the mysql root - # password from liberty onwards - backup_flags="--defaults-extra-file=/root/.my.cnf -u root --flush-privileges --all-databases --single-transaction" - # While not ideal, this step allows us to calculate exactly how much space the dump - # will need. Our main goal here is avoiding any chance of corruption due to disk space - # exhaustion - backup_size=$(mysqldump $backup_flags 2>/dev/null | wc -c) - database_size=$(du -cb /var/lib/mysql | tail -1 | awk '{ print $1 }') - free_space=$(df -B1 --output=avail "$MYSQL_BACKUP_DIR" | tail -1) - - # we need at least space for a new mysql database + dump of the existing one, - # times a small factor for additional safety room - # note: bash doesn't do floating point math or floats in if statements, - # so use python to apply the ratio and cast it back to integer - required_space=$(python -c "from __future__ import print_function; print(\"%d\" % int((($database_size + $backup_size) * $MYSQL_BACKUP_SIZE_RATIO)))") - if [ $required_space -ge $free_space ]; then - echo_error "Error: not enough free space in $MYSQL_BACKUP_DIR ($required_space bytes required)" - exit 1 - fi - mysqldump $backup_flags > "$MYSQL_BACKUP_DIR/openstack_database.sql" cp -rdp /etc/my.cnf* "$MYSQL_BACKUP_DIR" fi pcs resource disable httpd check_resource httpd stopped 1800 - pcs resource disable openstack-core - check_resource openstack-core stopped 1800 pcs resource disable redis check_resource redis stopped 600 - pcs resource disable mongod - check_resource mongod stopped 600 pcs resource disable rabbitmq check_resource rabbitmq stopped 600 - pcs resource disable memcached - check_resource memcached stopped 600 pcs resource disable galera check_resource galera stopped 600 # Disable all VIPs before stopping the cluster, so that pcs doesn't use one as a source address: @@ -115,6 +53,15 @@ if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname) pcs cluster stop --all fi +stop_or_disable_service mongod +check_resource mongod stopped 600 +stop_or_disable_service memcached +check_resource memcached stopped 600 + + + + + # Swift isn't controled by pacemaker systemctl_swift stop @@ -198,11 +145,3 @@ fi # Pin messages sent to compute nodes to kilo, these will be upgraded later crudini --set /etc/nova/nova.conf upgrade_levels compute "$upgrade_level_nova_compute" -# https://bugzilla.redhat.com/show_bug.cgi?id=1284047 -# Change-Id: Ib3f6c12ff5471e1f017f28b16b1e6496a4a4b435 -crudini --set /etc/ceilometer/ceilometer.conf DEFAULT rpc_backend rabbit -# https://bugzilla.redhat.com/show_bug.cgi?id=1284058 -# Ifd1861e3df46fad0e44ff9b5cbd58711bbc87c97 Swift Ceilometer middleware no longer exists -crudini --set /etc/swift/proxy-server.conf pipeline:main pipeline "catch_errors healthcheck cache ratelimit tempurl formpost authtoken keystone staticweb proxy-logging proxy-server" -# LP: 1615035, required only for M/N upgrade. -crudini --set /etc/nova/nova.conf DEFAULT scheduler_host_manager host_manager diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh index bc708cce..cfe5bcfe 100755 --- a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh +++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh @@ -3,10 +3,10 @@ set -eu cluster_form_timeout=600 -cluster_settle_timeout=600 +cluster_settle_timeout=1800 galera_sync_timeout=600 -if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then +if [[ -n $(is_bootstrap_node) ]]; then pcs cluster start --all tstart=$(date +%s) @@ -26,14 +26,16 @@ if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname) for vip in $(pcs resource show | grep ocf::heartbeat:IPaddr2 | grep Stopped | awk '{ print $1 }'); do pcs resource enable $vip - check_resource $vip started 60 + check_resource_pacemaker $vip started 60 done +fi - pcs resource enable galera - check_resource galera started 600 - pcs resource enable mongod - check_resource mongod started 600 +start_or_enable_service galera +check_resource galera started 600 +start_or_enable_service mongod +check_resource mongod started 600 +if [[ -n $(is_bootstrap_node) ]]; then tstart=$(date +%s) while ! clustercheck; do sleep 5 @@ -53,19 +55,18 @@ if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname) keystone-manage db_sync neutron-db-manage --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugin.ini upgrade head nova-manage db sync - nova-manage api_db sync - - pcs resource enable memcached - check_resource memcached started 600 - pcs resource enable rabbitmq - check_resource rabbitmq started 600 - pcs resource enable redis - check_resource redis started 600 - pcs resource enable openstack-core - check_resource openstack-core started 1800 - pcs resource enable httpd - check_resource httpd started 1800 + #TODO(marios):someone from sahara needs to check this: + # sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head fi -# Swift isn't controled by heat +start_or_enable_service memcached +check_resource memcached started 600 +start_or_enable_service rabbitmq +check_resource rabbitmq started 600 +start_or_enable_service redis +check_resource redis started 600 +start_or_enable_service httpd +check_resource httpd started 1800 + +# Swift isn't controled by pacemaker systemctl_swift start diff --git a/extraconfig/tasks/major_upgrade_pacemaker.yaml b/extraconfig/tasks/major_upgrade_pacemaker.yaml index 598d22d0..7244f949 100644 --- a/extraconfig/tasks/major_upgrade_pacemaker.yaml +++ b/extraconfig/tasks/major_upgrade_pacemaker.yaml @@ -1,16 +1,8 @@ -heat_template_version: 2014-10-16 +heat_template_version: 2016-10-14 description: 'Upgrade for Pacemaker deployments' parameters: - controller_servers: - type: json - compute_servers: - type: json - blockstorage_servers: - type: json - objectstorage_servers: - type: json - cephstorage_servers: + servers: type: json input_values: type: json @@ -54,9 +46,10 @@ resources: CephMonUpgradeDeployment: type: OS::Heat::SoftwareDeploymentGroup properties: - servers: {get_param: controller_servers} + servers: {get_param: servers, Controller} config: {get_resource: CephMonUpgradeConfig} input_values: {get_param: input_values} + update_policy: batch_create: max_batch_size: 1 rolling_update: @@ -82,6 +75,7 @@ resources: params: MYSQL_MAJOR_UPGRADE: {get_param: MySqlMajorUpgrade} - get_file: pacemaker_common_functions.sh + - get_file: major_upgrade_check.sh - get_file: major_upgrade_pacemaker_migrations.sh - get_file: major_upgrade_controller_pacemaker_1.sh @@ -89,7 +83,7 @@ resources: type: OS::Heat::SoftwareDeploymentGroup depends_on: CephMonUpgradeDeployment properties: - servers: {get_param: controller_servers} + servers: {get_param: servers, Controller} config: {get_resource: ControllerPacemakerUpgradeConfig_Step1} input_values: {get_param: input_values} @@ -103,7 +97,7 @@ resources: BlockStorageUpgradeDeployment: type: OS::Heat::SoftwareDeploymentGroup properties: - servers: {get_param: blockstorage_servers} + servers: {get_param: servers, BlockStorage} config: {get_resource: BlockStorageUpgradeConfig} input_values: {get_param: input_values} @@ -122,7 +116,7 @@ resources: type: OS::Heat::SoftwareDeploymentGroup depends_on: BlockStorageUpgradeDeployment properties: - servers: {get_param: controller_servers} + servers: {get_param: servers, Controller} config: {get_resource: ControllerPacemakerUpgradeConfig_Step2} input_values: {get_param: input_values} diff --git a/extraconfig/tasks/major_upgrade_pacemaker_init.yaml b/extraconfig/tasks/major_upgrade_pacemaker_init.j2.yaml index 623549a0..f6aa3066 100644 --- a/extraconfig/tasks/major_upgrade_pacemaker_init.yaml +++ b/extraconfig/tasks/major_upgrade_pacemaker_init.j2.yaml @@ -3,15 +3,7 @@ description: 'Upgrade for Pacemaker deployments' parameters: - controller_servers: - type: json - compute_servers: - type: json - blockstorage_servers: - type: json - objectstorage_servers: - type: json - cephstorage_servers: + servers: type: json input_values: type: json @@ -43,45 +35,12 @@ resources: - "if [[ -f /etc/resolv.conf.save ]] ; then rm /etc/resolv.conf.save; fi\n\n" - get_param: UpgradeInitCommand - UpgradeInitControllerDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - servers: {get_param: controller_servers} - config: {get_resource: UpgradeInitConfig} - input_values: {get_param: input_values} - - UpgradeInitComputeDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - servers: {get_param: compute_servers} - config: {get_resource: UpgradeInitConfig} - input_values: {get_param: input_values} - - UpgradeInitBlockStorageDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - servers: {get_param: blockstorage_servers} - config: {get_resource: UpgradeInitConfig} - input_values: {get_param: input_values} - - UpgradeInitObjectStorageDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - servers: {get_param: objectstorage_servers} - config: {get_resource: UpgradeInitConfig} - input_values: {get_param: input_values} - - UpgradeInitCephStorageDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - servers: {get_param: cephstorage_servers} - config: {get_resource: UpgradeInitConfig} - input_values: {get_param: input_values} - # TODO(jistr): for Mitaka->Newton upgrades and further we can use # map_merge with input_values instead of feeding params into scripts # via str_replace on bash snippets + # FIXME(shardy) we have hard-coded per-role *ScriptConfig's here + # Would be better to have a common config for all roles ComputeDeliverUpgradeScriptConfig: type: OS::Heat::SoftwareConfig properties: @@ -97,35 +56,32 @@ resources: UPGRADE_LEVEL_NOVA_COMPUTE: {get_param: UpgradeLevelNovaCompute} - get_file: major_upgrade_compute.sh - ComputeDeliverUpgradeScriptDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - servers: {get_param: compute_servers} - config: {get_resource: ComputeDeliverUpgradeScriptConfig} - input_values: {get_param: input_values} - ObjectStorageDeliverUpgradeScriptConfig: type: OS::Heat::SoftwareConfig properties: group: script config: {get_file: major_upgrade_object_storage.sh} - ObjectStorageDeliverUpgradeScriptDeployment: - type: OS::Heat::SoftwareDeploymentGroup - properties: - servers: {get_param: objectstorage_servers} - config: {get_resource: ObjectStorageDeliverUpgradeScriptConfig} - input_values: {get_param: input_values} - CephStorageDeliverUpgradeScriptConfig: type: OS::Heat::SoftwareConfig properties: group: script config: {get_file: major_upgrade_ceph_storage.sh} - CephStorageDeliverUpgradeScriptDeployment: +{% for role in roles %} + UpgradeInit{{role.name}}Deployment: + type: OS::Heat::SoftwareDeploymentGroup + properties: + servers: {get_param: [servers, {{role.name}}]} + config: {get_resource: UpgradeInitConfig} + input_values: {get_param: input_values} + + {% if not role.name in ['Controller', 'BlockStorage'] %} + {{role.name}}DeliverUpgradeScriptDeployment: type: OS::Heat::SoftwareDeploymentGroup properties: - servers: {get_param: cephstorage_servers} - config: {get_resource: CephStorageDeliverUpgradeScriptConfig} + servers: {get_param: [servers, {{role.name}}]} + config: {get_resource: {{role.name}}DeliverUpgradeScriptConfig} input_values: {get_param: input_values} + {% endif %} +{% endfor %} diff --git a/extraconfig/tasks/mitaka_to_newton_aodh_data_migration.yaml b/extraconfig/tasks/mitaka_to_newton_aodh_data_migration.yaml index 9414ac19..91406fba 100644 --- a/extraconfig/tasks/mitaka_to_newton_aodh_data_migration.yaml +++ b/extraconfig/tasks/mitaka_to_newton_aodh_data_migration.yaml @@ -4,15 +4,7 @@ description: > Software-config for performing aodh data migration parameters: - controller_servers: - type: json - compute_servers: - type: json - blockstorage_servers: - type: json - objectstorage_servers: - type: json - cephstorage_servers: + servers: type: json input_values: type: json @@ -28,6 +20,6 @@ resources: AodhMysqlMigrationScriptDeployment: type: OS::Heat::SoftwareDeploymentGroup properties: - servers: {get_param: controller_servers} + servers: {get_param: servers, Controller} config: {get_resource: AodhMysqlMigrationScriptConfig} input_values: {get_param: input_values} diff --git a/extraconfig/tasks/pacemaker_common_functions.sh b/extraconfig/tasks/pacemaker_common_functions.sh index 7d794c97..4f17b69a 100755 --- a/extraconfig/tasks/pacemaker_common_functions.sh +++ b/extraconfig/tasks/pacemaker_common_functions.sh @@ -2,51 +2,286 @@ set -eu -function check_resource { +DEBUG="true" # set false if the verbosity is a problem +SCRIPT_NAME=$(basename $0) +function log_debug { + if [[ $DEBUG = "true" ]]; then + echo "`date` $SCRIPT_NAME tripleo-upgrade $(facter hostname) $1" + fi +} + +function is_bootstrap_node { + if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then + log_debug "Node is bootstrap" + echo "true" + fi +} +function check_resource_pacemaker { if [ "$#" -ne 3 ]; then - echo_error "ERROR: check_resource function expects 3 parameters, $# given" - exit 1 + echo_error "ERROR: check_resource function expects 3 parameters, $# given" + exit 1 fi - service=$1 - state=$2 - timeout=$3 + local service=$1 + local state=$2 + local timeout=$3 + + if [[ -z $(is_bootstrap_node) ]] ; then + log_debug "Node isn't bootstrap, skipping check for $service to be $state here " + return + else + log_debug "Node is bootstrap checking $service to be $state here" + fi if [ "$state" = "stopped" ]; then - match_for_incomplete='Started' + match_for_incomplete='Started' else # started - match_for_incomplete='Stopped' + match_for_incomplete='Stopped' fi nodes_local=$(pcs status | grep ^Online | sed 's/.*\[ \(.*\) \]/\1/g' | sed 's/ /\|/g') if timeout -k 10 $timeout crm_resource --wait; then - node_states=$(pcs status --full | grep "$service" | grep -v Clone | { egrep "$nodes_local" || true; } ) - if echo "$node_states" | grep -q "$match_for_incomplete"; then - echo_error "ERROR: cluster finished transition but $service was not in $state state, exiting." - exit 1 - else - echo "$service has $state" - fi - else - echo_error "ERROR: cluster remained unstable for more than $timeout seconds, exiting." + node_states=$(pcs status --full | grep "$service" | grep -v Clone | { egrep "$nodes_local" || true; } ) + if echo "$node_states" | grep -q "$match_for_incomplete"; then + echo_error "ERROR: cluster finished transition but $service was not in $state state, exiting." exit 1 + else + echo "$service has $state" + fi + else + echo_error "ERROR: cluster remained unstable for more than $timeout seconds, exiting." + exit 1 + fi + +} + +function pcmk_running { + if [[ $(systemctl is-active pacemaker) = "active" ]] ; then + echo "true" + fi +} + +function is_systemd_unknown { + local service=$1 + if [[ $(systemctl is-active "$service") = "unknown" ]]; then + log_debug "$service found to be unkown to systemd" + echo "true" + fi +} + +function grep_is_cluster_controlled { + local service=$1 + if [[ -n $(systemctl status $service -l | grep Drop-In -A 5 | grep pacemaker) || + -n $(systemctl status $service -l | grep "Cluster Controlled $service") ]] ; then + log_debug "$service is pcmk managed from systemctl grep" + echo "true" + fi +} + + +function is_systemd_managed { + local service=$1 + #if we have pcmk check to see if it is managed there + if [[ -n $(pcmk_running) ]]; then + if [[ -z $(pcs status --full | grep $service) && -z $(is_systemd_unknown $service) ]] ; then + log_debug "$service found to be systemd managed from pcs status" + echo "true" + fi + else + # if it is "unknown" to systemd, then it is pacemaker managed + if [[ -n $(is_systemd_unknown $service) ]] ; then + return + elif [[ -z $(grep_is_cluster_controlled $service) ]] ; then + echo "true" + fi + fi +} + +function is_pacemaker_managed { + local service=$1 + #if we have pcmk check to see if it is managed there + if [[ -n $(pcmk_running) ]]; then + if [[ -n $(pcs status --full | grep $service) ]]; then + log_debug "$service found to be pcmk managed from pcs status" + echo "true" + fi + else + # if it is unknown to systemd, then it is pcmk managed + if [[ -n $(is_systemd_unknown $service) ]]; then + echo "true" + elif [[ -n $(grep_is_cluster_controlled $service) ]] ; then + echo "true" + fi + fi +} + +function is_managed { + local service=$1 + if [[ -n $(is_pacemaker_managed $service) || -n $(is_systemd_managed $service) ]]; then + echo "true" + fi +} + +function check_resource_systemd { + + if [ "$#" -ne 3 ]; then + echo_error "ERROR: check_resource function expects 3 parameters, $# given" + exit 1 fi + local service=$1 + local state=$2 + local timeout=$3 + local check_interval=3 + + if [ "$state" = "stopped" ]; then + match_for_incomplete='active' + else # started + match_for_incomplete='inactive' + fi + + log_debug "Going to check_resource_systemd for $service to be $state" + + #sanity check is systemd managed: + if [[ -z $(is_systemd_managed $service) ]]; then + echo "ERROR - $service not found to be systemd managed." + exit 1 + fi + + tstart=$(date +%s) + tend=$(( $tstart + $timeout )) + while (( $(date +%s) < $tend )); do + if [[ "$(systemctl is-active $service)" = $match_for_incomplete ]]; then + echo "$service not yet $state, sleeping $check_interval seconds." + sleep $check_interval + else + echo "$service is $state" + return + fi + done + + echo "Timed out waiting for $service to go to $state after $timeout seconds" + exit 1 +} + + +function check_resource { + local service=$1 + local pcmk_managed=$(is_pacemaker_managed $service) + local systemd_managed=$(is_systemd_managed $service) + + if [[ -n $pcmk_managed && -n $systemd_managed ]] ; then + log_debug "ERROR $service managed by both systemd and pcmk - SKIPPING" + return + fi + + if [[ -n $pcmk_managed ]]; then + check_resource_pacemaker $@ + return + elif [[ -n $systemd_managed ]]; then + check_resource_systemd $@ + return + fi + log_debug "ERROR cannot check_resource for $service, not managed here?" +} + +function manage_systemd_service { + local action=$1 + local service=$2 + log_debug "Going to systemctl $action $service" + systemctl $action $service +} + +function manage_pacemaker_service { + local action=$1 + local service=$2 + # not if pacemaker isn't running! + if [[ -z $(pcmk_running) ]]; then + echo "$(facter hostname) pacemaker not active, skipping $action $service here" + elif [[ -n $(is_bootstrap_node) ]]; then + log_debug "Going to pcs resource $action $service" + pcs resource $action $service + fi +} + +function stop_or_disable_service { + local service=$1 + local pcmk_managed=$(is_pacemaker_managed $service) + local systemd_managed=$(is_systemd_managed $service) + + if [[ -n $pcmk_managed && -n $systemd_managed ]] ; then + log_debug "Skipping stop_or_disable $service due to management conflict" + return + fi + + log_debug "Stopping or disabling $service" + if [[ -n $pcmk_managed ]]; then + manage_pacemaker_service disable $service + return + elif [[ -n $systemd_managed ]]; then + manage_systemd_service stop $service + return + fi + log_debug "ERROR: $service not managed here?" +} + +function start_or_enable_service { + local service=$1 + local pcmk_managed=$(is_pacemaker_managed $service) + local systemd_managed=$(is_systemd_managed $service) + + if [[ -n $pcmk_managed && -n $systemd_managed ]] ; then + log_debug "Skipping start_or_enable $service due to management conflict" + return + fi + + log_debug "Starting or enabling $service" + if [[ -n $pcmk_managed ]]; then + manage_pacemaker_service enable $service + return + elif [[ -n $systemd_managed ]]; then + manage_systemd_service start $service + return + fi + log_debug "ERROR $service not managed here?" +} + +function restart_service { + local service=$1 + local pcmk_managed=$(is_pacemaker_managed $service) + local systemd_managed=$(is_systemd_managed $service) + + if [[ -n $pcmk_managed && -n $systemd_managed ]] ; then + log_debug "ERROR $service managed by both systemd and pcmk - SKIPPING" + return + fi + + log_debug "Restarting $service" + if [[ -n $pcmk_managed ]]; then + manage_pacemaker_service restart $service + return + elif [[ -n $systemd_managed ]]; then + manage_systemd_service restart $service + return + fi + log_debug "ERROR $service not managed here?" } function echo_error { echo "$@" | tee /dev/fd2 } +# swift is a special case because it is/was never handled by pacemaker +# when stand-alone swift is used, only swift-proxy is running on controllers function systemctl_swift { services=( openstack-swift-account-auditor openstack-swift-account-reaper openstack-swift-account-replicator openstack-swift-account \ openstack-swift-container-auditor openstack-swift-container-replicator openstack-swift-container-updater openstack-swift-container \ openstack-swift-object-auditor openstack-swift-object-replicator openstack-swift-object-updater openstack-swift-object openstack-swift-proxy ) - action=$1 + local action=$1 case $action in stop) - services=$(systemctl | grep swift | grep running | awk '{print $1}') + services=$(systemctl | grep openstack-swift- | grep running | awk '{print $1}') ;; start) enable_swift_storage=$(hiera -c /etc/puppet/hiera.yaml 'enable_swift_storage') @@ -54,9 +289,11 @@ function systemctl_swift { services=( openstack-swift-proxy ) fi ;; - *) services=() ;; # for safetly, should never happen + *) echo "Unknown action $action passed to systemctl_swift" + exit 1 + ;; # shouldn't ever happen... esac - for S in ${services[@]}; do - systemctl $action $S + for service in ${services[@]}; do + manage_systemd_service $action $service done } diff --git a/extraconfig/tasks/pacemaker_resource_restart.sh b/extraconfig/tasks/pacemaker_resource_restart.sh index fd1fd0dc..3da7efec 100755 --- a/extraconfig/tasks/pacemaker_resource_restart.sh +++ b/extraconfig/tasks/pacemaker_resource_restart.sh @@ -2,12 +2,9 @@ set -eux -pacemaker_status=$(systemctl is-active pacemaker) - # Run if pacemaker is running, we're the bootstrap node, # and we're updating the deployment (not creating). -if [ "$pacemaker_status" = "active" -a \ - "$(hiera bootstrap_nodeid)" = "$(facter hostname)" ]; then +if [[ -n $(pcmk_running) && -n $(is_bootstrap_node) ]]; then TIMEOUT=600 SERVICES_TO_RESTART="$(ls /var/lib/tripleo/pacemaker-restarts)" @@ -25,5 +22,4 @@ if [ "$pacemaker_status" = "active" -a \ pcs resource restart --wait=$TIMEOUT $service rm -f /var/lib/tripleo/pacemaker-restarts/$service done - fi |