diff options
Diffstat (limited to 'extraconfig')
40 files changed, 1236 insertions, 298 deletions
diff --git a/extraconfig/all_nodes/mac_hostname.j2.yaml b/extraconfig/all_nodes/mac_hostname.j2.yaml index af6aa7f7..fcf022ae 100644 --- a/extraconfig/all_nodes/mac_hostname.j2.yaml +++ b/extraconfig/all_nodes/mac_hostname.j2.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: > Example extra config for cluster config @@ -34,7 +34,7 @@ resources: type: OS::Heat::SoftwareDeployments properties: name: CollectMacDeploymentsController - servers: {get_param: servers, {{role.name}}} + servers: {get_param: [servers, {{role.name}}]} config: {get_resource: CollectMacConfig} actions: ['CREATE'] # Only do this on CREATE {% endfor %} @@ -63,7 +63,7 @@ resources: type: OS::Heat::SoftwareDeployments properties: name: DistributeMacDeploymentsController - servers: {get_param: servers, Controller} + servers: {get_param: [servers, Controller]} config: {get_resource: DistributeMacConfig} input_values: # FIXME(shardy): It'd be more convenient if we could join these diff --git a/extraconfig/all_nodes/random_string.j2.yaml b/extraconfig/all_nodes/random_string.j2.yaml index 1c42cb85..77d4b381 100644 --- a/extraconfig/all_nodes/random_string.j2.yaml +++ b/extraconfig/all_nodes/random_string.j2.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: > Example extra config for cluster config @@ -34,7 +34,7 @@ resources: type: OS::Heat::SoftwareDeployments properties: name: RandomDeploymentsController - servers: {get_param: servers, Controller} + servers: {get_param: [servers, Controller]} config: {get_resource: RandomConfig} actions: ['CREATE'] # Only do this on CREATE input_values: @@ -44,7 +44,7 @@ resources: type: OS::Heat::SoftwareDeployments properties: name: RandomDeploymentsCompute - servers: {get_param: servers, Compute} + servers: {get_param: [servers, Compute]} config: {get_resource: RandomConfig} actions: ['CREATE'] # Only do this on CREATE input_values: diff --git a/extraconfig/all_nodes/swap-partition.j2.yaml b/extraconfig/all_nodes/swap-partition.j2.yaml index 014a96a1..b6fef79f 100644 --- a/extraconfig/all_nodes/swap-partition.j2.yaml +++ b/extraconfig/all_nodes/swap-partition.j2.yaml @@ -1,11 +1,7 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata -description: > - Extra config to add swap space to nodes. +description: Template file to add a swap partition to a node. -# Parameters passed from the parent template - note if you maintain -# out-of-tree templates they may require additional parameters if the -# in-tree templates add a new role. parameters: servers: type: json @@ -14,9 +10,7 @@ parameters: description: Swap partition label default: 'swap1' - resources: - SwapConfig: type: OS::Heat::SoftwareConfig properties: @@ -25,8 +19,13 @@ resources: #!/bin/bash set -eux swap_partition=$(realpath /dev/disk/by-label/$swap_partition_label) - swapon $swap_partition - echo "$swap_partition swap swap defaults 0 0" >> /etc/fstab + if [ -f "$swap_partition" ]; then + swapon $swap_partition + echo "$swap_partition swap swap defaults 0 0" >> /etc/fstab + else + echo "$swap_partition needs to be a valid path" + echo "Check that $swap_partition_label is a valid partition label" + fi inputs: - name: swap_partition_label description: Swap partition label @@ -37,7 +36,7 @@ resources: type: OS::Heat::SoftwareDeploymentGroup properties: config: {get_resource: SwapConfig} - servers: {get_param: servers, {{role.name}}} + servers: {get_param: [servers, {{role.name}}]} input_values: swap_partition_label: {get_param: swap_partition_label} actions: ["CREATE"] diff --git a/extraconfig/all_nodes/swap.j2.yaml b/extraconfig/all_nodes/swap.j2.yaml index 97149080..044f817c 100644 --- a/extraconfig/all_nodes/swap.j2.yaml +++ b/extraconfig/all_nodes/swap.j2.yaml @@ -1,11 +1,7 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata -description: > - Extra config to add swap space to nodes. +description: Template file to add a swap file to a node. -# Parameters passed from the parent template - note if you maintain -# out-of-tree templates they may require additional parameters if the -# in-tree templates add a new role. parameters: servers: type: json @@ -18,9 +14,7 @@ parameters: description: Full path to location of swap file default: '/swap' - resources: - SwapConfig: type: OS::Heat::SoftwareConfig properties: @@ -50,7 +44,7 @@ resources: type: OS::Heat::SoftwareDeploymentGroup properties: config: {get_resource: SwapConfig} - servers: {get_param: servers, {{role.name}}} + servers: {get_param: [servers, {{role.name}}]} input_values: swap_size_megabytes: {get_param: swap_size_megabytes} swap_path: {get_param: swap_path} diff --git a/extraconfig/post_deploy/default.yaml b/extraconfig/post_deploy/default.yaml index ddfe0243..4da54ead 100644 --- a/extraconfig/post_deploy/default.yaml +++ b/extraconfig/post_deploy/default.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: 'Extra Post Deployment Config' parameters: servers: diff --git a/extraconfig/post_deploy/example.yaml b/extraconfig/post_deploy/example.yaml index f83dff76..8ac7eb73 100644 --- a/extraconfig/post_deploy/example.yaml +++ b/extraconfig/post_deploy/example.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: > Example extra config for post-deployment diff --git a/extraconfig/post_deploy/example_run_on_update.yaml b/extraconfig/post_deploy/example_run_on_update.yaml index 234488af..738e263b 100644 --- a/extraconfig/post_deploy/example_run_on_update.yaml +++ b/extraconfig/post_deploy/example_run_on_update.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: > Example extra config for post-deployment, this re-runs every update diff --git a/extraconfig/post_deploy/undercloud_post.sh b/extraconfig/post_deploy/undercloud_post.sh new file mode 100755 index 00000000..8bcae1d3 --- /dev/null +++ b/extraconfig/post_deploy/undercloud_post.sh @@ -0,0 +1,126 @@ +#!/bin/bash +set -eux + +ln -sf /etc/puppet/hiera.yaml /etc/hiera.yaml + + +# WRITE OUT STACKRC +if [ ! -e /root/stackrc ]; then + touch /root/stackrc + chmod 0600 /root/stackrc + +cat >> /root/stackrc <<-EOF_CAT +export OS_PASSWORD=$admin_password +export OS_AUTH_URL=$auth_url +export OS_USERNAME=admin +export OS_TENANT_NAME=admin +export COMPUTE_API_VERSION=1.1 +export NOVA_VERSION=1.1 +export OS_BAREMETAL_API_VERSION=1.15 +export OS_NO_CACHE=True +export OS_CLOUDNAME=undercloud +EOF_CAT + + if [ -n "$ssl_certificate" ]; then +cat >> /root/stackrc <<-EOF_CAT +export PYTHONWARNINGS="ignore:Certificate has no, ignore:A true SSLContext object is not available" +EOF_CAT + fi +fi + +source /root/stackrc + +if [ ! -f /root/.ssh/authorized_keys ]; then + sudo mkdir -p /root/.ssh + sudo chmod 7000 /root/.ssh/ + sudo touch /root/.ssh/authorized_keys + sudo chmod 600 /root/.ssh/authorized_keys +fi + +if [ ! -f /root/.ssh/id_rsa ]; then + ssh-keygen -b 1024 -N '' -f /root/.ssh/id_rsa +fi + +if ! grep "$(cat /root/.ssh/id_rsa.pub)" /root/.ssh/authorized_keys; then + cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys +fi + +PHYSICAL_NETWORK=ctlplane + +ctlplane_id=$(openstack network list -f csv -c ID -c Name --quote none | tail -n +2 | grep ctlplane | cut -d, -f1) +subnet_ids=$(openstack subnet list -f csv -c ID --quote none | tail -n +2) +subnet_id= + +for subnet_id in $subnet_ids; do + network_id=$(openstack subnet show -f value -c network_id $subnet_id) + if [ "$network_id" = "$ctlplane_id" ]; then + break + fi +done + +net_create=1 +if [ -n "$subnet_id" ]; then + cidr=$(openstack subnet show $subnet_id -f value -c cidr) + if [ "$cidr" = "$undercloud_network_cidr" ]; then + net_create=0 + else + echo "New cidr $undercloud_network_cidr does not equal old cidr $cidr" + echo "Will attempt to delete and recreate subnet $subnet_id" + fi +fi + +if [ "$net_create" -eq "1" ]; then + # Delete the subnet and network to make sure it doesn't already exist + if openstack subnet list | grep start; then + openstack subnet delete $(openstack subnet list | grep start | awk '{print $4}') + fi + if openstack network show ctlplane; then + openstack network delete ctlplane + fi + + + NETWORK_ID=$(openstack network create --provider-network-type=flat --provider-physical-network=ctlplane ctlplane | grep " id " | awk '{print $4}') + + NAMESERVER_ARG="" + if [ -n "${undercloud_nameserver:-}" ]; then + NAMESERVER_ARG="--dns-nameserver $undercloud_nameserver" + fi + + openstack subnet create --network=$NETWORK_ID \ + --gateway=$undercloud_network_gateway \ + --subnet-range=$undercloud_network_cidr \ + --allocation-pool start=$undercloud_dhcp_start,end=$undercloud_dhcp_end \ + --host-route destination=169.254.169.254/32,gateway=$local_ip \ + $NAMESERVER_ARG ctlplane +fi + +# Disable nova quotas +openstack quota set --cores -1 --instances -1 --ram -1 $(openstack project show admin | awk '$2=="id" {print $4}') + +# MISTRAL WORKFLOW CONFIGURATION +if [ "$(hiera mistral_api_enabled)" = "true" ]; then + # load workflows + for workbook in $(openstack workbook list | grep tripleo | cut -f 2 -d ' '); do + openstack workbook delete $workbook + done + for workflow in $(openstack workflow list | grep tripleo | cut -f 2 -d ' '); do + openstack workflow delete $workflow + done + for workbook in $(ls /usr/share/openstack-tripleo-common/workbooks/*); do + openstack workbook create $workbook + done + + # Store the SNMP password in a mistral environment + if ! openstack workflow env show tripleo.undercloud-config &>/dev/null; then + TMP_MISTRAL_ENV=$(mktemp) + echo "{\"name\": \"tripleo.undercloud-config\", \"variables\": {\"undercloud_ceilometer_snmpd_password\": \"$snmp_readonly_user_password\"}}" > $TMP_MISTRAL_ENV + openstack workflow env create $TMP_MISTRAL_ENV + fi + +fi + +# IP forwarding is needed to allow the overcloud nodes access to the outside +# internet in cases where they are on an isolated network. +sysctl -w net.ipv4.ip_forward=1 +# Make it persistent +echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/ip-forward.conf diff --git a/extraconfig/post_deploy/undercloud_post.yaml b/extraconfig/post_deploy/undercloud_post.yaml new file mode 100644 index 00000000..38a9181e --- /dev/null +++ b/extraconfig/post_deploy/undercloud_post.yaml @@ -0,0 +1,93 @@ +heat_template_version: ocata + +description: > + Post-deployment for the TripleO undercloud + +parameters: + servers: + type: json + DeployedServerPortMap: + default: {} + type: json + UndercloudDhcpRangeStart: + type: string + default: '192.168.24.5' + UndercloudDhcpRangeEnd: + type: string + default: '192.168.24.24' + UndercloudNetworkCidr: + type: string + default: '192.168.24.0/24' + UndercloudNetworkGateway: + type: string + default: '192.168.24.1' + UndercloudNameserver: + type: string + default: '' + AdminPassword: #supplied by tripleo-undercloud-passwords.yaml + type: string + description: The password for the keystone admin account, used for monitoring, querying neutron etc. + hidden: True + SSLCertificate: + description: > + The content of the SSL certificate (without Key) in PEM format. + type: string + default: "" + hidden: True + SnmpdReadonlyUserPassword: + description: The user password for SNMPd with readonly rights running on all Overcloud nodes + type: string + hidden: true + +conditions: + + ssl_disabled: {equals : [{get_param: SSLCertificate}, ""]} + +resources: + + UndercloudPostConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + inputs: + - name: deploy_identifier + - name: local_ip + - name: undercloud_dhcp_start + - name: undercloud_dhcp_end + - name: undercloud_network_cidr + - name: undercloud_network_gateway + - name: undercloud_nameserver + - name: admin_password + - name: auth_url + - name: snmp_readonly_user_password + config: {get_file: ./undercloud_post.sh} + + UndercloudPostDeployment: + type: OS::Heat::SoftwareDeployments + properties: + servers: {get_param: servers} + config: {get_resource: UndercloudPostConfig} + input_values: + local_ip: {get_param: [DeployedServerPortMap, 'control_virtual_ip', fixed_ips, 0, ip_address]} + undercloud_dhcp_start: {get_param: UndercloudDhcpRangeStart} + undercloud_dhcp_end: {get_param: UndercloudDhcpRangeEnd} + undercloud_network_cidr: {get_param: UndercloudNetworkCidr} + undercloud_network_gateway: {get_param: UndercloudNetworkGateway} + undercloud_nameserver: {get_param: UndercloudNameserver} + ssl_certificate: {get_param: SSLCertificate} + admin_password: {get_param: AdminPassword} + snmp_readonly_user_password: {get_param: SnmpdReadonlyUserPassword} + # if SSL is enabled we use the public virtual ip as the stackrc endpoint + auth_url: + if: + - ssl_disabled + - list_join: + - '' + - - 'http://' + - {get_param: [DeployedServerPortMap, 'control_virtual_ip', fixed_ips, 0, ip_address]} + - ':5000/v2.0' + - list_join: + - '' + - - 'https://' + - {get_param: [DeployedServerPortMap, 'public_virtual_ip', fixed_ips, 0, ip_address]} + - ':13000/v2.0' diff --git a/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml b/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml index 7c65bd8b..fdf2e957 100644 --- a/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml +++ b/extraconfig/pre_deploy/rhel-registration/rhel-registration.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: > RHEL Registration and unregistration software deployments. diff --git a/extraconfig/pre_deploy/rhel-registration/scripts/rhel-registration b/extraconfig/pre_deploy/rhel-registration/scripts/rhel-registration index 1c9acd2b..2650a967 100644 --- a/extraconfig/pre_deploy/rhel-registration/scripts/rhel-registration +++ b/extraconfig/pre_deploy/rhel-registration/scripts/rhel-registration @@ -11,6 +11,7 @@ if [ -e $OK ] ; then exit 0 fi +retryCount=0 opts= attach_opts= sat5_opts= @@ -96,12 +97,28 @@ if [ -n "${REG_TYPE:-}" ]; then opts="$opts --type=$REG_TYPE" fi +function retry() { + if [[ $retryCount < 3 ]]; then + $@ + if ! [[ $? == 0 ]]; then + retryCount=$(echo $retryCount + 1 | bc) + echo "WARN: Failed to connect when running '$@', retrying..." + retry $@ + else + retryCount=0 + fi + else + echo "ERROR: Failed to connect after 3 attempts when running '$@'" + exit 1 + fi +} + function detect_satellite_version { ping_api=$REG_SAT_URL/katello/api/ping - if curl -L -k -s -D - -o /dev/null $ping_api | grep "200 OK"; then + if curl --retry 3 --retry-delay 10 --max-time 30 -L -k -s -D - -o /dev/null $ping_api | grep "200 OK"; then echo Satellite 6 detected at $REG_SAT_URL satellite_version=6 - elif curl -L -k -s -D - -o /dev/null $REG_SAT_URL/rhn/Login.do | grep "200 OK"; then + elif curl --retry 3 --retry-delay 10 --max-time 30 -L -k -s -D - -o /dev/null $REG_SAT_URL/rhn/Login.do | grep "200 OK"; then echo Satellite 5 detected at $REG_SAT_URL satellite_version=5 else @@ -112,28 +129,29 @@ function detect_satellite_version { case "${REG_METHOD:-}" in portal) - subscription-manager register $opts + retry subscription-manager register $opts if [ -z "${REG_AUTO_ATTACH:-}" -a -z "${REG_ACTIVATION_KEY:-}" ]; then - subscription-manager attach $attach_opts + retry subscription-manager attach $attach_opts fi - subscription-manager $repos + retry subscription-manager repos --disable '*' + retry subscription-manager $repos ;; satellite) detect_satellite_version if [ "$satellite_version" = "6" ]; then repos="$repos --enable ${satellite_repo}" - curl -L -k -O "$REG_SAT_URL/pub/katello-ca-consumer-latest.noarch.rpm" + curl --retry 3 --retry-delay 10 --max-time 30 -L -k -O "$REG_SAT_URL/pub/katello-ca-consumer-latest.noarch.rpm" rpm -Uvh katello-ca-consumer-latest.noarch.rpm || true - subscription-manager register $opts - subscription-manager $repos - yum install -y katello-agent || true # needed for errata reporting to satellite6 + retry subscription-manager register $opts + retry subscription-manager $repos + retry yum install -y katello-agent || true # needed for errata reporting to satellite6 katello-package-upload - subscription-manager repos --disable ${satellite_repo} + retry subscription-manager repos --disable ${satellite_repo} else pushd /usr/share/rhn/ - curl -k -O $REG_SAT_URL/pub/RHN-ORG-TRUSTED-SSL-CERT + curl --retry 3 --retry-delay 10 --max-time 30 -k -O $REG_SAT_URL/pub/RHN-ORG-TRUSTED-SSL-CERT popd - rhnreg_ks --serverUrl=$REG_SAT_URL/XMLRPC $sat5_opts + retry rhnreg_ks --serverUrl=$REG_SAT_URL/XMLRPC $sat5_opts fi ;; disable) diff --git a/extraconfig/pre_network/ansible_host_config.ansible b/extraconfig/pre_network/ansible_host_config.ansible new file mode 100644 index 00000000..c126c1a1 --- /dev/null +++ b/extraconfig/pre_network/ansible_host_config.ansible @@ -0,0 +1,58 @@ +--- +- name: Configuration to be applied before rebooting the node + connection: local + hosts: localhost + + tasks: + # Kernel Args Configuration + - block: + - name: Ensure the kernel args ( {{ _KERNEL_ARGS_ }} ) is present as TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS + lineinfile: + dest: /etc/default/grub + regexp: '^TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS.*' + insertafter: '^GRUB_CMDLINE_LINUX.*' + line: 'TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS=" {{ _KERNEL_ARGS_ }} "' + - name: Add TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS to the GRUB_CMDLINE_LINUX parameter + lineinfile: + dest: /etc/default/grub + line: 'GRUB_CMDLINE_LINUX="${GRUB_CMDLINE_LINUX:+$GRUB_CMDLINE_LINUX }${TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS}"' + insertafter: '^TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS.*' + - name: Generate grub config file + command: grub2-mkconfig -o /boot/grub2/grub.cfg + become: true + when: _KERNEL_ARGS_|default("") != "" + + # Tune-d Configuration + - block: + - name: Tune-d Configuration + lineinfile: + dest: /etc/tuned/cpu-partitioning-variables.conf + regexp: '^isolated_cores=.*' + line: 'isolated_cores={{ _HOST_CPUS_LIST_ }}' + when: _HOST_CPUS_LIST_|default("") != "" + + - name: Tune-d provile activation + shell: tuned-adm profile {{ _TUNED_PROFILE_NAME_ }} + become: true + when: _TUNED_PROFILE_NAME_|default("") != "" + + # Provisioning Network workaround + # The script will be executed before os-net-config, in which case, only Provisioning network will have IP + # BOOTPROTO of all interface config files (except provisioning), will be set to "none" to avoid reboot failing to acquire IP on other networks + - block: + - find: + paths: /etc/sysconfig/network-scripts/ + patterns: ifcfg-* + register: ifcfg_files + + - replace: + dest: "{{ item.path }}" + regexp: '^BOOTPROTO=.*' + replace: 'BOOTPROTO=none' + when: + - item.path | regex_replace('(^.*ifcfg-)(.*)', '\\2') != "lo" + # This condition will list all the interfaces except the one with valid IP (which is Provisioning network at this stage) + # Simpler Version - hostvars[inventory_hostname]['ansible_' + iface_name ]['ipv4']['address'] is undefined + - hostvars[inventory_hostname]['ansible_' + item.path | regex_replace('(^.*ifcfg-)(.*)', '\\2') ]['ipv4']['address'] is undefined + with_items: + - "{{ ifcfg_files.files }}" diff --git a/extraconfig/pre_network/config_then_reboot.yaml b/extraconfig/pre_network/config_then_reboot.yaml new file mode 100644 index 00000000..bb0b9511 --- /dev/null +++ b/extraconfig/pre_network/config_then_reboot.yaml @@ -0,0 +1,48 @@ +heat_template_version: ocata + +description: > + Do some configuration, then reboot - sometimes needed for early-boot + changes such as modifying kernel configuration + +parameters: + server: + type: string + +resources: + + SomeConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: | + #!/bin/bash + echo "did some config before reboot" > /root/pre-reboot-config + + SomeDeployment: + type: OS::Heat::SoftwareDeployment + properties: + name: SomeDeployment + server: {get_param: server} + config: {get_resource: SomeConfig} + actions: ['CREATE'] # Only do this on CREATE + + RebootConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: | + #!/bin/bash + # Stop os-collect-config to avoid any race collecting another + # deployment before reboot happens + systemctl stop os-collect-config.service + /sbin/reboot + + RebootDeployment: + type: OS::Heat::SoftwareDeployment + depends_on: SomeDeployment + properties: + name: RebootDeployment + server: {get_param: server} + config: {get_resource: RebootConfig} + actions: ['CREATE'] # Only do this on CREATE + signal_transport: NO_SIGNAL diff --git a/extraconfig/pre_network/host_config_and_reboot.role.j2.yaml b/extraconfig/pre_network/host_config_and_reboot.role.j2.yaml new file mode 100644 index 00000000..4ad53cb8 --- /dev/null +++ b/extraconfig/pre_network/host_config_and_reboot.role.j2.yaml @@ -0,0 +1,100 @@ +heat_template_version: ocata + +description: > + Do some configuration, then reboot - sometimes needed for early-boot + changes such as modifying kernel configuration + +parameters: + server: + type: string + {{role}}KernelArgs: + type: string + default: "" + {{role}}TunedProfileName: + type: string + default: "" + {{role}}HostCpusList: + type: string + default: "" + +conditions: + param_exists: + or: + - not: + equals: + - get_param: {{role}}KernelArgs + - "" + - not: + equals: + - get_param: {{role}}TunedProfileName + - "" + +resources: + + HostParametersConfig: + type: OS::Heat::SoftwareConfig + condition: param_exists + properties: + group: ansible + inputs: + - name: _KERNEL_ARGS_ + - name: _TUNED_PROFILE_NAME_ + - name: _HOST_CPUS_LIST_ + outputs: + - name: result + config: + get_file: ansible_host_config.ansible + + HostParametersDeployment: + type: OS::Heat::SoftwareDeployment + condition: param_exists + properties: + name: HostParametersDeployment + server: {get_param: server} + config: {get_resource: HostParametersConfig} + actions: ['CREATE'] # Only do this on CREATE + input_values: + _KERNEL_ARGS_: {get_param: {{role}}KernelArgs} + _TUNED_PROFILE_NAME_: {get_param: {{role}}TunedProfileName} + _HOST_CPUS_LIST_: {get_param: {{role}}HostCpusList} + + RebootConfig: + type: OS::Heat::SoftwareConfig + condition: param_exists + properties: + group: script + config: | + #!/bin/bash + # Stop os-collect-config to avoid any race collecting another + # deployment before reboot happens + systemctl stop os-collect-config.service + /sbin/reboot + + RebootDeployment: + type: OS::Heat::SoftwareDeployment + condition: param_exists + depends_on: HostParametersDeployment + properties: + name: RebootDeployment + server: {get_param: server} + config: {get_resource: RebootConfig} + actions: ['CREATE'] # Only do this on CREATE + signal_transport: NO_SIGNAL + +outputs: + result: + value: + get_attr: [HostParametersDeployment, result] + condition: param_exists + stdout: + value: + get_attr: [HostParametersDeployment, deploy_stdout] + condition: param_exists + stderr: + value: + get_attr: [HostParametersDeployment, deploy_stderr] + condition: param_exists + status_code: + value: + get_attr: [HostParametersDeployment, deploy_status_code] + condition: param_exists diff --git a/extraconfig/tasks/major_upgrade_block_storage.sh b/extraconfig/tasks/major_upgrade_block_storage.sh index 07666245..64c4457e 100644 --- a/extraconfig/tasks/major_upgrade_block_storage.sh +++ b/extraconfig/tasks/major_upgrade_block_storage.sh @@ -4,5 +4,8 @@ # set -eu +# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205 +special_case_ovs_upgrade_if_needed + yum -y install python-zaqarclient # needed for os-collect-config yum -y -q update diff --git a/extraconfig/tasks/major_upgrade_ceilometer_wsgi_mitaka_newton.yaml b/extraconfig/tasks/major_upgrade_ceilometer_wsgi_mitaka_newton.yaml new file mode 100644 index 00000000..cf5d7a84 --- /dev/null +++ b/extraconfig/tasks/major_upgrade_ceilometer_wsgi_mitaka_newton.yaml @@ -0,0 +1,62 @@ +heat_template_version: ocata + +description: > + Software-config for ceilometer configuration under httpd during upgrades + +parameters: + servers: + type: json + input_values: + type: json + description: input values for the software deployments +resources: + CeilometerWsgiMitakaNewtonPreUpgradeConfig: + type: OS::Heat::SoftwareConfig + properties: + group: puppet + config: + get_file: mitaka_to_newton_ceilometer_wsgi_upgrade.pp + + CeilometerWsgiMitakaNewtonUpgradeConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: + list_join: + - '' + - - "#!/bin/bash\n\nset -e\n\n" + - get_file: pacemaker_common_functions.sh + - get_file: major_upgrade_pacemaker_migrations.sh + - "disable_standalone_ceilometer_api\n\n" + + CeilometerWsgiMitakaNewtonPostUpgradeConfig: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: | + #!/bin/bash + set -e + /usr/bin/systemctl reload httpd + + CeilometerWsgiMitakaNewtonPreUpgradeDeployment: + type: OS::Heat::SoftwareDeploymentGroup + properties: + name: CeilometerWsgiMitakaNewtonPreUpgradeDeployment + servers: {get_param: [servers, Controller]} + config: {get_resource: CeilometerWsgiMitakaNewtonPreUpgradeConfig} + + CeilometerWsgiMitakaNewtonUpgradeConfigDeployment: + type: OS::Heat::SoftwareDeploymentGroup + depends_on: CeilometerWsgiMitakaNewtonPreUpgradeDeployment + properties: + name: CeilometerWsgiMitakaNewtonUpgradeConfigDeployment + servers: {get_param: [servers, Controller]} + config: {get_resource: CeilometerWsgiMitakaNewtonUpgradeConfig} + + CeilometerWsgiMitakaNewtonPostUpgradeDeployment: + type: OS::Heat::SoftwareDeploymentGroup + depends_on: CeilometerWsgiMitakaNewtonUpgradeConfigDeployment + properties: + name: CeilometerWsgiMitakaNewtonPostUpgradeDeployment + servers: {get_param: [servers, Controller]} + config: {get_resource: CeilometerWsgiMitakaNewtonPostUpgradeConfig} diff --git a/extraconfig/tasks/major_upgrade_ceph_mon.sh b/extraconfig/tasks/major_upgrade_ceph_mon.sh index 21a2b5bc..e0d160f1 100755 --- a/extraconfig/tasks/major_upgrade_ceph_mon.sh +++ b/extraconfig/tasks/major_upgrade_ceph_mon.sh @@ -5,7 +5,7 @@ set -o pipefail echo INFO: starting $(basename "$0") # Exit if not running -if ! pidof ceph-mon; then +if ! pidof ceph-mon &> /dev/null; then echo INFO: ceph-mon is not running, skipping exit 0 fi @@ -24,7 +24,7 @@ if [ ${CEPH_STATUS} = HEALTH_ERR ]; then fi # Useful when upgrading with OSDs num < replica size -if [ ${ignore_ceph_upgrade_warnings:-false} != "true" ]; then +if [[ ${ignore_ceph_upgrade_warnings:-False} != [Tt]rue ]]; then timeout 300 bash -c "while [ ${CEPH_STATUS} != HEALTH_OK ]; do echo WARNING: Waiting for Ceph cluster status to go HEALTH_OK; sleep 30; @@ -54,7 +54,7 @@ if [[ "$UPDATED_VERSION" =~ ^0\.94.* ]]; then elif [[ "$UPDATED_VERSION" =~ ^10\.2.* ]]; then # RPM could own some of these but we can't take risks on the pre-existing files for d in /var/lib/ceph/mon /var/log/ceph /var/run/ceph /etc/ceph; do - chown -R ceph:ceph $d || echo WARNING: chown of $d failed + chown -L -R ceph:ceph $d || echo WARNING: chown of $d failed done # Replay udev events with newer rules @@ -71,6 +71,10 @@ elif [[ "$UPDATED_VERSION" =~ ^10\.2.* ]]; then sleep 10; done" + # if tunables become legacy, cluster status will be HEALTH_WARN causing + # upgrade to fail on following node + ceph osd crush tunables default + echo INFO: Ceph was upgraded to Jewel else echo ERROR: Ceph was upgraded to an unknown release, daemon is stopped, need manual intervention diff --git a/extraconfig/tasks/major_upgrade_ceph_storage.sh b/extraconfig/tasks/major_upgrade_ceph_storage.sh index dc80a724..a745e723 100644 --- a/extraconfig/tasks/major_upgrade_ceph_storage.sh +++ b/extraconfig/tasks/major_upgrade_ceph_storage.sh @@ -8,7 +8,9 @@ set -o pipefail UPGRADE_SCRIPT=/root/tripleo_upgrade_node.sh -cat > $UPGRADE_SCRIPT << 'ENDOFCAT' +declare -f special_case_ovs_upgrade_if_needed > $UPGRADE_SCRIPT +# use >> here so we don't lose the declaration we added above +cat >> $UPGRADE_SCRIPT << 'ENDOFCAT' #!/bin/bash ### DO NOT MODIFY THIS FILE ### This file is automatically delivered to the ceph-storage nodes as part of the @@ -18,7 +20,7 @@ set -eu echo INFO: starting $(basename "$0") # Exit if not running -if ! pidof ceph-osd; then +if ! pidof ceph-osd &> /dev/null; then echo INFO: ceph-osd is not running, skipping exit 0 fi @@ -49,6 +51,8 @@ timeout 60 bash -c "while kill -0 ${OSD_PIDS} 2> /dev/null; do sleep 2; done" +special_case_ovs_upgrade_if_needed + # Update (Ceph to Jewel) yum -y install python-zaqarclient # needed for os-collect-config yum -y update @@ -63,12 +67,22 @@ if [[ "$UPDATED_VERSION" =~ ^0\.94.* ]]; then elif [[ "$UPDATED_VERSION" =~ ^10\.2.* ]]; then # RPM could own some of these but we can't take risks on the pre-existing files for d in /var/lib/ceph/osd /var/log/ceph /var/run/ceph /etc/ceph; do - chown -R ceph:ceph $d || echo WARNING: chown of $d failed + chown -L -R ceph:ceph $d || echo WARNING: chown of $d failed done # Replay udev events with newer rules udevadm trigger && udevadm settle + # If on ext4, we need to enforce lower values for name and namespace len + # or ceph-osd will refuse to start, see: http://tracker.ceph.com/issues/16187 + for OSD_ID in $OSD_IDS; do + OSD_FS=$(df -l --output=fstype /var/lib/ceph/osd/ceph-${OSD_ID} | tail -n +2) + if [ ${OSD_FS} = ext4 ]; then + crudini --set /etc/ceph/ceph.conf global osd_max_object_name_len 256 + crudini --set /etc/ceph/ceph.conf global osd_max_object_namespace_len 64 + fi + done + # Enable systemd unit systemctl enable ceph-osd.target for OSD_ID in $OSD_IDS; do diff --git a/extraconfig/tasks/major_upgrade_check.sh b/extraconfig/tasks/major_upgrade_check.sh index dc7ec71a..8bdff5e7 100755 --- a/extraconfig/tasks/major_upgrade_check.sh +++ b/extraconfig/tasks/major_upgrade_check.sh @@ -18,14 +18,8 @@ check_pcsd() fi } -check_disk_for_mysql_dump() +mysql_need_update() { - # Where to backup current database if mysql need to be upgraded - MYSQL_BACKUP_DIR=/var/tmp/mysql_upgrade_osp - MYSQL_TEMP_UPGRADE_BACKUP_DIR=/var/lib/mysql-temp-upgrade-backup - # Spare disk ratio for extra safety - MYSQL_BACKUP_SIZE_RATIO=1.2 - # Shall we upgrade mysql data directory during the stack upgrade? if [ "$mariadb_do_major_upgrade" = "auto" ]; then ret=$(is_mysql_upgrade_needed) @@ -40,6 +34,17 @@ check_disk_for_mysql_dump() else DO_MYSQL_UPGRADE=1 fi +} + +check_disk_for_mysql_dump() +{ + # Where to backup current database if mysql need to be upgraded + MYSQL_BACKUP_DIR=/var/tmp/mysql_upgrade_osp + MYSQL_TEMP_UPGRADE_BACKUP_DIR=/var/lib/mysql-temp-upgrade-backup + # Spare disk ratio for extra safety + MYSQL_BACKUP_SIZE_RATIO=1.2 + + mysql_need_update if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then if [ $DO_MYSQL_UPGRADE -eq 1 ]; then @@ -88,8 +93,8 @@ check_python_rpm() check_clean_cluster() { - if crm_mon -1 | grep -A3 Failed; then - echo_error "ERROR: upgrade cannot start with failed resources on the cluster. Clean them up before starting: pcs resource cleanup." + if pcs status | grep -q Stopped:; then + echo_error "ERROR: upgrade cannot start with stopped resources on the cluster. Make sure that all the resources are up and running." exit 1 fi } diff --git a/extraconfig/tasks/major_upgrade_compute.sh b/extraconfig/tasks/major_upgrade_compute.sh index a1df695f..7a3e1073 100644 --- a/extraconfig/tasks/major_upgrade_compute.sh +++ b/extraconfig/tasks/major_upgrade_compute.sh @@ -18,9 +18,16 @@ set -eu crudini --set /etc/nova/nova.conf upgrade_levels compute $upgrade_level_nova_compute +$(declare -f special_case_ovs_upgrade_if_needed) +special_case_ovs_upgrade_if_needed + yum -y install python-zaqarclient # needed for os-collect-config yum -y update +# Due to bug#1640177 we need to restart compute agent +echo "Restarting openstack ceilometer agent compute" +systemctl restart openstack-ceilometer-compute + ENDOFCAT # ensure the permissions are OK diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh index 2490ce27..080831ab 100755 --- a/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh +++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_1.sh @@ -2,11 +2,11 @@ set -eu -cluster_sync_timeout=1800 - check_cluster check_pcsd -check_clean_cluster +if [[ -n $(is_bootstrap_node) ]]; then + check_clean_cluster +fi check_python_rpm check_galera_root_password check_disk_for_mysql_dump @@ -15,141 +15,22 @@ check_disk_for_mysql_dump # nodes where a service fails to stop, which could be fatal during an upgrade # procedure. So we remember the stonith state. If it was enabled we reenable it # at the end of this script -STONITH_STATE=$(pcs property show stonith-enabled | grep "stonith-enabled" | awk '{ print $2 }') -pcs property set stonith-enabled=false - -# Migrate to HA NG -if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then - migrate_full_to_ng_ha -fi - -# After migrating the cluster to HA-NG the services not under pacemaker's control -# are still up and running. We need to stop them explicitely otherwise during the yum -# upgrade the rpm %post sections will try to do a systemctl try-restart <service>, which -# is going to take a long time because rabbit is down. By having the service stopped -# systemctl try-restart is a noop - -for $service in $(services_to_migrate); do - manage_systemd_service stop "${service%%-clone}" - check_resource_systemd "${service%%-clone}" stopped 600 -done - -# In case the mysql package is updated, the database on disk must be -# upgraded as well. This typically needs to happen during major -# version upgrades (e.g. 5.5 -> 5.6, 5.5 -> 10.1...) -# -# Because in-place upgrades are not supported across 2+ major versions -# (e.g. 5.5 -> 10.1), we rely on logical upgrades via dump/restore cycle -# https://bugzilla.redhat.com/show_bug.cgi?id=1341968 -# -# The default is to determine automatically if upgrade is needed based -# on mysql package versionning, but this can be overriden manually -# to support specific upgrade scenario - -if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then - if [ $DO_MYSQL_UPGRADE -eq 1 ]; then - mysqldump $backup_flags > "$MYSQL_BACKUP_DIR/openstack_database.sql" - cp -rdp /etc/my.cnf* "$MYSQL_BACKUP_DIR" +if [[ -n $(is_bootstrap_node) ]]; then + STONITH_STATE=$(pcs property show stonith-enabled | grep "stonith-enabled" | awk '{ print $2 }') + # We create this empty file if stonith was set to true so we can reenable stonith in step2 + rm -f /var/tmp/stonith-true + if [ $STONITH_STATE == "true" ]; then + touch /var/tmp/stonith-true fi - - pcs resource disable redis - check_resource redis stopped 600 - pcs resource disable rabbitmq - check_resource rabbitmq stopped 600 - pcs resource disable galera - check_resource galera stopped 600 - # Disable all VIPs before stopping the cluster, so that pcs doesn't use one as a source address: - # https://bugzilla.redhat.com/show_bug.cgi?id=1330688 - for vip in $(pcs resource show | grep ocf::heartbeat:IPaddr2 | grep Started | awk '{ print $1 }'); do - pcs resource disable $vip - check_resource $vip stopped 60 - done - pcs cluster stop --all + pcs property set stonith-enabled=false fi - -# Swift isn't controled by pacemaker -systemctl_swift stop - -tstart=$(date +%s) -while systemctl is-active pacemaker; do - sleep 5 - tnow=$(date +%s) - if (( tnow-tstart > cluster_sync_timeout )) ; then - echo_error "ERROR: cluster shutdown timed out" - exit 1 - fi -done - -# The reason we do an sql dump *and* we move the old dir out of -# the way is because it gives us an extra level of safety in case -# something goes wrong during the upgrade. Once the restore is -# successful we go ahead and remove it. If the directory exists -# we bail out as it means the upgrade process had issues in the last -# run. -if [ $DO_MYSQL_UPGRADE -eq 1 ]; then - if [ -d $MYSQL_TEMP_UPGRADE_BACKUP_DIR ]; then - echo_error "ERROR: mysql backup dir already exist" - exit 1 - fi - mv /var/lib/mysql $MYSQL_TEMP_UPGRADE_BACKUP_DIR -fi - -yum -y install python-zaqarclient # needed for os-collect-config -yum -y -q update - -# We need to ensure at least those two configuration settings, otherwise -# mariadb 10.1+ won't activate galera replication. -# wsrep_cluster_address must only be set though, its value does not -# matter because it's overriden by the galera resource agent. -cat >> /etc/my.cnf.d/galera.cnf <<EOF -[mysqld] -wsrep_on = ON -wsrep_cluster_address = gcomm://localhost -EOF - -if [ $DO_MYSQL_UPGRADE -eq 1 ]; then - # Scripts run via heat have no HOME variable set and this confuses - # mysqladmin - export HOME=/root - - mkdir /var/lib/mysql || /bin/true - chown mysql:mysql /var/lib/mysql - chmod 0755 /var/lib/mysql - restorecon -R /var/lib/mysql/ - mysql_install_db --datadir=/var/lib/mysql --user=mysql - chown -R mysql:mysql /var/lib/mysql/ - - if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then - mysqld_safe --wsrep-new-cluster & - # We have a populated /root/.my.cnf with root/password here so - # we need to temporarily rename it because the newly created - # db is empty and no root password is set - mv /root/.my.cnf /root/.my.cnf.temporary - timeout 60 sh -c 'while ! mysql -e "" &> /dev/null; do sleep 1; done' - mysql -u root < "$MYSQL_BACKUP_DIR/openstack_database.sql" - mv /root/.my.cnf.temporary /root/.my.cnf - mysqladmin -u root shutdown - # The import was successful so we may remove the folder - rm -r "$MYSQL_BACKUP_DIR" - fi -fi - -# If we reached here without error we can safely blow away the origin -# mysql dir from every controller - -# TODO: What if the upgrade fails on the bootstrap node, but not on -# this controller. Data may be lost. -if [ $DO_MYSQL_UPGRADE -eq 1 ]; then - rm -r $MYSQL_TEMP_UPGRADE_BACKUP_DIR -fi - -# Let's reset the stonith back to true if it was true, before starting the cluster -if [ $STONITH_STATE == "true" ]; then - pcs -f /var/lib/pacemaker/cib/cib.xml property set stonith-enabled=true +# Migrate to HA NG and fix up rabbitmq queues +# We fix up the rabbitmq ha queues after the migration because it will +# restart the rabbitmq resource. Doing it after the migration means no other +# services will be restart as there are no other constraints +if [[ -n $(is_bootstrap_node) ]]; then + migrate_full_to_ng_ha + rabbitmq_newton_ocata_upgrade fi -# Pin messages sent to compute nodes to kilo, these will be upgraded later -crudini --set /etc/nova/nova.conf upgrade_levels compute "$upgrade_level_nova_compute" - -crudini --set /etc/sahara/sahara.conf DEFAULT plugins ambari,cdh,mapr,vanilla,spark,storm diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh index 6bb2fa73..6bfe1239 100755 --- a/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh +++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_2.sh @@ -2,73 +2,175 @@ set -eu -cluster_form_timeout=600 -cluster_settle_timeout=1800 -galera_sync_timeout=600 +cluster_sync_timeout=1800 -if [[ -n $(is_bootstrap_node) ]]; then - pcs cluster start --all +# After migrating the cluster to HA-NG the services not under pacemaker's control +# are still up and running. We need to stop them explicitely otherwise during the yum +# upgrade the rpm %post sections will try to do a systemctl try-restart <service>, which +# is going to take a long time because rabbit is down. By having the service stopped +# systemctl try-restart is a noop +for service in $(services_to_migrate); do + manage_systemd_service stop "${service%%-clone}" + # So the reason for not reusing check_resource_systemd is that + # I have observed systemctl is-active returning unknown with at least + # one service that was stopped (See LP 1627254) + timeout=600 tstart=$(date +%s) - while pcs status 2>&1 | grep -E '(cluster is not currently running)|(OFFLINE:)'; do - sleep 5 - tnow=$(date +%s) - if (( tnow-tstart > cluster_form_timeout )) ; then - echo_error "ERROR: timed out forming the cluster" - exit 1 - fi + tend=$(( $tstart + $timeout )) + check_interval=3 + while (( $(date +%s) < $tend )); do + if [[ "$(systemctl is-active ${service%%-clone})" = "active" ]]; then + echo "$service still active, sleeping $check_interval seconds." + sleep $check_interval + else + # we do not care if it is inactive, unknown or failed as long as it is + # not running + break + fi + done +done - if ! timeout -k 10 $cluster_settle_timeout crm_resource --wait; then - echo_error "ERROR: timed out waiting for cluster to finish transition" - exit 1 +# In case the mysql package is updated, the database on disk must be +# upgraded as well. This typically needs to happen during major +# version upgrades (e.g. 5.5 -> 5.6, 5.5 -> 10.1...) +# +# Because in-place upgrades are not supported across 2+ major versions +# (e.g. 5.5 -> 10.1), we rely on logical upgrades via dump/restore cycle +# https://bugzilla.redhat.com/show_bug.cgi?id=1341968 +# +# The default is to determine automatically if upgrade is needed based +# on mysql package versionning, but this can be overriden manually +# to support specific upgrade scenario + +# Calling this function will set the DO_MYSQL_UPGRADE variable which is used +# later +mysql_need_update + +if [[ -n $(is_bootstrap_node) ]]; then + if [ $DO_MYSQL_UPGRADE -eq 1 ]; then + mysqldump $backup_flags > "$MYSQL_BACKUP_DIR/openstack_database.sql" + cp -rdp /etc/my.cnf* "$MYSQL_BACKUP_DIR" fi - for vip in $(pcs resource show | grep ocf::heartbeat:IPaddr2 | grep Stopped | awk '{ print $1 }'); do - pcs resource enable $vip - check_resource_pacemaker $vip started 60 + pcs resource disable redis + check_resource redis stopped 600 + pcs resource disable rabbitmq + check_resource rabbitmq stopped 600 + pcs resource disable galera + check_resource galera stopped 600 + pcs resource disable openstack-cinder-volume + check_resource openstack-cinder-volume stopped 600 + # Disable all VIPs before stopping the cluster, so that pcs doesn't use one as a source address: + # https://bugzilla.redhat.com/show_bug.cgi?id=1330688 + for vip in $(pcs resource show | grep ocf::heartbeat:IPaddr2 | grep Started | awk '{ print $1 }'); do + pcs resource disable $vip + check_resource $vip stopped 60 done + pcs cluster stop --all fi -start_or_enable_service galera -check_resource galera started 600 -if [[ -n $(is_bootstrap_node) ]]; then - tstart=$(date +%s) - while ! clustercheck; do - sleep 5 - tnow=$(date +%s) - if (( tnow-tstart > galera_sync_timeout )) ; then - echo_error "ERROR galera sync timed out" - exit 1 - fi - done +# Swift isn't controlled by pacemaker +systemctl_swift stop + +tstart=$(date +%s) +while systemctl is-active pacemaker; do + sleep 5 + tnow=$(date +%s) + if (( tnow-tstart > cluster_sync_timeout )) ; then + echo_error "ERROR: cluster shutdown timed out" + exit 1 + fi +done + +# The reason we do an sql dump *and* we move the old dir out of +# the way is because it gives us an extra level of safety in case +# something goes wrong during the upgrade. Once the restore is +# successful we go ahead and remove it. If the directory exists +# we bail out as it means the upgrade process had issues in the last +# run. +if [ $DO_MYSQL_UPGRADE -eq 1 ]; then + if [ -d $MYSQL_TEMP_UPGRADE_BACKUP_DIR ]; then + echo_error "ERROR: mysql backup dir already exist" + exit 1 + fi + mv /var/lib/mysql $MYSQL_TEMP_UPGRADE_BACKUP_DIR +fi + +# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205 +special_case_ovs_upgrade_if_needed + +yum -y install python-zaqarclient # needed for os-collect-config +yum -y -q update + +# We need to ensure at least those two configuration settings, otherwise +# mariadb 10.1+ won't activate galera replication. +# wsrep_cluster_address must only be set though, its value does not +# matter because it's overriden by the galera resource agent. +cat >> /etc/my.cnf.d/galera.cnf <<EOF +[mysqld] +wsrep_on = ON +wsrep_cluster_address = gcomm://localhost +EOF - # Run all the db syncs - # TODO: check if this can be triggered in puppet and removed from here - ceilometer-dbsync --config-file=/etc/ceilometer/ceilometer.conf - cinder-manage db sync - glance-manage --config-file=/etc/glance/glance-registry.conf db_sync - heat-manage --config-file /etc/heat/heat.conf db_sync - keystone-manage db_sync - neutron-db-manage --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugin.ini upgrade head - nova-manage db sync - #TODO(marios):someone from sahara needs to check this: - # sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head +if [ $DO_MYSQL_UPGRADE -eq 1 ]; then + # Scripts run via heat have no HOME variable set and this confuses + # mysqladmin + export HOME=/root + + mkdir /var/lib/mysql || /bin/true + chown mysql:mysql /var/lib/mysql + chmod 0755 /var/lib/mysql + restorecon -R /var/lib/mysql/ + mysql_install_db --datadir=/var/lib/mysql --user=mysql + chown -R mysql:mysql /var/lib/mysql/ + + if [ "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]; then + mysqld_safe --wsrep-new-cluster & + # We have a populated /root/.my.cnf with root/password here so + # we need to temporarily rename it because the newly created + # db is empty and no root password is set + mv /root/.my.cnf /root/.my.cnf.temporary + timeout 60 sh -c 'while ! mysql -e "" &> /dev/null; do sleep 1; done' + mysql -u root < "$MYSQL_BACKUP_DIR/openstack_database.sql" + mv /root/.my.cnf.temporary /root/.my.cnf + mysqladmin -u root shutdown + # The import was successful so we may remove the folder + rm -r "$MYSQL_BACKUP_DIR" + fi fi -start_or_enable_service rabbitmq -check_resource rabbitmq started 600 -start_or_enable_service redis -check_resource redis started 600 +# If we reached here without error we can safely blow away the origin +# mysql dir from every controller -# Swift isn't controled by pacemaker -systemctl_swift start +# TODO: What if the upgrade fails on the bootstrap node, but not on +# this controller. Data may be lost. +if [ $DO_MYSQL_UPGRADE -eq 1 ]; then + rm -r $MYSQL_TEMP_UPGRADE_BACKUP_DIR +fi + +# Let's reset the stonith back to true if it was true, before starting the cluster +if [[ -n $(is_bootstrap_node) ]]; then + if [ -f /var/tmp/stonith-true ]; then + pcs -f /var/lib/pacemaker/cib/cib.xml property set stonith-enabled=true + fi + rm -f /var/tmp/stonith-true +fi + +# Pin messages sent to compute nodes to kilo, these will be upgraded later +crudini --set /etc/nova/nova.conf upgrade_levels compute "$upgrade_level_nova_compute" +# https://bugzilla.redhat.com/show_bug.cgi?id=1284047 +# Change-Id: Ib3f6c12ff5471e1f017f28b16b1e6496a4a4b435 +crudini --set /etc/ceilometer/ceilometer.conf DEFAULT rpc_backend rabbit +# https://bugzilla.redhat.com/show_bug.cgi?id=1284058 +# Ifd1861e3df46fad0e44ff9b5cbd58711bbc87c97 Swift Ceilometer middleware no longer exists +crudini --set /etc/swift/proxy-server.conf pipeline:main pipeline "catch_errors healthcheck cache ratelimit tempurl formpost authtoken keystone staticweb proxy-logging proxy-server" +# LP: 1615035, required only for M/N upgrade. +crudini --set /etc/nova/nova.conf DEFAULT scheduler_host_manager host_manager +# LP: 1627450, required only for M/N upgrade +crudini --set /etc/nova/nova.conf DEFAULT scheduler_driver filter_scheduler + +crudini --set /etc/sahara/sahara.conf DEFAULT plugins ambari,cdh,mapr,vanilla,spark,storm -# We need to start the systemd services we explicitely stopped at step _1.sh -# FIXME: Should we let puppet during the convergence step do the service enabling or -# should we add it here? -for $service in $(services_to_migrate); do - manage_systemd_service stop "${service%%-clone}" - check_resource_systemd "${service%%-clone}" started 600 -done diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh new file mode 100755 index 00000000..a3cbd945 --- /dev/null +++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_3.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +set -eu + +cluster_form_timeout=600 +cluster_settle_timeout=1800 +galera_sync_timeout=600 + +if [[ -n $(is_bootstrap_node) ]]; then + pcs cluster start --all + + tstart=$(date +%s) + while pcs status 2>&1 | grep -E '(cluster is not currently running)|(OFFLINE:)'; do + sleep 5 + tnow=$(date +%s) + if (( tnow-tstart > cluster_form_timeout )) ; then + echo_error "ERROR: timed out forming the cluster" + exit 1 + fi + done + + if ! timeout -k 10 $cluster_settle_timeout crm_resource --wait; then + echo_error "ERROR: timed out waiting for cluster to finish transition" + exit 1 + fi + + for vip in $(pcs resource show | grep ocf::heartbeat:IPaddr2 | grep Stopped | awk '{ print $1 }'); do + pcs resource enable $vip + check_resource_pacemaker $vip started 60 + done +fi + +start_or_enable_service galera +check_resource galera started 600 +start_or_enable_service redis +check_resource redis started 600 +# We need mongod which is now a systemd service up and running before calling +# ceilometer-dbsync. There is still a race here: mongod might not be up on all nodes +# so ceilometer-dbsync will fail a couple of times before that. As it retries indefinitely +# we should be good. +# Due to LP Bug https://bugs.launchpad.net/tripleo/+bug/1627254 am using systemctl directly atm +systemctl start mongod +check_resource mongod started 600 + +if [[ -n $(is_bootstrap_node) ]]; then + tstart=$(date +%s) + while ! clustercheck; do + sleep 5 + tnow=$(date +%s) + if (( tnow-tstart > galera_sync_timeout )) ; then + echo_error "ERROR galera sync timed out" + exit 1 + fi + done + + # Run all the db syncs + # TODO: check if this can be triggered in puppet and removed from here + ceilometer-upgrade --config-file=/etc/ceilometer/ceilometer.conf --skip-gnocchi-resource-types + cinder-manage db sync + glance-manage db_sync + heat-manage --config-file /etc/heat/heat.conf db_sync + keystone-manage db_sync + neutron-db-manage upgrade heads + nova-manage db sync + nova-manage api_db sync + nova-manage db online_data_migrations + sahara-db-manage --config-file /etc/sahara/sahara.conf upgrade head +fi diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh new file mode 100755 index 00000000..d2cb9553 --- /dev/null +++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_4.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -eu + +start_or_enable_service rabbitmq +check_resource rabbitmq started 600 +start_or_enable_service redis +check_resource redis started 600 +start_or_enable_service openstack-cinder-volume +check_resource openstack-cinder-volume started 600 + +# start httpd so keystone is available for gnocchi +# upgrade to run. +systemctl start httpd + +# Swift isn't controled by pacemaker +systemctl_swift start diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh new file mode 100755 index 00000000..fa95f1f8 --- /dev/null +++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_5.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -eu + +if [[ -n $(is_bootstrap_node) ]]; then + # run gnocchi upgrade + gnocchi-upgrade +fi diff --git a/extraconfig/tasks/major_upgrade_controller_pacemaker_6.sh b/extraconfig/tasks/major_upgrade_controller_pacemaker_6.sh new file mode 100755 index 00000000..d569084d --- /dev/null +++ b/extraconfig/tasks/major_upgrade_controller_pacemaker_6.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -eu + +# We need to start the systemd services we explicitely stopped at step _1.sh +# FIXME: Should we let puppet during the convergence step do the service enabling or +# should we add it here? +services=$(services_to_migrate) +if [[ ${keep_sahara_services_on_upgrade} =~ [Ff]alse ]] ; then + services=${services%%openstack-sahara*} +fi +for service in $services; do + manage_systemd_service start "${service%%-clone}" + check_resource_systemd "${service%%-clone}" started 600 +done diff --git a/extraconfig/tasks/major_upgrade_object_storage.sh b/extraconfig/tasks/major_upgrade_object_storage.sh index f82457ce..d9d1b4d5 100644 --- a/extraconfig/tasks/major_upgrade_object_storage.sh +++ b/extraconfig/tasks/major_upgrade_object_storage.sh @@ -23,6 +23,8 @@ function systemctl_swift { done } +$(declare -f special_case_ovs_upgrade_if_needed) +special_case_ovs_upgrade_if_needed systemctl_swift stop diff --git a/extraconfig/tasks/major_upgrade_pacemaker.yaml b/extraconfig/tasks/major_upgrade_pacemaker.yaml index 7244f949..b63aafbd 100644 --- a/extraconfig/tasks/major_upgrade_pacemaker.yaml +++ b/extraconfig/tasks/major_upgrade_pacemaker.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2016-10-14 +heat_template_version: ocata description: 'Upgrade for Pacemaker deployments' parameters: @@ -22,6 +22,11 @@ parameters: type: boolean default: false description: If enabled, Ceph upgrade will be forced even though cluster or PGs status is not clean + KeepSaharaServicesOnUpgrade: + type: boolean + default: true + description: Whether to keep Sahara services when upgrading controller nodes from mitaka to newton + resources: # TODO(jistr): for Mitaka->Newton upgrades and further we can use @@ -46,7 +51,7 @@ resources: CephMonUpgradeDeployment: type: OS::Heat::SoftwareDeploymentGroup properties: - servers: {get_param: servers, Controller} + servers: {get_param: [servers, Controller]} config: {get_resource: CephMonUpgradeConfig} input_values: {get_param: input_values} update_policy: @@ -83,7 +88,7 @@ resources: type: OS::Heat::SoftwareDeploymentGroup depends_on: CephMonUpgradeDeployment properties: - servers: {get_param: servers, Controller} + servers: {get_param: [servers, Controller]} config: {get_resource: ControllerPacemakerUpgradeConfig_Step1} input_values: {get_param: input_values} @@ -92,12 +97,16 @@ resources: depends_on: ControllerPacemakerUpgradeDeployment_Step1 properties: group: script - config: {get_file: major_upgrade_block_storage.sh} + config: + list_join: + - '' + - - get_file: pacemaker_common_functions.sh + - get_file: major_upgrade_block_storage.sh BlockStorageUpgradeDeployment: type: OS::Heat::SoftwareDeploymentGroup properties: - servers: {get_param: servers, BlockStorage} + servers: {get_param: [servers, BlockStorage]} config: {get_resource: BlockStorageUpgradeConfig} input_values: {get_param: input_values} @@ -108,7 +117,20 @@ resources: config: list_join: - '' - - - get_file: pacemaker_common_functions.sh + - - str_replace: + template: | + #!/bin/bash + upgrade_level_nova_compute='UPGRADE_LEVEL_NOVA_COMPUTE' + params: + UPGRADE_LEVEL_NOVA_COMPUTE: {get_param: UpgradeLevelNovaCompute} + - str_replace: + template: | + #!/bin/bash + mariadb_do_major_upgrade='MYSQL_MAJOR_UPGRADE' + params: + MYSQL_MAJOR_UPGRADE: {get_param: MySqlMajorUpgrade} + - get_file: pacemaker_common_functions.sh + - get_file: major_upgrade_check.sh - get_file: major_upgrade_pacemaker_migrations.sh - get_file: major_upgrade_controller_pacemaker_2.sh @@ -116,7 +138,88 @@ resources: type: OS::Heat::SoftwareDeploymentGroup depends_on: BlockStorageUpgradeDeployment properties: - servers: {get_param: servers, Controller} + servers: {get_param: [servers, Controller]} config: {get_resource: ControllerPacemakerUpgradeConfig_Step2} input_values: {get_param: input_values} + ControllerPacemakerUpgradeConfig_Step3: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: + list_join: + - '' + - - get_file: pacemaker_common_functions.sh + - get_file: major_upgrade_pacemaker_migrations.sh + - get_file: major_upgrade_controller_pacemaker_3.sh + + ControllerPacemakerUpgradeDeployment_Step3: + type: OS::Heat::SoftwareDeploymentGroup + depends_on: ControllerPacemakerUpgradeDeployment_Step2 + properties: + servers: {get_param: [servers, Controller]} + config: {get_resource: ControllerPacemakerUpgradeConfig_Step3} + input_values: {get_param: input_values} + + ControllerPacemakerUpgradeConfig_Step4: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: + list_join: + - '' + - - get_file: pacemaker_common_functions.sh + - get_file: major_upgrade_pacemaker_migrations.sh + - get_file: major_upgrade_controller_pacemaker_4.sh + + ControllerPacemakerUpgradeDeployment_Step4: + type: OS::Heat::SoftwareDeploymentGroup + depends_on: ControllerPacemakerUpgradeDeployment_Step3 + properties: + servers: {get_param: [servers, Controller]} + config: {get_resource: ControllerPacemakerUpgradeConfig_Step4} + input_values: {get_param: input_values} + + ControllerPacemakerUpgradeConfig_Step5: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: + list_join: + - '' + - - get_file: pacemaker_common_functions.sh + - get_file: major_upgrade_pacemaker_migrations.sh + - get_file: major_upgrade_controller_pacemaker_5.sh + + ControllerPacemakerUpgradeDeployment_Step5: + type: OS::Heat::SoftwareDeploymentGroup + depends_on: ControllerPacemakerUpgradeDeployment_Step4 + properties: + servers: {get_param: [servers, Controller]} + config: {get_resource: ControllerPacemakerUpgradeConfig_Step5} + input_values: {get_param: input_values} + + ControllerPacemakerUpgradeConfig_Step6: + type: OS::Heat::SoftwareConfig + properties: + group: script + config: + list_join: + - '' + - - str_replace: + template: | + #!/bin/bash + keep_sahara_services_on_upgrade='KEEP_SAHARA_SERVICES_ON_UPGRADE' + params: + KEEP_SAHARA_SERVICES_ON_UPGRADE: {get_param: KeepSaharaServicesOnUpgrade} + - get_file: pacemaker_common_functions.sh + - get_file: major_upgrade_pacemaker_migrations.sh + - get_file: major_upgrade_controller_pacemaker_6.sh + + ControllerPacemakerUpgradeDeployment_Step6: + type: OS::Heat::SoftwareDeploymentGroup + depends_on: ControllerPacemakerUpgradeDeployment_Step5 + properties: + servers: {get_param: [servers, Controller]} + config: {get_resource: ControllerPacemakerUpgradeConfig_Step6} + input_values: {get_param: input_values} diff --git a/extraconfig/tasks/major_upgrade_pacemaker_init.j2.yaml b/extraconfig/tasks/major_upgrade_pacemaker_init.j2.yaml index f6aa3066..c308720b 100644 --- a/extraconfig/tasks/major_upgrade_pacemaker_init.j2.yaml +++ b/extraconfig/tasks/major_upgrade_pacemaker_init.j2.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: 'Upgrade for Pacemaker deployments' parameters: @@ -54,19 +54,28 @@ resources: upgrade_level_nova_compute='UPGRADE_LEVEL_NOVA_COMPUTE' params: UPGRADE_LEVEL_NOVA_COMPUTE: {get_param: UpgradeLevelNovaCompute} + - get_file: pacemaker_common_functions.sh - get_file: major_upgrade_compute.sh ObjectStorageDeliverUpgradeScriptConfig: type: OS::Heat::SoftwareConfig properties: group: script - config: {get_file: major_upgrade_object_storage.sh} + config: + list_join: + - '' + - - get_file: pacemaker_common_functions.sh + - get_file: major_upgrade_object_storage.sh CephStorageDeliverUpgradeScriptConfig: type: OS::Heat::SoftwareConfig properties: group: script - config: {get_file: major_upgrade_ceph_storage.sh} + config: + list_join: + - '' + - - get_file: pacemaker_common_functions.sh + - get_file: major_upgrade_ceph_storage.sh {% for role in roles %} UpgradeInit{{role.name}}Deployment: diff --git a/extraconfig/tasks/major_upgrade_pacemaker_migrations.sh b/extraconfig/tasks/major_upgrade_pacemaker_migrations.sh index b8c5321b..ae22a1e7 100644 --- a/extraconfig/tasks/major_upgrade_pacemaker_migrations.sh +++ b/extraconfig/tasks/major_upgrade_pacemaker_migrations.sh @@ -77,14 +77,12 @@ function services_to_migrate { openstack-aodh-evaluator-clone openstack-aodh-listener-clone openstack-aodh-notifier-clone - openstack-ceilometer-api-clone openstack-ceilometer-central-clone openstack-ceilometer-collector-clone openstack-ceilometer-notification-clone openstack-cinder-api-clone openstack-cinder-scheduler-clone openstack-glance-api-clone - openstack-glance-registry-clone openstack-gnocchi-metricd-clone openstack-gnocchi-statsd-clone openstack-heat-api-cfn-clone @@ -109,20 +107,17 @@ function services_to_migrate { # during the conversion # 2. Remove all the colocation constraints and then the ordering constraints, except the # ones related to haproxy/VIPs which exist in Newton as well -# 3. Remove all the resources that won't be managed by pacemaker in newton. Note that they -# will show up as ORPHANED but they will keep running normally via systemd. They will be -# enabled to start at boot by puppet during the converge step -# 4. Take the cluster out of maintenance-mode and do a resource cleanup +# 3. Take the cluster out of maintenance-mode +# 4. Remove all the resources that won't be managed by pacemaker in newton. The +# outcome will be +# that they are stopped and removed from pacemakers control +# 5. Do a resource cleanup to make sure the cluster is in a clean state function migrate_full_to_ng_ha { if [[ -n $(pcmk_running) ]]; then pcs property set maintenance-mode=true - # We are making sure here that the property has propagated everywhere - if ! timeout -k 10 300 crm_resource --wait; then - echo_error "ERROR: cluster remained unstable after setting maintenance-mode for more than 300 seconds, exiting." - exit 1 - fi - # First we go through all the colocation constraints (except the ones we want to keep, i.e. the haproxy/ip ones) - # and we remove those + + # First we go through all the colocation constraints (except the ones + # we want to keep, i.e. the haproxy/ip ones) and we remove those COL_CONSTRAINTS=$(pcs config show | sed -n '/^Colocation Constraints:$/,/^$/p' | grep -v "Colocation Constraints:" | egrep -v "ip-.*haproxy" | awk '{print $NF}' | cut -f2 -d: |cut -f1 -d\)) for constraint in $COL_CONSTRAINTS; do log_debug "Deleting colocation constraint $constraint from CIB" @@ -135,32 +130,35 @@ function migrate_full_to_ng_ha { log_debug "Deleting ordering constraint $constraint from CIB" pcs constraint remove "$constraint" done + # At this stage all the pacemaker resources are removed from the CIB. + # Once we remove the maintenance-mode those systemd resources will keep + # on running. They shall be systemd enabled via the puppet converge + # step later on + pcs property set maintenance-mode=false # At this stage there are no constraints whatsoever except the haproxy/ip ones - # which we want to keep. We now delete each resource that will move to systemd - # Note that the corresponding systemd resource will stay running, which means that - # later when we do the "yum update", things will be a bit slower because each - # "systemctl try-restart <service>" is not a no-op any longer because the service is up - # and running and it will be restarted with rabbitmq being down. + # which we want to keep. We now disable and then delete each resource + # that will move to systemd. + # We want the systemd resources be stopped before doing "yum update", + # that way "systemctl try-restart <service>" is no-op because the + # service was down already PCS_STATUS_OUTPUT="$(pcs status)" for resource in $(services_to_migrate) "delay-clone" "openstack-core-clone"; do if echo "$PCS_STATUS_OUTPUT" | grep "$resource"; then log_debug "Deleting $resource from the CIB" - - # We need to add --force because the cluster is in maintenance mode and the resource - # is unmanaged. The if serves to make this idempotent + if ! pcs resource disable "$resource" --wait=600; then + echo_error "ERROR: resource $resource failed to be disabled" + exit 1 + fi pcs resource delete --force "$resource" else - log_debug "Service $service not found as a pacemaker resource, not trying to delete." + log_debug "Service $resource not found as a pacemaker resource, not trying to delete." fi done - # At this stage all the pacemaker resources are removed from the CIB. Once we remove the - # maintenance-mode those systemd resources will keep on running. They shall be systemd enabled - # via the puppet converge step later on - pcs property set maintenance-mode=false - # We need to do a pcs resource cleanup here + crm_resource --wait to make sure the - # cluster is in a clean state before we stop everything, upgrade and restart everything + # We need to do a pcs resource cleanup here + crm_resource --wait to + # make sure the cluster is in a clean state before we stop everything, + # upgrade and restart everything pcs resource cleanup # We are making sure here that the cluster is stable before proceeding if ! timeout -k 10 600 crm_resource --wait; then @@ -169,3 +167,34 @@ function migrate_full_to_ng_ha { fi fi } + +function disable_standalone_ceilometer_api { + if [[ -n $(is_bootstrap_node) ]]; then + if [[ -n $(is_pacemaker_managed openstack-ceilometer-api) ]]; then + # Disable pacemaker resources for ceilometer-api + manage_pacemaker_service disable openstack-ceilometer-api + check_resource_pacemaker openstack-ceilometer-api stopped 600 + pcs resource delete openstack-ceilometer-api --wait=600 + fi + fi +} + + +# This function will make sure that the rabbitmq ha policies are converted from mitaka to newton +# In newton we had: Attributes: set_policy="ha-all ^(?!amq\.).* {"ha-mode":"all"}" +# In ocata we want: Attributes: set_policy="ha-all ^(?!amq\.).* {"ha-mode":"exactly","ha-params":2}" +# The nr "2" should be CEIL(N/2) where N is the number of Controllers (i.e. rabbit instances) +# Note that changing an attribute like this makes the rabbitmq resource restart +function rabbitmq_newton_ocata_upgrade { + if pcs resource show rabbitmq-clone | grep -q -E "Attributes:.*\"ha-mode\":\"all\""; then + # Number of controller is obtained by counting how many hostnames we + # have in controller_node_names hiera key + nr_controllers=$(($(hiera controller_node_names | grep -o "," |wc -l) + 1)) + nr_queues=$(($nr_controllers / 2 + ($nr_controllers % 2))) + if ! [ $nr_queues -gt 0 -a $nr_queues -le $nr_controllers ]; then + echo_error "ERROR: The nr. of HA queues during the M/N upgrade is out of range $nr_queues" + exit 1 + fi + pcs resource update rabbitmq set_policy='ha-all ^(?!amq\\.).* {"ha-mode":"exactly","ha-params":'"$nr_queues}" --wait=600 + fi +} diff --git a/extraconfig/tasks/mitaka_to_newton_aodh_data_migration.yaml b/extraconfig/tasks/mitaka_to_newton_aodh_data_migration.yaml index 91406fba..45933fb7 100644 --- a/extraconfig/tasks/mitaka_to_newton_aodh_data_migration.yaml +++ b/extraconfig/tasks/mitaka_to_newton_aodh_data_migration.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: > Software-config for performing aodh data migration @@ -20,6 +20,6 @@ resources: AodhMysqlMigrationScriptDeployment: type: OS::Heat::SoftwareDeploymentGroup properties: - servers: {get_param: servers, Controller} + servers: {get_param: [servers, Controller]} config: {get_resource: AodhMysqlMigrationScriptConfig} input_values: {get_param: input_values} diff --git a/extraconfig/tasks/mitaka_to_newton_ceilometer_wsgi_upgrade.pp b/extraconfig/tasks/mitaka_to_newton_ceilometer_wsgi_upgrade.pp new file mode 100644 index 00000000..a8d43663 --- /dev/null +++ b/extraconfig/tasks/mitaka_to_newton_ceilometer_wsgi_upgrade.pp @@ -0,0 +1,103 @@ +# Copyright 2015 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This puppet manifest is to be used only during a Mitaka->Newton upgrade +# It configures ceilometer to be run under httpd but it makes sure to not +# restart any services. This snippet needs to be called before init as a +# pre upgrade migration. + +Service <| + tag == 'ceilometer-service' +|> { + hasrestart => true, + restart => '/bin/true', + start => '/bin/true', + stop => '/bin/true', +} + +if $::hostname == downcase(hiera('bootstrap_nodeid')) { + $pacemaker_master = true + $sync_db = true +} else { + $pacemaker_master = false + $sync_db = false +} + +include ::tripleo::packages + + +if str2bool(hiera('mongodb::server::ipv6', false)) { + $mongo_node_ips_with_port_prefixed = prefix(hiera('mongodb_node_ips'), '[') + $mongo_node_ips_with_port = suffix($mongo_node_ips_with_port_prefixed, ']:27017') +} else { + $mongo_node_ips_with_port = suffix(hiera('mongodb_node_ips'), ':27017') +} +$mongodb_replset = hiera('mongodb::server::replset') +$mongo_node_string = join($mongo_node_ips_with_port, ',') +$database_connection = "mongodb://${mongo_node_string}/ceilometer?replicaSet=${mongodb_replset}" + +$rabbit_hosts = hiera('rabbitmq_node_ips', undef) +$rabbit_port = hiera('ceilometer::rabbit_port', 5672) +$rabbit_endpoints = suffix(any2array(normalize_ip_for_uri($rabbit_hosts)), ":${rabbit_port}") + +class { '::ceilometer' : + rabbit_hosts => $rabbit_endpoints, +} + +class {'::ceilometer::db': + database_connection => $database_connection, +} + +if $sync_db { + include ::ceilometer::db::sync +} + +include ::ceilometer::config + +class { '::ceilometer::api': + enabled => true, + service_name => 'httpd', + keystone_password => hiera('ceilometer::keystone::auth::password'), + identity_uri => hiera('ceilometer::keystone::authtoken::auth_url'), + auth_uri => hiera('ceilometer::keystone::authtoken::auth_uri'), + keystone_tenant => hiera('ceilometer::keystone::authtoken::project_name'), +} + +class { '::apache' : + service_enable => false, + service_manage => true, + service_restart => '/bin/true', + purge_configs => false, + purge_vhost_dir => false, +} + +# To ensure existing ports are not overridden +class { '::aodh::wsgi::apache': + servername => $::hostname, + ssl => false, +} +class { '::gnocchi::wsgi::apache': + servername => $::hostname, + ssl => false, +} + +class { '::keystone::wsgi::apache': + servername => $::hostname, + ssl => false, +} +class { '::ceilometer::wsgi::apache': + servername => $::hostname, + ssl => false, +} diff --git a/extraconfig/tasks/pacemaker_common_functions.sh b/extraconfig/tasks/pacemaker_common_functions.sh index 4f17b69a..aae4a2de 100755 --- a/extraconfig/tasks/pacemaker_common_functions.sh +++ b/extraconfig/tasks/pacemaker_common_functions.sh @@ -284,7 +284,7 @@ function systemctl_swift { services=$(systemctl | grep openstack-swift- | grep running | awk '{print $1}') ;; start) - enable_swift_storage=$(hiera -c /etc/puppet/hiera.yaml 'enable_swift_storage') + enable_swift_storage=$(hiera -c /etc/puppet/hiera.yaml tripleo::profile::base::swift::storage::enable_swift_storage) if [[ $enable_swift_storage != "true" ]]; then services=( openstack-swift-proxy ) fi @@ -297,3 +297,27 @@ function systemctl_swift { manage_systemd_service $action $service done } + +# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205 +function special_case_ovs_upgrade_if_needed { + if [[ -n $(rpm -q --scripts openvswitch | awk '/postuninstall/,/*/' | grep "systemctl.*try-restart") ]]; then + echo "Manual upgrade of openvswitch - restart in postun detected" + rm -rf OVS_UPGRADE + mkdir OVS_UPGRADE && pushd OVS_UPGRADE + echo "Attempting to downloading latest openvswitch with yumdownloader" + yumdownloader --resolve openvswitch + for pkg in $(ls -1 *.rpm); do + if rpm -U --test $pkg 2>&1 | grep "already installed" ; then + echo "Looks like newer version of $pkg is already installed, skipping" + else + echo "Updating $pkg with nopostun option" + rpm -U --replacepkgs --nopostun $pkg + fi + done + popd + else + echo "Skipping manual upgrade of openvswitch - no restart in postun detected" + fi + +} + diff --git a/extraconfig/tasks/pacemaker_resource_restart.sh b/extraconfig/tasks/pacemaker_resource_restart.sh index 3da7efec..49d39bc8 100755 --- a/extraconfig/tasks/pacemaker_resource_restart.sh +++ b/extraconfig/tasks/pacemaker_resource_restart.sh @@ -4,11 +4,14 @@ set -eux # Run if pacemaker is running, we're the bootstrap node, # and we're updating the deployment (not creating). -if [[ -n $(pcmk_running) && -n $(is_bootstrap_node) ]]; then + +RESTART_FOLDER="/var/lib/tripleo/pacemaker-restarts" + +if [[ -d "$RESTART_FOLDER" && -n $(pcmk_running) && -n $(is_bootstrap_node) ]]; then TIMEOUT=600 - SERVICES_TO_RESTART="$(ls /var/lib/tripleo/pacemaker-restarts)" PCS_STATUS_OUTPUT="$(pcs status)" + SERVICES_TO_RESTART="$(ls $RESTART_FOLDER)" for service in $SERVICES_TO_RESTART; do if ! echo "$PCS_STATUS_OUTPUT" | grep $service; then @@ -20,6 +23,11 @@ if [[ -n $(pcmk_running) && -n $(is_bootstrap_node) ]]; then for service in $SERVICES_TO_RESTART; do echo "Restarting $service..." pcs resource restart --wait=$TIMEOUT $service - rm -f /var/lib/tripleo/pacemaker-restarts/$service + rm -f "$RESTART_FOLDER"/$service done + +fi + +if [ $(systemctl is-active haproxy) = "active" ]; then + systemctl reload haproxy fi diff --git a/extraconfig/tasks/post_puppet_pacemaker.yaml b/extraconfig/tasks/post_puppet_pacemaker.yaml index b62502f8..a63868c9 100644 --- a/extraconfig/tasks/post_puppet_pacemaker.yaml +++ b/extraconfig/tasks/post_puppet_pacemaker.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: 'Post-Puppet Config for Pacemaker deployments' parameters: diff --git a/extraconfig/tasks/post_puppet_pacemaker_restart.yaml b/extraconfig/tasks/post_puppet_pacemaker_restart.yaml index 52760c87..475a6688 100644 --- a/extraconfig/tasks/post_puppet_pacemaker_restart.yaml +++ b/extraconfig/tasks/post_puppet_pacemaker_restart.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: 'Post-Puppet restart config for Pacemaker deployments' parameters: diff --git a/extraconfig/tasks/pre_puppet_pacemaker.yaml b/extraconfig/tasks/pre_puppet_pacemaker.yaml index 82546588..aa7514f9 100644 --- a/extraconfig/tasks/pre_puppet_pacemaker.yaml +++ b/extraconfig/tasks/pre_puppet_pacemaker.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: 'Pre-Puppet Config for Pacemaker deployments' parameters: diff --git a/extraconfig/tasks/yum_update.sh b/extraconfig/tasks/yum_update.sh index b045e5ea..74af7b02 100755 --- a/extraconfig/tasks/yum_update.sh +++ b/extraconfig/tasks/yum_update.sh @@ -44,6 +44,27 @@ fi pacemaker_status=$(systemctl is-active pacemaker) +# Fix the redis/rabbit resource start/stop timeouts. See https://bugs.launchpad.net/tripleo/+bug/1633455 +# and https://bugs.launchpad.net/tripleo/+bug/1634851 +if [[ "$pacemaker_status" == "active" && \ + "$(hiera -c /etc/puppet/hiera.yaml bootstrap_nodeid)" = "$(facter hostname)" ]] ; then + if pcs resource show rabbitmq | grep -E "start.*timeout=100"; then + pcs resource update rabbitmq op start timeout=200s + fi + if pcs resource show rabbitmq | grep -E "stop.*timeout=90"; then + pcs resource update rabbitmq op stop timeout=200s + fi + if pcs resource show redis | grep -E "start.*timeout=120"; then + pcs resource update redis op start timeout=200s + fi + if pcs resource show redis | grep -E "stop.*timeout=120"; then + pcs resource update redis op stop timeout=200s + fi +fi + +# Special-case OVS for https://bugs.launchpad.net/tripleo/+bug/1635205 +special_case_ovs_upgrade_if_needed + if [[ "$pacemaker_status" == "active" ]] ; then echo "Pacemaker running, stopping cluster node and doing full package update" node_count=$(pcs status xml | grep -o "<nodes_configured.*/>" | grep -o 'number="[0-9]*"' | grep -o "[0-9]*") @@ -54,8 +75,9 @@ if [[ "$pacemaker_status" == "active" ]] ; then pcs cluster stop fi else - echo "Upgrading openstack-puppet-modules" + echo "Upgrading openstack-puppet-modules and its dependencies" yum -q -y update openstack-puppet-modules + yum deplist openstack-puppet-modules | awk '/dependency/{print $2}' | xargs yum -q -y update echo "Upgrading other packages is handled by config management tooling" echo -n "true" > $heat_outputs_path.update_managed_packages exit 0 @@ -70,6 +92,17 @@ return_code=$? echo "$result" echo "yum return code: $return_code" +# Writes any changes caused by alterations to os-net-config and bounces the +# interfaces *before* restarting the cluster. +os-net-config -c /etc/os-net-config/config.json -v --detailed-exit-codes +RETVAL=$? +if [[ $RETVAL == 2 ]]; then + echo "os-net-config: interface configuration files updated successfully" +elif [[ $RETVAL != 0 ]]; then + echo "ERROR: os-net-config configuration failed" + exit $RETVAL +fi + if [[ "$pacemaker_status" == "active" ]] ; then echo "Starting cluster node" pcs cluster start diff --git a/extraconfig/tasks/yum_update.yaml b/extraconfig/tasks/yum_update.yaml index d313ca9f..8cff838e 100644 --- a/extraconfig/tasks/yum_update.yaml +++ b/extraconfig/tasks/yum_update.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: > Software-config for performing package updates using yum @@ -9,7 +9,12 @@ resources: type: OS::Heat::SoftwareConfig properties: group: script - config: {get_file: yum_update.sh} + config: + list_join: + - '' + - - get_file: pacemaker_common_functions.sh + - get_file: yum_update.sh + inputs: - name: update_identifier description: yum will only run for previously unused values of update_identifier diff --git a/extraconfig/tasks/yum_update_noop.yaml b/extraconfig/tasks/yum_update_noop.yaml index b759d9c5..9400c1d2 100644 --- a/extraconfig/tasks/yum_update_noop.yaml +++ b/extraconfig/tasks/yum_update_noop.yaml @@ -1,4 +1,4 @@ -heat_template_version: 2014-10-16 +heat_template_version: ocata description: 'No-op yum update task' resources: |