From 644dc6a6cd8c4a309352a2b2602d757f4acbf63b Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Sat, 14 Oct 2017 22:47:45 +0200 Subject: states: Break on error, retry states up to 5 times While applying scenario states, break on error, and retry failed state up to 5 times. Apply the same behavior for `salt.sh`. Add new deploy parameter, '-D', backed up by 'CI_DEBUG' env var, which gates deploy sh scripts logging (set -x). Also extend '-f' deploy parameter, allowing it to be specified more than once; the first occurence will skip infra VM creation, but still sync reclass & other config from local repo, while a second occurence will also disable config sync. To prevent glusterfs client state from failing due to non-existent nova user/group, move it after nova:compute's nova state is applied. Change-Id: I234e126e16be0e133d878957bd88fed946955de8 Signed-off-by: Alexandru Avadanii (cherry picked from commit 23f9ef727310a8ef449372313785241df6557559) --- ci/deploy.sh | 51 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 18 deletions(-) (limited to 'ci') diff --git a/ci/deploy.sh b/ci/deploy.sh index 926d52bdb..908909d20 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -1,4 +1,4 @@ -#!/bin/bash -ex +#!/bin/bash -e # shellcheck disable=SC2034,SC2154,SC1090,SC1091 ############################################################################## # Copyright (c) 2017 Ericsson AB, Mirantis Inc., Enea AB and others. @@ -13,8 +13,13 @@ # BEGIN of Exit handlers # do_exit () { + local RC=$? clean - echo "Exiting ..." + if [ ${RC} -eq 0 ]; then + notify "\n[OK] MCP: Openstack installation finished succesfully!\n\n" 2 + else + notify "\n[ERROR] MCP: Openstack installation threw a fatal error!\n\n" + fi } # # End of Exit handlers @@ -32,14 +37,16 @@ $(notify "$(basename "$0"): Deploy the Fuel@OPNFV MCP stack" 3) $(notify "USAGE:" 2) $(basename "$0") -b base-uri -l lab-name -p pod-name -s deploy-scenario \\ [-B PXE Bridge [-B Mgmt Bridge [-B Internal Bridge [-B Public Bridge]]]] \\ - [-S storage-dir] [-L /path/to/log/file.tar.gz] [-f] [-F] [-e] [-d] + [-S storage-dir] [-L /path/to/log/file.tar.gz] \\ + [-f [-f]] [-F] [-e] [-d] [-D] $(notify "OPTIONS:" 2) -b Base-uri for the stack-configuration structure -B Bridge(s): 1st usage = PXE, 2nd = Mgmt, 3rd = Internal, 4th = Public -d Dry-run + -D Debug logging -e Do not launch environment deployment - -f Deploy on existing Salt master + -f Deploy on existing Salt master (use twice to also skip config sync) -F Do only create a Salt master -h Print this message and exit -l Lab-name @@ -73,8 +80,12 @@ $(notify "Input parameters to the build script are:" 2) while "mcpcontrol" is used to provision the infrastructure VMs only. The default is 'pxebr'. -d Dry-run - Produce deploy config files, but do not execute deploy +-D Debug logging - Enable extra logging in sh deploy scripts (set -x) -e Do not launch environment deployment --f Deploy on existing Salt master +-f Deploy on existing Salt master. It will skip infrastructure VM creation, + but it will still sync reclass configuration from current repo to Salt + Master node. If specified twice (e.g. -f -f), config sync will also be + skipped. -F Do only create a Salt master -h Print this message and exit -L Deployment log path and name, eg. -L /home/jenkins/job.log.tar.gz @@ -124,6 +135,7 @@ clean() { ############################################################################## # BEGIN of variables to customize # +CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x REPO_ROOT_PATH=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")/..") DEPLOY_DIR=$(cd "${REPO_ROOT_PATH}/mcp/scripts"; pwd) STORAGE_DIR=$(cd "${REPO_ROOT_PATH}/mcp/deploy/images"; pwd) @@ -150,7 +162,7 @@ source "${DEPLOY_DIR}/globals.sh" # set +x OPNFV_BRIDGE_IDX=0 -while getopts "b:B:dfFl:L:p:s:S:he" OPTION +while getopts "b:B:dDfFl:L:p:s:S:he" OPTION do case $OPTION in b) @@ -169,15 +181,18 @@ do if [ -n "${bridge}" ]; then OPNFV_BRIDGES[${OPNFV_BRIDGE_IDX}]="${bridge}" fi - OPNFV_BRIDGE_IDX=$((OPNFV_BRIDGE_IDX + 1)) + ((OPNFV_BRIDGE_IDX+=1)) done IFS=${OIFS} ;; d) DRY_RUN=1 ;; + D) + CI_DEBUG=1 + ;; f) - USE_EXISTING_INFRA=1 + ((USE_EXISTING_INFRA+=1)) ;; F) INFRA_CREATION_ONLY=1 @@ -234,7 +249,7 @@ if [ -z "${TARGET_LAB}" ] || [ -z "${TARGET_POD}" ] || \ exit 1 fi -set -x +[[ "${CI_DEBUG}" =~ (false|0) ]] || set -x # Enable the automatic exit trap trap do_exit SIGINT SIGTERM EXIT @@ -289,8 +304,8 @@ fi # Check scenario file existence SCENARIO_DIR="../config/scenario" if [ ! -f "${SCENARIO_DIR}/${DEPLOY_TYPE}/${DEPLOY_SCENARIO}.yaml" ]; then - notify "[WARN] ${DEPLOY_SCENARIO}.yaml not found! \ - Setting simplest scenario (os-nosdn-nofeature-noha)\n" 3 + notify "[WARN] ${DEPLOY_SCENARIO}.yaml not found!\n" 3 + notify "[WARN] Setting simplest scenario (os-nosdn-nofeature-noha)\n" 3 DEPLOY_SCENARIO='os-nosdn-nofeature-noha' if [ ! -f "${SCENARIO_DIR}/${DEPLOY_TYPE}/${DEPLOY_SCENARIO}.yaml" ]; then notify "[ERROR] Scenario definition file is missing!\n" 1>&2 @@ -310,7 +325,7 @@ source lib.sh eval "$(parse_yaml "${SCENARIO_DIR}/defaults-$(uname -i).yaml")" eval "$(parse_yaml "${SCENARIO_DIR}/${DEPLOY_TYPE}/${DEPLOY_SCENARIO}.yaml")" eval "$(parse_yaml "${LOCAL_PDF_RECLASS}")" -set -x +[[ "${CI_DEBUG}" =~ (false|0) ]] || set -x export CLUSTER_DOMAIN=${cluster_domain} @@ -358,7 +373,7 @@ notify "[NOTE] Using bridges: ${OPNFV_BRIDGES[*]}\n" 2 if [ ${DRY_RUN} -eq 1 ]; then notify "[NOTE] Dry run, skipping all deployment tasks\n" 2 1>&2 exit 0 -elif [ ${USE_EXISTING_INFRA} -eq 1 ]; then +elif [ ${USE_EXISTING_INFRA} -gt 0 ]; then notify "[NOTE] Use existing infra\n" 2 1>&2 check_connection else @@ -370,7 +385,9 @@ else update_mcpcontrol_network start_vms virtual_nodes check_connection - ./salt.sh "${LOCAL_PDF_RECLASS}" +fi +if [ ${USE_EXISTING_INFRA} -lt 2 ]; then + wait_for 5 "./salt.sh ${LOCAL_PDF_RECLASS}" fi # Openstack cluster setup @@ -381,8 +398,8 @@ else for state in "${cluster_states[@]}"; do notify "[STATE] Applying state: ${state}\n" 2 # shellcheck disable=SC2086,2029 - ssh ${SSH_OPTS} "${SSH_SALT}" \ - sudo "/root/fuel/mcp/config/states/${state} || true" + wait_for 5 "ssh ${SSH_OPTS} ${SSH_SALT} \ + sudo /root/fuel/mcp/config/states/${state}" done fi @@ -390,8 +407,6 @@ fi popd > /dev/null -notify "\n[DONE] MCP: Openstack installation finished succesfully!\n\n" 2 - # # END of main ############################################################################## -- cgit 1.2.3-korg