diff options
Diffstat (limited to 'xci/files/xci-lib.sh')
| -rw-r--r-- | xci/files/xci-lib.sh | 298 |
1 files changed, 298 insertions, 0 deletions
diff --git a/xci/files/xci-lib.sh b/xci/files/xci-lib.sh new file mode 100644 index 00000000..860153b9 --- /dev/null +++ b/xci/files/xci-lib.sh @@ -0,0 +1,298 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 SUSE LINUX GmbH. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# Avoid double sourcing the file +[[ -n ${XCI_LIB_SOURCED:-} ]] && return 0 || export XCI_LIB_SOURCED=1 + +function usage() { + echo " +Usage: $(basename ${0}) [-i <idf>] [-p <pdf>] + + -h: This message + -i: Installer Descriptor File (IDF). (Default ${XCI_PATH}/xci/var/idf.yml) + -p: Pod Descriptor File (PDF). (Default ${XCI_PATH}/xci/var/pdf.yml) + " + exit 0 +} + +function parse_cmdline_opts() { + IDF=${XCI_PATH}/xci/var/idf.yml + PDF=${XCI_PATH}/xci/var/pdf.yml + + while getopts ":hi:p:" o; do + case "${o}" in + i) IDF="${OPTARG}" ;; + p) PDF="${OPTARG}" ;; + h) usage ;; + *) echo "ERROR: Invalid option '-${OPTARG}'"; usage ;; + esac + done + + # Do all the exports + export PDF=$(realpath ${PDF}) + export IDF=$(realpath ${IDF}) +} + +function bootstrap_xci_env() { + # Declare our virtualenv + export XCI_VENV=${XCI_PATH}/venv/ + # source user vars + source $XCI_PATH/xci/config/user-vars + # source pinned versions + source $XCI_PATH/xci/config/pinned-versions + # source flavor configuration + source "$XCI_PATH/xci/config/${XCI_FLAVOR}-vars" + # source installer configuration + source "$XCI_PATH/xci/installer/${INSTALLER_TYPE}/env" &>/dev/null || true + # source xci configuration + source $XCI_PATH/xci/config/env-vars + # baremetal variable to true if the vendor in the pdf is not libvirt + grep -o vendor.* ${PDF} | grep -q libvirt && export BAREMETAL=false || export BAREMETAL=true +} + +function install_ansible() { + set -eu + + # Use the upper-constraints file from the pinned requirements repository. + local uc="https://raw.githubusercontent.com/openstack/requirements/${OPENSTACK_REQUIREMENTS_VERSION}/upper-constraints.txt" + local osa_uc="https://raw.githubusercontent.com/openstack/openstack-ansible/${OPENSTACK_OSA_VERSION}/global-requirement-pins.txt" + local install_map + + declare -A PKG_MAP + + # workaround: for latest bindep to work, it needs to use en_US local + export LANG="C" + + CHECK_CMD_PKGS=( + gcc + libffi + libopenssl + lsb-release + make + net-tools + python-devel + python + python-pyyaml + venv + wget + curl + ) + + source /etc/os-release || source /usr/lib/os-release + case ${ID,,} in + *suse*) + OS_FAMILY="Suse" + INSTALLER_CMD="sudo -H -E zypper -q install -y --no-recommends" + CHECK_CMD="zypper search --match-exact --installed" + PKG_MAP=( + [gcc]=gcc + [libffi]=libffi-devel + [libopenssl]=libopenssl-devel + [lsb-release]=lsb-release + [make]=make + [net-tools]=net-tools + [pip]=python-pip + [python]=python + [python-devel]=python-devel + [python-pyyaml]=python-PyYAML + [venv]=python-virtualenv + [wget]=wget + [curl]=curl + ) + EXTRA_PKG_DEPS=( python-xml ) + sudo zypper -n ref + # NOTE (cinerama): we can't install python without removing this package + # if it exists + if $(${CHECK_CMD} patterns-openSUSE-minimal_base-conflicts &> /dev/null); then + sudo -H zypper remove -y patterns-openSUSE-minimal_base-conflicts + fi + ;; + + ubuntu|debian) + OS_FAMILY="Debian" + export DEBIAN_FRONTEND=noninteractive + INSTALLER_CMD="sudo -H -E apt-get -y -q=3 install" + CHECK_CMD="dpkg -l" + PKG_MAP=( + [gcc]=gcc + [libffi]=libffi-dev + [libopenssl]=libssl-dev + [lsb-release]=lsb-release + [make]=make + [net-tools]=net-tools + [pip]=python-pip + [python]=python-minimal + [python-devel]=libpython-dev + [python-pyyaml]=python-yaml + [venv]=python-virtualenv + [wget]=wget + [curl]=curl + ) + EXTRA_PKG_DEPS=( apt-utils ) + sudo apt-get update -qq > /dev/null + ;; + + rhel|fedora|centos) + OS_FAMILY="RedHat" + PKG_MANAGER=$(which dnf || which yum) + INSTALLER_CMD="sudo -H -E ${PKG_MANAGER} -q -y install" + CHECK_CMD="rpm -q" + PKG_MAP=( + [gcc]=gcc + [libffi]=libffi-devel + [libopenssl]=openssl-devel + [lsb-release]=redhat-lsb + [make]=make + [net-tools]=net-tools + [pip]=python2-pip + [python]=python + [python-devel]=python-devel + [python-pyyaml]=PyYAML + [venv]=python-virtualenv + [wget]=wget + [curl]=curl + ) + sudo $PKG_MANAGER updateinfo > /dev/null + EXTRA_PKG_DEPS=( deltarpm ) + ;; + + *) echo "ERROR: Supported package manager not found. Supported: apt, dnf, yum, zypper"; exit 1;; + esac + + # Build instllation map + for pkgmap in ${CHECK_CMD_PKGS[@]}; do + install_map+=(${PKG_MAP[$pkgmap]} ) + done + + install_map+=(${EXTRA_PKG_DEPS[@]} ) + + ${INSTALLER_CMD} ${install_map[@]} > /dev/null + + # We need to prepare our virtualenv now + virtualenv --quiet --no-site-packages ${XCI_VENV} + set +u + source ${XCI_VENV}/bin/activate + set -u + + # We are inside the virtualenv now so we should be good to use pip and python from it. + pip -q install --upgrade pip==9.0.3 # We need a version which supports the '-c' parameter + pip -q install --upgrade -c $uc -c $osa_uc ara==0.16.4 virtualenv pip setuptools shade ansible==$XCI_ANSIBLE_PIP_VERSION ansible-lint==3.4.21 + + ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))") + export ANSIBLE_CALLBACK_PLUGINS="/etc/ansible/roles/plugins/callback:${ara_location}/plugins/callbacks" +} + +ansible_lint() { + set -eu + local playbooks_dir=(xci/playbooks xci/installer/osa/playbooks xci/installer/kubespray/playbooks xci/installer/osh/playbooks) + # Extract role from scenario information + local testing_role=$(sed -n "/^- scenario: ${DEPLOY_SCENARIO}$/,/^$/p" ${XCI_PATH}/xci/opnfv-scenario-requirements.yml | grep role | rev | cut -d '/' -f -1 | rev) + + # clear XCI_CACHE + rm -rf ${XCI_CACHE}/repos/openstack-ansible-tests + + # Clone OSA rules too + git clone --quiet --depth 1 https://github.com/openstack/openstack-ansible-tests.git \ + ${XCI_CACHE}/repos/openstack-ansible-tests + + # Because of https://github.com/willthames/ansible-lint/issues/306, ansible-lint does not understand + # import and includes yet so we need to trick it with a fake playbook so we can test our roles. We + # only test the role for the scenario we are testing + echo "Building testing playbook for role: ${testing_role}" + cat > ${XCI_PATH}/xci/playbooks/test-playbook.yml << EOF + - name: Testing playbook + hosts: localhost + roles: + - ${testing_role} +EOF + + # Only check our own playbooks + for dir in ${playbooks_dir[@]}; do + for play in $(ls ${XCI_PATH}/${dir}/*.yml); do + echo -en "Checking '${play}' playbook..." + ansible-lint --nocolor -R -r \ + ${XCI_CACHE}/repos/openstack-ansible-tests/ansible-lint ${play} + echo -en "[OK]\n" + done + done + + # Remove testing playbook + rm ${XCI_PATH}/xci/playbooks/test-playbook.yml +} + +collect_xci_logs() { + echo "----------------------------------" + echo "Info: Collecting XCI logs" + echo "----------------------------------" + + # Create the ARA log directory and store the sqlite source database + mkdir -p ${LOG_PATH}/ara/ ${LOG_PATH}/opnfv/ara + + rsync -q -a "${HOME}/.ara/ansible.sqlite" "${LOG_PATH}/ara/" + rsync -q -a root@${OPNFV_HOST_IP}:releng-xci/${LOG_PATH#$XCI_PATH/}/ ${LOG_PATH}/opnfv/ &> /dev/null || true + rsync -q -a root@${OPNFV_HOST_IP}:.ara/ansible.sqlite ${LOG_PATH}/opnfv/ara/ &> /dev/null || true + + sudo -H -E bash -c 'chown ${SUDO_UID}:${SUDO_GID} -R ${LOG_PATH}/' +} + +submit_bug_report() { + cd ${XCI_PATH} + echo "" + echo "-------------------------------------------------------------------------" + echo "Oh nooooo! The XCI deployment failed miserably :-(" + echo "" + echo "If you need help, please choose one of the following options" + echo "* #opnfv-pharos @ freenode network" + echo "* opnfv-tech-discuss mailing list (https://lists.opnfv.org/mailman/listinfo/opnfv-tech-discuss)" + echo " - Please prefix the subject with [XCI]" + echo "* https://jira.opnfv.org (Release Engineering project)" + echo "" + echo "Do not forget to submit the following information on your bug report:" + echo "" + git diff --quiet && echo "releng-xci tree status: clean" || echo "releng-xci tree status: local modifications" + echo "opnfv/releng-xci version: $(git rev-parse HEAD)" + echo "openstack/bifrost version: $OPENSTACK_BIFROST_VERSION" + echo "openstack/openstack-ansible version: $OPENSTACK_OSA_VERSION" + echo "xci flavor: $XCI_FLAVOR" + echo "xci installer: $INSTALLER_TYPE" + echo "xci scenario: $DEPLOY_SCENARIO" + echo "Environment variables:" + env | grep --color=never '\(OPNFV\|XCI\|INSTALLER_TYPE\|OPENSTACK\|SCENARIO\|ANSIBLE\|BIFROST\|DIB\)' + echo "-------------------------------------------------------------------------" +} + +log_xci_information() { + local scenario_version scenario_sha + + cd ${XCI_SCENARIOS_CACHE}/${DEPLOY_SCENARIO} + scenario_sha=$(git rev-parse HEAD) + scenario_version=$(git describe --exact 2>/dev/null || echo "master") + cd - + echo "Info: Starting XCI Deployment" + echo "Info: Deployment parameters" + echo "-------------------------------------------------------------------------" + echo "OPNFV scenario: $DEPLOY_SCENARIO" + echo "Scenario version: ${scenario_version} (sha: ${scenario_sha})" + echo "xci flavor: $XCI_FLAVOR" + echo "xci installer: $INSTALLER_TYPE" + echo "infra deployment: $INFRA_DEPLOYMENT" + echo "opnfv/releng-xci version: $(git rev-parse HEAD)" + [[ "$INFRA_DEPLOYMENT" == "bifrost" ]] && echo "openstack/bifrost version: $OPENSTACK_BIFROST_VERSION" + [[ "$INSTALLER_TYPE" == "osa" ]] && echo "openstack/openstack-ansible version: $OPENSTACK_OSA_VERSION" + [[ "$INSTALLER_TYPE" == "kubespray" ]] && echo "kubespray version: $KUBESPRAY_VERSION" + [[ "$INSTALLER_TYPE" == "osh" ]] && echo "kubespray version: $KUBESPRAY_VERSION" + echo "-------------------------------------------------------------------------" +} + +exit_trap() { + submit_bug_report + collect_xci_logs +} + +# vim: set ts=4 sw=4 expandtab: |