From 434466fafd40485eefba1c59af105ecf793a3c25 Mon Sep 17 00:00:00 2001 From: Markos Chandras Date: Fri, 23 Mar 2018 10:01:47 +0000 Subject: xci: install: Move all pip installations to virtualenv Now that all pip installations are managed in the same way we can move them inside a global virtual environment to further isolate the XCI artifacts from the rest of the system. Moreover, we further simplify the initial package installation to install everything at once instead of calling the package manager for every single package that we need. Change-Id: I6a170d2439fae8b0653f3141e0e8bb8ead67657e Signed-off-by: Markos Chandras --- xci/files/install-lib.sh | 70 +++++++------------------- xci/files/xci-destroy-env.sh | 4 +- xci/infra/bifrost/scripts/bifrost-provision.sh | 5 +- xci/xci-deploy.sh | 2 + 4 files changed, 27 insertions(+), 54 deletions(-) diff --git a/xci/files/install-lib.sh b/xci/files/install-lib.sh index 2679233d..5c2dd288 100644 --- a/xci/files/install-lib.sh +++ b/xci/files/install-lib.sh @@ -17,6 +17,7 @@ function install_ansible() { # Use the upper-constraints file from the pinned requirements repository. local requirements_sha=$(awk '/requirements_git_install_branch:/ {print $2}' ${XCI_PATH}/xci/installer/osa/files/openstack_services.yml) local uc="https://raw.githubusercontent.com/openstack/requirements/${requirements_sha}/upper-constraints.txt" + local install_map declare -A PKG_MAP @@ -49,6 +50,7 @@ function install_ansible() { [lsb-release]=lsb-release [make]=make [net-tools]=net-tools + [pip]=python-pip [python]=python [python-devel]=python-devel [venv]=python-virtualenv @@ -75,12 +77,13 @@ function install_ansible() { [lsb-release]=lsb-release [make]=make [net-tools]=net-tools + [pip]=python-pip [python]=python-minimal [python-devel]=libpython-dev [venv]=python-virtualenv [wget]=wget ) - EXTRA_PKG_DEPS=() + EXTRA_PKG_DEPS=( apt-utils ) sudo apt-get update ;; @@ -96,6 +99,7 @@ function install_ansible() { [lsb-release]=redhat-lsb [make]=make [net-tools]=net-tools + [pip]=python2-pip [python]=python [python-devel]=python-devel [venv]=python-virtualenv @@ -108,51 +112,15 @@ function install_ansible() { *) echo "ERROR: Supported package manager not found. Supported: apt, dnf, yum, zypper"; exit 1;; esac - if ! $(python --version &>/dev/null); then - ${INSTALLER_CMD} ${PKG_MAP[python]} - fi - if ! $(gcc -v &>/dev/null); then - ${INSTALLER_CMD} ${PKG_MAP[gcc]} - fi - if ! $(wget --version &>/dev/null); then - ${INSTALLER_CMD} ${PKG_MAP[wget]} - fi - - if ! $(python -m virtualenv --version &>/dev/null); then - ${INSTALLER_CMD} ${PKG_MAP[venv]} - fi - - for pkg in ${CHECK_CMD_PKGS[@]}; do - if ! $(${CHECK_CMD} ${PKG_MAP[$pkg]} &>/dev/null); then - ${INSTALLER_CMD} ${PKG_MAP[$pkg]} - fi + # Build instllation map + for pkgmap in ${CHECK_CMD_PKGS[@]}; do + install_map+=(${PKG_MAP[$pkgmap]} ) done - if [ -n "${EXTRA_PKG_DEPS-}" ]; then - for pkg in ${EXTRA_PKG_DEPS}; do - if ! $(${CHECK_CMD} ${pkg} &>/dev/null); then - ${INSTALLER_CMD} ${pkg} - fi - done - fi + install_map+=(${EXTRA_PKG_DEPS[@]} ) + + ${INSTALLER_CMD} ${install_map[@]} - # If we're using a venv, we need to work around sudo not - # keeping the path even with -E. - PYTHON=$(which python) - - # To install python packages, we need pip. - # - # We can't use the apt packaged version of pip since - # older versions of pip are incompatible with - # requests, one of our indirect dependencies (bug 1459947). - # - # Note(cinerama): We use pip to install an updated pip plus our - # other python requirements. pip breakages can seriously impact us, - # so we've chosen to install/upgrade pip here rather than in - # requirements (which are synced automatically from the global ones) - # so we can quickly and easily adjust version parameters. - # See bug 1536627. - # # Note(cinerama): If pip is linked to pip3, the rest of the install # won't work. Remove the alternatives. This is due to ansible's # python 2.x requirement. @@ -160,16 +128,16 @@ function install_ansible() { sudo -H update-alternatives --remove pip $(readlink -f /etc/alternatives/pip) fi - if ! which pip; then - wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py - sudo -H -E ${PYTHON} /tmp/get-pip.py - fi + # We need to prepare our virtualenv now + virtualenv --quiet --no-site-packages ${XCI_VENV} + set +u + source ${XCI_VENV}/bin/activate + set -u - PIP=$(which pip) - echo "Using pip: $(${PIP} --version)" - ${PIP} -q install --user --upgrade -c $uc ara virtualenv pip setuptools ansible==$XCI_ANSIBLE_PIP_VERSION + # We are inside the virtualenv now so we should be good to use pip and python from it. + pip -q install --upgrade -c $uc ara virtualenv pip setuptools ansible==$XCI_ANSIBLE_PIP_VERSION - ara_location=$(${PYTHON} -c "import os,ara; print(os.path.dirname(ara.__file__))") + ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))") export ANSIBLE_CALLBACK_PLUGINS="/etc/ansible/roles/plugins/callback:${ara_location}/plugins/callbacks" } diff --git a/xci/files/xci-destroy-env.sh b/xci/files/xci-destroy-env.sh index 8b811d0f..2e183bd7 100755 --- a/xci/files/xci-destroy-env.sh +++ b/xci/files/xci-destroy-env.sh @@ -23,7 +23,7 @@ rm -rf ${HOME}/.config/openstack rm -rf ${HOME}/.ansible # bifrost installs everything on venv so we need to look there if virtualbmc is not installed on the host. -if which vbmc &>/dev/null || { [[ -e /opt/stack/bifrost/bin/activate ]] && source /opt/stack/bifrost/bin/activate; }; then +if which vbmc &>/dev/null || { [[ -e ${XCI_VENV}/bifrost/bin/activate ]] && source ${XCI_VENV}/bifrost/bin/activate; }; then # Delete all libvirt VMs and hosts from vbmc (look for a port number) for vm in $(vbmc list | awk '/[0-9]/{{ print $2 }}'); do if which virsh &>/dev/null; then @@ -81,4 +81,6 @@ service ironic-api restart || true service ironic-conductor start || true service ironic-inspector restart || true +rm -rf ${XCI_VENV} + # vim: set ts=4 sw=4 expandtab: diff --git a/xci/infra/bifrost/scripts/bifrost-provision.sh b/xci/infra/bifrost/scripts/bifrost-provision.sh index 939acb58..eedf350f 100755 --- a/xci/infra/bifrost/scripts/bifrost-provision.sh +++ b/xci/infra/bifrost/scripts/bifrost-provision.sh @@ -13,7 +13,8 @@ set -o pipefail export PYTHONUNBUFFERED=1 SCRIPT_HOME="$(cd "$(dirname "$0")" && pwd)" BIFROST_HOME=$SCRIPT_HOME/.. -ENABLE_VENV="false" +ENABLE_VENV="true" +export VENV=${XCI_VENV}/bifrost PROVISION_WAIT_TIMEOUT=${PROVISION_WAIT_TIMEOUT:-3600} # This is normally exported by XCI env but we should initialize it here # in case we run this script on its own for debug purposes @@ -93,7 +94,7 @@ if [[ -e ${XCI_PATH}/deployment_image.qcow2 ]]; then fi # Install missing dependencies -$(which pip) install -q --user --upgrade -r "$(dirname $0)/../requirements.txt" +pip install -q --upgrade -r "$(dirname $0)/../requirements.txt" # Change working directory cd $BIFROST_HOME/playbooks diff --git a/xci/xci-deploy.sh b/xci/xci-deploy.sh index fa9f3351..618377ec 100755 --- a/xci/xci-deploy.sh +++ b/xci/xci-deploy.sh @@ -59,6 +59,8 @@ fi #------------------------------------------------------------------------------- # find where are we export XCI_PATH="$(git rev-parse --show-toplevel)" +# Declare our virtualenv +export XCI_VENV=${XCI_PATH}/venv/ # source user vars source $XCI_PATH/xci/config/user-vars # source pinned versions -- cgit 1.2.3-korg