Merge "xci: install: Move all pip installations to virtualenv"

4 files changed, 27 insertions, 54 deletions

diff --git a/xci/files/install-lib.sh b/xci/files/install-lib.sh
index 8ac6d612..48233968 100644
--- a/xci/files/install-lib.sh
+++ b/xci/files/install-lib.sh
@@ -17,6 +17,7 @@ function install_ansible() {
     # Use the upper-constraints file from the pinned requirements repository.
     local requirements_sha=$(awk '/requirements_git_install_branch:/ {print $2}' ${XCI_PATH}/xci/installer/osa/files/openstack_services.yml)
     local uc="https://raw.githubusercontent.com/openstack/requirements/${requirements_sha}/upper-constraints.txt"
+    local install_map
 
     declare -A PKG_MAP
 
@@ -49,6 +50,7 @@ function install_ansible() {
             [lsb-release]=lsb-release
             [make]=make
             [net-tools]=net-tools
+            [pip]=python-pip
             [python]=python
             [python-devel]=python-devel
             [venv]=python-virtualenv
@@ -75,12 +77,13 @@ function install_ansible() {
             [lsb-release]=lsb-release
             [make]=make
             [net-tools]=net-tools
+            [pip]=python-pip
             [python]=python-minimal
             [python-devel]=libpython-dev
             [venv]=python-virtualenv
             [wget]=wget
         )
-        EXTRA_PKG_DEPS=()
+        EXTRA_PKG_DEPS=( apt-utils )
         sudo apt-get update
         ;;
 
@@ -96,6 +99,7 @@ function install_ansible() {
             [lsb-release]=redhat-lsb
             [make]=make
             [net-tools]=net-tools
+            [pip]=python2-pip
             [python]=python
             [python-devel]=python-devel
             [venv]=python-virtualenv
@@ -108,51 +112,15 @@ function install_ansible() {
         *) echo "ERROR: Supported package manager not found.  Supported: apt, dnf, yum, zypper"; exit 1;;
     esac
 
-    if ! $(python --version &>/dev/null); then
-        ${INSTALLER_CMD} ${PKG_MAP[python]}
-    fi
-    if ! $(gcc -v &>/dev/null); then
-        ${INSTALLER_CMD} ${PKG_MAP[gcc]}
-    fi
-    if ! $(wget --version &>/dev/null); then
-        ${INSTALLER_CMD} ${PKG_MAP[wget]}
-    fi
-
-    if ! $(python -m virtualenv --version &>/dev/null); then
-        ${INSTALLER_CMD} ${PKG_MAP[venv]}
-    fi
-
-    for pkg in ${CHECK_CMD_PKGS[@]}; do
-        if ! $(${CHECK_CMD} ${PKG_MAP[$pkg]} &>/dev/null); then
-            ${INSTALLER_CMD} ${PKG_MAP[$pkg]}
-        fi
+    # Build instllation map
+    for pkgmap in ${CHECK_CMD_PKGS[@]}; do
+        install_map+=(${PKG_MAP[$pkgmap]} )
     done
 
-    if [ -n "${EXTRA_PKG_DEPS-}" ]; then
-        for pkg in ${EXTRA_PKG_DEPS}; do
-            if ! $(${CHECK_CMD} ${pkg} &>/dev/null); then
-              ${INSTALLER_CMD} ${pkg}
-            fi
-        done
-    fi
+    install_map+=(${EXTRA_PKG_DEPS[@]} )
+
+    ${INSTALLER_CMD} ${install_map[@]}
 
-    # If we're using a venv, we need to work around sudo not
-    # keeping the path even with -E.
-    PYTHON=$(which python)
-
-    # To install python packages, we need pip.
-    #
-    # We can't use the apt packaged version of pip since
-    # older versions of pip are incompatible with
-    # requests, one of our indirect dependencies (bug 1459947).
-    #
-    # Note(cinerama): We use pip to install an updated pip plus our
-    # other python requirements. pip breakages can seriously impact us,
-    # so we've chosen to install/upgrade pip here rather than in
-    # requirements (which are synced automatically from the global ones)
-    # so we can quickly and easily adjust version parameters.
-    # See bug 1536627.
-    #
     # Note(cinerama): If pip is linked to pip3, the rest of the install
     # won't work. Remove the alternatives. This is due to ansible's
     # python 2.x requirement.
@@ -160,16 +128,16 @@ function install_ansible() {
         sudo -H update-alternatives --remove pip $(readlink -f /etc/alternatives/pip)
     fi
 
-    if ! which pip; then
-        wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py
-        sudo -H -E ${PYTHON} /tmp/get-pip.py
-    fi
+    # We need to prepare our virtualenv now
+    virtualenv --quiet --no-site-packages ${XCI_VENV}
+    set +u
+    source ${XCI_VENV}/bin/activate
+    set -u
 
-    PIP=$(which pip)
-    echo "Using pip: $(${PIP} --version)"
-    ${PIP} -q install --user --upgrade -c $uc ara virtualenv pip setuptools ansible==$XCI_ANSIBLE_PIP_VERSION
+    # We are inside the virtualenv now so we should be good to use pip and python from it.
+    pip -q install --upgrade -c $uc ara virtualenv pip setuptools ansible==$XCI_ANSIBLE_PIP_VERSION
 
-    ara_location=$(${PYTHON} -c "import os,ara; print(os.path.dirname(ara.__file__))")
+    ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))")
     export ANSIBLE_CALLBACK_PLUGINS="/etc/ansible/roles/plugins/callback:${ara_location}/plugins/callbacks"
 }
 
diff --git a/xci/files/xci-destroy-env.sh b/xci/files/xci-destroy-env.sh
index 8b811d0f..2e183bd7 100755
--- a/xci/files/xci-destroy-env.sh
+++ b/xci/files/xci-destroy-env.sh
@@ -23,7 +23,7 @@ rm -rf ${HOME}/.config/openstack
 rm -rf ${HOME}/.ansible
 
 # bifrost installs everything on venv so we need to look there if virtualbmc is not installed on the host.
-if which vbmc &>/dev/null || { [[ -e /opt/stack/bifrost/bin/activate ]] && source /opt/stack/bifrost/bin/activate; }; then
+if which vbmc &>/dev/null || { [[ -e ${XCI_VENV}/bifrost/bin/activate ]] && source ${XCI_VENV}/bifrost/bin/activate; }; then
     # Delete all libvirt VMs and hosts from vbmc (look for a port number)
     for vm in $(vbmc list | awk '/[0-9]/{{ print $2 }}'); do
         if which virsh &>/dev/null; then
@@ -81,4 +81,6 @@ service ironic-api restart || true
 service ironic-conductor start || true
 service ironic-inspector restart || true
 
+rm -rf ${XCI_VENV}
+
 # vim: set ts=4 sw=4 expandtab:
diff --git a/xci/infra/bifrost/scripts/bifrost-provision.sh b/xci/infra/bifrost/scripts/bifrost-provision.sh
index e1a7e57c..2d773227 100755
--- a/xci/infra/bifrost/scripts/bifrost-provision.sh
+++ b/xci/infra/bifrost/scripts/bifrost-provision.sh
@@ -13,7 +13,8 @@ set -o pipefail
 export PYTHONUNBUFFERED=1
 SCRIPT_HOME="$(cd "$(dirname "$0")" && pwd)"
 BIFROST_HOME=$SCRIPT_HOME/..
-ENABLE_VENV="false"
+ENABLE_VENV="true"
+export VENV=${XCI_VENV}/bifrost
 PROVISION_WAIT_TIMEOUT=${PROVISION_WAIT_TIMEOUT:-3600}
 # This is normally exported by XCI env but we should initialize it here
 # in case we run this script on its own for debug purposes
@@ -97,7 +98,7 @@ if [[ -e ${XCI_PATH}/deployment_image.qcow2 ]]; then
 fi
 
 # Install missing dependencies
-$(which pip) install -q --user --upgrade -r "$(dirname $0)/../requirements.txt"
+pip install -q --upgrade -r "$(dirname $0)/../requirements.txt"
 
 # Change working directory
 cd $BIFROST_HOME/playbooks
diff --git a/xci/xci-deploy.sh b/xci/xci-deploy.sh
index fa9f3351..618377ec 100755
--- a/xci/xci-deploy.sh
+++ b/xci/xci-deploy.sh
@@ -59,6 +59,8 @@ fi
 #-------------------------------------------------------------------------------
 # find where are we
 export XCI_PATH="$(git rev-parse --show-toplevel)"
+# Declare our virtualenv
+export XCI_VENV=${XCI_PATH}/venv/
 # source user vars
 source $XCI_PATH/xci/config/user-vars
 # source pinned versions