xci: install: Move all pip installations to virtualenv

Now that all pip installations are managed in the same way we can
move them inside a global virtual environment to further isolate the
XCI artifacts from the rest of the system. Moreover, we further simplify
the initial package installation to install everything at once instead
of calling the package manager for every single package that we need.

Change-Id: I6a170d2439fae8b0653f3141e0e8bb8ead67657e
Signed-off-by: Markos Chandras <mchandras@suse.de>

4 files changed, 27 insertions, 54 deletions

diff --git a/xci/files/install-lib.sh b/xci/files/install-lib.sh
index 2679233d..5c2dd288 100644
--- a/xci/files/install-lib.sh
+++ b/xci/files/install-lib.sh
@@ -17,6 +17,7 @@ function install_ansible() {
     # Use the upper-constraints file from the pinned requirements repository.
     local requirements_sha=$(awk '/requirements_git_install_branch:/ {print $2}' ${XCI_PATH}/xci/installer/osa/files/openstack_services.yml)
     local uc="https://raw.githubusercontent.com/openstack/requirements/${requirements_sha}/upper-constraints.txt"
+    local install_map
 
     declare -A PKG_MAP
 
@@ -49,6 +50,7 @@ function install_ansible() {
             [lsb-release]=lsb-release
             [make]=make
             [net-tools]=net-tools
+            [pip]=python-pip
             [python]=python
             [python-devel]=python-devel
             [venv]=python-virtualenv
@@ -75,12 +77,13 @@ function install_ansible() {
             [lsb-release]=lsb-release
             [make]=make
             [net-tools]=net-tools
+            [pip]=python-pip
             [python]=python-minimal
             [python-devel]=libpython-dev
             [venv]=python-virtualenv
             [wget]=wget
         )
-        EXTRA_PKG_DEPS=()
+        EXTRA_PKG_DEPS=( apt-utils )
         sudo apt-get update
         ;;
 
@@ -96,6 +99,7 @@ function install_ansible() {
             [lsb-release]=redhat-lsb
             [make]=make
             [net-tools]=net-tools
+            [pip]=python2-pip
             [python]=python
             [python-devel]=python-devel
             [venv]=python-virtualenv
@@ -108,51 +112,15 @@ function install_ansible() {
         *) echo "ERROR: Supported package manager not found.  Supported: apt, dnf, yum, zypper"; exit 1;;
     esac
 
-    if ! $(python --version &>/dev/null); then
-        ${INSTALLER_CMD} ${PKG_MAP[python]}
-    fi
-    if ! $(gcc -v &>/dev/null); then
-        ${INSTALLER_CMD} ${PKG_MAP[gcc]}
-    fi
-    if ! $(wget --version &>/dev/null); then
-        ${INSTALLER_CMD} ${PKG_MAP[wget]}
-    fi
-
-    if ! $(python -m virtualenv --version &>/dev/null); then
-        ${INSTALLER_CMD} ${PKG_MAP[venv]}
-    fi
-
-    for pkg in ${CHECK_CMD_PKGS[@]}; do
-        if ! $(${CHECK_CMD} ${PKG_MAP[$pkg]} &>/dev/null); then
-            ${INSTALLER_CMD} ${PKG_MAP[$pkg]}
-        fi
+    # Build instllation map
+    for pkgmap in ${CHECK_CMD_PKGS[@]}; do
+        install_map+=(${PKG_MAP[$pkgmap]} )
     done
 
-    if [ -n "${EXTRA_PKG_DEPS-}" ]; then
-        for pkg in ${EXTRA_PKG_DEPS}; do
-            if ! $(${CHECK_CMD} ${pkg} &>/dev/null); then
-              ${INSTALLER_CMD} ${pkg}
-            fi
-        done
-    fi
+    install_map+=(${EXTRA_PKG_DEPS[@]} )
+
+    ${INSTALLER_CMD} ${install_map[@]}
 
-    # If we're using a venv, we need to work around sudo not
-    # keeping the path even with -E.
-    PYTHON=$(which python)
-
-    # To install python packages, we need pip.
-    #
-    # We can't use the apt packaged version of pip since
-    # older versions of pip are incompatible with
-    # requests, one of our indirect dependencies (bug 1459947).
-    #
-    # Note(cinerama): We use pip to install an updated pip plus our
-    # other python requirements. pip breakages can seriously impact us,
-    # so we've chosen to install/upgrade pip here rather than in
-    # requirements (which are synced automatically from the global ones)
-    # so we can quickly and easily adjust version parameters.
-    # See bug 1536627.
-    #
     # Note(cinerama): If pip is linked to pip3, the rest of the install
     # won't work. Remove the alternatives. This is due to ansible's
     # python 2.x requirement.
@@ -160,16 +128,16 @@ function install_ansible() {
         sudo -H update-alternatives --remove pip $(readlink -f /etc/alternatives/pip)
     fi
 
-    if ! which pip; then
-        wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py
-        sudo -H -E ${PYTHON} /tmp/get-pip.py
-    fi
+    # We need to prepare our virtualenv now
+    virtualenv --quiet --no-site-packages ${XCI_VENV}
+    set +u
+    source ${XCI_VENV}/bin/activate
+    set -u
 
-    PIP=$(which pip)
-    echo "Using pip: $(${PIP} --version)"
-    ${PIP} -q install --user --upgrade -c $uc ara virtualenv pip setuptools ansible==$XCI_ANSIBLE_PIP_VERSION
+    # We are inside the virtualenv now so we should be good to use pip and python from it.
+    pip -q install --upgrade -c $uc ara virtualenv pip setuptools ansible==$XCI_ANSIBLE_PIP_VERSION
 
-    ara_location=$(${PYTHON} -c "import os,ara; print(os.path.dirname(ara.__file__))")
+    ara_location=$(python -c "import os,ara; print(os.path.dirname(ara.__file__))")
     export ANSIBLE_CALLBACK_PLUGINS="/etc/ansible/roles/plugins/callback:${ara_location}/plugins/callbacks"
 }
 
diff --git a/xci/files/xci-destroy-env.sh b/xci/files/xci-destroy-env.sh
index 8b811d0f..2e183bd7 100755
--- a/xci/files/xci-destroy-env.sh
+++ b/xci/files/xci-destroy-env.sh
@@ -23,7 +23,7 @@ rm -rf ${HOME}/.config/openstack
 rm -rf ${HOME}/.ansible
 
 # bifrost installs everything on venv so we need to look there if virtualbmc is not installed on the host.
-if which vbmc &>/dev/null || { [[ -e /opt/stack/bifrost/bin/activate ]] && source /opt/stack/bifrost/bin/activate; }; then
+if which vbmc &>/dev/null || { [[ -e ${XCI_VENV}/bifrost/bin/activate ]] && source ${XCI_VENV}/bifrost/bin/activate; }; then
     # Delete all libvirt VMs and hosts from vbmc (look for a port number)
     for vm in $(vbmc list | awk '/[0-9]/{{ print $2 }}'); do
         if which virsh &>/dev/null; then
@@ -81,4 +81,6 @@ service ironic-api restart || true
 service ironic-conductor start || true
 service ironic-inspector restart || true
 
+rm -rf ${XCI_VENV}
+
 # vim: set ts=4 sw=4 expandtab:
diff --git a/xci/infra/bifrost/scripts/bifrost-provision.sh b/xci/infra/bifrost/scripts/bifrost-provision.sh
index 939acb58..eedf350f 100755
--- a/xci/infra/bifrost/scripts/bifrost-provision.sh
+++ b/xci/infra/bifrost/scripts/bifrost-provision.sh
@@ -13,7 +13,8 @@ set -o pipefail
 export PYTHONUNBUFFERED=1
 SCRIPT_HOME="$(cd "$(dirname "$0")" && pwd)"
 BIFROST_HOME=$SCRIPT_HOME/..
-ENABLE_VENV="false"
+ENABLE_VENV="true"
+export VENV=${XCI_VENV}/bifrost
 PROVISION_WAIT_TIMEOUT=${PROVISION_WAIT_TIMEOUT:-3600}
 # This is normally exported by XCI env but we should initialize it here
 # in case we run this script on its own for debug purposes
@@ -93,7 +94,7 @@ if [[ -e ${XCI_PATH}/deployment_image.qcow2 ]]; then
 fi
 
 # Install missing dependencies
-$(which pip) install -q --user --upgrade -r "$(dirname $0)/../requirements.txt"
+pip install -q --upgrade -r "$(dirname $0)/../requirements.txt"
 
 # Change working directory
 cd $BIFROST_HOME/playbooks
diff --git a/xci/xci-deploy.sh b/xci/xci-deploy.sh
index fa9f3351..618377ec 100755
--- a/xci/xci-deploy.sh
+++ b/xci/xci-deploy.sh
@@ -59,6 +59,8 @@ fi
 #-------------------------------------------------------------------------------
 # find where are we
 export XCI_PATH="$(git rev-parse --show-toplevel)"
+# Declare our virtualenv
+export XCI_VENV=${XCI_PATH}/venv/
 # source user vars
 source $XCI_PATH/xci/config/user-vars
 # source pinned versions