diff options
| -rwxr-xr-x | xci/scripts/vm/build-dib-os.sh | 43 | ||||
| -rwxr-xr-x | xci/scripts/vm/start-new-vm.sh | 106 |
2 files changed, 110 insertions, 39 deletions
diff --git a/xci/scripts/vm/build-dib-os.sh b/xci/scripts/vm/build-dib-os.sh new file mode 100755 index 00000000..7547d40e --- /dev/null +++ b/xci/scripts/vm/build-dib-os.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2017 SUSE LINUX GmbH. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +ONE_DISTRO=${1:-} + +docker_cmd="sudo docker" +# See if we can run docker as regular user. +docker ps &> /dev/null && docker_cmd="docker" +docker_name="docker_xci_builder_${ONE_DISTRO:-all}" + +# Destroy previous containers +if eval $docker_cmd ps -a | grep -q ${docker_name} &>/dev/null; then + echo "Destroying previous container..." + eval $docker_cmd rm -f ${docker_name} +fi + +# Prepare new working directory +dib_workdir="${XCI_CACHE_DIR:-${HOME}/.cache/opnfv_xci_deploy}/clean_vm/images" +[[ ! -d $dib_workdir ]] && mkdir -p $dib_workdir +chmod 777 -R $dib_workdir + +# Record our information +uid=$(id -u) +gid=$(id -g) + +echo "Getting the latest docker image..." +eval $docker_cmd pull hwoarang/docker-dib-xci:latest + +# Get rid of stale files +rm -rf $dib_workdir/*.qcow2 $dib_workdir/*.sha256.txt $dib_workdir/*.d +echo "Initiating dib build..." +eval $docker_cmd run --name ${docker_name} \ + --rm --privileged=true -e ONE_DISTRO=${ONE_DISTRO} \ + -t -v $dib_workdir:$dib_workdir -w $dib_workdir \ + hwoarang/docker-dib-xci '/usr/bin/do-build.sh' +sudo chown $uid:$gid $dib_workdir/${ONE_DISTRO}.qcow2 diff --git a/xci/scripts/vm/start-new-vm.sh b/xci/scripts/vm/start-new-vm.sh index 673d2b70..4ad41f64 100755 --- a/xci/scripts/vm/start-new-vm.sh +++ b/xci/scripts/vm/start-new-vm.sh @@ -16,6 +16,11 @@ set -e export JENKINS_HOME="${JENKINS_HOME:-${HOME}}" export DEFAULT_XCI_TEST=${DEFAULT_XCI_TEST:-false} +# JIT Build of OS image to load on the clean VM +export XCI_BUILD_CLEAN_VM_OS=${XCI_BUILD_CLEAN_VM_OS:-true} +# Use cached (possibly outdated) images instead of always using the latest +# ones. +export XCI_UPDATE_CLEAN_VM_OS=${XCI_UPDATE_CLEAN_VM_OS:-false} grep -q -i ^Y$ /sys/module/kvm_intel/parameters/nested || { echo "Nested virtualization is not enabled but it's needed for XCI to work"; exit 1; } @@ -27,6 +32,42 @@ usage() { """ } +update_clean_vm_files() { + local opnfv_url="http://artifacts.opnfv.org/releng/xci/images" + local vm_cache=${XCI_CACHE_DIR}/clean_vm/images + local sha_local shafile=${vm_cache}/${OS}.qcow2.sha256.txt + local sha_remote="${opnfv_url}/${OS}.qcow2.sha256.txt" + local image_remote="${opnfv_url}/${OS}.qcow2" + + get_new_vm_files() { + rm -rf ${vm_cache}/${OS}* + wget --quiet ${image_remote} + wget --quiet ${sha_remote} + } + + # There are 3 reasons why we want to fetch files from the GS storage + # 1) We don't have a shafile locally (clean cache?) + # 2) We have one but it's not the latest one + # 3) We have one but the qcow2 is corrupted + cd ${vm_cache} + if [[ -e ${shafile} ]]; then + echo "Found local ${OS} files in cache..." + sha_local=$(awk '{print $1}' $shafile) + if $XCI_UPDATE_CLEAN_VM_OS; then + echo "Updating local copies of ${OS}..." + ! curl -s ${sha_remote} | grep -q ${sha_local} && \ + get_new_vm_files + fi + echo "Verifying integrity of ${OS} files..." + ! sha256sum --status -c $shafile && get_new_vm_files + else + get_new_vm_files + fi + echo "Final integrity check of ${OS} files..." + sha256sum --status -c $shafile + cd - &> /dev/null +} + [[ $# -ne 1 ]] && usage && exit 1 declare -r CPU=host @@ -37,6 +78,7 @@ declare -r VM_NAME=${1}_xci_vm declare -r OS=${1} declare -r NETWORK="jenkins-test" declare -r BASE_PATH=$(dirname $(readlink -f $0) | sed "s@/xci/.*@@") +declare -r XCI_CACHE_DIR=${HOME}/.cache/opnfv_xci_deploy echo "Preparing new virtual machine '${VM_NAME}'..." @@ -66,45 +108,32 @@ echo "Ensuring libvirt and docker services are running..." sudo systemctl -q start libvirtd sudo systemctl -q start docker -echo "Building new ${OS} image..." - -_retries=20 -while [[ $_retries -gt 0 ]]; do - if pgrep -a docker | grep -q docker-dib-xci &> /dev/null; then - echo "There is another dib process running... ($_retries retries left)" - sleep 60 - (( _retries = _retries - 1 )) - else - docker_cmd="sudo docker" - # See if we can run docker as regular user. - docker ps &> /dev/null && docker_cmd="docker" - docker_name="docker_xci_builder_${OS}" - # Destroy previous containers - if eval $docker_cmd ps -a | grep -q ${docker_name} &>/dev/null; then - echo "Destroying previous container..." - eval $docker_cmd rm -f ${docker_name} +echo "Preparing XCI cache..." +mkdir -p ${XCI_CACHE_DIR}/ ${XCI_CACHE_DIR}/clean_vm/images/ + +if ${XCI_BUILD_CLEAN_VM_OS}; then + echo "Building new ${OS} image..." + _retries=20 + while [[ $_retries -gt 0 ]]; do + if pgrep -a docker | grep -q docker-dib-xci &> /dev/null; then + echo "There is another dib process running... ($_retries retries left)" + sleep 60 + (( _retries = _retries - 1 )) + else + ${BASE_PATH}/xci/scripts/vm/build-dib-os.sh ${OS} + break fi - # Prepare new working directory - dib_workdir="$(pwd)/docker_dib_xci_workdir" - [[ ! -d $dib_workdir ]] && mkdir $dib_workdir - chmod 777 -R $dib_workdir - uid=$(id -u) - gid=$(id -g) - # Get rid of stale files - rm -rf $dib_workdir/*.qcow2 $dib_workdir/*.sha256.txt $dib_workdir/*.d - echo "Getting the latest docker image..." - eval $docker_cmd pull hwoarang/docker-dib-xci:latest - echo "Initiating dib build..." - eval $docker_cmd run --name ${docker_name} \ - --rm --privileged=true -e ONE_DISTRO=${OS} \ - -t -v $dib_workdir:$dib_workdir -w $dib_workdir \ - hwoarang/docker-dib-xci '/usr/bin/do-build.sh' - sudo chown $uid:$gid $dib_workdir/${OS}.qcow2 - declare -r OS_IMAGE_FILE=$dib_workdir/${OS}.qcow2 + done +else + echo "Retrieving ${OS} image files..." + update_clean_vm_files +fi - break - fi -done +# Doesn't matter if we just built an image or got one from artifacts. In both +# cases there should be a copy in the cache so copy it over. +sudo rm -f ${BASE_PATH}/${OS}.qcow2 +cp ${XCI_CACHE_DIR}/clean_vm/images/${OS}.qcow2 ${BASE_PATH}/ +declare -r OS_IMAGE_FILE=${OS}.qcow2 [[ ! -e ${OS_IMAGE_FILE} ]] && echo "${OS_IMAGE_FILE} not found! This should never happen!" && exit 1 @@ -227,9 +256,8 @@ do_copy() { rsync -a \ --exclude "${VM_NAME}*" \ --exclude "${OS}*" \ - --exclude "$dib_workdir*" \ --exclude "build.log" \ - -e "$vm_ssh" ${BASE_PATH}/* $_ip:~/releng-xci/ + -e "$vm_ssh" ${BASE_PATH}/ $_ip:~/releng-xci/ } do_copy |