xci: scripts: Add script to use clean virtual machines for testing

These scripts will prepare a clean OS image based on diskimage-builder
and launch a new VM with it. The purpose of that is to use this virtual
machine as a 'host' to run XCI on it. This way we can ensure that all
tests are being executed on the same clean environment which is ideal
for reproducing and debugging purposes. We also commit the ssh keypair
so we can build it in the image and use it to control the virtual
machine without user interaction. Needless to say that this keypair MUST
NOT be used for anything else apart from the dib build process.

Change-Id: Idec0fc06c82435adc1a4d569b8e578616998de5f
Signed-off-by: Markos Chandras <mchandras@suse.de>

5 files changed, 286 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
index a7cb9e3f..7c952f34 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,3 +36,8 @@ testapi_venv/
 .tox
 *.retry
 job_output/
+# Clear VM files
+*.qcow2
+*.sha256.txt
+build.log
+*.d/
diff --git a/xci/scripts/vm/build-dib-os.sh b/xci/scripts/vm/build-dib-os.sh
new file mode 100755
index 00000000..47d0a905
--- /dev/null
+++ b/xci/scripts/vm/build-dib-os.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2017 SUSE LINUX GmbH.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -e
+
+# This only works on ubuntu hosts
+lsb_release -i | grep -q -i ubuntu || { echo "This script only works on Ubuntu distros"; exit 1; }
+
+declare -A flavors=( ["ubuntu-minimal"]="xenial" ["opensuse-minimal"]="42.3" ["centos-minimal"]="7" )
+elements="vm simple-init devuser growroot openssh-server"
+declare -r one_distro=${1}
+if [[ -n ${one_distro} ]]; then
+	case ${one_distro} in
+		centos|ubuntu|opensuse) : ;;
+		*) echo "unsupported distribution"; exit 1 ;;
+	esac
+fi
+
+# devuser logins
+echo "Configuring devuser..."
+export DIB_DEV_USER_USERNAME=devuser
+export DIB_DEV_USER_PWDLESS_SUDO=1
+export DIB_DEV_USER_AUTHORIZED_KEYS=$HOME/.ssh/id_rsa_for_dib.pub
+export DIB_DEV_USER_PASSWORD=linux
+
+echo "Installing base dependencies..."
+sudo apt-get install -y -q=3 yum yum-utils rpm zypper kpartx python-pip debootstrap gnupg2
+
+echo "Installing diskimage-builder"
+
+sudo -H pip install -q -U diskimage-builder
+
+echo "Removing old files..."
+sudo rm -rf *.qcow2 *.sha256.txt
+
+do_build() {
+	local image=${1}-minimal
+	local image_name=${1}
+	echo "Building ${image}-${flavors[$image]}..."
+	export DIB_RELEASE=${flavors[$image]}
+	# Some defaults
+	export DIB_YUM_MINIMAL_CREATE_INTERFACES=1 # centos dhcp setup
+	disk-image-create --no-tmpfs -o ${image_name}.qcow2 ${elements} $image
+	sha256sum ${image_name}.qcow2 > ${image_name}.sha256.txt
+	echo "Done!"
+}
+
+if [[ -n ${one_distro} ]]; then
+	do_build ${one_distro}
+else
+	for image in "${!flavors[@]}"; do
+		image_name=${image/-minimal}
+		do_build $image_name
+	done
+fi
+
+exit 0
diff --git a/xci/scripts/vm/id_rsa_for_dib b/xci/scripts/vm/id_rsa_for_dib
new file mode 100644
index 00000000..bbc2ec58
--- /dev/null
+++ b/xci/scripts/vm/id_rsa_for_dib
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAwJboPeJys/zK6orZUwnwx9seICIk47HTA3fE2RuhmVHYaEpw
+/wi+0NNtxBh79Bjrxcd2psYTNHF9odktc7iNdGABNhsBk7zYlIT4VzYeG41eTy2J
+4C5ny+ArbMj4XCnFq5nlf1wiIjqOk4tvTMXfoHvx73biN6h2O62JrzAljzgDMEmm
+TO3W1HN6dfFNKc3DQfYiyLKLOy4CXuEbvq6e4o32lz3ylNZEAJNQ1Qfo2cfuWALJ
+B8TWEYL9Oxf2XM5w5Wll8bLzLoQlVzkw2TXFGCcWOZczRCREEzZqOhOOtkLqVTZD
++rFlwMAYqbHQMuYd+dC5zOwwYI5Nmjwe3xsWPwIDAQABAoIBAHvrXeob4ARR/1rl
+FqsotUZThlHUxnm//O+5S+8KqXNWx3FrnMNtJCKIOJO50WAh/OsYyg7LWqYe9GCE
+totET7h24bCntd8S3yTJXmX3S24dC3bU7WdTVLRJPIfV1ICw3CDreFD3vnrZxM3J
+JUd39slzlxunGi4H9nO12ObhivlYSZyBryf9WhSdveKV0bfKT+wiZBu786Qv7kad
+6JYm7WTK1qio6szt3Yqs9sIAolJBDK8bOH6fBNYgT/U6vWTS3MX4xH9sTZt8QUWP
+80J5HInYEatQDAJ4GQgLBrC53+UKqKuvqM1ZRddjnpqbkBph2KJ+yeR51HAhZDuE
+2ufgp9ECgYEA739wjbV+Fp2Xpyz5uldq220j5JvoiKP+g6GJwaPmnwg8ZVNxHxrV
+XfrjKGcCfE22pwReiyFW5iEHKDFJnvH2UZ6fAS7I8wtZ/kK6/SIerXDmYawqKy65
+sVAdwE6kIfICfnSxjKyZG6lGjRXfd7JiWCeOWbFyEFHkt/Nw2QBPJvcCgYEAzdwK
+QOKv7MlP3zTD5Z4u6rsADT5Tynu5SZrET0SR0GrNcIi5fmtc3MRbaCMWOFCHX90u
+s4E7Y5ArXoCgMYaovGJP939uoRy8i6/G4gFuXDejINKyER61xoPQWZFjuo1qrprz
+dPBccfwr6vXrO1Ho0LEruQ4kI4evYUd5yChXUPkCgYA5vyaqZIqEdXctjgCg15ci
+VUyEQLtuxuwFXRvUHurqiZfWbqol79hmkqnXAg9tXWefGcSaE23aizgojsnLk1fr
+Ei4QoPE++Qu37OsdQe7XrxM9lb+NLfUENecWiV+19552UzjETJGWQ9iJCC8bs11X
+GCKQqeJw4O4fFImvKQ3svQKBgQCcaXr08AhhblgcrRfHKJhXS3xvk5LQv476CPjt
+jZkI9Y1g2tGt519ocl1NU5N3eGBTi1gHlcKNFKaPKEYPO1SLxlsiHxXsnkv4Cref
+Ux0KPk/ZCZ24ek/9JHwzAc/3l6mePyeKC3MCPGsWHIH90W+G2YbyFXeJ83di/47d
+cSFe6QKBgFvN6woRxXJVcfDZ0dSo0PiG5Td0oYq6l7I9k9IbsXXUe1WHhiN4FECE
+PLA4ChVGINpQ6MKJF6SvY3ufEE28SV9jhswrg3qW/uKtaLiIbz/AgadijVJnxeQl
+d9b3OoUpN199cOz0EVul7jNHJdcElrr447tRBNrb41IwLEGEIqM+
+-----END RSA PRIVATE KEY-----
diff --git a/xci/scripts/vm/id_rsa_for_dib.pub b/xci/scripts/vm/id_rsa_for_dib.pub
new file mode 100644
index 00000000..33c66c86
--- /dev/null
+++ b/xci/scripts/vm/id_rsa_for_dib.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAlug94nKz/MrqitlTCfDH2x4gIiTjsdMDd8TZG6GZUdhoSnD/CL7Q023EGHv0GOvFx3amxhM0cX2h2S1zuI10YAE2GwGTvNiUhPhXNh4bjV5PLYngLmfL4CtsyPhcKcWrmeV/XCIiOo6Ti29Mxd+ge/HvduI3qHY7rYmvMCWPOAMwSaZM7dbUc3p18U0pzcNB9iLIsos7LgJe4Ru+rp7ijfaXPfKU1kQAk1DVB+jZx+5YAskHxNYRgv07F/ZcznDlaWXxsvMuhCVXOTDZNcUYJxY5lzNEJEQTNmo6E462QupVNkP6sWXAwBipsdAy5h350LnM7DBgjk2aPB7fGxY/ jenkins@pod20-node4
diff --git a/xci/scripts/vm/start-new-vm.sh b/xci/scripts/vm/start-new-vm.sh
new file mode 100755
index 00000000..5101fc9f
--- /dev/null
+++ b/xci/scripts/vm/start-new-vm.sh
@@ -0,0 +1,190 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2017 SUSE LINUX GmbH.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+set -e
+
+lsb_release -i | grep -q -i ubuntu || { echo "This script only works on Ubuntu distros"; exit 1; }
+
+export DEFAULT_XCI_TEST=${DEFAULT_XCI_TEST:-false}
+
+grep -q -i ^Y$ /sys/module/kvm_intel/parameters/nested || { echo "Nested virtualization is not enabled but it's needed for XCI to work"; exit 1; }
+
+usage() {
+	echo """
+	$0 <distro>
+
+	distro must be one of 'ubuntu', 'opensuse', 'centos'
+	"""
+}
+
+[[ $# -ne 1 ]] && usage && exit 1
+
+declare -r CPU=host
+declare -r NCPUS=24
+declare -r MEMORY=49152
+declare -r DISK=500
+declare -r NAME=${1}
+declare -r NETWORK="jenkins-test"
+declare -r BASE_PATH=$(dirname $(readlink -f $0) | sed "s@/xci.*@@")
+
+echo "Preparing new virtual machine '${NAME}'..."
+
+# NOTE(hwoarang) This should be removed when we move the dib images to a central place
+echo "Building '${NAME}' image (tail build.log for progress and failures)..."
+$BASE_PATH/xci/scripts/vm/build-dib-os.sh ${NAME} > build.log 2>&1
+
+[[ ! -e ${1}.qcow2 ]] && echo "${1}.qcow2 not found! This should never happen!" && exit 1
+
+sudo apt-get install -y -q=3 virt-manager qemu-kvm libvirt-bin qemu-utils
+sudo systemctl -q start libvirtd
+
+echo "Resizing disk image '${NAME}' to ${DISK}G..."
+qemu-img resize ${NAME}.qcow2 ${DISK}G
+
+echo "Creating new network '${NETWORK}' if it does not exist already..."
+if ! sudo virsh net-list --name | grep -q ${NETWORK}; then
+	cat > /tmp/${NETWORK}.xml <<EOF
+<network>
+  <name>${NETWORK}</name>
+  <forward mode='nat'>
+    <nat>
+      <port start='1024' end='65535'/>
+    </nat>
+  </forward>
+  <bridge name='jenkins_br0' std='off' delay='0'/>
+  <ip address='192.168.140.1' netmask='255.255.255.0'>
+    <dhcp>
+      <range start='192.168.140.2' end='192.168.140.254'/>
+    </dhcp>
+  </ip>
+</network>
+EOF
+	sudo virsh net-define /tmp/${NETWORK}.xml
+	sudo virsh net-autostart ${NETWORK}
+	sudo virsh net-start ${NETWORK}
+fi
+
+echo "Destroying previous instances if necessary..."
+sudo virsh destroy ${NAME} || true
+sudo virsh undefine ${NAME} || true
+
+echo "Installing virtual machine '${NAME}'..."
+sudo virt-install -n ${NAME} --memory ${MEMORY} --vcpus ${NCPUS} --cpu ${CPU} \
+	--import --disk=${NAME}.qcow2 --network network=${NETWORK} \
+	--graphics none --hvm --noautoconsole
+
+_retries=30
+while [[ $_retries -ne 0 ]]; do
+	_ip=$(sudo virsh domifaddr ${NAME} | grep -o --colour=never 192.168.140.[[:digit:]]* | cat )
+	if [[ -z ${_ip} ]]; then
+		echo "Waiting for '${NAME}' virtual machine to boot ($_retries retries left)..."
+		sleep 5
+		(( _retries = _retries - 1 ))
+	else
+		break
+	fi
+done
+[[ -n $_ip ]] && echo "'${NAME}' virtual machine is online at $_ip"
+[[ -z $_ip ]] && echo "'${NAME}' virtual machine did not boot on time" && exit 1
+
+# Fix up perms if needed to make ssh happy
+chmod 600 ${BASE_PATH}/xci/scripts/vm/id_rsa_for_dib*
+# Remove it from known_hosts
+ssh-keygen -R $_ip || true
+ssh-keygen -R ${NAME}_xci_vm || true
+
+declare -r vm_ssh="ssh -o StrictHostKeyChecking=no -i ${BASE_PATH}/xci/scripts/vm/id_rsa_for_dib -l devuser"
+
+_retries=30
+_ssh_exit=0
+
+echo "Verifying operational status..."
+while [[ $_retries -ne 0 ]]; do
+	if eval $vm_ssh $_ip "sudo cat /etc/os-release" 2>/dev/null; then
+		_ssh_exit=$?
+		break;
+	else
+		_ssh_exit=$?
+		sleep 5
+		(( _retries = _retries - 1 ))
+	fi
+done
+[[ $_ssh_exit != 0 ]] && echo "Failed to SSH to the virtual machine '${NAME}'! This should never happen!" && exit 1
+
+echo "Congratulations! Your shiny new '${NAME}' virtual machine is fully operational! Enjoy!"
+
+echo "Adding ${NAME}_xci_vm entry to /etc/hosts"
+sudo sed -i "/.*${NAME}_xci_vm.*/d" /etc/hosts
+sudo bash -c "echo '${_ip} ${NAME}_xci_vm' >> /etc/hosts"
+
+echo "Dropping a minimal .ssh/config file"
+cat > $HOME/.ssh/config<<EOF
+Host *
+StrictHostKeyChecking no
+IdentityFile ${BASE_PATH}/xci/scripts/vm/id_rsa_for_dib
+
+Host *_xci_vm
+User devuser
+
+Host *_xci_vm_opnfv
+User root
+TCPKeepAlive yes
+StrictHostKeyChecking no
+ProxyCommand ssh -l devuser \$(echo %h | sed 's/_opnfv//') 'nc 192.168.122.2 %p'
+EOF
+
+echo "Preparing test environment..."
+# Start with good dns
+$vm_ssh $_ip 'sudo bash -c "echo nameserver 8.8.8.8 > /etc/resolv.conf"'
+$vm_ssh $_ip 'sudo bash -c "echo nameserver 8.8.4.4 >> /etc/resolv.conf"'
+# Need to copy releng-xci to the vm so we can execute stuff
+do_copy() {
+	rsync -a \
+		--exclude "${NAME}*" \
+		--exclude "build.log" \
+		-e "$vm_ssh" ${BASE_PATH} $_ip:~/
+}
+
+do_copy
+# Copy keypair
+$vm_ssh $_ip "cp --preserve=all ~/releng-xci/xci/scripts/vm/id_rsa_for_dib /home/devuser/.ssh/id_rsa"
+$vm_ssh $_ip "cp --preserve=all ~/releng-xci/xci/scripts/vm/id_rsa_for_dib.pub /home/devuser/.ssh/id_rsa.pub"
+
+set +e
+
+_has_test=true
+echo "Verifying test script exists..."
+$vm_ssh $_ip "bash -c 'stat ~/$(basename ${BASE_PATH})/run_jenkins_test.sh'"
+if [[ $? != 0 ]]; then
+	echo "Failed to find a 'run_jenkins_test.sh' script..."
+	if ${DEFAULT_XCI_TEST}; then
+		echo "Creating a default test case to run xci-deploy.sh"
+		cat > ${BASE_PATH}/run_jenkins_test.sh <<EOF
+#!/bin/bash
+cd releng-xci/xci
+./xci-deploy.sh
+EOF
+		# Copy again
+		do_copy
+	else
+		_has_test=false
+	fi
+fi
+
+if ${_has_test}; then
+	echo "Running test..."
+	$vm_ssh $_ip "bash ~/$(basename ${BASE_PATH})/run_jenkins_test.sh"
+	xci_error=$?
+else
+	echo "No jenkins test was found. The virtual machine will remain idle!"
+	xci_error=0
+fi
+
+exit $xci_error