diff options
author | Georg Kunz <georg.kunz@est.tech> | 2020-08-24 23:09:51 +0200 |
---|---|---|
committer | Rihab Banday <rihab.banday@ericsson.com> | 2020-09-10 10:29:28 +0000 |
commit | cb3895c81c6931482e688a820013f6926644221d (patch) | |
tree | 82f7aa83cb8e407b6437353686dd6177c7227e54 | |
parent | 42b76853c5ba0ad53aa04ae6cab4bd1de2db7b42 (diff) |
Implementing jump VM create via ansible
This patch reimplements the creation of the jump VM as ansible
playbook. The advantage of this approach is a more deterministic
behavior and the ability to render configuration files using
templates based on the pdf and idf.
Moreover, this patch allows to overwrite the VENDOR variable.
This is necessary to allow deployment in different labs via
CI.
The BMRA configuration has not yet been transformed to ansible
template and needs to follow in subsequent patches.
Signed-off-by: Georg Kunz <georg.kunz@est.tech>
Signed-off-by: Victor Morales <v.morales@samsung.com>
Change-Id: I2cde41dcecec7480bddf71ed864f06244a89f1f3
Reviewed-on: https://gerrit.opnfv.org/gerrit/c/kuberef/+/70882
Tested-by: jenkins-ci <jenkins-opnfv-ci@opnfv.org>
Reviewed-by: Victor Morales <chipahuac@hotmail.com>
-rw-r--r-- | .gitignore | 6 | ||||
-rw-r--r-- | bindep.txt | 5 | ||||
-rw-r--r-- | deploy.env | 2 | ||||
-rwxr-xr-x | deploy.sh | 32 | ||||
-rwxr-xr-x | functions.sh | 131 | ||||
-rw-r--r-- | hw_config/ericsson-pod1/pdf.yaml | 11 | ||||
-rw-r--r-- | hw_config/intel/idf.yaml | 2 | ||||
-rw-r--r-- | inventory/.keep | 0 | ||||
-rw-r--r-- | inventory/group_vars/all/global.yaml | 24 | ||||
-rw-r--r-- | inventory/localhost.ini | 5 | ||||
-rw-r--r-- | playbooks/bootstrap.yaml | 17 | ||||
-rw-r--r-- | playbooks/jump-vm.yaml | 16 | ||||
-rw-r--r-- | playbooks/roles/bootstrap/tasks/main.yaml | 20 | ||||
-rw-r--r-- | playbooks/roles/jump-vm/tasks/main.yaml | 99 | ||||
-rw-r--r-- | playbooks/roles/jump-vm/templates/meta-data.j2 | 1 | ||||
-rw-r--r-- | playbooks/roles/jump-vm/templates/network-config.j2 | 14 | ||||
-rw-r--r-- | playbooks/roles/jump-vm/templates/user-data.j2 | 13 | ||||
-rw-r--r-- | requirements.txt | 2 |
18 files changed, 297 insertions, 103 deletions
@@ -1,3 +1,9 @@ meta-data user-data *.swp +**/.DS_Store +**/._.DS_Store +kuberef/inventory/group_vars/all/idf.yaml +kuberef/inventory/group_vars/all/pdf.yaml +images +workspace diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..f989692 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,5 @@ +python3-all-dev [platform:dpkg] +python3-all [platform:dpkg] +python-all-dev [platform:dpkg] +python-devel [platform:rpm !platform:centos-8] +python3-devel [platform:rpm] @@ -1,6 +1,6 @@ # Define environment variables -export VENDOR=intel +export VENDOR=${VENDOR:-intel} export INSTALLER=bmra # Name of host bridge to which the VM is connected to (used for PXE) @@ -1,7 +1,7 @@ #!/bin/bash # SPDX-license-identifier: Apache-2.0 ############################################################################## -# Copyright (c) +# Copyright (c) Ericsson AB and others # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -16,8 +16,8 @@ set -o nounset # Please refer to README for detailed information. # Get path information -DIRECTORY=$(readlink -f "$0") -CURRENTPATH=$(dirname "$DIRECTORY") +CURRENTPATH=$(git rev-parse --show-toplevel) +export CURRENTPATH # Source env variables & functions # shellcheck source=./deploy.env @@ -30,24 +30,34 @@ source "$CURRENTPATH/functions.sh" # --------------------------------------------------------------------- check_prerequisites -# Clean up leftovers -clean_up +# --------------------------------------------------------------------- +# bootstrap install prerequisites +# --------------------------------------------------------------------- +ansible-playbook -i "$CURRENTPATH"/inventory/localhost.ini \ + "$CURRENTPATH"/playbooks/bootstrap.yaml -# The next two functions require that you know your pxe network configuration -# and IP of resulting jumphost VM in advance. This IP/MAC info also then needs to -# be added in PDF & IDF files (not supported yet via this script) -# Create jumphost VM & setup PXE network -create_jump -setup_PXE_network +# --------------------------------------------------------------------- +# Create jump VM from which the installation is performed +# --------------------------------------------------------------------- +ansible-playbook -i "$CURRENTPATH"/inventory/localhost.ini \ + "$CURRENTPATH"/playbooks/jump-vm.yaml +# --------------------------------------------------------------------- # Copy files needed by Infra engine & BMRA in the jumphost VM +# --------------------------------------------------------------------- copy_files_jump +# --------------------------------------------------------------------- # Provision remote hosts +# --------------------------------------------------------------------- provision_hosts +# --------------------------------------------------------------------- # Setup networking (Adapt according to your network setup) +# --------------------------------------------------------------------- setup_network +# --------------------------------------------------------------------- # Provision k8s cluster (currently BMRA) +# --------------------------------------------------------------------- provision_k8s diff --git a/functions.sh b/functions.sh index 4d75ee1..1229d32 100755 --- a/functions.sh +++ b/functions.sh @@ -1,26 +1,13 @@ #!/bin/bash # SPDX-license-identifier: Apache-2.0 ############################################################################## -# Copyright (c) +# Copyright (c) Ericsson AB and others # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -# Clean up -clean_up() { - if sudo virsh list --all | grep " ${VM_NAME} .*running" ; then - sudo virsh destroy "$VM_NAME" - fi - if sudo virsh list --all | grep " ${VM_NAME} " ; then - sudo virsh undefine "$VM_NAME" - fi - sudo rm -rf "/var/lib/libvirt/images/$VM_NAME" - sleep 5 -} - - check_prerequisites() { echo "Info : Check prerequisites" @@ -69,84 +56,60 @@ check_prerequisites() { echo "ERROR : Libvirt not found. Please install." exit 1 fi -} - -# Create jumphost VM -create_jump() { -# Create VM image - sudo mkdir -p "/var/lib/libvirt/images/$VM_NAME" - sudo qemu-img create -f qcow2 \ - -o backing_file=/var/lib/libvirt/images/ubuntu-18.04.qcow2 \ - "/var/lib/libvirt/images/$VM_NAME/$VM_NAME.qcow2" 10G - -# Create VM cloud-init configuration files - cat <<EOL > user-data - #cloud-config - users: - - name: $USERNAME - ssh-authorized-keys: - - $(cat "$HOME/.ssh/id_rsa.pub") - sudo: ['ALL=(ALL) NOPASSWD:ALL'] - groups: sudo - shell: /bin/bash -EOL - cat <<EOL > meta-data - local-hostname: $VM_NAME -EOL - -# Create VM - sudo genisoimage -output "/var/lib/libvirt/images/$VM_NAME/$VM_NAME-cidata.iso" \ - -volid cidata -joliet -rock user-data meta-data - sudo virt-customize -a "/var/lib/libvirt/images/$VM_NAME/$VM_NAME.qcow2" \ - --root-password password:"$ROOT_PASSWORD" - sudo virt-install --connect qemu:///system --name "$VM_NAME" \ - --ram 4096 --vcpus=4 --os-type linux --os-variant ubuntu16.04 \ - --disk path="/var/lib/libvirt/images/$VM_NAME/$VM_NAME.qcow2",format=qcow2 \ - --disk "/var/lib/libvirt/images/$VM_NAME/$VM_NAME-cidata.iso",device=cdrom \ - --import --network network=default --network bridge="$BRIDGE",model=rtl8139 --noautoconsole - jumpbox_ip=$(get_vm_ip) - i=0 - while [ -z "$jumpbox_ip" ]; do - sleep $((++i)) - jumpbox_ip=$(get_vm_ip) - done - i=0 - until nc -w5 -z "$jumpbox_ip" 22; do - sleep $((++i)) - done + #------------------------------------------------------------------------------- + # Check if user belongs to libvirt's group + #------------------------------------------------------------------------------- + libvirt_group="libvirt" + # shellcheck disable=SC1091 + source /etc/os-release || source /usr/lib/os-release + if [ "${ID,,}" == "ubuntu" ] && [ "$VERSION_ID" == "16.04" ]; then + libvirt_group+="d" + fi + if ! groups | grep " $libvirt_group "; then + echo "ERROR : $(id -nu) user doesn't belong to $libvirt_group group." + exit 1 + fi } # Get jumphost VM IP -get_vm_ip() { - sudo virsh domifaddr "$VM_NAME" | awk 'FNR == 3 {gsub(/\/.*/, ""); print $4}' -} +get_host_pxe_ip() { + local PXE_NETWORK + local PXE_IF_INDEX + local PXE_IF_IP + + host=$1 + if [[ "$host" == "" ]]; then + echo "ERROR : get_ip - host parameter not provided" + exit 1 + fi -# Setup PXE network -setup_PXE_network() { -# Extract configuration from PDF/IDF - PXE_IF=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.pxe_interface) - PXE_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config.oob.interface) - if [[ -z $PXE_IF || -z $PXE_IF_INDEX ]]; then - echo 'one or more variables in IDF are undefined' + PXE_NETWORK=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.pxe_network) + if [[ "$PXE_NETWORK" == "" ]]; then + echo "ERROR : PXE network for jump VM not defined in IDF." exit 1 fi - PXE_IF_IP=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml jumphost.interfaces.["$PXE_IF_INDEX"].address) - PXE_IF_MAC=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml jumphost.interfaces.["$PXE_IF_INDEX"].mac_address) - if [[ -z $PXE_IF_IP || -z $PXE_IF_MAC ]]; then - echo 'one or more variables in PDF are incorrect' + + PXE_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config."$PXE_NETWORK".interface) + if [[ "$PXE_IF_INDEX" == "" ]]; then + echo "ERROR : Index of PXE interface not found in IDF." exit 1 fi - export NETMASK=255.255.255.0 -# SSH to jumphost - # shellcheck disable=SC2087 - ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF -sudo ifconfig $PXE_IF up -sudo ifconfig $PXE_IF $PXE_IF_IP netmask $NETMASK -sudo ifconfig $PXE_IF hw ether $PXE_IF_MAC -EOF + + PXE_IF_IP=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/pdf.yaml "$host".interfaces["$PXE_IF_INDEX"].address) + if [[ "$PXE_IF_IP" == "" ]]; then + echo "ERROR : IP of PXE interface not found in PDF." + exit 1 + fi + echo "$PXE_IF_IP" } +get_vm_ip() { + ip=$(get_host_pxe_ip "jumphost") + echo "$ip" +} + + # Copy files needed by Infra engine & BMRA in the jumphost VM copy_files_jump() { scp -r -o StrictHostKeyChecking=no \ @@ -174,10 +137,8 @@ EOF # Setup networking on provisioned hosts (Adapt setup_network.sh according to your network setup) setup_network() { -# Extract IPs of provisioned nodes from PDF/IDF. When running this function standalone, ensure -# to set $PXE_IF_INDEX - MASTER_IP=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml nodes.[0].interfaces.["$PXE_IF_INDEX"].address) - WORKER_IP=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml nodes.[1].interfaces.["$PXE_IF_INDEX"].address) + MASTER_IP=$(get_host_pxe_ip "nodes[0]") + WORKER_IP=$(get_host_pxe_ip "nodes[1]") # SSH to jumphost # shellcheck disable=SC2087 ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF diff --git a/hw_config/ericsson-pod1/pdf.yaml b/hw_config/ericsson-pod1/pdf.yaml index 48933f1..7006788 100644 --- a/hw_config/ericsson-pod1/pdf.yaml +++ b/hw_config/ericsson-pod1/pdf.yaml @@ -9,12 +9,12 @@ ############################################################################## version: 1.0 details: - pod_owner: Nordix - contact: infra@nordix.org - lab: Nordix OpenLab - location: N/A + pod_owner: Ericsson + contact: georg.kunz@ericsson.com + lab: OPNFV lab + location: Sweden type: baremetal - link: http://wiki.nordix.org/ + link: N/A ############################################################################## jumphost: name: kuberef-jump @@ -40,6 +40,7 @@ jumphost: mac_address: 52:54:00:4a:e8:2d - name: 'ens3' address: 100.64.200.254 + mac_address: 52:54:00:4a:e8:2f ############################################################################## nodes: - name: node1 diff --git a/hw_config/intel/idf.yaml b/hw_config/intel/idf.yaml index 328fe09..ca5e4f2 100644 --- a/hw_config/intel/idf.yaml +++ b/hw_config/intel/idf.yaml @@ -53,7 +53,7 @@ engine: pxe_network: oob # net_config network to be used for the internet access - public_network: oob + public_network: public # interface to be used by the PXE pxe_interface: ens4 diff --git a/inventory/.keep b/inventory/.keep new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/inventory/.keep diff --git a/inventory/group_vars/all/global.yaml b/inventory/group_vars/all/global.yaml new file mode 100644 index 0000000..312fcda --- /dev/null +++ b/inventory/group_vars/all/global.yaml @@ -0,0 +1,24 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) Ericsson AB and others +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# root directory of kuberef repository +kuberef_root: "{{ lookup('env', 'CURRENTPATH') }}" + +# path to inventory folder where the initial +inventory_path: "{{ kuberef_root }}/inventory" + +# workspace where all work-in-progress files get stored +workspace: "{{ kuberef_root }}/workspace" + +# directory for storing base images acting as backing file +images_path: "{{ kuberef_root }}/images" + +# public SSH key for use by kuberef installation +pub_key: "{{ lookup('env', 'HOME') }}/.ssh/id_rsa.pub" diff --git a/inventory/localhost.ini b/inventory/localhost.ini new file mode 100644 index 0000000..72f71a5 --- /dev/null +++ b/inventory/localhost.ini @@ -0,0 +1,5 @@ +[all:vars] +ansible_ssh_extra_args=' -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' + +[localhost] +127.0.0.1 ansible_connection=local diff --git a/playbooks/bootstrap.yaml b/playbooks/bootstrap.yaml new file mode 100644 index 0000000..4ef2baa --- /dev/null +++ b/playbooks/bootstrap.yaml @@ -0,0 +1,17 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) Ericsson AB and others +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: localhost + connection: local + gather_facts: false + become: false + + roles: + - role: bootstrap diff --git a/playbooks/jump-vm.yaml b/playbooks/jump-vm.yaml new file mode 100644 index 0000000..bdad81c --- /dev/null +++ b/playbooks/jump-vm.yaml @@ -0,0 +1,16 @@ +--- +############################################################################## +# Copyright (c) Ericsson AB and others +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: localhost + connection: local + gather_facts: false + become: false + + roles: + - role: jump-vm diff --git a/playbooks/roles/bootstrap/tasks/main.yaml b/playbooks/roles/bootstrap/tasks/main.yaml new file mode 100644 index 0000000..ff20552 --- /dev/null +++ b/playbooks/roles/bootstrap/tasks/main.yaml @@ -0,0 +1,20 @@ +--- +############################################################################## +# Copyright (c) Ericsson AB and others +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# collected PDF and IDF are copied into group_vars +# folder in order to ensure we have single PDF and IDF +- name: Copy collected PDF and IDF to group_vars + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + force: true + mode: 0644 + with_items: + - {src: "{{ kuberef_root }}/hw_config/{{ lookup('env', 'VENDOR') }}/pdf.yaml", dest: "{{ inventory_path }}/group_vars/all/pdf.yaml"} + - {src: "{{ kuberef_root }}/hw_config/{{ lookup('env', 'VENDOR') }}/idf.yaml", dest: "{{ inventory_path }}/group_vars/all/idf.yaml"} diff --git a/playbooks/roles/jump-vm/tasks/main.yaml b/playbooks/roles/jump-vm/tasks/main.yaml new file mode 100644 index 0000000..f5f6154 --- /dev/null +++ b/playbooks/roles/jump-vm/tasks/main.yaml @@ -0,0 +1,99 @@ +--- +############################################################################## +# Copyright (c) Ericsson AB and others +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: get all running VMs + virt: + command: list_vms + state: running + register: running_vms + +- name: shutdown existing jump VM + virt: + name: "{{ jumphost.name }}" + command: destroy + when: + jumphost.name in running_vms.list_vms + +- name: get all shutdown VMs + virt: + command: list_vms + state: shutdown + register: shutdown_vms + +- name: undefine existing jump VM + virt: + name: "{{ jumphost.name }}" + command: undefine + when: + jumphost.name in shutdown_vms.list_vms + +- name: clean workspace + file: + path: "{{ workspace }}" + state: absent + +- name: create workspace if it does not exist + file: + path: "{{ workspace }}" + state: directory + mode: '0755' + +- name: create directory for base images + file: + path: "{{ images_path }}" + state: directory + mode: '0755' + +- name: download Ubuntu image for jump VM + get_url: + url: https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img + dest: "{{ images_path }}/bionic-server-cloudimg-amd64.img" + force: false + mode: '0666' + +- name: create new VM image from base image + command: "qemu-img create -f qcow2 -o backing_file={{ images_path }}/bionic-server-cloudimg-amd64.img {{ workspace }}/{{ jumphost.name }}.qcow2 10G" + +- name: render config files for jump VM from templates + template: + src: "{{ kuberef_root }}/playbooks/roles/jump-vm/templates/{{ item }}.j2" + dest: "{{ workspace }}/{{ item }}" + mode: 0644 + with_items: + - network-config + - user-data + - meta-data + +- name: create config drive + command: "genisoimage -output {{ workspace }}/{{ jumphost.name }}-cidata.iso -volid cidata -joliet -rock \ + {{ workspace }}/user-data {{ workspace }}/meta-data {{ workspace }}/network-config" + +# currently commented out because of portability issues between Centos and Ubuntu +# - name: setting root password for debugging +# become: true +# command: "virt-customize -a {{ workspace }}/{{ jumphost.name }}.qcow2 --root-password password:'root'" + +- name: define jump VM + command: "virt-install --connect qemu:///system --name {{ jumphost.name }} \ + --ram 4096 --vcpus=4 --os-type linux --os-variant ubuntu16.04 \ + --disk path={{ workspace }}/kuberef-jump.qcow2,format=qcow2 \ + --disk {{ workspace }}/kuberef-jump-cidata.iso,device=cdrom \ + --network network=default,model=virtio,mac='{{ jumphost.interfaces[engine.net_config[engine.public_network].interface].mac_address }}' \ + --network bridge=pxebr,model=virtio,mac='{{ jumphost.interfaces[engine.net_config[engine.pxe_network].interface].mac_address }}' \ + --import --noautoconsole" + +- name: start jump VM + virt: + name: "{{ jumphost.name }}" + state: running + +- name: wait for VM to be reachable + wait_for: + host: "{{ jumphost.interfaces[idf.net_config[engine.pxe_network].interface].address }}" + port: 22 diff --git a/playbooks/roles/jump-vm/templates/meta-data.j2 b/playbooks/roles/jump-vm/templates/meta-data.j2 new file mode 100644 index 0000000..b4cc5e8 --- /dev/null +++ b/playbooks/roles/jump-vm/templates/meta-data.j2 @@ -0,0 +1 @@ +local-hostname: {{ jumphost.name }} diff --git a/playbooks/roles/jump-vm/templates/network-config.j2 b/playbooks/roles/jump-vm/templates/network-config.j2 new file mode 100644 index 0000000..ceded54 --- /dev/null +++ b/playbooks/roles/jump-vm/templates/network-config.j2 @@ -0,0 +1,14 @@ +version: 1 +config: + - type: physical + name: "{{ jumphost.interfaces[idf.net_config[engine.public_network].interface].name }}" + mac_address: "{{ jumphost.interfaces[idf.net_config[engine.public_network].interface].mac_address }}" + subnets: + - type: dhcp + - type: physical + name: "{{ jumphost.interfaces[idf.net_config[engine.pxe_network].interface].name }}" + mac_address: "{{ jumphost.interfaces[idf.net_config[engine.pxe_network].interface].mac_address }}" + subnets: + - type: static + address: "{{ jumphost.interfaces[idf.net_config[engine.pxe_network].interface].address }}" + netmask: "{{ idf.net_config[engine.pxe_network].mask }}" diff --git a/playbooks/roles/jump-vm/templates/user-data.j2 b/playbooks/roles/jump-vm/templates/user-data.j2 new file mode 100644 index 0000000..648f8d1 --- /dev/null +++ b/playbooks/roles/jump-vm/templates/user-data.j2 @@ -0,0 +1,13 @@ + #cloud-config + users: + - name: ubuntu + ssh-authorized-keys: + - {{ lookup('file', pub_key ) }} + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + groups: sudo + shell: /bin/bash + runcmd: + # this is requried in labs where the PXE network is different from + # the public network. Without internet connectivity, the installation + # of BMRA fails + - [ iptables, -t, nat, -A, POSTROUTING, -o, ens3, -j, MASQUERADE ] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c1409de --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +ansible +libvirt-python |