diff options
| -rw-r--r-- | .gitignore | 6 | ||||
| -rw-r--r-- | bindep.txt | 5 | ||||
| -rw-r--r-- | deploy.env | 2 | ||||
| -rwxr-xr-x | deploy.sh | 32 | ||||
| -rwxr-xr-x | functions.sh | 131 | ||||
| -rw-r--r-- | hw_config/ericsson-pod1/pdf.yaml | 11 | ||||
| -rw-r--r-- | hw_config/intel/idf.yaml | 2 | ||||
| -rw-r--r-- | inventory/.keep | 0 | ||||
| -rw-r--r-- | inventory/group_vars/all/global.yaml | 24 | ||||
| -rw-r--r-- | inventory/localhost.ini | 5 | ||||
| -rw-r--r-- | playbooks/bootstrap.yaml | 17 | ||||
| -rw-r--r-- | playbooks/jump-vm.yaml | 16 | ||||
| -rw-r--r-- | playbooks/roles/bootstrap/tasks/main.yaml | 20 | ||||
| -rw-r--r-- | playbooks/roles/jump-vm/tasks/main.yaml | 99 | ||||
| -rw-r--r-- | playbooks/roles/jump-vm/templates/meta-data.j2 | 1 | ||||
| -rw-r--r-- | playbooks/roles/jump-vm/templates/network-config.j2 | 14 | ||||
| -rw-r--r-- | playbooks/roles/jump-vm/templates/user-data.j2 | 13 | ||||
| -rw-r--r-- | requirements.txt | 2 |
18 files changed, 297 insertions, 103 deletions
@@ -1,3 +1,9 @@ meta-data user-data *.swp +**/.DS_Store +**/._.DS_Store +kuberef/inventory/group_vars/all/idf.yaml +kuberef/inventory/group_vars/all/pdf.yaml +images +workspace diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 0000000..f989692 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,5 @@ +python3-all-dev [platform:dpkg] +python3-all [platform:dpkg] +python-all-dev [platform:dpkg] +python-devel [platform:rpm !platform:centos-8] +python3-devel [platform:rpm] @@ -1,6 +1,6 @@ # Define environment variables -export VENDOR=intel +export VENDOR=${VENDOR:-intel} export INSTALLER=bmra # Name of host bridge to which the VM is connected to (used for PXE) @@ -1,7 +1,7 @@ #!/bin/bash # SPDX-license-identifier: Apache-2.0 ############################################################################## -# Copyright (c) +# Copyright (c) Ericsson AB and others # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -16,8 +16,8 @@ set -o nounset # Please refer to README for detailed information. # Get path information -DIRECTORY=$(readlink -f "$0") -CURRENTPATH=$(dirname "$DIRECTORY") +CURRENTPATH=$(git rev-parse --show-toplevel) +export CURRENTPATH # Source env variables & functions # shellcheck source=./deploy.env @@ -30,24 +30,34 @@ source "$CURRENTPATH/functions.sh" # --------------------------------------------------------------------- check_prerequisites -# Clean up leftovers -clean_up +# --------------------------------------------------------------------- +# bootstrap install prerequisites +# --------------------------------------------------------------------- +ansible-playbook -i "$CURRENTPATH"/inventory/localhost.ini \ + "$CURRENTPATH"/playbooks/bootstrap.yaml -# The next two functions require that you know your pxe network configuration -# and IP of resulting jumphost VM in advance. This IP/MAC info also then needs to -# be added in PDF & IDF files (not supported yet via this script) -# Create jumphost VM & setup PXE network -create_jump -setup_PXE_network +# --------------------------------------------------------------------- +# Create jump VM from which the installation is performed +# --------------------------------------------------------------------- +ansible-playbook -i "$CURRENTPATH"/inventory/localhost.ini \ + "$CURRENTPATH"/playbooks/jump-vm.yaml +# --------------------------------------------------------------------- # Copy files needed by Infra engine & BMRA in the jumphost VM +# --------------------------------------------------------------------- copy_files_jump +# --------------------------------------------------------------------- # Provision remote hosts +# --------------------------------------------------------------------- provision_hosts +# --------------------------------------------------------------------- # Setup networking (Adapt according to your network setup) +# --------------------------------------------------------------------- setup_network +# --------------------------------------------------------------------- # Provision k8s cluster (currently BMRA) +# --------------------------------------------------------------------- provision_k8s diff --git a/functions.sh b/functions.sh index 4d75ee1..1229d32 100755 --- a/functions.sh +++ b/functions.sh @@ -1,26 +1,13 @@ #!/bin/bash # SPDX-license-identifier: Apache-2.0 ############################################################################## -# Copyright (c) +# Copyright (c) Ericsson AB and others # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -# Clean up -clean_up() { - if sudo virsh list --all | grep " ${VM_NAME} .*running" ; then - sudo virsh destroy "$VM_NAME" - fi - if sudo virsh list --all | grep " ${VM_NAME} " ; then - sudo virsh undefine "$VM_NAME" - fi - sudo rm -rf "/var/lib/libvirt/images/$VM_NAME" - sleep 5 -} - - check_prerequisites() { echo "Info : Check prerequisites" @@ -69,84 +56,60 @@ check_prerequisites() { echo "ERROR : Libvirt not found. Please install." exit 1 fi -} - -# Create jumphost VM -create_jump() { -# Create VM image - sudo mkdir -p "/var/lib/libvirt/images/$VM_NAME" - sudo qemu-img create -f qcow2 \ - -o backing_file=/var/lib/libvirt/images/ubuntu-18.04.qcow2 \ - "/var/lib/libvirt/images/$VM_NAME/$VM_NAME.qcow2" 10G - -# Create VM cloud-init configuration files - cat <<EOL > user-data - #cloud-config - users: - - name: $USERNAME - ssh-authorized-keys: - - $(cat "$HOME/.ssh/id_rsa.pub") - sudo: ['ALL=(ALL) NOPASSWD:ALL'] - groups: sudo - shell: /bin/bash -EOL - cat <<EOL > meta-data - local-hostname: $VM_NAME -EOL - -# Create VM - sudo genisoimage -output "/var/lib/libvirt/images/$VM_NAME/$VM_NAME-cidata.iso" \ - -volid cidata -joliet -rock user-data meta-data - sudo virt-customize -a "/var/lib/libvirt/images/$VM_NAME/$VM_NAME.qcow2" \ - --root-password password:"$ROOT_PASSWORD" - sudo virt-install --connect qemu:///system --name "$VM_NAME" \ - --ram 4096 --vcpus=4 --os-type linux --os-variant ubuntu16.04 \ - --disk path="/var/lib/libvirt/images/$VM_NAME/$VM_NAME.qcow2",format=qcow2 \ - --disk "/var/lib/libvirt/images/$VM_NAME/$VM_NAME-cidata.iso",device=cdrom \ - --import --network network=default --network bridge="$BRIDGE",model=rtl8139 --noautoconsole - jumpbox_ip=$(get_vm_ip) - i=0 - while [ -z "$jumpbox_ip" ]; do - sleep $((++i)) - jumpbox_ip=$(get_vm_ip) - done - i=0 - until nc -w5 -z "$jumpbox_ip" 22; do - sleep $((++i)) - done + #------------------------------------------------------------------------------- + # Check if user belongs to libvirt's group + #------------------------------------------------------------------------------- + libvirt_group="libvirt" + # shellcheck disable=SC1091 + source /etc/os-release || source /usr/lib/os-release + if [ "${ID,,}" == "ubuntu" ] && [ "$VERSION_ID" == "16.04" ]; then + libvirt_group+="d" + fi + if ! groups | grep " $libvirt_group "; then + echo "ERROR : $(id -nu) user doesn't belong to $libvirt_group group." + exit 1 + fi } # Get jumphost VM IP -get_vm_ip() { - sudo virsh domifaddr "$VM_NAME" | awk 'FNR == 3 {gsub(/\/.*/, ""); print $4}' -} +get_host_pxe_ip() { + local PXE_NETWORK + local PXE_IF_INDEX + local PXE_IF_IP + + host=$1 + if [[ "$host" == "" ]]; then + echo "ERROR : get_ip - host parameter not provided" + exit 1 + fi -# Setup PXE network -setup_PXE_network() { -# Extract configuration from PDF/IDF - PXE_IF=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.pxe_interface) - PXE_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config.oob.interface) - if [[ -z $PXE_IF || -z $PXE_IF_INDEX ]]; then - echo 'one or more variables in IDF are undefined' + PXE_NETWORK=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.pxe_network) + if [[ "$PXE_NETWORK" == "" ]]; then + echo "ERROR : PXE network for jump VM not defined in IDF." exit 1 fi - PXE_IF_IP=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml jumphost.interfaces.["$PXE_IF_INDEX"].address) - PXE_IF_MAC=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml jumphost.interfaces.["$PXE_IF_INDEX"].mac_address) - if [[ -z $PXE_IF_IP || -z $PXE_IF_MAC ]]; then - echo 'one or more variables in PDF are incorrect' + + PXE_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config."$PXE_NETWORK".interface) + if [[ "$PXE_IF_INDEX" == "" ]]; then + echo "ERROR : Index of PXE interface not found in IDF." exit 1 fi - export NETMASK=255.255.255.0 -# SSH to jumphost - # shellcheck disable=SC2087 - ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF -sudo ifconfig $PXE_IF up -sudo ifconfig $PXE_IF $PXE_IF_IP netmask $NETMASK -sudo ifconfig $PXE_IF hw ether $PXE_IF_MAC -EOF + + PXE_IF_IP=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/pdf.yaml "$host".interfaces["$PXE_IF_INDEX"].address) + if [[ "$PXE_IF_IP" == "" ]]; then + echo "ERROR : IP of PXE interface not found in PDF." + exit 1 + fi + echo "$PXE_IF_IP" } +get_vm_ip() { + ip=$(get_host_pxe_ip "jumphost") + echo "$ip" +} + + # Copy files needed by Infra engine & BMRA in the jumphost VM copy_files_jump() { scp -r -o StrictHostKeyChecking=no \ @@ -174,10 +137,8 @@ EOF # Setup networking on provisioned hosts (Adapt setup_network.sh according to your network setup) setup_network() { -# Extract IPs of provisioned nodes from PDF/IDF. When running this function standalone, ensure -# to set $PXE_IF_INDEX - MASTER_IP=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml nodes.[0].interfaces.["$PXE_IF_INDEX"].address) - WORKER_IP=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml nodes.[1].interfaces.["$PXE_IF_INDEX"].address) + MASTER_IP=$(get_host_pxe_ip "nodes[0]") + WORKER_IP=$(get_host_pxe_ip "nodes[1]") # SSH to jumphost # shellcheck disable=SC2087 ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF diff --git a/hw_config/ericsson-pod1/pdf.yaml b/hw_config/ericsson-pod1/pdf.yaml index 48933f1..7006788 100644 --- a/hw_config/ericsson-pod1/pdf.yaml +++ b/hw_config/ericsson-pod1/pdf.yaml @@ -9,12 +9,12 @@ ############################################################################## version: 1.0 details: - pod_owner: Nordix - contact: infra@nordix.org - lab: Nordix OpenLab - location: N/A + pod_owner: Ericsson + contact: georg.kunz@ericsson.com + lab: OPNFV lab + location: Sweden type: baremetal - link: http://wiki.nordix.org/ + link: N/A ############################################################################## jumphost: name: kuberef-jump @@ -40,6 +40,7 @@ jumphost: mac_address: 52:54:00:4a:e8:2d - name: 'ens3' address: 100.64.200.254 + mac_address: 52:54:00:4a:e8:2f ############################################################################## nodes: - name: node1 diff --git a/hw_config/intel/idf.yaml b/hw_config/intel/idf.yaml index 328fe09..ca5e4f2 100644 --- a/hw_config/intel/idf.yaml +++ b/hw_config/intel/idf.yaml @@ -53,7 +53,7 @@ engine: pxe_network: oob # net_config network to be used for the internet access - public_network: oob + public_network: public # interface to be used by the PXE pxe_interface: ens4 diff --git a/inventory/.keep b/inventory/.keep new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/inventory/.keep diff --git a/inventory/group_vars/all/global.yaml b/inventory/group_vars/all/global.yaml new file mode 100644 index 0000000..312fcda --- /dev/null +++ b/inventory/group_vars/all/global.yaml @@ -0,0 +1,24 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) Ericsson AB and others +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# root directory of kuberef repository +kuberef_root: "{{ lookup('env', 'CURRENTPATH') }}" + +# path to inventory folder where the initial +inventory_path: "{{ kuberef_root }}/inventory" + +# workspace where all work-in-progress files get stored +workspace: "{{ kuberef_root }}/workspace" + +# directory for storing base images acting as backing file +images_path: "{{ kuberef_root }}/images" + +# public SSH key for use by kuberef installation +pub_key: "{{ lookup('env', 'HOME') }}/.ssh/id_rsa.pub" diff --git a/inventory/localhost.ini b/inventory/localhost.ini new file mode 100644 index 0000000..72f71a5 --- /dev/null +++ b/inventory/localhost.ini @@ -0,0 +1,5 @@ +[all:vars] +ansible_ssh_extra_args=' -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' + +[localhost] +127.0.0.1 ansible_connection=local diff --git a/playbooks/bootstrap.yaml b/playbooks/bootstrap.yaml new file mode 100644 index 0000000..4ef2baa --- /dev/null +++ b/playbooks/bootstrap.yaml @@ -0,0 +1,17 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) Ericsson AB and others +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: localhost + connection: local + gather_facts: false + become: false + + roles: + - role: bootstrap diff --git a/playbooks/jump-vm.yaml b/playbooks/jump-vm.yaml new file mode 100644 index 0000000..bdad81c --- /dev/null +++ b/playbooks/jump-vm.yaml @@ -0,0 +1,16 @@ +--- +############################################################################## +# Copyright (c) Ericsson AB and others +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- hosts: localhost + connection: local + gather_facts: false + become: false + + roles: + - role: jump-vm diff --git a/playbooks/roles/bootstrap/tasks/main.yaml b/playbooks/roles/bootstrap/tasks/main.yaml new file mode 100644 index 0000000..ff20552 --- /dev/null +++ b/playbooks/roles/bootstrap/tasks/main.yaml @@ -0,0 +1,20 @@ +--- +############################################################################## +# Copyright (c) Ericsson AB and others +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +# collected PDF and IDF are copied into group_vars +# folder in order to ensure we have single PDF and IDF +- name: Copy collected PDF and IDF to group_vars + copy: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + force: true + mode: 0644 + with_items: + - {src: "{{ kuberef_root }}/hw_config/{{ lookup('env', 'VENDOR') }}/pdf.yaml", dest: "{{ inventory_path }}/group_vars/all/pdf.yaml"} + - {src: "{{ kuberef_root }}/hw_config/{{ lookup('env', 'VENDOR') }}/idf.yaml", dest: "{{ inventory_path }}/group_vars/all/idf.yaml"} diff --git a/playbooks/roles/jump-vm/tasks/main.yaml b/playbooks/roles/jump-vm/tasks/main.yaml new file mode 100644 index 0000000..f5f6154 --- /dev/null +++ b/playbooks/roles/jump-vm/tasks/main.yaml @@ -0,0 +1,99 @@ +--- +############################################################################## +# Copyright (c) Ericsson AB and others +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: get all running VMs + virt: + command: list_vms + state: running + register: running_vms + +- name: shutdown existing jump VM + virt: + name: "{{ jumphost.name }}" + command: destroy + when: + jumphost.name in running_vms.list_vms + +- name: get all shutdown VMs + virt: + command: list_vms + state: shutdown + register: shutdown_vms + +- name: undefine existing jump VM + virt: + name: "{{ jumphost.name }}" + command: undefine + when: + jumphost.name in shutdown_vms.list_vms + +- name: clean workspace + file: + path: "{{ workspace }}" + state: absent + +- name: create workspace if it does not exist + file: + path: "{{ workspace }}" + state: directory + mode: '0755' + +- name: create directory for base images + file: + path: "{{ images_path }}" + state: directory + mode: '0755' + +- name: download Ubuntu image for jump VM + get_url: + url: https://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img + dest: "{{ images_path }}/bionic-server-cloudimg-amd64.img" + force: false + mode: '0666' + +- name: create new VM image from base image + command: "qemu-img create -f qcow2 -o backing_file={{ images_path }}/bionic-server-cloudimg-amd64.img {{ workspace }}/{{ jumphost.name }}.qcow2 10G" + +- name: render config files for jump VM from templates + template: + src: "{{ kuberef_root }}/playbooks/roles/jump-vm/templates/{{ item }}.j2" + dest: "{{ workspace }}/{{ item }}" + mode: 0644 + with_items: + - network-config + - user-data + - meta-data + +- name: create config drive + command: "genisoimage -output {{ workspace }}/{{ jumphost.name }}-cidata.iso -volid cidata -joliet -rock \ + {{ workspace }}/user-data {{ workspace }}/meta-data {{ workspace }}/network-config" + +# currently commented out because of portability issues between Centos and Ubuntu +# - name: setting root password for debugging +# become: true +# command: "virt-customize -a {{ workspace }}/{{ jumphost.name }}.qcow2 --root-password password:'root'" + +- name: define jump VM + command: "virt-install --connect qemu:///system --name {{ jumphost.name }} \ + --ram 4096 --vcpus=4 --os-type linux --os-variant ubuntu16.04 \ + --disk path={{ workspace }}/kuberef-jump.qcow2,format=qcow2 \ + --disk {{ workspace }}/kuberef-jump-cidata.iso,device=cdrom \ + --network network=default,model=virtio,mac='{{ jumphost.interfaces[engine.net_config[engine.public_network].interface].mac_address }}' \ + --network bridge=pxebr,model=virtio,mac='{{ jumphost.interfaces[engine.net_config[engine.pxe_network].interface].mac_address }}' \ + --import --noautoconsole" + +- name: start jump VM + virt: + name: "{{ jumphost.name }}" + state: running + +- name: wait for VM to be reachable + wait_for: + host: "{{ jumphost.interfaces[idf.net_config[engine.pxe_network].interface].address }}" + port: 22 diff --git a/playbooks/roles/jump-vm/templates/meta-data.j2 b/playbooks/roles/jump-vm/templates/meta-data.j2 new file mode 100644 index 0000000..b4cc5e8 --- /dev/null +++ b/playbooks/roles/jump-vm/templates/meta-data.j2 @@ -0,0 +1 @@ +local-hostname: {{ jumphost.name }} diff --git a/playbooks/roles/jump-vm/templates/network-config.j2 b/playbooks/roles/jump-vm/templates/network-config.j2 new file mode 100644 index 0000000..ceded54 --- /dev/null +++ b/playbooks/roles/jump-vm/templates/network-config.j2 @@ -0,0 +1,14 @@ +version: 1 +config: + - type: physical + name: "{{ jumphost.interfaces[idf.net_config[engine.public_network].interface].name }}" + mac_address: "{{ jumphost.interfaces[idf.net_config[engine.public_network].interface].mac_address }}" + subnets: + - type: dhcp + - type: physical + name: "{{ jumphost.interfaces[idf.net_config[engine.pxe_network].interface].name }}" + mac_address: "{{ jumphost.interfaces[idf.net_config[engine.pxe_network].interface].mac_address }}" + subnets: + - type: static + address: "{{ jumphost.interfaces[idf.net_config[engine.pxe_network].interface].address }}" + netmask: "{{ idf.net_config[engine.pxe_network].mask }}" diff --git a/playbooks/roles/jump-vm/templates/user-data.j2 b/playbooks/roles/jump-vm/templates/user-data.j2 new file mode 100644 index 0000000..648f8d1 --- /dev/null +++ b/playbooks/roles/jump-vm/templates/user-data.j2 @@ -0,0 +1,13 @@ + #cloud-config + users: + - name: ubuntu + ssh-authorized-keys: + - {{ lookup('file', pub_key ) }} + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + groups: sudo + shell: /bin/bash + runcmd: + # this is requried in labs where the PXE network is different from + # the public network. Without internet connectivity, the installation + # of BMRA fails + - [ iptables, -t, nat, -A, POSTROUTING, -o, ens3, -j, MASQUERADE ] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c1409de --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +ansible +libvirt-python |