diff options
| -rw-r--r-- | deploy.env | 7 | ||||
| -rwxr-xr-x | deploy.sh | 9 | ||||
| -rwxr-xr-x | functions.sh | 70 | ||||
| -rw-r--r-- | hw_config/equinix-metal/idf.yaml | 40 | ||||
| -rw-r--r-- | hw_config/equinix-metal/pdf.yaml | 34 | ||||
| -rw-r--r-- | inventory/group_vars/all/global.yaml | 3 | ||||
| -rw-r--r-- | playbooks/pre-install.yaml | 15 | ||||
| -rw-r--r-- | playbooks/roles/bmra-config/templates/inventory.ini | 1 | ||||
| -rw-r--r-- | playbooks/roles/jump-vm/tasks/main.yaml | 32 | ||||
| -rw-r--r-- | playbooks/roles/jump-vm/templates/network-config.j2 | 2 | ||||
| -rw-r--r-- | playbooks/roles/jump-vm/templates/user-data.j2 | 27 | ||||
| -rw-r--r-- | playbooks/roles/pre-install/molecule/default/converge.yml | 16 | ||||
| -rw-r--r-- | playbooks/roles/pre-install/molecule/default/molecule.yml | 26 | ||||
| -rw-r--r-- | playbooks/roles/pre-install/molecule/default/tests/test_default.py | 26 | ||||
| -rw-r--r-- | playbooks/roles/pre-install/tasks/main.yml | 20 | ||||
| -rw-r--r-- | playbooks/roles/pre-install/vars/RedHat.yml | 14 | ||||
| -rw-r--r-- | test-requirements.txt | 5 |
17 files changed, 315 insertions, 32 deletions
@@ -3,11 +3,14 @@ export VENDOR=${VENDOR:-ericsson-pod2} export INSTALLER=bmra +# Deployment type. Supports "full" and "k8s" +export DEPLOYMENT=${DEPLOYMENT:-full} + # Name of host bridge to which the VM is connected to (used for PXE) export BRIDGE=pxebr # Jump VM details export VM_NAME=$(yq r $CURRENTPATH/hw_config/$VENDOR/pdf.yaml jumphost.name) -export USERNAME=ubuntu -export PROJECT_ROOT="/home/ubuntu" +export USERNAME=${USERNAME:-ubuntu} +export PROJECT_ROOT="${PROJECT_ROOT:-/home/ubuntu}" export ROOT_PASSWORD="root" # This is used for throubleshooting purposes @@ -52,13 +52,12 @@ copy_files_jump # --------------------------------------------------------------------- # Provision remote hosts -# --------------------------------------------------------------------- -provision_hosts - -# --------------------------------------------------------------------- # Setup networking (Adapt according to your network setup) # --------------------------------------------------------------------- -setup_network +if [[ "$DEPLOYMENT" == "full" ]]; then + provision_hosts + setup_network +fi # --------------------------------------------------------------------- # Provision k8s cluster (currently BMRA) diff --git a/functions.sh b/functions.sh index 20c4d5d..4e06523 100755 --- a/functions.sh +++ b/functions.sh @@ -31,9 +31,17 @@ check_prerequisites() { info "Check prerequisites" #------------------------------------------------------------------------------- + # Check for DEPLOYMENT type + #------------------------------------------------------------------------------- + DEPLOYMENT=${DEPLOYMENT:-full} + if ! [[ "$DEPLOYMENT" =~ ^(full|k8s)$ ]]; then + error "Unsupported value for DEPLOYMENT ($DEPLOYMENT)" + fi + + #------------------------------------------------------------------------------- # We shouldn't be running as root #------------------------------------------------------------------------------- - if [[ "$(whoami)" == "root" ]]; then + if [[ "$(whoami)" == "root" ]] && [[ "$DEPLOYMENT" != "k8s" ]]; then error "This script must not be run as root! Please switch to a regular user before running the script." fi @@ -59,7 +67,7 @@ check_prerequisites() { #------------------------------------------------------------------------------- # Check if some tools are installed #------------------------------------------------------------------------------- - for tool in ansible yq virsh; do + for tool in ansible yq virsh jq; do if ! command -v "$tool" &> /dev/null; then error "$tool not found. Please install." fi @@ -88,7 +96,7 @@ get_host_pxe_ip() { host=$1 assert_non_empty "$host" "get_ip - host parameter not provided" - PXE_NETWORK=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.pxe_network) + PXE_NETWORK=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.pxe_network) assert_non_empty "$PXE_NETWORK" "PXE network for jump VM not defined in IDF." PXE_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config."$PXE_NETWORK".interface) @@ -100,17 +108,51 @@ get_host_pxe_ip() { echo "$PXE_IF_IP" } +# Get public MAC for VM +get_host_pub_mac() { + local PUB_NETWORK + local PUB_IF_INDEX + local PUB_IF_MAC + + host=$1 + assert_non_empty "$host" "get_mac - host parameter not provided" + + PUB_NETWORK=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/idf.yaml engine.public_network) + assert_non_empty "$PUB_NETWORK" "Public network for jump VM not defined in IDF." + + PUB_IF_INDEX=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/idf.yaml idf.net_config."$PUB_NETWORK".interface) + assert_non_empty "$PUB_IF_INDEX" "Index of public interface not found in IDF." + + PUB_IF_MAC=$(yq r "$CURRENTPATH"/hw_config/"${VENDOR}"/pdf.yaml "$host".interfaces["$PUB_IF_INDEX"].mac_address) + assert_non_empty "$PUB_IF_MAC" "MAC of public interface not found in PDF." + echo "$PUB_IF_MAC" +} + # Get jumphost VM IP get_vm_ip() { - ip=$(get_host_pxe_ip "jumphost") + if [[ "$DEPLOYMENT" == "full" ]]; then + ip=$(get_host_pxe_ip "jumphost") + else + mac=$(get_host_pub_mac "jumphost") + JUMPHOST_NAME=$(yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml jumphost.name) + ipblock=$(virsh domifaddr "$JUMPHOST_NAME" --full | grep "$mac" | awk '{print $4}' | tail -n 1) + assert_non_empty "$ipblock" "IP subnet for VM not available." + ip="${ipblock%/*}" + fi echo "$ip" } # Copy files needed by Infra engine & BMRA in the jumphost VM copy_files_jump() { + vm_ip="$(get_vm_ip)" scp -r -o StrictHostKeyChecking=no \ "$CURRENTPATH"/{hw_config/"$VENDOR"/,sw_config/"$INSTALLER"/} \ - "$USERNAME@$(get_vm_ip):$PROJECT_ROOT" + "$USERNAME@${vm_ip}:$PROJECT_ROOT" + if [[ "$DEPLOYMENT" != "full" ]]; then + scp -r -o StrictHostKeyChecking=no \ + ~/.ssh/id_rsa \ + "$USERNAME@${vm_ip}:.ssh/id_rsa" + fi } # Host Provisioning @@ -147,8 +189,20 @@ EOF done } +# Get IPs of target nodes (used for installing dependencies) +get_target_ips() { + yq r "$CURRENTPATH"/hw_config/"$VENDOR"/pdf.yaml nodes[*].interfaces[*].address +} + # k8s Provisioning (currently BMRA) provision_k8s() { + ansible_cmd='/bin/bash -c "' + if [[ "$DEPLOYMENT" == "k8s" ]]; then + ansible-playbook -i "$CURRENTPATH"/sw_config/bmra/inventory.ini "$CURRENTPATH"/playbooks/pre-install.yaml + ansible_cmd+='pip install --upgrade pip==9.0.3; yum -y remove python-netaddr; pip install netaddr==0.7.19; ' + fi + ansible_cmd+='ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/cluster.yml"' + # shellcheck disable=SC2087 ssh -o StrictHostKeyChecking=no -tT "$USERNAME"@"$(get_vm_ip)" << EOF # Install BMRA @@ -164,8 +218,10 @@ if [ ! -d "${PROJECT_ROOT}/container-experience-kits" ]; then git clone --recurse-submodules --depth 1 https://github.com/intel/container-experience-kits.git -b v1.4.1 ${PROJECT_ROOT}/container-experience-kits/ cp -r ${PROJECT_ROOT}/container-experience-kits/examples/group_vars ${PROJECT_ROOT}/container-experience-kits/ #TODO Remove this once the reported issue is fixed in the next BMRA Release - sed -i '/\openshift/a \ extra_args: --ignore-installed PyYAML' \ + if [[ "$DEPLOYMENT" == "full" ]]; then + sed -i '/\openshift/a \ extra_args: --ignore-installed PyYAML' \ ${PROJECT_ROOT}/container-experience-kits/roles/net-attach-defs-create/tasks/main.yml + fi fi cp ${PROJECT_ROOT}/${INSTALLER}/inventory.ini \ ${PROJECT_ROOT}/container-experience-kits/ @@ -175,7 +231,7 @@ sudo docker run --rm \ -e ANSIBLE_CONFIG=/bmra/ansible.cfg \ -v ${PROJECT_ROOT}/container-experience-kits:/bmra \ -v ~/.ssh/:/root/.ssh/ rihabbanday/bmra-install:centos \ -ansible-playbook -i /bmra/inventory.ini /bmra/playbooks/cluster.yml +${ansible_cmd} EOF } diff --git a/hw_config/equinix-metal/idf.yaml b/hw_config/equinix-metal/idf.yaml new file mode 100644 index 0000000..2c61bb6 --- /dev/null +++ b/hw_config/equinix-metal/idf.yaml @@ -0,0 +1,40 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2020 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +idf: + version: 0.1 + net_config: &net_config + public: + interface: 0 + kubespray: &idf_kubespray + nodes_roles: + node1: [kube-master, etcd] + node2: [kube-node] + groups: + k8s-cluster: + - kube-node + - kube-master + hostnames: + # Update with hostnames of servers (node1: master, node2: worker) + node1: kref-mst + node2: kref-wrk + +engine: + pod_name: kref-jmp + net_config: *net_config + + # net_config network to be used by the PXE + pxe_network: public + + # net_config network to be used for the internet access + public_network: public + + installers: + kubespray: *idf_kubespray diff --git a/hw_config/equinix-metal/pdf.yaml b/hw_config/equinix-metal/pdf.yaml new file mode 100644 index 0000000..61afc27 --- /dev/null +++ b/hw_config/equinix-metal/pdf.yaml @@ -0,0 +1,34 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2020 +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +version: 1.0 +details: + pod_owner: Equinix Metal + contact: Equinix Metal + lab: EWR1 + location: Parsipanny, USA + type: baremetal + link: https://console.equinix.com/ +jumphost: + name: kuberef-jump + interfaces: + - name: 'nic1' + mac_address: "52:54:00:4a:e8:2d" +nodes: + - name: node1 + interfaces: + - name: 'nic1' + # Update with public IP of target server 1 (master) + address: <Server IP> + - name: node2 + interfaces: + - name: 'nic1' + # Update with public IP of target server 2 (worker) + address: <Server IP> diff --git a/inventory/group_vars/all/global.yaml b/inventory/group_vars/all/global.yaml index 312fcda..c88d794 100644 --- a/inventory/group_vars/all/global.yaml +++ b/inventory/group_vars/all/global.yaml @@ -22,3 +22,6 @@ images_path: "{{ kuberef_root }}/images" # public SSH key for use by kuberef installation pub_key: "{{ lookup('env', 'HOME') }}/.ssh/id_rsa.pub" + +# deployment type for kuberef +deployment_type: "{{ lookup('env', 'DEPLOYMENT') }}" diff --git a/playbooks/pre-install.yaml b/playbooks/pre-install.yaml new file mode 100644 index 0000000..5f07a63 --- /dev/null +++ b/playbooks/pre-install.yaml @@ -0,0 +1,15 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2020 Samsung Electronics +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: Install requirements on nodes + hosts: all + gather_facts: true + roles: + - role: pre-install diff --git a/playbooks/roles/bmra-config/templates/inventory.ini b/playbooks/roles/bmra-config/templates/inventory.ini index 48ca5fd..7a9c6ab 100644 --- a/playbooks/roles/bmra-config/templates/inventory.ini +++ b/playbooks/roles/bmra-config/templates/inventory.ini @@ -10,7 +10,6 @@ {{ idf.kubespray.hostnames[node.name] }} {% endif %} {% endfor %} - {% endfor %} [k8s-cluster:children] diff --git a/playbooks/roles/jump-vm/tasks/main.yaml b/playbooks/roles/jump-vm/tasks/main.yaml index 2c0cc37..c5738c3 100644 --- a/playbooks/roles/jump-vm/tasks/main.yaml +++ b/playbooks/roles/jump-vm/tasks/main.yaml @@ -33,6 +33,16 @@ when: jumphost.name in shutdown_vms.list_vms +- name: remove dhcp leases + shell: | + jq 'del( .[] | select(.domain == "{{ jumphost.name }}" or .hostname == "{{ jumphost.name }}"))' /var/lib/libvirt/dnsmasq/virbr0.{{ item }} > /tmp/{{ item }}.tmp + mv /tmp/{{ item }}.tmp /var/lib/libvirt/dnsmasq/virbr0.{{ item }} + with_items: + - status + - macs + become: true + when: deployment_type == 'k8s' + - name: clean workspace file: path: "{{ workspace }}" @@ -71,8 +81,9 @@ - meta-data - name: create config drive - command: "genisoimage -output {{ workspace }}/{{ jumphost.name }}-cidata.iso -volid cidata -joliet -rock \ - {{ workspace }}/user-data {{ workspace }}/meta-data {{ workspace }}/network-config" + command: "genisoimage -output {{ workspace }}/{{ jumphost.name }}-cidata.iso -volid cidata -joliet -rock \ + {{ workspace }}/user-data {{ workspace }}/meta-data \ + {{ workspace + '/network-config' if deployment_type == 'full' else '' }}" # currently commented out because of portability issues between Centos and Ubuntu # - name: setting root password for debugging @@ -85,7 +96,7 @@ --disk path={{ workspace }}/kuberef-jump.qcow2,format=qcow2 \ --disk {{ workspace }}/kuberef-jump-cidata.iso,device=cdrom \ --network network=default,model=virtio,mac='{{ jumphost.interfaces[engine.net_config[engine.public_network].interface].mac_address }}' \ - --network bridge=pxebr,model=rtl8139,mac='{{ jumphost.interfaces[engine.net_config[engine.pxe_network].interface].mac_address }}' \ + {{ '--network bridge=pxebr,model=rtl8139,mac=' + jumphost.interfaces[engine.net_config[engine.pxe_network].interface].mac_address if deployment_type == 'full' else '' }} \ --import --noautoconsole" - name: start jump VM @@ -93,7 +104,20 @@ name: "{{ jumphost.name }}" state: running +- name: wait for ip to be visible + shell: virsh domifaddr "{{ jumphost.name }}" --full | grep "{{ jumphost.interfaces[engine.net_config[engine.public_network].interface].mac_address }}" | awk '{print $4}' | tail -n 1 + register: ipblock + retries: 30 + delay: 1 + until: ipblock.stdout != "" + when: deployment_type == 'k8s' + +- name: store ip + set_fact: + vm_ip: "{{ ipblock.stdout.split('/') }}" + when: deployment_type == 'k8s' + - name: wait for VM to be reachable wait_for: - host: "{{ jumphost.interfaces[idf.net_config[engine.pxe_network].interface].address }}" + host: "{{ ( vm_ip is defined and vm_ip.0 or '' ) if 'k8s' in deployment_type else jumphost.interfaces[idf.net_config[engine.pxe_network].interface].address | default('') }}" port: 22 diff --git a/playbooks/roles/jump-vm/templates/network-config.j2 b/playbooks/roles/jump-vm/templates/network-config.j2 index ceded54..8587a10 100644 --- a/playbooks/roles/jump-vm/templates/network-config.j2 +++ b/playbooks/roles/jump-vm/templates/network-config.j2 @@ -1,3 +1,4 @@ +{% if deployment_type == 'full' %} version: 1 config: - type: physical @@ -12,3 +13,4 @@ config: - type: static address: "{{ jumphost.interfaces[idf.net_config[engine.pxe_network].interface].address }}" netmask: "{{ idf.net_config[engine.pxe_network].mask }}" +{% endif %} diff --git a/playbooks/roles/jump-vm/templates/user-data.j2 b/playbooks/roles/jump-vm/templates/user-data.j2 index 648f8d1..2fad2e3 100644 --- a/playbooks/roles/jump-vm/templates/user-data.j2 +++ b/playbooks/roles/jump-vm/templates/user-data.j2 @@ -1,13 +1,14 @@ - #cloud-config - users: - - name: ubuntu - ssh-authorized-keys: - - {{ lookup('file', pub_key ) }} - sudo: ['ALL=(ALL) NOPASSWD:ALL'] - groups: sudo - shell: /bin/bash - runcmd: - # this is requried in labs where the PXE network is different from - # the public network. Without internet connectivity, the installation - # of BMRA fails - - [ iptables, -t, nat, -A, POSTROUTING, -o, ens3, -j, MASQUERADE ] +#jinja2:lstrip_blocks: True +#cloud-config +users: + - name: {{ lookup('env', 'USERNAME') }} + ssh-authorized-keys: + - {{ lookup('file', pub_key ) }} + sudo: ['ALL=(ALL) NOPASSWD:ALL'] + groups: sudo + shell: /bin/bash +runcmd: + # this is requried in labs where the PXE network is different from + # the public network. Without internet connectivity, the installation + # of BMRA fails + - [ iptables, -t, nat, -A, POSTROUTING, -o, ens3, -j, MASQUERADE ] diff --git a/playbooks/roles/pre-install/molecule/default/converge.yml b/playbooks/roles/pre-install/molecule/default/converge.yml new file mode 100644 index 0000000..c7904cf --- /dev/null +++ b/playbooks/roles/pre-install/molecule/default/converge.yml @@ -0,0 +1,16 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2020 Samsung Electronics +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: Converge + hosts: all + gather_facts: true + roles: + - role: pre-install + inventory_path: /home/kuberef/inventory diff --git a/playbooks/roles/pre-install/molecule/default/molecule.yml b/playbooks/roles/pre-install/molecule/default/molecule.yml new file mode 100644 index 0000000..a43f45f --- /dev/null +++ b/playbooks/roles/pre-install/molecule/default/molecule.yml @@ -0,0 +1,26 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2020 Samsung Electronics +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +dependency: + name: galaxy +driver: + name: docker +lint: | + set -e + yamllint -c ../../../.yamllint . + ansible-lint +platforms: + - name: centos_7 + image: centos:7 +provisioner: + name: ansible +verifier: + name: testinfra + lint: + name: flake8 diff --git a/playbooks/roles/pre-install/molecule/default/tests/test_default.py b/playbooks/roles/pre-install/molecule/default/tests/test_default.py new file mode 100644 index 0000000..c992994 --- /dev/null +++ b/playbooks/roles/pre-install/molecule/default/tests/test_default.py @@ -0,0 +1,26 @@ +# Copyright 2020 Samsung Electronics +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.# +# + +import os +import pytest + +import testinfra.utils.ansible_runner + +testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner( + os.environ['MOLECULE_INVENTORY_FILE'] +).get_hosts('all') +def test_requirements_installed(host): + for pkg in ["lshw", "pciutils", "ethtool"]: + assert host.package(pkg).is_installed diff --git a/playbooks/roles/pre-install/tasks/main.yml b/playbooks/roles/pre-install/tasks/main.yml new file mode 100644 index 0000000..db6534d --- /dev/null +++ b/playbooks/roles/pre-install/tasks/main.yml @@ -0,0 +1,20 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2020 Samsung Electronics +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +- name: Load distro variables + include_vars: + file: "{{ ansible_os_family }}.yml" + +- name: Install BRMA requirements + become: true + package: + name: "{{ item }}" + state: present + with_items: "{{ bmra_pkgs }}" diff --git a/playbooks/roles/pre-install/vars/RedHat.yml b/playbooks/roles/pre-install/vars/RedHat.yml new file mode 100644 index 0000000..5adac50 --- /dev/null +++ b/playbooks/roles/pre-install/vars/RedHat.yml @@ -0,0 +1,14 @@ +--- +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2020 Samsung Electronics +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +bmra_pkgs: + - lshw + - pciutils + - ethtool diff --git a/test-requirements.txt b/test-requirements.txt index 1580d0c..cb560ac 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,3 +5,8 @@ yamllint # LGPLv3 bashate # Apache-2.0 rstcheck # MIT +molecule # MIT +molecule-docker # MIT +docker # Apache-2.0 +testinfra # Apache-2.0 +ansible-lint # MIT |