diff options
Diffstat (limited to 'mcp/scripts')
24 files changed, 1267 insertions, 754 deletions
diff --git a/mcp/scripts/.gitignore b/mcp/scripts/.gitignore index a7f658e4e..d89d60722 100644 --- a/mcp/scripts/.gitignore +++ b/mcp/scripts/.gitignore @@ -1,3 +1,5 @@ mcp.rsa* -user-data.*.sh +user-data.sh xdf_data.sh +docker-compose/files/entrypoint_maas.sh +docker-compose/files/hosts diff --git a/mcp/scripts/docker-compose/docker-compose.yaml.j2 b/mcp/scripts/docker-compose/docker-compose.yaml.j2 new file mode 100644 index 000000000..04dc93f98 --- /dev/null +++ b/mcp/scripts/docker-compose/docker-compose.yaml.j2 @@ -0,0 +1,102 @@ +############################################################################## +# Copyright (c) 2018 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +{%- import 'net_map.j2' as nm with context %} +{%- import 'net_macros.j2' as ma with context %} +{#- conf.MCPCONTROL_NET & co are mandatory, defaults are set via globals.sh #} +{%- set net_mcpcontrol = [conf.MCPCONTROL_NET, conf.MCPCONTROL_PREFIX] | join("/") %} +version: '2' +services: + opnfv-fuel-salt-master: + container_name: "fuel" + image: "opnfv/fuel:saltmaster-reclass-{{ conf.MCP_DOCKER_TAG }}" + restart: always + networks: + mcpcontrol: + ipv4_address: {{ conf.SALT_MASTER }} + pxebr: + ipv4_address: {{ nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_config_pxe_admin_address') +1) }} + mgmt: + ipv4_address: {{ nm.net_mgmt | ipnet_hostaddr(nm.start_ip[nm.net_mgmt] + nm.net_mgmt_hosts.index('opnfv_infra_config_address') +1) }} + volumes: + - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket:ro + - {{ conf.MCP_REPO_ROOT_PATH }}:/root/fuel + - {{ conf.MCP_REPO_ROOT_PATH }}/mcp/scripts/docker-compose/files/entrypoint.sh:/entrypoint.sh + - {{ conf.MCP_STORAGE_DIR }}/pod_config.yml:/root/pod_config.yml + - {{ conf.MCP_STORAGE_DIR }}/nodes:/srv/salt/reclass/nodes + - {{ conf.MCP_STORAGE_DIR }}/pki:/etc/pki + - {{ conf.MCP_STORAGE_DIR }}/salt:/etc/salt + - {{ conf.MCP_STORAGE_DIR }}/hosts:/etc/hosts +{%- if conf.MCP_VCP or '-vcp-' in conf.MCP_DEPLOY_SCENARIO %} + - {{ conf.MCP_STORAGE_DIR }}/base_image_opnfv_fuel_vcp.img:/srv/salt/env/prd/salt/files/control/images/base_image_opnfv_fuel_vcp.img +{%- endif %} + hostname: cfg01 + domainname: {{ conf.cluster.domain }} + privileged: true + dns: +{%- for server in nm.dns_public %} + - {{ server }} +{%- endfor %} +{%- if nm.cluster.has_baremetal_nodes %} + opnfv-fuel-maas: + container_name: "maas" + image: "opnfv/fuel:saltminion-maas-{{ conf.MCP_DOCKER_TAG }}" + restart: always + networks: + mcpcontrol: + ipv4_address: {{ conf.MAAS_IP }} + pxebr: + ipv4_address: {{ nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_maas_node01_deploy_address') +1) }} + mgmt: + ipv4_address: {{ nm.net_mgmt | ipnet_hostaddr(nm.start_ip[nm.net_mgmt] + nm.net_mgmt_hosts.index('opnfv_infra_maas_node01_address') +1) }} + volumes: + - /lib/modules:/lib/modules:ro + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket:ro + - {{ conf.MCP_REPO_ROOT_PATH }}/mcp/scripts/docker-compose/files/entrypoint_maas.sh:/entrypoint.sh:ro + - {{ conf.MCP_STORAGE_DIR }}/hosts:/etc/hosts:ro + - {{ conf.MCP_STORAGE_DIR }}/mas01/etc/iptables:/etc/iptables + - {{ conf.MCP_STORAGE_DIR }}/mas01/var/lib/postgresql:/var/lib/postgresql + - {{ conf.MCP_STORAGE_DIR }}/mas01/var/lib/maas:/var/lib/maas + - {{ conf.MCP_STORAGE_DIR }}/mas01/var/spool/maas-proxy:/var/spool/maas-proxy + - {{ conf.MCP_STORAGE_DIR }}/mas01/etc/maas:/etc/maas + hostname: mas01 + domainname: {{ conf.cluster.domain }} + privileged: true + dns: +{%- for server in nm.dns_public %} + - {{ server }} +{%- endfor %} + ports: + - 5240:5240 +{%- endif %} +networks: + mcpcontrol: + driver: bridge + driver_opts: + com.docker.network.driver.mtu: 9000 + ipam: + config: + - subnet: {{ net_mcpcontrol }} + pxebr: + driver: macvlan + driver_opts: + parent: veth_mcp1 # Always untagged + ipam: + config: + - subnet: {{ nm.net_admin }} + mgmt: + driver: macvlan + driver_opts: +{%- if conf.idf.fuel.jumphost.get('trunks', {}).get('mgmt', False) %} + parent: {{ ma.interface_str('veth_mcp3', nm.vlan_mgmt) }} +{%- else %} + parent: veth_mcp3 # Untagged by default +{%- endif %} + ipam: + config: + - subnet: {{ nm.net_mgmt }} diff --git a/mcp/scripts/docker-compose/files/entrypoint.sh b/mcp/scripts/docker-compose/files/entrypoint.sh new file mode 100755 index 000000000..baf1f65d2 --- /dev/null +++ b/mcp/scripts/docker-compose/files/entrypoint.sh @@ -0,0 +1,104 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2018 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +mkdir -p /etc/salt/{master.d,minion.d,proxy.d} + +if [ ! -z "$SALT_EXT_PILLAR" ]; then + cp -avr "/tmp/${SALT_EXT_PILLAR}.conf" /etc/salt/master.d/ +fi + +if [ ! -f /home/ubuntu/.ssh/authorized_keys ]; then + install -D -o ubuntu /root/fuel/mcp/scripts/mcp.rsa.pub \ + /home/ubuntu/.ssh/authorized_keys + mkdir -p /root/.ssh/ + echo 'User ubuntu' > /root/.ssh/config + echo 'IdentityFile /root/fuel/mcp/scripts/mcp.rsa' >> /root/.ssh/config +fi + +# salt state does not properly configure file_roots in master.conf, hard set it +cp -a /root/fuel/mcp/scripts/docker-compose/files/opnfv_master.conf \ + /etc/salt/master.d/opnfv_master.conf +echo -e 'master: localhost\nmine_interval: 15' > /etc/salt/minion.d/opnfv_slave.conf + +# NOTE: Most Salt and/or reclass tools have issues traversing Docker mounts +# or detecting them as directories inside the container. +# For now, let's do a lot of copy operations to bypass this. +# Later, we will inject the OPNFV patched reclass model during image build. +rm -rf /srv/salt/reclass/classes/* +cp -ar /root/fuel/mcp/reclass/classes/* /srv/salt/reclass/classes +cp -ar /root/fuel/mcp/reclass/nodes/* /srv/salt/reclass/nodes +# Sensitive data should stay out of /root/fuel, which is exposed via Jenkins WS +cp -a /root/pod_config.yml \ + /srv/salt/reclass/classes/cluster/all-mcp-arch-common/opnfv/pod_config.yml + +# OPNFV formulas +prefix=/srv/salt/formula/salt-formulas +rm -f /root/fuel/mcp/salt-formulas/*/.git +cp -ar /root/fuel/mcp/salt-formulas/* ${prefix}/ +for formula in 'armband' 'opendaylight' 'tacker' 'quagga'; do + ln -sf /root/fuel/mcp/salt-formulas/salt-formula-${formula}/* \ + /srv/salt/env/prd/ +done + +# Re-create classes.service links that we destroyed above +for formula in ${prefix}/*; do + if [ -e "${formula}/metadata/service" ] && [[ ! $formula =~ \. ]]; then + ln -sf "${formula}/metadata/service" \ + "/srv/salt/reclass/classes/service/${formula#${prefix}/salt-formula-}" + fi +done + +# Create links for salt-formula-* packages to mimic git-style salt-formulas +for artifact in /usr/share/salt-formulas/env/_*/*; do + ln -sf "${artifact}" "/srv/salt/env/prd/${artifact#/usr/share/salt-formulas/env/}" +done +for artifact in /usr/share/salt-formulas/env/*; do + if [[ ! ${artifact} =~ ^_ ]]; then + ln -sf "${artifact}" "/srv/salt/env/prd/$(basename ${artifact})" + fi +done +for formula in /usr/share/salt-formulas/reclass/service/*; do + ln -sf "${formula}" "/srv/salt/reclass/classes/service/$(basename ${formula})" +done + +# Temporary link rocky configs to stein +for f in /srv/salt/env/prd/*/files/rocky; do + if [ ! -d "$f/../stein" ]; then + ln -sf "$f" "$f/../stein" + fi +done + +# Tini init system resembles upstart very much, but needs a little adjustment +sed -i -e "s|return 'start/running' in |return 'is running' in |" \ + -e "s|ret = _default_runlevel|return _default_runlevel|" \ + /usr/lib/python2.7/dist-packages/salt/modules/upstart.py + +# Workaround for: https://github.com/salt-formulas/reclass/issues/77 +sed -i -e 's|\(ignore_overwritten_missing_references\)defaults.|\1|' \ + /usr/local/lib/python2.7/dist-packages/reclass/settings.py + +# Remove broken symlinks in /srv/salt, silences recurring warnings +find -L /srv/salt /srv/salt/env/prd/_* -maxdepth 1 -type l -delete + +# Fix up any permissions after above file shuffling +chown root:root -R /srv/salt + +# Docker-ce 19.x+ workaround for broken domainname setup +# shellcheck source=/dev/null +source /root/fuel/mcp/scripts/xdf_data.sh +hostname -b "cfg01.${CLUSTER_DOMAIN}" + +service ssh start +service salt-minion start + +if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then + exec /usr/bin/salt-master --log-file-level=quiet --log-level=info "$@" +else + exec "$@" +fi diff --git a/mcp/scripts/docker-compose/files/entrypoint_maas.sh.j2 b/mcp/scripts/docker-compose/files/entrypoint_maas.sh.j2 new file mode 100644 index 000000000..d92eeb017 --- /dev/null +++ b/mcp/scripts/docker-compose/files/entrypoint_maas.sh.j2 @@ -0,0 +1,62 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2019 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +{%- import 'net_map.j2' as nm with context %} +{%- set pxebr_addr = nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_maas_node01_deploy_address') +1) %} +if [ ! -e /var/lib/postgresql/*/main ]; then + cp -ar /var/lib/opnfv/{postgresql,maas} /var/lib/ + cp -ar /var/lib/opnfv/etc/maas /etc/ +fi +chown -R maas:maas /var/lib/maas /etc/maas +chown -R postgres:postgres /var/lib/postgresql +chown -R proxy:proxy /var/spool/maas-proxy + +if [ ! -f /etc/sysctl.d/99-salt.conf ]; then + echo 'net.ipv4.ip_forward = 1' > /etc/sysctl.d/99-salt.conf +fi + +cat <<-EOF | tee /etc/resolv.conf +{%- for server in nm.dns_public %} +nameserver {{ server }} +{%- endfor %} +EOF + +cat <<-EOF | tee /etc/salt/minion.d/opnfv.conf +id: mas01.{{ conf.cluster.domain }} +master: {{ conf.SALT_MASTER }} +grains: + virtual_subtype: Docker_ +EOF +rm -f /etc/salt/minion.d/99-master-address.conf + +# Work around MaaS issues with PXE/admin using jumbo frames +MAAS_MTU_SERVICE="/etc/systemd/system/maas-mtu.service" +cat <<-EOF | tee "${MAAS_MTU_SERVICE}" +[Unit] +Requires=network-online.target +After=network-online.target +[Service] +ExecStart=/bin/sh -ec '\ + /sbin/ifconfig $(/sbin/ip addr | /bin/grep -Po "{{ pxebr_addr }}.* \K(.*)") mtu 1500' +EOF +ln -sf "${MAAS_MTU_SERVICE}" "/etc/systemd/system/multi-user.target.wants/" + +# Configure mass-region-controller if not already done previously +[ ! -e /var/lib/maas/secret ] || exit 0 +MAAS_FIXUP_SERVICE="/etc/systemd/system/maas-fixup.service" +cat <<-EOF | tee "${MAAS_FIXUP_SERVICE}" +[Unit] +After=postgresql.service +[Service] +ExecStart=/bin/sh -ec '\ + echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections && \ + /var/lib/dpkg/info/maas-region-controller.config configure && \ + /var/lib/dpkg/info/maas-region-controller.postinst configure' +EOF +ln -sf "${MAAS_FIXUP_SERVICE}" "/etc/systemd/system/multi-user.target.wants/" +rm "/usr/sbin/policy-rc.d" diff --git a/mcp/scripts/docker-compose/files/hosts.j2 b/mcp/scripts/docker-compose/files/hosts.j2 new file mode 100644 index 000000000..b42c5a088 --- /dev/null +++ b/mcp/scripts/docker-compose/files/hosts.j2 @@ -0,0 +1,7 @@ +{{ conf.SALT_MASTER }} cfg01.{{ conf.cluster.domain }} +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +fe00::0 ip6-localnet +ff00::0 ip6-mcastprefix +ff02::1 ip6-allnodes +ff02::2 ip6-allrouters diff --git a/mcp/scripts/docker-compose/files/opnfv_master.conf b/mcp/scripts/docker-compose/files/opnfv_master.conf new file mode 100644 index 000000000..8e6d1af28 --- /dev/null +++ b/mcp/scripts/docker-compose/files/opnfv_master.conf @@ -0,0 +1,21 @@ +worker_threads: 20 +timeout: 15 +gather_job_timeout: 30 +max_open_files: 15000 + +file_roots: + base: + - /srv/salt/env/prd + prd: + - /srv/salt/env/prd + dev: + - /srv/salt/env/dev + - /srv/salt/env/prd + +user: root +file_recv: True + +open_mode: True + +peer: + .*: ['x509.sign_remote_certificate'] diff --git a/mcp/scripts/globals.sh b/mcp/scripts/globals.sh index 54f015cf6..e5d1decc6 100644 --- a/mcp/scripts/globals.sh +++ b/mcp/scripts/globals.sh @@ -1,6 +1,6 @@ #!/bin/bash -e ############################################################################## -# Copyright (c) 2017 Ericsson AB, Mirantis Inc., Enea AB and others. +# Copyright (c) 2018 Ericsson AB, Mirantis Inc., Enea AB and others. # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -12,6 +12,8 @@ export CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x export SSH_KEY=${SSH_KEY:-"/var/lib/opnfv/mcp.rsa"} export SALT_MASTER=${INSTALLER_IP:-10.20.0.2} export SALT_MASTER_USER=${SALT_MASTER_USER:-ubuntu} +export VIRSH=${VIRSH:-'virsh --connect qemu:///system'} +export MCP_KERNEL_VER=${MCP_KERNEL_VER:-5.0.0-37} # Derived from INSTALLER_IP export MCPCONTROL_NET=${MCPCONTROL_NET:-${SALT_MASTER%.*}.0} @@ -36,7 +38,7 @@ function notify() { function notify_i() { tput setaf "${2:-1}" || true echo -en "${1:-"[WARN] Unsupported opt arg: $3\\n"}" - tput sgr0 + tput sgr0 || true } # same as `notify` + extra '\n' before and after; diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh index ce5db251f..6455a65c0 100644 --- a/mcp/scripts/lib.sh +++ b/mcp/scripts/lib.sh @@ -1,5 +1,5 @@ #!/bin/bash -e -# shellcheck disable=SC2155,SC1001,SC2015,SC2128 +# shellcheck disable=SC2155,SC2015 ############################################################################## # Copyright (c) 2018 Mirantis Inc., Enea AB and others. # All rights reserved. This program and the accompanying materials @@ -8,501 +8,9 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## # -# Library of shell functions +# Library of common shell functions used by build/deploy scripts, states etc. # -function generate_ssh_key { - local mcp_ssh_key=$(basename "${SSH_KEY}") - local user=${USER} - if [ -n "${SUDO_USER}" ] && [ "${SUDO_USER}" != 'root' ]; then - user=${SUDO_USER} - fi - - if [ -f "${SSH_KEY}" ]; then - cp "${SSH_KEY}" . - ssh-keygen -f "${mcp_ssh_key}" -y > "${mcp_ssh_key}.pub" - fi - - [ -f "${mcp_ssh_key}" ] || ssh-keygen -f "${mcp_ssh_key}" -N '' - sudo install -D -o "${user}" -m 0600 "${mcp_ssh_key}" "${SSH_KEY}" -} - -function get_base_image { - local base_image=$1 - local image_dir=$2 - - mkdir -p "${image_dir}" - wget --progress=dot:giga -P "${image_dir}" -N "${base_image}" -} - -function __kernel_modules { - # Load mandatory kernel modules: loop, nbd - local image_dir=$1 - test -e /dev/loop-control || sudo modprobe loop - if sudo modprobe nbd max_part=8 || sudo modprobe -f nbd max_part=8; then - return 0 - fi - if [ -e /dev/nbd0 ]; then return 0; fi # nbd might be inbuilt - # CentOS (or RHEL family in general) do not provide 'nbd' out of the box - echo "[WARN] 'nbd' kernel module cannot be loaded!" - if [ ! -e /etc/redhat-release ]; then - echo "[ERROR] Non-RHEL system detected, aborting!" - echo "[ERROR] Try building 'nbd' manually or install it from a 3rd party." - exit 1 - fi - - # Best-effort attempt at building a non-maintaned kernel module - local __baseurl - local __subdir - local __uname_r=$(uname -r) - local __uname_m=$(uname -m) - if [ "${__uname_m}" = 'x86_64' ]; then - __baseurl='http://vault.centos.org/centos' - __subdir='Source/SPackages' - __srpm="kernel-${__uname_r%.${__uname_m}}.src.rpm" - else - __baseurl='http://vault.centos.org/altarch' - __subdir="Source/${__uname_m}/Source/SPackages" - # NOTE: fmt varies across releases (e.g. kernel-alt-4.11.0-44.el7a.src.rpm) - __srpm="kernel-alt-${__uname_r%.${__uname_m}}.src.rpm" - fi - - local __found='n' - local __versions=$(curl -s "${__baseurl}/" | grep -Po 'href="\K7\.[\d\.]+') - for ver in ${__versions}; do - for comp in os updates; do - local url="${__baseurl}/${ver}/${comp}/${__subdir}/${__srpm}" - if wget "${url}" -O "${image_dir}/${__srpm}" > /dev/null 2>&1; then - __found='y'; break 2 - fi - done - done - - if [ "${__found}" = 'n' ]; then - echo "[ERROR] Can't find the linux kernel SRPM for: ${__uname_r}" - echo "[ERROR] 'nbd' module cannot be built, aborting!" - echo "[ERROR] Try 'yum upgrade' or building 'nbd' krn module manually ..." - exit 1 - fi - - rpm -ivh "${image_dir}/${__srpm}" 2> /dev/null - mkdir -p ~/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} - # shellcheck disable=SC2016 - echo '%_topdir %(echo $HOME)/rpmbuild' > ~/.rpmmacros - ( - cd ~/rpmbuild/SPECS - rpmbuild -bp --nodeps --target="${__uname_m}" kernel*.spec - cd ~/rpmbuild/BUILD/"${__srpm%.src.rpm}"/linux-* - sed -i 's/^.*\(CONFIG_BLK_DEV_NBD\).*$/\1=m/g' .config - # http://centosfaq.org/centos/nbd-does-not-compile-for-3100-514262el7x86_64 - if grep -Rq 'REQ_TYPE_DRV_PRIV' drivers/block; then - sed -i 's/REQ_TYPE_SPECIAL/REQ_TYPE_DRV_PRIV/g' drivers/block/nbd.c - fi - gunzip -c "/boot/symvers-${__uname_r}.gz" > Module.symvers - make prepare modules_prepare - make M=drivers/block -j - modinfo drivers/block/nbd.ko - sudo mkdir -p "/lib/modules/${__uname_r}/extra/" - sudo cp drivers/block/nbd.ko "/lib/modules/${__uname_r}/extra/" - ) - sudo depmod -a - sudo modprobe nbd max_part=8 || sudo modprobe -f nbd max_part=8 -} - -function mount_image { - local image=$1 - local image_dir=$2 - OPNFV_MNT_DIR="${image_dir}/ubuntu" - - # Find free nbd, loop devices - for dev in '/sys/class/block/nbd'*; do - if [ "$(cat "${dev}/size")" = '0' ]; then - OPNFV_NBD_DEV=/dev/$(basename "${dev}") - break - fi - done - OPNFV_LOOP_DEV=$(sudo losetup -f) - OPNFV_MAP_DEV=/dev/mapper/$(basename "${OPNFV_NBD_DEV}")p1 - export OPNFV_MNT_DIR OPNFV_LOOP_DEV - [ -n "${OPNFV_NBD_DEV}" ] && [ -n "${OPNFV_LOOP_DEV}" ] || exit 1 - qemu-img resize "${image_dir}/${image}" 3G - sudo qemu-nbd --connect="${OPNFV_NBD_DEV}" --aio=native --cache=none \ - "${image_dir}/${image}" - sudo kpartx -av "${OPNFV_NBD_DEV}" - sleep 5 # /dev/nbdNp1 takes some time to come up - # Hardcode partition index to 1, unlikely to change for Ubuntu UCA image - if sudo growpart "${OPNFV_NBD_DEV}" 1; then - sudo kpartx -u "${OPNFV_NBD_DEV}" - sudo e2fsck -pf "${OPNFV_MAP_DEV}" - sudo resize2fs "${OPNFV_MAP_DEV}" - fi - # grub-update does not like /dev/nbd*, so use a loop device to work around it - sudo losetup "${OPNFV_LOOP_DEV}" "${OPNFV_MAP_DEV}" - mkdir -p "${OPNFV_MNT_DIR}" - sudo mount "${OPNFV_LOOP_DEV}" "${OPNFV_MNT_DIR}" - sudo mount -t proc proc "${OPNFV_MNT_DIR}/proc" - sudo mount -t sysfs sys "${OPNFV_MNT_DIR}/sys" - sudo mount -o bind /dev "${OPNFV_MNT_DIR}/dev" - sudo mkdir -p "${OPNFV_MNT_DIR}/run/resolvconf" - sudo cp /etc/resolv.conf "${OPNFV_MNT_DIR}/run/resolvconf" - echo "GRUB_DISABLE_OS_PROBER=true" | \ - sudo tee -a "${OPNFV_MNT_DIR}/etc/default/grub" - sudo sed -i -e 's/^\(GRUB_TIMEOUT\)=.*$/\1=1/g' -e 's/^GRUB_HIDDEN.*$//g' \ - "${OPNFV_MNT_DIR}/etc/default/grub" -} - -function apt_repos_pkgs_image { - local apt_key_urls=(${1//,/ }) - local all_repos=(${2//,/ }) - local pkgs_i=(${3//,/ }) - local pkgs_r=(${4//,/ }) - [ -n "${OPNFV_MNT_DIR}" ] || exit 1 - - # APT keys - if [ "${#apt_key_urls[@]}" -gt 0 ]; then - for apt_key in "${apt_key_urls[@]}"; do - sudo chroot "${OPNFV_MNT_DIR}" /bin/bash -c \ - "wget -qO - '${apt_key}' | apt-key add -" - done - fi - # Additional repositories - for repo_line in "${all_repos[@]}"; do - # <repo_name>|<repo prio>|deb|[arch=<arch>]|<repo url>|<dist>|<repo comp> - local repo=(${repo_line//|/ }) - [ "${#repo[@]}" -gt 5 ] || continue - # NOTE: Names and formatting are compatible with Salt linux.system.repo - cat <<-EOF | sudo tee "${OPNFV_MNT_DIR}/etc/apt/preferences.d/${repo[0]}" - - Package: * - Pin: release a=${repo[-2]} - Pin-Priority: ${repo[1]} - - EOF - echo "${repo[@]:2}" | sudo tee \ - "${OPNFV_MNT_DIR}/etc/apt/sources.list.d/${repo[0]}.list" - done - # Install packages - if [ "${#pkgs_i[@]}" -gt 0 ]; then - sudo DEBIAN_FRONTEND="noninteractive" \ - chroot "${OPNFV_MNT_DIR}" apt-get update - sudo DEBIAN_FRONTEND="noninteractive" FLASH_KERNEL_SKIP="true" \ - chroot "${OPNFV_MNT_DIR}" apt-get install -y "${pkgs_i[@]}" - fi - # Remove packages - if [ "${#pkgs_r[@]}" -gt 0 ]; then - sudo DEBIAN_FRONTEND="noninteractive" FLASH_KERNEL_SKIP="true" \ - chroot "${OPNFV_MNT_DIR}" apt-get purge -y "${pkgs_r[@]}" - fi - # Disable cloud-init metadata service datasource - sudo mkdir -p "${OPNFV_MNT_DIR}/etc/cloud/cloud.cfg.d" - echo "datasource_list: [ NoCloud, None ]" | sudo tee \ - "${OPNFV_MNT_DIR}/etc/cloud/cloud.cfg.d/95_real_datasources.cfg" -} - -function cleanup_mounts { - # Remove any mounts, loop and/or nbd devs created while patching base image - if [ -n "${OPNFV_MNT_DIR}" ] && [ -d "${OPNFV_MNT_DIR}" ]; then - if [ -f "${OPNFV_MNT_DIR}/boot/grub/grub.cfg" ]; then - # Grub thinks it's running from a live CD - sudo sed -i -e 's/^\s*set root=.*$//g' -e 's/^\s*loopback.*$//g' \ - "${OPNFV_MNT_DIR}/boot/grub/grub.cfg" - fi - sudo rm -f "${OPNFV_MNT_DIR}/run/resolvconf/resolv.conf" - sync - if mountpoint -q "${OPNFV_MNT_DIR}"; then - sudo umount -l "${OPNFV_MNT_DIR}" || true - fi - fi - if [ -n "${OPNFV_LOOP_DEV}" ] && \ - sudo losetup "${OPNFV_LOOP_DEV}" 1>&2 > /dev/null; then - sudo losetup -d "${OPNFV_LOOP_DEV}" - fi - if [ -n "${OPNFV_NBD_DEV}" ]; then - sudo kpartx -d "${OPNFV_NBD_DEV}" || true - sudo qemu-nbd -d "${OPNFV_NBD_DEV}" || true - fi -} - -function cleanup_uefi { - # Clean up Ubuntu boot entry if cfg01, kvm nodes online from previous deploy - local cmd_str="ssh ${SSH_OPTS} ${SSH_SALT}" - ping -c 1 -w 1 "${SALT_MASTER}" || return 0 - [ ! "$(hostname)" = 'cfg01' ] || cmd_str='eval' - ${cmd_str} "sudo salt -C 'kvm* or cmp*' cmd.run \ - \"which efibootmgr > /dev/null 2>&1 && \ - efibootmgr | grep -oP '(?<=Boot)[0-9]+(?=.*ubuntu)' | \ - xargs -I{} efibootmgr --delete-bootnum --bootnum {}; \ - rm -rf /boot/efi/*\"" || true -} - -function cleanup_vms { - # clean up existing nodes - for node in $(virsh list --name | grep -P '\w{3}\d{2}'); do - virsh destroy "${node}" - done - for node in $(virsh list --name --all | grep -P '\w{3}\d{2}'); do - virsh domblklist "${node}" | awk '/^.da/ {print $2}' | \ - xargs --no-run-if-empty -I{} sudo rm -f {} - virsh undefine "${node}" --remove-all-storage --nvram - done -} - -function prepare_vms { - local base_image=$1; shift - local image_dir=$1; shift - local repos_pkgs_str=$1; shift # ^-sep list of repos, pkgs to install/rm - local vnodes=("$@") - local image=base_image_opnfv_fuel.img - local vcp_image=${image%.*}_vcp.img - local _o=${base_image/*\/} - local _h=$(echo "${repos_pkgs_str}.$(md5sum "${image_dir}/${_o}")" | \ - md5sum | cut -c -8) - local _tmp - - cleanup_uefi - cleanup_vms - get_base_image "${base_image}" "${image_dir}" - IFS='^' read -r -a repos_pkgs <<< "${repos_pkgs_str}" - - echo "[INFO] Lookup cache / build patched base image for fingerprint: ${_h}" - _tmp="${image%.*}.${_h}.img" - if [ "${image_dir}/${_tmp}" -ef "${image_dir}/${image}" ]; then - echo "[INFO] Patched base image found" - else - rm -f "${image_dir}/${image%.*}"* - if [[ ! "${repos_pkgs_str}" =~ ^\^+$ ]]; then - echo "[INFO] Patching base image ..." - cp "${image_dir}/${_o}" "${image_dir}/${_tmp}" - __kernel_modules "${image_dir}" - mount_image "${_tmp}" "${image_dir}" - apt_repos_pkgs_image "${repos_pkgs[@]:0:4}" - cleanup_mounts - else - echo "[INFO] No patching required, using vanilla base image" - ln -sf "${image_dir}/${_o}" "${image_dir}/${_tmp}" - fi - ln -sf "${image_dir}/${_tmp}" "${image_dir}/${image}" - fi - - # Create config ISO and resize OS disk image for each foundation node VM - for node in "${vnodes[@]}"; do - if [[ "${node}" =~ ^(cfg01|mas01) ]]; then - user_data='user-data.mcp.sh' - else - user_data='user-data.admin.sh' - fi - ./create-config-drive.sh -k "$(basename "${SSH_KEY}").pub" \ - -u "${user_data}" -h "${node}" "${image_dir}/mcp_${node}.iso" - cp "${image_dir}/${image}" "${image_dir}/mcp_${node}.qcow2" - qemu-img resize "${image_dir}/mcp_${node}.qcow2" 100G - # Prepare dedicated drive for cinder on cmp nodes - if [[ "${node}" =~ ^(cmp) ]]; then - qemu-img create "${image_dir}/mcp_${node}_storage.qcow2" 100G - fi - done - - # VCP VMs base image specific changes - if [[ ! "${repos_pkgs_str}" =~ \^{3}$ ]] && [ -n "${repos_pkgs[*]:4}" ]; then - echo "[INFO] Lookup cache / build patched VCP image for md5sum: ${_h}" - _tmp="${vcp_image%.*}.${_h}.img" - if [ "${image_dir}/${_tmp}" -ef "${image_dir}/${vcp_image}" ]; then - echo "[INFO] Patched VCP image found" - else - echo "[INFO] Patching VCP image ..." - cp "${image_dir}/${image}" "${image_dir}/${_tmp}" - __kernel_modules "${image_dir}" - mount_image "${_tmp}" "${image_dir}" - apt_repos_pkgs_image "${repos_pkgs[@]:4:4}" - cleanup_mounts - ln -sf "${image_dir}/${_tmp}" "${image_dir}/${vcp_image}" - fi - fi -} - -function jumpserver_pkg_install { - if [ -n "$(command -v apt-get)" ]; then - pkg_type='deb'; pkg_cmd='sudo apt-get install -y' - else - pkg_type='rpm'; pkg_cmd='sudo yum install -y --skip-broken' - fi - eval "$(parse_yaml "./requirements_${pkg_type}.yaml")" - for section in 'common' "$(uname -i)"; do - section_var="requirements_pkg_${section}[*]" - pkg_list+=" ${!section_var}" - done - # shellcheck disable=SC2086 - ${pkg_cmd} ${pkg_list} -} - -function jumpserver_check_requirements { - # shellcheck disable=SC2178 - local vnodes=$1; shift - local br=("$@") - local err_br_not_found='Linux bridge not found!' - local err_br_virsh_net='is a virtual network, Linux bridge expected!' - local warn_br_endpoint="Endpoints might be inaccessible from external hosts!" - # MaaS requires a Linux bridge for PXE/admin - if [[ "${vnodes}" =~ mas01 ]]; then - if ! brctl showmacs "${br[0]}" >/dev/null 2>&1; then - notify_e "[ERROR] PXE/admin (${br[0]}) ${err_br_not_found}" - fi - # Assume virsh network name matches bridge name (true if created by us) - if virsh net-info "${br[0]}" >/dev/null 2>&1; then - notify_e "[ERROR] ${br[0]} ${err_br_virsh_net}" - fi - fi - # If virtual nodes are present, public should be a Linux bridge - if [ "$(echo "${vnodes}" | wc -w)" -gt 2 ]; then - if ! brctl showmacs "${br[3]}" >/dev/null 2>&1; then - if [[ "${vnodes}" =~ mas01 ]]; then - # Baremetal nodes *require* a proper public network - notify_e "[ERROR] Public (${br[3]}) ${err_br_not_found}" - else - notify_n "[WARN] Public (${br[3]}) ${err_br_not_found}" 3 - notify_n "[WARN] ${warn_br_endpoint}" 3 - fi - fi - if virsh net-info "${br[3]}" >/dev/null 2>&1; then - if [[ "${vnodes}" =~ mas01 ]]; then - notify_e "[ERROR] ${br[3]} ${err_br_virsh_net}" - else - notify_n "[WARN] ${br[3]} ${err_br_virsh_net}" 3 - notify_n "[WARN] ${warn_br_endpoint}" 3 - fi - fi - fi -} - -function create_networks { - local vnode_networks=("$@") - # create required networks, including constant "mcpcontrol" - for net in "mcpcontrol" "${vnode_networks[@]}"; do - if virsh net-info "${net}" >/dev/null 2>&1; then - virsh net-destroy "${net}" || true - virsh net-undefine "${net}" - fi - # in case of custom network, host should already have the bridge in place - if [ -f "virsh_net/net_${net}.xml" ] && \ - [ ! -d "/sys/class/net/${net}/bridge" ]; then - virsh net-define "virsh_net/net_${net}.xml" - virsh net-autostart "${net}" - virsh net-start "${net}" - fi - done -} - -function create_vms { - local image_dir=$1; shift - # vnode data should be serialized with the following format: - # '<name0>,<ram0>,<vcpu0>|<name1>,<ram1>,<vcpu1>[...]' - IFS='|' read -r -a vnodes <<< "$1"; shift - - # AArch64: prepare arch specific arguments - local virt_extra_args="" - if [ "$(uname -i)" = "aarch64" ]; then - # No Cirrus VGA on AArch64, use virtio instead - virt_extra_args="$virt_extra_args --video=virtio" - fi - - # create vms with specified options - for serialized_vnode_data in "${vnodes[@]}"; do - IFS=',' read -r -a vnode_data <<< "${serialized_vnode_data}" - - # prepare network args - local vnode_networks=("$@") - if [[ "${vnode_data[0]}" =~ ^(cfg01|mas01) ]]; then - net_args=" --network network=mcpcontrol,model=virtio" - # 3rd interface gets connected to PXE/Admin Bridge (cfg01, mas01) - vnode_networks[2]="${vnode_networks[0]}" - else - net_args=" --network bridge=${vnode_networks[0]},model=virtio" - fi - for net in "${vnode_networks[@]:1}"; do - net_args="${net_args} --network bridge=${net},model=virtio" - done - - # dedicated storage drive for cinder on cmp nodes - virt_extra_storage= - if [[ "${vnode_data[0]}" =~ ^(cmp) ]]; then - virt_extra_storage="--disk path=${image_dir}/mcp_${vnode_data[0]}_storage.qcow2,format=qcow2,bus=virtio,cache=none,io=native" - fi - - # shellcheck disable=SC2086 - virt-install --name "${vnode_data[0]}" \ - --ram "${vnode_data[1]}" --vcpus "${vnode_data[2]}" \ - --cpu host-passthrough --accelerate ${net_args} \ - --disk path="${image_dir}/mcp_${vnode_data[0]}.qcow2",format=qcow2,bus=virtio,cache=none,io=native \ - ${virt_extra_storage} \ - --os-type linux --os-variant none \ - --boot hd --vnc --console pty --autostart --noreboot \ - --disk path="${image_dir}/mcp_${vnode_data[0]}.iso",device=cdrom \ - --noautoconsole \ - ${virt_extra_args} - done -} - -function update_mcpcontrol_network { - # set static ip address for salt master node, MaaS node - local cmac=$(virsh domiflist cfg01 2>&1| awk '/mcpcontrol/ {print $5; exit}') - local amac=$(virsh domiflist mas01 2>&1| awk '/mcpcontrol/ {print $5; exit}') - virsh net-update "mcpcontrol" add ip-dhcp-host \ - "<host mac='${cmac}' name='cfg01' ip='${SALT_MASTER}'/>" --live --config - [ -z "${amac}" ] || virsh net-update "mcpcontrol" add ip-dhcp-host \ - "<host mac='${amac}' name='mas01' ip='${MAAS_IP}'/>" --live --config -} - -function start_vms { - local vnodes=("$@") - - # start vms - for node in "${vnodes[@]}"; do - virsh start "${node}" - sleep $((RANDOM%5+1)) - done -} - -function check_connection { - local total_attempts=60 - local sleep_time=5 - - set +e - echo '[INFO] Attempting to get into Salt master ...' - - # wait until ssh on Salt master is available - # shellcheck disable=SC2034 - for attempt in $(seq "${total_attempts}"); do - # shellcheck disable=SC2086 - ssh ${SSH_OPTS} "ubuntu@${SALT_MASTER}" uptime - case $? in - 0) echo "${attempt}> Success"; break ;; - *) echo "${attempt}/${total_attempts}> ssh server ain't ready yet, waiting for ${sleep_time} seconds ..." ;; - esac - sleep $sleep_time - done - set -e -} - -function parse_yaml { - local prefix=$2 - local s - local w - local fs - s='[[:space:]]*' - w='[a-zA-Z0-9_]*' - fs="$(echo @|tr @ '\034')" - sed -e 's|---||g' -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ - -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" "$1" | - awk -F"$fs" '{ - indent = length($1)/2; - vname[indent] = $2; - for (i in vname) {if (i > indent) {delete vname[i]}} - if (length($3) > 0) { - vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")} - printf("%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, $3); - } - }' | sed 's/_=/+=/g' -} - function wait_for { # Execute in a subshell to prevent local variable override during recursion ( @@ -527,16 +35,18 @@ function wait_for { ) } -function do_sysctl_cfg { - local _conf='/etc/sysctl.d/99-opnfv-fuel-bridge.conf' - # https://wiki.libvirt.org/page/Net.bridge.bridge-nf-call_and_sysctl.conf - if modprobe br_netfilter bridge; then - echo 'net.bridge.bridge-nf-call-arptables = 0' |& sudo tee "${_conf}" - echo 'net.bridge.bridge-nf-call-iptables = 0' |& sudo tee -a "${_conf}" - echo 'net.bridge.bridge-nf-call-ip6tables = 0' |& sudo tee -a "${_conf}" - # Some distros / sysadmins explicitly blacklist br_netfilter - sudo sysctl -q -p "${_conf}" || true - fi +function cleanup_uefi { + # Clean up Ubuntu boot entry if cfg01, baremetal nodes online from previous deploy + local cmd_str="ssh ${SSH_OPTS} ${SSH_SALT}" + ping -c 1 -w 1 "${SALT_MASTER}" || return 0 + [ ! "$(hostname)" = 'cfg01' ] || cmd_str='eval' + ${cmd_str} "sudo salt -C 'G@virtual:physical and not cfg01*' cmd.run \ + \"which efibootmgr > /dev/null 2>&1 && \ + efibootmgr | grep -oP '(?<=Boot)[0-9]+(?=.*ubuntu)' | \ + xargs -I{} efibootmgr --delete-bootnum --bootnum {}; \ + rm -rf /boot/efi/*\"" || true + + ${cmd_str} "sudo salt -C 'G@virtual:physical and not cfg01*' cmd.run 'shutdown now'" || true } function get_nova_compute_pillar_data { diff --git a/mcp/scripts/lib_jump_common.sh b/mcp/scripts/lib_jump_common.sh new file mode 100644 index 000000000..c2bd46649 --- /dev/null +++ b/mcp/scripts/lib_jump_common.sh @@ -0,0 +1,213 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2018 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# +# Library of shell functions used by build / deploy scripts on jumpserver: +# - distro package requirements installation (e.g. DEB, RPM); +# - other package requirements from custom sources (e.g. docker); +# - jumpserver prerequisites validation (e.g. network bridges); +# - distro configuration (e.g. udev, sysctl); +# etc. + +############################################################################## +# private helper functions +############################################################################## + +function __parse_yaml { + local prefix=$2 + local s + local w + local fs + s='[[:space:]]*' + w='[a-zA-Z0-9_]*' + fs="$(echo @|tr @ '\034')" + sed -e 's|---||g' -ne "s|^\($s\)\($w\)$s:$s\"\(.*\)\"$s\$|\1$fs\2$fs\3|p" \ + -e "s|^\($s\)\($w\)$s[:-]$s\(.*\)$s\$|\1$fs\2$fs\3|p" "$1" | + awk -F"$fs" '{ + indent = length($1)/2; + vname[indent] = $2; + for (i in vname) {if (i > indent) {delete vname[i]}} + if (length($3) > 0) { + vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")} + printf("%s%s%s=(\"%s\")\n", "'"$prefix"'",vn, $2, $3); + } + }' | sed 's/_=/+=/g' +} + +############################################################################## +# public functions +############################################################################## + +function jumpserver_pkg_install { + local req_type=$1 + if [ -n "$(command -v apt-get)" ]; then + pkg_type='deb'; pkg_cmd='sudo apt-get install -y' + else + pkg_type='rpm'; pkg_cmd='sudo yum install -y --skip-broken' + fi + eval "$(__parse_yaml "./requirements_${pkg_type}.yaml")" + for section in 'common' "$(uname -i)"; do + section_var="${req_type}_${section}[*]" + pkg_list+=" ${!section_var}" + done + # shellcheck disable=SC2086 + ${pkg_cmd} ${pkg_list} +} + +function jumpserver_check_requirements { + # shellcheck disable=SC2178 + local states=$1; shift + # shellcheck disable=SC2178 + local vnodes=$1; shift + local br=("$@") + local err_br_not_found='Linux bridge not found!' + local err_br_virsh_net='is a virtual network, Linux bridge expected!' + local warn_br_endpoint="Endpoints might be inaccessible from external hosts!" + # MaaS requires a Linux bridge for PXE/admin + if [[ "${states}" =~ maas ]]; then + if ! brctl showmacs "${br[0]}" >/dev/null 2>&1; then + notify_e "[ERROR] PXE/admin (${br[0]}) ${err_br_not_found}" + fi + # Assume virsh network name matches bridge name (true if created by us) + if ${VIRSH} net-info "${br[0]}" >/dev/null 2>&1; then + notify_e "[ERROR] ${br[0]} ${err_br_virsh_net}" + fi + fi + # If virtual nodes are present, public should be a Linux bridge + if [ -n "${vnodes}" ]; then + if ! brctl showmacs "${br[3]}" >/dev/null 2>&1; then + if [[ "${states}" =~ maas ]]; then + # Baremetal nodes *require* a proper public network + notify_e "[ERROR] Public (${br[3]}) ${err_br_not_found}" + else + notify_n "[WARN] Public (${br[3]}) ${err_br_not_found}" 3 + notify_n "[WARN] ${warn_br_endpoint}" 3 + fi + fi + if ${VIRSH} net-info "${br[3]}" >/dev/null 2>&1; then + if [[ "${states}" =~ maas ]]; then + notify_e "[ERROR] ${br[3]} ${err_br_virsh_net}" + else + notify_n "[WARN] ${br[3]} ${err_br_virsh_net}" 3 + notify_n "[WARN] ${warn_br_endpoint}" 3 + fi + fi + # https://bugs.launchpad.net/ubuntu/+source/qemu/+bug/1797332 + if lsb_release -d | grep -q -e 'Ubuntu 16.04'; then + if uname -r | grep -q -e '^4\.4\.'; then + notify_n "[WARN] Host kernel too old; nested virtualization issues!" 3 + notify_n "[WARN] apt install linux-generic-hwe-16.04 && reboot" 3 + notify_e "[ERROR] Please upgrade the kernel and reboot!" + fi + fi + fi +} + +function docker_install { + local image_dir=$1 + # Mininum effort attempt at installing Docker if missing + if ! docker --version; then + curl -fsSL https://get.docker.com -o get-docker.sh + sudo sh get-docker.sh + rm get-docker.sh + # On RHEL distros, the Docker service should be explicitly started + sudo systemctl start docker + else + DOCKER_VER=$(docker version --format '{{.Server.Version}}') + if [ "${DOCKER_VER%%.*}" -lt 2 ]; then + notify_e "[ERROR] Docker version ${DOCKER_VER} is too old, please upgrade it." + fi + fi + # Distro-provided docker-compose might be simply broken (Ubuntu 16.04, CentOS 7) + if ! docker-compose --version > /dev/null 2>&1 || \ + [ "$(docker-compose version --short | tr -d '.')" -lt 1220 ] && \ + [ "$(uname -m)" = 'x86_64' ]; then + COMPOSE_BIN="${image_dir}/docker-compose" + COMPOSE_VERSION='1.22.0' + notify_n "[WARN] Using docker-compose ${COMPOSE_VERSION} in ${COMPOSE_BIN}" 3 + if [ ! -e "${COMPOSE_BIN}" ]; then + COMPOSE_URL="https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}" + sudo curl -L "${COMPOSE_URL}/docker-compose-$(uname -s)-$(uname -m)" -o "${COMPOSE_BIN}" + sudo chmod +x "${COMPOSE_BIN}" + fi + fi +} + +function e2fsprogs_install { + local image_dir=$1 + E2FS_VER=$(e2fsck -V 2>&1 | grep -Pzo "e2fsck \K1\.\d{2}") + if [ "${E2FS_VER//./}" -lt 143 ]; then + E2FS_TGZ="${image_dir}/e2fsprogs.tar.gz" + E2FS_VER='1.43.9' + E2FS_URL="https://git.kernel.org/pub/scm/fs/ext2/e2fsprogs.git/snapshot/e2fsprogs-${E2FS_VER}.tar.gz" + notify_n "[WARN] Using e2fsprogs ${E2FS_VER} from ${E2FS_TGZ}" 3 + if [ ! -e "${E2FS_TGZ}" ]; then + curl -L "${E2FS_URL}" -o "${E2FS_TGZ}" + mkdir -p "${image_dir}/e2fsprogs" + tar xzf "${E2FS_TGZ}" -C "${image_dir}/e2fsprogs" --strip-components=1 + cd "${image_dir}/e2fsprogs" || exit 1 + ./configure + make + cd - || exit 1 + fi + fi +} + +function virtinst_install { + local image_dir=$1 + VIRT_VER=$(virt-install --version 2>&1) + if [ "${VIRT_VER//./}" -lt 140 ]; then + VIRT_TGZ="${image_dir}/virt-manager.tar.gz" + VIRT_VER='1.4.3' + VIRT_URL="https://github.com/virt-manager/virt-manager/archive/v${VIRT_VER}.tar.gz" + notify_n "[WARN] Using virt-install ${VIRT_VER} from ${VIRT_TGZ}" 3 + if [ ! -e "${VIRT_TGZ}" ]; then + curl -L "${VIRT_URL}" -o "${VIRT_TGZ}" + mkdir -p "${image_dir}/virt-manager" + tar xzf "${VIRT_TGZ}" -C "${image_dir}/virt-manager" --strip-components=1 + fi + fi +} + +function do_udev_cfg { + local _conf='/etc/udev/rules.d/99-opnfv-fuel-vnet-mtu.rules' + # http://linuxaleph.blogspot.com/2013/01/how-to-network-jumbo-frames-to-kvm-guest.html + echo 'SUBSYSTEM=="net", ACTION=="add|change", KERNEL=="vnet*", RUN+="/bin/sh -c '"'/bin/sleep 1; /sbin/ip link set %k mtu 9000'\"" |& sudo tee "${_conf}" + echo 'SUBSYSTEM=="net", ACTION=="add|change", KERNEL=="*-nic", RUN+="/bin/sh -c '"'/bin/sleep 1; /sbin/ip link set %k mtu 9000'\"" |& sudo tee -a "${_conf}" + sudo udevadm control --reload + sudo udevadm trigger +} + +function do_sysctl_cfg { + local _conf='/etc/sysctl.d/99-opnfv-fuel-bridge.conf' + # https://wiki.libvirt.org/page/Net.bridge.bridge-nf-call_and_sysctl.conf + if modprobe br_netfilter bridge; then + echo 'net.bridge.bridge-nf-call-arptables = 0' |& sudo tee "${_conf}" + echo 'net.bridge.bridge-nf-call-iptables = 0' |& sudo tee -a "${_conf}" + echo 'net.bridge.bridge-nf-call-ip6tables = 0' |& sudo tee -a "${_conf}" + # Some distros / sysadmins explicitly blacklist br_netfilter + sudo sysctl -q -p "${_conf}" || true + fi +} + +function generate_ssh_key { + # shellcheck disable=SC2155 + local mcp_ssh_key=$(basename "${SSH_KEY}") + local user=${USER} + if [ -n "${SUDO_USER}" ] && [ "${SUDO_USER}" != 'root' ]; then + user=${SUDO_USER} + fi + + if [ -f "${SSH_KEY}" ]; then + cp "${SSH_KEY}" . + ssh-keygen -f "${mcp_ssh_key}" -y > "${mcp_ssh_key}.pub" + fi + + [ -f "${mcp_ssh_key}" ] || ssh-keygen -f "${mcp_ssh_key}" -N '' + sudo install -D -o "${user}" -m 0600 "${mcp_ssh_key}" "${SSH_KEY}" +} diff --git a/mcp/scripts/lib_jump_deploy.sh b/mcp/scripts/lib_jump_deploy.sh new file mode 100644 index 000000000..3dc3c4436 --- /dev/null +++ b/mcp/scripts/lib_jump_deploy.sh @@ -0,0 +1,591 @@ +#!/bin/bash -e +# shellcheck disable=SC2155,SC1001,SC2015,SC2128 +############################################################################## +# Copyright (c) 2018 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +# +# Library of shell functions used by deploy script on jumpserver: +# - base cloud image (used by FN VMs and VCP VMs) processing: +# * download; +# * tooling for offline image modification (without libguestfs); +# * package pre-installation (requires nbd, loop krn mods); +# - virtualized hosts processing: +# * virsh-managed VMs boilerplate; +# * salt master container tooling; +# * virsh & docker network plumbing; +# etc. + +############################################################################## +# private helper functions +############################################################################## + +function __get_base_image { + local base_image=$1 + local image_dir=$2 + + mkdir -p "${image_dir}" + wget --progress=dot:giga -P "${image_dir}" -N "${base_image}" +} + +function __kernel_modules { + # Load mandatory kernel modules: loop, nbd + local image_dir=$1 + test -e /dev/loop-control || sudo modprobe loop + if sudo modprobe nbd max_part=8 || sudo modprobe -f nbd max_part=8; then + return 0 + fi + if [ -e /dev/nbd0 ]; then return 0; fi # nbd might be inbuilt + # CentOS (or RHEL family in general) do not provide 'nbd' out of the box + echo "[WARN] 'nbd' kernel module cannot be loaded!" + if [ ! -e /etc/redhat-release ]; then + echo "[ERROR] Non-RHEL system detected, aborting!" + echo "[ERROR] Try building 'nbd' manually or install it from a 3rd party." + exit 1 + fi + + # Best-effort attempt at building a non-maintaned kernel module + local __baseurl='http://vault.centos.org/centos' + local __subdir='Source/SPackages' + local __uname_r=$(uname -r) + local __uname_m=$(uname -m) + if [ "${__uname_m}" = 'x86_64' ]; then + __srpm="kernel-${__uname_r%.${__uname_m}}.src.rpm" + else + # NOTE: fmt varies across releases (e.g. kernel-alt-4.11.0-44.el7a.src.rpm) + __srpm="kernel-alt-${__uname_r%.${__uname_m}}.src.rpm" + fi + + local __found='n' + local __versions=$(curl -s "${__baseurl}/" | grep -Po 'href="\K7\.[\d\.]+') + for ver in ${__versions}; do + for comp in os updates; do + local url="${__baseurl}/${ver}/${comp}/${__subdir}/${__srpm}" + if wget "${url}" -O "${image_dir}/${__srpm}" > /dev/null 2>&1; then + __found='y'; break 2 + fi + done + done + + if [ "${__found}" = 'n' ]; then + echo "[ERROR] Can't find the linux kernel SRPM for: ${__uname_r}" + echo "[ERROR] 'nbd' module cannot be built, aborting!" + echo "[ERROR] Try 'yum upgrade' or building 'nbd' krn module manually ..." + exit 1 + fi + + rpm -ivh "${image_dir}/${__srpm}" 2> /dev/null + mkdir -p ~/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} + # shellcheck disable=SC2016 + echo '%_topdir %(echo $HOME)/rpmbuild' > ~/.rpmmacros + ( + cd ~/rpmbuild/SPECS + rpmbuild -bp --nodeps --target="${__uname_m}" kernel*.spec + cd ~/rpmbuild/BUILD/"${__srpm%.src.rpm}"/linux-* + sed -i 's/^.*\(CONFIG_BLK_DEV_NBD\).*$/\1=m/g' .config + # http://centosfaq.org/centos/nbd-does-not-compile-for-3100-514262el7x86_64 + if grep -Rq 'REQ_TYPE_DRV_PRIV' drivers/block; then + sed -i 's/REQ_TYPE_SPECIAL/REQ_TYPE_DRV_PRIV/g' drivers/block/nbd.c + fi + gunzip -c "/boot/symvers-${__uname_r}.gz" > Module.symvers + make prepare modules_prepare + make M=drivers/block -j + modinfo drivers/block/nbd.ko + sudo mkdir -p "/lib/modules/${__uname_r}/extra/" + sudo cp drivers/block/nbd.ko "/lib/modules/${__uname_r}/extra/" + ) + sudo depmod -a + sudo modprobe nbd max_part=8 || sudo modprobe -f nbd max_part=8 +} + +function __mount_image { + local image=$1 + local image_dir=$2 + OPNFV_MNT_DIR="${image_dir}/mnt" + + # Find free nbd, loop devices + for dev in '/sys/class/block/nbd'*; do + if [ "$(cat "${dev}/size")" = '0' ]; then + OPNFV_NBD_DEV=/dev/$(basename "${dev}") + break + fi + done + OPNFV_LOOP_DEV=$(sudo losetup -f) + OPNFV_MAP_DEV=/dev/mapper/$(basename "${OPNFV_NBD_DEV}")p1 + export OPNFV_MNT_DIR OPNFV_LOOP_DEV + [ -n "${OPNFV_NBD_DEV}" ] && [ -n "${OPNFV_LOOP_DEV}" ] || exit 1 + [[ "${MCP_OS:-}" =~ centos ]] || \ + qemu-img resize "${image_dir}/${image}" 3G + sudo qemu-nbd --connect="${OPNFV_NBD_DEV}" --aio=native --cache=none \ + "${image_dir}/${image}" + sudo kpartx -av "${OPNFV_NBD_DEV}" + # Hardcode partition index to 1, unlikely to change for Ubuntu UCA image + sudo partx -uvn 1:1 "${OPNFV_NBD_DEV}" + if [[ "${MCP_OS:-}" =~ ubuntu ]] && sudo growpart "${OPNFV_NBD_DEV}" 1 + then + if [ -e "${image_dir}/e2fsprogs" ]; then + E2FSCK_PREFIX="${image_dir}/e2fsprogs/e2fsck/" + RESIZE_PREFIX="${image_dir}/e2fsprogs/resize/" + fi + sudo kpartx -u "${OPNFV_NBD_DEV}" + sudo "${E2FSCK_PREFIX}e2fsck" -pf "${OPNFV_MAP_DEV}" + sudo "${RESIZE_PREFIX}resize2fs" "${OPNFV_MAP_DEV}" + else + sleep 5 # /dev/nbdNp1 takes some time to come up + fi + sudo partx -d "${OPNFV_NBD_DEV}" + mkdir -p "${OPNFV_MNT_DIR}" + if [ "$(uname -i)" = "aarch64" ] && [[ "${MCP_OS:-}" =~ centos ]]; then + # AArch64 CentOS cloud image contains a broken shim binary + # https://bugzilla.redhat.com/show_bug.cgi?id=1527283 + sudo mount "${OPNFV_MAP_DEV}" "${OPNFV_MNT_DIR}" + sudo cp -f --remove-destination "${OPNFV_MNT_DIR}/EFI/BOOT/fbaa64.efi" \ + "${OPNFV_MNT_DIR}/EFI/BOOT/BOOTAA64.EFI" + sudo umount -l "${OPNFV_MNT_DIR}" + # AArch64 CentOS cloud image has root partition at index 4 instead of 1 + sudo mount "${OPNFV_MAP_DEV/p1/p4}" "${OPNFV_MNT_DIR}" + else + # grub-update does not like /dev/nbd*, so use a loop device to work around it + sudo losetup "${OPNFV_LOOP_DEV}" "${OPNFV_MAP_DEV}" + sudo mount "${OPNFV_LOOP_DEV}" "${OPNFV_MNT_DIR}" + fi + sudo mount -t proc proc "${OPNFV_MNT_DIR}/proc" + sudo mount -t sysfs sys "${OPNFV_MNT_DIR}/sys" + sudo mount -o bind /dev "${OPNFV_MNT_DIR}/dev" + if [[ "${MCP_OS:-}" =~ ubuntu1804 ]]; then + # Ubuntu Bionic (18.04) or newer defaults to using netplan.io, revert it + sudo mkdir -p "${OPNFV_MNT_DIR}/run/systemd/resolve" + echo "nameserver ${dns_public}" | sudo tee \ + "${OPNFV_MNT_DIR}/run/systemd/resolve/stub-resolv.conf" + sudo chroot "${OPNFV_MNT_DIR}" systemctl stop \ + systemd-networkd.socket systemd-networkd \ + networkd-dispatcher systemd-networkd-wait-online systemd-resolved + sudo chroot "${OPNFV_MNT_DIR}" systemctl disable \ + systemd-networkd.socket systemd-networkd \ + networkd-dispatcher systemd-networkd-wait-online systemd-resolved + sudo chroot "${OPNFV_MNT_DIR}" systemctl mask \ + systemd-networkd.socket systemd-networkd \ + networkd-dispatcher systemd-networkd-wait-online systemd-resolved + sudo chroot "${OPNFV_MNT_DIR}" apt --assume-yes purge nplan netplan.io + echo "source /etc/network/interfaces.d/*" | \ + sudo tee "${OPNFV_MNT_DIR}/etc/network/interfaces" + elif [[ "${MCP_OS:-}" =~ centos ]]; then + sudo sed -i -e 's/^\(SELINUX\)=.*$/\1=permissive/g' \ + "${OPNFV_MNT_DIR}/etc/selinux/config" + fi + sudo rm -f "${OPNFV_MNT_DIR}/etc/resolv.conf" + echo "nameserver ${dns_public}" | sudo tee \ + "${OPNFV_MNT_DIR}/etc/resolv.conf" + echo "GRUB_DISABLE_OS_PROBER=true" | \ + sudo tee -a "${OPNFV_MNT_DIR}/etc/default/grub" + sudo sed -i -e 's/^\(GRUB_TIMEOUT\)=.*$/\1=1/g' -e 's/^GRUB_HIDDEN.*$//g' \ + "${OPNFV_MNT_DIR}/etc/default/grub" +} + +function __apt_repos_pkgs_image { + local apt_key_urls=(${1//,/ }) + local all_repos=(${2//,/ }) + local pkgs_i=(${3//,/ }) + local pkgs_r=(${4//,/ }) + [ -n "${OPNFV_MNT_DIR}" ] || exit 1 + + # NOTE: We don't support (yet) some features for non-APT repos: keys, prio + + # APT keys + if [[ "${MCP_OS:-}" =~ ubuntu ]] && [ "${#apt_key_urls[@]}" -gt 0 ]; then + for apt_key in "${apt_key_urls[@]}"; do + sudo chroot "${OPNFV_MNT_DIR}" /bin/bash -c \ + "wget -qO - '${apt_key}' | apt-key add -" + done + fi + # Additional repositories + for repo_line in "${all_repos[@]}"; do + # <repo_name>|<repo prio>|deb|[arch=<arch>]|<repo url>|<dist>|<repo comp> + local repo=(${repo_line//|/ }) + + if [[ "${MCP_OS:-}" =~ centos ]]; then + cat <<-EOF | sudo tee "${OPNFV_MNT_DIR}/etc/yum.repos.d/${repo[0]}.repo" + [${repo[0]}] + baseurl=${repo[3]} + enabled=1 + gpgcheck=0 + EOF + continue + fi + [ "${#repo[@]}" -gt 5 ] || continue + # NOTE: Names and formatting are compatible with Salt linux.system.repo + cat <<-EOF | sudo tee "${OPNFV_MNT_DIR}/etc/apt/preferences.d/${repo[0]}" + + Package: * + Pin: release a=${repo[-2]} + Pin-Priority: ${repo[1]} + + EOF + echo "${repo[@]:2}" | sudo tee \ + "${OPNFV_MNT_DIR}/etc/apt/sources.list.d/${repo[0]}.list" + done + # Install packages + if [ "${#pkgs_i[@]}" -gt 0 ]; then + if [[ "${MCP_OS:-}" =~ ubuntu ]]; then + sudo DEBIAN_FRONTEND="noninteractive" \ + chroot "${OPNFV_MNT_DIR}" apt-get update + sudo DEBIAN_FRONTEND="noninteractive" FLASH_KERNEL_SKIP="true" \ + chroot "${OPNFV_MNT_DIR}" apt-get install -y "${pkgs_i[@]}" + else + sudo chroot "${OPNFV_MNT_DIR}" yum install -y "${pkgs_i[@]}" + fi + fi + # Remove packages + if [ "${#pkgs_r[@]}" -gt 0 ]; then + if [[ "${MCP_OS:-}" =~ ubuntu ]]; then + sudo DEBIAN_FRONTEND="noninteractive" FLASH_KERNEL_SKIP="true" \ + chroot "${OPNFV_MNT_DIR}" apt-get purge -y "${pkgs_r[@]}" + else + sudo chroot "${OPNFV_MNT_DIR}" yum remove -y "${pkgs_r[@]}" + fi + fi + # Disable cloud-init metadata service datasource + sudo mkdir -p "${OPNFV_MNT_DIR}/etc/cloud/cloud.cfg.d" + echo "datasource_list: [ NoCloud, None ]" | sudo tee \ + "${OPNFV_MNT_DIR}/etc/cloud/cloud.cfg.d/95_real_datasources.cfg" +} + +function __cleanup_vms { + # clean up existing nodes + for node in $(${VIRSH} list --name | grep -P '\w{3}\d{2}'); do + ${VIRSH} destroy "${node}" 2>/dev/null || true + done + for node in $(${VIRSH} list --name --all | grep -P '\w{3}\d{2}'); do + ${VIRSH} domblklist "${node}" | awk '/^.da/ {print $2}' | \ + xargs --no-run-if-empty -I{} sudo rm -f {} + ${VIRSH} undefine "${node}" --remove-all-storage --nvram || \ + ${VIRSH} undefine "${node}" --remove-all-storage + done +} + +############################################################################## +# public functions +############################################################################## + +function prepare_vms { + local base_image_f=$1; shift + local base_image=${base_image_f%.xz} + local image_dir=$1; shift + local repos_pkgs_str=$1; shift # ^-sep list of repos, pkgs to install/rm + local image=base_image_opnfv_fuel.img + local vcp_image=${image%.*}_vcp.img + local _o=${base_image/*\/} + [ -n "${image_dir}" ] || exit 1 + + cleanup_uefi + __cleanup_vms + __get_base_image "${base_image_f}" "${image_dir}" + [ "${base_image}" == "${base_image_f}" ] || unxz -fk "${image_dir}/${_o}.xz" + IFS='^' read -r -a repos_pkgs <<< "${repos_pkgs_str}" + + local _h=$(echo "${repos_pkgs_str}.$(md5sum "${image_dir}/${_o}")" | \ + md5sum | cut -c -8) + local _tmp="${image%.*}.${_h}.img" + echo "[INFO] Lookup cache / build patched base image for fingerprint: ${_h}" + if [ "${image_dir}/${_tmp}" -ef "${image_dir}/${image}" ]; then + echo "[INFO] Patched base image found" + else + # shellcheck disable=SC2115 + rm -rf "${image_dir}/${image%.*}"* + if [[ ! "${repos_pkgs_str}" =~ ^\^+$ ]]; then + echo "[INFO] Patching base image ..." + cp "${image_dir}/${_o}" "${image_dir}/${_tmp}" + __kernel_modules "${image_dir}" + __mount_image "${_tmp}" "${image_dir}" + __apt_repos_pkgs_image "${repos_pkgs[@]:0:4}" + cleanup_mounts + else + echo "[INFO] No patching required, using vanilla base image" + ln -sf "${image_dir}/${_o}" "${image_dir}/${_tmp}" + fi + ln -sf "${image_dir}/${_tmp}" "${image_dir}/${image}" + fi + + # VCP VMs base image specific changes + if [[ ! "${repos_pkgs_str}" =~ \^{3}$ ]] && [ -n "${repos_pkgs[*]:4}" ]; then + echo "[INFO] Lookup cache / build patched VCP image for md5sum: ${_h}" + _tmp="${vcp_image%.*}.${_h}.img" + if [ "${image_dir}/${_tmp}" -ef "${image_dir}/${vcp_image}" ]; then + echo "[INFO] Patched VCP image found" + else + echo "[INFO] Patching VCP image ..." + cp "${image_dir}/${image}" "${image_dir}/${_tmp}" + __kernel_modules "${image_dir}" + __mount_image "${_tmp}" "${image_dir}" + __apt_repos_pkgs_image "${repos_pkgs[@]:4:4}" + cleanup_mounts + ln -sf "${image_dir}/${_tmp}" "${image_dir}/${vcp_image}" + fi + fi +} + +function create_networks { + local all_vnode_networks=("$@") + # create required networks + for net in "mcpcontrol" "${all_vnode_networks[@]}"; do + if ${VIRSH} net-info "${net}" >/dev/null 2>&1; then + ${VIRSH} net-destroy "${net}" || true + ${VIRSH} net-undefine "${net}" + fi + # in case of custom network, host should already have the bridge in place + if [ -f "virsh_net/net_${net}.xml" ] && \ + [ ! -d "/sys/class/net/${net}/bridge" ]; then + ${VIRSH} net-define "virsh_net/net_${net}.xml" + ${VIRSH} net-autostart "${net}" + ${VIRSH} net-start "${net}" + fi + done + + sudo ip link del veth_mcp0 || true + sudo ip link del veth_mcp2 || true + # Create systemd service for veth creation after reboot + FUEL_VETHC_SERVICE="/etc/systemd/system/opnfv-fuel-vethc.service" + FUEL_VETHA_SERVICE="/etc/systemd/system/opnfv-fuel-vetha.service" + test -f /usr/sbin/ip && PREFIX=/usr/sbin || PREFIX=/sbin + cat <<-EOF | sudo tee "${FUEL_VETHC_SERVICE}" + [Unit] + After=libvirtd.service + Before=docker.service + [Service] + ExecStart=/bin/sh -ec '\ + ${PREFIX}/ip link add veth_mcp0 type veth peer name veth_mcp1 && \ + ${PREFIX}/ip link add veth_mcp2 type veth peer name veth_mcp3 && \ + ${PREFIX}/ip link set veth_mcp0 up mtu 9000 && \ + ${PREFIX}/ip link set veth_mcp1 up mtu 9000 && \ + ${PREFIX}/ip link set veth_mcp2 up mtu 9000 && \ + ${PREFIX}/ip link set veth_mcp3 up mtu 9000' + EOF + cat <<-EOF | sudo tee "${FUEL_VETHA_SERVICE}" + [Unit] + StartLimitInterval=200 + StartLimitBurst=10 + After=opnfv-fuel-vethc.service + [Service] + Restart=on-failure + RestartSec=10 + ExecStartPre=/bin/sh -ec '\ + ${PREFIX}/brctl showstp ${all_vnode_networks[0]} > /dev/null 2>&1 && \ + ${PREFIX}/brctl showstp ${all_vnode_networks[1]} > /dev/null 2>&1' + ExecStart=/bin/sh -ec '\ + ${PREFIX}/brctl addif ${all_vnode_networks[0]} veth_mcp0 && \ + ${PREFIX}/brctl addif ${all_vnode_networks[1]} veth_mcp2 && \ + while ${PREFIX}/ip rule del to ${SALT_MASTER} iif docker0 table 200 2>/dev/null; do true; done && \ + ${PREFIX}/ip rule add to ${SALT_MASTER} iif docker0 table 200 && \ + ${PREFIX}/ip route replace ${SALT_MASTER} dev ${all_vnode_networks[0]} table 200' + EOF + sudo ln -sf "${FUEL_VETHC_SERVICE}" "/etc/systemd/system/multi-user.target.wants/" + sudo ln -sf "${FUEL_VETHA_SERVICE}" "/etc/systemd/system/multi-user.target.wants/" + sudo systemctl daemon-reload + sudo systemctl restart opnfv-fuel-vethc + sudo systemctl restart opnfv-fuel-vetha +} + +function cleanup_all { + local image_dir=$1; shift + local all_vnode_networks=("$@") + [ ! -e "${image_dir}/docker-compose" ] || COMPOSE_PREFIX="${image_dir}/" + + cleanup_uefi + __cleanup_vms + sudo ip link del veth_mcp0 || true + sudo ip link del veth_mcp2 || true + for net in "mcpcontrol" "${all_vnode_networks[@]}"; do + if ${VIRSH} net-info "${net}" >/dev/null 2>&1; then + ${VIRSH} net-destroy "${net}" || true + ${VIRSH} net-undefine "${net}" + fi + done + sudo rm -f "/etc/systemd/system/multi-user.target.wants/opnfv-fuel"* \ + "/etc/systemd/system/opnfv-fuel"* + sudo systemctl daemon-reload + "${COMPOSE_PREFIX}docker-compose" -f docker-compose/docker-compose.yaml down +} + +function create_vms { + local image_dir=$1; shift + local image=base_image_opnfv_fuel.img + # vnode data should be serialized with the following format: + # <name0>,<disks0>,<ram0>,<vcpu0>[,<sockets0>,<cores0>,<threads0>[,<cell0name0>,<cell0memory0>, + # <cell0cpus0>,<cell1name0>,<cell1memory0>,<cell1cpus0>]]|<name1>,...' + IFS='|' read -r -a vnodes <<< "$1"; shift + + # AArch64: prepare arch specific arguments + local virt_extra_args="" + if [ "$(uname -i)" = "aarch64" ]; then + # No Cirrus VGA on AArch64, use virtio instead + virt_extra_args="$virt_extra_args --video=virtio" + fi + + # create vms with specified options + for serialized_vnode_data in "${vnodes[@]}"; do + if [ -z "${serialized_vnode_data}" ]; then continue; fi + IFS=',' read -r -a vnode_data <<< "${serialized_vnode_data}" + IFS=';' read -r -a disks_data <<< "${vnode_data[1]}" + + # Create config ISO and resize OS disk image for each foundation node VM + ./create-config-drive.sh -k "$(basename "${SSH_KEY}").pub" \ + -u 'user-data.sh' -h "${vnode_data[0]}" "${image_dir}/mcp_${vnode_data[0]}.iso" + cp "${image_dir}/${image}" "${image_dir}/mcp_${vnode_data[0]}.qcow2" + qemu-img resize "${image_dir}/mcp_${vnode_data[0]}.qcow2" "${disks_data[0]}" + # Prepare additional drives if present + idx=0 + virt_extra_storage= + for dsize in "${disks_data[@]:1}"; do + ((idx+=1)) + qcow_file="${image_dir}/mcp_${vnode_data[0]}_${idx}.qcow2" + qemu-img create "${qcow_file}" "${dsize}" + virt_extra_storage+=" --disk path=${qcow_file},format=qcow2,bus=virtio,cache=none,io=native" + done + + # prepare VM CPU model, count, topology (optional), NUMA cells (optional, requires topo) + local virt_cpu_args=' --cpu host-passthrough' + local idx=7 # cell0.name index in serialized data + while [ -n "${vnode_data[${idx}]}" ]; do + virt_cpu_args+=",${vnode_data[${idx}]}.memory=${vnode_data[$((idx + 1))]}" + virt_cpu_args+=",${vnode_data[${idx}]}.cpus=${vnode_data[$((idx + 2))]}" + idx=$((idx + 3)) + done + virt_cpu_args+=" --vcpus vcpus=${vnode_data[3]}" + if [ -n "${vnode_data[6]}" ]; then + virt_cpu_args+=",sockets=${vnode_data[4]},cores=${vnode_data[5]},threads=${vnode_data[6]}" + fi + + # prepare network args + local vnode_networks=("$@") + local net_args= + for net in "${vnode_networks[@]}"; do + net_args="${net_args} --network bridge=${net},model=virtio" + done + + [ ! -e "${image_dir}/virt-manager" ] || VIRT_PREFIX="${image_dir}/virt-manager/" + # shellcheck disable=SC2086 + ${VIRT_PREFIX}${VIRSH/virsh/virt-install} --name "${vnode_data[0]}" \ + ${virt_cpu_args} --accelerate \ + ${net_args} \ + --ram "${vnode_data[2]}" \ + --disk path="${image_dir}/mcp_${vnode_data[0]}.qcow2",format=qcow2,bus=virtio,cache=none,io=native \ + ${virt_extra_storage} \ + --os-type linux --os-variant none \ + --boot hd --vnc --console pty --autostart --noreboot \ + --disk path="${image_dir}/mcp_${vnode_data[0]}.iso",device=cdrom \ + --noautoconsole \ + ${virt_extra_args} + done +} + +function reset_vms { + local vnodes=("$@") + local cmd_str="ssh ${SSH_OPTS} ${SSH_SALT}" + + # reset non-infrastructure vms, wait for them to come back online + for node in "${vnodes[@]}"; do + ${VIRSH} reset "${node}" + done + for node in "${vnodes[@]}"; do + wait_for 20.0 "${cmd_str} sudo salt -C '${node}*' saltutil.sync_all" + done +} + +function start_vms { + local vnodes=("$@") + + # start vms + for node in "${vnodes[@]}"; do + ${VIRSH} start "${node}" + sleep $((RANDOM%5+1)) + done +} + +function prepare_containers { + local image_dir=$1 + [ -n "${image_dir}" ] || exit 1 + [ -n "${MCP_REPO_ROOT_PATH}" ] || exit 1 + [ ! -e "${image_dir}/docker-compose" ] || COMPOSE_PREFIX="${image_dir}/" + + "${COMPOSE_PREFIX}docker-compose" -f docker-compose/docker-compose.yaml down + if [[ ! "${MCP_DOCKER_TAG}" =~ 'verify' ]]; then + "${COMPOSE_PREFIX}docker-compose" -f docker-compose/docker-compose.yaml pull + fi + # overwrite hosts only on first container up, to preserve cluster nodes + sudo cp docker-compose/files/hosts "${image_dir}/hosts" + sudo rm -rf "${image_dir}/"{salt,pki,mas01/etc} "${image_dir}/nodes/"* + find "${image_dir}/mas01/var/lib/" \ + -mindepth 2 -maxdepth 2 -not -name boot-resources \ + -exec sudo rm -rf {} \; || true + mkdir -p "${image_dir}/"{salt/master.d,salt/minion.d} + + if grep -q -e 'maas' 'docker-compose/docker-compose.yaml'; then + # Apparmor workaround for bind9 inside Docker containers using AUFS + for profile in 'usr.sbin.ntpd' 'usr.sbin.named' \ + 'usr.sbin.dhcpd' 'usr.sbin.tcpdump' 'usr.bin.tcpdump'; do + if [ -e "/etc/apparmor.d/${profile}" ] && \ + [ ! -e "/etc/apparmor.d/disable/${profile}" ]; then + sudo ln -sf "/etc/apparmor.d/${profile}" "/etc/apparmor.d/disable/" + sudo apparmor_parser -R "/etc/apparmor.d/${profile}" || true + fi + done + fi +} + +function start_containers { + local image_dir=$1 + [ -n "${image_dir}" ] || exit 1 + [ ! -e "${image_dir}/docker-compose" ] || COMPOSE_PREFIX="${image_dir}/" + if grep -q -e 'maas' 'docker-compose/docker-compose.yaml'; then + chmod +x docker-compose/files/entrypoint*.sh + fi + "${COMPOSE_PREFIX}docker-compose" -f docker-compose/docker-compose.yaml up -d +} + +function check_connection { + local total_attempts=60 + local sleep_time=5 + + set +e + echo '[INFO] Attempting to get into Salt master ...' + + # wait until ssh on Salt master is available + # shellcheck disable=SC2034 + for attempt in $(seq "${total_attempts}"); do + # shellcheck disable=SC2086 + ssh ${SSH_OPTS} "ubuntu@${SALT_MASTER}" uptime + case $? in + 0) echo "${attempt}> Success"; break ;; + *) echo "${attempt}/${total_attempts}> ssh server ain't ready yet, waiting for ${sleep_time} seconds ..." ;; + esac + sleep $sleep_time + done + set -e +} + +function cleanup_mounts { + # Remove any mounts, loop and/or nbd devs created while patching base image + if [ -n "${OPNFV_MNT_DIR}" ] && [ -d "${OPNFV_MNT_DIR}" ]; then + if [ -f "${OPNFV_MNT_DIR}/boot/grub/grub.cfg" ]; then + # Grub thinks it's running from a live CD + sudo sed -i -e 's/^\s*set root=.*$//g' -e 's/^\s*loopback.*$//g' \ + "${OPNFV_MNT_DIR}/boot/grub/grub.cfg" + fi + sync + if mountpoint -q "${OPNFV_MNT_DIR}"; then + sudo umount -l "${OPNFV_MNT_DIR}" || true + fi + fi + if [ -n "${OPNFV_LOOP_DEV}" ] && \ + sudo losetup "${OPNFV_LOOP_DEV}" 1>&2 > /dev/null; then + sudo losetup -d "${OPNFV_LOOP_DEV}" + fi + if [ -n "${OPNFV_NBD_DEV}" ]; then + sudo partx -d "${OPNFV_NBD_DEV}" || true + sudo kpartx -d "${OPNFV_NBD_DEV}" || true + sudo qemu-nbd -d "${OPNFV_NBD_DEV}" || true + fi +} diff --git a/mcp/scripts/lib_template.sh b/mcp/scripts/lib_template.sh index 0fbe628b7..822d14116 100644 --- a/mcp/scripts/lib_template.sh +++ b/mcp/scripts/lib_template.sh @@ -23,16 +23,14 @@ function do_templates_scenario { local target_lab=$1; shift local target_pod=$1; shift local lab_config_uri=$1; shift - local scenario_dir=$1 + local scenario_dir=$1; shift + local extra_yaml=("$@") BASE_CONFIG_PDF="${lab_config_uri}/labs/${target_lab}/${target_pod}.yaml" BASE_CONFIG_IDF="${lab_config_uri}/labs/${target_lab}/idf-${target_pod}.yaml" LOCAL_PDF="${image_dir}/$(basename "${BASE_CONFIG_PDF}")" LOCAL_IDF="${image_dir}/$(basename "${BASE_CONFIG_IDF}")" - # Make sample PDF/IDF available via default lab-config (pharos submodule) - ln -sf "$(readlink -f "../config/labs/local")" "./pharos/labs/" - # Expand scenario file and main reclass input (pod_config.yaml) based on PDF if ! curl --create-dirs -o "${LOCAL_PDF}" "${BASE_CONFIG_PDF}"; then notify_e "[ERROR] Could not retrieve PDF (Pod Descriptor File)!" @@ -49,18 +47,22 @@ function do_templates_scenario { notify_e "[ERROR] IDF does not match yaml schema!" fi fi - if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \ - -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then - notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!" - fi printenv | \ awk '/^(SALT|MCP|MAAS).*=/ { gsub(/=/,": "); print }' >> "${LOCAL_PDF}" j2args=$(find "${scenario_dir}" -name '*.j2' -exec echo -j {} \;) # shellcheck disable=SC2086 - if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \ + if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \ -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then notify_e '[ERROR] Could not convert j2 scenario definitions!' fi + for _yaml in "${extra_yaml[@]}"; do + awk '/^---$/{f=1;next;}f' "${_yaml}" >> "${LOCAL_PDF}" + done + if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \ + -i "$(dirname "$(readlink -f "${PHAROS_IA}")")" \ + -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then + notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!" + fi } # Expand reclass and virsh network templates based on PDF + IDF + others @@ -78,10 +80,11 @@ function do_templates_cluster { awk '/^---$/{f=1;next;}f' "${_yaml}" >> "${LOCAL_PDF}" done # shellcheck disable=SC2046 - j2args=$(find "${RECLASS_CLUSTER_DIR}" "$(readlink -f virsh_net)" $(readlink -f ./*j2) \ + j2args=$(find "${RECLASS_CLUSTER_DIR}" "$(readlink -f virsh_net)" \ + "$(readlink -f docker-compose)" $(readlink -f ./*j2) \ -name '*.j2' -exec echo -j {} \;) # shellcheck disable=SC2086 - if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \ + if ! python3 "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \ -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then notify_e '[ERROR] Could not convert PDF to network definitions!' fi diff --git a/mcp/scripts/pharos b/mcp/scripts/pharos -Subproject c7034503e2bb8da20f2b5c678719c7c7fbe6e61 +Subproject 061b5588d40253193eddf76139c361d62e6fbeb diff --git a/mcp/scripts/requirements_deb.yaml b/mcp/scripts/requirements_deb.yaml index 04ddcf631..58fc533b0 100644 --- a/mcp/scripts/requirements_deb.yaml +++ b/mcp/scripts/requirements_deb.yaml @@ -6,31 +6,41 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## --- -requirements_pkg: +build: + # Common pkgs required for all builds, no matter the type, arch etc. + common: + - curl + - git + - make + - python3-pip +deploy: # Common pkgs required for all deploys, no matter the type, arch etc. common: - bridge-utils + - build-essential - cloud-guest-utils - cpu-checker - curl + - docker-compose - e2fsprogs - git - kpartx + - libglib2.0-bin - libvirt-bin - - make - mkisofs - qemu-kvm - rsync - uuid-runtime - virtinst # python is indirectly required for PDF parsing - - python - - python-ipaddress - - python-jinja2 - - python-yaml - - python-jsonschema + - python3 + - python3-jinja2 + - python3-yaml + - python3-jsonschema # Optional, arch-specific requirements, matched by key name = $(uname -m) aarch64: # AArch64 VMs use AAVMF (guest UEFI) - ipxe-qemu - qemu-efi + # AArch64 CentOS cloud image is archived with xz + - xz-utils diff --git a/mcp/scripts/requirements_rpm.yaml b/mcp/scripts/requirements_rpm.yaml index 3df4d0a80..b2ee0dcd1 100644 --- a/mcp/scripts/requirements_rpm.yaml +++ b/mcp/scripts/requirements_rpm.yaml @@ -6,13 +6,20 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## --- -requirements_pkg: +build: + # Common pkgs required for all builds, no matter the type, arch etc. + common: + - git + - make + - python3-pip +deploy: # Common pkgs required for all deploys, no matter the type, arch etc. common: - bc - bridge-utils - cloud-utils-growpart - curl + - docker-compose - elfutils-libelf-devel - e2fsprogs - genisoimage @@ -30,12 +37,13 @@ requirements_pkg: - virt-install - wget # For python is indirectly required for PDF parsing - - python - - python-ipaddress - - python-jinja2 - - python-yaml - - python-jsonschema + - python3 + - python36-jinja2 + - python36-yaml + - python36-jsonschema # Optional, arch-specific requirements, matched by key name = $(uname -m) aarch64: # AArch64 VMs use AAVMF (guest UEFI) - AAVMF + # AArch64 CentOS cloud image is archived with xz + - xz diff --git a/mcp/scripts/salt.sh b/mcp/scripts/salt.sh deleted file mode 100755 index aecfecea4..000000000 --- a/mcp/scripts/salt.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/bash -e -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -# -# Deploy Salt Master -# - -CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x -F_GIT_ROOT=$(git rev-parse --show-toplevel) -F_GIT_DIR=$(cd "${F_GIT_ROOT}/mcp" && git rev-parse --git-dir) -F_GIT_SUBD=${F_GIT_ROOT#${F_GIT_DIR%%/.git*}} -OPNFV_TMP_DIR="/home/${SALT_MASTER_USER}/opnfv" -OPNFV_GIT_DIR="/root/opnfv" -OPNFV_FUEL_DIR="/root/fuel" # Should be in sync with patch.sh, scripts patches -OPNFV_RDIR="reclass/classes/cluster/all-mcp-arch-common" -OPNFV_VCP_IMG="mcp/scripts/base_image_opnfv_fuel_vcp.img" -OPNFV_VCP_DIR="/srv/salt/env/prd/salt/files/control/images" -LOCAL_GIT_DIR="${F_GIT_ROOT%${F_GIT_SUBD}}" -LOCAL_PDF_RECLASS=$1; shift -# shellcheck disable=SC2116,SC2086 -LOCAL_VIRT_NODES=$(echo ${*//cfg01/}) # unquoted to filter space -NODE_MASK="${LOCAL_VIRT_NODES// /|}" - -# push to cfg01 current git repo first (including submodules), at ~ubuntu/opnfv -# later we move it to ~root/opnfv (and ln as ~root/fuel); delete the temp clone -remote_tmp="${SSH_SALT}:$(basename "${OPNFV_TMP_DIR}")" -rsync -Erl --delete -e "ssh ${SSH_OPTS}" \ - --exclude-from="${F_GIT_ROOT}/.gitignore" \ - "${LOCAL_GIT_DIR}/" "${remote_tmp}/" -if [ -n "${LOCAL_PDF_RECLASS}" ] && [ -f "${LOCAL_PDF_RECLASS}" ]; then - rsync -e "ssh ${SSH_OPTS}" "${LOCAL_PDF_RECLASS}" \ - "${remote_tmp}${F_GIT_SUBD}/mcp/${OPNFV_RDIR}/opnfv/" -fi -local_vcp_img=$(dirname "${LOCAL_PDF_RECLASS}")/$(basename "${OPNFV_VCP_IMG}") -if [ -e "${local_vcp_img}" ]; then - rsync -L -e "ssh ${SSH_OPTS}" "${local_vcp_img}" \ - "${remote_tmp}${F_GIT_SUBD}/${OPNFV_VCP_IMG}" -fi - -# ssh to cfg01 -# shellcheck disable=SC2086,2087 -ssh ${SSH_OPTS} "${SSH_SALT}" bash -s -e << SALT_INSTALL_END - sudo -i - set -e - export TERM=${TERM} - export CI_DEBUG=${CI_DEBUG}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x - - echo -n 'Checking out cloud-init has finished running ...' - while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo -n '.'; sleep 1; done - echo ' done' - - mkdir -p /srv/salt /usr/share/salt-formulas/reclass - rm -rf ${OPNFV_GIT_DIR} ${OPNFV_FUEL_DIR} - mv ${OPNFV_TMP_DIR} ${OPNFV_GIT_DIR} && chown -R root.root ${OPNFV_GIT_DIR} - find ${OPNFV_GIT_DIR} -name '.git' -type f | while read f_git; do - sed -i 's@${LOCAL_GIT_DIR}@${OPNFV_GIT_DIR}@g' \$f_git - done - ln -sf ${OPNFV_GIT_DIR}${F_GIT_SUBD} ${OPNFV_FUEL_DIR} - ln -sf ${OPNFV_FUEL_DIR}/mcp/reclass /srv/salt - ln -sf ${OPNFV_FUEL_DIR}/mcp/deploy/scripts /srv/salt - ln -sf ${OPNFV_FUEL_DIR}/mcp/scripts/mcp.rsa $(dirname "${OPNFV_FUEL_DIR}") - cd /srv/salt/${OPNFV_RDIR} && rm -f arch && ln -sf "\$(uname -i)" arch - - cp -r ${OPNFV_FUEL_DIR}/mcp/metadata/service /usr/share/salt-formulas/reclass - cd /srv/salt/reclass/classes/service && \ - ln -sf /usr/share/salt-formulas/reclass/service/opendaylight - - # Armband APT-MK nightly/extra repo for forked & extended reclass - wget -qO - https://linux.enea.com/apt-mk/public.gpg | apt-key add - - echo 'deb http://linux.enea.com/apt-mk/xenial nightly extra' > \ - '/etc/apt/sources.list.d/armband_mcp_extra.list' - apt-get update - - cd /srv/salt/scripts - export DEBIAN_FRONTEND=noninteractive - echo 'Dpkg::Use-Pty "0";' > /etc/apt/apt.conf.d/90silence-dpkg - OLD_DOMAIN=\$(grep -sPzo "id: cfg01\.\K(\S*)" /etc/salt/minion.d/minion.conf) || true - BOOTSTRAP_SALTSTACK_OPTS=" -r -dX stable 2016.11 " \ - MASTER_HOSTNAME=cfg01.${CLUSTER_DOMAIN} DISTRIB_REVISION=nightly \ - EXTRA_FORMULAS="nfs" \ - ./salt-master-init.sh - salt-key -Ay - - cp -r ${OPNFV_FUEL_DIR}/mcp/salt-formulas/* /usr/share/salt-formulas/env - cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list formulas - cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list reclass - - source ${OPNFV_FUEL_DIR}/mcp/scripts/lib.sh - wait_for 3.0 "salt-call state.apply salt" - - # In case scenario changed (and implicitly domain name), re-register minions - if [ -n "\${OLD_DOMAIN}" ] && [ "\${OLD_DOMAIN}" != "${CLUSTER_DOMAIN}" ]; then - salt "*.\${OLD_DOMAIN}" cmd.run "grep \${OLD_DOMAIN} -sRl /etc/salt | \ - xargs --no-run-if-empty sed -i 's/\${OLD_DOMAIN}/${CLUSTER_DOMAIN}/g'; \ - service salt-minion restart" || true - salt-key -yd "*.\${OLD_DOMAIN}" - salt-key -Ay - fi - - # Init specific to VMs on FN (all for virtual, cfg|mas for baremetal) - wait_for 3.0 'salt -C "cfg01*" state.apply linux' - if [[ "${LOCAL_VIRT_NODES}" =~ mas ]]; then - wait_for 3.0 'salt -C "mas*" test.ping' - else - wait_for 3.0 '(for n in ${LOCAL_VIRT_NODES}; do salt -C \${n}.* test.ping || exit; done)' - fi - wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" saltutil.sync_all' - wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.apply salt' - - wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" state.sls linux.system,linux.storage' - wait_for 2.0 'salt -C "E@^(${NODE_MASK}).*" state.sls linux.network' || true - salt -C "E@^(${NODE_MASK}).*" state.sls opnfv.route_wrapper - salt -C "E@^(${NODE_MASK}).*" system.reboot - wait_for 90.0 'salt -C "E@^(${NODE_MASK}).*" test.ping' - wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" pkg.upgrade refresh=False dist_upgrade=True' - - wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.sls ntp' - - if [ -f "${OPNFV_FUEL_DIR}/${OPNFV_VCP_IMG}" ]; then - mkdir -p "${OPNFV_VCP_DIR}" - mv "${OPNFV_FUEL_DIR}/${OPNFV_VCP_IMG}" "${OPNFV_VCP_DIR}/" - fi -SALT_INSTALL_END diff --git a/mcp/scripts/sysinfo_print.sh b/mcp/scripts/sysinfo_print.sh index bff36aff4..f07e7a5cb 100755 --- a/mcp/scripts/sysinfo_print.sh +++ b/mcp/scripts/sysinfo_print.sh @@ -10,9 +10,20 @@ # Collect jump server system information for deploy debugging # +# HW info +cat /proc/cpuinfo +free -mh +df -h + # Network info brctl show +ip a +route -n +sudo iptables -S # Distro & pkg info cat /etc/*-release uname -a + +# Misc info +sudo losetup -a diff --git a/mcp/scripts/user-data.admin.sh.j2 b/mcp/scripts/user-data.admin.sh.j2 deleted file mode 100644 index d9b86c79c..000000000 --- a/mcp/scripts/user-data.admin.sh.j2 +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2018 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -rm /etc/salt/minion_id -rm -f /etc/salt/pki/minion/minion_master.pub -echo "id: $(hostname).{{ conf.cluster.domain }}" > /etc/salt/minion -{#- should be in sync with 'opnfv_infra_config_pxe_address' in 'pharos/config/installers/fuel/pod_config.yml.j2 #} -echo "master: {{ conf.idf.net_config.admin.network | ipaddr_index(2) }}" >> /etc/salt/minion -service salt-minion restart diff --git a/mcp/scripts/user-data.mcp.sh.j2 b/mcp/scripts/user-data.mcp.sh.j2 deleted file mode 100644 index bd80961e6..000000000 --- a/mcp/scripts/user-data.mcp.sh.j2 +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -############################################################################## -# Copyright (c) 2018 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -rm /etc/salt/minion_id -rm -f /etc/salt/pki/minion/minion_master.pub -echo "id: $(hostname).{{ conf.cluster.domain }}" > /etc/salt/minion -echo "master: {{ conf.SALT_MASTER }}" >> /etc/salt/minion -service salt-minion restart diff --git a/mcp/scripts/user-data.sh.j2 b/mcp/scripts/user-data.sh.j2 new file mode 100644 index 000000000..8b80e32d0 --- /dev/null +++ b/mcp/scripts/user-data.sh.j2 @@ -0,0 +1,20 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2018 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +{%- import 'net_map.j2' as nm with context %} +rm -f /etc/salt/minion_id /etc/salt/pki/minion/minion_master.pub +echo "id: $(hostname).{{ conf.cluster.domain }}" > /etc/salt/minion +echo "master: {{ nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_config_pxe_admin_address') +1) }}" >> /etc/salt/minion +ldconfig +{%- if 'ubuntu1804' in conf.MCP_OS %} +systemctl unmask networking.service || true +systemctl enable networking.service || true +systemctl start networking.service || true +{%- endif %} +systemctl enable salt-minion.service +systemctl restart salt-minion.service diff --git a/mcp/scripts/virsh_net/net_mcpcontrol.xml.j2 b/mcp/scripts/virsh_net/net_mcpcontrol.xml.j2 deleted file mode 100644 index 95424db4e..000000000 --- a/mcp/scripts/virsh_net/net_mcpcontrol.xml.j2 +++ /dev/null @@ -1,19 +0,0 @@ -<!-- - Copyright (c) 2017 Mirantis Inc., Enea AB and others. - - All rights reserved. This program and the accompanying materials - are made available under the terms of the Apache License, Version 2.0 - which accompanies this distribution, and is available at - http://www.apache.org/licenses/LICENSE-2.0 ---> -{#- conf.MCPCONTROL_NET & co are mandatory, defaults are set via globals.sh #} -<network> - <name>mcpcontrol</name> - <bridge name="mcpcontrol"/> - <forward mode="nat"/> - <ip address="{{ conf.MCPCONTROL_NET | ipaddr_index(1) }}" netmask="{{ conf.MCPCONTROL_PREFIX | netmask }}"> - <dhcp> - <range start="{{ conf.MCPCONTROL_NET | ipaddr_index(2) }}" end="{{ conf.MCPCONTROL_NET | ipaddr_index(254) }}"/> - </dhcp> - </ip> -</network> diff --git a/mcp/scripts/virsh_net/net_mgmt.xml.j2 b/mcp/scripts/virsh_net/net_mgmt.xml.j2 index a558293fa..23b32f2f4 100644 --- a/mcp/scripts/virsh_net/net_mgmt.xml.j2 +++ b/mcp/scripts/virsh_net/net_mgmt.xml.j2 @@ -6,15 +6,9 @@ which accompanies this distribution, and is available at http://www.apache.org/licenses/LICENSE-2.0 --> -{%- if conf.idf.net_config.mgmt is defined %} - {%- set mgmt_network = conf.idf.net_config.mgmt.network %} - {%- set mgmt_prefix = conf.idf.net_config.mgmt.mask %} -{%- else %} - {%- set mgmt_network = '172.16.10.0' %} - {%- set mgmt_prefix = '24' %} -{%- endif %} +{%- import 'net_map.j2' as nm with context %} <network> <name>mgmt</name> <bridge name="mgmt"/> - <ip address="{{ mgmt_network | ipaddr_index(1) }}" netmask="{{ mgmt_prefix | netmask }}"/> + <ip address="{{ nm.net_mgmt | ipnet_hostmin }}" netmask="{{ nm.net_mgmt | ipnet_netmask }}"/> </network> diff --git a/mcp/scripts/virsh_net/net_public.xml.j2 b/mcp/scripts/virsh_net/net_public.xml.j2 index 737b638b3..a9e8349f4 100644 --- a/mcp/scripts/virsh_net/net_public.xml.j2 +++ b/mcp/scripts/virsh_net/net_public.xml.j2 @@ -6,27 +6,15 @@ which accompanies this distribution, and is available at http://www.apache.org/licenses/LICENSE-2.0 --> -{%- set cluster = {'has_virtual_nodes': False} %} -{%- for node in conf.nodes %} - {%- if not cluster.has_virtual_nodes and node.node.type == 'virtual' %} - {%- do cluster.update({'has_virtual_nodes': True}) %} - {%- endif %} -{%- endfor %} -{%- if conf.idf.net_config.public is defined %} - {%- set public_network = conf.idf.net_config.public.network %} - {%- set public_prefix = conf.idf.net_config.public.mask %} -{%- else %} - {%- set public_network = '10.16.0.0' %} - {%- set public_prefix = '24' %} -{%- endif %} +{%- import 'net_map.j2' as nm with context %} <network> <name>public</name> <bridge name="public"/> -{%- if cluster.has_virtual_nodes %} +{%- if nm.cluster.has_virtual_nodes %} {#- Ideally, jumpserver would have a real Linux bridge we will hook to. In case it doesn't, we use this virsh network as a *mock* public. The *mock* public should NOT overlap with the real public in any way. #} <forward mode="nat"/> - <ip address="{{ public_network | ipaddr_index(1) }}" netmask="{{ public_prefix | netmask }}"/> + <ip address="{{ nm.net_public | ipnet_hostmin }}" netmask="{{ nm.net_public | ipnet_netmask }}"/> {%- endif %} </network> diff --git a/mcp/scripts/virsh_net/net_pxebr.xml.j2 b/mcp/scripts/virsh_net/net_pxebr.xml.j2 index f82780cf7..1c48e9b1a 100644 --- a/mcp/scripts/virsh_net/net_pxebr.xml.j2 +++ b/mcp/scripts/virsh_net/net_pxebr.xml.j2 @@ -6,21 +6,14 @@ which accompanies this distribution, and is available at http://www.apache.org/licenses/LICENSE-2.0 --> -{%- if conf.idf.net_config.admin is defined %} - {%- set pxebr_network = conf.idf.net_config.admin.network %} - {%- set pxebr_prefix = conf.idf.net_config.admin.mask %} -{%- else %} - {%- set pxebr_network = '192.168.11.0' %} - {%- set pxebr_prefix = '24' %} -{%- endif %} +{%- import 'net_map.j2' as nm with context %} <network> <name>pxebr</name> <forward mode="nat"/> <bridge name="pxebr"/> - <ip address="{{ pxebr_network | ipaddr_index(1) }}" netmask="{{ pxebr_prefix | netmask }}"> - <!-- NOTE: .254 is harcoded for now (for /24 prefix), should be computed instead. --> + <ip address="{{ nm.net_admin_gw }}" netmask="{{ nm.net_admin | ipnet_netmask }}"> <dhcp> - <range start="{{ pxebr_network | ipaddr_index(4) }}" end="{{ pxebr_network | ipaddr_index(254) }}"/> + <range start="{{ nm.net_admin_pool_start }}" end="{{ nm.net_admin_pool_end }}"/> </dhcp> </ip> </network> diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2 index 8c9d5d969..6aca36f32 100644 --- a/mcp/scripts/xdf_data.sh.j2 +++ b/mcp/scripts/xdf_data.sh.j2 @@ -11,7 +11,14 @@ # Data derived from XDF (PDF/IDF/SDF/etc), used as input in deploy.sh # +{%- import 'net_map.j2' as nm with context -%} +{%- set cluster_states = conf.cluster.states if conf.MCP_NO_DEPLOY_ENVIRONMENT < 2 else [] -%} {%- set arch = conf[conf.MCP_JUMP_ARCH] -%} +{%- set V = conf.virtual -%} +{%- set section_map = { + 'control': nm.ctl01.idx, + 'compute': nm.cmp001.idx +} -%} {%- macro bash_arr(_l) -%} ({%- for n in _l -%}'{{ n }}' {% endfor -%}) @@ -24,25 +31,60 @@ {#- Pack all vnode data as string -#} {%- macro serialize_vnodes() -%} - {%- set V = conf.virtual -%} {%- set arr = [] -%} - {%- for n in V.nodes -%} - {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%} - {%- do arr.append(pack([n, V[n].ram or arch.default.ram, - V[n].vcpus or arch.default.vcpus])) -%} + {%- for section in section_map -%} + {%- for n in V.nodes[section] or [] -%} + {%- if ( section_map[section] < conf.nodes | length and + conf.nodes[section_map[section] + loop.index0].node.type == 'virtual' ) -%} + {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%} + {%- set cpu_topo = 'cpu_topology' in V[n] and not conf.MCP_CMP_SS -%} + {%- if 'numa' in V[n] and cpu_topo -%} + {%- for k, v in V[n].numa.items() -%} + {%- set c = pack([k, v.memory, v.cpus]) -%} + {%- do V[n].update({'s_numa': c if 's_numa' not in V[n] else pack([c, V[n].s_numa])}) -%} + {%- endfor -%} + {%- endif -%} + {%- do arr.append(pack([n, V[n].disks or arch.default.disks, + V[n].ram or arch.default.ram, + V[n].vcpus or arch.default.vcpus, + '' if not cpu_topo else pack([ + V[n].cpu_topology.sockets, + V[n].cpu_topology.cores, + V[n].cpu_topology.threads, + '' if 's_numa' not in V[n] else V[n].s_numa])])) -%} + {%- endif -%} + {%- endfor -%} {%- endfor -%} '{{ pack(arr, '|') }}' {%- endmacro -%} -{#- Pack apt_pkg data as string -#} -{%- macro serialize_apt_pkg() -%} +{#- Return a bash array of node names or a Salt query, optionally filtered by type #} +{%- macro filter_nodes(type, output_as_query = False, sections = section_map) -%} {%- set arr = [] -%} - {%- set sections = [arch.common] -%} - {%- if conf.MCP_VCP -%} - {%- do sections.append(arch.control) -%} + {%- for section in sections -%} + {%- for n in V.nodes[section] or [] -%} + {%- if ( section_map[section] < conf.nodes | length and + conf.nodes[section_map[section] + loop.index0].node.type in type ) -%} + {%- do arr.append(n) -%} + {%- endif -%} + {%- endfor -%} + {%- endfor -%} + {%- if output_as_query -%} + '{{ arr | join('* or ') }}*' + {%- else -%} + {{ bash_arr(arr) }} + {%- endif -%} +{%- endmacro -%} + +{#- Pack repo_pkg data as string -#} +{%- macro serialize_repo_pkg() -%} + {%- set arr = [] -%} + {%- set sections = [arch[conf.MCP_OS].common] -%} + {%- if conf.MCP_VCP or '-vcp-' in conf.MCP_DEPLOY_SCENARIO -%} + {%- do sections.append(arch[conf.MCP_OS].control) -%} {%- endif -%} {%- for c in sections -%} - {%- do arr.append(pack([pack(c.apt['keys']), pack(c.apt.repos), + {%- do arr.append(pack([pack(c.repo['keys']), pack(c.repo.repos), pack(c.pkg.install), pack(c.pkg.remove)], '^')) -%} {%- endfor -%} '{{ pack(arr, '^') }}' @@ -58,14 +100,18 @@ OPNFV_BRIDGES=( ) export CLUSTER_DOMAIN={{ conf.cluster.domain }} -cluster_states={{ bash_arr(conf.cluster.states) }} -virtual_nodes={{ bash_arr(conf.virtual.nodes) }} -base_image={{ arch.base_image }} +dns_public={{ nm.dns_public[0] }} +cluster_states={{ bash_arr(arch.default.cluster.states + cluster_states) }} +virtual_nodes={{ filter_nodes('virtual') }} +control_nodes_query={{ filter_nodes(['baremetal', 'virtual'], True, ['control']) }} +base_image={{ arch[conf.MCP_OS].base_image }} -# Serialize vnode data as '<name0>,<ram0>,<vcpu0>|<name1>,<ram1>,<vcpu1>[...]' +# Serialize vnode data as: +# <name0>,<ram0>,<vcpu0>[,<sockets0>,<cores0>,<threads0>[,<cell0name0>,<cell0memory0>, +# <cell0cpus0>,<cell1name0>,<cell1memory0>,<cell1cpus0>]]|<name1>,...' virtual_nodes_data={{ serialize_vnodes() }} # Serialize repos, packages to (pre-)install/remove for: # - foundation node VM base image (virtual: all VMs, baremetal: cfg01|mas01) # - virtualized control plane VM base image (only when VCP is used) -virtual_repos_pkgs={{ serialize_apt_pkg() }} +virtual_repos_pkgs={{ serialize_repo_pkg() }} |