diff options
Diffstat (limited to 'mcp/scripts')
| -rw-r--r-- | mcp/scripts/.gitignore | 1 | ||||
| -rw-r--r-- | mcp/scripts/docker-compose/docker-compose.yaml.j2 | 55 | ||||
| -rwxr-xr-x | mcp/scripts/docker-compose/files/entrypoint.sh | 5 | ||||
| -rw-r--r-- | mcp/scripts/docker-compose/files/entrypoint_maas.sh.j2 | 62 | ||||
| -rw-r--r-- | mcp/scripts/lib_jump_common.sh | 10 | ||||
| -rw-r--r-- | mcp/scripts/lib_jump_deploy.sh | 65 | ||||
| -rw-r--r-- | mcp/scripts/user-data.sh.j2 | 7 | ||||
| -rw-r--r-- | mcp/scripts/virsh_net/net_mcpcontrol.xml.j2 | 20 | ||||
| -rw-r--r-- | mcp/scripts/xdf_data.sh.j2 | 8 |
9 files changed, 145 insertions, 88 deletions
diff --git a/mcp/scripts/.gitignore b/mcp/scripts/.gitignore index 6a95545c4..c21c3d9de 100644 --- a/mcp/scripts/.gitignore +++ b/mcp/scripts/.gitignore @@ -1,3 +1,4 @@ mcp.rsa* user-data.sh xdf_data.sh +docker-compose/files/entrypoint_maas.sh diff --git a/mcp/scripts/docker-compose/docker-compose.yaml.j2 b/mcp/scripts/docker-compose/docker-compose.yaml.j2 index 891d55938..bc8b3e80a 100644 --- a/mcp/scripts/docker-compose/docker-compose.yaml.j2 +++ b/mcp/scripts/docker-compose/docker-compose.yaml.j2 @@ -22,33 +22,60 @@ services: mgmt: ipv4_address: {{ nm.net_mgmt | ipnet_hostaddr(nm.start_ip[nm.net_mgmt] + nm.net_mgmt_hosts.index('opnfv_infra_config_address') +1) }} volumes: - - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket:ro - - {{ conf.MCP_REPO_ROOT_PATH }}:/root/fuel - - {{ conf.MCP_REPO_ROOT_PATH }}/mcp/scripts/docker-compose/files/entrypoint.sh:/entrypoint.sh - - {{ conf.MCP_STORAGE_DIR }}/pod_config.yml:/root/pod_config.yml - - {{ conf.MCP_STORAGE_DIR }}/nodes:/srv/salt/reclass/nodes - - {{ conf.MCP_STORAGE_DIR }}/pki:/etc/pki - - {{ conf.MCP_STORAGE_DIR }}/salt:/etc/salt - - {{ conf.MCP_STORAGE_DIR }}/hosts:/etc/hosts + - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket:ro + - {{ conf.MCP_REPO_ROOT_PATH }}:/root/fuel + - {{ conf.MCP_REPO_ROOT_PATH }}/mcp/scripts/docker-compose/files/entrypoint.sh:/entrypoint.sh + - {{ conf.MCP_STORAGE_DIR }}/pod_config.yml:/root/pod_config.yml + - {{ conf.MCP_STORAGE_DIR }}/nodes:/srv/salt/reclass/nodes + - {{ conf.MCP_STORAGE_DIR }}/pki:/etc/pki + - {{ conf.MCP_STORAGE_DIR }}/salt:/etc/salt + - {{ conf.MCP_STORAGE_DIR }}/hosts:/etc/hosts {%- if conf.MCP_VCP %} - - {{ conf.MCP_STORAGE_DIR }}/base_image_opnfv_fuel_vcp.img:/srv/salt/env/prd/salt/files/control/images/base_image_opnfv_fuel_vcp.img + - {{ conf.MCP_STORAGE_DIR }}/base_image_opnfv_fuel_vcp.img:/srv/salt/env/prd/salt/files/control/images/base_image_opnfv_fuel_vcp.img {%- endif %} hostname: cfg01 domainname: {{ conf.cluster.domain }} privileged: true +{%- if nm.cluster.has_baremetal_nodes %} + opnfv-fuel-maas: + container_name: "maas" + image: "opnfv/fuel:saltminion-maas-{{ conf.MCP_DOCKER_TAG }}" + networks: + mcpcontrol: + ipv4_address: {{ conf.MAAS_IP }} + pxebr: + ipv4_address: {{ nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_maas_node01_deploy_address') +1) }} + mgmt: + ipv4_address: {{ nm.net_mgmt | ipnet_hostaddr(nm.start_ip[nm.net_mgmt] + nm.net_mgmt_hosts.index('opnfv_infra_maas_node01_address') +1) }} + volumes: + - /lib/modules:/lib/modules:ro + - /sys/fs/cgroup:/sys/fs/cgroup:ro + - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket:ro + - {{ conf.MCP_REPO_ROOT_PATH }}/mcp/scripts/docker-compose/files/entrypoint_maas.sh:/entrypoint.sh:ro + - {{ conf.MCP_STORAGE_DIR }}/hosts:/etc/hosts:ro + - {{ conf.MCP_STORAGE_DIR }}/mas01/etc/iptables:/etc/iptables + - {{ conf.MCP_STORAGE_DIR }}/mas01/var/lib/postgresql:/var/lib/postgresql + - {{ conf.MCP_STORAGE_DIR }}/mas01/var/lib/maas:/var/lib/maas + - {{ conf.MCP_STORAGE_DIR }}/mas01/var/spool/maas-proxy:/var/spool/maas-proxy + - {{ conf.MCP_STORAGE_DIR }}/mas01/etc/maas:/etc/maas + hostname: mas01 + domainname: {{ conf.cluster.domain }} + privileged: true + ports: + - 5240:5240 +{%- endif %} networks: mcpcontrol: - driver: macvlan + driver: bridge driver_opts: - parent: veth_mcp1 # Always untagged + com.docker.network.driver.mtu: 9000 ipam: config: - subnet: {{ net_mcpcontrol }} - gateway: {{ net_mcpcontrol | ipnet_hostaddr(1) }} pxebr: driver: macvlan driver_opts: - parent: veth_mcp3 # Always untagged + parent: veth_mcp1 # Always untagged ipam: config: - subnet: {{ nm.net_admin }} @@ -58,7 +85,7 @@ networks: {%- if conf.idf.fuel.jumphost.get('trunks', {}).get('mgmt', False) %} parent: {{ ma.interface_str('veth_mcp5', nm.vlan_mgmt) }} {%- else %} - parent: veth_mcp5 # Untagged by default + parent: veth_mcp3 # Untagged by default {%- endif %} ipam: config: diff --git a/mcp/scripts/docker-compose/files/entrypoint.sh b/mcp/scripts/docker-compose/files/entrypoint.sh index 9830ea15f..a0f72e2d1 100755 --- a/mcp/scripts/docker-compose/files/entrypoint.sh +++ b/mcp/scripts/docker-compose/files/entrypoint.sh @@ -21,11 +21,6 @@ if [ ! -f /home/ubuntu/.ssh/authorized_keys ]; then echo 'IdentityFile /root/fuel/mcp/scripts/mcp.rsa' >> /root/.ssh/config fi -if ! grep -q localhost /etc/hosts; then - # overwrite hosts only on first container up, to preserve cluster nodes - cp -a /root/fuel/mcp/scripts/docker-compose/files/hosts /etc/hosts -fi - # salt state does not properly configure file_roots in master.conf, hard set it cp -a /root/fuel/mcp/scripts/docker-compose/files/opnfv_master.conf \ /etc/salt/master.d/opnfv_master.conf diff --git a/mcp/scripts/docker-compose/files/entrypoint_maas.sh.j2 b/mcp/scripts/docker-compose/files/entrypoint_maas.sh.j2 new file mode 100644 index 000000000..23b8d8f30 --- /dev/null +++ b/mcp/scripts/docker-compose/files/entrypoint_maas.sh.j2 @@ -0,0 +1,62 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2019 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +{%- import 'net_map.j2' as nm with context %} +{%- set pxebr_addr = nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_maas_node01_deploy_address') +1) %} +if [ ! -e /var/lib/postgresql/*/main ]; then + cp -ar /var/lib/opnfv/{postgresql,maas} /var/lib/ + cp -ar /var/lib/opnfv/etc/{ssh,maas} /etc/ +fi +chown -R maas:maas /var/lib/maas +chown -R postgres:postgres /var/lib/postgresql +chown -R proxy:proxy /var/spool/maas-proxy + +if [ ! -f /etc/sysctl.d/99-salt.conf ]; then + echo 'net.ipv4.ip_forward = 1' > /etc/sysctl.d/99-salt.conf +fi + +cat <<-EOF | tee /etc/resolv.conf +{%- for server in nm.dns_public %} +nameserver {{ server }} +{%- endfor %} +EOF + +cat <<-EOF | tee /etc/salt/minion.d/opnfv.conf +id: mas01.{{ conf.cluster.domain }} +master: {{ conf.SALT_MASTER }} +grains: + virtual_subtype: Docker_ +EOF +rm -f /etc/salt/minion.d/99-master-address.conf + +# Work around MaaS issues with PXE/admin using jumbo frames +MAAS_MTU_SERVICE="/etc/systemd/system/maas-mtu.service" +cat <<-EOF | tee "${MAAS_MTU_SERVICE}" +[Unit] +Requires=network-online.target +After=network-online.target +[Service] +ExecStart=/bin/sh -ec '\ + /sbin/ifconfig $(/sbin/ip addr | /bin/grep -Po "{{ pxebr_addr }}.* \K(.*)") mtu 1500' +EOF +ln -sf "${MAAS_MTU_SERVICE}" "/etc/systemd/system/multi-user.target.wants/" + +# Configure mass-region-controller if not already done previously +[ ! -e /var/lib/maas/secret ] || exit 0 +MAAS_FIXUP_SERVICE="/etc/systemd/system/maas-fixup.service" +cat <<-EOF | tee "${MAAS_FIXUP_SERVICE}" +[Unit] +After=postgresql.service +[Service] +ExecStart=/bin/sh -ec '\ + echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections && \ + /var/lib/dpkg/info/maas-region-controller.config configure && \ + /var/lib/dpkg/info/maas-region-controller.postinst configure' +EOF +ln -sf "${MAAS_FIXUP_SERVICE}" "/etc/systemd/system/multi-user.target.wants/" +rm "/usr/sbin/policy-rc.d" diff --git a/mcp/scripts/lib_jump_common.sh b/mcp/scripts/lib_jump_common.sh index 5b09c29ab..b89a33d71 100644 --- a/mcp/scripts/lib_jump_common.sh +++ b/mcp/scripts/lib_jump_common.sh @@ -61,13 +61,15 @@ function jumpserver_pkg_install { function jumpserver_check_requirements { # shellcheck disable=SC2178 + local states=$1; shift + # shellcheck disable=SC2178 local vnodes=$1; shift local br=("$@") local err_br_not_found='Linux bridge not found!' local err_br_virsh_net='is a virtual network, Linux bridge expected!' local warn_br_endpoint="Endpoints might be inaccessible from external hosts!" # MaaS requires a Linux bridge for PXE/admin - if [[ "${vnodes}" =~ mas01 ]]; then + if [[ "${states}" =~ maas ]]; then if ! brctl showmacs "${br[0]}" >/dev/null 2>&1; then notify_e "[ERROR] PXE/admin (${br[0]}) ${err_br_not_found}" fi @@ -77,9 +79,9 @@ function jumpserver_check_requirements { fi fi # If virtual nodes are present, public should be a Linux bridge - if [ "$(echo "${vnodes}" | wc -w)" -gt 2 ]; then + if [ -n "${vnodes}" ]; then if ! brctl showmacs "${br[3]}" >/dev/null 2>&1; then - if [[ "${vnodes}" =~ mas01 ]]; then + if [[ "${states}" =~ maas ]]; then # Baremetal nodes *require* a proper public network notify_e "[ERROR] Public (${br[3]}) ${err_br_not_found}" else @@ -88,7 +90,7 @@ function jumpserver_check_requirements { fi fi if ${VIRSH} net-info "${br[3]}" >/dev/null 2>&1; then - if [[ "${vnodes}" =~ mas01 ]]; then + if [[ "${states}" =~ maas ]]; then notify_e "[ERROR] ${br[3]} ${err_br_virsh_net}" else notify_n "[WARN] ${br[3]} ${err_br_virsh_net}" 3 diff --git a/mcp/scripts/lib_jump_deploy.sh b/mcp/scripts/lib_jump_deploy.sh index 9c4d8fb5a..bce54ade2 100644 --- a/mcp/scripts/lib_jump_deploy.sh +++ b/mcp/scripts/lib_jump_deploy.sh @@ -216,9 +216,6 @@ function prepare_vms { local image=base_image_opnfv_fuel.img local vcp_image=${image%.*}_vcp.img local _o=${base_image/*\/} - local _h=$(echo "${repos_pkgs_str}.$(md5sum "${image_dir}/${_o}")" | \ - md5sum | cut -c -8) - local _tmp [ -n "${image_dir}" ] || exit 1 cleanup_uefi @@ -226,8 +223,10 @@ function prepare_vms { __get_base_image "${base_image}" "${image_dir}" IFS='^' read -r -a repos_pkgs <<< "${repos_pkgs_str}" + local _h=$(echo "${repos_pkgs_str}.$(md5sum "${image_dir}/${_o}")" | \ + md5sum | cut -c -8) + local _tmp="${image%.*}.${_h}.img" echo "[INFO] Lookup cache / build patched base image for fingerprint: ${_h}" - _tmp="${image%.*}.${_h}.img" if [ "${image_dir}/${_tmp}" -ef "${image_dir}/${image}" ]; then echo "[INFO] Patched base image found" else @@ -278,9 +277,9 @@ function prepare_vms { } function create_networks { - local all_vnode_networks=("mcpcontrol" "$@") - # create required networks, including constant "mcpcontrol" - for net in "${all_vnode_networks[@]}"; do + local all_vnode_networks=("$@") + # create required networks + for net in "mcpcontrol" "${all_vnode_networks[@]}"; do if ${VIRSH} net-info "${net}" >/dev/null 2>&1; then ${VIRSH} net-destroy "${net}" || true ${VIRSH} net-undefine "${net}" @@ -293,8 +292,8 @@ function create_networks { ${VIRSH} net-start "${net}" fi done - # create veth pairs for relevant networks (mcpcontrol, pxebr, mgmt) - for i in $(seq 0 2 4); do + # create veth pairs for relevant networks (pxebr, mgmt) + for i in $(seq 0 2 2); do sudo ip link del "veth_mcp$i" || true sudo ip link add "veth_mcp$i" type veth peer name "veth_mcp$((i+1))" sudo ip link set "veth_mcp$i" up mtu 9000 @@ -337,14 +336,8 @@ function create_vms { # prepare network args local vnode_networks=("$@") - if [[ "${vnode_data[0]}" =~ ^(cfg01|mas01) ]]; then - net_args=" --network network=mcpcontrol,model=virtio" - # 3rd interface gets connected to PXE/Admin Bridge (cfg01, mas01) - vnode_networks[2]="${vnode_networks[0]}" - else - net_args=" --network bridge=${vnode_networks[0]},model=virtio" - fi - for net in "${vnode_networks[@]:1}"; do + local net_args= + for net in "${vnode_networks[@]}"; do net_args="${net_args} --network bridge=${net},model=virtio" done @@ -370,27 +363,16 @@ function create_vms { done } -function update_mcpcontrol_network { - # set static ip address for salt master node, MaaS node - local amac=$(${VIRSH} domiflist mas01 2>&1| awk '/mcpcontrol/ {print $5; exit}') - [ -z "${amac}" ] || ${VIRSH} net-update "mcpcontrol" add ip-dhcp-host \ - "<host mac='${amac}' name='mas01' ip='${MAAS_IP}'/>" --live --config -} - function reset_vms { local vnodes=("$@") local cmd_str="ssh ${SSH_OPTS} ${SSH_SALT}" # reset non-infrastructure vms, wait for them to come back online for node in "${vnodes[@]}"; do - if [[ ! "${node}" =~ (cfg01|mas01) ]]; then - ${VIRSH} reset "${node}" - fi + ${VIRSH} reset "${node}" done for node in "${vnodes[@]}"; do - if [[ ! "${node}" =~ (cfg01|mas01) ]]; then - wait_for 20.0 "${cmd_str} sudo salt -C '${node}*' saltutil.sync_all" - fi + wait_for 20.0 "${cmd_str} sudo salt -C '${node}*' saltutil.sync_all" done } @@ -414,9 +396,26 @@ function prepare_containers { if [[ ! "${MCP_DOCKER_TAG}" =~ 'verify' ]]; then "${COMPOSE_PREFIX}docker-compose" -f docker-compose/docker-compose.yaml pull fi - sudo rm -rf "${image_dir}/"{salt,hosts,pki} "${image_dir}/nodes/"* - mkdir -p "${image_dir}/salt/"{master.d,minion.d} - touch "${image_dir}/hosts" + # overwrite hosts only on first container up, to preserve cluster nodes + sudo cp docker-compose/files/hosts "${image_dir}/hosts" + sudo rm -rf "${image_dir}/"{salt,pki,mas01/etc} "${image_dir}/nodes/"* + find "${image_dir}/mas01/var/lib/" \ + -mindepth 2 -maxdepth 2 -not -name boot-resources \ + -exec sudo rm -rf {} \; || true + mkdir -p "${image_dir}/"{salt/master.d,salt/minion.d} + + if grep -q -e 'maas' 'docker-compose/docker-compose.yaml'; then + chmod +x docker-compose/files/entrypoint*.sh + # Apparmor workaround for bind9 inside Docker containers using AUFS + for profile in 'usr.sbin.ntpd' 'usr.sbin.named' \ + 'usr.sbin.dhcpd' 'usr.bin.tcpdump'; do + if [ -e "/etc/apparmor.d/${profile}" ] && \ + [ ! -e "/etc/apparmor.d/disable/${profile}" ]; then + sudo ln -sf "/etc/apparmor.d/${profile}" "/etc/apparmor.d/disable/" + sudo apparmor_parser -R "/etc/apparmor.d/${profile}" || true + fi + done + fi } function start_containers { diff --git a/mcp/scripts/user-data.sh.j2 b/mcp/scripts/user-data.sh.j2 index cebf3bc61..d77773260 100644 --- a/mcp/scripts/user-data.sh.j2 +++ b/mcp/scripts/user-data.sh.j2 @@ -7,13 +7,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## {%- import 'net_map.j2' as nm with context %} -{%- set infra_nodes = conf[conf.MCP_JUMP_ARCH].default.virtual.nodes.infra %} rm /etc/salt/minion_id rm -f /etc/salt/pki/minion/minion_master.pub echo "id: $(hostname).{{ conf.cluster.domain }}" > /etc/salt/minion -if [[ "{{ infra_nodes or [] | join(' ') }}" =~ $(hostname) ]]; then - echo "master: {{ conf.SALT_MASTER }}" >> /etc/salt/minion -else - echo "master: {{ nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_config_pxe_admin_address') +1) }}" >> /etc/salt/minion -fi +echo "master: {{ nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_config_pxe_admin_address') +1) }}" >> /etc/salt/minion service salt-minion restart diff --git a/mcp/scripts/virsh_net/net_mcpcontrol.xml.j2 b/mcp/scripts/virsh_net/net_mcpcontrol.xml.j2 deleted file mode 100644 index 46798a3c6..000000000 --- a/mcp/scripts/virsh_net/net_mcpcontrol.xml.j2 +++ /dev/null @@ -1,20 +0,0 @@ -<!-- - Copyright (c) 2017 Mirantis Inc., Enea AB and others. - - All rights reserved. This program and the accompanying materials - are made available under the terms of the Apache License, Version 2.0 - which accompanies this distribution, and is available at - http://www.apache.org/licenses/LICENSE-2.0 ---> -{#- conf.MCPCONTROL_NET & co are mandatory, defaults are set via globals.sh #} -{%- set net_mcpcontrol = [conf.MCPCONTROL_NET, conf.MCPCONTROL_PREFIX] | join("/") %} -<network> - <name>mcpcontrol</name> - <bridge name="mcpcontrol"/> - <forward mode="nat"/> - <ip address="{{ net_mcpcontrol | ipnet_hostaddr(1) }}" netmask="{{ net_mcpcontrol | ipnet_netmask }}"> - <dhcp> - <range start="{{ net_mcpcontrol | ipnet_hostaddr(3) }}" end="{{ net_mcpcontrol | ipnet_hostmax }}"/> - </dhcp> - </ip> -</network> diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2 index 4db5593e3..30e41e95c 100644 --- a/mcp/scripts/xdf_data.sh.j2 +++ b/mcp/scripts/xdf_data.sh.j2 @@ -15,9 +15,7 @@ {%- set cluster_states = conf.cluster.states if conf.MCP_NO_DEPLOY_ENVIRONMENT < 2 else [] -%} {%- set arch = conf[conf.MCP_JUMP_ARCH] -%} {%- set V = conf.virtual -%} -{%- do V.nodes.update(arch.default.virtual.nodes) -%} {%- set section_map = { - 'infra': -1, 'control': nm.ctl01.idx, 'compute': nm.cmp001.idx } -%} @@ -36,7 +34,7 @@ {%- set arr = [] -%} {%- for section in section_map -%} {%- for n in V.nodes[section] or [] -%} - {%- if section_map[section] < 0 or conf.nodes[section_map[section] + loop.index0].node.type == 'virtual' -%} + {%- if conf.nodes[section_map[section] + loop.index0].node.type == 'virtual' -%} {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%} {%- set cpu_topo = 'cpu_topology' in V[n] and not conf.MCP_CMP_SS -%} {%- if 'numa' in V[n] and cpu_topo -%} @@ -63,9 +61,7 @@ {%- set arr = [] -%} {%- for section in sections -%} {%- for n in V.nodes[section] or [] -%} - {%- if ( (section_map[section] < 0 and type == 'virtual') or - (section_map[section] >= 0 and - conf.nodes[section_map[section] + loop.index0].node.type in type) ) -%} + {%- if conf.nodes[section_map[section] + loop.index0].node.type in type -%} {%- do arr.append(n) -%} {%- endif -%} {%- endfor -%} |