[baremetal] Containerize MaaS

- replace mas01 VM with a Docker container;
- drop `mcpcontrol` virsh-managed network, including special handling
  previously required for it across all scripts;
- drop infrastructure VMs handling from scripts, the only VMs we still
  handle are cluster VMs for virtual and/or hybrid deployments;
- drop SSH server from mas01;
- stop running linux state on mas01, as all prerequisites are properly
  handled durin Docker build or via entrypoint.sh - for completeness,
  we still keep pillar data in sync with the actual contents of mas01
  configuration, so running the state manually would still work;
- make port 5240 available on the jumpserver for MaaS dashboard access;
- docs: update diagrams and text to reflect the new changes;

Change-Id: I6d9424995e9a90c530fd7577edf401d552bab929
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

9 files changed, 145 insertions, 88 deletions

diff --git a/mcp/scripts/.gitignore b/mcp/scripts/.gitignore
index 6a95545c4..c21c3d9de 100644
--- a/mcp/scripts/.gitignore
+++ b/mcp/scripts/.gitignore
@@ -1,3 +1,4 @@
 mcp.rsa*
 user-data.sh
 xdf_data.sh
+docker-compose/files/entrypoint_maas.sh
diff --git a/mcp/scripts/docker-compose/docker-compose.yaml.j2 b/mcp/scripts/docker-compose/docker-compose.yaml.j2
index 891d55938..bc8b3e80a 100644
--- a/mcp/scripts/docker-compose/docker-compose.yaml.j2
+++ b/mcp/scripts/docker-compose/docker-compose.yaml.j2
@@ -22,33 +22,60 @@ services:
       mgmt:
         ipv4_address: {{ nm.net_mgmt | ipnet_hostaddr(nm.start_ip[nm.net_mgmt] + nm.net_mgmt_hosts.index('opnfv_infra_config_address') +1) }}
     volumes:
-       - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket:ro
-       - {{ conf.MCP_REPO_ROOT_PATH }}:/root/fuel
-       - {{ conf.MCP_REPO_ROOT_PATH }}/mcp/scripts/docker-compose/files/entrypoint.sh:/entrypoint.sh
-       - {{ conf.MCP_STORAGE_DIR }}/pod_config.yml:/root/pod_config.yml
-       - {{ conf.MCP_STORAGE_DIR }}/nodes:/srv/salt/reclass/nodes
-       - {{ conf.MCP_STORAGE_DIR }}/pki:/etc/pki
-       - {{ conf.MCP_STORAGE_DIR }}/salt:/etc/salt
-       - {{ conf.MCP_STORAGE_DIR }}/hosts:/etc/hosts
+      - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket:ro
+      - {{ conf.MCP_REPO_ROOT_PATH }}:/root/fuel
+      - {{ conf.MCP_REPO_ROOT_PATH }}/mcp/scripts/docker-compose/files/entrypoint.sh:/entrypoint.sh
+      - {{ conf.MCP_STORAGE_DIR }}/pod_config.yml:/root/pod_config.yml
+      - {{ conf.MCP_STORAGE_DIR }}/nodes:/srv/salt/reclass/nodes
+      - {{ conf.MCP_STORAGE_DIR }}/pki:/etc/pki
+      - {{ conf.MCP_STORAGE_DIR }}/salt:/etc/salt
+      - {{ conf.MCP_STORAGE_DIR }}/hosts:/etc/hosts
 {%- if conf.MCP_VCP %}
-       - {{ conf.MCP_STORAGE_DIR }}/base_image_opnfv_fuel_vcp.img:/srv/salt/env/prd/salt/files/control/images/base_image_opnfv_fuel_vcp.img
+      - {{ conf.MCP_STORAGE_DIR }}/base_image_opnfv_fuel_vcp.img:/srv/salt/env/prd/salt/files/control/images/base_image_opnfv_fuel_vcp.img
 {%- endif %}
     hostname: cfg01
     domainname: {{ conf.cluster.domain }}
     privileged: true
+{%- if nm.cluster.has_baremetal_nodes %}
+  opnfv-fuel-maas:
+    container_name: "maas"
+    image: "opnfv/fuel:saltminion-maas-{{ conf.MCP_DOCKER_TAG }}"
+    networks:
+      mcpcontrol:
+        ipv4_address: {{ conf.MAAS_IP }}
+      pxebr:
+        ipv4_address: {{ nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_maas_node01_deploy_address') +1) }}
+      mgmt:
+        ipv4_address: {{ nm.net_mgmt | ipnet_hostaddr(nm.start_ip[nm.net_mgmt] + nm.net_mgmt_hosts.index('opnfv_infra_maas_node01_address') +1) }}
+    volumes:
+      - /lib/modules:/lib/modules:ro
+      - /sys/fs/cgroup:/sys/fs/cgroup:ro
+      - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket:ro
+      - {{ conf.MCP_REPO_ROOT_PATH }}/mcp/scripts/docker-compose/files/entrypoint_maas.sh:/entrypoint.sh:ro
+      - {{ conf.MCP_STORAGE_DIR }}/hosts:/etc/hosts:ro
+      - {{ conf.MCP_STORAGE_DIR }}/mas01/etc/iptables:/etc/iptables
+      - {{ conf.MCP_STORAGE_DIR }}/mas01/var/lib/postgresql:/var/lib/postgresql
+      - {{ conf.MCP_STORAGE_DIR }}/mas01/var/lib/maas:/var/lib/maas
+      - {{ conf.MCP_STORAGE_DIR }}/mas01/var/spool/maas-proxy:/var/spool/maas-proxy
+      - {{ conf.MCP_STORAGE_DIR }}/mas01/etc/maas:/etc/maas
+    hostname: mas01
+    domainname: {{ conf.cluster.domain }}
+    privileged: true
+    ports:
+      - 5240:5240
+{%- endif %}
 networks:
   mcpcontrol:
-    driver: macvlan
+    driver: bridge
     driver_opts:
-      parent: veth_mcp1  # Always untagged
+      com.docker.network.driver.mtu: 9000
     ipam:
       config:
         - subnet: {{ net_mcpcontrol }}
-          gateway: {{ net_mcpcontrol | ipnet_hostaddr(1) }}
   pxebr:
     driver: macvlan
     driver_opts:
-      parent: veth_mcp3  # Always untagged
+      parent: veth_mcp1  # Always untagged
     ipam:
       config:
         - subnet: {{ nm.net_admin }}
@@ -58,7 +85,7 @@ networks:
 {%- if conf.idf.fuel.jumphost.get('trunks', {}).get('mgmt', False) %}
       parent: {{ ma.interface_str('veth_mcp5', nm.vlan_mgmt) }}
 {%- else %}
-      parent: veth_mcp5  # Untagged by default
+      parent: veth_mcp3  # Untagged by default
 {%- endif %}
     ipam:
       config:
diff --git a/mcp/scripts/docker-compose/files/entrypoint.sh b/mcp/scripts/docker-compose/files/entrypoint.sh
index 9830ea15f..a0f72e2d1 100755
--- a/mcp/scripts/docker-compose/files/entrypoint.sh
+++ b/mcp/scripts/docker-compose/files/entrypoint.sh
@@ -21,11 +21,6 @@ if [ ! -f /home/ubuntu/.ssh/authorized_keys ]; then
     echo 'IdentityFile /root/fuel/mcp/scripts/mcp.rsa' >> /root/.ssh/config
 fi
 
-if ! grep -q localhost /etc/hosts; then
-    # overwrite hosts only on first container up, to preserve cluster nodes
-    cp -a /root/fuel/mcp/scripts/docker-compose/files/hosts /etc/hosts
-fi
-
 # salt state does not properly configure file_roots in master.conf, hard set it
 cp -a /root/fuel/mcp/scripts/docker-compose/files/opnfv_master.conf \
       /etc/salt/master.d/opnfv_master.conf
diff --git a/mcp/scripts/docker-compose/files/entrypoint_maas.sh.j2 b/mcp/scripts/docker-compose/files/entrypoint_maas.sh.j2
new file mode 100644
index 000000000..23b8d8f30
--- /dev/null
+++ b/mcp/scripts/docker-compose/files/entrypoint_maas.sh.j2
@@ -0,0 +1,62 @@
+#!/bin/bash -e
+##############################################################################
+# Copyright (c) 2019 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+{%- import 'net_map.j2' as nm with context %}
+{%- set pxebr_addr = nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_maas_node01_deploy_address') +1) %}
+if [ ! -e /var/lib/postgresql/*/main ]; then
+    cp -ar /var/lib/opnfv/{postgresql,maas} /var/lib/
+    cp -ar /var/lib/opnfv/etc/{ssh,maas} /etc/
+fi
+chown -R maas:maas /var/lib/maas
+chown -R postgres:postgres /var/lib/postgresql
+chown -R proxy:proxy /var/spool/maas-proxy
+
+if [ ! -f /etc/sysctl.d/99-salt.conf ]; then
+    echo 'net.ipv4.ip_forward = 1' > /etc/sysctl.d/99-salt.conf
+fi
+
+cat <<-EOF | tee /etc/resolv.conf
+{%- for server in nm.dns_public %}
+nameserver {{ server }}
+{%- endfor %}
+EOF
+
+cat <<-EOF | tee /etc/salt/minion.d/opnfv.conf
+id: mas01.{{ conf.cluster.domain }}
+master: {{ conf.SALT_MASTER }}
+grains:
+  virtual_subtype: Docker_
+EOF
+rm -f /etc/salt/minion.d/99-master-address.conf
+
+# Work around MaaS issues with PXE/admin using jumbo frames
+MAAS_MTU_SERVICE="/etc/systemd/system/maas-mtu.service"
+cat <<-EOF | tee "${MAAS_MTU_SERVICE}"
+[Unit]
+Requires=network-online.target
+After=network-online.target
+[Service]
+ExecStart=/bin/sh -ec '\
+  /sbin/ifconfig $(/sbin/ip addr | /bin/grep -Po "{{ pxebr_addr }}.* \K(.*)") mtu 1500'
+EOF
+ln -sf "${MAAS_MTU_SERVICE}" "/etc/systemd/system/multi-user.target.wants/"
+
+# Configure mass-region-controller if not already done previously
+[ ! -e /var/lib/maas/secret ] || exit 0
+MAAS_FIXUP_SERVICE="/etc/systemd/system/maas-fixup.service"
+cat <<-EOF | tee "${MAAS_FIXUP_SERVICE}"
+[Unit]
+After=postgresql.service
+[Service]
+ExecStart=/bin/sh -ec '\
+  echo "debconf debconf/frontend select Noninteractive" | debconf-set-selections && \
+  /var/lib/dpkg/info/maas-region-controller.config configure && \
+  /var/lib/dpkg/info/maas-region-controller.postinst configure'
+EOF
+ln -sf "${MAAS_FIXUP_SERVICE}" "/etc/systemd/system/multi-user.target.wants/"
+rm "/usr/sbin/policy-rc.d"
diff --git a/mcp/scripts/lib_jump_common.sh b/mcp/scripts/lib_jump_common.sh
index 5b09c29ab..b89a33d71 100644
--- a/mcp/scripts/lib_jump_common.sh
+++ b/mcp/scripts/lib_jump_common.sh
@@ -61,13 +61,15 @@ function jumpserver_pkg_install {
 
 function jumpserver_check_requirements {
   # shellcheck disable=SC2178
+  local states=$1; shift
+  # shellcheck disable=SC2178
   local vnodes=$1; shift
   local br=("$@")
   local err_br_not_found='Linux bridge not found!'
   local err_br_virsh_net='is a virtual network, Linux bridge expected!'
   local warn_br_endpoint="Endpoints might be inaccessible from external hosts!"
   # MaaS requires a Linux bridge for PXE/admin
-  if [[ "${vnodes}" =~ mas01 ]]; then
+  if [[ "${states}" =~ maas ]]; then
     if ! brctl showmacs "${br[0]}" >/dev/null 2>&1; then
       notify_e "[ERROR] PXE/admin (${br[0]}) ${err_br_not_found}"
     fi
@@ -77,9 +79,9 @@ function jumpserver_check_requirements {
     fi
   fi
   # If virtual nodes are present, public should be a Linux bridge
-  if [ "$(echo "${vnodes}" | wc -w)" -gt 2 ]; then
+  if [ -n "${vnodes}" ]; then
     if ! brctl showmacs "${br[3]}" >/dev/null 2>&1; then
-      if [[ "${vnodes}" =~ mas01 ]]; then
+      if [[ "${states}" =~ maas ]]; then
         # Baremetal nodes *require* a proper public network
         notify_e "[ERROR] Public (${br[3]}) ${err_br_not_found}"
       else
@@ -88,7 +90,7 @@ function jumpserver_check_requirements {
       fi
     fi
     if ${VIRSH} net-info "${br[3]}" >/dev/null 2>&1; then
-      if [[ "${vnodes}" =~ mas01 ]]; then
+      if [[ "${states}" =~ maas ]]; then
         notify_e "[ERROR] ${br[3]} ${err_br_virsh_net}"
       else
         notify_n "[WARN] ${br[3]} ${err_br_virsh_net}" 3
diff --git a/mcp/scripts/lib_jump_deploy.sh b/mcp/scripts/lib_jump_deploy.sh
index 9c4d8fb5a..bce54ade2 100644
--- a/mcp/scripts/lib_jump_deploy.sh
+++ b/mcp/scripts/lib_jump_deploy.sh
@@ -216,9 +216,6 @@ function prepare_vms {
   local image=base_image_opnfv_fuel.img
   local vcp_image=${image%.*}_vcp.img
   local _o=${base_image/*\/}
-  local _h=$(echo "${repos_pkgs_str}.$(md5sum "${image_dir}/${_o}")" | \
-             md5sum | cut -c -8)
-  local _tmp
   [ -n "${image_dir}" ] || exit 1
 
   cleanup_uefi
@@ -226,8 +223,10 @@ function prepare_vms {
   __get_base_image "${base_image}" "${image_dir}"
   IFS='^' read -r -a repos_pkgs <<< "${repos_pkgs_str}"
 
+  local _h=$(echo "${repos_pkgs_str}.$(md5sum "${image_dir}/${_o}")" | \
+             md5sum | cut -c -8)
+  local _tmp="${image%.*}.${_h}.img"
   echo "[INFO] Lookup cache / build patched base image for fingerprint: ${_h}"
-  _tmp="${image%.*}.${_h}.img"
   if [ "${image_dir}/${_tmp}" -ef "${image_dir}/${image}" ]; then
     echo "[INFO] Patched base image found"
   else
@@ -278,9 +277,9 @@ function prepare_vms {
 }
 
 function create_networks {
-  local all_vnode_networks=("mcpcontrol" "$@")
-  # create required networks, including constant "mcpcontrol"
-  for net in "${all_vnode_networks[@]}"; do
+  local all_vnode_networks=("$@")
+  # create required networks
+  for net in "mcpcontrol" "${all_vnode_networks[@]}"; do
     if ${VIRSH} net-info "${net}" >/dev/null 2>&1; then
       ${VIRSH} net-destroy "${net}" || true
       ${VIRSH} net-undefine "${net}"
@@ -293,8 +292,8 @@ function create_networks {
       ${VIRSH} net-start "${net}"
     fi
   done
-  # create veth pairs for relevant networks (mcpcontrol, pxebr, mgmt)
-  for i in $(seq 0 2 4); do
+  # create veth pairs for relevant networks (pxebr, mgmt)
+  for i in $(seq 0 2 2); do
     sudo ip link del "veth_mcp$i" || true
     sudo ip link add "veth_mcp$i" type veth peer name "veth_mcp$((i+1))"
     sudo ip link set "veth_mcp$i" up mtu 9000
@@ -337,14 +336,8 @@ function create_vms {
 
     # prepare network args
     local vnode_networks=("$@")
-    if [[ "${vnode_data[0]}" =~ ^(cfg01|mas01) ]]; then
-      net_args=" --network network=mcpcontrol,model=virtio"
-      # 3rd interface gets connected to PXE/Admin Bridge (cfg01, mas01)
-      vnode_networks[2]="${vnode_networks[0]}"
-    else
-      net_args=" --network bridge=${vnode_networks[0]},model=virtio"
-    fi
-    for net in "${vnode_networks[@]:1}"; do
+    local net_args=
+    for net in "${vnode_networks[@]}"; do
       net_args="${net_args} --network bridge=${net},model=virtio"
     done
 
@@ -370,27 +363,16 @@ function create_vms {
   done
 }
 
-function update_mcpcontrol_network {
-  # set static ip address for salt master node, MaaS node
-  local amac=$(${VIRSH} domiflist mas01 2>&1| awk '/mcpcontrol/ {print $5; exit}')
-  [ -z "${amac}" ] || ${VIRSH} net-update "mcpcontrol" add ip-dhcp-host \
-    "<host mac='${amac}' name='mas01' ip='${MAAS_IP}'/>" --live --config
-}
-
 function reset_vms {
   local vnodes=("$@")
   local cmd_str="ssh ${SSH_OPTS} ${SSH_SALT}"
 
   # reset non-infrastructure vms, wait for them to come back online
   for node in "${vnodes[@]}"; do
-    if [[ ! "${node}" =~ (cfg01|mas01) ]]; then
-      ${VIRSH} reset "${node}"
-    fi
+    ${VIRSH} reset "${node}"
   done
   for node in "${vnodes[@]}"; do
-    if [[ ! "${node}" =~ (cfg01|mas01) ]]; then
-      wait_for 20.0 "${cmd_str} sudo salt -C '${node}*' saltutil.sync_all"
-    fi
+    wait_for 20.0 "${cmd_str} sudo salt -C '${node}*' saltutil.sync_all"
   done
 }
 
@@ -414,9 +396,26 @@ function prepare_containers {
   if [[ ! "${MCP_DOCKER_TAG}" =~ 'verify' ]]; then
     "${COMPOSE_PREFIX}docker-compose" -f docker-compose/docker-compose.yaml pull
   fi
-  sudo rm -rf "${image_dir}/"{salt,hosts,pki} "${image_dir}/nodes/"*
-  mkdir -p "${image_dir}/salt/"{master.d,minion.d}
-  touch "${image_dir}/hosts"
+  # overwrite hosts only on first container up, to preserve cluster nodes
+  sudo cp docker-compose/files/hosts "${image_dir}/hosts"
+  sudo rm -rf "${image_dir}/"{salt,pki,mas01/etc} "${image_dir}/nodes/"*
+  find "${image_dir}/mas01/var/lib/" \
+    -mindepth 2 -maxdepth 2 -not -name boot-resources \
+    -exec sudo rm -rf {} \; || true
+  mkdir -p "${image_dir}/"{salt/master.d,salt/minion.d}
+
+  if grep -q -e 'maas' 'docker-compose/docker-compose.yaml'; then
+    chmod +x docker-compose/files/entrypoint*.sh
+    # Apparmor workaround for bind9 inside Docker containers using AUFS
+    for profile in 'usr.sbin.ntpd' 'usr.sbin.named' \
+                   'usr.sbin.dhcpd' 'usr.bin.tcpdump'; do
+      if [ -e "/etc/apparmor.d/${profile}" ] && \
+       [ ! -e "/etc/apparmor.d/disable/${profile}" ]; then
+        sudo ln -sf "/etc/apparmor.d/${profile}" "/etc/apparmor.d/disable/"
+        sudo apparmor_parser -R "/etc/apparmor.d/${profile}" || true
+      fi
+    done
+  fi
 }
 
 function start_containers {
diff --git a/mcp/scripts/user-data.sh.j2 b/mcp/scripts/user-data.sh.j2
index cebf3bc61..d77773260 100644
--- a/mcp/scripts/user-data.sh.j2
+++ b/mcp/scripts/user-data.sh.j2
@@ -7,13 +7,8 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 {%- import 'net_map.j2' as nm with context %}
-{%- set infra_nodes = conf[conf.MCP_JUMP_ARCH].default.virtual.nodes.infra %}
 rm /etc/salt/minion_id
 rm -f /etc/salt/pki/minion/minion_master.pub
 echo "id: $(hostname).{{ conf.cluster.domain }}" > /etc/salt/minion
-if [[ "{{ infra_nodes or [] | join(' ') }}" =~ $(hostname) ]]; then
-  echo "master: {{ conf.SALT_MASTER }}" >> /etc/salt/minion
-else
-  echo "master: {{ nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_config_pxe_admin_address') +1) }}" >> /etc/salt/minion
-fi
+echo "master: {{ nm.net_admin | ipnet_hostaddr(nm.start_ip[nm.net_admin] + nm.net_admin_hosts.index('opnfv_infra_config_pxe_admin_address') +1) }}" >> /etc/salt/minion
 service salt-minion restart
diff --git a/mcp/scripts/virsh_net/net_mcpcontrol.xml.j2 b/mcp/scripts/virsh_net/net_mcpcontrol.xml.j2
deleted file mode 100644
index 46798a3c6..000000000
--- a/mcp/scripts/virsh_net/net_mcpcontrol.xml.j2
+++ /dev/null
@@ -1,20 +0,0 @@
-<!--
- Copyright (c) 2017 Mirantis Inc., Enea AB and others.
-
- All rights reserved. This program and the accompanying materials
- are made available under the terms of the Apache License, Version 2.0
- which accompanies this distribution, and is available at
- http://www.apache.org/licenses/LICENSE-2.0
--->
-{#- conf.MCPCONTROL_NET & co are mandatory, defaults are set via globals.sh #}
-{%- set net_mcpcontrol = [conf.MCPCONTROL_NET, conf.MCPCONTROL_PREFIX] | join("/") %}
-<network>
-  <name>mcpcontrol</name>
-  <bridge name="mcpcontrol"/>
-  <forward mode="nat"/>
-  <ip address="{{ net_mcpcontrol | ipnet_hostaddr(1) }}" netmask="{{ net_mcpcontrol | ipnet_netmask }}">
-    <dhcp>
-      <range start="{{ net_mcpcontrol | ipnet_hostaddr(3) }}" end="{{ net_mcpcontrol | ipnet_hostmax }}"/>
-    </dhcp>
-  </ip>
-</network>
diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2
index 4db5593e3..30e41e95c 100644
--- a/mcp/scripts/xdf_data.sh.j2
+++ b/mcp/scripts/xdf_data.sh.j2
@@ -15,9 +15,7 @@
 {%- set cluster_states = conf.cluster.states if conf.MCP_NO_DEPLOY_ENVIRONMENT < 2 else [] -%}
 {%- set arch = conf[conf.MCP_JUMP_ARCH] -%}
 {%- set V = conf.virtual -%}
-{%- do V.nodes.update(arch.default.virtual.nodes) -%}
 {%- set section_map = {
-  'infra': -1,
   'control': nm.ctl01.idx,
   'compute': nm.cmp001.idx
 } -%}
@@ -36,7 +34,7 @@
   {%- set arr = [] -%}
   {%- for section in section_map -%}
     {%- for n in V.nodes[section] or [] -%}
-      {%- if section_map[section] < 0 or conf.nodes[section_map[section] + loop.index0].node.type == 'virtual' -%}
+      {%- if conf.nodes[section_map[section] + loop.index0].node.type == 'virtual' -%}
         {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%}
         {%- set cpu_topo = 'cpu_topology' in V[n] and not conf.MCP_CMP_SS -%}
         {%- if 'numa' in V[n] and cpu_topo -%}
@@ -63,9 +61,7 @@
   {%- set arr = [] -%}
   {%- for section in sections -%}
     {%- for n in V.nodes[section] or [] -%}
-      {%- if ( (section_map[section] < 0 and type == 'virtual') or
-               (section_map[section] >= 0 and
-                conf.nodes[section_map[section] + loop.index0].node.type in type) ) -%}
+      {%- if conf.nodes[section_map[section] + loop.index0].node.type in type -%}
         {%- do arr.append(n) -%}
       {%- endif -%}
     {%- endfor -%}