[vnode parsing] Move core logic to j2

j2/python is easier to read and manipulate strings, although it does
need some special care about undefined dict keys.

With this in place, deploy.sh only contains the higher level logic for
the deployment process.

- merge arch-specific default configuration files into a singular
  file with arch name as main dict key of old config (also avoids
  creating duplicate 'virtual' YAML keys in $LOCAL_PDF);
- move template handling to separate <lib_template.sh>;
- decouple tight bash ordering of scenario expansion -> parse_yaml ->
  variable export (e.g. CLUSTER_DOMAIN) -> re-use in cluster j2s;
  however we can't parse *all* j2s in one go, as scenario j2s might
  expand to YAMLs needed while expanding cluster j2;
- split `do_templates` into separate functions for each stage, with
  no coupling between them other then call order;

Change-Id: I4b5e804094c00e5e918caf769fd85fa52181ad76
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

7 files changed, 168 insertions, 97 deletions

diff --git a/mcp/config/scenario/defaults-x86_64.yaml b/mcp/config/scenario/defaults-x86_64.yaml
deleted file mode 100644
index 18b0826f9..000000000
--- a/mcp/config/scenario/defaults-x86_64.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
----
-base_image: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
-virtual:
-  default:
-    vcpus: 2
-    ram: 4096
-  common:
-    apt:
-      keys:
-        - https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.11/SALTSTACK-GPG-KEY.pub
-      repos:
-        # <repo name> <repo prio> deb [arch=<arch>] <repo url> <repo dist> <repo comp>
-        - saltstack 500 deb [arch=amd64] http://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.11 xenial main
-    pkg:
-      install:
-        - salt-minion
-  control:
-    pkg:
-      install:
-        - cloud-init
diff --git a/mcp/config/scenario/defaults-aarch64.yaml b/mcp/config/scenario/defaults.yaml
index 1efa12ae4..e6915d468 100644
--- a/mcp/config/scenario/defaults-aarch64.yaml
+++ b/mcp/config/scenario/defaults.yaml
@@ -1,13 +1,33 @@
 ##############################################################################
-# Copyright (c) 2017 Mirantis Inc. and others.
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 ---
-base_image: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-arm64-uefi1.img
-virtual:
+x86_64:
+  base_image: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
+  default:
+    vcpus: 2
+    ram: 4096
+  common:
+    apt:
+      keys:
+        - https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.11/SALTSTACK-GPG-KEY.pub
+      repos:
+        # <repo name> <repo prio> deb [arch=<arch>] <repo url> <repo dist> <repo comp>
+        - saltstack 500 deb [arch=amd64] http://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.11 xenial main
+    pkg:
+      install:
+        - salt-minion
+  control:
+    apt: ~
+    pkg:
+      install:
+        - cloud-init
+aarch64:
+  base_image: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-arm64-uefi1.img
   default:
     vcpus: 6
     ram: 4096
@@ -25,6 +45,7 @@ virtual:
         - linux-headers-generic-hwe-16.04-edge
         - salt-minion
   control:
+    apt: ~
     pkg:
       install:
         - cloud-init
diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh
index e8edf9587..5be4c556d 100644
--- a/mcp/scripts/lib.sh
+++ b/mcp/scripts/lib.sh
@@ -1,7 +1,7 @@
 #!/bin/bash -e
 # shellcheck disable=SC2155,SC1001,SC2015,SC2128
 ##############################################################################
-# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
 # which accompanies this distribution, and is available at
@@ -543,65 +543,3 @@ function get_nova_compute_pillar_data {
     echo "${value}"
   fi
 }
-
-function do_templates() {
-  local git_repo_root=$1; shift
-  local image_dir=$1; shift
-  local target_lab=$1; shift
-  local target_pod=$1; shift
-  local lab_config_uri=$1; shift
-  local scenario_dir=${1:-}
-
-  RECLASS_CLUSTER_DIR=$(cd "${git_repo_root}/mcp/reclass/classes/cluster"; pwd)
-  PHAROS_GEN_CFG="./pharos/config/utils/generate_config.py"
-  PHAROS_IA=$(readlink -f "./pharos/config/installers/fuel/pod_config.yml.j2")
-  PHAROS_VALIDATE_SCHEMA_SCRIPT="./pharos/config/utils/validate_schema.py"
-  PHAROS_SCHEMA_PDF="./pharos/config/pdf/pod1.schema.yaml"
-  PHAROS_SCHEMA_IDF="./pharos/config/pdf/idf-pod1.schema.yaml"
-  BASE_CONFIG_PDF="${lab_config_uri}/labs/${target_lab}/${target_pod}.yaml"
-  BASE_CONFIG_IDF="${lab_config_uri}/labs/${target_lab}/idf-${target_pod}.yaml"
-  LOCAL_PDF="${image_dir}/$(basename "${BASE_CONFIG_PDF}")"
-  LOCAL_IDF="${image_dir}/$(basename "${BASE_CONFIG_IDF}")"
-
-  # Two-stage expansion, first stage handles pod_config and scenarios only
-  if [ -n "${scenario_dir}" ]; then
-    # Make sample PDF/IDF available via default lab-config (pharos submodule)
-    ln -sf "$(readlink -f "../config/labs/local")" "./pharos/labs/"
-
-    # Expand scenario file and main reclass input (pod_config.yaml) based on PDF
-    if ! curl --create-dirs -o "${LOCAL_PDF}" "${BASE_CONFIG_PDF}"; then
-      notify_e "[ERROR] Could not retrieve PDF (Pod Descriptor File)!"
-    elif ! curl -o "${LOCAL_IDF}" "${BASE_CONFIG_IDF}"; then
-      notify_e "[ERROR] Could not retrieve IDF (Installer Descriptor File)!"
-    fi
-    # Check first if configuration files are valid
-    if [[ ! "$target_pod" =~ "virtual" ]]; then
-      if ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_PDF}" \
-        -s "${PHAROS_SCHEMA_PDF}"; then
-        notify_e "[ERROR] PDF does not match yaml schema!"
-      elif ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_IDF}" \
-        -s "${PHAROS_SCHEMA_IDF}"; then
-        notify_e "[ERROR] IDF does not match yaml schema!"
-      fi
-    fi
-    if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \
-      -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then
-      notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!"
-    fi
-    template_dirs="${scenario_dir}"
-    template_err_str='Could not convert j2 scenario definitions!'
-  else
-    # Expand reclass and virsh network templates based on PDF + IDF
-    printenv | \
-      awk '/^(SALT|MCP|MAAS|CLUSTER).*=/ { gsub(/=/,": "); print }' >> "${LOCAL_PDF}"
-    template_dirs="${RECLASS_CLUSTER_DIR} $(readlink -f virsh_net) $(readlink -f ./*j2)"
-    template_err_str='Could not convert PDF to network definitions!'
-  fi
-  # shellcheck disable=SC2086
-  j2args=$(find $template_dirs -name '*.j2' -exec echo -j {} \;)
-  # shellcheck disable=SC2086
-  if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
-    -i "$(dirname "${PHAROS_IA}")"; then
-    notify_e "[ERROR] ${template_err_str}"
-  fi
-}
diff --git a/mcp/scripts/lib_template.sh b/mcp/scripts/lib_template.sh
new file mode 100644
index 000000000..0fbe628b7
--- /dev/null
+++ b/mcp/scripts/lib_template.sh
@@ -0,0 +1,88 @@
+#!/bin/bash -e
+# shellcheck disable=SC2155,SC1001,SC2015,SC2128
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+#
+# Library of shell functions dedicated to j2 template handling
+#
+
+PHAROS_GEN_CFG='./pharos/config/utils/generate_config.py'
+PHAROS_IA='./pharos/config/installers/fuel/pod_config.yml.j2'
+PHAROS_VALIDATE_SCHEMA_SCRIPT='./pharos/config/utils/validate_schema.py'
+PHAROS_SCHEMA_PDF='./pharos/config/pdf/pod1.schema.yaml'
+PHAROS_SCHEMA_IDF='./pharos/config/pdf/idf-pod1.schema.yaml'
+
+# Handles pod_config and scenarios only
+function do_templates_scenario {
+  local image_dir=$1; shift
+  local target_lab=$1; shift
+  local target_pod=$1; shift
+  local lab_config_uri=$1; shift
+  local scenario_dir=$1
+
+  BASE_CONFIG_PDF="${lab_config_uri}/labs/${target_lab}/${target_pod}.yaml"
+  BASE_CONFIG_IDF="${lab_config_uri}/labs/${target_lab}/idf-${target_pod}.yaml"
+  LOCAL_PDF="${image_dir}/$(basename "${BASE_CONFIG_PDF}")"
+  LOCAL_IDF="${image_dir}/$(basename "${BASE_CONFIG_IDF}")"
+
+  # Make sample PDF/IDF available via default lab-config (pharos submodule)
+  ln -sf "$(readlink -f "../config/labs/local")" "./pharos/labs/"
+
+  # Expand scenario file and main reclass input (pod_config.yaml) based on PDF
+  if ! curl --create-dirs -o "${LOCAL_PDF}" "${BASE_CONFIG_PDF}"; then
+    notify_e "[ERROR] Could not retrieve PDF (Pod Descriptor File)!"
+  elif ! curl -o "${LOCAL_IDF}" "${BASE_CONFIG_IDF}"; then
+    notify_e "[ERROR] Could not retrieve IDF (Installer Descriptor File)!"
+  fi
+  # Check first if configuration files are valid
+  if [[ ! "$target_pod" =~ "virtual" ]]; then
+    if ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_PDF}" \
+      -s "${PHAROS_SCHEMA_PDF}"; then
+      notify_e "[ERROR] PDF does not match yaml schema!"
+    elif ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_IDF}" \
+      -s "${PHAROS_SCHEMA_IDF}"; then
+      notify_e "[ERROR] IDF does not match yaml schema!"
+    fi
+  fi
+  if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \
+    -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then
+    notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!"
+  fi
+  printenv | \
+    awk '/^(SALT|MCP|MAAS).*=/ { gsub(/=/,": "); print }' >> "${LOCAL_PDF}"
+  j2args=$(find "${scenario_dir}" -name '*.j2' -exec echo -j {} \;)
+  # shellcheck disable=SC2086
+  if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
+    -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then
+    notify_e '[ERROR] Could not convert j2 scenario definitions!'
+  fi
+}
+
+# Expand reclass and virsh network templates based on PDF + IDF + others
+function do_templates_cluster {
+  local image_dir=$1; shift
+  local target_lab=$1; shift
+  local target_pod=$1; shift
+  local git_repo_root=$1; shift
+  local extra_yaml=("$@")
+
+  RECLASS_CLUSTER_DIR=$(cd "${git_repo_root}/mcp/reclass/classes/cluster"; pwd)
+  LOCAL_PDF="${image_dir}/${target_pod}.yaml"
+
+  for _yaml in "${extra_yaml[@]}"; do
+    awk '/^---$/{f=1;next;}f' "${_yaml}" >> "${LOCAL_PDF}"
+  done
+  # shellcheck disable=SC2046
+  j2args=$(find "${RECLASS_CLUSTER_DIR}" "$(readlink -f virsh_net)" $(readlink -f ./*j2) \
+           -name '*.j2' -exec echo -j {} \;)
+  # shellcheck disable=SC2086
+  if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
+    -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then
+    notify_e '[ERROR] Could not convert PDF to network definitions!'
+  fi
+}
diff --git a/mcp/scripts/user-data.admin.sh.j2 b/mcp/scripts/user-data.admin.sh.j2
index b4c89b29c..d9b86c79c 100644
--- a/mcp/scripts/user-data.admin.sh.j2
+++ b/mcp/scripts/user-data.admin.sh.j2
@@ -8,7 +8,7 @@
 ##############################################################################
 rm /etc/salt/minion_id
 rm -f /etc/salt/pki/minion/minion_master.pub
-echo "id: $(hostname).{{ conf.CLUSTER_DOMAIN }}" > /etc/salt/minion
+echo "id: $(hostname).{{ conf.cluster.domain }}" > /etc/salt/minion
 {#- should be in sync with 'opnfv_infra_config_pxe_address' in 'pharos/config/installers/fuel/pod_config.yml.j2 #}
 echo "master: {{ conf.idf.net_config.admin.network | ipaddr_index(2) }}" >> /etc/salt/minion
 service salt-minion restart
diff --git a/mcp/scripts/user-data.mcp.sh.j2 b/mcp/scripts/user-data.mcp.sh.j2
index 4bd0d1d09..bd80961e6 100644
--- a/mcp/scripts/user-data.mcp.sh.j2
+++ b/mcp/scripts/user-data.mcp.sh.j2
@@ -1,6 +1,6 @@
 #!/bin/bash
 ##############################################################################
-# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
 # are made available under the terms of the Apache License, Version 2.0
 # which accompanies this distribution, and is available at
@@ -8,6 +8,6 @@
 ##############################################################################
 rm /etc/salt/minion_id
 rm -f /etc/salt/pki/minion/minion_master.pub
-echo "id: $(hostname).{{ conf.CLUSTER_DOMAIN }}" > /etc/salt/minion
+echo "id: $(hostname).{{ conf.cluster.domain }}" > /etc/salt/minion
 echo "master: {{ conf.SALT_MASTER }}" >> /etc/salt/minion
 service salt-minion restart
diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2
index 9479f2356..ad3c0fe1e 100644
--- a/mcp/scripts/xdf_data.sh.j2
+++ b/mcp/scripts/xdf_data.sh.j2
@@ -1,4 +1,5 @@
 #!/bin/bash -e
+# shellcheck disable=SC2034
 ##############################################################################
 # Copyright (c) 2018 Mirantis Inc., Enea AB and others.
 # All rights reserved. This program and the accompanying materials
@@ -10,11 +11,61 @@
 # Data derived from XDF (PDF/IDF/SDF/etc), used as input in deploy.sh
 #
 
-# Determine bridge names based on IDF, where all bridges are now mandatory
+{%- set arch = conf[conf.MCP_JUMP_ARCH] -%}
+
+{%- macro bash_arr(_l) -%}
+  ({%- for n in _l -%}'{{ n }}' {% endfor -%})
+{%- endmacro -%}
+
+{#- Pack list as `sep`-separated string, replacing spaces with '|' -#}
+{%- macro pack(x = [], sep = ',') -%}
+  {{ x | join(sep) | replace(' ', '|') }}
+{%- endmacro -%}
+
+{#- Pack all vnode data as string -#}
+{%- macro serialize_vnodes() -%}
+  {%- set V = conf.virtual -%}
+  {%- set arr = [] -%}
+  {%- for n in V.nodes -%}
+    {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%}
+    {%- do arr.append(pack([n, V[n].ram or arch.default.ram,
+                               V[n].vcpus or arch.default.vcpus])) -%}
+  {%- endfor -%}
+  '{{ pack(arr, '|') }}'
+{%- endmacro -%}
+
+{#- Pack apt_pkg data as string -#}
+{%- macro serialize_apt_pkg() -%}
+  {%- set arr = [] -%}
+  {%- set sections = [arch.common] -%}
+  {%- if 'virtual_control_plane' in conf.cluster.states -%}
+    {%- do sections.append(arch.control) -%}
+  {%- endif -%}
+  {%- for c in sections -%}
+    {%- do arr.append(pack([pack(c.apt['keys']), pack(c.apt.repos),
+                            pack(c.pkg.install), pack(c.pkg.remove)], '^')) -%}
+  {%- endfor -%}
+  '{{ pack(arr, '^') }}'
+{%- endmacro -%}
+
 {%- set bridges = conf.idf.fuel.jumphost.bridges %}
+# Determine bridge names based on IDF, where all bridges are now mandatory
 OPNFV_BRIDGES=(
   '{{ bridges.admin or "pxebr" }}'
   '{{ bridges.mgmt or "mgmt" }}'
   '{{ bridges.private or "internal" }}'
   '{{ bridges.public or "public" }}'
 )
+
+export CLUSTER_DOMAIN={{ conf.cluster.domain }}
+cluster_states={{ bash_arr(conf.cluster.states) }}
+virtual_nodes={{ bash_arr(conf.virtual.nodes) }}
+base_image={{ arch.base_image }}
+
+# Serialize vnode data as '<name0>,<ram0>,<vcpu0>|<name1>,<ram1>,<vcpu1>[...]'
+virtual_nodes_data={{ serialize_vnodes() }}
+
+# Serialize repos, packages to (pre-)install/remove for:
+# - foundation node VM base image (virtual: all VMs, baremetal: cfg01|mas01)
+# - virtualized control plane VM base image (only when VCP is used)
+virtual_repos_pkgs={{ serialize_apt_pkg() }}