[scenarios] Differentiate virtual node roles

Split scenario yaml definitions for virtual.nodes based on node
role ('infra', 'control' or 'compute'), to be leveraged later to
contruct node lists based on said role.

This moves the responsability of filtering node names in scenario
files (based on 'virtual' or 'baremetal' type) to xdf_data.sh.j2,
simplifying scenario templates.

By keeping all nodes (both virtual and baremetal) in scenario files,
we can later determine the role (and implicitly the hostname) for a
MaaS-managed node based on its index in the virtual.nodes.control
structure.

JIRA: FUEL-382

Change-Id: I1f83a307631f4166ee1c57ef598c44876b962f97
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

11 files changed, 121 insertions, 88 deletions

diff --git a/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2
index 26938262e..7eb604e65 100644
--- a/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2
+++ b/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2
@@ -21,25 +21,22 @@ cluster:
     - networks
 virtual:
   nodes:
-    - cfg01
+    infra:
+      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
-    - mas01
-{%- endif %}
-{#- Most likely, controllers will always have the same type and number (3) #}
-{%- if conf.nodes[nm.ctl01.idx].node.type == 'virtual' %}
-    - kvm01
-    - kvm02
-    - kvm03
-{%- endif %}
-{#- Later, we might have mixed computes here, for hybrid multi-arch testing #}
-{%- if conf.nodes[nm.cmp001.idx].node.type == 'virtual' %}
-    - cmp001
-    - cmp002
+      - mas01
 {%- endif %}
+    control:
+      - kvm01
+      - kvm02
+      - kvm03
+    compute:
+      - cmp001
+      - cmp002
+  # Below values are only used for each node if said node is virtual
   cfg01:
     vcpus: 4
     ram: 6144
-  # Below values are only used when nodes are defined in virtual.nodes above
   mas01:
     vcpus: 4
     ram: 6144
diff --git a/mcp/config/scenario/os-nosdn-nofeature-noha.yaml b/mcp/config/scenario/os-nosdn-nofeature-noha.yaml
index 5c5e77ceb..8a3a33db0 100644
--- a/mcp/config/scenario/os-nosdn-nofeature-noha.yaml
+++ b/mcp/config/scenario/os-nosdn-nofeature-noha.yaml
@@ -14,11 +14,14 @@ cluster:
     - networks
 virtual:
   nodes:
-    - cfg01
-    - ctl01
-    - cmp001
-    - cmp002
-    - gtw01
+    infra:
+      - cfg01
+    control:
+      - ctl01
+      - gtw01
+    compute:
+      - cmp001
+      - cmp002
   ctl01:
     vcpus: 4
     ram: 14336
diff --git a/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 b/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2
index f9ab7c3fb..da9938ce0 100644
--- a/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2
+++ b/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2
@@ -22,25 +22,22 @@ cluster:
     - networks
 virtual:
   nodes:
-    - cfg01
+    infra:
+      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
-    - mas01
-{%- endif %}
-{#- Most likely, controllers will always have the same type and number (3) #}
-{%- if conf.nodes[nm.ctl01.idx].node.type == 'virtual' %}
-    - kvm01
-    - kvm02
-    - kvm03
-{%- endif %}
-{#- Later, we might have mixed computes here, for hybrid multi-arch testing #}
-{%- if conf.nodes[nm.cmp001.idx].node.type == 'virtual' %}
-    - cmp001
-    - cmp002
+      - mas01
 {%- endif %}
+    control:
+      - kvm01
+      - kvm02
+      - kvm03
+    compute:
+      - cmp001
+      - cmp002
+  # Below values are only used for each node if said node is virtual
   cfg01:
     vcpus: 4
     ram: 6144
-  # Below values are only used when nodes are defined in virtual.nodes above
   mas01:
     vcpus: 4
     ram: 6144
diff --git a/mcp/config/scenario/os-nosdn-ovs-noha.yaml b/mcp/config/scenario/os-nosdn-ovs-noha.yaml
index 19e7946d1..795b73064 100644
--- a/mcp/config/scenario/os-nosdn-ovs-noha.yaml
+++ b/mcp/config/scenario/os-nosdn-ovs-noha.yaml
@@ -15,11 +15,14 @@ cluster:
     - networks
 virtual:
   nodes:
-    - cfg01
-    - ctl01
-    - cmp001
-    - cmp002
-    - gtw01
+    infra:
+      - cfg01
+    control:
+      - ctl01
+      - gtw01
+    compute:
+      - cmp001
+      - cmp002
   ctl01:
     vcpus: 4
     ram: 14336
diff --git a/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2
index c48f18c51..fbedbc6b9 100644
--- a/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2
+++ b/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2
@@ -22,25 +22,22 @@ cluster:
     - networks
 virtual:
   nodes:
-    - cfg01
+    infra:
+      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
-    - mas01
-{%- endif %}
-{#- Most likely, controllers will always have the same type and number (3) #}
-{%- if conf.nodes[nm.ctl01.idx].node.type == 'virtual' %}
-    - kvm01
-    - kvm02
-    - kvm03
-{%- endif %}
-{#- Later, we might have mixed computes here, for hybrid multi-arch testing #}
-{%- if conf.nodes[nm.cmp001.idx].node.type == 'virtual' %}
-    - cmp001
-    - cmp002
+      - mas01
 {%- endif %}
+    control:
+      - kvm01
+      - kvm02
+      - kvm03
+    compute:
+      - cmp001
+      - cmp002
+  # Below values are only used for each node if said node is virtual
   cfg01:
     vcpus: 4
     ram: 6144
-  # Below values are only used when nodes are defined in virtual.nodes above
   mas01:
     vcpus: 4
     ram: 6144
diff --git a/mcp/config/scenario/os-odl-nofeature-noha.yaml b/mcp/config/scenario/os-odl-nofeature-noha.yaml
index ec04ee87a..d0a490ef9 100644
--- a/mcp/config/scenario/os-odl-nofeature-noha.yaml
+++ b/mcp/config/scenario/os-odl-nofeature-noha.yaml
@@ -15,12 +15,15 @@ cluster:
     - networks
 virtual:
   nodes:
-    - cfg01
-    - ctl01
-    - cmp001
-    - cmp002
-    - gtw01
-    - odl01
+    infra:
+      - cfg01
+    control:
+      - ctl01
+      - gtw01
+      - odl01
+    compute:
+      - cmp001
+      - cmp002
   ctl01:
     vcpus: 4
     ram: 14336
diff --git a/mcp/config/scenario/os-odl-ovs-noha.yaml b/mcp/config/scenario/os-odl-ovs-noha.yaml
index 083e7e646..29e6842ac 100644
--- a/mcp/config/scenario/os-odl-ovs-noha.yaml
+++ b/mcp/config/scenario/os-odl-ovs-noha.yaml
@@ -16,12 +16,15 @@ cluster:
     - networks
 virtual:
   nodes:
-    - cfg01
-    - ctl01
-    - cmp001
-    - cmp002
-    - gtw01
-    - odl01
+    infra:
+      - cfg01
+    control:
+      - ctl01
+      - gtw01
+      - odl01
+    compute:
+      - cmp001
+      - cmp002
   ctl01:
     vcpus: 4
     ram: 14336
diff --git a/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2
index 218b5dece..f81779b8a 100644
--- a/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2
+++ b/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2
@@ -21,25 +21,22 @@ cluster:
     - networks
 virtual:
   nodes:
-    - cfg01
+    infra:
+      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
-    - mas01
-{%- endif %}
-{#- Most likely, controllers will always have the same type and number (3) #}
-{%- if conf.nodes[nm.ctl01.idx].node.type == 'virtual' %}
-    - kvm01
-    - kvm02
-    - kvm03
-{%- endif %}
-{#- Later, we might have mixed computes here, for hybrid multi-arch testing #}
-{%- if conf.nodes[nm.cmp001.idx].node.type == 'virtual' %}
-    - cmp001
-    - cmp002
+      - mas01
 {%- endif %}
+    control:
+      - kvm01
+      - kvm02
+      - kvm03
+    compute:
+      - cmp001
+      - cmp002
+  # Below values are only used for each node if said node is virtual
   cfg01:
     vcpus: 4
     ram: 6144
-  # Below values are only used when nodes are defined in virtual.nodes above
   mas01:
     vcpus: 4
     ram: 6144
diff --git a/mcp/config/scenario/os-ovn-nofeature-noha.yaml b/mcp/config/scenario/os-ovn-nofeature-noha.yaml
index 7036340bf..fc4086670 100644
--- a/mcp/config/scenario/os-ovn-nofeature-noha.yaml
+++ b/mcp/config/scenario/os-ovn-nofeature-noha.yaml
@@ -13,10 +13,13 @@ cluster:
     - networks
 virtual:
   nodes:
-    - cfg01
-    - ctl01
-    - cmp001
-    - cmp002
+    infra:
+      - cfg01
+    control:
+      - ctl01
+    compute:
+      - cmp001
+      - cmp002
   ctl01:
     vcpus: 4
     ram: 14336
diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh
index 9c12bff64..4e4f36908 100644
--- a/mcp/scripts/lib.sh
+++ b/mcp/scripts/lib.sh
@@ -407,6 +407,7 @@ function create_vms {
 
   # create vms with specified options
   for serialized_vnode_data in "${vnodes[@]}"; do
+    if [ -z "${serialized_vnode_data}" ]; then continue; fi
     IFS=',' read -r -a vnode_data <<< "${serialized_vnode_data}"
 
     # prepare network args
diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2
index 8c9d5d969..bc0107761 100644
--- a/mcp/scripts/xdf_data.sh.j2
+++ b/mcp/scripts/xdf_data.sh.j2
@@ -11,7 +11,14 @@
 # Data derived from XDF (PDF/IDF/SDF/etc), used as input in deploy.sh
 #
 
+{%- import 'net_map.j2' as nm with context -%}
 {%- set arch = conf[conf.MCP_JUMP_ARCH] -%}
+{%- set V = conf.virtual -%}
+{%- set section_map = {
+  'infra': -1,
+  'control': nm.ctl01.idx,
+  'compute': nm.cmp001.idx
+} -%}
 
 {%- macro bash_arr(_l) -%}
   ({%- for n in _l -%}'{{ n }}' {% endfor -%})
@@ -24,16 +31,38 @@
 
 {#- Pack all vnode data as string -#}
 {%- macro serialize_vnodes() -%}
-  {%- set V = conf.virtual -%}
   {%- set arr = [] -%}
-  {%- for n in V.nodes -%}
-    {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%}
-    {%- do arr.append(pack([n, V[n].ram or arch.default.ram,
-                               V[n].vcpus or arch.default.vcpus])) -%}
+  {%- for section in section_map -%}
+    {%- for n in V.nodes[section] -%}
+      {%- if section_map[section] < 0 or conf.nodes[section_map[section] + loop.index0].node.type == 'virtual' -%}
+        {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%}
+        {%- do arr.append(pack([n, V[n].ram or arch.default.ram,
+                                   V[n].vcpus or arch.default.vcpus])) -%}
+      {%- endif -%}
+    {%- endfor -%}
   {%- endfor -%}
   '{{ pack(arr, '|') }}'
 {%- endmacro -%}
 
+{#- Return a bash array of node names or a Salt query, optionally filtered by type #}
+{%- macro filter_nodes(type, output_as_query = False, sections = section_map) -%}
+  {%- set arr = [] -%}
+  {%- for section in sections -%}
+    {%- for n in V.nodes[section] -%}
+      {%- if ( (section_map[section] < 0 and type == 'virtual') or
+               (section_map[section] >= 0 and
+                conf.nodes[section_map[section] + loop.index0].node.type in type) ) -%}
+        {%- do arr.append(n) -%}
+      {%- endif -%}
+    {%- endfor -%}
+  {%- endfor -%}
+  {%- if output_as_query -%}
+  '{{ arr | join('* or ') }}*'
+  {%- else -%}
+  {{ bash_arr(arr) }}
+  {%- endif -%}
+{%- endmacro -%}
+
 {#- Pack apt_pkg data as string -#}
 {%- macro serialize_apt_pkg() -%}
   {%- set arr = [] -%}
@@ -59,7 +88,7 @@ OPNFV_BRIDGES=(
 
 export CLUSTER_DOMAIN={{ conf.cluster.domain }}
 cluster_states={{ bash_arr(conf.cluster.states) }}
-virtual_nodes={{ bash_arr(conf.virtual.nodes) }}
+virtual_nodes={{ filter_nodes('virtual') }}
 base_image={{ arch.base_image }}
 
 # Serialize vnode data as '<name0>,<ram0>,<vcpu0>|<name1>,<ram1>,<vcpu1>[...]'