[FN VM] Reboot VMs on jump, wait for all online

- apply `linux` state on cfg01 first, so PXE/admin IP is added and
  FN VM minions are available;
- add barrier and wait for all FN VMs to register with cfg01;
- use batch-mode execution while applying `linux.network` on FN VMs;
- retry all states executed via <salt.sh> on FN VMs;

JIRA: FUEL-310

Change-Id: I72e1c565370072500df1d486fe76e6315f583c75
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

5 files changed, 35 insertions, 14 deletions

diff --git a/mcp/config/states/dpdk b/mcp/config/states/dpdk
index 5ae2aac48..281b78fed 100755
--- a/mcp/config/states/dpdk
+++ b/mcp/config/states/dpdk
@@ -12,8 +12,5 @@ CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
 # shellcheck disable=SC1090
 source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib.sh"
 
-salt -I 'nova:compute' system.reboot
-wait_for 90.0 "salt -I 'nova:compute' test.ping"
-
 salt -I 'nova:compute' alternatives.set ovs-vswitchd /usr/lib/openvswitch-switch-dpdk/ovs-vswitchd-dpdk
 salt -I 'nova:compute' service.restart openvswitch-switch
diff --git a/mcp/config/states/maas b/mcp/config/states/maas
index 8f7a86611..dcbdf98cc 100755
--- a/mcp/config/states/maas
+++ b/mcp/config/states/maas
@@ -77,7 +77,7 @@ salt -C 'mas01*' state.apply maas.machines
 # cleanup outdated salt keys
 sleep 30
 salt-key --out yaml | awk '!/^(minions|- cfg01|- mas01)/ {print $2}' | \
-  xargs -I{} salt-key -yd {}
+  xargs --no-run-if-empty -I{} salt-key -yd {}
 
 # MaaS node deployment
 wait_for 10 maas_fixup
diff --git a/mcp/reclass/classes/cluster/virtual-mcp-pike-common-noha/infra/config.yml b/mcp/reclass/classes/cluster/virtual-mcp-pike-common-noha/infra/config.yml
index 7e3a49491..908ba7719 100644
--- a/mcp/reclass/classes/cluster/virtual-mcp-pike-common-noha/infra/config.yml
+++ b/mcp/reclass/classes/cluster/virtual-mcp-pike-common-noha/infra/config.yml
@@ -33,6 +33,11 @@ parameters:
   linux:
     network:
       interface:
+        dhcp:
+          enabled: true
+          type: eth
+          proto: dhcp
+          name: ${_param:opnfv_fn_vm_primary_interface}
         single_int:
           enabled: true
           name: ${_param:opnfv_fn_vm_secondary_interface}
@@ -40,6 +45,13 @@ parameters:
           proto: static
           address: ${_param:single_address}
           netmask: 255.255.255.0
+        pxe:
+          enabled: true
+          type: eth
+          proto: static
+          name: ${_param:opnfv_fn_vm_tertiary_interface}
+          address: ${_param:opnfv_infra_config_pxe_address}
+          netmask: 255.255.255.0
   salt:
     master:
       file_recv: true
diff --git a/mcp/reclass/classes/cluster/virtual-mcp-pike-common-noha/openstack_control.yml b/mcp/reclass/classes/cluster/virtual-mcp-pike-common-noha/openstack_control.yml
index 98256745f..9defb0070 100644
--- a/mcp/reclass/classes/cluster/virtual-mcp-pike-common-noha/openstack_control.yml
+++ b/mcp/reclass/classes/cluster/virtual-mcp-pike-common-noha/openstack_control.yml
@@ -54,6 +54,11 @@ parameters:
           version: latest
     network:
       interface:
+        dhcp_int:
+          enabled: true
+          name: ${_param:opnfv_fn_vm_primary_interface}
+          proto: dhcp
+          type: eth
         single_int:
           enabled: true
           name: ${_param:opnfv_fn_vm_secondary_interface}
diff --git a/mcp/scripts/salt.sh b/mcp/scripts/salt.sh
index 03218e27f..87f7a164e 100755
--- a/mcp/scripts/salt.sh
+++ b/mcp/scripts/salt.sh
@@ -16,15 +16,14 @@ F_GIT_DIR=$(cd "${F_GIT_ROOT}/mcp" && git rev-parse --git-dir)
 F_GIT_SUBD=${F_GIT_ROOT#${F_GIT_DIR%%/.git*}}
 OPNFV_TMP_DIR="/home/${SALT_MASTER_USER}/opnfv"
 OPNFV_GIT_DIR="/root/opnfv"
-OPNFV_FUEL_DIR="/root/fuel"
+OPNFV_FUEL_DIR="/root/fuel" # Should be in sync with patch.sh, scripts patches
 OPNFV_RDIR="reclass/classes/cluster/all-mcp-arch-common"
 OPNFV_VCP_IMG="mcp/scripts/base_image_opnfv_fuel_vcp.img"
 OPNFV_VCP_DIR="/srv/salt/env/prd/salt/files/control/images"
 LOCAL_GIT_DIR="${F_GIT_ROOT%${F_GIT_SUBD}}"
-LOCAL_PDF_RECLASS=$1
-NODE_MASK='*'
-
-[[ "${CLUSTER_DOMAIN}" =~ virtual ]] || NODE_MASK='mas01*'
+LOCAL_PDF_RECLASS=$1; shift
+LOCAL_VIRT_NODES=$(echo ${*//cfg01/}) # unquoted to filter space
+NODE_MASK="${LOCAL_VIRT_NODES// /|}"
 
 # push to cfg01 current git repo first (including submodules), at ~ubuntu/opnfv
 # later we move it to ~root/opnfv (and ln as ~root/fuel); delete the temp clone
@@ -96,14 +95,22 @@ ssh ${SSH_OPTS} "${SSH_SALT}" bash -s -e << SALT_INSTALL_END
   fi
 
   # Init specific to VMs on FN (all for virtual, cfg|mas for baremetal)
-  salt -C "${NODE_MASK} or cfg01*" saltutil.sync_all
-  wait_for 3.0 'salt -C "${NODE_MASK} or cfg01*" state.apply salt'
   wait_for 3.0 'salt -C "cfg01*" state.apply linux'
+  if [[ "${LOCAL_VIRT_NODES}" =~ mas ]]; then
+    wait_for 3.0 'salt -C "mas*" test.ping'
+  else
+    wait_for 3.0 'for n in ${LOCAL_VIRT_NODES}; do salt -C \${n}.* test.ping; done'
+  fi
+  wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" saltutil.sync_all'
+  wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.apply salt'
 
-  salt -C "${NODE_MASK} and not cfg01*" state.sls linux || true
-  salt -C "${NODE_MASK} and not cfg01*" pkg.upgrade refresh=False
+  wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" state.sls linux.system,linux.storage'
+  salt -C "E@^(${NODE_MASK}).*" state.sls linux.network -b 1 || true
+  salt -C "E@^(${NODE_MASK}).*" system.reboot
+  wait_for 90.0 'salt -C "E@^(${NODE_MASK}).*" test.ping'
+  wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" pkg.upgrade refresh=False'
 
-  salt -C "${NODE_MASK} or cfg01*" state.sls ntp
+  wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.sls ntp'
 
   if [ -f "${OPNFV_FUEL_DIR}/${OPNFV_VCP_IMG}" ]; then
     mkdir -p "${OPNFV_VCP_DIR}"