[docker] Switch to containerized Salt Master

* Refactor OPNFV salt-formulas mechanism to resemble upstream git
  structure:
  - git submodules: add new submodule for each formula we patch;
  - create salt-formula-x directories for OPNFV formulas;
  - move mcp/metadata/service contents to their each formula subdir;
  - use `make patches-import` for patches previously handled by
    patch.sh;
  - retire patch.sh
* states: add virtual_init:
  - mostly based on old salt.sh, which is now obsolete;
  - exclude salt-master service restart (it would kill the container);
* scenarios: cleanup (rm cfg01 virtual node def), adopt virtual_init;
* reclass: align our model with prebuilt container's Salt config:
  - drop linux:network pillar data (handled by Docker);
  - stop applying linux.system state on cfg01;
  - align salt user homedir;
  - drop salt-formula packages (preprovisioned);
* minor plumbing in deploy.sh and lib.sh;

JIRA: FUEL-383

Change-Id: I28708a9b399d3f19012212c71966ebda9d6fc0ac
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

10 files changed, 48 insertions, 21 deletions

diff --git a/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2
index 7eb604e65..55da8f74b 100644
--- a/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2
+++ b/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2
@@ -10,6 +10,7 @@
 cluster:
   domain: mcp-ovs-ha.local
   states:
+    - virtual_init
 {%- if nm.cluster.has_baremetal_nodes %}
     - maas
     - baremetal_init
@@ -22,7 +23,6 @@ cluster:
 virtual:
   nodes:
     infra:
-      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
       - mas01
 {%- endif %}
@@ -34,9 +34,6 @@ virtual:
       - cmp001
       - cmp002
   # Below values are only used for each node if said node is virtual
-  cfg01:
-    vcpus: 4
-    ram: 6144
   mas01:
     vcpus: 4
     ram: 6144
diff --git a/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 b/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2
index a0ad96b87..cd214a5cd 100644
--- a/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2
+++ b/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2
@@ -10,6 +10,7 @@
 cluster:
   domain: mcp-ovs-noha.local
   states:
+    - virtual_init
 {%- if nm.cluster.has_baremetal_nodes %}
     - maas
     - baremetal_init
@@ -20,7 +21,6 @@ cluster:
 virtual:
   nodes:
     infra:
-      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
       - mas01
 {%- endif %}
diff --git a/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 b/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2
index da9938ce0..0f7a7bc34 100644
--- a/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2
+++ b/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2
@@ -10,6 +10,7 @@
 cluster:
   domain: mcp-ovs-dpdk-ha.local
   states:
+    - virtual_init
 {%- if nm.cluster.has_baremetal_nodes %}
     - maas
     - baremetal_init
@@ -23,7 +24,6 @@ cluster:
 virtual:
   nodes:
     infra:
-      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
       - mas01
 {%- endif %}
@@ -35,9 +35,6 @@ virtual:
       - cmp001
       - cmp002
   # Below values are only used for each node if said node is virtual
-  cfg01:
-    vcpus: 4
-    ram: 6144
   mas01:
     vcpus: 4
     ram: 6144
diff --git a/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2 b/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2
index bee0e926f..bf37f93ea 100644
--- a/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2
+++ b/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2
@@ -10,6 +10,7 @@
 cluster:
   domain: mcp-ovs-dpdk-noha.local
   states:
+    - virtual_init
 {%- if nm.cluster.has_baremetal_nodes %}
     - maas
     - baremetal_init
@@ -21,7 +22,6 @@ cluster:
 virtual:
   nodes:
     infra:
-      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
       - mas01
 {%- endif %}
diff --git a/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2
index fbedbc6b9..b31dc136e 100644
--- a/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2
+++ b/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2
@@ -10,6 +10,7 @@
 cluster:
   domain: mcp-odl-ha.local
   states:
+    - virtual_init
 {%- if nm.cluster.has_baremetal_nodes %}
     - maas
     - baremetal_init
@@ -23,7 +24,6 @@ cluster:
 virtual:
   nodes:
     infra:
-      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
       - mas01
 {%- endif %}
@@ -35,9 +35,6 @@ virtual:
       - cmp001
       - cmp002
   # Below values are only used for each node if said node is virtual
-  cfg01:
-    vcpus: 4
-    ram: 6144
   mas01:
     vcpus: 4
     ram: 6144
diff --git a/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2 b/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2
index 7df28c2b3..2a10a3163 100644
--- a/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2
+++ b/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2
@@ -10,6 +10,7 @@
 cluster:
   domain: mcp-odl-noha.local
   states:
+    - virtual_init
 {%- if nm.cluster.has_baremetal_nodes %}
     - maas
     - baremetal_init
@@ -21,7 +22,6 @@ cluster:
 virtual:
   nodes:
     infra:
-      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
       - mas01
 {%- endif %}
diff --git a/mcp/config/scenario/os-odl-ovs-noha.yaml.j2 b/mcp/config/scenario/os-odl-ovs-noha.yaml.j2
index c81d35c51..62c120b26 100644
--- a/mcp/config/scenario/os-odl-ovs-noha.yaml.j2
+++ b/mcp/config/scenario/os-odl-ovs-noha.yaml.j2
@@ -10,6 +10,7 @@
 cluster:
   domain: mcp-odl-noha.local
   states:
+    - virtual_init
 {%- if nm.cluster.has_baremetal_nodes %}
     - maas
     - baremetal_init
@@ -22,7 +23,6 @@ cluster:
 virtual:
   nodes:
     infra:
-      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
       - mas01
 {%- endif %}
diff --git a/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2
index f81779b8a..021548f63 100644
--- a/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2
+++ b/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2
@@ -10,6 +10,7 @@
 cluster:
   domain: mcp-ovn-ha.local
   states:
+    - virtual_init
 {%- if nm.cluster.has_baremetal_nodes %}
     - maas
     - baremetal_init
@@ -22,7 +23,6 @@ cluster:
 virtual:
   nodes:
     infra:
-      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
       - mas01
 {%- endif %}
@@ -34,9 +34,6 @@ virtual:
       - cmp001
       - cmp002
   # Below values are only used for each node if said node is virtual
-  cfg01:
-    vcpus: 4
-    ram: 6144
   mas01:
     vcpus: 4
     ram: 6144
diff --git a/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2 b/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2
index fee47d7eb..635106355 100644
--- a/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2
+++ b/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2
@@ -10,6 +10,7 @@
 cluster:
   domain: mcp-ovn-noha.local
   states:
+    - virtual_init
 {%- if nm.cluster.has_baremetal_nodes %}
     - maas
     - baremetal_init
@@ -19,7 +20,6 @@ cluster:
 virtual:
   nodes:
     infra:
-      - cfg01
 {%- if nm.cluster.has_baremetal_nodes %}
       - mas01
 {%- endif %}
diff --git a/mcp/config/states/virtual_init b/mcp/config/states/virtual_init
new file mode 100755
index 000000000..147e6b05e
--- /dev/null
+++ b/mcp/config/states/virtual_init
@@ -0,0 +1,39 @@
+#!/bin/bash -e
+# shellcheck disable=SC1090
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
+
+# shellcheck disable=SC1090
+source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib.sh"
+source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/xdf_data.sh"
+
+CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
+# shellcheck disable=SC2154,SC2086,SC2116
+LOCAL_VIRT_NODES=$(echo ${virtual_nodes[*]}) # unquoted to filter space
+NODE_MASK="${LOCAL_VIRT_NODES// /|}"
+
+# wait_for 3.0 "salt-call state.apply salt exclude='[{id: salt_master_service}]'"
+wait_for 5.0 "salt-call state.sls reclass,linux.network"
+wait_for 3.0 "salt -C '*' saltutil.refresh_pillar"
+
+# NOTE: domain name changes are not yet supported without a clean redeploy
+
+# Init specific to VMs on FN (all for virtual, mas for baremetal)
+wait_for 3.0 "(for n in ${LOCAL_VIRT_NODES}; do salt -C \${n}.* test.ping || exit; done)"
+
+wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' state.sls linux.system,linux.storage"
+wait_for 2.0 "salt -C 'E@^(${NODE_MASK}).*' state.sls linux.network"
+salt -C "E@^(${NODE_MASK}).*" system.reboot
+wait_for 90.0 "salt -C 'E@^(${NODE_MASK}).*' test.ping"
+wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' pkg.upgrade refresh=False dist_upgrade=True"
+wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' saltutil.sync_all"
+wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' state.apply salt"
+
+wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' state.sls ntp"