[docker] Switch to containerized Salt Master

* Refactor OPNFV salt-formulas mechanism to resemble upstream git
  structure:
  - git submodules: add new submodule for each formula we patch;
  - create salt-formula-x directories for OPNFV formulas;
  - move mcp/metadata/service contents to their each formula subdir;
  - use `make patches-import` for patches previously handled by
    patch.sh;
  - retire patch.sh
* states: add virtual_init:
  - mostly based on old salt.sh, which is now obsolete;
  - exclude salt-master service restart (it would kill the container);
* scenarios: cleanup (rm cfg01 virtual node def), adopt virtual_init;
* reclass: align our model with prebuilt container's Salt config:
  - drop linux:network pillar data (handled by Docker);
  - stop applying linux.system state on cfg01;
  - align salt user homedir;
  - drop salt-formula packages (preprovisioned);
* minor plumbing in deploy.sh and lib.sh;

JIRA: FUEL-383

Change-Id: I28708a9b399d3f19012212c71966ebda9d6fc0ac
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

8 files changed, 161 insertions, 42 deletions

diff --git a/mcp/patches/patch.sh b/mcp/patches/patch.sh
deleted file mode 100755
index bb48dcd07..000000000
--- a/mcp/patches/patch.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash -e
-##############################################################################
-# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x
-
-if [ -r "$1" ]; then
-  while IFS=': ' read -r p_dest p_file; do
-    if ! patch --dry-run -Rd "${p_dest}" -r - -s -p1 < \
-      "/root/fuel/mcp/patches/${p_file}" > /dev/null; then
-        patch -d "${p_dest}" -p1 < "/root/fuel/mcp/patches/${p_file}"
-    fi
-  done < <(grep -vE '^#' "${1}" | grep -E "^.*${2}.*: ")
-fi
diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list
deleted file mode 100644
index baa15d79d..000000000
--- a/mcp/patches/patches.list
+++ /dev/null
@@ -1,12 +0,0 @@
-##############################################################################
-# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-/usr/share/salt-formulas/env: 0002-maas-region-skip-credentials-update.patch
-/usr/share/salt-formulas/env: 0008-Handle-extra-environment-variables.patch
-/usr/share/salt-formulas/env: 0010-maas-region-allow-timeout-override.patch
-/usr/share/salt-formulas/env: 0011-system.repo-Debian-Add-keyserver-proxy-support.patch
-/usr/share/salt-formulas/env: 0015-Set-ovs-bridges-as-L3-interfaces.patch
diff --git a/mcp/patches/0008-Handle-extra-environment-variables.patch b/mcp/patches/salt-formula-keystone/0001-Handle-extra-environment-variables.patch
index 8df3227f2..2108587b1 100644
--- a/mcp/patches/0008-Handle-extra-environment-variables.patch
+++ b/mcp/patches/salt-formula-keystone/0001-Handle-extra-environment-variables.patch
@@ -11,6 +11,9 @@ Date: Mon, 12 Mar 2018 17:43:09 +0400
 Subject: [PATCH] Handle extra environment variables
 
 Change-Id: Ieae46ac65041630759c82238a8a5ce0535c454b2
+---
+ keystone/files/keystonercv3 | 3 +++
+ 1 file changed, 3 insertions(+)
 
 diff --git a/keystone/files/keystonercv3 b/keystone/files/keystonercv3
 index 1b7f378..984c8a2 100644
diff --git a/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch
index dec29e6a7..4ad51e3e0 100644
--- a/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch
+++ b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch
@@ -37,7 +37,7 @@ Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
  1 file changed, 47 insertions(+), 8 deletions(-)
 
 diff --git a/linux/system/repo.sls b/linux/system/repo.sls
-index 9664129..d9f567e 100644
+index 0c0b026..82661f0 100644
 --- a/linux/system/repo.sls
 +++ b/linux/system/repo.sls
 @@ -7,18 +7,27 @@ linux_repo_prereq_pkgs:
diff --git a/mcp/patches/0015-Set-ovs-bridges-as-L3-interfaces.patch b/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch
index a7b366b94..ee82474b1 100644
--- a/mcp/patches/0015-Set-ovs-bridges-as-L3-interfaces.patch
+++ b/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch
@@ -11,10 +11,16 @@ Date: Wed, 28 Feb 2018 17:54:28 +0400
 Subject: [PATCH] Set ovs bridges as L3 interfaces
 
 Change-Id: I1e83129cc184cf481bea21d7aa452bf60d9e0499
+---
+ linux/files/ovs_bridge      | 19 +++++++++++++++++++
+ linux/files/ovs_port        |  7 ++++++-
+ linux/network/interface.sls | 28 ++++++++++++++++++++++++++++
+ 3 files changed, 53 insertions(+), 1 deletion(-)
+ create mode 100644 linux/files/ovs_bridge
 
 diff --git a/linux/files/ovs_bridge b/linux/files/ovs_bridge
 new file mode 100644
-index 0000000..c609e45
+index 0000000..073c91c
 --- /dev/null
 +++ b/linux/files/ovs_bridge
 @@ -0,0 +1,19 @@
@@ -38,14 +44,14 @@ index 0000000..c609e45
 +  dns-nameservers {{ bridge.name_servers | join(' ') }}
 +  {%- endif %}
 diff --git a/linux/files/ovs_port b/linux/files/ovs_port
-index 222ca8e..efb0307 100644
+index 222ca8e..03072cd 100644
 --- a/linux/files/ovs_port
 +++ b/linux/files/ovs_port
 @@ -1,6 +1,11 @@
+-auto {{ port_name }}
 +# With systemd, adding OVS bridges as 'auto' can cause race conditions
 +# https://github.com/openvswitch/ovs/blob/master/debian/openvswitch-switch.README.Debian
 +# auto {{ port_name }}
--auto {{ port_name }}
  allow-{{ port.bridge }} {{ port_name }}
  iface {{ port_name }} inet {{ port.get('proto', 'manual') }}
 +{%- if '.' in port_name %}
@@ -55,7 +61,7 @@ index 222ca8e..efb0307 100644
  mtu {{ port.get('mtu', '1500') }}
  ovs_bridge {{ port.bridge }}
 diff --git a/linux/network/interface.sls b/linux/network/interface.sls
-index 180f912..dcb295b 100644
+index c2d2a23..01131fa 100644
 --- a/linux/network/interface.sls
 +++ b/linux/network/interface.sls
 @@ -91,6 +91,34 @@ add_int_{{ int_name }}_to_ovs_dpdk_bridge_{{ interface_name }}:
diff --git a/mcp/patches/0002-maas-region-skip-credentials-update.patch b/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch
index 1d226e4cd..718f25b4e 100644
--- a/mcp/patches/0002-maas-region-skip-credentials-update.patch
+++ b/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch
@@ -17,12 +17,14 @@ updating credentials.
 
 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
 ---
+ maas/region.sls | 7 +++----
+ 1 file changed, 3 insertions(+), 4 deletions(-)
 
 diff --git a/maas/region.sls b/maas/region.sls
-index d3227ca..8a2243d 100644
+index 684fda4..5844f22 100644
 --- a/maas/region.sls
 +++ b/maas/region.sls
-@@ -6,10 +6,9 @@
+@@ -6,10 +6,9 @@ maas_region_packages:
      - names: {{ region.pkgs }}
 
  /etc/maas/regiond.conf:
diff --git a/mcp/patches/0010-maas-region-allow-timeout-override.patch b/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch
index c6f9e3a52..aef087d92 100644
--- a/mcp/patches/0010-maas-region-allow-timeout-override.patch
+++ b/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch
@@ -35,18 +35,20 @@ node's PXE physical interface, but that overcomplicates things.
 blocksize to be 1008. We can force it to be 1464 and gain some performance due
 to MTU beeing 1500 (i.e. allow bigger packets).
 
-
 JIRA: FUEL-316
 
 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
 ---
+ maas/region.sls | 40 ++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 40 insertions(+)
 
 diff --git a/maas/region.sls b/maas/region.sls
+index 5844f22..d844bb4 100644
 --- a/maas/region.sls
 +++ b/maas/region.sls
-@@ -19,6 +19,46 @@
-   - require:
-     - pkg: maas_region_packages
+@@ -38,6 +38,46 @@ restore_maas_database_{{ region.database.name }}:
+
+ {%- endif %}
 
 +maas_timeout_commissioning:
 +  file.replace:
diff --git a/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch b/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch
new file mode 100644
index 000000000..9f13c4616
--- /dev/null
+++ b/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch
@@ -0,0 +1,137 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sun, 19 Aug 2018 05:38:27 +0200
+Subject: [PATCH] Add machines.delete & co, pxe_nat sls
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ maas/machines/delete.sls                  | 20 ++++++++++++
+ maas/machines/mark_broken_fixed.sls       | 20 ++++++++++++
+ maas/machines/override_failed_testing.sls | 20 ++++++++++++
+ maas/pxe_nat.sls                          | 37 +++++++++++++++++++++++
+ 4 files changed, 97 insertions(+)
+ create mode 100644 maas/machines/delete.sls
+ create mode 100644 maas/machines/mark_broken_fixed.sls
+ create mode 100644 maas/machines/override_failed_testing.sls
+ create mode 100644 maas/pxe_nat.sls
+
+diff --git a/maas/machines/delete.sls b/maas/machines/delete.sls
+new file mode 100644
+index 0000000..2903f92
+--- /dev/null
++++ b/maas/machines/delete.sls
+@@ -0,0 +1,20 @@
++##############################################################################
++# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
++# All rights reserved. This program and the accompanying materials
++# are made available under the terms of the Apache License, Version 2.0
++# which accompanies this distribution, and is available at
++# http://www.apache.org/licenses/LICENSE-2.0
++##############################################################################
++{%- from "maas/map.jinja" import region with context %}
++
++maas_login_admin:
++  cmd.run:
++  - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
++  - unless: 'test -e /var/lib/maas/.maas_credentials'
++
++# TODO: implement delete_machine via _modules/maas.py
++delete_machine:
++  cmd.run:
++  - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine delete {{ pillar['system_id'] }}"
++  - require:
++    - cmd: maas_login_admin
+diff --git a/maas/machines/mark_broken_fixed.sls b/maas/machines/mark_broken_fixed.sls
+new file mode 100644
+index 0000000..46691bb
+--- /dev/null
++++ b/maas/machines/mark_broken_fixed.sls
+@@ -0,0 +1,20 @@
++##############################################################################
++# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
++# All rights reserved. This program and the accompanying materials
++# are made available under the terms of the Apache License, Version 2.0
++# which accompanies this distribution, and is available at
++# http://www.apache.org/licenses/LICENSE-2.0
++##############################################################################
++{%- from "maas/map.jinja" import region with context %}
++
++maas_login_admin:
++  cmd.run:
++  - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
++  - unless: 'test -e /var/lib/maas/.maas_credentials'
++
++# TODO: implement mark_broken_fixed_machine via _modules/maas.py
++mark_broken_fixed_machine:
++  cmd.run:
++  - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine mark-broken {{ pillar['system_id'] }} && sleep 10 && maas opnfv machine mark-fixed {{ pillar['system_id'] }} && maas opnfv machine test {{ pillar['system_id'] }} testing_scripts=fio"
++  - require:
++    - cmd: maas_login_admin
+diff --git a/maas/machines/override_failed_testing.sls b/maas/machines/override_failed_testing.sls
+new file mode 100644
+index 0000000..e7fe1d2
+--- /dev/null
++++ b/maas/machines/override_failed_testing.sls
+@@ -0,0 +1,20 @@
++##############################################################################
++# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
++# All rights reserved. This program and the accompanying materials
++# are made available under the terms of the Apache License, Version 2.0
++# which accompanies this distribution, and is available at
++# http://www.apache.org/licenses/LICENSE-2.0
++##############################################################################
++{%- from "maas/map.jinja" import region with context %}
++
++maas_login_admin:
++  cmd.run:
++  - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
++  - unless: 'test -e /var/lib/maas/.maas_credentials'
++
++# TODO: implement override_failed_testing via _modules/maas.py
++mark_broken_fixed_machine:
++  cmd.run:
++  - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine override-failed-testing {{ pillar['system_id'] }}"
++  - require:
++    - cmd: maas_login_admin
+diff --git a/maas/pxe_nat.sls b/maas/pxe_nat.sls
+new file mode 100644
+index 0000000..8a03c4f
+--- /dev/null
++++ b/maas/pxe_nat.sls
+@@ -0,0 +1,37 @@
++##############################################################################
++# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
++# All rights reserved. This program and the accompanying materials
++# are made available under the terms of the Apache License, Version 2.0
++# which accompanies this distribution, and is available at
++# http://www.apache.org/licenses/LICENSE-2.0
++##############################################################################
++net.ipv4.ip_forward:
++  sysctl.present:
++    - value: 1
++
++iptables_pxe_nat:
++  iptables.append:
++    - table: nat
++    - chain: POSTROUTING
++    - jump: MASQUERADE
++    - destination: 0/0
++    - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }}
++    - save: True
++
++iptables_pxe_source:
++  iptables.append:
++    - table: filter
++    - chain: INPUT
++    - jump: ACCEPT
++    - destination: 0/0
++    - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }}
++    - save: True
++
++iptables_pxe_destination:
++  iptables.append:
++    - table: filter
++    - chain: INPUT
++    - jump: ACCEPT
++    - destination: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }}
++    - source: 0/0
++    - save: True