[docker] Switch to containerized Salt Master

* Refactor OPNFV salt-formulas mechanism to resemble upstream git
  structure:
  - git submodules: add new submodule for each formula we patch;
  - create salt-formula-x directories for OPNFV formulas;
  - move mcp/metadata/service contents to their each formula subdir;
  - use `make patches-import` for patches previously handled by
    patch.sh;
  - retire patch.sh
* states: add virtual_init:
  - mostly based on old salt.sh, which is now obsolete;
  - exclude salt-master service restart (it would kill the container);
* scenarios: cleanup (rm cfg01 virtual node def), adopt virtual_init;
* reclass: align our model with prebuilt container's Salt config:
  - drop linux:network pillar data (handled by Docker);
  - stop applying linux.system state on cfg01;
  - align salt user homedir;
  - drop salt-formula packages (preprovisioned);
* minor plumbing in deploy.sh and lib.sh;

JIRA: FUEL-383

Change-Id: I28708a9b399d3f19012212c71966ebda9d6fc0ac
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

3 files changed, 272 insertions, 0 deletions

diff --git a/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch b/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch
new file mode 100644
index 000000000..718f25b4e
--- /dev/null
+++ b/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch
@@ -0,0 +1,40 @@
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+: Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+:
+: All rights reserved. This program and the accompanying materials
+: are made available under the terms of the Apache License, Version 2.0
+: which accompanies this distribution, and is available at
+: http://www.apache.org/licenses/LICENSE-2.0
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sat, 5 Aug 2017 02:03:01 +0200
+Subject: [PATCH] maas: region: skip credentials update
+
+Password update for maas psql database breaks ulterior acesses
+to maas-region syncdb.
+For now, limit regiond.conf changes to maas_url, and skip
+updating credentials.
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ maas/region.sls | 7 +++----
+ 1 file changed, 3 insertions(+), 4 deletions(-)
+
+diff --git a/maas/region.sls b/maas/region.sls
+index 684fda4..5844f22 100644
+--- a/maas/region.sls
++++ b/maas/region.sls
+@@ -6,10 +6,9 @@ maas_region_packages:
+     - names: {{ region.pkgs }}
+
+ /etc/maas/regiond.conf:
+-  file.managed:
+-  - source: salt://maas/files/regiond.conf
+-  - template: jinja
+-  - group: maas
++  file.replace:
++  - pattern: ^maas_url.*$
++  - repl: "maas_url: http://{{ region.bind.host }}:5240/MAAS"
+   - require:
+     - pkg: maas_region_packages
+
diff --git a/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch b/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch
new file mode 100644
index 000000000..aef087d92
--- /dev/null
+++ b/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch
@@ -0,0 +1,95 @@
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+: Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+:
+: All rights reserved. This program and the accompanying materials
+: are made available under the terms of the Apache License, Version 2.0
+: which accompanies this distribution, and is available at
+: http://www.apache.org/licenses/LICENSE-2.0
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sat, 23 Sep 2017 02:03:01 +0200
+Subject: [PATCH] maas: region: allow timeout override
+
+Two changes are squashed into a single patch so we can test for
+reverse-applying the cummulative diff.
+
+1. Poor implementation that allows overiding comissioning/deploying
+timeouts, which are hardcoded in MaaS source code.
+
+Add the following reclass _param:
+- opnfv_maas_timeout_comissioning;
+- opnfv_maas_timeout_deploying;
+
+The defaults are now 10/15 minutes (MaaS defaults to 20/40 min).
+
+2. maas: region: interface: default mode DHCP
+
+Switch MaaS interface mode default from AUTO to DHCP.
+This way, MaaS will not statically assign IPs *outside* the dynamic
+reserved range, but instead DHCP will assign them *inside* the range.
+
+One alternative would be to handle this via API calls, for each
+node's PXE physical interface, but that overcomplicates things.
+
+3. MaaS blocksize: For some reason the python library behind tftp forces ip
+blocksize to be 1008. We can force it to be 1464 and gain some performance due
+to MTU beeing 1500 (i.e. allow bigger packets).
+
+JIRA: FUEL-316
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ maas/region.sls | 40 ++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 40 insertions(+)
+
+diff --git a/maas/region.sls b/maas/region.sls
+index 5844f22..d844bb4 100644
+--- a/maas/region.sls
++++ b/maas/region.sls
+@@ -38,6 +38,46 @@ restore_maas_database_{{ region.database.name }}:
+
+ {%- endif %}
+
++maas_timeout_commissioning:
++  file.replace:
++    - name: "/usr/lib/python3/dist-packages/maasserver/node_status.py"
++    - pattern: '^(\s+NODE_STATUS.COMMISSIONING:)\s+\d+,$'
++    - repl: \1 {{ salt['pillar.get']('_param:opnfv_maas_timeout_comissioning', '20') }},
++    - require:
++      - pkg: maas_region_packages
++    - require_in:
++      - service: maas_region_services
++
++maas_timeout_deploying:
++  file.replace:
++    - name: "/usr/lib/python3/dist-packages/maasserver/node_status.py"
++    - pattern: '^(\s+NODE_STATUS.DEPLOYING:)\s+\d+,$'
++    - repl: \1 {{ salt['pillar.get']('_param:opnfv_maas_timeout_deploying', '40') }},
++    - require:
++      - pkg: maas_region_packages
++    - require_in:
++      - service: maas_region_services
++
++maas_ip_blksize_force:
++  file.replace:
++    - name: "/usr/lib/python3/dist-packages/tftp/bootstrap.py"
++    - pattern: 'int_blksize = min\(\(int_blksize, MAX_BLOCK_SIZE\)\)'
++    - repl: 'int_blksize = 1464'
++    - require:
++      - pkg: maas_region_packages
++    - require_in:
++      - service: maas_region_services
++
++maas_interface_default_mode_dhcp:
++  file.replace:
++    - name: "/usr/lib/python3/dist-packages/maasserver/models/node.py"
++    - pattern: '^(\s+INTERFACE_LINK_TYPE)\.AUTO'
++    - repl: \1.DHCP
++    - require:
++      - pkg: maas_region_packages
++    - require_in:
++      - service: maas_region_services
++
+ {%- if region.get('enable_iframe', False)  %}
+
+ /etc/apache2/conf-enabled/maas-http.conf:
diff --git a/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch b/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch
new file mode 100644
index 000000000..9f13c4616
--- /dev/null
+++ b/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch
@@ -0,0 +1,137 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sun, 19 Aug 2018 05:38:27 +0200
+Subject: [PATCH] Add machines.delete & co, pxe_nat sls
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ maas/machines/delete.sls                  | 20 ++++++++++++
+ maas/machines/mark_broken_fixed.sls       | 20 ++++++++++++
+ maas/machines/override_failed_testing.sls | 20 ++++++++++++
+ maas/pxe_nat.sls                          | 37 +++++++++++++++++++++++
+ 4 files changed, 97 insertions(+)
+ create mode 100644 maas/machines/delete.sls
+ create mode 100644 maas/machines/mark_broken_fixed.sls
+ create mode 100644 maas/machines/override_failed_testing.sls
+ create mode 100644 maas/pxe_nat.sls
+
+diff --git a/maas/machines/delete.sls b/maas/machines/delete.sls
+new file mode 100644
+index 0000000..2903f92
+--- /dev/null
++++ b/maas/machines/delete.sls
+@@ -0,0 +1,20 @@
++##############################################################################
++# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
++# All rights reserved. This program and the accompanying materials
++# are made available under the terms of the Apache License, Version 2.0
++# which accompanies this distribution, and is available at
++# http://www.apache.org/licenses/LICENSE-2.0
++##############################################################################
++{%- from "maas/map.jinja" import region with context %}
++
++maas_login_admin:
++  cmd.run:
++  - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
++  - unless: 'test -e /var/lib/maas/.maas_credentials'
++
++# TODO: implement delete_machine via _modules/maas.py
++delete_machine:
++  cmd.run:
++  - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine delete {{ pillar['system_id'] }}"
++  - require:
++    - cmd: maas_login_admin
+diff --git a/maas/machines/mark_broken_fixed.sls b/maas/machines/mark_broken_fixed.sls
+new file mode 100644
+index 0000000..46691bb
+--- /dev/null
++++ b/maas/machines/mark_broken_fixed.sls
+@@ -0,0 +1,20 @@
++##############################################################################
++# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
++# All rights reserved. This program and the accompanying materials
++# are made available under the terms of the Apache License, Version 2.0
++# which accompanies this distribution, and is available at
++# http://www.apache.org/licenses/LICENSE-2.0
++##############################################################################
++{%- from "maas/map.jinja" import region with context %}
++
++maas_login_admin:
++  cmd.run:
++  - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
++  - unless: 'test -e /var/lib/maas/.maas_credentials'
++
++# TODO: implement mark_broken_fixed_machine via _modules/maas.py
++mark_broken_fixed_machine:
++  cmd.run:
++  - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine mark-broken {{ pillar['system_id'] }} && sleep 10 && maas opnfv machine mark-fixed {{ pillar['system_id'] }} && maas opnfv machine test {{ pillar['system_id'] }} testing_scripts=fio"
++  - require:
++    - cmd: maas_login_admin
+diff --git a/maas/machines/override_failed_testing.sls b/maas/machines/override_failed_testing.sls
+new file mode 100644
+index 0000000..e7fe1d2
+--- /dev/null
++++ b/maas/machines/override_failed_testing.sls
+@@ -0,0 +1,20 @@
++##############################################################################
++# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
++# All rights reserved. This program and the accompanying materials
++# are made available under the terms of the Apache License, Version 2.0
++# which accompanies this distribution, and is available at
++# http://www.apache.org/licenses/LICENSE-2.0
++##############################################################################
++{%- from "maas/map.jinja" import region with context %}
++
++maas_login_admin:
++  cmd.run:
++  - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials"
++  - unless: 'test -e /var/lib/maas/.maas_credentials'
++
++# TODO: implement override_failed_testing via _modules/maas.py
++mark_broken_fixed_machine:
++  cmd.run:
++  - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine override-failed-testing {{ pillar['system_id'] }}"
++  - require:
++    - cmd: maas_login_admin
+diff --git a/maas/pxe_nat.sls b/maas/pxe_nat.sls
+new file mode 100644
+index 0000000..8a03c4f
+--- /dev/null
++++ b/maas/pxe_nat.sls
+@@ -0,0 +1,37 @@
++##############################################################################
++# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
++# All rights reserved. This program and the accompanying materials
++# are made available under the terms of the Apache License, Version 2.0
++# which accompanies this distribution, and is available at
++# http://www.apache.org/licenses/LICENSE-2.0
++##############################################################################
++net.ipv4.ip_forward:
++  sysctl.present:
++    - value: 1
++
++iptables_pxe_nat:
++  iptables.append:
++    - table: nat
++    - chain: POSTROUTING
++    - jump: MASQUERADE
++    - destination: 0/0
++    - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }}
++    - save: True
++
++iptables_pxe_source:
++  iptables.append:
++    - table: filter
++    - chain: INPUT
++    - jump: ACCEPT
++    - destination: 0/0
++    - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }}
++    - save: True
++
++iptables_pxe_destination:
++  iptables.append:
++    - table: filter
++    - chain: INPUT
++    - jump: ACCEPT
++    - destination: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }}
++    - source: 0/0
++    - save: True