[docker] Switch to containerized Salt Master

* Refactor OPNFV salt-formulas mechanism to resemble upstream git
  structure:
  - git submodules: add new submodule for each formula we patch;
  - create salt-formula-x directories for OPNFV formulas;
  - move mcp/metadata/service contents to their each formula subdir;
  - use `make patches-import` for patches previously handled by
    patch.sh;
  - retire patch.sh
* states: add virtual_init:
  - mostly based on old salt.sh, which is now obsolete;
  - exclude salt-master service restart (it would kill the container);
* scenarios: cleanup (rm cfg01 virtual node def), adopt virtual_init;
* reclass: align our model with prebuilt container's Salt config:
  - drop linux:network pillar data (handled by Docker);
  - stop applying linux.system state on cfg01;
  - align salt user homedir;
  - drop salt-formula packages (preprovisioned);
* minor plumbing in deploy.sh and lib.sh;

JIRA: FUEL-383

Change-Id: I28708a9b399d3f19012212c71966ebda9d6fc0ac
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

1 files changed, 157 insertions, 0 deletions

diff --git a/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch
new file mode 100644
index 000000000..4ad51e3e0
--- /dev/null
+++ b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch
@@ -0,0 +1,157 @@
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+: Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+:
+: All rights reserved. This program and the accompanying materials
+: are made available under the terms of the Apache License, Version 2.0
+: which accompanies this distribution, and is available at
+: http://www.apache.org/licenses/LICENSE-2.0
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sun, 3 Jun 2018 19:28:18 +0200
+Subject: [PATCH] system.repo: Debian: Use proxy for keyservers
+
+Previously, when fetching GPG keys for APT keyring, either using
+public key download & import (as for default repos) or via keyserver,
+we relied on simple `curl` calls or passed it down to Salt aptpkg
+module.
+To be able to retrieve APT keys behind a proxy, one used to have to
+configure the proxy for the Salt minion, which does not yet have
+`no_proxy` support (either *all* or *no* traffic hits the proxy).
+
+When `linux:system:proxy` http(s) proxies are set:
+- no longer pass key configuration to Salt aptpkg (until it properly
+  supports `no_proxy`);
+- handle all keys explicitly with `curl` and `apt-key`;
+- set 'http(s)_proxy' env vars for `cmd.wait` calls;
+
+If `linux:system:proxy` is not defined, the behavior is
+unchanged for backwards compatibility.
+
+NOTE: If present, per-repo proxies are also used for keyserver access.
+
+system.repo: Fix conditions order for Debian proxy
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ linux/system/repo.sls | 55 ++++++++++++++++++++++++++++++++++++-------
+ 1 file changed, 47 insertions(+), 8 deletions(-)
+
+diff --git a/linux/system/repo.sls b/linux/system/repo.sls
+index 0c0b026..82661f0 100644
+--- a/linux/system/repo.sls
++++ b/linux/system/repo.sls
+@@ -7,18 +7,27 @@ linux_repo_prereq_pkgs:
+   - pkgs: {{ system.pkgs }}
+   {%- endif %}
+
++  {%- set proxies = {'system': {}, 'repo': {}} %}
++
+   # global proxy setup
+   {%- if grains.os_family == 'Debian' %}
++
+     {%- if system.proxy.get('pkg', {}).get('enabled', False) %}
++
++      {%- do proxies.system.update({'https': system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true)}) %}
++      {%- do proxies.system.update({'http': system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true)}) %}
++      {%- do proxies.system.update({'ftp': system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true)}) %}
++
+ /etc/apt/apt.conf.d/99proxies-salt:
+   file.managed:
+   - template: jinja
+   - source: salt://linux/files/apt.conf.d_proxies
+   - defaults:
+       external_host: False
+-      https: {{ system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true) }}
+-      http: {{ system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true) }}
+-      ftp: {{ system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true) }}
++      https: {{ proxies.system.https }}
++      http: {{ proxies.system.http }}
++      ftp: {{ proxies.system.ftp }}
++
+     {%- else %}
+ /etc/apt/apt.conf.d/99proxies-salt:
+   file.absent
+@@ -43,16 +52,20 @@ purge_sources_list_d_repos:
+ # per repository proxy setup
+       {%- if repo.get('proxy', {}).get('enabled', False) %}
+         {%- set external_host = repo.proxy.get('host', None) or repo.source.split('/')[2] %}
++        {%- do proxies.repo.update({'https': repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true)}) %}
++        {%- do proxies.repo.update({'http': repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true)}) %}
++        {%- do proxies.repo.update({'ftp': repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true)}) %}
+ /etc/apt/apt.conf.d/99proxies-salt-{{ name }}:
+   file.managed:
+   - template: jinja
+   - source: salt://linux/files/apt.conf.d_proxies
+   - defaults:
+       external_host: {{ external_host }}
+-      https: {{ repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), True) }}
+-      http: {{ repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), True) }}
+-      ftp: {{ repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), True) }}
++      https: {{ proxies.repo.https }}
++      http: {{ proxies.repo.http }}
++      ftp: {{ proxies.repo.ftp }}
+       {%- else %}
++        {%- do proxies.repo.update({'https': None, 'http': None, 'ftp': None}) %}
+ /etc/apt/apt.conf.d/99proxies-salt-{{ name }}:
+   file.absent
+       {%- endif %}
+@@ -104,6 +117,13 @@ linux_repo_{{ name }}_key:
+         {% else %}
+       - pkgrepo: linux_repo_{{ name }}
+         {% endif %}
++    - env:
++        {%- if proxies.repo.get('https', None) or proxies.system.get('https', None) %}
++      - https_proxy: {{ proxies.repo.get('https', None) or proxies.system.get('https', None) }}
++        {%- endif %}
++        {%- if proxies.repo.get('http', None) or proxies.system.get('http', None) %}
++      - http_proxy: {{ proxies.repo.get('http', None) or proxies.system.get('http', None) }}
++        {%- endif %}
+       {%- endif %}
+
+       {%- if repo.get('default', False) %}
+@@ -111,6 +131,9 @@ linux_repo_{{ name }}_key:
+       {%- else %}
+
+         {%- if repo.get('enabled', True) %}
++          {%- set use_proxy = ( ( proxies.repo.get('https', None) or proxies.system.get('https', None) or
++                                  proxies.repo.get('http', None) or proxies.system.get('http', None) ) and
++                                  repo.key_id is defined and repo.key_server is defined ) %}
+ linux_repo_{{ name }}:
+   pkgrepo.managed:
+   - refresh_db: False
+@@ -126,10 +149,10 @@ linux_repo_{{ name }}:
+             {%- endif %}
+   - file: /etc/apt/sources.list.d/{{ name }}.list
+   - clean_file: {{ repo.get('clean_file', True) }}
+-            {%- if repo.key_id is defined %}
++            {%- if not use_proxy and repo.key_id is defined %}
+   - keyid: {{ repo.key_id }}
+             {%- endif %}
+-            {%- if repo.key_server is defined %}
++            {%- if not use_proxy and repo.key_server is defined %}
+   - keyserver: {{ repo.key_server }}
+             {%- endif %}
+             {%- if repo.key_url is defined and (grains['saltversioninfo'] >= [2017, 7] or repo.key_url.startswith('salt://')) %}
+@@ -143,6 +166,22 @@ linux_repo_{{ name }}:
+     - file: purge_sources_list_d_repos
+             {%- endif %}
+           {%- endif %}
++          {%- if use_proxy and repo.key_id is defined and repo.key_server is defined %}
++linux_repo_{{ name }}_key:
++  cmd.run:
++    - name: "apt-key adv --keyserver {{ repo.key_server }} --recv {{ repo.key_id }}"
++    - unless: 'test -e /etc/apt/sources.list.d/{{ name }}.list'
++    - require_in:
++      - pkgrepo: linux_repo_{{ name }}
++    - env:
++            {%- if proxies.repo.get('https', None) or proxies.system.get('https', None) %}
++      - https_proxy: {{ proxies.repo.get('https', None) or proxies.system.get('https', None) }}
++            {%- endif %}
++            {%- if proxies.repo.get('http', None) or proxies.system.get('http', None) %}
++      - http_proxy: {{ proxies.repo.get('http', None) or proxies.system.get('http', None) }}
++            {%- endif %}
++          {%- endif %}
++        {#- repo.enabled is false #}
+         {%- else %}
+ linux_repo_{{ name }}:
+   pkgrepo.absent: