[docker] Switch to containerized Salt Master

* Refactor OPNFV salt-formulas mechanism to resemble upstream git
  structure:
  - git submodules: add new submodule for each formula we patch;
  - create salt-formula-x directories for OPNFV formulas;
  - move mcp/metadata/service contents to their each formula subdir;
  - use `make patches-import` for patches previously handled by
    patch.sh;
  - retire patch.sh
* states: add virtual_init:
  - mostly based on old salt.sh, which is now obsolete;
  - exclude salt-master service restart (it would kill the container);
* scenarios: cleanup (rm cfg01 virtual node def), adopt virtual_init;
* reclass: align our model with prebuilt container's Salt config:
  - drop linux:network pillar data (handled by Docker);
  - stop applying linux.system state on cfg01;
  - align salt user homedir;
  - drop salt-formula packages (preprovisioned);
* minor plumbing in deploy.sh and lib.sh;

JIRA: FUEL-383

Change-Id: I28708a9b399d3f19012212c71966ebda9d6fc0ac
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

2 files changed, 258 insertions, 0 deletions

diff --git a/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch
new file mode 100644
index 000000000..4ad51e3e0
--- /dev/null
+++ b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch
@@ -0,0 +1,157 @@
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+: Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+:
+: All rights reserved. This program and the accompanying materials
+: are made available under the terms of the Apache License, Version 2.0
+: which accompanies this distribution, and is available at
+: http://www.apache.org/licenses/LICENSE-2.0
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Sun, 3 Jun 2018 19:28:18 +0200
+Subject: [PATCH] system.repo: Debian: Use proxy for keyservers
+
+Previously, when fetching GPG keys for APT keyring, either using
+public key download & import (as for default repos) or via keyserver,
+we relied on simple `curl` calls or passed it down to Salt aptpkg
+module.
+To be able to retrieve APT keys behind a proxy, one used to have to
+configure the proxy for the Salt minion, which does not yet have
+`no_proxy` support (either *all* or *no* traffic hits the proxy).
+
+When `linux:system:proxy` http(s) proxies are set:
+- no longer pass key configuration to Salt aptpkg (until it properly
+  supports `no_proxy`);
+- handle all keys explicitly with `curl` and `apt-key`;
+- set 'http(s)_proxy' env vars for `cmd.wait` calls;
+
+If `linux:system:proxy` is not defined, the behavior is
+unchanged for backwards compatibility.
+
+NOTE: If present, per-repo proxies are also used for keyserver access.
+
+system.repo: Fix conditions order for Debian proxy
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ linux/system/repo.sls | 55 ++++++++++++++++++++++++++++++++++++-------
+ 1 file changed, 47 insertions(+), 8 deletions(-)
+
+diff --git a/linux/system/repo.sls b/linux/system/repo.sls
+index 0c0b026..82661f0 100644
+--- a/linux/system/repo.sls
++++ b/linux/system/repo.sls
+@@ -7,18 +7,27 @@ linux_repo_prereq_pkgs:
+   - pkgs: {{ system.pkgs }}
+   {%- endif %}
+
++  {%- set proxies = {'system': {}, 'repo': {}} %}
++
+   # global proxy setup
+   {%- if grains.os_family == 'Debian' %}
++
+     {%- if system.proxy.get('pkg', {}).get('enabled', False) %}
++
++      {%- do proxies.system.update({'https': system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true)}) %}
++      {%- do proxies.system.update({'http': system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true)}) %}
++      {%- do proxies.system.update({'ftp': system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true)}) %}
++
+ /etc/apt/apt.conf.d/99proxies-salt:
+   file.managed:
+   - template: jinja
+   - source: salt://linux/files/apt.conf.d_proxies
+   - defaults:
+       external_host: False
+-      https: {{ system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true) }}
+-      http: {{ system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true) }}
+-      ftp: {{ system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true) }}
++      https: {{ proxies.system.https }}
++      http: {{ proxies.system.http }}
++      ftp: {{ proxies.system.ftp }}
++
+     {%- else %}
+ /etc/apt/apt.conf.d/99proxies-salt:
+   file.absent
+@@ -43,16 +52,20 @@ purge_sources_list_d_repos:
+ # per repository proxy setup
+       {%- if repo.get('proxy', {}).get('enabled', False) %}
+         {%- set external_host = repo.proxy.get('host', None) or repo.source.split('/')[2] %}
++        {%- do proxies.repo.update({'https': repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true)}) %}
++        {%- do proxies.repo.update({'http': repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true)}) %}
++        {%- do proxies.repo.update({'ftp': repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true)}) %}
+ /etc/apt/apt.conf.d/99proxies-salt-{{ name }}:
+   file.managed:
+   - template: jinja
+   - source: salt://linux/files/apt.conf.d_proxies
+   - defaults:
+       external_host: {{ external_host }}
+-      https: {{ repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), True) }}
+-      http: {{ repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), True) }}
+-      ftp: {{ repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), True) }}
++      https: {{ proxies.repo.https }}
++      http: {{ proxies.repo.http }}
++      ftp: {{ proxies.repo.ftp }}
+       {%- else %}
++        {%- do proxies.repo.update({'https': None, 'http': None, 'ftp': None}) %}
+ /etc/apt/apt.conf.d/99proxies-salt-{{ name }}:
+   file.absent
+       {%- endif %}
+@@ -104,6 +117,13 @@ linux_repo_{{ name }}_key:
+         {% else %}
+       - pkgrepo: linux_repo_{{ name }}
+         {% endif %}
++    - env:
++        {%- if proxies.repo.get('https', None) or proxies.system.get('https', None) %}
++      - https_proxy: {{ proxies.repo.get('https', None) or proxies.system.get('https', None) }}
++        {%- endif %}
++        {%- if proxies.repo.get('http', None) or proxies.system.get('http', None) %}
++      - http_proxy: {{ proxies.repo.get('http', None) or proxies.system.get('http', None) }}
++        {%- endif %}
+       {%- endif %}
+
+       {%- if repo.get('default', False) %}
+@@ -111,6 +131,9 @@ linux_repo_{{ name }}_key:
+       {%- else %}
+
+         {%- if repo.get('enabled', True) %}
++          {%- set use_proxy = ( ( proxies.repo.get('https', None) or proxies.system.get('https', None) or
++                                  proxies.repo.get('http', None) or proxies.system.get('http', None) ) and
++                                  repo.key_id is defined and repo.key_server is defined ) %}
+ linux_repo_{{ name }}:
+   pkgrepo.managed:
+   - refresh_db: False
+@@ -126,10 +149,10 @@ linux_repo_{{ name }}:
+             {%- endif %}
+   - file: /etc/apt/sources.list.d/{{ name }}.list
+   - clean_file: {{ repo.get('clean_file', True) }}
+-            {%- if repo.key_id is defined %}
++            {%- if not use_proxy and repo.key_id is defined %}
+   - keyid: {{ repo.key_id }}
+             {%- endif %}
+-            {%- if repo.key_server is defined %}
++            {%- if not use_proxy and repo.key_server is defined %}
+   - keyserver: {{ repo.key_server }}
+             {%- endif %}
+             {%- if repo.key_url is defined and (grains['saltversioninfo'] >= [2017, 7] or repo.key_url.startswith('salt://')) %}
+@@ -143,6 +166,22 @@ linux_repo_{{ name }}:
+     - file: purge_sources_list_d_repos
+             {%- endif %}
+           {%- endif %}
++          {%- if use_proxy and repo.key_id is defined and repo.key_server is defined %}
++linux_repo_{{ name }}_key:
++  cmd.run:
++    - name: "apt-key adv --keyserver {{ repo.key_server }} --recv {{ repo.key_id }}"
++    - unless: 'test -e /etc/apt/sources.list.d/{{ name }}.list'
++    - require_in:
++      - pkgrepo: linux_repo_{{ name }}
++    - env:
++            {%- if proxies.repo.get('https', None) or proxies.system.get('https', None) %}
++      - https_proxy: {{ proxies.repo.get('https', None) or proxies.system.get('https', None) }}
++            {%- endif %}
++            {%- if proxies.repo.get('http', None) or proxies.system.get('http', None) %}
++      - http_proxy: {{ proxies.repo.get('http', None) or proxies.system.get('http', None) }}
++            {%- endif %}
++          {%- endif %}
++        {#- repo.enabled is false #}
+         {%- else %}
+ linux_repo_{{ name }}:
+   pkgrepo.absent:
diff --git a/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch b/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch
new file mode 100644
index 000000000..ee82474b1
--- /dev/null
+++ b/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch
@@ -0,0 +1,101 @@
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+: Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+:
+: All rights reserved. This program and the accompanying materials
+: are made available under the terms of the Apache License, Version 2.0
+: which accompanies this distribution, and is available at
+: http://www.apache.org/licenses/LICENSE-2.0
+::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+From: Michael Polenchuk <mpolenchuk@mirantis.com>
+Date: Wed, 28 Feb 2018 17:54:28 +0400
+Subject: [PATCH] Set ovs bridges as L3 interfaces
+
+Change-Id: I1e83129cc184cf481bea21d7aa452bf60d9e0499
+---
+ linux/files/ovs_bridge      | 19 +++++++++++++++++++
+ linux/files/ovs_port        |  7 ++++++-
+ linux/network/interface.sls | 28 ++++++++++++++++++++++++++++
+ 3 files changed, 53 insertions(+), 1 deletion(-)
+ create mode 100644 linux/files/ovs_bridge
+
+diff --git a/linux/files/ovs_bridge b/linux/files/ovs_bridge
+new file mode 100644
+index 0000000..073c91c
+--- /dev/null
++++ b/linux/files/ovs_bridge
+@@ -0,0 +1,19 @@
++auto {{ bridge_name }}
++allow-ovs {{ bridge_name }}
++iface {{ bridge_name }} inet static
++  ovs_type OVSBridge
++  address {{ bridge.address }}
++  netmask {{ bridge.netmask }}
++  mtu {{ bridge.get('mtu', '1500') }}
++  {%- if bridge.use_interfaces is defined %}
++  ovs_ports {{ bridge.use_interfaces|join(' ') }}
++  {%- endif %}
++  {%- if bridge.datapath_type is defined %}
++  ovs_extra set Bridge ${IFACE} datapath_type={{ bridge.datapath_type }}
++  {%- endif %}
++  {%- if bridge.gateway is defined %}
++  gateway {{ bridge.gateway }}
++  {%- endif %}
++  {%- if bridge.name_servers is defined %}
++  dns-nameservers {{ bridge.name_servers | join(' ') }}
++  {%- endif %}
+diff --git a/linux/files/ovs_port b/linux/files/ovs_port
+index 222ca8e..03072cd 100644
+--- a/linux/files/ovs_port
++++ b/linux/files/ovs_port
+@@ -1,6 +1,11 @@
+-auto {{ port_name }}
++# With systemd, adding OVS bridges as 'auto' can cause race conditions
++# https://github.com/openvswitch/ovs/blob/master/debian/openvswitch-switch.README.Debian
++# auto {{ port_name }}
+ allow-{{ port.bridge }} {{ port_name }}
+ iface {{ port_name }} inet {{ port.get('proto', 'manual') }}
++{%- if '.' in port_name %}
++vlan-raw-device {{ port_name.split('.')[0] }}
++{%- endif %}
+ ovs_type {{ port.get('ovs_port_type', 'OVSIntPort') }}
+ mtu {{ port.get('mtu', '1500') }}
+ ovs_bridge {{ port.bridge }}
+diff --git a/linux/network/interface.sls b/linux/network/interface.sls
+index c2d2a23..01131fa 100644
+--- a/linux/network/interface.sls
++++ b/linux/network/interface.sls
+@@ -91,6 +91,34 @@ add_int_{{ int_name }}_to_ovs_dpdk_bridge_{{ interface_name }}:
+ ovs_bridge_{{ interface_name }}:
+   openvswitch_bridge.present:
+   - name: {{ interface_name }}
++{%- if interface.get('proto', 'manual') == 'static' %}
++  file.managed:
++  - name: /etc/network/interfaces.u/ifcfg-{{ interface_name }}
++  - makedirs: True
++  - source: salt://linux/files/ovs_bridge
++  - defaults:
++      bridge: {{ interface|yaml }}
++      bridge_name: {{ interface_name }}
++  - template: jinja
++
++linux_interfaces_include_{{ interface_name }}:
++  file.prepend:
++  - name: /etc/network/interfaces
++  - text: |
++      source /etc/network/interfaces.d/*
++      # Workaround for Upstream-Bug: https://github.com/saltstack/salt/issues/40262
++      source /etc/network/interfaces.u/*
++
++ovs_bridge_up_{{ interface_name }}:
++  cmd.run:
++  - name: ifup {{ interface_name }}
++  - require:
++    - file: ovs_bridge_{{ interface_name }}
++    - openvswitch_bridge: ovs_bridge_{{ interface_name }}
++    - file: linux_interfaces_final_include
++  - unless:
++    - ip link show {{ interface_name }} | grep -q '\<UP\>'
++{%- endif %}
+
+ {# add linux network interface into OVS bridge #}
+ {%- for int_name, int in network.interface.items() %}