[patch] system.repo: Add keyserver proxy support

Instead of defining a http proxy for all salt-minion traffic, which
also includes some Openstack API accesses we can't filter (no_proxy
is not yet supported), add & leverage support for proxy configuration
during APT keyserver access / key download.

JIRA: FUEL-331

Change-Id: I9470807633596c610cfafb141b139ddda2ff096b
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>

6 files changed, 122 insertions, 8 deletions

diff --git a/mcp/config/states/baremetal_init b/mcp/config/states/baremetal_init
index 9718a4176..cd254ddd3 100755
--- a/mcp/config/states/baremetal_init
+++ b/mcp/config/states/baremetal_init
@@ -25,7 +25,6 @@ salt -C 'kvm* or cmp*' file.replace $debian_ip_source \
 salt -C 'kvm*' pkg.install bridge-utils
 salt -C 'kvm*' state.apply linux.network
 salt -C 'kvm* or cmp*' state.apply salt.minion
-salt -C 'kvm* or cmp*' service.force_reload salt-minion
 salt -C 'cmp*' state.apply linux.system
 salt -C 'cmp*' state.apply linux.network || true
 
diff --git a/mcp/config/states/virtual_control_plane b/mcp/config/states/virtual_control_plane
index 0607b318b..75347d918 100755
--- a/mcp/config/states/virtual_control_plane
+++ b/mcp/config/states/virtual_control_plane
@@ -59,7 +59,6 @@ salt -C 'E@^(?!cfg01|mas01|kvm|cmp00).*' cp.get_file \
   "salt://maas/files/$(basename "${APT_CONF_D_CURTIN}")" "${APT_CONF_D_CURTIN}"
 
 wait_for 10 "salt -C 'E@^(?!cfg01|mas01|kvm|cmp00).*' state.apply salt"
-wait_for 10 "salt -C 'E@^(?!cfg01|mas01|kvm|cmp00).*' service.force_reload salt-minion"
 wait_for 10.0 "salt -C 'E@^(?!cfg01|mas01|kvm|cmp00).*' state.apply linux,ntp"
 
 wait_for 10 "salt -C 'E@^(?!cfg01|mas01|kvm|cmp00).*' ssh.set_auth_key ${SUDO_USER} \
diff --git a/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch b/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch
new file mode 100644
index 000000000..8fb9bafe7
--- /dev/null
+++ b/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch
@@ -0,0 +1,108 @@
+From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+Date: Mon, 22 Jan 2018 00:28:09 +0100
+Subject: [PATCH] system.repo: Debian: Add keyserver proxy support
+
+Introduce a new, optional set of parameters to configure the proxy
+used for key fetching / keyserver access under:
+linux:system:proxy:keyserver:http(s).
+
+Previously, when fetching GPG keys for APT keyring, either using
+public key download & import (as for default repos) or via keyserver,
+we relied on simple `curl` calls or passed it down to Salt aptpkg
+module.
+To be able to retrieve APT keys behind a proxy, one used to have to
+configure the proxy for the Salt minion, which does not yet have
+`no_proxy` support (either *all* or *no* traffic hits the proxy).
+
+When the new http(s) proxy param is set:
+- no longer pass key configuration to Salt aptpkg (until it properly
+  supports `no_proxy`);
+- handle all keys explicitly with `curl` and `apt-key`;
+- set 'http(s)_proxy' env vars for `cmd.run`/`cmd.wait` calls;
+
+If linux:system:proxy:keyserver is not defined, the behavior is
+unchanged for backwards compatibility.
+
+Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
+---
+ README.rst            | 16 ++++++++++++++++
+ linux/system/repo.sls | 38 ++++++++++++++++++++++++++++++++++++++
+ 2 files changed, 54 insertions(+)
+
+diff --git a/linux/system/repo.sls b/linux/system/repo.sls
+index 5d4d059..964db3f 100644
+--- a/linux/system/repo.sls
++++ b/linux/system/repo.sls
+@@ -96,13 +96,48 @@ linux_repo_{{ name }}_key:
+     - name: "curl -s {{ repo.key_url }} | apt-key add -"
+     - watch:
+       - file: default_repo_list
++    - env:
++      - http_proxy: {{ system.proxy.get('keyserver', {}).get('http', None) }}
++      - https_proxy: {{ system.proxy.get('keyserver', {}).get('https', None) }}
+
+ {%- endif %}
+
++{#- repo.default is false #}
+ {%- else %}
+
+ {%- if repo.get('enabled', True) %}
+
++{%- if system.proxy.keyserver is defined %}
++
++{%- if repo.get('key') %}
++
++linux_repo_{{ name }}_key:
++  cmd.run:
++    - name: "echo '{{ repo.key }}' | apt-key add -"
++
++{%- elif repo.key_url|default(False) %}
++
++linux_repo_{{ name }}_key:
++  cmd.run:
++    - name: "curl -s {{ repo.key_url }} | apt-key add -"
++    - env:
++      - http_proxy: {{ system.proxy.get('keyserver', {}).get('http', None) }}
++      - https_proxy: {{ system.proxy.get('keyserver', {}).get('https', None) }}
++
++{%- elif repo.key_id is defined and repo.key_server is defined %}
++
++linux_repo_{{ name }}_key:
++  cmd.run:
++    - name: "apt-key adv --keyserver {{ repo.key_server }} --recv {{ repo.key_id }}"
++    - env:
++      - http_proxy: {{ system.proxy.get('keyserver', {}).get('http', None) }}
++      - https_proxy: {{ system.proxy.get('keyserver', {}).get('https', None) }}
++
++{%- endif %}
++
++{#- system.proxy.keyserver #}
++{%- endif %}
++
+ linux_repo_{{ name }}:
+   pkgrepo.managed:
+   {%- if repo.ppa is defined %}
+@@ -115,6 +150,7 @@ linux_repo_{{ name }}:
+   {%- endif %}
+   - file: /etc/apt/sources.list.d/{{ name }}.list
+   - clean_file: {{ repo.clean|default(True) }}
++  {%- if system.proxy.keyserver is not defined %}
+   {%- if repo.key_id is defined %}
+   - keyid: {{ repo.key_id }}
+   {%- endif %}
+@@ -124,6 +160,7 @@ linux_repo_{{ name }}:
+   {%- if repo.key_url is defined %}
+   - key_url: {{ repo.key_url }}
+   {%- endif %}
++  {%- endif %}
+   - consolidate: {{ repo.get('consolidate', False) }}
+   - clean_file: {{ repo.get('clean_file', False) }}
+   - refresh_db: {{ repo.get('refresh_db', True) }}
+@@ -140,6 +177,7 @@ linux_repo_{{ name }}:
+   {%- endif %}
+   {%- endif %}
+
++{#- repo.enabled is false #}
+ {%- else %}
+
+ linux_repo_{{ name }}_absent:
diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list
index 284f1bcec..1b3bfeab0 100644
--- a/mcp/patches/patches.list
+++ b/mcp/patches/patches.list
@@ -15,4 +15,5 @@
 /usr/share/salt-formulas/env: 0008-Handle-file_recv-option.patch
 /usr/share/salt-formulas/env: 0009-controller-Use-keystoneclient-to-check-project-ID.patch
 /usr/share/salt-formulas/env: 0010-maas-region-allow-timeout-override.patch
+/usr/share/salt-formulas/env: 0011-system.repo-Debian-Add-keyserver-proxy-support.patch
 /usr/share/salt-formulas/env: 0012-linux.storage.lvm-Disable-filter.patch
diff --git a/mcp/reclass/classes/cluster/baremetal-mcp-pike-common-ha/include/proxy.yml b/mcp/reclass/classes/cluster/baremetal-mcp-pike-common-ha/include/proxy.yml
index 5d7475812..3c324db8b 100644
--- a/mcp/reclass/classes/cluster/baremetal-mcp-pike-common-ha/include/proxy.yml
+++ b/mcp/reclass/classes/cluster/baremetal-mcp-pike-common-ha/include/proxy.yml
@@ -16,8 +16,9 @@ parameters:
   #       https_proxy: http://${_param:infra_maas_node01_deploy_address}:8000
   #       no_proxy:
   #         - .local
-  salt:
-    minion:
+  linux:
+    system:
       proxy:
-        host: ${_param:infra_maas_node01_deploy_address}
-        port: 8000
+        keyserver:
+          http: http://${_param:infra_maas_node01_deploy_address}:8000
+          https: http://${_param:infra_maas_node01_deploy_address}:8000
diff --git a/mcp/salt-formulas/opendaylight/server.sls b/mcp/salt-formulas/opendaylight/server.sls
index 30c337c00..1f0475062 100644
--- a/mcp/salt-formulas/opendaylight/server.sls
+++ b/mcp/salt-formulas/opendaylight/server.sls
@@ -6,9 +6,17 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 {% from "opendaylight/map.jinja" import server with context %}
+{% from "linux/map.jinja" import system with context %}
 
 {%- if server.enabled %}
 
+opendaylight_repo_key:
+  cmd.run:
+    - name: "apt-key adv --keyserver keyserver.ubuntu.com --recv 49B07274951063870A8B7EAE7B8AA1A344C05248"
+    - env:
+      - http_proxy: {{ system.proxy.get('keyserver', {}).get('http', None) }}
+      - https_proxy: {{ system.proxy.get('keyserver', {}).get('https', None) }}
+
 opendaylight_repo:
   pkgrepo.managed:
   # NOTE(armband): PPA handling behind proxy broken, define it explicitly
@@ -17,8 +25,6 @@ opendaylight_repo:
   - human_name: opendaylight-ppa
   - name: deb http://ppa.launchpad.net/odl-team/nitrogen/ubuntu xenial main
   - file: /etc/apt/sources.list.d/odl-team-ubuntu-nitrogen-xenial.list
-  - keyid: 49B07274951063870A8B7EAE7B8AA1A344C05248
-  - keyserver: keyserver.ubuntu.com
 
 opendaylight:
   pkg.installed: