From ab18375a629010525ac15bc11ce2d4e4cf393fe9 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Sat, 18 Aug 2018 03:46:13 +0200 Subject: [docker] Switch to containerized Salt Master * Refactor OPNFV salt-formulas mechanism to resemble upstream git structure: - git submodules: add new submodule for each formula we patch; - create salt-formula-x directories for OPNFV formulas; - move mcp/metadata/service contents to their each formula subdir; - use `make patches-import` for patches previously handled by patch.sh; - retire patch.sh * states: add virtual_init: - mostly based on old salt.sh, which is now obsolete; - exclude salt-master service restart (it would kill the container); * scenarios: cleanup (rm cfg01 virtual node def), adopt virtual_init; * reclass: align our model with prebuilt container's Salt config: - drop linux:network pillar data (handled by Docker); - stop applying linux.system state on cfg01; - align salt user homedir; - drop salt-formula packages (preprovisioned); * minor plumbing in deploy.sh and lib.sh; JIRA: FUEL-383 Change-Id: I28708a9b399d3f19012212c71966ebda9d6fc0ac Signed-off-by: Alexandru Avadanii --- ...stem.repo-Debian-Use-proxy-for-keyservers.patch | 157 +++++++++++++++++++++ .../0002-Set-ovs-bridges-as-L3-interfaces.patch | 101 +++++++++++++ 2 files changed, 258 insertions(+) create mode 100644 mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch create mode 100644 mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch (limited to 'mcp/patches/salt-formula-linux') diff --git a/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch new file mode 100644 index 000000000..4ad51e3e0 --- /dev/null +++ b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch @@ -0,0 +1,157 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2018 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii +Date: Sun, 3 Jun 2018 19:28:18 +0200 +Subject: [PATCH] system.repo: Debian: Use proxy for keyservers + +Previously, when fetching GPG keys for APT keyring, either using +public key download & import (as for default repos) or via keyserver, +we relied on simple `curl` calls or passed it down to Salt aptpkg +module. +To be able to retrieve APT keys behind a proxy, one used to have to +configure the proxy for the Salt minion, which does not yet have +`no_proxy` support (either *all* or *no* traffic hits the proxy). + +When `linux:system:proxy` http(s) proxies are set: +- no longer pass key configuration to Salt aptpkg (until it properly + supports `no_proxy`); +- handle all keys explicitly with `curl` and `apt-key`; +- set 'http(s)_proxy' env vars for `cmd.wait` calls; + +If `linux:system:proxy` is not defined, the behavior is +unchanged for backwards compatibility. + +NOTE: If present, per-repo proxies are also used for keyserver access. + +system.repo: Fix conditions order for Debian proxy + +Signed-off-by: Alexandru Avadanii +--- + linux/system/repo.sls | 55 ++++++++++++++++++++++++++++++++++++------- + 1 file changed, 47 insertions(+), 8 deletions(-) + +diff --git a/linux/system/repo.sls b/linux/system/repo.sls +index 0c0b026..82661f0 100644 +--- a/linux/system/repo.sls ++++ b/linux/system/repo.sls +@@ -7,18 +7,27 @@ linux_repo_prereq_pkgs: + - pkgs: {{ system.pkgs }} + {%- endif %} + ++ {%- set proxies = {'system': {}, 'repo': {}} %} ++ + # global proxy setup + {%- if grains.os_family == 'Debian' %} ++ + {%- if system.proxy.get('pkg', {}).get('enabled', False) %} ++ ++ {%- do proxies.system.update({'https': system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true)}) %} ++ {%- do proxies.system.update({'http': system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true)}) %} ++ {%- do proxies.system.update({'ftp': system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true)}) %} ++ + /etc/apt/apt.conf.d/99proxies-salt: + file.managed: + - template: jinja + - source: salt://linux/files/apt.conf.d_proxies + - defaults: + external_host: False +- https: {{ system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true) }} +- http: {{ system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true) }} +- ftp: {{ system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true) }} ++ https: {{ proxies.system.https }} ++ http: {{ proxies.system.http }} ++ ftp: {{ proxies.system.ftp }} ++ + {%- else %} + /etc/apt/apt.conf.d/99proxies-salt: + file.absent +@@ -43,16 +52,20 @@ purge_sources_list_d_repos: + # per repository proxy setup + {%- if repo.get('proxy', {}).get('enabled', False) %} + {%- set external_host = repo.proxy.get('host', None) or repo.source.split('/')[2] %} ++ {%- do proxies.repo.update({'https': repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true)}) %} ++ {%- do proxies.repo.update({'http': repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true)}) %} ++ {%- do proxies.repo.update({'ftp': repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true)}) %} + /etc/apt/apt.conf.d/99proxies-salt-{{ name }}: + file.managed: + - template: jinja + - source: salt://linux/files/apt.conf.d_proxies + - defaults: + external_host: {{ external_host }} +- https: {{ repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), True) }} +- http: {{ repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), True) }} +- ftp: {{ repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), True) }} ++ https: {{ proxies.repo.https }} ++ http: {{ proxies.repo.http }} ++ ftp: {{ proxies.repo.ftp }} + {%- else %} ++ {%- do proxies.repo.update({'https': None, 'http': None, 'ftp': None}) %} + /etc/apt/apt.conf.d/99proxies-salt-{{ name }}: + file.absent + {%- endif %} +@@ -104,6 +117,13 @@ linux_repo_{{ name }}_key: + {% else %} + - pkgrepo: linux_repo_{{ name }} + {% endif %} ++ - env: ++ {%- if proxies.repo.get('https', None) or proxies.system.get('https', None) %} ++ - https_proxy: {{ proxies.repo.get('https', None) or proxies.system.get('https', None) }} ++ {%- endif %} ++ {%- if proxies.repo.get('http', None) or proxies.system.get('http', None) %} ++ - http_proxy: {{ proxies.repo.get('http', None) or proxies.system.get('http', None) }} ++ {%- endif %} + {%- endif %} + + {%- if repo.get('default', False) %} +@@ -111,6 +131,9 @@ linux_repo_{{ name }}_key: + {%- else %} + + {%- if repo.get('enabled', True) %} ++ {%- set use_proxy = ( ( proxies.repo.get('https', None) or proxies.system.get('https', None) or ++ proxies.repo.get('http', None) or proxies.system.get('http', None) ) and ++ repo.key_id is defined and repo.key_server is defined ) %} + linux_repo_{{ name }}: + pkgrepo.managed: + - refresh_db: False +@@ -126,10 +149,10 @@ linux_repo_{{ name }}: + {%- endif %} + - file: /etc/apt/sources.list.d/{{ name }}.list + - clean_file: {{ repo.get('clean_file', True) }} +- {%- if repo.key_id is defined %} ++ {%- if not use_proxy and repo.key_id is defined %} + - keyid: {{ repo.key_id }} + {%- endif %} +- {%- if repo.key_server is defined %} ++ {%- if not use_proxy and repo.key_server is defined %} + - keyserver: {{ repo.key_server }} + {%- endif %} + {%- if repo.key_url is defined and (grains['saltversioninfo'] >= [2017, 7] or repo.key_url.startswith('salt://')) %} +@@ -143,6 +166,22 @@ linux_repo_{{ name }}: + - file: purge_sources_list_d_repos + {%- endif %} + {%- endif %} ++ {%- if use_proxy and repo.key_id is defined and repo.key_server is defined %} ++linux_repo_{{ name }}_key: ++ cmd.run: ++ - name: "apt-key adv --keyserver {{ repo.key_server }} --recv {{ repo.key_id }}" ++ - unless: 'test -e /etc/apt/sources.list.d/{{ name }}.list' ++ - require_in: ++ - pkgrepo: linux_repo_{{ name }} ++ - env: ++ {%- if proxies.repo.get('https', None) or proxies.system.get('https', None) %} ++ - https_proxy: {{ proxies.repo.get('https', None) or proxies.system.get('https', None) }} ++ {%- endif %} ++ {%- if proxies.repo.get('http', None) or proxies.system.get('http', None) %} ++ - http_proxy: {{ proxies.repo.get('http', None) or proxies.system.get('http', None) }} ++ {%- endif %} ++ {%- endif %} ++ {#- repo.enabled is false #} + {%- else %} + linux_repo_{{ name }}: + pkgrepo.absent: diff --git a/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch b/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch new file mode 100644 index 000000000..ee82474b1 --- /dev/null +++ b/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch @@ -0,0 +1,101 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2018 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Michael Polenchuk +Date: Wed, 28 Feb 2018 17:54:28 +0400 +Subject: [PATCH] Set ovs bridges as L3 interfaces + +Change-Id: I1e83129cc184cf481bea21d7aa452bf60d9e0499 +--- + linux/files/ovs_bridge | 19 +++++++++++++++++++ + linux/files/ovs_port | 7 ++++++- + linux/network/interface.sls | 28 ++++++++++++++++++++++++++++ + 3 files changed, 53 insertions(+), 1 deletion(-) + create mode 100644 linux/files/ovs_bridge + +diff --git a/linux/files/ovs_bridge b/linux/files/ovs_bridge +new file mode 100644 +index 0000000..073c91c +--- /dev/null ++++ b/linux/files/ovs_bridge +@@ -0,0 +1,19 @@ ++auto {{ bridge_name }} ++allow-ovs {{ bridge_name }} ++iface {{ bridge_name }} inet static ++ ovs_type OVSBridge ++ address {{ bridge.address }} ++ netmask {{ bridge.netmask }} ++ mtu {{ bridge.get('mtu', '1500') }} ++ {%- if bridge.use_interfaces is defined %} ++ ovs_ports {{ bridge.use_interfaces|join(' ') }} ++ {%- endif %} ++ {%- if bridge.datapath_type is defined %} ++ ovs_extra set Bridge ${IFACE} datapath_type={{ bridge.datapath_type }} ++ {%- endif %} ++ {%- if bridge.gateway is defined %} ++ gateway {{ bridge.gateway }} ++ {%- endif %} ++ {%- if bridge.name_servers is defined %} ++ dns-nameservers {{ bridge.name_servers | join(' ') }} ++ {%- endif %} +diff --git a/linux/files/ovs_port b/linux/files/ovs_port +index 222ca8e..03072cd 100644 +--- a/linux/files/ovs_port ++++ b/linux/files/ovs_port +@@ -1,6 +1,11 @@ +-auto {{ port_name }} ++# With systemd, adding OVS bridges as 'auto' can cause race conditions ++# https://github.com/openvswitch/ovs/blob/master/debian/openvswitch-switch.README.Debian ++# auto {{ port_name }} + allow-{{ port.bridge }} {{ port_name }} + iface {{ port_name }} inet {{ port.get('proto', 'manual') }} ++{%- if '.' in port_name %} ++vlan-raw-device {{ port_name.split('.')[0] }} ++{%- endif %} + ovs_type {{ port.get('ovs_port_type', 'OVSIntPort') }} + mtu {{ port.get('mtu', '1500') }} + ovs_bridge {{ port.bridge }} +diff --git a/linux/network/interface.sls b/linux/network/interface.sls +index c2d2a23..01131fa 100644 +--- a/linux/network/interface.sls ++++ b/linux/network/interface.sls +@@ -91,6 +91,34 @@ add_int_{{ int_name }}_to_ovs_dpdk_bridge_{{ interface_name }}: + ovs_bridge_{{ interface_name }}: + openvswitch_bridge.present: + - name: {{ interface_name }} ++{%- if interface.get('proto', 'manual') == 'static' %} ++ file.managed: ++ - name: /etc/network/interfaces.u/ifcfg-{{ interface_name }} ++ - makedirs: True ++ - source: salt://linux/files/ovs_bridge ++ - defaults: ++ bridge: {{ interface|yaml }} ++ bridge_name: {{ interface_name }} ++ - template: jinja ++ ++linux_interfaces_include_{{ interface_name }}: ++ file.prepend: ++ - name: /etc/network/interfaces ++ - text: | ++ source /etc/network/interfaces.d/* ++ # Workaround for Upstream-Bug: https://github.com/saltstack/salt/issues/40262 ++ source /etc/network/interfaces.u/* ++ ++ovs_bridge_up_{{ interface_name }}: ++ cmd.run: ++ - name: ifup {{ interface_name }} ++ - require: ++ - file: ovs_bridge_{{ interface_name }} ++ - openvswitch_bridge: ovs_bridge_{{ interface_name }} ++ - file: linux_interfaces_final_include ++ - unless: ++ - ip link show {{ interface_name }} | grep -q '\' ++{%- endif %} + + {# add linux network interface into OVS bridge #} + {%- for int_name, int in network.interface.items() %} -- cgit 1.2.3-korg