diff options
Diffstat (limited to 'mcp/patches')
47 files changed, 2109 insertions, 836 deletions
diff --git a/mcp/patches/0003-maas-region-force-artifact-download.patch b/mcp/patches/0003-maas-region-force-artifact-download.patch deleted file mode 100644 index b61a3d1b8..000000000 --- a/mcp/patches/0003-maas-region-force-artifact-download.patch +++ /dev/null @@ -1,73 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Sat, 5 Aug 2017 02:03:01 +0200 -Subject: [PATCH] maas: region: force artifact download - -MaaS configuration fails until all required artifacts are in place, -including bootloaders and target images. - -Hack around this by forcing an explicit artifact sync. - -NOTE: This is probably achievable through existing maas salt custom -module (py) and/or minor rework on that. -This fixup should be temporary at best. - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - -diff --git a/maas/region.sls b/maas/region.sls ---- a/maas/region.sls -+++ b/maas/region.sls -@@ -109,11 +109,19 @@ - cmd.run: - - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" - -+maas_force_artifact_sync: -+ cmd.script: -+ - name: salt://maas/files/maas-artifact-sync.sh -+ - template: jinja -+ - shell: /bin/bash -+ - require: -+ - cmd: maas_login_admin -+ - maas_config: - module.run: - - name: maas.process_maas_config - - require: -- - cmd: maas_login_admin -+ - cmd: maas_force_artifact_sync - - maas_commissioning_scripts: - module.run: -diff --git a/maas/files/maas-artifact-sync.sh b/maas/files/maas-artifact-sync.sh -new file mode 100644 ---- /dev/null -+++ b/maas/files/maas-artifact-sync.sh -@@ -0,0 +1,20 @@ -+{%- from "maas/map.jinja" import region with context %} -+#!/bin/bash -+function wait_for { -+ local total_attempts=$1; shift -+ local cmdstr=$@ -+ local sleep_time=10 -+ echo -e "\n[NOTE] Waiting for cmd to return success: ${cmdstr}\n" -+ for attempt in $(seq "${total_attempts}"); do -+ eval "${cmdstr}" && break || true -+ echo -n '.'; sleep "${sleep_time}" -+ done -+} -+maas login {{ region.admin.username }} \ -+ http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < \ -+ /var/lib/maas/.maas_credentials || exit 1 -+# wait max 5 min for service up, 15 min image download, 5 min region to rack sync -+wait_for 30 "maas opnfv boot-resources import" -+wait_for 90 "! maas opnfv boot-resources is-importing | grep -q 'true'" -+maas opnfv rack-controllers import-boot-images || exit 3 -+wait_for 30 "test -d /var/lib/maas/boot-resources/current/ubuntu/amd64" diff --git a/mcp/patches/0004-network.dpdk-Move-ifcfg-br-prv-to-interfaces.u.patch b/mcp/patches/0004-network.dpdk-Move-ifcfg-br-prv-to-interfaces.u.patch deleted file mode 100644 index 180a9ddee..000000000 --- a/mcp/patches/0004-network.dpdk-Move-ifcfg-br-prv-to-interfaces.u.patch +++ /dev/null @@ -1,41 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Sun, 25 Feb 2018 03:43:49 +0100 -Subject: [PATCH] network.dpdk: Move ifcfg-br-prv to interfaces.u - -Workaround for Upstream-Bug: -https://github.com/saltstack/salt/issues/40262 - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - linux/network/dpdk.sls | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/linux/network/dpdk.sls b/linux/network/dpdk.sls -index 05fe05f..cc0ff9e 100644 ---- a/linux/network/dpdk.sls -+++ b/linux/network/dpdk.sls -@@ -128,7 +128,7 @@ linux_network_dpdk_bridge_interface_{{ interface_name }}: - After=openvswitch-switch.service - - {# enforce ip address and mtu for ovs dpdk br-prv #} --/etc/network/interfaces.d/ifcfg-{{ interface_name }}: -+/etc/network/interfaces.u/ifcfg-{{ interface_name }}: - file.managed: - - contents: | - auto {{ interface_name }} -@@ -138,6 +138,7 @@ linux_network_dpdk_bridge_interface_{{ interface_name }}: - {%- if interface.mtu is defined %} - mtu {{ interface.mtu }} - {%- endif %} -+ - makedirs: True - - require: - - file: /etc/systemd/system/ifup@{{ interface_name }}.service.d/override.conf - diff --git a/mcp/patches/0005-maas-module-Obtain-fabric-ID-from-CIDR.patch b/mcp/patches/0005-maas-module-Obtain-fabric-ID-from-CIDR.patch deleted file mode 100644 index 4a81786b2..000000000 --- a/mcp/patches/0005-maas-module-Obtain-fabric-ID-from-CIDR.patch +++ /dev/null @@ -1,65 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Sat, 19 Aug 2017 02:03:01 +0200 -Subject: [PATCH] maas: module: Obtain fabric ID from CIDR - -MaaS subnet update requires specifying the correct fabric via reclass, -which we used to hardcode in our OPNFV reclass model to fabric-2. -However, fabric index numbers are not deterministic, so the old -method is unreliable. - -Update MaaS custom py module to determine fabric name/ID on the -fly, based on CIDR matching (assuming we don't have CIDR conflicts). - -This change maintains backwards compatibility: -- if fabric is specified via reclass model, it will be used as-is; -- if fabric is not specified via reclass model, we try to deduce it - based on CIDR; if no match is found, the old default ('') is used; - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Signed-off-by: Guillermo Herrero <Guillermo.Herrero@enea.com> ---- - -diff --git a/_modules/maas.py b/_modules/maas.py ---- a/_modules/maas.py -+++ b/_modules/maas.py -@@ -126,6 +126,8 @@ - - def process_single(name, config_data): - self._update = False -+ if isinstance(config_data, dict) and 'name' in config_data: -+ name = config_data['name'] - try: - data = self.fill_data(name, config_data, **extra) - if data is None: -@@ -198,7 +198,8 @@ - def fill_data(self, name, subnet, fabrics): - data = { - 'name': name, -- 'fabric': str(fabrics[subnet.get('fabric', '')]), -+ 'fabric': str(fabrics[subnet.get('fabric', -+ self._get_fabric_from_cidr(subnet.get('cidr')))]), - 'cidr': subnet.get('cidr'), - 'gateway_ip': subnet['gateway_ip'], - } -@@ -215,6 +216,13 @@ - self._process_iprange(res_json['id']) - return response - -+ def _get_fabric_from_cidr(self, cidr): -+ subnets = json.loads(self._maas.get(u'api/2.0/subnets/').read()) -+ for subnet in subnets: -+ if subnet['cidr'] == cidr: -+ return subnet['vlan']['fabric'] -+ return '' -+ - def _process_iprange(self, subnet_id): - ipranges = json.loads(self._maas.get(u'api/2.0/ipranges/').read()) - LOG.warn('all %s ipranges %s', subnet_id, ipranges) diff --git a/mcp/patches/0006-maas-module-Add-VLAN-DHCP-enable-support.patch b/mcp/patches/0006-maas-module-Add-VLAN-DHCP-enable-support.patch deleted file mode 100644 index d2f531168..000000000 --- a/mcp/patches/0006-maas-module-Add-VLAN-DHCP-enable-support.patch +++ /dev/null @@ -1,89 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Sat, 19 Aug 2017 02:03:01 +0200 -Subject: [PATCH] maas: module: Add VLAN DHCP enable support - -MaaS custom py module does not support VLAN configuration. -This should be implemented by adding a dedicated class for VLAN. -However, we are only interested in updating an existign VLAN to -enable DHCP on an existing IP range (set up via subnet configuration), -so extend existing subnet handling to include basic VLAN update. - -NOTE: Design-wise, this is hacky, and its only purpose is to allow -setting 'dhcp_on=True' for an existing VLAN. - -Example reclass model usage: -maas: - region: - subnets: - 192.168.11.0/24: - # ... - vlans: - untagged: - vid: 0 - dhcp_on: true - primary_rack: mas01 - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - -diff --git a/_modules/maas.py b/_modules/maas.py -index d3227ca..8a2243d 100644 ---- a/_modules/maas.py -+++ b/_modules/maas.py -@@ -204,6 +204,7 @@ - 'gateway_ip': subnet['gateway_ip'], - } - self._iprange = subnet['iprange'] -+ self._vlans = subnet['vlans'] - return data - - def update(self, new, old): -@@ -214,6 +215,7 @@ - response = super(Subnet, self).send(data) - res_json = json.loads(response) - self._process_iprange(res_json['id']) -+ self._process_dhcp_vlans_update(data) - return response - - def _get_fabric_from_cidr(self, cidr): -@@ -248,6 +250,32 @@ - else: - self._maas.post(u'api/2.0/ipranges/', None, **data) - -+ def _process_dhcp_vlans_update(self, subnet_data): -+ fabric_vlans = json.loads(self._maas.get(u'api/2.0/fabrics/{0}/vlans/' -+ .format(subnet_data['fabric'])).read()) -+ LOG.warn('all fabric %s vlans %s', subnet_data['fabric'], fabric_vlans) -+ for vlan_name, vlan_data in self._vlans.iteritems(): -+ update = False -+ old_data = None -+ for fabric_vlan in fabric_vlans: -+ if fabric_vlan['vid'] == vlan_data['vid']: -+ update = True -+ old_data = fabric_vlan -+ break -+ data = { -+ 'mtu': str(vlan_data.get('mtu', 1500)), -+ 'dhcp_on': str(vlan_data.get('dhcp_on')), -+ 'primary_rack': str(vlan_data.get('primary_rack')), -+ 'secondary_rack': str(vlan_data.get('secondary_rack', '')) -+ } -+ if update: -+ LOG.warn('UPDATING %s %s', data, old_data) -+ self._maas.put(u'api/2.0/fabrics/{0}/vlans/{1}/' -+ .format(subnet_data['fabric'], old_data['vid']), -+ **data) -+ else: -+ LOG.warn('MISSING vlan %s, not doing anything', data) -+ - - class DHCPSnippet(MaasObject): - def __init__(self): diff --git a/mcp/patches/0007-network.interface-Fix-ifup-OVS-port-with-route.patch b/mcp/patches/0007-network.interface-Fix-ifup-OVS-port-with-route.patch deleted file mode 100644 index 2a5e29e9e..000000000 --- a/mcp/patches/0007-network.interface-Fix-ifup-OVS-port-with-route.patch +++ /dev/null @@ -1,39 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Sun, 31 Dec 2017 16:06:30 +0100 -Subject: [PATCH] network.interface: Fix ifup OVS port with route - -This change attempts to fix the following sequence for nodes with -OVS ports that also have routes defined: -- state.sls linux.network; system.reboot; state.sls linux.network; - -If an OVS port also has a route defined, a route cfg file is -created in </etc/network/if.{down,up}.d/>, which is sourced after -system reboot before we (eventually) re-run `interface.sls`, leaving -the OVS port in UP state, so `ifup` would fail. - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - linux/network/interface.sls | 2 ++ - 1 file changed, 2 insertions(+) - -diff --git a/linux/network/interface.sls b/linux/network/interface.sls -index 921ceac..3bba64e 100644 ---- a/linux/network/interface.sls -+++ b/linux/network/interface.sls -@@ -146,6 +146,8 @@ ovs_port_up_{{ interface_name }}: - - file: ovs_port_{{ interface_name }}_line2 - - openvswitch_bridge: ovs_bridge_{{ interface.bridge }} - - file: linux_interfaces_final_include -+ - unless: -+ - ip link show {{ interface_name }} | grep -q '\<UP\>' - - {%- endif %} - diff --git a/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch b/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch deleted file mode 100644 index 48e945acd..000000000 --- a/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch +++ /dev/null @@ -1,122 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Mon, 22 Jan 2018 00:28:09 +0100 -Subject: [PATCH] system.repo: Debian: Add keyserver proxy support - -Introduce a new, optional set of parameters to configure the proxy -used for key fetching / keyserver access under: -linux:system:proxy:keyserver:http(s). - -Previously, when fetching GPG keys for APT keyring, either using -public key download & import (as for default repos) or via keyserver, -we relied on simple `curl` calls or passed it down to Salt aptpkg -module. -To be able to retrieve APT keys behind a proxy, one used to have to -configure the proxy for the Salt minion, which does not yet have -`no_proxy` support (either *all* or *no* traffic hits the proxy). - -When the new http(s) proxy param is set: -- no longer pass key configuration to Salt aptpkg (until it properly - supports `no_proxy`); -- handle all keys explicitly with `curl` and `apt-key`; -- set 'http(s)_proxy' env vars for `cmd.run`/`cmd.wait` calls; - -If linux:system:proxy:keyserver is not defined, the behavior is -unchanged for backwards compatibility. - -To allow runtime decisions whether the keyserver proxy should be used -add an additional condition for it to match the first nameserver. -This allows us to mix virtual nodes with MaaS-provisioned nodes in -Fuel@OPNFV, while keeping the ext_pillar common. - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - README.rst | 16 ++++++++++++++++ - linux/system/repo.sls | 40 ++++++++++++++++++++++++++++++++++++++++ - 2 files changed, 56 insertions(+) - -diff --git a/linux/system/repo.sls b/linux/system/repo.sls ---- a/linux/system/repo.sls -+++ b/linux/system/repo.sls -@@ -96,13 +96,50 @@ linux_repo_{{ name }}_key: - - name: "curl -s {{ repo.key_url }} | apt-key add -" - - watch: - - file: default_repo_list -+{%- if system.proxy.keyserver is defined and grains['dns']['nameservers'][0] in system.proxy.keyserver.http %} -+ - env: -+ - http_proxy: {{ system.proxy.get('keyserver', {}).get('http', '') }} -+ - https_proxy: {{ system.proxy.get('keyserver', {}).get('https', '') }} -+{%- endif %} - - {%- endif %} - -+{#- repo.default is false #} - {%- else %} - - {%- if repo.get('enabled', True) %} - -+{%- if system.proxy.keyserver is defined and grains['dns']['nameservers'][0] in system.proxy.keyserver.http %} -+ -+{%- if repo.get('key') %} -+ -+linux_repo_{{ name }}_key: -+ cmd.run: -+ - name: "echo '{{ repo.key }}' | apt-key add -" -+ -+{%- elif repo.key_url|default(False) %} -+ -+linux_repo_{{ name }}_key: -+ cmd.run: -+ - name: "curl -s {{ repo.key_url }} | apt-key add -" -+ - env: -+ - http_proxy: {{ system.proxy.get('keyserver', {}).get('http', '') }} -+ - https_proxy: {{ system.proxy.get('keyserver', {}).get('https', '') }} -+ -+{%- elif repo.key_id is defined and repo.key_server is defined %} -+ -+linux_repo_{{ name }}_key: -+ cmd.run: -+ - name: "apt-key adv --keyserver {{ repo.key_server }} --recv {{ repo.key_id }}" -+ - env: -+ - http_proxy: {{ system.proxy.get('keyserver', {}).get('http', '') }} -+ - https_proxy: {{ system.proxy.get('keyserver', {}).get('https', '') }} -+ -+{%- endif %} -+ -+{#- system.proxy.keyserver #} -+{%- endif %} -+ - linux_repo_{{ name }}: - pkgrepo.managed: - {%- if repo.ppa is defined %} -@@ -115,6 +152,7 @@ linux_repo_{{ name }}: - {%- endif %} - - file: /etc/apt/sources.list.d/{{ name }}.list - - clean_file: {{ repo.clean|default(True) }} -+ {%- if system.proxy.keyserver is not defined or grains['dns']['nameservers'][0] not in system.proxy.keyserver.http %} - {%- if repo.key_id is defined %} - - keyid: {{ repo.key_id }} - {%- endif %} -@@ -124,6 +162,7 @@ linux_repo_{{ name }}: - {%- if repo.key_url is defined %} - - key_url: {{ repo.key_url }} - {%- endif %} -+ {%- endif %} - - consolidate: {{ repo.get('consolidate', False) }} - - clean_file: {{ repo.get('clean_file', False) }} - - refresh_db: {{ repo.get('refresh_db', True) }} -@@ -140,6 +179,7 @@ linux_repo_{{ name }}: - {%- endif %} - {%- endif %} - -+{#- repo.enabled is false #} - {%- else %} - - linux_repo_{{ name }}_absent: diff --git a/mcp/patches/0012-routes-Skip-network-restart-on-noifupdown.patch b/mcp/patches/0012-routes-Skip-network-restart-on-noifupdown.patch deleted file mode 100644 index fb42512ea..000000000 --- a/mcp/patches/0012-routes-Skip-network-restart-on-noifupdown.patch +++ /dev/null @@ -1,37 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Mon, 8 Jan 2018 05:09:11 +0100 -Subject: [PATCH] routes: Skip network restart on 'noifupdown' - -Previously, setting up routes did not allow passing 'require_reboot', -so each route change would lead to a networking service restart, -rendering interface configuration options like 'noifupdown' useless. -Allow disabling network restart per-interface using the existing -'noifupdown' option. - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - linux/network/interface.sls | 3 +++ - 1 file changed, 3 insertions(+) - -diff --git a/linux/network/interface.sls b/linux/network/interface.sls -index 921ceac..6ebc670 100644 ---- a/linux/network/interface.sls -+++ b/linux/network/interface.sls -@@ -338,6 +338,9 @@ linux_network_{{ interface_name }}_routes: - gateway: {{ route.gateway }} - {%- endif %} - {%- endfor %} -+ {%- if interface.noifupdown is defined %} -+ - require_reboot: {{ interface.noifupdown }} -+ {%- endif %} - - {%- endif %} - diff --git a/mcp/patches/0014-reclass.storage.node-Merge-duplicate-nodes.patch b/mcp/patches/0014-reclass.storage.node-Merge-duplicate-nodes.patch deleted file mode 100644 index a9d5b6589..000000000 --- a/mcp/patches/0014-reclass.storage.node-Merge-duplicate-nodes.patch +++ /dev/null @@ -1,44 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Sun, 03 Dec 2017 22:03:01 +0200 -Subject: [PATCH] reclass.storage.node: Merge duplicate nodes - -Reclass does not support duplicate nodes in top pillar, so merge all -nodes with the same name into a single node, inheriting classes from -all instances. - -This allows using multiple "system.reclass.storage.system.*_cluster" -classes for the same node, based on re-using the name (hostname). - -NOTE: defaults.merge module does not merge lists (e.g. for classes), -so handle that case separately. - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - -diff --git a/reclass/storage/node.sls b/reclass/storage/node.sls ---- a/reclass/storage/node.sls -+++ b/reclass/storage/node.sls -@@ -1,7 +1,15 @@ - {%- from "reclass/map.jinja" import storage with context %} - {%- if storage.enabled %} - -+{%- set storage_by_name = {} %} - {%- for node_name, node in storage.get('node', {}).iteritems() %} -+{%- set new_node_name = node.get('name', node_name) %} -+{%- set new_node = storage_by_name.get(new_node_name, {'classes': []}) %} -+{%- do salt['defaults.merge'](node, {'classes': new_node.classes + node.classes}) %} -+{%- do salt['defaults.merge'](storage_by_name, {new_node_name: node}) %} -+{%- endfor %} -+ -+{%- for node_name, node in storage_by_name.iteritems() %} - - {%- if node.repeat is defined %} - diff --git a/mcp/patches/Makefile b/mcp/patches/Makefile index be3eb9ee5..e0a1c34ce 100644 --- a/mcp/patches/Makefile +++ b/mcp/patches/Makefile @@ -25,7 +25,7 @@ FPATCHES = $(shell find ${F_PATCH_DIR} -name '*.patch') # In order to keep things sort of separate, we should only pass up (to main # Makefile) the fully-patched repos, and gather any fingerprinting info here. -# Fuel@OPNFV relies on upstream git repos (one per component) in 1 of 2 ways: +# OPNFV Fuel relies on upstream git repos (one per component) in 1 of 2 ways: # - pinned down to tag objects (e.g. "9.0.1") # - tracking upstream remote HEAD on a stable or master branch # FIXME(alav): Should we support mixed cases? (e.g. pin down only fuel-main) diff --git a/mcp/patches/README.rst b/mcp/patches/README.rst index 735b70341..28af0e8f7 100644 --- a/mcp/patches/README.rst +++ b/mcp/patches/README.rst @@ -1,30 +1,30 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. SPDX-License-Identifier: CC-BY-4.0 -.. (c) 2017 Mirantis Inc., Enea AB and others. +.. (c) 2018 Mirantis Inc., Enea AB and others. ========================================== -Fuel@OPNFV submodule fetching and patching +OPNFV Fuel Submodule Fetching and Patching ========================================== This directory holds submodule fetching/patching scripts, intended for -working with upstream Fuel/MCP components (e.g.: reclass-system-salt-model) in -developing/applying OPNFV patches (backports, custom fixes etc.). +working with upstream Fuel/MCP components (e.g.: ``reclass-system-salt-model``) +in developing/applying OPNFV patches (backports, custom fixes etc.). The scripts should be friendly to the following 2 use-cases: - - development work: easily cloning, binding repos to specific commits, - remote tracking, patch development etc.; - - to provide parent build scripts an easy method of tracking upstream - references and applying OPNFV patches on top; +- development work: easily cloning, binding repos to specific commits, + remote tracking, patch development etc.; +- to provide parent build scripts an easy method of tracking upstream + references and applying OPNFV patches on top; Also, we need to support at least the following modes of operations: - - submodule bind - each submodule patches will be based on the commit ID - saved in the .gitmodules config file; - - remote tracking - each submodule will sync with the upstream remote - and patches will be applied on top of <sub_remote>/<sub_branch>/HEAD; +- submodule bind - each submodule patches will be based on the commit ID + saved in the ``.gitmodules`` config file; +- remote tracking - each submodule will sync with the upstream remote + and patches will be applied on top of ``<sub_remote>/<sub_branch>/HEAD``; -Workflow (development) +Workflow (Development) ====================== The standard development workflow should look as follows: @@ -32,114 +32,116 @@ The standard development workflow should look as follows: Decide whether remote tracking should be active or not ------------------------------------------------------ -NOTE: Setting the following var to any non-empty str enables remote track. +.. NOTE:: -NOTE: Leaving unset will enable remote track for anything but stable branch. + Setting the following var to any non-empty str enables remote track. - .. code-block:: bash +.. code-block:: console - $ export FUEL_TRACK_REMOTES="" + developer@machine:~/fuel$ export FUEL_TRACK_REMOTES="" Initialize git submodules ------------------------- -All Fuel sub-projects are registered as submodules. +All Fuel direct dependency projects are registered as submodules. If remote tracking is active, upstream remote is queried and latest remote -branch HEAD is fetched. Otherwise, checkout commit IDs from .gitmodules. +branch ``HEAD`` is fetched. Otherwise, checkout commit IDs from ``.gitmodules``. - .. code-block:: bash +.. code-block:: console - $ make sub + developer@machine:~/fuel$ make -C mcp/patches sub -Apply patches from `patches/<sub-project>/*` to respective submodules ---------------------------------------------------------------------- +Apply patches from ``patches/<sub-project>/*`` to respective submodules +----------------------------------------------------------------------- This will result in creation of: -- a tag called `${FUEL_MAIN_TAG}-opnfv-root` at the same commit as Fuel@OPNFV - upstream reference (bound to git submodule OR tracking remote HEAD); -- a new branch `opnfv-fuel` which will hold all the OPNFV patches, - each patch is applied on this new branch with `git-am`; -- a tag called `${FUEL_MAIN_TAG}-opnfv` at `opnfv-fuel/HEAD`; +- a tag called ``${F_OPNFV_TAG}-root`` at the same commit as OPNFV Fuel + upstream reference (bound to git submodule OR tracking remote ``HEAD``); +- a new branch ``nightly`` which will hold all the OPNFV patches, + each patch is applied on this new branch with ``git-am``; +- a tag called ``${F_OPNFV_TAG}`` at ``nightly/HEAD``; +- for each (sub)directory of ``patches/<sub-project>``, another pair of tags + ``${F_OPNFV_TAG}-<sub-directory>-fuel/patch-root`` and + ``${F_OPNFV_TAG}-<sub-directory>-fuel/patch`` are also created; - .. code-block:: bash +.. code-block:: console - $ make patches-import + developer@machine:~/fuel$ make -C mcp/patches patches-import Modify sub-projects for whatever you need ----------------------------------------- -Commit your changes when you want them taken into account in the build. +To add/change OPNFV-specific patches for a sub-project: -Re-create patches +- commit your changes inside the git submodule(s); +- move the git tag to the new reference so ``make patches-export`` will + pick up the new commit later; + +.. code-block:: console + + developer@machine:~/fuel$ cd ./path/to/submodule + developer@machine:~/fuel/path/to/submodule$ # ... + developer@machine:~/fuel/path/to/submodule$ git commit + developer@machine:~/fuel/path/to/submodule$ git tag -f ${F_OPNFV_TAG}-fuel/patch + +Re-create Patches ----------------- -Each commit on `opnfv-fuel` branch of each subproject will be -exported to `patches/subproject/` via `git format-patch`. +Each commit on ``nightly`` branch of each subproject will be +exported to ``patches/subproject/`` via ``git format-patch``. + +.. NOTE:: + + Only commit submodule file changes when you need to bump upstream refs. -NOTE: Only commit (-f) submodules when you need to bump upstream ref. +.. WARNING:: -NOTE: DO NOT commit patched submodules! + DO NOT commit patched submodules! - .. code-block:: bash +.. code-block:: console - $ make patches-export + developer@machine:~/fuel$ make -C mcp/patches patches-export patches-copyright -Clean workbench branches and tags +Clean Workbench Branches and Tags --------------------------------- - .. code-block:: bash +.. code-block:: console - $ make clean + developer@machine:~/fuel$ make -C mcp/patches clean -De-initialize submodules and force a clean clone +De-initialize Submodules and Force a Clean Clone ------------------------------------------------ - .. code-block:: bash +.. code-block:: console - $ make deepclean + developer@machine:~/fuel$ make -C mcp/patches deepclean -Sub-project maintenance +Sub-project Maintenance ======================= -Adding a new submodule +Adding a New Submodule ---------------------- -If you need to add another subproject, you can do it with `git submodule`. -Make sure that you specify branch (with `-b`), short name (with `--name`): - - .. code-block:: bash - - $ git submodule -b master add --name reclass-system-salt-model - https://github.com/Mirantis/reclass-system-salt-model - relative/path/to/submodule - -Working with remote tracking for upgrading Fuel components ----------------------------------------------------------- - -Enable remote tracking as described above, which at `make sub` will update -ALL submodules (e.g. reclass-system-salt-model) to remote branch (set in -.gitmodules) HEAD. +If you need to add another subproject, you can do it with ``git submodule``. +Make sure that you specify branch (with ``-b``), short name (with ``--name``): -* If upstream has NOT already tagged a new version, we can still work on - our patches, make sure they apply etc., then check for new upstream - changes (and that our patches still apply on top of them) by: +.. code-block:: console -* If upstream has already tagged a new version we want to pick up, checkout - the new tag in each submodule: + developer@machine:~/fuel$ git submodule -b master add --name reclass-system-salt-model \ + https://github.com/Mirantis/reclass-system-salt-model \ + mcp/reclass/classes/system -* Once satisfied with the patch and submodule changes, commit them: +Working with Remote Tracking +---------------------------- - - enforce FUEL_TRACK_REMOTES to "yes" if you want to constatly use the - latest remote branch HEAD (as soon as upstream pushes a change on that - branch, our next build will automatically include it - risk of our - patches colliding with new upstream changes); - - stage patch changes if any; - - if submodule tags have been updated (relevant when remote tracking is - disabled, i.e. we have a stable upstream baseline), add submodules; +Enable remote tracking as described above, which at ``make sub`` will update +ALL submodules (e.g. ``reclass-system-salt-model``) to remote branch (set in +``.gitmodules``) ``HEAD``. - .. code-block:: bash +.. WARNING:: - $ make deepclean patches-import - $ git submodule foreach 'git checkout <newtag>' - $ make deepclean sub && git add -f relative/path/to/submodule + Enforce ``FUEL_TRACK_REMOTES`` to ``yes`` only if you want to constatly + use the latest remote branch ``HEAD`` (as soon as upstream pushes a change + on that branch, our next build will automatically include it - risk of our + patches colliding with new upstream changes) - for **ALL** submodules. diff --git a/mcp/patches/config.mk b/mcp/patches/config.mk index 260cbf829..5e5d3b327 100644 --- a/mcp/patches/config.mk +++ b/mcp/patches/config.mk @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2015,2016,2017 Ericsson AB, Enea AB and others. +# Copyright (c) 2018 Ericsson AB, Enea AB and others. # stefan.k.berg@ericsson.com # jonas.bjurel@ericsson.com # All rights reserved. This program and the accompanying materials @@ -18,6 +18,5 @@ F_GIT_DIR := $(shell git rev-parse --git-dir) F_PATCH_DIR := $(shell pwd) F_OPNFV_TAG := master-opnfv -# for the patches applying purposes (empty git config in docker build container) export GIT_COMMITTER_NAME?=Fuel OPNFV export GIT_COMMITTER_EMAIL?=fuel@opnfv.org diff --git a/mcp/patches/docker/0001-tasks.py-Allow-passing-extra-build-arg.patch b/mcp/patches/docker/0001-tasks.py-Allow-passing-extra-build-arg.patch new file mode 100644 index 000000000..9abbfb0b2 --- /dev/null +++ b/mcp/patches/docker/0001-tasks.py-Allow-passing-extra-build-arg.patch @@ -0,0 +1,59 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Sun, 9 Sep 2018 23:18:39 +0200 +Subject: [PATCH] tasks.py: Allow passing extra --build-arg + +E.g. A, B with values "a", respectively "b c": +$ invoke build saltmaster-reclass --build-arg-extra='A="a" B="b c"' +will be passed down as: +--build-arg A="a" --build-arg B="b c" + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + tasks.py | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/tasks.py b/tasks.py +index d6bf3bc..59e217d 100644 +--- a/tasks.py ++++ b/tasks.py +@@ -3,6 +3,7 @@ + + + from invoke import Collection, task ++from shlex import split + from string import Template + import re + import ast +@@ -25,7 +26,7 @@ def all(ctx, dry=False, push=False, dry_targets=False, filter=None, **kwargs): + dry=dry, push=push, dry_targets=dry_targets, filter=filter, **kwargs) + + @task +-def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, push=False, dry=False, dry_targets=False, **kwargs): ++def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, push=False, dry=False, dry_targets=False, build_arg_extra='', **kwargs): + + kwargs['dist'] = dist + kwargs['dist_rel'] = dist_rel +@@ -35,6 +36,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, + kwargs['require'] = require + kwargs['salt'] = salt + kwargs['target'] = target ++ kwargs['build_arg_extra'] = ' --build-arg '.join([''] + split(build_arg_extra.replace('"', '"\\"'))) + # command formating + update + fmt = {'tag': ''} + fmt.update(ctx.dockermake) +@@ -60,6 +62,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, + \t--requires ${requires} \ + \t--build-arg SALT_VERSION="${salt}" \ + \t--build-arg SALT_FORMULA_VERSION="${formula_rev}" \ ++ \t${build_arg_extra} \ + \t${push} ${options} \ + ${fin}""").safe_substitute(fmt) + ctx.run(cmd.replace(' ', '')) diff --git a/mcp/patches/docker/0002-OPNFV-tag-convention-alignment.patch b/mcp/patches/docker/0002-OPNFV-tag-convention-alignment.patch new file mode 100644 index 000000000..683dc4600 --- /dev/null +++ b/mcp/patches/docker/0002-OPNFV-tag-convention-alignment.patch @@ -0,0 +1,78 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Tue, 21 Aug 2018 02:42:35 +0200 +Subject: [PATCH] OPNFV tag convention alignment + +* (arch)- prefix Docker tags + Hacky implementation that only detects 'arm64', otherwise it will + default to 'amd64'. + Note that Docker arch notation uses 'arm64' instead of 'aarch64'. +* Override Docker tag to align with OPNFV format +* use Docker repository: opnfv/fuel + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + invoke.yml | 2 +- + tasks.py | 7 +++++-- + 2 files changed, 6 insertions(+), 3 deletions(-) + +diff --git a/invoke.yml b/invoke.yml +index 0bd793e..291f4c7 100644 +--- a/invoke.yml ++++ b/invoke.yml +@@ -65,7 +65,7 @@ target: + + dockermake: + destination: 'images' +- repository: 'docker.io/epcim/salt' ++ repository: 'docker.io/opnfv/fuel' + # options: '--no-cache' + # options: '--bust-cache reclass' + # options: '--registry-user epcim' +diff --git a/tasks.py b/tasks.py +index 59e217d..8afbde9 100644 +--- a/tasks.py ++++ b/tasks.py +@@ -7,6 +7,7 @@ from shlex import split + from string import Template + import re + import ast ++import platform + + # TODOs: + # - WIP - use namespace to expose images/targets as inoke tasks # image = Collection('image') +@@ -26,8 +27,9 @@ def all(ctx, dry=False, push=False, dry_targets=False, filter=None, **kwargs): + dry=dry, push=push, dry_targets=dry_targets, filter=filter, **kwargs) + + @task +-def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, push=False, dry=False, dry_targets=False, build_arg_extra='', **kwargs): ++def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, formula_rev=None, opnfv_tag='latest', push=False, dry=False, dry_targets=False, build_arg_extra='', **kwargs): + ++ kwargs['arch'] = 'arm64' if platform.machine() == 'aarch64' else 'amd64' + kwargs['dist'] = dist + kwargs['dist_rel'] = dist_rel + kwargs['dry'] = True if dry_targets or dry else False +@@ -37,6 +39,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, + kwargs['salt'] = salt + kwargs['target'] = target + kwargs['build_arg_extra'] = ' --build-arg '.join([''] + split(build_arg_extra.replace('"', '"\\"'))) ++ kwargs['opnfv_tag'] = opnfv_tag + # command formating + update + fmt = {'tag': ''} + fmt.update(ctx.dockermake) +@@ -58,7 +61,7 @@ def build(ctx, target, require=[], dist='debian', dist_rel='stretch', salt=None, + # execute + cmd = Template(""" + ${dry}docker-make -f DockerMake.${dist}.yml -u ${repository}: --name ${target} \ +- \t-t ${dist}-${dist_rel}${tag} \ ++ \t-t ${arch}-${opnfv_tag} \ + \t--requires ${requires} \ + \t--build-arg SALT_VERSION="${salt}" \ + \t--build-arg SALT_FORMULA_VERSION="${formula_rev}" \ diff --git a/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch new file mode 100644 index 000000000..27df86e8d --- /dev/null +++ b/mcp/patches/docker/0003-OPNFV-package-installation-Ubuntu-user.patch @@ -0,0 +1,112 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Wed, 22 Aug 2018 01:35:06 +0200 +Subject: [PATCH] OPNFV package installation, Ubuntu user + +* Install OpenSSH server (and client), so other OPNFV projects can + easily connect to the Salt master node; +* Install 'salt-formula-gnocchi' distro package (empty git formula + workaround); +* While at it, create 'ubuntu' user so other OPNFV projects don't + have to switch to 'root' login; +* Preinstall `salt_minion_dependency_packages` and + `salt_minion_reclass_dependencies`; +* Pin dockermake to v0.8 to allow using python3.5 for virtualenv, + since python 3.6 is not easily available for Ubuntu Xenial + jump/build hosts. +* Pin jinja2 to 2.11.0 to bypass [1]. + +[1] https://github.com/saltstack/salt/issues/46594 + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + DockerMake.yml | 39 ++++++++++++++++++++++++++++++++++++++- + Pipfile | 4 ++-- + 2 files changed, 40 insertions(+), 3 deletions(-) + +diff --git a/DockerMake.yml b/DockerMake.yml +index 2c75586..f30024c 100644 +--- a/DockerMake.yml ++++ b/DockerMake.yml +@@ -29,6 +29,7 @@ common-cleanup: + + common: + build: | ++ ARG CACHE_INVALIDATE="0" + RUN echo "Layer with common packages" \ + && eval ${LAYER_PKGUPDT} \ + && apt-get upgrade -qy \ +@@ -94,7 +95,7 @@ salt-formulas: + build: | + ARG SALT_FORMULA_VERSION="master" + ENV SALT_FORMULA_VERSION $SALT_FORMULA_VERSION +- ARG SALT_FORMULA_SOURCES="https://github.com/salt-formulas https://github.com/saltstack-formulas" ++ ARG SALT_FORMULA_SOURCES="https://github.com/salt-formulas" + ENV SALT_FORMULA_SOURCES $SALT_FORMULA_SOURCES + ARG SALT_FORMULAS_BASE="/srv/salt/formula" + ENV SALT_FORMULAS_BASE $SALT_FORMULAS_BASE +@@ -108,6 +109,42 @@ salt-formulas: + && bash -c 'source /srv/salt/formula-fetch.sh && setupPyEnv && fetchAll' \ + && eval ${LAYER_CLEANUP} + ++opnfv: ++ build: | ++ # Some formulas have empty git repos, use the package version ++ ARG SALT_FORMULA_VERSION="nightly" ++ ENV SALT_FORMULA_VERSION $SALT_FORMULA_VERSION ++ ARG SALT_FORMULA_LIST="salt-formula-gnocchi" ++ ENV SALT_FORMULA_LIST $SALT_FORMULA_LIST ++ ENV APT_REPOSITORY "deb [arch=amd64] http://apt.mirantis.com/xenial ${SALT_FORMULA_VERSION} salt" ++ ENV APT_REPOSITORY_GPG "http://apt.mirantis.com/public.gpg" ++ RUN echo "Layer extra salt-formulas packages" \ ++ && echo "$APT_REPOSITORY" | tee /etc/apt/sources.list.d/salt-formulas.list >/dev/null \ ++ && curl -sL $APT_REPOSITORY_GPG | $SUDO apt-key add - \ ++ && eval ${LAYER_PKGUPDT} \ ++ && ${LAYER_INSTALL} ${SALT_FORMULA_LIST} -y --fix-missing -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \ ++ && eval ${LAYER_CLEANUP} ++ RUN echo "Layer with OPNFV packages" \ ++ && eval ${LAYER_PKGUPDT} \ ++ && ${LAYER_INSTALL} \ ++ gawk \ ++ inetutils-ping \ ++ kmod \ ++ net-tools \ ++ openssh-server \ ++ python-futures \ ++ python-m2crypto \ ++ python-msgpack \ ++ python-netaddr \ ++ python-oauth \ ++ python-psutil \ ++ python-yaml \ ++ && useradd -m ubuntu \ ++ && echo 'ubuntu ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ubuntu \ ++ && python -m pip install -U jinja2==2.11.0 \ ++ && eval ${LAYER_CLEANUP} ++ ++ + wheel: + requires: + - base +diff --git a/Pipfile b/Pipfile +index d3e8d66..340e125 100644 +--- a/Pipfile ++++ b/Pipfile +@@ -7,8 +7,8 @@ name = "pypi" + + [packages] + pygithub = "*" +-docker-make = {git = "https://github.com/avirshup/DockerMake"} +-dockermake = {git = "https://github.com/avirshup/DockerMake"} ++docker-make = {git = "https://github.com/avirshup/DockerMake", ref = "aeac230fd5ab468d806bf42b120aa97f91eb40a2"} ++dockermake = {git = "https://github.com/avirshup/DockerMake", ref = "aeac230fd5ab468d806bf42b120aa97f91eb40a2"} + + [requires] + python_version = "3.6" diff --git a/mcp/patches/docker/0004-reclass-Set-ignore_overwritten_missing_references.patch b/mcp/patches/docker/0004-reclass-Set-ignore_overwritten_missing_references.patch new file mode 100644 index 000000000..0af1e6967 --- /dev/null +++ b/mcp/patches/docker/0004-reclass-Set-ignore_overwritten_missing_references.patch @@ -0,0 +1,34 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Tue, 20 Nov 2018 23:16:44 +0200 +Subject: [PATCH] reclass: Set ignore_overwritten_missing_references + +The recent changes in `reclass` 1.6.x allow configuring this new +option; however the default for it is broken - see [1]. + +[1] https://github.com/salt-formulas/reclass/issues/77 + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + files/reclass/reclass-config.yml | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/files/reclass/reclass-config.yml b/files/reclass/reclass-config.yml +index 00aba49..f4da04c 100644 +--- a/files/reclass/reclass-config.yml ++++ b/files/reclass/reclass-config.yml +@@ -3,6 +3,7 @@ inventory_base_uri: /srv/salt/reclass + pretty_print: True + output: yaml + ++ignore_overwritten_missing_references: True + ignore_class_notfound: True + ignore_class_regexp: + - 'service.*' diff --git a/mcp/patches/docker/0005-Add-saltminion-maas-build-target.patch b/mcp/patches/docker/0005-Add-saltminion-maas-build-target.patch new file mode 100644 index 000000000..889fdcd43 --- /dev/null +++ b/mcp/patches/docker/0005-Add-saltminion-maas-build-target.patch @@ -0,0 +1,108 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Wed, 30 Jan 2019 17:21:03 +0100 +Subject: [PATCH] Add saltminion-maas build target + +--- + DockerMake.yml | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++ + invoke.yml | 5 ++++ + 2 files changed, 69 insertions(+) + +diff --git a/DockerMake.yml b/DockerMake.yml +index f30024c..a23541c 100644 +--- a/DockerMake.yml ++++ b/DockerMake.yml +@@ -50,6 +50,70 @@ common: + && eval ${LAYER_CLEANUP} + + ++maas: ++ requires: ++ - python ++ build: | ++ ENV container docker ++ ARG SALT_VERSION="stable" ++ ENV SALT_VERSION $SALT_VERSION ++ ARG SALT_BOOTSTRAP_OPTS="-PdX ${SALT_VERSION}" ++ ENV SALT_BOOTSTRAP_OPTS $SALT_BOOTSTRAP_OPTS ++ RUN echo "Layer with MaaS, salt-minion packages" \ ++ && eval ${LAYER_PKGUPDT} \ ++ && apt-get upgrade -qy \ ++ && ${LAYER_INSTALL} maas-common systemd \ ++ && find /etc/systemd/system \ ++ /lib/systemd/system \ ++ -path '*.wants/*' \ ++ -not -name '*journald*' \ ++ -not -name '*systemd-tmpfiles*' \ ++ -not -name '*systemd-user-sessions*' \ ++ -exec rm \{} \; \ ++ && systemctl set-default multi-user.target \ ++ && ${LAYER_INSTALL} \ ++ at \ ++ avahi-utils \ ++ dbconfig-pgsql \ ++ ipmitool \ ++ iptables \ ++ iptables-persistent \ ++ iputils-ping \ ++ kmod \ ++ lsb-release \ ++ maas-cli \ ++ maas-dns \ ++ maas-rack-controller \ ++ maas-region-api \ ++ postgresql \ ++ python-futures \ ++ python-m2crypto \ ++ python-msgpack \ ++ python-netaddr \ ++ python-oauth \ ++ python-pip \ ++ python-psutil \ ++ python-setuptools \ ++ python-yaml \ ++ sysfsutils \ ++ tcpdump \ ++ && apt-get download maas-region-controller \ ++ && dpkg-deb --extract maas-region-controller*.deb maas-region-controller \ ++ && dpkg-deb --control maas-region-controller*.deb maas-region-controller/DEBIAN \ ++ && mkdir -p /var/lib/opnfv/etc \ ++ && mv maas-region-controller/DEBIAN/postinst /var/lib/opnfv/maas-region-controller.postinst \ ++ && dpkg-deb --build maas-region-controller \ ++ && dpkg -i maas-region-controller.deb \ ++ && rm -rf maas* \ ++ && mv /var/lib/maas /var/lib/postgresql /var/lib/opnfv/ \ ++ && mv /etc/maas /etc/ssh /var/lib/opnfv/etc/ \ ++ && mv /var/lib/opnfv/maas-region-controller.postinst /var/lib/dpkg/info/ \ ++ && python -m pip install -U jinja2==2.11.0 \ ++ && curl -qL https://raw.githubusercontent.com/saltstack/salt-bootstrap/stable/bootstrap-salt.sh | $SUDO sh -s -- -A cfg01 ${SALT_BOOTSTRAP_OPTS} \ ++ && eval ${LAYER_CLEANUP} ++ ENTRYPOINT ["/bin/bash", "-c", "/entrypoint.sh && exec /lib/systemd/systemd"] ++ ++ + salt: + requires: + - python +diff --git a/invoke.yml b/invoke.yml +index 291f4c7..a2f4614 100644 +--- a/invoke.yml ++++ b/invoke.yml +@@ -62,6 +62,11 @@ target: + # - salt + # require: + # - saltclass ++ saltminion-maas: ++ matrix: ++ - dist ++ require: ++ - maas + + dockermake: + destination: 'images' diff --git a/mcp/patches/docker/0006-Use-archive.repo.saltstack.com-repos.patch b/mcp/patches/docker/0006-Use-archive.repo.saltstack.com-repos.patch new file mode 100644 index 000000000..1889e3a9b --- /dev/null +++ b/mcp/patches/docker/0006-Use-archive.repo.saltstack.com-repos.patch @@ -0,0 +1,42 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Thu, 18 Jun 2020 16:33:35 +0200 +Subject: [PATCH] Use archive.repo.saltstack.com repos + +Saltstack packages for 2017.7 have been archives, use the appropiate +repository URL. + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + DockerMake.yml | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/DockerMake.yml b/DockerMake.yml +index a23541c..5910613 100644 +--- a/DockerMake.yml ++++ b/DockerMake.yml +@@ -109,7 +109,7 @@ maas: + && mv /etc/maas /etc/ssh /var/lib/opnfv/etc/ \ + && mv /var/lib/opnfv/maas-region-controller.postinst /var/lib/dpkg/info/ \ + && python -m pip install -U jinja2==2.11.0 \ +- && curl -qL https://raw.githubusercontent.com/saltstack/salt-bootstrap/stable/bootstrap-salt.sh | $SUDO sh -s -- -A cfg01 ${SALT_BOOTSTRAP_OPTS} \ ++ && curl -qL https://raw.githubusercontent.com/saltstack/salt-bootstrap/stable/bootstrap-salt.sh | $SUDO sh -s -- -A cfg01 -R archive.repo.saltstack.com ${SALT_BOOTSTRAP_OPTS} \ + && eval ${LAYER_CLEANUP} + ENTRYPOINT ["/bin/bash", "-c", "/entrypoint.sh && exec /lib/systemd/systemd"] + +@@ -127,7 +127,7 @@ salt: + RUN echo "Layer salt" \ + && eval ${LAYER_PKGUPDT} \ + && mkdir -p /var/run/salt /var/cache/salt /var/log/salt /etc/salt/pki/master/minions /srv/salt/formula /srv/salt/env \ +- && curl -qL https://raw.githubusercontent.com/saltstack/salt-bootstrap/stable/bootstrap-salt.sh | $SUDO sh -s -- ${SALT_BOOTSTRAP_PKGS} -M -A localhost ${SALT_BOOTSTRAP_OPTS} \ ++ && curl -qL https://raw.githubusercontent.com/saltstack/salt-bootstrap/stable/bootstrap-salt.sh | $SUDO sh -s -- ${SALT_BOOTSTRAP_PKGS} -R archive.repo.saltstack.com -M -A localhost ${SALT_BOOTSTRAP_OPTS} \ + && if ! getent passwd salt > /dev/null;then useradd --system salt; fi \ + && chown -R salt:salt /etc/salt /var/cache/salt /var/log/salt /var/run/salt \ + && eval ${LAYER_CLEANUP} diff --git a/mcp/patches/fuel-patch-copyright.template b/mcp/patches/fuel-patch-copyright.template index e92458082..d50bdbe8a 100644 --- a/mcp/patches/fuel-patch-copyright.template +++ b/mcp/patches/fuel-patch-copyright.template @@ -1,5 +1,5 @@ :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. : : All rights reserved. This program and the accompanying materials : are made available under the terms of the Apache License, Version 2.0 diff --git a/mcp/patches/patch.sh b/mcp/patches/patch.sh deleted file mode 100755 index bb48dcd07..000000000 --- a/mcp/patches/patch.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x - -if [ -r "$1" ]; then - while IFS=': ' read -r p_dest p_file; do - if ! patch --dry-run -Rd "${p_dest}" -r - -s -p1 < \ - "/root/fuel/mcp/patches/${p_file}" > /dev/null; then - patch -d "${p_dest}" -p1 < "/root/fuel/mcp/patches/${p_file}" - fi - done < <(grep -vE '^#' "${1}" | grep -E "^.*${2}.*: ") -fi diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list deleted file mode 100644 index dfb0df773..000000000 --- a/mcp/patches/patches.list +++ /dev/null @@ -1,18 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -/usr/share/salt-formulas/env: 0002-maas-region-skip-credentials-update.patch -/usr/share/salt-formulas/env: 0003-maas-region-force-artifact-download.patch -/usr/share/salt-formulas/env: 0004-network.dpdk-Move-ifcfg-br-prv-to-interfaces.u.patch -/usr/share/salt-formulas/env: 0005-maas-module-Obtain-fabric-ID-from-CIDR.patch -/usr/share/salt-formulas/env: 0006-maas-module-Add-VLAN-DHCP-enable-support.patch -/usr/share/salt-formulas/env: 0007-network.interface-Fix-ifup-OVS-port-with-route.patch -/usr/share/salt-formulas/env: 0008-Handle-extra-environment-variables.patch -/usr/share/salt-formulas/env: 0010-maas-region-allow-timeout-override.patch -/usr/share/salt-formulas/env: 0011-system.repo-Debian-Add-keyserver-proxy-support.patch -/usr/share/salt-formulas/env: 0012-routes-Skip-network-restart-on-noifupdown.patch -/usr/share/salt-formulas/env: 0015-Set-ovs-bridges-as-L3-interfaces.patch diff --git a/mcp/patches/patches_init.list b/mcp/patches/patches_init.list deleted file mode 100644 index 94488cf9a..000000000 --- a/mcp/patches/patches_init.list +++ /dev/null @@ -1,8 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -/usr/share/salt-formulas/env: 0014-reclass.storage.node-Merge-duplicate-nodes.patch diff --git a/mcp/patches/reclass-system-salt-model/0001-Use-keystone-v3-endpoints-by-default.patch b/mcp/patches/reclass-system-salt-model/0001-Use-keystone-v3-endpoints-by-default.patch index 514ea59e4..f003c2031 100644 --- a/mcp/patches/reclass-system-salt-model/0001-Use-keystone-v3-endpoints-by-default.patch +++ b/mcp/patches/reclass-system-salt-model/0001-Use-keystone-v3-endpoints-by-default.patch @@ -20,10 +20,10 @@ Change-Id: I7e9a1b180f4e0ddb24ec72ed9f08c9e2580c7897 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keystone/client/single.yml b/keystone/client/single.yml -index a79ed7ec..86b4e09e 100644 +index b8ab7f1b..98cf359c 100644 --- a/keystone/client/single.yml +++ b/keystone/client/single.yml -@@ -4,7 +4,7 @@ classes: +@@ -5,7 +5,7 @@ classes: - system.keystone.client.service.glance - system.keystone.client.service.heat - system.keystone.client.service.heat-cfn diff --git a/mcp/patches/salt-formula-aodh/0001-Extend-apache-service-state.patch b/mcp/patches/salt-formula-aodh/0001-Extend-apache-service-state.patch new file mode 100644 index 000000000..9bf85efa0 --- /dev/null +++ b/mcp/patches/salt-formula-aodh/0001-Extend-apache-service-state.patch @@ -0,0 +1,47 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Michael Polenchuk <mpolenchuk@mirantis.com> +Date: Wed, 10 Jul 2019 10:18:58 +0400 +Subject: [PATCH] Extend apache service state + +diff --git a/aodh/server.sls b/aodh/server.sls +index 04b0395..258e5fe 100644 +--- a/aodh/server.sls ++++ b/aodh/server.sls +@@ -183,20 +183,16 @@ aodh_api_config: + + {%- endif %} + +-aodh_apache_restart: +- service.running: +- - enable: true +- - name: apache2 +- {%- if grains.get('noservices') %} +- - onlyif: /bin/false +- {%- endif %} +- - watch: +- - file: /etc/aodh/aodh.conf +- {%- if pillar.get('apache', {}).get('server', {}).get('site', {}).aodh is defined %} +- - apache_enable_aodh_wsgi +- {%- else %} +- - file: aodh_api_apache_config +- {%- endif %} ++extend: ++ apache_service: ++ service.running: ++ - watch: ++ - file: /etc/aodh/aodh.conf ++ {%- if pillar.get('apache', {}).get('server', {}).get('site', {}).aodh is defined %} ++ - apache_enable_aodh_wsgi ++ {%- else %} ++ - file: aodh_api_apache_config ++ {%- endif %} + + {%- endif %} + diff --git a/mcp/patches/salt-formula-cinder/0001-Support-stein-version.patch b/mcp/patches/salt-formula-cinder/0001-Support-stein-version.patch new file mode 100644 index 000000000..bae93ce6f --- /dev/null +++ b/mcp/patches/salt-formula-cinder/0001-Support-stein-version.patch @@ -0,0 +1,25 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Michael Polenchuk <mpolenchuk@mirantis.com> +Date: Tue, 9 Jul 2019 14:23:14 +0400 +Subject: [PATCH] Support stein version + +diff --git a/cinder/controller.sls b/cinder/controller.sls +index 31cb53c..f58e261 100644 +--- a/cinder/controller.sls ++++ b/cinder/controller.sls +@@ -72,7 +72,7 @@ cinder_controller_packages: + {%- endif %} + + {# Starting from ocata api running undder apache, so dedicated loggong.conf is not needed #} +-{%- if controller.version not in ('ocata','pike','queens', 'rocky') %} ++{%- if controller.version not in ('ocata','pike','queens', 'rocky', 'stein') %} + {%- do cinder_log_services.append('cinder-api') %} + {%- endif %} + diff --git a/mcp/patches/salt-formula-horizon/0001-Support-stein-version.patch b/mcp/patches/salt-formula-horizon/0001-Support-stein-version.patch new file mode 100644 index 000000000..3ae93638d --- /dev/null +++ b/mcp/patches/salt-formula-horizon/0001-Support-stein-version.patch @@ -0,0 +1,35 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Tue, 9 Jul 2019 14:54:50 +0400 +Subject: [PATCH] Support stein version + +--- + horizon/files/local_settings/stein_settings.py | 1 + + horizon/files/policy/stein | 1 + + 2 files changed, 2 insertions(+) + create mode 120000 horizon/files/local_settings/stein_settings.py + create mode 120000 horizon/files/policy/stein + +diff --git a/horizon/files/local_settings/stein_settings.py b/horizon/files/local_settings/stein_settings.py +new file mode 120000 +index 0000000..183e9a8 +--- /dev/null ++++ b/horizon/files/local_settings/stein_settings.py +@@ -0,0 +1 @@ ++rocky_settings.py +\ No newline at end of file +diff --git a/horizon/files/policy/stein b/horizon/files/policy/stein +new file mode 120000 +index 0000000..c8dbb8c +--- /dev/null ++++ b/horizon/files/policy/stein +@@ -0,0 +1 @@ ++rocky +\ No newline at end of file diff --git a/mcp/patches/salt-formula-horizon/0002-Align-packages-with-Stein-reqs.patch b/mcp/patches/salt-formula-horizon/0002-Align-packages-with-Stein-reqs.patch new file mode 100644 index 000000000..90a0057cb --- /dev/null +++ b/mcp/patches/salt-formula-horizon/0002-Align-packages-with-Stein-reqs.patch @@ -0,0 +1,30 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Mon, 30 Dec 2019 16:15:04 +0100 +Subject: [PATCH] Align packages with Stein reqs + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + horizon/map.jinja | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/horizon/map.jinja b/horizon/map.jinja +index 3779d69..0cebe0c 100644 +--- a/horizon/map.jinja ++++ b/horizon/map.jinja +@@ -6,7 +6,7 @@ + + {%- if not salt['pillar.get']('horizon:server:cache:backend') %} + {%- if salt['pillar.get']('horizon:server:version') not in ['mitaka','newton'] %} +- {%- set cache_pkgs = ['python-pylibmc'] %} ++ {%- set cache_pkgs = ['python3-pylibmc'] %} + {%- set default_cache_backend = 'django.core.cache.backends.memcached.PyLibMCCache' %} + {%- else %} + {% if grains['os_family'] == 'RedHat' %} diff --git a/mcp/patches/0008-Handle-extra-environment-variables.patch b/mcp/patches/salt-formula-keystone/0001-Handle-extra-environment-variables.patch index 8df3227f2..2108587b1 100644 --- a/mcp/patches/0008-Handle-extra-environment-variables.patch +++ b/mcp/patches/salt-formula-keystone/0001-Handle-extra-environment-variables.patch @@ -11,6 +11,9 @@ Date: Mon, 12 Mar 2018 17:43:09 +0400 Subject: [PATCH] Handle extra environment variables Change-Id: Ieae46ac65041630759c82238a8a5ce0535c454b2 +--- + keystone/files/keystonercv3 | 3 +++ + 1 file changed, 3 insertions(+) diff --git a/keystone/files/keystonercv3 b/keystone/files/keystonercv3 index 1b7f378..984c8a2 100644 diff --git a/mcp/patches/0015-Set-ovs-bridges-as-L3-interfaces.patch b/mcp/patches/salt-formula-linux/0001-Set-ovs-bridges-as-L3-interfaces.patch index c55a3727c..9479a990b 100644 --- a/mcp/patches/0015-Set-ovs-bridges-as-L3-interfaces.patch +++ b/mcp/patches/salt-formula-linux/0001-Set-ovs-bridges-as-L3-interfaces.patch @@ -1,5 +1,5 @@ :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. : : All rights reserved. This program and the accompanying materials : are made available under the terms of the Apache License, Version 2.0 @@ -10,33 +10,51 @@ From: Michael Polenchuk <mpolenchuk@mirantis.com> Date: Wed, 28 Feb 2018 17:54:28 +0400 Subject: [PATCH] Set ovs bridges as L3 interfaces -Change-Id: I1e83129cc184cf481bea21d7aa452bf60d9e0499 +--- + linux/files/ovs_bridge | 22 ++++++++++++++++++++++ + linux/files/ovs_port | 8 +++++++- + linux/network/interface.sls | 34 +++++++++++++++++++++++++++++++++- + 3 files changed, 62 insertions(+), 2 deletions(-) + create mode 100644 linux/files/ovs_bridge diff --git a/linux/files/ovs_bridge b/linux/files/ovs_bridge new file mode 100644 -index 0000000..c609e45 +index 0000000..4718b91 --- /dev/null +++ b/linux/files/ovs_bridge -@@ -0,0 +1,13 @@ -+auto {{ bridge_name }} +@@ -0,0 +1,22 @@ ++# With systemd, adding OVS bridges as 'auto' can cause race conditions ++# https://github.com/openvswitch/ovs/blob/master/debian/openvswitch-switch.README.Debian ++# auto {{ bridge_name }} +allow-ovs {{ bridge_name }} -+iface {{ bridge_name }} inet static ++iface {{ bridge_name }} inet {{ bridge.get('proto', 'static' if bridge.address is defined else 'manual') }} + ovs_type OVSBridge ++ {%- if bridge.address is defined %} + address {{ bridge.address }} + netmask {{ bridge.netmask }} -+ mtu {{ bridge.get('mtu', '1500') }} ++ {%- endif %} + {%- if bridge.use_interfaces is defined %} + ovs_ports {{ bridge.use_interfaces|join(' ') }} + {%- endif %} + {%- if bridge.datapath_type is defined %} + ovs_extra set Bridge ${IFACE} datapath_type={{ bridge.datapath_type }} + {%- endif %} ++ {%- if bridge.gateway is defined %} ++ gateway {{ bridge.gateway }} ++ {%- endif %} ++ {%- if bridge.name_servers is defined %} ++ dns-nameservers {{ bridge.name_servers | join(' ') }} ++ {%- endif %} diff --git a/linux/files/ovs_port b/linux/files/ovs_port -index 222ca8e..efb0307 100644 +index 222ca8e..41821b7 100644 --- a/linux/files/ovs_port +++ b/linux/files/ovs_port -@@ -1,6 +1,9 @@ - auto {{ port_name }} +@@ -1,6 +1,12 @@ +-auto {{ port_name }} ++# With systemd, adding OVS bridges as 'auto' can cause race conditions ++# https://github.com/openvswitch/ovs/blob/master/debian/openvswitch-switch.README.Debian ++# OVS ports will be automatically ifup-ed when ifup-ing the OVS bridge ++# auto {{ port_name }} allow-{{ port.bridge }} {{ port_name }} iface {{ port_name }} inet {{ port.get('proto', 'manual') }} +{%- if '.' in port_name %} @@ -46,14 +64,13 @@ index 222ca8e..efb0307 100644 mtu {{ port.get('mtu', '1500') }} ovs_bridge {{ port.bridge }} diff --git a/linux/network/interface.sls b/linux/network/interface.sls -index 180f912..dcb295b 100644 +index a39fc37..8bce092 100644 --- a/linux/network/interface.sls +++ b/linux/network/interface.sls -@@ -91,6 +91,34 @@ add_int_{{ int_name }}_to_ovs_dpdk_bridge_{{ interface_name }}: +@@ -92,6 +92,35 @@ add_int_{{ int_name }}_to_ovs_dpdk_bridge_{{ interface_name }}: ovs_bridge_{{ interface_name }}: openvswitch_bridge.present: - name: {{ interface_name }} -+{%- if interface.get('proto', 'manual') == 'static' %} + file.managed: + - name: /etc/network/interfaces.u/ifcfg-{{ interface_name }} + - makedirs: True @@ -73,14 +90,35 @@ index 180f912..dcb295b 100644 + +ovs_bridge_up_{{ interface_name }}: + cmd.run: -+ - name: ifup {{ interface_name }} ++ - name: ifup --ignore-errors {{ interface_name }} + - require: + - file: ovs_bridge_{{ interface_name }} + - openvswitch_bridge: ovs_bridge_{{ interface_name }} + - file: linux_interfaces_final_include -+ - unless: -+ - ip link show {{ interface_name }} | grep -q '\<UP\>' -+{%- endif %} ++ {%- if network.noifupdown|d(false) or interface.noifupdown|d(false) %} ++ - onlyif: /bin/false ++ {%- else %} ++ - unless: grep -qFx up /sys/class/net/{{ interface_name }}/operstate ++ {%- endif %} {# add linux network interface into OVS bridge #} {%- for int_name, int in network.interface.items() %} +@@ -102,7 +131,7 @@ ovs_bridge_{{ interface_name }}: + + add_int_{{ int_name }}_to_ovs_bridge_{{ interface_name }}: + cmd.run: +- - unless: ovs-vsctl show | grep {{ int_name }} ++ - unless: ovs-vsctl list-ports {{ interface_name }} | grep -qFx {{ int_name }} + - name: ovs-vsctl{%- if network.ovs_nowait %} --no-wait{%- endif %} add-port {{ interface_name }} {{ int_name }} + + {%- endif %} +@@ -176,6 +205,9 @@ ovs_port_{{ interface_name }}_line2: + ovs_port_up_{{ interface_name }}: + cmd.run: + - name: ifup {{ interface_name }} ++ {%- if network.noifupdown|d(false) or interface.noifupdown|d(false) %} ++ - onlyif: /bin/false ++ {%- endif %} + - require: + - file: ovs_port_{{ interface_name }} + - file: ovs_port_{{ interface_name }}_line1 diff --git a/mcp/patches/salt-formula-linux/0002-network-Bring-in-basic-VPP-support.patch b/mcp/patches/salt-formula-linux/0002-network-Bring-in-basic-VPP-support.patch new file mode 100644 index 000000000..5166be56a --- /dev/null +++ b/mcp/patches/salt-formula-linux/0002-network-Bring-in-basic-VPP-support.patch @@ -0,0 +1,139 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Mon, 10 Dec 2018 16:16:26 +0100 +Subject: [PATCH] network: Bring in basic VPP support + +For now, we only care about DPDK-backed VPP ports. + +TODO: +- README.rst: VPP port usage + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + linux/files/vpp_commands.txt | 4 +++ + linux/files/vpp_startup.conf | 47 ++++++++++++++++++++++++++++++++++++ + linux/network/dpdk.sls | 38 +++++++++++++++++++++++++++++ + 3 files changed, 89 insertions(+) + create mode 100644 linux/files/vpp_commands.txt + create mode 100644 linux/files/vpp_startup.conf + +diff --git a/linux/files/vpp_commands.txt b/linux/files/vpp_commands.txt +new file mode 100644 +index 0000000..2ce4726 +--- /dev/null ++++ b/linux/files/vpp_commands.txt +@@ -0,0 +1,4 @@ ++{%- from "linux/map.jinja" import network with context %} ++{%- if network.vpp.commands is defined %} ++{{ network.vpp.commands }} ++{%- endif %} +diff --git a/linux/files/vpp_startup.conf b/linux/files/vpp_startup.conf +new file mode 100644 +index 0000000..fcfc1fd +--- /dev/null ++++ b/linux/files/vpp_startup.conf +@@ -0,0 +1,47 @@ ++{%- from "linux/map.jinja" import network with context %} ++unix { ++ cli-listen /run/vpp/cli.sock ++ log /var/log/vpp.log ++ full-coredump ++ nodaemon ++ startup-config /etc/vpp/commands.txt ++{%- if network.vpp.gid is defined %} ++ gid {{ network.vpp.gid }} ++{%- endif %} ++} ++api-trace { ++ on ++} ++{%- if network.vpp.gid is defined %} ++api-segment { ++ gid {{ network.vpp.gid }} ++} ++{%- endif %} ++cpu { ++{%- if network.vpp.main_core is defined %} ++ main-core {{ network.vpp.main_core }} ++{%- endif %} ++{%- if network.vpp.corelist_workers is defined %} ++ corelist-workers {{ network.vpp.corelist_workers }} ++{%- endif %} ++{%- if network.vpp.skip_core is defined %} ++ skip-core {{ network.vpp.skip_core }} ++{%- endif %} ++{%- if network.vpp.workers is defined %} ++ workers {{ network.vpp.workers }} ++{%- endif %} ++} ++dpdk { ++{%- if network.vpp.decimal_interface_names is defined %} ++ decimal-interface-names ++{%- endif %} ++{%- if network.vpp.dpdk_socket_mem is defined %} ++ socket-mem {{ network.vpp.dpdk_socket_mem }} ++{%- endif %} ++ ## Whitelist specific interface by specifying PCI address ++{%- for interface_name, interface in network.interface.items() %} ++{%- if 'dpdk_vpp_port' in interface.type and interface.pci is defined %} ++ dev {{ interface.pci }} ++{%- endif %} ++{%- endfor %} ++} +diff --git a/linux/network/dpdk.sls b/linux/network/dpdk.sls +index 786f7c8..09453c6 100644 +--- a/linux/network/dpdk.sls ++++ b/linux/network/dpdk.sls +@@ -32,6 +32,45 @@ linux_network_dpdk_service: + - watch: + - file: /etc/dpdk/interfaces + ++{%- if network.vpp is defined %} ++ ++vpp_pkgs: ++ pkg.installed: ++ - pkgs: ++ - vpp ++ - vpp-plugin-core ++ - vpp-plugin-dpdk ++ - bridge-utils ++ ++/etc/vpp/commands.txt: ++ file.managed: ++ - source: salt://linux/files/vpp_commands.txt ++ - template: jinja ++ - require: ++ - pkg: vpp_pkgs ++ ++/etc/vpp/startup.conf: ++ file.managed: ++ - source: salt://linux/files/vpp_startup.conf ++ - template: jinja ++ - require: ++ - pkg: vpp_pkgs ++ ++/etc/sysctl.d/80-vpp.conf: ++ file.managed: ++ - contents: ++ - '# Disabled by salt-formula-linux' ++ ++linux_network_vpp_service: ++ service.running: ++ - enable: true ++ - name: vpp ++ - watch: ++ - file: /etc/vpp/startup.conf ++ - file: /etc/vpp/commands.txt ++ ++{%- endif %} ++ + {%- if network.openvswitch is defined %} + + openvswitch_dpdk_pkgs: diff --git a/mcp/patches/salt-formula-linux/0003-OVS-Fix-Debian-service-deps-OVS-bridge-ifup.patch b/mcp/patches/salt-formula-linux/0003-OVS-Fix-Debian-service-deps-OVS-bridge-ifup.patch new file mode 100644 index 000000000..70b699a76 --- /dev/null +++ b/mcp/patches/salt-formula-linux/0003-OVS-Fix-Debian-service-deps-OVS-bridge-ifup.patch @@ -0,0 +1,97 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Fri, 25 Jan 2019 21:20:04 +0100 +Subject: [PATCH] OVS: Fix Debian service deps, OVS bridge ifup + +Fix OVS vs Linux bridge race condition: +- OVS services should start before networking service; +- OVS services should start after DPDK service (if present); +- networking service should ifup OVS bridges (and automatically their + OVS ports if present) after Linux interfaces/bridges; +- br-prv should be handled by OVS to avoid another race condition, + so use 'allow-ovs br-prv' instead of 'auto'; + +NOTE: +- OVS ports/bridges should NOT be configured as auto for this to work; +- OVS services correspond to OVS 2.9 or newer, since before that + ovsdb-server was called openvswitch-nonetwork. +- we also need to take care of one particularly ugly circular dep: + ovs-vswitchd --> ovsdb-server -(default dep)-> sysinit.target --> + cloud-init.service --> networking.service --> ovs-vswitchd + We'll just set 'DefaultDependencies=no' for ovs services, although + this might require explicitly adding back some of the indirect + dependencies of sysinit.target. + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + linux/network/dpdk.sls | 2 +- + linux/network/interface.sls | 36 ++++++++++++++++++++++++++++++++++++ + 2 files changed, 37 insertions(+), 1 deletion(-) + +diff --git a/linux/network/dpdk.sls b/linux/network/dpdk.sls +index 09453c6..e866909 100644 +--- a/linux/network/dpdk.sls ++++ b/linux/network/dpdk.sls +@@ -199,7 +199,7 @@ linux_network_dpdk_bridge_interface_{{ interface_name }}: + /etc/network/interfaces.u/ifcfg-{{ interface_name }}: + file.managed: + - contents: | +- auto {{ interface_name }} ++ allow-ovs {{ interface_name }} + iface {{ interface_name }} inet static + address {{ interface.address }} + netmask {{ interface.netmask }} +diff --git a/linux/network/interface.sls b/linux/network/interface.sls +index 8bce092..11db5be 100644 +--- a/linux/network/interface.sls ++++ b/linux/network/interface.sls +@@ -24,6 +24,42 @@ linux_network_bridge_pkgs: + - pkgs: {{ network.bridge_pkgs }} + {%- endif %} + ++{%- if network.bridge == 'openvswitch' and grains.os_family == 'Debian' %} ++ ++{# create drop-in dpdk, networking dependency for ovs services #} ++/etc/systemd/system/ovsdb-server.service.d/override.conf: ++ file.managed: ++ - makedirs: true ++ - require: ++ - pkg: linux_network_bridge_pkgs ++ - contents: | ++ [Unit] ++ After=dpdk.service ++ Before=networking.service ++ DefaultDependencies=no ++ ++/etc/systemd/system/ovs-vswitchd.service.d/override.conf: ++ file.managed: ++ - makedirs: true ++ - require: ++ - pkg: linux_network_bridge_pkgs ++ - contents: | ++ [Unit] ++ Before=networking.service ++ DefaultDependencies=no ++ ++{# Debian/Ubuntu won't automatically ifup OVS bridges, workaround #} ++/etc/systemd/system/networking.service.d/ovs_workaround.conf: ++ file.managed: ++ - makedirs: true ++ - require: ++ - pkg: linux_network_bridge_pkgs ++ - contents: | ++ [Service] ++ ExecStart=/sbin/ifup --allow=ovs -a --read-environment ++ ++{%- endif %} ++ + {%- endif %} + + {%- for f in network.get('concat_iface_files', []) %} diff --git a/mcp/patches/salt-formula-linux/0004-dpdk-Handle-per-port-memory-model.patch b/mcp/patches/salt-formula-linux/0004-dpdk-Handle-per-port-memory-model.patch new file mode 100644 index 000000000..b54650007 --- /dev/null +++ b/mcp/patches/salt-formula-linux/0004-dpdk-Handle-per-port-memory-model.patch @@ -0,0 +1,25 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Michael Polenchuk <mpolenchuk@mirantis.com> +Date: Tue, 30 Apr 2019 12:59:54 +0400 +Subject: [PATCH] [dpdk] Handle per port memory model + + +diff --git a/linux/network/dpdk.sls b/linux/network/dpdk.sls +index 786f7c8..c6f3020 100644 +--- a/linux/network/dpdk.sls ++++ b/linux/network/dpdk.sls +@@ -49,6 +49,7 @@ linux_network_dpdk_ovs_service: + - unless: 'ovs-vsctl get Open_vSwitch . other_config | grep "dpdk-init=\"true\""' + + {%- set ovs_options = [ ++ 'per-port-memory="'+network.openvswitch.per_port_memory|d('false')+'"', + "pmd-cpu-mask=\""+network.openvswitch.pmd_cpu_mask+"\"", + "dpdk-socket-mem=\""+network.openvswitch.dpdk_socket_mem+"\"", + "dpdk-lcore-mask=\""+network.openvswitch.dpdk_lcore_mask+"\"", diff --git a/mcp/patches/salt-formula-linux/0005-network-RHEL-Set-bridge-for-member-interfaces.patch b/mcp/patches/salt-formula-linux/0005-network-RHEL-Set-bridge-for-member-interfaces.patch new file mode 100644 index 000000000..67004601c --- /dev/null +++ b/mcp/patches/salt-formula-linux/0005-network-RHEL-Set-bridge-for-member-interfaces.patch @@ -0,0 +1,44 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Tue, 11 Jun 2019 03:57:29 +0200 +Subject: [PATCH] network: RHEL: Set bridge for member interfaces + +For RHEL, bridge member interfaces need to be passed down the +bridge name, so lookup all bridges, find the one containing the +current interface (if any) and use it when calling +`network.managed`. + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + linux/network/interface.sls | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/linux/network/interface.sls b/linux/network/interface.sls +index 11db5be..ccac6d7 100644 +--- a/linux/network/interface.sls ++++ b/linux/network/interface.sls +@@ -303,6 +303,17 @@ linux_interface_{{ interface_name }}: + - cmd: ovs_port_up_{{ network }} + {%- endfor %} + {%- endif %} ++ {%- if grains.os_family == 'RedHat' %} ++ {%- set br_todo = [] %} ++ {%- for br_name, br_info in network.interface.items() %} ++ {%- if br_info.type == 'bridge' and interface_name in br_info.get('use_interfaces', []) %} ++ {%- do br_todo.append(br_info.get('name', br_name)) %} ++ {%- endif %} ++ {%- endfor %} ++ {%- if br_todo %} ++ - bridge: {{ br_todo | first }} ++ {%- endif %} ++ {%- endif %} + {%- if interface.type == 'bridge' %} + - bridge: {{ interface_name }} + - delay: 0 diff --git a/mcp/patches/salt-formula-linux/0006-dpdk-Remove-invalid-vhost-options.patch b/mcp/patches/salt-formula-linux/0006-dpdk-Remove-invalid-vhost-options.patch new file mode 100644 index 000000000..acc3c181f --- /dev/null +++ b/mcp/patches/salt-formula-linux/0006-dpdk-Remove-invalid-vhost-options.patch @@ -0,0 +1,25 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Michael Polenchuk <mpolenchuk@mirantis.com> +Date: Tue, 30 Jul 2019 13:24:41 +0400 +Subject: [PATCH] [dpdk] Remove invalid vhost options + +diff --git a/linux/network/dpdk.sls b/linux/network/dpdk.sls +index 786f7c8..c929e43 100644 +--- a/linux/network/dpdk.sls ++++ b/linux/network/dpdk.sls +@@ -52,7 +52,7 @@ linux_network_dpdk_ovs_service: + "pmd-cpu-mask=\""+network.openvswitch.pmd_cpu_mask+"\"", + "dpdk-socket-mem=\""+network.openvswitch.dpdk_socket_mem+"\"", + "dpdk-lcore-mask=\""+network.openvswitch.dpdk_lcore_mask+"\"", +- "dpdk-extra=\"-n "+network.openvswitch.memory_channels+" --vhost-owner libvirt-qemu:kvm --vhost-perm 0664\"" ++ "dpdk-extra=\"-n "+network.openvswitch.memory_channels+"\"" + ] + %} + diff --git a/mcp/patches/0002-maas-region-skip-credentials-update.patch b/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch index 1d226e4cd..eb607cf81 100644 --- a/mcp/patches/0002-maas-region-skip-credentials-update.patch +++ b/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch @@ -1,5 +1,5 @@ :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. : : All rights reserved. This program and the accompanying materials : are made available under the terms of the Apache License, Version 2.0 @@ -17,12 +17,14 @@ updating credentials. Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> --- + maas/region.sls | 7 +++---- + 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/maas/region.sls b/maas/region.sls -index d3227ca..8a2243d 100644 +index 066490c..de5054a 100644 --- a/maas/region.sls +++ b/maas/region.sls -@@ -6,10 +6,9 @@ +@@ -6,10 +6,9 @@ maas_region_packages: - names: {{ region.pkgs }} /etc/maas/regiond.conf: diff --git a/mcp/patches/0010-maas-region-allow-timeout-override.patch b/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch index c6f9e3a52..3d8deff60 100644 --- a/mcp/patches/0010-maas-region-allow-timeout-override.patch +++ b/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch @@ -1,5 +1,5 @@ :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. : : All rights reserved. This program and the accompanying materials : are made available under the terms of the Apache License, Version 2.0 @@ -35,18 +35,20 @@ node's PXE physical interface, but that overcomplicates things. blocksize to be 1008. We can force it to be 1464 and gain some performance due to MTU beeing 1500 (i.e. allow bigger packets). - JIRA: FUEL-316 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> --- + maas/region.sls | 30 ++++++++++++++++++++++++++++++ + 1 file changed, 30 insertions(+) diff --git a/maas/region.sls b/maas/region.sls +index de5054a..4a7f6cc 100644 --- a/maas/region.sls +++ b/maas/region.sls -@@ -19,6 +19,46 @@ - - require: - - pkg: maas_region_packages +@@ -38,6 +38,36 @@ restore_maas_database_{{ region.database.name }}: + + {%- endif %} +maas_timeout_commissioning: + file.replace: @@ -68,16 +70,6 @@ diff --git a/maas/region.sls b/maas/region.sls + - require_in: + - service: maas_region_services + -+maas_ip_blksize_force: -+ file.replace: -+ - name: "/usr/lib/python3/dist-packages/tftp/bootstrap.py" -+ - pattern: 'int_blksize = min\(\(int_blksize, MAX_BLOCK_SIZE\)\)' -+ - repl: 'int_blksize = 1464' -+ - require: -+ - pkg: maas_region_packages -+ - require_in: -+ - service: maas_region_services -+ +maas_interface_default_mode_dhcp: + file.replace: + - name: "/usr/lib/python3/dist-packages/maasserver/models/node.py" diff --git a/mcp/patches/salt-formula-maas/0003-Extend-wait_for-maas.py-wait_for_-attempts-arg.patch b/mcp/patches/salt-formula-maas/0003-Extend-wait_for-maas.py-wait_for_-attempts-arg.patch new file mode 100644 index 000000000..efd7cc2fd --- /dev/null +++ b/mcp/patches/salt-formula-maas/0003-Extend-wait_for-maas.py-wait_for_-attempts-arg.patch @@ -0,0 +1,268 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Sun, 23 Sep 2018 03:57:27 +0200 +Subject: [PATCH] Extend wait_for maas.py, wait_for_* attempts arg + +1. maas.py: Extend wait_for states with timeout param + +Extend the wait_for states with a timeout parameter. +The timeout value is taken from reclass pillar data if +defined. Oterwise, the states use the default value. +Based on Ting's PR [1], slightly refactored. + +2. maas.py: Extend `req_status` support to multiple values + +Previously, req_status could be one of the MaaS status strings, e.g. +'Ready'. Extend matching to '|'-separated statuses (e.g. +'Ready|Deployed') to allow idempotency in MaaS machine commissioning +and deployment cycles. + +Also provide a `maas.machines.wait_for_ready_or_deployed` sls. + +3. maas.py: wait_for_*: Add attempts arg + +Introduce a new parameter that allows a maximum number of automatic +recovery attempts for the common failures w/ machine operations. +If not present in pillar data, it defaults to 0 (OFF). + +Common error states, possible cause and automatic recovery pattern: +* New + - usually indicates issues with BMC connectivity (no network route, + but on rare occassions it happens due to MaaS API being flaky); + - fix: delete the machine, (re)process machine definitions; +* Failed commissioning + - various causes, usually a simple retry works; + - fix: delete the machine, (re)process machine definitions; +* Failed testing + - incompatible hardware, missing drivers etc. + - usually consistent and board-specific; + - fix: override failed testing +* Allocated + - on rare ocassions nodes get stuck in this state instead 'Deploy'; + - fix: mark-broken, mark-fixed, if it failed at least once before + perform a fio test (fixes another unrelated spurious issue with + encrypted disks from previous deployments), (re)deploy machines; +* Failed deployment + - various causes, usually a simple retry works; + - fix: same as for nodes stuck in 'Allocated'; + +[1] https://github.com/salt-formulas/salt-formula-maas/pull/34 + +Change-Id: Ifb7dd9f8fcfbbed557e47d8fdffb1f963604fb15 +Signed-off-by: ting wu <ting.wu@enea.com> +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + README.rst | 9 +++- + _modules/maas.py | 52 +++++++++++++++++--- + maas/machines/wait_for_deployed.sls | 2 + + maas/machines/wait_for_ready.sls | 3 ++ + maas/machines/wait_for_ready_or_deployed.sls | 15 ++++++ + maas/map.jinja | 4 ++ + tests/pillar/maas_region.sls | 4 ++ + 7 files changed, 81 insertions(+), 8 deletions(-) + create mode 100644 maas/machines/wait_for_ready_or_deployed.sls + +diff --git a/README.rst b/README.rst +index 20da43e..78d8aef 100644 +--- a/README.rst ++++ b/README.rst +@@ -622,12 +622,16 @@ Wait for status of selected machine's: + machines: + - kvm01 + - kvm02 +- timeout: 1200 # in seconds ++ timeout: {{ region.timeout.ready }} ++ attempts: {{ region.timeout.attempts }} + req_status: "Ready" + - require: + - cmd: maas_login_admin + ... + ++The timeout setting is taken from the reclass pillar data. ++If the pillar data is not defined, it will use the default value. ++ + If module run w/\o any extra paremeters, + ``wait_for_machines_ready`` will wait for defined in salt + machines. In this case, it is usefull to skip some machines: +@@ -642,7 +646,8 @@ machines. In this case, it is usefull to skip some machines: + module.run: + - name: maas.wait_for_machine_status + - kwargs: +- timeout: 1200 # in seconds ++ timeout: {{ region.timeout.deployed }} ++ attempts: {{ region.timeout.attempts }} + req_status: "Deployed" + ignore_machines: + - kvm01 # in case it's broken or whatever +diff --git a/_modules/maas.py b/_modules/maas.py +index c02f104..bb70576 100644 +--- a/_modules/maas.py ++++ b/_modules/maas.py +@@ -670,7 +670,7 @@ class DeployMachines(MaasObject): + if machine['status'] == self.DEPLOYED: + return + if machine['status'] != self.READY: +- raise Exception('Not in ready state') ++ return + data = { + 'system_id': machine['system_id'], + } +@@ -921,6 +921,7 @@ class MachinesStatus(MaasObject): + req_status: string; Polling status + machines: list; machine names + ignore_machines: list; machine names ++ attempts: max number of automatic hard retries + :ret: True + Exception - if something fail/timeout reached + """ +@@ -929,6 +930,8 @@ class MachinesStatus(MaasObject): + req_status = kwargs.get("req_status", "Ready") + to_discover = kwargs.get("machines", None) + ignore_machines = kwargs.get("ignore_machines", None) ++ attempts = kwargs.get("attempts", 0) ++ failed_attempts = {} + if not to_discover: + try: + to_discover = __salt__['config.get']('maas')['region'][ +@@ -943,11 +946,46 @@ class MachinesStatus(MaasObject): + while len(total) <= len(to_discover): + for m in to_discover: + for discovered in MachinesStatus.execute()['machines']: +- if m == discovered['hostname'] and \ +- discovered['status'].lower() == req_status.lower(): +- if m in total: ++ if m == discovered['hostname'] and m in total: ++ req_status_list = req_status.lower().split('|') ++ if discovered['status'].lower() in req_status_list: + total.remove(m) +- ++ elif attempts > 0 and (m not in failed_attempts or ++ failed_attempts[m] < attempts): ++ status = discovered['status'] ++ sid = discovered['system_id'] ++ cls._maas = _create_maas_client() ++ if status in ['Failed commissioning', 'New']: ++ LOG.info('Machine {0} deleted'.format(sid)) ++ cls._maas.delete(u'api/2.0/machines/{0}/' ++ .format(sid)) ++ Machine().process() ++ elif status in ['Failed testing']: ++ data = {} ++ LOG.info('Machine {0} overriden'.format(sid)) ++ action = 'override_failed_testing' ++ cls._maas.post(u'api/2.0/machines/{0}/' ++ .format(sid), action, **data) ++ elif status in ['Failed deployment', 'Allocated']: ++ data = {} ++ LOG.info('Machine {0} mark broken'.format(sid)) ++ cls._maas.post(u'api/2.0/machines/{0}/' ++ .format(sid), 'mark_broken', **data) ++ time.sleep(poll_time) ++ LOG.info('Machine {0} mark fixed'.format(sid)) ++ cls._maas.post(u'api/2.0/machines/{0}/' ++ .format(sid), 'mark_fixed', **data) ++ if m in failed_attempts and failed_attempts[m]: ++ LOG.info('Machine {0} fio test'.format(sid)) ++ data['testing_scripts'] = 'fio' ++ cls._maas.post(u'api/2.0/machines/{0}/' ++ .format(sid), 'commission', **data) ++ DeployMachines().process() ++ else: ++ continue ++ if m not in failed_attempts: ++ failed_attempts[m] = 0 ++ failed_attempts[m] = failed_attempts[m] + 1 + if len(total) <= 0: + LOG.debug( + "Machines:{} are:{}".format(to_discover, req_status)) +@@ -959,7 +997,9 @@ class MachinesStatus(MaasObject): + "Waiting status:{} " + "for machines:{}" + "\nsleep for:{}s " +- "Timeout:{}s".format(req_status, total, poll_time, timeout)) ++ "Timeout:{}s ({}s left)" ++ .format(req_status, total, poll_time, timeout, ++ timeout - (time.time() - started_at))) + time.sleep(poll_time) + + +diff --git a/maas/machines/wait_for_deployed.sls b/maas/machines/wait_for_deployed.sls +index ebeedac..a646fdb 100644 +--- a/maas/machines/wait_for_deployed.sls ++++ b/maas/machines/wait_for_deployed.sls +@@ -9,5 +9,7 @@ wait_for_machines_deployed: + - name: maas.wait_for_machine_status + - kwargs: + req_status: "Deployed" ++ timeout: {{ region.timeout.deployed }} ++ attempts: {{ region.timeout.attempts }} + - require: + - cmd: maas_login_admin +diff --git a/maas/machines/wait_for_ready.sls b/maas/machines/wait_for_ready.sls +index c5d3c28..d8a2963 100644 +--- a/maas/machines/wait_for_ready.sls ++++ b/maas/machines/wait_for_ready.sls +@@ -7,5 +7,8 @@ maas_login_admin: + wait_for_machines_ready: + module.run: + - name: maas.wait_for_machine_status ++ - kwargs: ++ timeout: {{ region.timeout.ready }} ++ attempts: {{ region.timeout.attempts }} + - require: + - cmd: maas_login_admin +diff --git a/maas/machines/wait_for_ready_or_deployed.sls b/maas/machines/wait_for_ready_or_deployed.sls +new file mode 100644 +index 0000000..db3dcc4 +--- /dev/null ++++ b/maas/machines/wait_for_ready_or_deployed.sls +@@ -0,0 +1,15 @@ ++{%- from "maas/map.jinja" import region with context %} ++ ++maas_login_admin: ++ cmd.run: ++ - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" ++ ++wait_for_machines_ready_or_deployed: ++ module.run: ++ - name: maas.wait_for_machine_status ++ - kwargs: ++ req_status: "Ready|Deployed" ++ timeout: {{ region.timeout.ready }} ++ attempts: {{ region.timeout.attempts }} ++ - require: ++ - cmd: maas_login_admin +diff --git a/maas/map.jinja b/maas/map.jinja +index 0671435..1e6ac07 100644 +--- a/maas/map.jinja ++++ b/maas/map.jinja +@@ -22,6 +22,10 @@ Debian: + bind: + host: 0.0.0.0 + port: 80 ++ timeout: ++ ready: 1200 ++ deployed: 7200 ++ attempts: 0 + {%- endload %} + + {%- set region = salt['grains.filter_by'](region_defaults, merge=salt['pillar.get']('maas:region', {})) %} +diff --git a/tests/pillar/maas_region.sls b/tests/pillar/maas_region.sls +index d3325eb..d710216 100644 +--- a/tests/pillar/maas_region.sls ++++ b/tests/pillar/maas_region.sls +@@ -34,3 +34,7 @@ maas: + password: password + username: maas + salt_master_ip: 127.0.0.1 ++ timeout: ++ deployed: 900 ++ ready: 900 ++ attempts: 2 diff --git a/mcp/patches/salt-formula-maas/0004-curtin-Tune-default-salt-minion-config.patch b/mcp/patches/salt-formula-maas/0004-curtin-Tune-default-salt-minion-config.patch new file mode 100644 index 000000000..978f4ecb6 --- /dev/null +++ b/mcp/patches/salt-formula-maas/0004-curtin-Tune-default-salt-minion-config.patch @@ -0,0 +1,43 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Mon, 5 Nov 2018 16:47:13 +0100 +Subject: [PATCH] curtin: Tune default salt-minion config + +* Sync AArch64 configuration with the changes introduced by commit + c6a12de on amd64 + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + maas/files/curtin_userdata_arm64_generic_xenial | 12 ++++++++++-- + 1 file changed, 10 insertions(+), 2 deletions(-) + +diff --git a/maas/files/curtin_userdata_arm64_generic_xenial b/maas/files/curtin_userdata_arm64_generic_xenial +index af9a047..1b2e401 100644 +--- a/maas/files/curtin_userdata_arm64_generic_xenial ++++ b/maas/files/curtin_userdata_arm64_generic_xenial +@@ -32,8 +32,16 @@ late_commands: + apt_03_update: ["curtin", "in-target", "--", "apt-get", "update"] + salt_01_install: ["curtin", "in-target", "--", "apt-get", "-y", "install", "salt-minion", "python-futures"] + salt_02_hostname_set: ["curtin", "in-target", "--", "echo", "{% raw %}{{node.hostname}}{% endraw %}.{{pillar.linux.system.domain}}"] +- salt_03_hostname_get: ["curtin", "in-target", "--", "sh", "-c", "echo 'id: {% raw %}{{node.hostname}}{% endraw %}.{{pillar.linux.system.domain}}' >> /etc/salt/minion"] +- salt_04_master: ["curtin", "in-target", "--", "sh", "-c", "echo 'master: {{ salt_master_ip }}' >> /etc/salt/minion"] ++ salt_03_hostname_get: ["curtin", "in-target", "--", "sh", "-c", "echo 'id: {% raw %}{{node.hostname}}{% endraw %}.{{pillar.linux.system.domain}}' >> /etc/salt/minion.d/minion.conf"] ++ salt_04_master: ["curtin", "in-target", "--", "sh", "-c", "echo 'master: {{ salt_master_ip }}' >> /etc/salt/minion.d/minion.conf"] ++ salt_05_max_event_size: ["curtin", "in-target", "--", "sh", "-c", "echo 'max_event_size: 100000000' >> /etc/salt/minion.d/minion.conf"] ++ salt_06_acceptance_wait_time_max: ["curtin", "in-target", "--", "sh", "-c", "echo 'acceptance_wait_time_max: 60' >> /etc/salt/minion.d/minion.conf"] ++ salt_07_acceptance_wait_time: ["curtin", "in-target", "--", "sh", "-c", "echo 'acceptance_wait_time: 10' >> /etc/salt/minion.d/minion.conf"] ++ salt_08_random_reauth_delay: ["curtin", "in-target", "--", "sh", "-c", "echo 'random_reauth_delay: 270' >> /etc/salt/minion.d/minion.conf"] ++ salt_09_recon_default: ["curtin", "in-target", "--", "sh", "-c", "echo 'recon_default: 1000' >> /etc/salt/minion.d/minion.conf"] ++ salt_10_recon_max: ["curtin", "in-target", "--", "sh", "-c", "echo 'recon_max: 60000' >> /etc/salt/minion.d/minion.conf"] ++ salt_11_recon_randomize: ["curtin", "in-target", "--", "sh", "-c", "echo 'recon_randomize: True' >> /etc/salt/minion.d/minion.conf"] ++ salt_12_auth_timeout: ["curtin", "in-target", "--", "sh", "-c", "echo 'auth_timeout: 60' >> /etc/salt/minion.d/minion.conf"] + {% raw %} + {{if third_party_drivers and driver}} + driver_00_key_get: curtin in-target -- sh -c "/bin/echo -en '{{key_string}}' > /tmp/maas-{{driver['package']}}.gpg" diff --git a/mcp/patches/salt-formula-maas/0005-Implement-tags-support.patch b/mcp/patches/salt-formula-maas/0005-Implement-tags-support.patch new file mode 100644 index 000000000..09d21d022 --- /dev/null +++ b/mcp/patches/salt-formula-maas/0005-Implement-tags-support.patch @@ -0,0 +1,95 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Sat, 29 Dec 2018 23:09:01 +0100 +Subject: [PATCH] Implement 'tags' support + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + README.rst | 5 +++++ + _modules/maas.py | 26 ++++++++++++++++++++++++++ + maas/region.sls | 8 ++++++++ + 3 files changed, 39 insertions(+) + +diff --git a/README.rst b/README.rst +index 78d8aef..8c35458 100644 +--- a/README.rst ++++ b/README.rst +@@ -181,6 +181,11 @@ Single MAAS region service [single UI/API]: + mac: '66:55:44:33:22:11' + commissioning_scripts: + 00-maas-05-simplify-network-interfaces: /etc/maas/files/commisioning_scripts/00-maas-05-simplify-network-interfaces ++ tags: ++ aarch64_hugepages_1g: ++ comment: 'Enable 1G pagesizes on aarch64' ++ definition: '//capability[@id="asimd"]' ++ kernel_opts: 'default_hugepagesz=1G hugepagesz=1G' + maas_config: + # domain: mydomain.local # This function broken + http_proxy: http://192.168.0.10:3142 +diff --git a/_modules/maas.py b/_modules/maas.py +index bb70576..0cda8dd 100644 +--- a/_modules/maas.py ++++ b/_modules/maas.py +@@ -876,6 +876,28 @@ class Domain(MaasObject): + return ret + + ++class Tags(MaasObject): ++ def __init__(self): ++ super(Tags, self).__init__() ++ self._all_elements_url = u'api/2.0/tags/' ++ self._create_url = u'api/2.0/tags/' ++ self._config_path = 'region.tags' ++ self._update_url = u'api/2.0/tags/{0}/' ++ self._update_key = 'name' ++ ++ def fill_data(self, name, tag_data): ++ data = { ++ 'name': name, ++ } ++ for key in ['comment', 'definition', 'kernel_opts']: ++ if key in tag_data: ++ data[key] = tag_data[key] ++ return data ++ ++ def update(self, new, old): ++ return new ++ ++ + class MachinesStatus(MaasObject): + @classmethod + def execute(cls, objects_name=None): +@@ -1065,5 +1087,9 @@ def process_sshprefs(): + return SSHPrefs().process() + + ++def process_tags(): ++ return Tags().process() ++ ++ + def wait_for_machine_status(**kwargs): + return MachinesStatus.wait_for_machine_status(**kwargs) +diff --git a/maas/region.sls b/maas/region.sls +index 4a7f6cc..52fb952 100644 +--- a/maas/region.sls ++++ b/maas/region.sls +@@ -444,4 +444,12 @@ maas_sshkey_{{ idx }}: + {% endfor %} + {%- endif %} + ++{%- if region.get('tags', False) %} ++maas_tags: ++ module.run: ++ - name: maas.process_tags ++ - require: ++ - cmd: maas_login_admin ++{%- endif %} ++ + {%- endif %} diff --git a/mcp/patches/salt-formula-maas/0006-curtin-Add-Bionic-support.patch b/mcp/patches/salt-formula-maas/0006-curtin-Add-Bionic-support.patch new file mode 100644 index 000000000..c69507b20 --- /dev/null +++ b/mcp/patches/salt-formula-maas/0006-curtin-Add-Bionic-support.patch @@ -0,0 +1,231 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Wed, 16 Oct 2019 15:02:39 +0200 +Subject: [PATCH] curtin: Add Bionic support + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + .../curtin_userdata_amd64_generic_bionic | 84 +++++++++++++++++++ + .../curtin_userdata_arm64_generic_bionic | 79 +++++++++++++++++ + maas/region.sls | 24 ++++++ + 3 files changed, 187 insertions(+) + create mode 100644 maas/files/curtin_userdata_amd64_generic_bionic + create mode 100644 maas/files/curtin_userdata_arm64_generic_bionic + +diff --git a/maas/files/curtin_userdata_amd64_generic_bionic b/maas/files/curtin_userdata_amd64_generic_bionic +new file mode 100644 +index 0000000..11af3cf +--- /dev/null ++++ b/maas/files/curtin_userdata_amd64_generic_bionic +@@ -0,0 +1,84 @@ ++{%- from "maas/map.jinja" import cluster with context %} ++{%- raw %} ++#cloud-config ++debconf_selections: ++ maas: | ++ {{for line in str(curtin_preseed).splitlines()}} ++ {{line}} ++ {{endfor}} ++early_commands: ++ thin_tools_install: ["sh", "-c", "apt-get update --quiet && apt-get --assume-yes install thin-provisioning-tools"] ++{{if third_party_drivers and driver}} ++ {{py: key_string = ''.join(['\\x%x' % x for x in map(ord, driver['key_binary'])])}} ++ driver_00_get_key: /bin/echo -en '{{key_string}}' > /tmp/maas-{{driver['package']}}.gpg ++ driver_01_add_key: ["apt-key", "add", "/tmp/maas-{{driver['package']}}.gpg"] ++ driver_02_add: ["add-apt-repository", "-y", "deb {{driver['repository']}} {{node.get_distro_series()}} main"] ++ driver_03_update_install: ["sh", "-c", "apt-get update --quiet && apt-get --assume-yes install {{driver['package']}}"] ++ driver_04_load: ["sh", "-c", "depmod && modprobe {{driver['module']}}"] ++{{endif}} ++late_commands: ++ maas: [wget, '--no-proxy', {{node_disable_pxe_url|escape.json}}, '--post-data', {{node_disable_pxe_data|escape.json}}, '-O', '/dev/null'] ++{% endraw %} ++ ++{%- if cluster.get('saltstack_repo_key', False) %} ++ {% set salt_repo_key = salt['hashutil.base64_b64encode'](cluster.saltstack_repo_key) %} ++ apt_00_set_gpg: ["curtin", "in-target", "--", "sh", "-c", "echo '{{salt_repo_key}}' | base64 -d | apt-key add -"] ++{%- endif %} ++{% if cluster.saltstack_repo_bionic.startswith('deb') %} ++ {%- set saltstack_repo = cluster.saltstack_repo_bionic -%} ++{%- else %} ++ {%- set saltstack_repo = 'deb [arch=amd64] ' + cluster.saltstack_repo_bionic -%} ++{%- endif %} ++ ++ apt_01_set_repo: ["curtin", "in-target", "--", "sh", "-c", "echo '{{ saltstack_repo }}' >> /etc/apt/sources.list.d/mcp_saltstack.list"] ++ apt_03_update: ["curtin", "in-target", "--", "apt-get", "update"] ++{%- if salt['pillar.get']('maas:cluster:curtin_vars:amd64:bionic:extra_pkgs:enabled')|default(false) %} ++ {% for pkg in pillar.maas.cluster.curtin_vars.amd64.bionic.extra_pkgs.pkgs -%} ++ apt_04_install_pkgs_{{ loop.index }}: ["curtin", "in-target", "--", "apt-get", "-y", "install", "{{ pkg }}"] ++ {% endfor %} ++{%- endif %} ++{%- if salt['pillar.get']('maas:cluster:curtin_vars:amd64:bionic:kernel_package:enabled')|default(false) %} ++ apt_05_kernel_old_absent: ["curtin", "in-target", "--", "sh", "-c", "dpkg -l '*linux-generic-*[0-9]*' '*linux-image-*[0-9]*' '*linux-headers-*[0-9]*' '*linux-image-extra-*[0-9]*' '*linux-modules-extra-*[0-9]*' 'linux-generic' 'linux-image-generic' 'linux-headers-generic' 2>/dev/null | grep -E '^ii' | awk '{print $2}' | grep -v '{{ pillar.maas.cluster.curtin_vars.amd64.bionic.kernel_package.value.split('-')[2:-1] | join('-') }}' | xargs dpkg --purge --force-depends"] ++{%- endif %} ++ salt_01_install: ["curtin", "in-target", "--", "apt-get", "-y", "install", "salt-minion", "ifupdown", "cloud-init", "dnsmasq"] ++ salt_02_hostname_set: ["curtin", "in-target", "--", "echo", "{% raw %}{{node.hostname}}{% endraw %}.{{pillar.linux.system.domain}}"] ++ salt_03_hostname_get: ["curtin", "in-target", "--", "sh", "-c", "echo 'id: {% raw %}{{node.hostname}}{% endraw %}.{{pillar.linux.system.domain}}' >> /etc/salt/minion.d/minion.conf"] ++ salt_04_master: ["curtin", "in-target", "--", "sh", "-c", "echo 'master: {{ salt_master_ip }}' >> /etc/salt/minion.d/minion.conf"] ++ salt_05_max_event_size: ["curtin", "in-target", "--", "sh", "-c", "echo 'max_event_size: 100000000' >> /etc/salt/minion.d/minion.conf"] ++ salt_06_acceptance_wait_time_max: ["curtin", "in-target", "--", "sh", "-c", "echo 'acceptance_wait_time_max: 60' >> /etc/salt/minion.d/minion.conf"] ++ salt_07_acceptance_wait_time: ["curtin", "in-target", "--", "sh", "-c", "echo 'acceptance_wait_time: 10' >> /etc/salt/minion.d/minion.conf"] ++ salt_08_random_reauth_delay: ["curtin", "in-target", "--", "sh", "-c", "echo 'random_reauth_delay: 270' >> /etc/salt/minion.d/minion.conf"] ++ salt_09_recon_default: ["curtin", "in-target", "--", "sh", "-c", "echo 'recon_default: 1000' >> /etc/salt/minion.d/minion.conf"] ++ salt_10_recon_max: ["curtin", "in-target", "--", "sh", "-c", "echo 'recon_max: 60000' >> /etc/salt/minion.d/minion.conf"] ++ salt_11_recon_randomize: ["curtin", "in-target", "--", "sh", "-c", "echo 'recon_randomize: True' >> /etc/salt/minion.d/minion.conf"] ++ salt_12_auth_timeout: ["curtin", "in-target", "--", "sh", "-c", "echo 'auth_timeout: 60' >> /etc/salt/minion.d/minion.conf"] ++ ++ salt_20_bionic_nplan_stop: ["curtin", "in-target", "--", "systemctl", "stop", "systemd-networkd.socket", "systemd-networkd", "networkd-dispatcher", "systemd-networkd-wait-online", "systemd-resolved"] ++ salt_21_bionic_nplan_disable: ["curtin", "in-target", "--", "systemctl", "disable", "systemd-networkd.socket", "systemd-networkd", "networkd-dispatcher", "systemd-networkd-wait-online", "systemd-resolved"] ++ salt_22_bionic_nplan_mask: ["curtin", "in-target", "--", "systemctl", "mask", "systemd-networkd.socket", "systemd-networkd", "networkd-dispatcher", "systemd-networkd-wait-online", "systemd-resolved"] ++ salt_23_bionic_nplan_purge: ["curtin", "in-target", "--", "apt", "--assume-yes", "purge", "nplan", "netplan.io"] ++ salt_24_bionic_interfaces: ["curtin", "in-target", "--", "sh", "-c", "echo 'source /etc/network/interfaces.d/*' >> /etc/network/interfaces"] ++ salt_25_bionic_networking_unmask: ["curtin", "in-target", "--", "systemctl", "unmask", "networking.service"] ++ salt_26_bionic_networking_enable: ["curtin", "in-target", "--", "systemctl", "enable", "networking.service"] ++ salt_27_bionic_networking_start: ["curtin", "in-target", "--", "systemctl", "start", "networking.service"] ++ ++{%- if salt['pillar.get']('maas:cluster:curtin_vars:amd64:bionic:kernel_package:enabled')|default(false) %} ++kernel: ++ package: {{ pillar.maas.cluster.curtin_vars.amd64.bionic.kernel_package.value }} ++{%- endif %} ++ ++{% raw %} ++{{if third_party_drivers and driver}} ++ driver_00_key_get: curtin in-target -- sh -c "/bin/echo -en '{{key_string}}' > /tmp/maas-{{driver['package']}}.gpg" ++ driver_02_key_add: ["curtin", "in-target", "--", "apt-key", "add", "/tmp/maas-{{driver['package']}}.gpg"] ++ driver_03_add: ["curtin", "in-target", "--", "add-apt-repository", "-y", "deb {{driver['repository']}} {{node.get_distro_series()}} main"] ++ driver_04_update_install: ["curtin", "in-target", "--", "apt-get", "update", "--quiet"] ++ driver_05_install: ["curtin", "in-target", "--", "apt-get", "-y", "install", "{{driver['package']}}"] ++ driver_06_depmod: ["curtin", "in-target", "--", "depmod"] ++ driver_07_update_initramfs: ["curtin", "in-target", "--", "update-initramfs", "-u"] ++{{endif}} ++{% endraw %} ++ ++{# ++# vim: ft=jinja ++#} +diff --git a/maas/files/curtin_userdata_arm64_generic_bionic b/maas/files/curtin_userdata_arm64_generic_bionic +new file mode 100644 +index 0000000..006d8c2 +--- /dev/null ++++ b/maas/files/curtin_userdata_arm64_generic_bionic +@@ -0,0 +1,79 @@ ++{%- from "maas/map.jinja" import cluster with context %} ++{% raw %} ++#cloud-config ++debconf_selections: ++ maas: | ++ {{for line in str(curtin_preseed).splitlines()}} ++ {{line}} ++ {{endfor}} ++early_commands: ++ thin_tools_install: ["sh", "-c", "apt-get update --quiet && apt-get --assume-yes install thin-provisioning-tools"] ++{{if third_party_drivers and driver}} ++ {{py: key_string = ''.join(['\\x%x' % x for x in map(ord, driver['key_binary'])])}} ++ driver_00_get_key: /bin/echo -en '{{key_string}}' > /tmp/maas-{{driver['package']}}.gpg ++ driver_01_add_key: ["apt-key", "add", "/tmp/maas-{{driver['package']}}.gpg"] ++ driver_02_add: ["add-apt-repository", "-y", "deb {{driver['repository']}} {{node.get_distro_series()}} main"] ++ driver_03_update_install: ["sh", "-c", "apt-get update --quiet && apt-get --assume-yes install {{driver['package']}}"] ++ driver_04_load: ["sh", "-c", "depmod && modprobe {{driver['module']}}"] ++{{endif}} ++late_commands: ++ maas: [wget, '--no-proxy', {{node_disable_pxe_url|escape.json}}, '--post-data', {{node_disable_pxe_data|escape.json}}, '-O', '/dev/null'] ++{% endraw %} ++{%- if cluster.get('saltstack_repo_key', False) %} ++ {% set salt_repo_key = salt['hashutil.base64_b64encode'](cluster.saltstack_repo_key) %} ++ apt_00_set_gpg: ["curtin", "in-target", "--", "sh", "-c", "echo '{{salt_repo_key}}' | base64 -d | apt-key add -"] ++{%- endif %} ++{% if cluster.saltstack_repo_bionic.startswith('deb') %} ++ {%- set saltstack_repo = cluster.saltstack_repo_bionic -%} ++{%- else %} ++ {%- set saltstack_repo = 'deb [arch=amd64] ' + cluster.saltstack_repo_bionic -%} ++{%- endif %} ++{#- NOTE: Re-use amd64 repos on arm64 since most packages are arch independent #} ++ apt_01_set_repo: ["curtin", "in-target", "--", "sh", "-c", "echo '{{ saltstack_repo }}' >> /etc/apt/sources.list.d/mcp_saltstack.list"] ++ apt_03_update: ["curtin", "in-target", "--", "apt-get", "update"] ++{%- if salt['pillar.get']('maas:cluster:curtin_vars:arm64:bionic:extra_pkgs:enabled')|default(false) %} ++ {% for pkg in pillar.maas.cluster.curtin_vars.arm64.bionic.extra_pkgs.pkgs -%} ++ apt_04_install_pkgs_{{ loop.index }}: ["curtin", "in-target", "--", "apt-get", "-y", "install", "{{ pkg }}"] ++ {% endfor %} ++{%- endif %} ++{%- if salt['pillar.get']('maas:cluster:curtin_vars:arm64:bionic:kernel_package:enabled')|default(false) %} ++ apt_05_kernel_old_absent: ["curtin", "in-target", "--", "sh", "-c", "dpkg -l '*linux-generic-*[0-9]*' '*linux-image-*[0-9]*' '*linux-headers-*[0-9]*' '*linux-image-extra-*[0-9]*' '*linux-modules-extra-*[0-9]*' 'linux-generic' 'linux-image-generic' 'linux-headers-generic' 2>/dev/null | grep -E '^ii' | awk '{print $2}' | grep -v '{{ pillar.maas.cluster.curtin_vars.arm64.bionic.kernel_package.value.split('-')[2:-1] | join('-') }}' | xargs dpkg --purge --force-depends"] ++{%- endif %} ++ salt_01_install: ["curtin", "in-target", "--", "apt-get", "-y", "install", "salt-minion", "python-futures", "ifupdown", "cloud-init", "dnsmasq"] ++ salt_02_hostname_set: ["curtin", "in-target", "--", "echo", "{% raw %}{{node.hostname}}{% endraw %}.{{pillar.linux.system.domain}}"] ++ salt_03_hostname_get: ["curtin", "in-target", "--", "sh", "-c", "echo 'id: {% raw %}{{node.hostname}}{% endraw %}.{{pillar.linux.system.domain}}' >> /etc/salt/minion.d/minion.conf"] ++ salt_04_master: ["curtin", "in-target", "--", "sh", "-c", "echo 'master: {{ salt_master_ip }}' >> /etc/salt/minion.d/minion.conf"] ++ salt_05_max_event_size: ["curtin", "in-target", "--", "sh", "-c", "echo 'max_event_size: 100000000' >> /etc/salt/minion.d/minion.conf"] ++ salt_06_acceptance_wait_time_max: ["curtin", "in-target", "--", "sh", "-c", "echo 'acceptance_wait_time_max: 60' >> /etc/salt/minion.d/minion.conf"] ++ salt_07_acceptance_wait_time: ["curtin", "in-target", "--", "sh", "-c", "echo 'acceptance_wait_time: 10' >> /etc/salt/minion.d/minion.conf"] ++ salt_08_random_reauth_delay: ["curtin", "in-target", "--", "sh", "-c", "echo 'random_reauth_delay: 270' >> /etc/salt/minion.d/minion.conf"] ++ salt_09_recon_default: ["curtin", "in-target", "--", "sh", "-c", "echo 'recon_default: 1000' >> /etc/salt/minion.d/minion.conf"] ++ salt_10_recon_max: ["curtin", "in-target", "--", "sh", "-c", "echo 'recon_max: 60000' >> /etc/salt/minion.d/minion.conf"] ++ salt_11_recon_randomize: ["curtin", "in-target", "--", "sh", "-c", "echo 'recon_randomize: True' >> /etc/salt/minion.d/minion.conf"] ++ salt_12_auth_timeout: ["curtin", "in-target", "--", "sh", "-c", "echo 'auth_timeout: 60' >> /etc/salt/minion.d/minion.conf"] ++ ++ salt_20_bionic_nplan_stop: ["curtin", "in-target", "--", "systemctl", "stop", "systemd-networkd.socket", "systemd-networkd", "networkd-dispatcher", "systemd-networkd-wait-online", "systemd-resolved"] ++ salt_21_bionic_nplan_disable: ["curtin", "in-target", "--", "systemctl", "disable", "systemd-networkd.socket", "systemd-networkd", "networkd-dispatcher", "systemd-networkd-wait-online", "systemd-resolved"] ++ salt_22_bionic_nplan_mask: ["curtin", "in-target", "--", "systemctl", "mask", "systemd-networkd.socket", "systemd-networkd", "networkd-dispatcher", "systemd-networkd-wait-online", "systemd-resolved"] ++ salt_23_bionic_nplan_purge: ["curtin", "in-target", "--", "apt", "--assume-yes", "purge", "nplan", "netplan.io"] ++ salt_24_bionic_interfaces: ["curtin", "in-target", "--", "sh", "-c", "echo 'source /etc/network/interfaces.d/*' >> /etc/network/interfaces"] ++ salt_25_bionic_networking_unmask: ["curtin", "in-target", "--", "systemctl", "unmask", "networking.service"] ++ salt_26_bionic_networking_enable: ["curtin", "in-target", "--", "systemctl", "enable", "networking.service"] ++ salt_27_bionic_networking_start: ["curtin", "in-target", "--", "systemctl", "start", "networking.service"] ++ ++{%- if salt['pillar.get']('maas:cluster:curtin_vars:arm64:bionic:kernel_package:enabled')|default(false) %} ++kernel: ++ package: {{ pillar.maas.cluster.curtin_vars.arm64.bionic.kernel_package.value }} ++{%- endif %} ++ ++{% raw %} ++{{if third_party_drivers and driver}} ++ driver_00_key_get: curtin in-target -- sh -c "/bin/echo -en '{{key_string}}' > /tmp/maas-{{driver['package']}}.gpg" ++ driver_02_key_add: ["curtin", "in-target", "--", "apt-key", "add", "/tmp/maas-{{driver['package']}}.gpg"] ++ driver_03_add: ["curtin", "in-target", "--", "add-apt-repository", "-y", "deb {{driver['repository']}} {{node.get_distro_series()}} main"] ++ driver_04_update_install: ["curtin", "in-target", "--", "apt-get", "update", "--quiet"] ++ driver_05_install: ["curtin", "in-target", "--", "apt-get", "-y", "install", "{{driver['package']}}"] ++ driver_06_depmod: ["curtin", "in-target", "--", "depmod"] ++ driver_07_update_initramfs: ["curtin", "in-target", "--", "update-initramfs", "-u"] ++{{endif}} ++{% endraw %} +diff --git a/maas/region.sls b/maas/region.sls +index 52fb952..ca876ee 100644 +--- a/maas/region.sls ++++ b/maas/region.sls +@@ -138,6 +138,30 @@ maas_apache_headers: + - require: + - pkg: maas_region_packages + ++/etc/maas/preseeds/curtin_userdata_amd64_generic_bionic: ++ file.managed: ++ - source: salt://maas/files/curtin_userdata_amd64_generic_bionic ++ - template: jinja ++ - user: root ++ - group: root ++ - mode: 644 ++ - context: ++ salt_master_ip: {{ region.salt_master_ip }} ++ - require: ++ - pkg: maas_region_packages ++ ++/etc/maas/preseeds/curtin_userdata_arm64_generic_bionic: ++ file.managed: ++ - source: salt://maas/files/curtin_userdata_arm64_generic_bionic ++ - template: jinja ++ - user: root ++ - group: root ++ - mode: 644 ++ - context: ++ salt_master_ip: {{ region.salt_master_ip }} ++ - require: ++ - pkg: maas_region_packages ++ + Configure /root/.pgpass for MAAS: + file.managed: + - name: /root/.pgpass diff --git a/mcp/patches/salt-formula-maas/0007-region-s-syncdb-migrate-for-MaaS-2.4-compatibility.patch b/mcp/patches/salt-formula-maas/0007-region-s-syncdb-migrate-for-MaaS-2.4-compatibility.patch new file mode 100644 index 000000000..f0a70ffc4 --- /dev/null +++ b/mcp/patches/salt-formula-maas/0007-region-s-syncdb-migrate-for-MaaS-2.4-compatibility.patch @@ -0,0 +1,30 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Mon, 27 Jan 2020 17:10:04 +0100 +Subject: [PATCH] region: s/syncdb/migrate/ for MaaS 2.4 compatibility + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + maas/region.sls | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/maas/region.sls b/maas/region.sls +index ca876ee..31e9906 100644 +--- a/maas/region.sls ++++ b/maas/region.sls +@@ -186,7 +186,7 @@ maas_region_services: + maas_region_syncdb: + cmd.run: + - names: +- - maas-region syncdb --noinput ++ - maas-region migrate --noinput + - require: + - file: /etc/maas/regiond.conf + {%- if grains['saltversioninfo'][0] >= 2017 and grains['saltversioninfo'][1] >= 7 %} diff --git a/mcp/patches/salt-formula-neutron/0001-Bring-in-basic-VPP-support.patch b/mcp/patches/salt-formula-neutron/0001-Bring-in-basic-VPP-support.patch new file mode 100644 index 000000000..d790ede3d --- /dev/null +++ b/mcp/patches/salt-formula-neutron/0001-Bring-in-basic-VPP-support.patch @@ -0,0 +1,195 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Thu, 6 Dec 2018 18:25:42 +0100 +Subject: [PATCH] Bring in basic VPP support + +TODO: +- update README + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> + +diff --git a/neutron/agents/_vpp.sls b/neutron/agents/_vpp.sls +new file mode 100644 +index 0000000..c1845d0 +--- /dev/null ++++ b/neutron/agents/_vpp.sls +@@ -0,0 +1,25 @@ ++{%- if pillar.neutron.gateway is defined %} ++{%- from "neutron/map.jinja" import gateway as neutron with context %} ++{%- else %} ++{%- from "neutron/map.jinja" import compute as neutron with context %} ++{%- endif %} ++ ++{%- if 'vpp' in neutron.get('backend', {}).get('mechanism', []) %} ++ ++/etc/neutron/plugins/ml2/ml2_conf.ini: ++ file.managed: ++ - source: salt://neutron/files/{{ neutron.version }}/_ml2_conf.vpp.ini ++ - mode: 0640 ++ - user: root ++ - group: neutron ++ - template: jinja ++ ++vpp: ++ service.running: ++ - enable: True ++ ++vpp-agent: ++ service.running: ++ - enable: True ++ ++{%- endif %} +diff --git a/neutron/compute.sls b/neutron/compute.sls +index 708a51d..3767011 100644 +--- a/neutron/compute.sls ++++ b/neutron/compute.sls +@@ -111,7 +111,9 @@ neutron_metadata_agent: + {%- if compute.opendaylight is defined %} + {%- include "neutron/opendaylight/client.sls" %} + {%- else %} ++ {#- We can reuse this for ml2_vpp and ignore openvswitch_agent.ini #} + {%- include "neutron/ml2_ovs/init.sls" %} ++ {%- include "neutron/agents/_vpp.sls" %} + {%- endif %} + + {%- elif compute.backend.engine == "ovn" %} +diff --git a/neutron/files/rocky/_ml2_conf.vpp.ini b/neutron/files/rocky/_ml2_conf.vpp.ini +new file mode 100644 +index 0000000..2373f64 +--- /dev/null ++++ b/neutron/files/rocky/_ml2_conf.vpp.ini +@@ -0,0 +1,41 @@ ++{%- if pillar.neutron.server is defined %} ++{%- from "neutron/map.jinja" import server as neutron with context %} ++{%- elif pillar.neutron.gateway is defined %} ++{%- from "neutron/map.jinja" import gateway as neutron with context %} ++{%- else %} ++{%- from "neutron/map.jinja" import compute as neutron with context %} ++{%- endif %} ++ ++{%- if 'vpp' in neutron.get('backend', {}).get('mechanism', []) %} ++ ++{%- set physnets_vpp = [] %} ++{%- set mechanism_vpp = neutron.backend.mechanism.vpp %} ++{%- for physnet, params in neutron.backend.get('physnets', {}).iteritems() %} ++{%- if params.get('vpp_interface', False) %} ++{%- do physnets_vpp.append([physnet, params.get('vpp_interface')]|join(":")) %} ++{%- endif %} ++{%- endfor %} ++{%- if not physnets_vpp %} ++{%- do physnets_vpp.append('physnet1:tap-0') %} ++{%- endif %} ++ ++{%- if pillar.neutron.server is not defined %} ++[ml2] ++type_drivers = flat,vlan ++{%- endif %} ++ ++[ml2_vpp] ++jwt_signing = False ++etcd_insecure_explicit_disable_https = True ++l3_hosts = {{ mechanism_vpp.get('l3_hosts', '127.0.0.1') }} ++enable_l3_ha = False ++gpe_locators = ++gpe_src_cidr = ++enable_vpp_restart = False ++etcd_pass = {{ mechanism_vpp.get('etcd_pass', '') }} ++etcd_user = {{ mechanism_vpp.get('etcd_user', '') }} ++etcd_port = {{ mechanism_vpp.get('etcd_port', 2379) }} ++etcd_host = {{ mechanism_vpp.get('etcd_host', '127.0.0.1') }} ++physnets = {{ ','.join(physnets_vpp) }} ++ ++{%- endif %} +diff --git a/neutron/files/rocky/ml2_conf.ini b/neutron/files/rocky/ml2_conf.ini +index a9a598f..4429c80 100644 +--- a/neutron/files/rocky/ml2_conf.ini ++++ b/neutron/files/rocky/ml2_conf.ini +@@ -27,6 +27,9 @@ agent_boot_time = {{ server.get('agent_boot_time', 180) }} + # List of network type driver entrypoints to be loaded from the + # neutron.ml2.type_drivers namespace. (list value) + #type_drivers = local,flat,vlan,gre,vxlan,geneve ++{%- if 'vpp' in server.backend.get('mechanism', []) %} ++type_drivers = flat,vlan ++{%- endif %} + + # Ordered list of network_types to allocate as tenant networks. The default + # value 'local' is useful for single-box testing but provides no connectivity +@@ -239,6 +242,7 @@ neutron_sync_mode = {{ _ovn.neutron_sync_mode|default('repair') }} + enable_distributed_floating_ip = {{ server.dvr|default('false') }} + {%- endif %} + ++{%- include "neutron/files/rocky/_ml2_conf.vpp.ini" %} + + {%- if server.backend.opendaylight|default(False) %} + [ml2_odl] +diff --git a/neutron/gateway.sls b/neutron/gateway.sls +index e51990a..95d4d07 100644 +--- a/neutron/gateway.sls ++++ b/neutron/gateway.sls +@@ -40,6 +40,8 @@ haproxy: + + {%- endif %} + ++{%- include "neutron/agents/_vpp.sls" %} ++ + {%- if gateway.l2gw is defined %} + {%- include "neutron/agents/_l2gw.sls" %} + {%- endif %} +diff --git a/neutron/map.jinja b/neutron/map.jinja +index 9e6cb36..17cd5b3 100644 +--- a/neutron/map.jinja ++++ b/neutron/map.jinja +@@ -14,9 +14,13 @@ + {%- do compute_pkgs_ovn.extend(['neutron-common', 'python-networking-ovn', 'haproxy']) %} + {%- endif %} + {%- set linuxbridge_enabled = pillar.neutron.compute is defined and pillar.neutron.compute.get('backend', {}).get('mechanism', {}).get('lb', {}).get('driver', {}) == "linuxbridge" %} ++{%- set vpp_enabled = 'vpp' in pillar.neutron.get('compute', {}).get('backend', {}).get('mechanism', []) %} + {%- if linuxbridge_enabled %} + {%- set pkgs_cmp = ['neutron-linuxbridge-agent'] %} + {%- set services_cmp = ['neutron-linuxbridge-agent'] %} ++{%- elif vpp_enabled %} ++{%- set pkgs_cmp = ['vpp-agent'] %} ++{%- set services_cmp = ['vpp-agent'] %} + {%- else %} + {%- set pkgs_cmp = ['neutron-openvswitch-agent', 'python-pycadf'] %} + {%- set services_cmp = ['neutron-openvswitch-agent'] %} +@@ -73,6 +77,7 @@ + {%- set opendaylight_enabled = pillar.neutron.gateway is defined and pillar.neutron.gateway.opendaylight is defined %} + {%- set linuxbridge_enabled = pillar.neutron.gateway is defined and pillar.neutron.gateway.get('backend', {}).get('mechanism', {}).get('lb', {}).get('driver', {}) == "linuxbridge" %} + {%- set dhcp_enabled = pillar.neutron.gateway is defined and pillar.neutron.gateway.get('dhcp_agent_enabled', True) %} ++{%- set vpp_enabled = 'vpp' in pillar.neutron.get('gateway', {}).get('backend', {}).get('mechanism', []) %} + {%- set pkgs_list = ['neutron-metadata-agent'] %} + + {%- set services_list = ['neutron-metadata-agent'] %} +@@ -83,6 +88,13 @@ + {%- if linuxbridge_enabled %} + {%- do pkgs_list.extend(['neutron-linuxbridge-agent', 'neutron-l3-agent']) %} + {%- do services_list.extend(['neutron-linuxbridge-agent', 'neutron-l3-agent']) %} ++{%- elif vpp_enabled %} ++{%- do pkgs_list.extend(['vpp-agent']) %} ++{%- do services_list.extend(['vpp-agent']) %} ++{%- if 'vpp-router' not in pillar.neutron.gateway.backend.get('router', '') %} ++{%- do pkgs_list.extend(['neutron-l3-agent']) %} ++{%- do services_list.extend(['neutron-l3-agent']) %} ++{%- endif %} + {%- elif not opendaylight_enabled %} + {%- do pkgs_list.extend(['neutron-openvswitch-agent', 'neutron-l3-agent']) %} + {%- do services_list.extend(['neutron-openvswitch-agent', 'neutron-l3-agent']) %} +@@ -132,6 +144,10 @@ + {%- set server_services_list = ['neutron-server'] %} + {%- do server_services_list.append('neutron-rpc-server') if wsgi_enabled %} + ++{%- if 'vpp' in pillar.neutron.get('server', {}).get('backend', {}).get('mechanism', []) %} ++{%- do server_pkgs_list.extend(['python3-networking-vpp']) %} ++{%- endif %} ++ + {% set server = salt['grains.filter_by']({ + 'BaseDefaults': default_params, + 'Debian': { diff --git a/mcp/patches/salt-formula-neutron/0002-Align-packages-with-stein-reqs.patch b/mcp/patches/salt-formula-neutron/0002-Align-packages-with-stein-reqs.patch new file mode 100644 index 000000000..6dcd91a3c --- /dev/null +++ b/mcp/patches/salt-formula-neutron/0002-Align-packages-with-stein-reqs.patch @@ -0,0 +1,114 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Michael Polenchuk <mpolenchuk@mirantis.com> +Date: Thu, 11 Jul 2019 11:52:53 +0400 +Subject: [PATCH] Align packages with stein reqs + +diff --git a/neutron/map.jinja b/neutron/map.jinja +index 17cd5b3..ed1831a 100644 +--- a/neutron/map.jinja ++++ b/neutron/map.jinja +@@ -11,7 +11,7 @@ + {%- if os_family == "Debian" %} + {%- set compute_pkgs_ovn = ['ovn-common', 'ovn-host'] %} + {%- if pillar.neutron.compute is defined and pillar.neutron.compute.metadata is defined %} +-{%- do compute_pkgs_ovn.extend(['neutron-common', 'python-networking-ovn', 'haproxy']) %} ++{%- do compute_pkgs_ovn.extend(['neutron-common', 'python3-networking-ovn', 'haproxy']) %} + {%- endif %} + {%- set linuxbridge_enabled = pillar.neutron.compute is defined and pillar.neutron.compute.get('backend', {}).get('mechanism', {}).get('lb', {}).get('driver', {}) == "linuxbridge" %} + {%- set vpp_enabled = 'vpp' in pillar.neutron.get('compute', {}).get('backend', {}).get('mechanism', []) %} +@@ -22,7 +22,7 @@ + {%- set pkgs_cmp = ['vpp-agent'] %} + {%- set services_cmp = ['vpp-agent'] %} + {%- else %} +-{%- set pkgs_cmp = ['neutron-openvswitch-agent', 'python-pycadf'] %} ++{%- set pkgs_cmp = ['neutron-openvswitch-agent'] %} + {%- set services_cmp = ['neutron-openvswitch-agent'] %} + {%- endif %} + {%- endif %} +@@ -33,7 +33,7 @@ + 'Debian': { + 'pkgs': pkgs_cmp, + 'pkgs_ovn': compute_pkgs_ovn, +- 'pkgs_bagpipe': ['python-networking-bagpipe'], ++ 'pkgs_bagpipe': ['python3-networking-bagpipe'], + 'services': services_cmp, + 'services_ovn': ['ovn-host'], + 'dpdk': false, +@@ -138,9 +138,9 @@ + + {%- set sfc_enabled = pillar.neutron.get('server', {}).get('sfc', {}).get('enabled', False) %} + {%- set wsgi_enabled = pillar.neutron.get('server', {}).get('wsgi', {}).get('enabled', False) %} +-{%- set server_pkgs_list = ['python-neutron-lbaas'] %} ++{%- set server_pkgs_list = [] %} + {%- do server_pkgs_list.append('uwsgi-plugin-python' if wsgi_enabled else 'neutron-server') %} +-{%- do server_pkgs_list.append('python-networking-sfc') if sfc_enabled %} ++{%- do server_pkgs_list.append('python3-networking-sfc') if sfc_enabled %} + {%- set server_services_list = ['neutron-server'] %} + {%- do server_services_list.append('neutron-rpc-server') if wsgi_enabled %} + +@@ -152,11 +152,11 @@ + 'BaseDefaults': default_params, + 'Debian': { + 'pkgs': server_pkgs_list, +- 'pkgs_ovn': ['python-networking-ovn', 'ovn-common', 'ovn-central'], ++ 'pkgs_ovn': ['python3-networking-ovn', 'ovn-common', 'ovn-central'], + 'pkgs_ml2': ['neutron-plugin-ml2'], +- 'pkgs_l2gw': ['python-networking-l2gw'], +- 'pkgs_bgpvpn': ['python-networking-bgpvpn'], +- 'pkgs_bagpipe': ['python-networking-bagpipe'], ++ 'pkgs_l2gw': ['python3-networking-l2gw'], ++ 'pkgs_bgpvpn': ['python3-networking-bgpvpn'], ++ 'pkgs_bagpipe': ['python3-networking-bagpipe'], + 'services': server_services_list, + 'services_ovn': ['openvswitch-switch', 'ovn-central'], + 'notification': {}, +@@ -204,7 +204,7 @@ + + {% set client = salt['grains.filter_by']({ + 'Debian': { +- 'pkgs': ['python-neutronclient'], ++ 'pkgs': ['python3-neutronclient'], + 'enabled': false + }, + 'RedHat': { +diff --git a/neutron/opendaylight/client.sls b/neutron/opendaylight/client.sls +index fb5829f..be9aff6 100644 +--- a/neutron/opendaylight/client.sls ++++ b/neutron/opendaylight/client.sls +@@ -4,7 +4,7 @@ + {%- from "neutron/map.jinja" import compute as neutron with context %} + {%- endif %} + +-python-networking-odl: ++python3-networking-odl: + pkg.installed + + {%- if not grains.get('noservices', False) %} +@@ -31,6 +31,6 @@ neutron_odl_ovs_hostconfig: + cmd.run: + - name: 'neutron-odl-ovs-hostconfig {{ ovs_hostconfig|join(' ') }}' + - require: +- - pkg: python-networking-odl ++ - pkg: python3-networking-odl + + {%- endif %} +diff --git a/neutron/server.sls b/neutron/server.sls +index 99bbdef..692e0ce 100644 +--- a/neutron/server.sls ++++ b/neutron/server.sls +@@ -115,7 +115,7 @@ ml2_plugin_link: + {%- endif %} + + {%- if server.backend.get('opendaylight', False) %} +-python-networking-odl: ++python3-networking-odl: + pkg.installed: + - require_in: + - pkg: neutron_server_packages diff --git a/mcp/patches/salt-formula-rabbitmq/0001-Stop-epmd.socket-before-relaunching-rabbit-service.patch b/mcp/patches/salt-formula-rabbitmq/0001-Stop-epmd.socket-before-relaunching-rabbit-service.patch new file mode 100644 index 000000000..f7b85f5ab --- /dev/null +++ b/mcp/patches/salt-formula-rabbitmq/0001-Stop-epmd.socket-before-relaunching-rabbit-service.patch @@ -0,0 +1,34 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Fri, 18 Oct 2019 17:07:13 +0200 +Subject: [PATCH] Stop epmd.socket before relaunching rabbit service + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + rabbitmq/server/service.sls | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/rabbitmq/server/service.sls b/rabbitmq/server/service.sls +index 3e95a30..05f2eb1 100644 +--- a/rabbitmq/server/service.sls ++++ b/rabbitmq/server/service.sls +@@ -58,6 +58,12 @@ rabbitmq_limits_systemd: + - require: + - pkg: rabbitmq_server + ++rabbitmq_epmd_socket: ++ service.dead: ++ - name: epmd.socket ++ - require: ++ - pkg: rabbitmq_server ++ + {%- endif %} + + {%- if server.secret_key is defined and not grains.get('noservices', False) %} diff --git a/mcp/patches/salt-formula-redis/0001-Add-Ubuntu-Bionic-support.patch b/mcp/patches/salt-formula-redis/0001-Add-Ubuntu-Bionic-support.patch new file mode 100644 index 000000000..1c06bd9d3 --- /dev/null +++ b/mcp/patches/salt-formula-redis/0001-Add-Ubuntu-Bionic-support.patch @@ -0,0 +1,44 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2019 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +Date: Thu, 24 Oct 2019 23:04:16 +0200 +Subject: [PATCH] Add Ubuntu Bionic support + +Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> +--- + redis/map.jinja | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/redis/map.jinja b/redis/map.jinja +index 299d5a4..7cbceb4 100755 +--- a/redis/map.jinja ++++ b/redis/map.jinja +@@ -37,6 +37,9 @@ + 'xenial': { + 'version': '3.0', + }, ++ 'bionic': { ++ 'version': '3.0', ++ }, + }, grain='oscodename', merge=salt['pillar.get']('redis:server'))) %} + + {% set cluster = salt['grains.filter_by']({ +@@ -60,4 +63,12 @@ + 'port': '26379' + } + }, ++ 'bionic': { ++ 'pkgs': ['redis-sentinel'], ++ 'service': 'redis-sentinel', ++ 'sentinel': { ++ 'address': '127.0.0.1', ++ 'port': '26379' ++ } ++ }, + }, grain='oscodename', merge=salt['pillar.get']('redis:cluster'))) %} diff --git a/mcp/patches/scripts/0001-salt-master-setup.sh-Allow-arm64-salt-bootstrap.patch b/mcp/patches/scripts/0001-salt-master-setup.sh-Allow-arm64-salt-bootstrap.patch deleted file mode 100644 index 498743b77..000000000 --- a/mcp/patches/scripts/0001-salt-master-setup.sh-Allow-arm64-salt-bootstrap.patch +++ /dev/null @@ -1,59 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Fri, 1 Sep 2017 00:48:26 +0200 -Subject: [PATCH] salt-master-setup.sh: Allow arm64 salt-bootstrap - -Upstream commit [1] broke Salt bootstrap on AArch64, by -introducing an architecture condition that is too strict to allow -Debian package installation (even if we provide our own repo). - -Add "arm64" to the list of supported architectures. This needs -to be done on the fly, as the bootstrap script is fetched using -`curl` from <salt-master-setup.sh>. - -[1] https://github.com/saltstack/salt-bootstrap/commit/caa6d7d - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - salt-master-setup.sh | 8 ++++---- - 1 file changed, 4 insertions(+), 4 deletions(-) - -diff --git a/salt-master-setup.sh b/salt-master-setup.sh -index cba21fb..0dd3036 100755 ---- a/salt-master-setup.sh -+++ b/salt-master-setup.sh -@@ -237,12 +237,12 @@ install_salt_master_pkg() - debian) - $SUDO apt-get install -y git - which reclass || $SUDO apt install -qqq -y reclass -- curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true -+ curl -L https://bootstrap.saltstack.com | sed 's@"amd64")@"amd64"|"arm64")@g' | $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true - ;; - rhel) - yum install -y git - which reclass || $SUDO yum install -y reclass -- curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true -+ curl -L https://bootstrap.saltstack.com | sed 's@"amd64")@"amd64"|"arm64")@g' | $SUDO sh -s -- -M ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true - ;; - esac - -@@ -310,10 +310,10 @@ install_salt_minion_pkg() - - case $PLATFORM_FAMILY in - debian) -- curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true -+ curl -L https://bootstrap.saltstack.com | sed 's@"amd64")@"amd64"|"arm64")@g' | $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true - ;; - rhel) -- curl -L https://bootstrap.saltstack.com | $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true -+ curl -L https://bootstrap.saltstack.com | sed 's@"amd64")@"amd64"|"arm64")@g' | $SUDO sh -s -- ${BOOTSTRAP_SALTSTACK_OPTS} &>/dev/null || true - ;; - esac - diff --git a/mcp/patches/scripts/0002-salt-master-init.sh-Apply-OPNFV-Fuel-patches.patch b/mcp/patches/scripts/0002-salt-master-init.sh-Apply-OPNFV-Fuel-patches.patch deleted file mode 100644 index 6c7cbb6e8..000000000 --- a/mcp/patches/scripts/0002-salt-master-init.sh-Apply-OPNFV-Fuel-patches.patch +++ /dev/null @@ -1,35 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Fri, 8 Dec 2017 20:30:46 +0100 -Subject: [PATCH] salt-master-init.sh: Apply OPNFV Fuel patches - -Some of Fuel@OPNFV patches need to be applied before the reclass -storage.node state is ran for Salt Master, i.e. between installing -salt-formula-* packages and configuring the Salt Master salt services. - -JIRA: FUEL-310 - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - salt-master-init.sh | 1 + - 1 file changed, 1 insertion(+) - -diff --git a/salt-master-init.sh b/salt-master-init.sh -index 343324c..a4ec138 100755 ---- a/salt-master-init.sh -+++ b/salt-master-init.sh -@@ -284,6 +284,7 @@ options - system_config - - saltmaster_bootstrap &&\ -+ /root/fuel/mcp/patches/patch.sh /root/fuel/mcp/patches/patches_init.list formulas &&\ - saltmaster_init &&\ - - verify_salt_minions diff --git a/mcp/patches/scripts/0003-salt-master-setup-Group-APT-install-formulas.patch b/mcp/patches/scripts/0003-salt-master-setup-Group-APT-install-formulas.patch deleted file mode 100644 index d5a9bf3c4..000000000 --- a/mcp/patches/scripts/0003-salt-master-setup-Group-APT-install-formulas.patch +++ /dev/null @@ -1,68 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii <Alexandru.Avadanii@enea.com> -Date: Tue, 30 Jan 2018 01:23:54 +0100 -Subject: [PATCH] salt-master-setup: Group APT install formulas - -Instead of calling `apt install` for each salt formula package that -we miss, construct a list and install them all at once. - -While at it, disable colored output on terminals that don't support -it, like vt220 (used in OPNFV CI). - -Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com> ---- - salt-master-init.sh | 6 ++++++ - salt-master-setup.sh | 12 +++++++----- - 2 files changed, 13 insertions(+), 5 deletions(-) - -diff --git a/salt-master-init.sh b/salt-master-init.sh -index a4ec138..db90a22 100755 ---- a/salt-master-init.sh -+++ b/salt-master-init.sh -@@ -24,6 +24,12 @@ options() { - export $(find $path -maxdepth 1 -name '*.env' 2> /dev/null | xargs --no-run-if-empty cat ) > /dev/null - done; - -+ # If terminal does not support color output, stop using it -+ if ! tput setaf 1 && tput sgr0; then -+ SALT_OPTS="${SALT_OPTS/--force-color/--no-color}" -+ return 0 -+ fi -+ - export MAGENTA='\033[0;95m' - export YELLOW='\033[1;33m' - export BLUE='\033[0;35m' -diff --git a/salt-master-setup.sh b/salt-master-setup.sh -index 0dd3036..a0c6311 100755 ---- a/salt-master-setup.sh -+++ b/salt-master-setup.sh -@@ -347,15 +347,17 @@ install_salt_formula_pkg() - # Set essentials if FORMULAS_SALT_MASTER is not defined at all - [ -z ${FORMULAS_SALT_MASTER+x} ] && declare -a FORMULAS_SALT_MASTER=("linux" "reclass" "salt" "memcached") - for formula_service in "${FORMULAS_SALT_MASTER[@]}"; do -- echo -e "\nConfiguring salt formula ${formula_service} ...\n" -+ echo -e "Configuring salt formula ${formula_service} ..." - [ ! -d "${FORMULAS_PATH}/env/${formula_service}" ] && \ -- if ! $SUDO apt-get install -y salt-formula-${formula_service}; then -- echo -e "\nInstall salt-formula-${formula_service} failed.\n" -- exit 1 -- fi -+ _FORMULAS_SALT_MASTER="${_FORMULAS_SALT_MASTER} salt-formula-${formula_service}" -+ # Create links first, install pkgs later - [ ! -L "/srv/salt/reclass/classes/service/${formula_service}" ] && \ - ln -sf ${FORMULAS_PATH}/reclass/service/${formula_service} /srv/salt/reclass/classes/service/${formula_service} - done -+ if ! $SUDO apt-get install -qqq -y ${_FORMULAS_SALT_MASTER}; then -+ echo -e "\nInstall ${_FORMULAS_SALT_MASTER} failed.\n" -+ exit 1 -+ fi - ;; - rhel) - # TODO |