From ab18375a629010525ac15bc11ce2d4e4cf393fe9 Mon Sep 17 00:00:00 2001 From: Alexandru Avadanii Date: Sat, 18 Aug 2018 03:46:13 +0200 Subject: [docker] Switch to containerized Salt Master * Refactor OPNFV salt-formulas mechanism to resemble upstream git structure: - git submodules: add new submodule for each formula we patch; - create salt-formula-x directories for OPNFV formulas; - move mcp/metadata/service contents to their each formula subdir; - use `make patches-import` for patches previously handled by patch.sh; - retire patch.sh * states: add virtual_init: - mostly based on old salt.sh, which is now obsolete; - exclude salt-master service restart (it would kill the container); * scenarios: cleanup (rm cfg01 virtual node def), adopt virtual_init; * reclass: align our model with prebuilt container's Salt config: - drop linux:network pillar data (handled by Docker); - stop applying linux.system state on cfg01; - align salt user homedir; - drop salt-formula packages (preprovisioned); * minor plumbing in deploy.sh and lib.sh; JIRA: FUEL-383 Change-Id: I28708a9b399d3f19012212c71966ebda9d6fc0ac Signed-off-by: Alexandru Avadanii --- .gitmodules | 16 +++ ci/deploy.sh | 11 +- mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 | 5 +- .../scenario/os-nosdn-nofeature-noha.yaml.j2 | 2 +- mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 | 5 +- mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2 | 2 +- mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 | 5 +- mcp/config/scenario/os-odl-nofeature-noha.yaml.j2 | 2 +- mcp/config/scenario/os-odl-ovs-noha.yaml.j2 | 2 +- mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 | 5 +- mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2 | 2 +- mcp/config/states/virtual_init | 39 +++++ .../service/opendaylight/server/single.yml | 20 --- mcp/metadata/service/opendaylight/support.yml | 19 --- mcp/metadata/service/tacker/server/single.yml | 34 ----- .../0002-maas-region-skip-credentials-update.patch | 38 ----- .../0008-Handle-extra-environment-variables.patch | 25 ---- .../0010-maas-region-allow-timeout-override.patch | 93 ------------ ...m.repo-Debian-Add-keyserver-proxy-support.patch | 157 --------------------- .../0015-Set-ovs-bridges-as-L3-interfaces.patch | 95 ------------- mcp/patches/patch.sh | 19 --- mcp/patches/patches.list | 12 -- .../0001-Handle-extra-environment-variables.patch | 28 ++++ ...stem.repo-Debian-Use-proxy-for-keyservers.patch | 157 +++++++++++++++++++++ .../0002-Set-ovs-bridges-as-L3-interfaces.patch | 101 +++++++++++++ .../0001-maas-region-skip-credentials-update.patch | 40 ++++++ .../0002-maas-region-allow-timeout-override.patch | 95 +++++++++++++ .../0003-Add-machines.delete-co-pxe_nat-sls.patch | 137 ++++++++++++++++++ .../all-mcp-arch-common/infra/config_pdf.yml.j2 | 9 +- .../cluster/mcp-common-ha/infra/config.yml.j2 | 31 ---- .../cluster/mcp-common-noha/infra/config.yml.j2 | 31 ---- mcp/salt-formulas/maas/machines/delete.sls | 20 --- .../maas/machines/mark_broken_fixed.sls | 20 --- .../maas/machines/override_failed_testing.sls | 20 --- mcp/salt-formulas/maas/pxe_nat.sls | 37 ----- mcp/salt-formulas/opendaylight/files/jetty.xml | 132 ----------------- mcp/salt-formulas/opendaylight/files/setenv.shell | 21 --- mcp/salt-formulas/opendaylight/init.sls | 12 -- mcp/salt-formulas/opendaylight/map.jinja | 28 ---- mcp/salt-formulas/opendaylight/server.sls | 113 --------------- mcp/salt-formulas/salt-formula-keystone | 1 + mcp/salt-formulas/salt-formula-linux | 1 + mcp/salt-formulas/salt-formula-maas | 1 + mcp/salt-formulas/salt-formula-nova | 1 + .../metadata/service/server/single.yml | 20 +++ .../metadata/service/support.yml | 19 +++ .../opendaylight/files/jetty.xml | 132 +++++++++++++++++ .../opendaylight/files/setenv.shell | 21 +++ .../opendaylight/init.sls | 12 ++ .../opendaylight/map.jinja | 28 ++++ .../opendaylight/server.sls | 113 +++++++++++++++ .../metadata/service/server/single.yml | 34 +++++ .../salt-formula-tacker/tacker/files/tacker.conf | 45 ++++++ .../tacker/files/tacker.systemd | 20 +++ .../salt-formula-tacker/tacker/init.sls | 11 ++ .../salt-formula-tacker/tacker/map.jinja | 21 +++ .../salt-formula-tacker/tacker/server.sls | 56 ++++++++ mcp/salt-formulas/tacker/files/tacker.conf | 45 ------ mcp/salt-formulas/tacker/files/tacker.systemd | 20 --- mcp/salt-formulas/tacker/init.sls | 11 -- mcp/salt-formulas/tacker/map.jinja | 21 --- mcp/salt-formulas/tacker/server.sls | 56 -------- mcp/scripts/lib.sh | 43 +++++- mcp/scripts/salt.sh | 132 ----------------- mcp/scripts/xdf_data.sh.j2 | 4 +- 65 files changed, 1210 insertions(+), 1298 deletions(-) create mode 100755 mcp/config/states/virtual_init delete mode 100644 mcp/metadata/service/opendaylight/server/single.yml delete mode 100644 mcp/metadata/service/opendaylight/support.yml delete mode 100644 mcp/metadata/service/tacker/server/single.yml delete mode 100644 mcp/patches/0002-maas-region-skip-credentials-update.patch delete mode 100644 mcp/patches/0008-Handle-extra-environment-variables.patch delete mode 100644 mcp/patches/0010-maas-region-allow-timeout-override.patch delete mode 100644 mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch delete mode 100644 mcp/patches/0015-Set-ovs-bridges-as-L3-interfaces.patch delete mode 100755 mcp/patches/patch.sh delete mode 100644 mcp/patches/patches.list create mode 100644 mcp/patches/salt-formula-keystone/0001-Handle-extra-environment-variables.patch create mode 100644 mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch create mode 100644 mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch create mode 100644 mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch create mode 100644 mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch create mode 100644 mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch delete mode 100644 mcp/salt-formulas/maas/machines/delete.sls delete mode 100644 mcp/salt-formulas/maas/machines/mark_broken_fixed.sls delete mode 100644 mcp/salt-formulas/maas/machines/override_failed_testing.sls delete mode 100644 mcp/salt-formulas/maas/pxe_nat.sls delete mode 100644 mcp/salt-formulas/opendaylight/files/jetty.xml delete mode 100644 mcp/salt-formulas/opendaylight/files/setenv.shell delete mode 100644 mcp/salt-formulas/opendaylight/init.sls delete mode 100644 mcp/salt-formulas/opendaylight/map.jinja delete mode 100644 mcp/salt-formulas/opendaylight/server.sls create mode 160000 mcp/salt-formulas/salt-formula-keystone create mode 160000 mcp/salt-formulas/salt-formula-linux create mode 160000 mcp/salt-formulas/salt-formula-maas create mode 160000 mcp/salt-formulas/salt-formula-nova create mode 100644 mcp/salt-formulas/salt-formula-opendaylight/metadata/service/server/single.yml create mode 100644 mcp/salt-formulas/salt-formula-opendaylight/metadata/service/support.yml create mode 100644 mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/jetty.xml create mode 100644 mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/setenv.shell create mode 100644 mcp/salt-formulas/salt-formula-opendaylight/opendaylight/init.sls create mode 100644 mcp/salt-formulas/salt-formula-opendaylight/opendaylight/map.jinja create mode 100644 mcp/salt-formulas/salt-formula-opendaylight/opendaylight/server.sls create mode 100644 mcp/salt-formulas/salt-formula-tacker/metadata/service/server/single.yml create mode 100644 mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.conf create mode 100644 mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.systemd create mode 100644 mcp/salt-formulas/salt-formula-tacker/tacker/init.sls create mode 100644 mcp/salt-formulas/salt-formula-tacker/tacker/map.jinja create mode 100644 mcp/salt-formulas/salt-formula-tacker/tacker/server.sls delete mode 100644 mcp/salt-formulas/tacker/files/tacker.conf delete mode 100644 mcp/salt-formulas/tacker/files/tacker.systemd delete mode 100644 mcp/salt-formulas/tacker/init.sls delete mode 100644 mcp/salt-formulas/tacker/map.jinja delete mode 100644 mcp/salt-formulas/tacker/server.sls delete mode 100755 mcp/scripts/salt.sh diff --git a/.gitmodules b/.gitmodules index d3501c3ec..767cf14df 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,3 +14,19 @@ path = docker url = https://github.com/epcim/docker-salt-formulas branch = master +[submodule "salt-formula-linux"] + path = mcp/salt-formulas/salt-formula-linux + url = https://github.com/salt-formulas/salt-formula-linux + branch = master +[submodule "salt-formula-nova"] + path = mcp/salt-formulas/salt-formula-nova + url = https://github.com/salt-formulas/salt-formula-nova + branch = master +[submodule "salt-formula-keystone"] + path = mcp/salt-formulas/salt-formula-keystone + url = https://github.com/salt-formulas/salt-formula-keystone + branch = master +[submodule "salt-formula-maas"] + path = mcp/salt-formulas/salt-formula-maas + url = https://github.com/salt-formulas/salt-formula-maas + branch = master diff --git a/ci/deploy.sh b/ci/deploy.sh index 47662f796..d3882a834 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -241,7 +241,7 @@ if [ ${USE_EXISTING_PKGS} -eq 1 ]; then else notify "[NOTE] Installing required distro pkgs" 2 jumpserver_pkg_install 'deploy' - docker_install + docker_install "${MCP_STORAGE_DIR}" fi if ! virsh list >/dev/null 2>&1; then @@ -289,22 +289,21 @@ if [ ${DRY_RUN} -eq 1 ]; then exit 0 elif [ ${USE_EXISTING_INFRA} -gt 0 ]; then notify "[NOTE] Use existing infra" 2 - check_connection else prepare_vms "${base_image}" "${MCP_STORAGE_DIR}" "${virtual_repos_pkgs}" \ "${virtual_nodes[@]}" + prepare_containers "${MCP_STORAGE_DIR}" create_networks "${OPNFV_BRIDGES[@]}" do_sysctl_cfg do_udev_cfg create_vms "${MCP_STORAGE_DIR}" "${virtual_nodes_data}" "${OPNFV_BRIDGES[@]}" update_mcpcontrol_network start_vms "${virtual_nodes[@]}" - check_connection -fi -if [ ${USE_EXISTING_INFRA} -lt 2 ]; then - wait_for 5 "./salt.sh ${MCP_STORAGE_DIR}/pod_config.yml ${virtual_nodes[*]}" fi +start_containers "${MCP_STORAGE_DIR}" +check_connection + # Openstack cluster setup set +x if [ ${INFRA_CREATION_ONLY} -eq 1 ] || [ ${NO_DEPLOY_ENVIRONMENT} -eq 1 ]; then diff --git a/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 index 7eb604e65..55da8f74b 100644 --- a/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 +++ b/mcp/config/scenario/os-nosdn-nofeature-ha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovs-ha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -22,7 +23,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} @@ -34,9 +34,6 @@ virtual: - cmp001 - cmp002 # Below values are only used for each node if said node is virtual - cfg01: - vcpus: 4 - ram: 6144 mas01: vcpus: 4 ram: 6144 diff --git a/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 b/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 index a0ad96b87..cd214a5cd 100644 --- a/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 +++ b/mcp/config/scenario/os-nosdn-nofeature-noha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovs-noha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -20,7 +21,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} diff --git a/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 b/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 index da9938ce0..0f7a7bc34 100644 --- a/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 +++ b/mcp/config/scenario/os-nosdn-ovs-ha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovs-dpdk-ha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -23,7 +24,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} @@ -35,9 +35,6 @@ virtual: - cmp001 - cmp002 # Below values are only used for each node if said node is virtual - cfg01: - vcpus: 4 - ram: 6144 mas01: vcpus: 4 ram: 6144 diff --git a/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2 b/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2 index bee0e926f..bf37f93ea 100644 --- a/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2 +++ b/mcp/config/scenario/os-nosdn-ovs-noha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovs-dpdk-noha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -21,7 +22,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} diff --git a/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 index fbedbc6b9..b31dc136e 100644 --- a/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 +++ b/mcp/config/scenario/os-odl-nofeature-ha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-odl-ha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -23,7 +24,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} @@ -35,9 +35,6 @@ virtual: - cmp001 - cmp002 # Below values are only used for each node if said node is virtual - cfg01: - vcpus: 4 - ram: 6144 mas01: vcpus: 4 ram: 6144 diff --git a/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2 b/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2 index 7df28c2b3..2a10a3163 100644 --- a/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2 +++ b/mcp/config/scenario/os-odl-nofeature-noha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-odl-noha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -21,7 +22,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} diff --git a/mcp/config/scenario/os-odl-ovs-noha.yaml.j2 b/mcp/config/scenario/os-odl-ovs-noha.yaml.j2 index c81d35c51..62c120b26 100644 --- a/mcp/config/scenario/os-odl-ovs-noha.yaml.j2 +++ b/mcp/config/scenario/os-odl-ovs-noha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-odl-noha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -22,7 +23,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} diff --git a/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 b/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 index f81779b8a..021548f63 100644 --- a/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 +++ b/mcp/config/scenario/os-ovn-nofeature-ha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovn-ha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -22,7 +23,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} @@ -34,9 +34,6 @@ virtual: - cmp001 - cmp002 # Below values are only used for each node if said node is virtual - cfg01: - vcpus: 4 - ram: 6144 mas01: vcpus: 4 ram: 6144 diff --git a/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2 b/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2 index fee47d7eb..635106355 100644 --- a/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2 +++ b/mcp/config/scenario/os-ovn-nofeature-noha.yaml.j2 @@ -10,6 +10,7 @@ cluster: domain: mcp-ovn-noha.local states: + - virtual_init {%- if nm.cluster.has_baremetal_nodes %} - maas - baremetal_init @@ -19,7 +20,6 @@ cluster: virtual: nodes: infra: - - cfg01 {%- if nm.cluster.has_baremetal_nodes %} - mas01 {%- endif %} diff --git a/mcp/config/states/virtual_init b/mcp/config/states/virtual_init new file mode 100755 index 000000000..147e6b05e --- /dev/null +++ b/mcp/config/states/virtual_init @@ -0,0 +1,39 @@ +#!/bin/bash -e +# shellcheck disable=SC1090 +############################################################################## +# Copyright (c) 2018 Mirantis Inc., Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x + +# shellcheck disable=SC1090 +source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib.sh" +source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/xdf_data.sh" + +CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x +# shellcheck disable=SC2154,SC2086,SC2116 +LOCAL_VIRT_NODES=$(echo ${virtual_nodes[*]}) # unquoted to filter space +NODE_MASK="${LOCAL_VIRT_NODES// /|}" + +# wait_for 3.0 "salt-call state.apply salt exclude='[{id: salt_master_service}]'" +wait_for 5.0 "salt-call state.sls reclass,linux.network" +wait_for 3.0 "salt -C '*' saltutil.refresh_pillar" + +# NOTE: domain name changes are not yet supported without a clean redeploy + +# Init specific to VMs on FN (all for virtual, mas for baremetal) +wait_for 3.0 "(for n in ${LOCAL_VIRT_NODES}; do salt -C \${n}.* test.ping || exit; done)" + +wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' state.sls linux.system,linux.storage" +wait_for 2.0 "salt -C 'E@^(${NODE_MASK}).*' state.sls linux.network" +salt -C "E@^(${NODE_MASK}).*" system.reboot +wait_for 90.0 "salt -C 'E@^(${NODE_MASK}).*' test.ping" +wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' pkg.upgrade refresh=False dist_upgrade=True" +wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' saltutil.sync_all" +wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' state.apply salt" + +wait_for 3.0 "salt -C 'E@^(${NODE_MASK}).*' state.sls ntp" diff --git a/mcp/metadata/service/opendaylight/server/single.yml b/mcp/metadata/service/opendaylight/server/single.yml deleted file mode 100644 index 3a762b39a..000000000 --- a/mcp/metadata/service/opendaylight/server/single.yml +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc. and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## ---- -applications: - - opendaylight -parameters: - _param: - version: oxygen - opendaylight: - server: - enabled: 'True' - version: ${_param:version} - repo: 'odl-team/${_param:version}' - logging: - engine: syslog diff --git a/mcp/metadata/service/opendaylight/support.yml b/mcp/metadata/service/opendaylight/support.yml deleted file mode 100644 index bbb204315..000000000 --- a/mcp/metadata/service/opendaylight/support.yml +++ /dev/null @@ -1,19 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc. and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## ---- -parameters: - opendaylight: - _support: - collectd: - enabled: false - heka: - enabled: false - sensu: - enabled: false - sphinx: - enabled: false diff --git a/mcp/metadata/service/tacker/server/single.yml b/mcp/metadata/service/tacker/server/single.yml deleted file mode 100644 index 597d9f8fb..000000000 --- a/mcp/metadata/service/tacker/server/single.yml +++ /dev/null @@ -1,34 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Mirantis Inc. and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## ---- -classes: - - service.git.client -parameters: - _param: - keystone_tacker_endpoint_type: internalURL - tacker: - server: - bind_host: ${_param:cluster_local_address} - enabled: true - branch: stable/queens - database: - engine: mysql - host: ${_param:single_address} - port: 3306 - name: tacker - user: tacker - password: ${_param:mysql_tacker_password} - identity: - engine: keystone - region: RegionOne - host: ${_param:single_address} - port: 35357 - user: tacker - password: ${_param:keystone_tacker_password} - tenant: service - endpoint_type: ${_param:keystone_tacker_endpoint_type} diff --git a/mcp/patches/0002-maas-region-skip-credentials-update.patch b/mcp/patches/0002-maas-region-skip-credentials-update.patch deleted file mode 100644 index 1d226e4cd..000000000 --- a/mcp/patches/0002-maas-region-skip-credentials-update.patch +++ /dev/null @@ -1,38 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii -Date: Sat, 5 Aug 2017 02:03:01 +0200 -Subject: [PATCH] maas: region: skip credentials update - -Password update for maas psql database breaks ulterior acesses -to maas-region syncdb. -For now, limit regiond.conf changes to maas_url, and skip -updating credentials. - -Signed-off-by: Alexandru Avadanii ---- - -diff --git a/maas/region.sls b/maas/region.sls -index d3227ca..8a2243d 100644 ---- a/maas/region.sls -+++ b/maas/region.sls -@@ -6,10 +6,9 @@ - - names: {{ region.pkgs }} - - /etc/maas/regiond.conf: -- file.managed: -- - source: salt://maas/files/regiond.conf -- - template: jinja -- - group: maas -+ file.replace: -+ - pattern: ^maas_url.*$ -+ - repl: "maas_url: http://{{ region.bind.host }}:5240/MAAS" - - require: - - pkg: maas_region_packages - diff --git a/mcp/patches/0008-Handle-extra-environment-variables.patch b/mcp/patches/0008-Handle-extra-environment-variables.patch deleted file mode 100644 index 8df3227f2..000000000 --- a/mcp/patches/0008-Handle-extra-environment-variables.patch +++ /dev/null @@ -1,25 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Michael Polenchuk -Date: Mon, 12 Mar 2018 17:43:09 +0400 -Subject: [PATCH] Handle extra environment variables - -Change-Id: Ieae46ac65041630759c82238a8a5ce0535c454b2 - -diff --git a/keystone/files/keystonercv3 b/keystone/files/keystonercv3 -index 1b7f378..984c8a2 100644 ---- a/keystone/files/keystonercv3 -+++ b/keystone/files/keystonercv3 -@@ -11,3 +11,6 @@ export OS_REGION_NAME={{ server.region }} - export OS_INTERFACE=internal - export OS_ENDPOINT_TYPE="internal" - export OS_CACERT="{{ server.cacert }}" -+{%- for opt, value in server.get('openrc_extra', {}).iteritems() %} -+{{ "export %s=%s"|format(opt|upper, value) }} -+{%- endfor %} diff --git a/mcp/patches/0010-maas-region-allow-timeout-override.patch b/mcp/patches/0010-maas-region-allow-timeout-override.patch deleted file mode 100644 index c6f9e3a52..000000000 --- a/mcp/patches/0010-maas-region-allow-timeout-override.patch +++ /dev/null @@ -1,93 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii -Date: Sat, 23 Sep 2017 02:03:01 +0200 -Subject: [PATCH] maas: region: allow timeout override - -Two changes are squashed into a single patch so we can test for -reverse-applying the cummulative diff. - -1. Poor implementation that allows overiding comissioning/deploying -timeouts, which are hardcoded in MaaS source code. - -Add the following reclass _param: -- opnfv_maas_timeout_comissioning; -- opnfv_maas_timeout_deploying; - -The defaults are now 10/15 minutes (MaaS defaults to 20/40 min). - -2. maas: region: interface: default mode DHCP - -Switch MaaS interface mode default from AUTO to DHCP. -This way, MaaS will not statically assign IPs *outside* the dynamic -reserved range, but instead DHCP will assign them *inside* the range. - -One alternative would be to handle this via API calls, for each -node's PXE physical interface, but that overcomplicates things. - -3. MaaS blocksize: For some reason the python library behind tftp forces ip -blocksize to be 1008. We can force it to be 1464 and gain some performance due -to MTU beeing 1500 (i.e. allow bigger packets). - - -JIRA: FUEL-316 - -Signed-off-by: Alexandru Avadanii ---- - -diff --git a/maas/region.sls b/maas/region.sls ---- a/maas/region.sls -+++ b/maas/region.sls -@@ -19,6 +19,46 @@ - - require: - - pkg: maas_region_packages - -+maas_timeout_commissioning: -+ file.replace: -+ - name: "/usr/lib/python3/dist-packages/maasserver/node_status.py" -+ - pattern: '^(\s+NODE_STATUS.COMMISSIONING:)\s+\d+,$' -+ - repl: \1 {{ salt['pillar.get']('_param:opnfv_maas_timeout_comissioning', '20') }}, -+ - require: -+ - pkg: maas_region_packages -+ - require_in: -+ - service: maas_region_services -+ -+maas_timeout_deploying: -+ file.replace: -+ - name: "/usr/lib/python3/dist-packages/maasserver/node_status.py" -+ - pattern: '^(\s+NODE_STATUS.DEPLOYING:)\s+\d+,$' -+ - repl: \1 {{ salt['pillar.get']('_param:opnfv_maas_timeout_deploying', '40') }}, -+ - require: -+ - pkg: maas_region_packages -+ - require_in: -+ - service: maas_region_services -+ -+maas_ip_blksize_force: -+ file.replace: -+ - name: "/usr/lib/python3/dist-packages/tftp/bootstrap.py" -+ - pattern: 'int_blksize = min\(\(int_blksize, MAX_BLOCK_SIZE\)\)' -+ - repl: 'int_blksize = 1464' -+ - require: -+ - pkg: maas_region_packages -+ - require_in: -+ - service: maas_region_services -+ -+maas_interface_default_mode_dhcp: -+ file.replace: -+ - name: "/usr/lib/python3/dist-packages/maasserver/models/node.py" -+ - pattern: '^(\s+INTERFACE_LINK_TYPE)\.AUTO' -+ - repl: \1.DHCP -+ - require: -+ - pkg: maas_region_packages -+ - require_in: -+ - service: maas_region_services -+ - {%- if region.get('enable_iframe', False) %} - - /etc/apache2/conf-enabled/maas-http.conf: diff --git a/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch b/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch deleted file mode 100644 index dec29e6a7..000000000 --- a/mcp/patches/0011-system.repo-Debian-Add-keyserver-proxy-support.patch +++ /dev/null @@ -1,157 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Alexandru Avadanii -Date: Sun, 3 Jun 2018 19:28:18 +0200 -Subject: [PATCH] system.repo: Debian: Use proxy for keyservers - -Previously, when fetching GPG keys for APT keyring, either using -public key download & import (as for default repos) or via keyserver, -we relied on simple `curl` calls or passed it down to Salt aptpkg -module. -To be able to retrieve APT keys behind a proxy, one used to have to -configure the proxy for the Salt minion, which does not yet have -`no_proxy` support (either *all* or *no* traffic hits the proxy). - -When `linux:system:proxy` http(s) proxies are set: -- no longer pass key configuration to Salt aptpkg (until it properly - supports `no_proxy`); -- handle all keys explicitly with `curl` and `apt-key`; -- set 'http(s)_proxy' env vars for `cmd.wait` calls; - -If `linux:system:proxy` is not defined, the behavior is -unchanged for backwards compatibility. - -NOTE: If present, per-repo proxies are also used for keyserver access. - -system.repo: Fix conditions order for Debian proxy - -Signed-off-by: Alexandru Avadanii ---- - linux/system/repo.sls | 55 ++++++++++++++++++++++++++++++++++++------- - 1 file changed, 47 insertions(+), 8 deletions(-) - -diff --git a/linux/system/repo.sls b/linux/system/repo.sls -index 9664129..d9f567e 100644 ---- a/linux/system/repo.sls -+++ b/linux/system/repo.sls -@@ -7,18 +7,27 @@ linux_repo_prereq_pkgs: - - pkgs: {{ system.pkgs }} - {%- endif %} - -+ {%- set proxies = {'system': {}, 'repo': {}} %} -+ - # global proxy setup - {%- if grains.os_family == 'Debian' %} -+ - {%- if system.proxy.get('pkg', {}).get('enabled', False) %} -+ -+ {%- do proxies.system.update({'https': system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true)}) %} -+ {%- do proxies.system.update({'http': system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true)}) %} -+ {%- do proxies.system.update({'ftp': system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true)}) %} -+ - /etc/apt/apt.conf.d/99proxies-salt: - file.managed: - - template: jinja - - source: salt://linux/files/apt.conf.d_proxies - - defaults: - external_host: False -- https: {{ system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true) }} -- http: {{ system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true) }} -- ftp: {{ system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true) }} -+ https: {{ proxies.system.https }} -+ http: {{ proxies.system.http }} -+ ftp: {{ proxies.system.ftp }} -+ - {%- else %} - /etc/apt/apt.conf.d/99proxies-salt: - file.absent -@@ -43,16 +52,20 @@ purge_sources_list_d_repos: - # per repository proxy setup - {%- if repo.get('proxy', {}).get('enabled', False) %} - {%- set external_host = repo.proxy.get('host', None) or repo.source.split('/')[2] %} -+ {%- do proxies.repo.update({'https': repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true)}) %} -+ {%- do proxies.repo.update({'http': repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true)}) %} -+ {%- do proxies.repo.update({'ftp': repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true)}) %} - /etc/apt/apt.conf.d/99proxies-salt-{{ name }}: - file.managed: - - template: jinja - - source: salt://linux/files/apt.conf.d_proxies - - defaults: - external_host: {{ external_host }} -- https: {{ repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), True) }} -- http: {{ repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), True) }} -- ftp: {{ repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), True) }} -+ https: {{ proxies.repo.https }} -+ http: {{ proxies.repo.http }} -+ ftp: {{ proxies.repo.ftp }} - {%- else %} -+ {%- do proxies.repo.update({'https': None, 'http': None, 'ftp': None}) %} - /etc/apt/apt.conf.d/99proxies-salt-{{ name }}: - file.absent - {%- endif %} -@@ -104,6 +117,13 @@ linux_repo_{{ name }}_key: - {% else %} - - pkgrepo: linux_repo_{{ name }} - {% endif %} -+ - env: -+ {%- if proxies.repo.get('https', None) or proxies.system.get('https', None) %} -+ - https_proxy: {{ proxies.repo.get('https', None) or proxies.system.get('https', None) }} -+ {%- endif %} -+ {%- if proxies.repo.get('http', None) or proxies.system.get('http', None) %} -+ - http_proxy: {{ proxies.repo.get('http', None) or proxies.system.get('http', None) }} -+ {%- endif %} - {%- endif %} - - {%- if repo.get('default', False) %} -@@ -111,6 +131,9 @@ linux_repo_{{ name }}_key: - {%- else %} - - {%- if repo.get('enabled', True) %} -+ {%- set use_proxy = ( ( proxies.repo.get('https', None) or proxies.system.get('https', None) or -+ proxies.repo.get('http', None) or proxies.system.get('http', None) ) and -+ repo.key_id is defined and repo.key_server is defined ) %} - linux_repo_{{ name }}: - pkgrepo.managed: - - refresh_db: False -@@ -126,10 +149,10 @@ linux_repo_{{ name }}: - {%- endif %} - - file: /etc/apt/sources.list.d/{{ name }}.list - - clean_file: {{ repo.get('clean_file', True) }} -- {%- if repo.key_id is defined %} -+ {%- if not use_proxy and repo.key_id is defined %} - - keyid: {{ repo.key_id }} - {%- endif %} -- {%- if repo.key_server is defined %} -+ {%- if not use_proxy and repo.key_server is defined %} - - keyserver: {{ repo.key_server }} - {%- endif %} - {%- if repo.key_url is defined and (grains['saltversioninfo'] >= [2017, 7] or repo.key_url.startswith('salt://')) %} -@@ -143,6 +166,22 @@ linux_repo_{{ name }}: - - file: purge_sources_list_d_repos - {%- endif %} - {%- endif %} -+ {%- if use_proxy and repo.key_id is defined and repo.key_server is defined %} -+linux_repo_{{ name }}_key: -+ cmd.run: -+ - name: "apt-key adv --keyserver {{ repo.key_server }} --recv {{ repo.key_id }}" -+ - unless: 'test -e /etc/apt/sources.list.d/{{ name }}.list' -+ - require_in: -+ - pkgrepo: linux_repo_{{ name }} -+ - env: -+ {%- if proxies.repo.get('https', None) or proxies.system.get('https', None) %} -+ - https_proxy: {{ proxies.repo.get('https', None) or proxies.system.get('https', None) }} -+ {%- endif %} -+ {%- if proxies.repo.get('http', None) or proxies.system.get('http', None) %} -+ - http_proxy: {{ proxies.repo.get('http', None) or proxies.system.get('http', None) }} -+ {%- endif %} -+ {%- endif %} -+ {#- repo.enabled is false #} - {%- else %} - linux_repo_{{ name }}: - pkgrepo.absent: diff --git a/mcp/patches/0015-Set-ovs-bridges-as-L3-interfaces.patch b/mcp/patches/0015-Set-ovs-bridges-as-L3-interfaces.patch deleted file mode 100644 index a7b366b94..000000000 --- a/mcp/patches/0015-Set-ovs-bridges-as-L3-interfaces.patch +++ /dev/null @@ -1,95 +0,0 @@ -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -: Copyright (c) 2018 Mirantis Inc., Enea AB and others. -: -: All rights reserved. This program and the accompanying materials -: are made available under the terms of the Apache License, Version 2.0 -: which accompanies this distribution, and is available at -: http://www.apache.org/licenses/LICENSE-2.0 -:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -From: Michael Polenchuk -Date: Wed, 28 Feb 2018 17:54:28 +0400 -Subject: [PATCH] Set ovs bridges as L3 interfaces - -Change-Id: I1e83129cc184cf481bea21d7aa452bf60d9e0499 - -diff --git a/linux/files/ovs_bridge b/linux/files/ovs_bridge -new file mode 100644 -index 0000000..c609e45 ---- /dev/null -+++ b/linux/files/ovs_bridge -@@ -0,0 +1,19 @@ -+auto {{ bridge_name }} -+allow-ovs {{ bridge_name }} -+iface {{ bridge_name }} inet static -+ ovs_type OVSBridge -+ address {{ bridge.address }} -+ netmask {{ bridge.netmask }} -+ mtu {{ bridge.get('mtu', '1500') }} -+ {%- if bridge.use_interfaces is defined %} -+ ovs_ports {{ bridge.use_interfaces|join(' ') }} -+ {%- endif %} -+ {%- if bridge.datapath_type is defined %} -+ ovs_extra set Bridge ${IFACE} datapath_type={{ bridge.datapath_type }} -+ {%- endif %} -+ {%- if bridge.gateway is defined %} -+ gateway {{ bridge.gateway }} -+ {%- endif %} -+ {%- if bridge.name_servers is defined %} -+ dns-nameservers {{ bridge.name_servers | join(' ') }} -+ {%- endif %} -diff --git a/linux/files/ovs_port b/linux/files/ovs_port -index 222ca8e..efb0307 100644 ---- a/linux/files/ovs_port -+++ b/linux/files/ovs_port -@@ -1,6 +1,11 @@ -+# With systemd, adding OVS bridges as 'auto' can cause race conditions -+# https://github.com/openvswitch/ovs/blob/master/debian/openvswitch-switch.README.Debian -+# auto {{ port_name }} --auto {{ port_name }} - allow-{{ port.bridge }} {{ port_name }} - iface {{ port_name }} inet {{ port.get('proto', 'manual') }} -+{%- if '.' in port_name %} -+vlan-raw-device {{ port_name.split('.')[0] }} -+{%- endif %} - ovs_type {{ port.get('ovs_port_type', 'OVSIntPort') }} - mtu {{ port.get('mtu', '1500') }} - ovs_bridge {{ port.bridge }} -diff --git a/linux/network/interface.sls b/linux/network/interface.sls -index 180f912..dcb295b 100644 ---- a/linux/network/interface.sls -+++ b/linux/network/interface.sls -@@ -91,6 +91,34 @@ add_int_{{ int_name }}_to_ovs_dpdk_bridge_{{ interface_name }}: - ovs_bridge_{{ interface_name }}: - openvswitch_bridge.present: - - name: {{ interface_name }} -+{%- if interface.get('proto', 'manual') == 'static' %} -+ file.managed: -+ - name: /etc/network/interfaces.u/ifcfg-{{ interface_name }} -+ - makedirs: True -+ - source: salt://linux/files/ovs_bridge -+ - defaults: -+ bridge: {{ interface|yaml }} -+ bridge_name: {{ interface_name }} -+ - template: jinja -+ -+linux_interfaces_include_{{ interface_name }}: -+ file.prepend: -+ - name: /etc/network/interfaces -+ - text: | -+ source /etc/network/interfaces.d/* -+ # Workaround for Upstream-Bug: https://github.com/saltstack/salt/issues/40262 -+ source /etc/network/interfaces.u/* -+ -+ovs_bridge_up_{{ interface_name }}: -+ cmd.run: -+ - name: ifup {{ interface_name }} -+ - require: -+ - file: ovs_bridge_{{ interface_name }} -+ - openvswitch_bridge: ovs_bridge_{{ interface_name }} -+ - file: linux_interfaces_final_include -+ - unless: -+ - ip link show {{ interface_name }} | grep -q '\' -+{%- endif %} - - {# add linux network interface into OVS bridge #} - {%- for int_name, int in network.interface.items() %} diff --git a/mcp/patches/patch.sh b/mcp/patches/patch.sh deleted file mode 100755 index bb48dcd07..000000000 --- a/mcp/patches/patch.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -e -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x - -if [ -r "$1" ]; then - while IFS=': ' read -r p_dest p_file; do - if ! patch --dry-run -Rd "${p_dest}" -r - -s -p1 < \ - "/root/fuel/mcp/patches/${p_file}" > /dev/null; then - patch -d "${p_dest}" -p1 < "/root/fuel/mcp/patches/${p_file}" - fi - done < <(grep -vE '^#' "${1}" | grep -E "^.*${2}.*: ") -fi diff --git a/mcp/patches/patches.list b/mcp/patches/patches.list deleted file mode 100644 index baa15d79d..000000000 --- a/mcp/patches/patches.list +++ /dev/null @@ -1,12 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -/usr/share/salt-formulas/env: 0002-maas-region-skip-credentials-update.patch -/usr/share/salt-formulas/env: 0008-Handle-extra-environment-variables.patch -/usr/share/salt-formulas/env: 0010-maas-region-allow-timeout-override.patch -/usr/share/salt-formulas/env: 0011-system.repo-Debian-Add-keyserver-proxy-support.patch -/usr/share/salt-formulas/env: 0015-Set-ovs-bridges-as-L3-interfaces.patch diff --git a/mcp/patches/salt-formula-keystone/0001-Handle-extra-environment-variables.patch b/mcp/patches/salt-formula-keystone/0001-Handle-extra-environment-variables.patch new file mode 100644 index 000000000..2108587b1 --- /dev/null +++ b/mcp/patches/salt-formula-keystone/0001-Handle-extra-environment-variables.patch @@ -0,0 +1,28 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2018 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Michael Polenchuk +Date: Mon, 12 Mar 2018 17:43:09 +0400 +Subject: [PATCH] Handle extra environment variables + +Change-Id: Ieae46ac65041630759c82238a8a5ce0535c454b2 +--- + keystone/files/keystonercv3 | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/keystone/files/keystonercv3 b/keystone/files/keystonercv3 +index 1b7f378..984c8a2 100644 +--- a/keystone/files/keystonercv3 ++++ b/keystone/files/keystonercv3 +@@ -11,3 +11,6 @@ export OS_REGION_NAME={{ server.region }} + export OS_INTERFACE=internal + export OS_ENDPOINT_TYPE="internal" + export OS_CACERT="{{ server.cacert }}" ++{%- for opt, value in server.get('openrc_extra', {}).iteritems() %} ++{{ "export %s=%s"|format(opt|upper, value) }} ++{%- endfor %} diff --git a/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch new file mode 100644 index 000000000..4ad51e3e0 --- /dev/null +++ b/mcp/patches/salt-formula-linux/0001-system.repo-Debian-Use-proxy-for-keyservers.patch @@ -0,0 +1,157 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2018 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii +Date: Sun, 3 Jun 2018 19:28:18 +0200 +Subject: [PATCH] system.repo: Debian: Use proxy for keyservers + +Previously, when fetching GPG keys for APT keyring, either using +public key download & import (as for default repos) or via keyserver, +we relied on simple `curl` calls or passed it down to Salt aptpkg +module. +To be able to retrieve APT keys behind a proxy, one used to have to +configure the proxy for the Salt minion, which does not yet have +`no_proxy` support (either *all* or *no* traffic hits the proxy). + +When `linux:system:proxy` http(s) proxies are set: +- no longer pass key configuration to Salt aptpkg (until it properly + supports `no_proxy`); +- handle all keys explicitly with `curl` and `apt-key`; +- set 'http(s)_proxy' env vars for `cmd.wait` calls; + +If `linux:system:proxy` is not defined, the behavior is +unchanged for backwards compatibility. + +NOTE: If present, per-repo proxies are also used for keyserver access. + +system.repo: Fix conditions order for Debian proxy + +Signed-off-by: Alexandru Avadanii +--- + linux/system/repo.sls | 55 ++++++++++++++++++++++++++++++++++++------- + 1 file changed, 47 insertions(+), 8 deletions(-) + +diff --git a/linux/system/repo.sls b/linux/system/repo.sls +index 0c0b026..82661f0 100644 +--- a/linux/system/repo.sls ++++ b/linux/system/repo.sls +@@ -7,18 +7,27 @@ linux_repo_prereq_pkgs: + - pkgs: {{ system.pkgs }} + {%- endif %} + ++ {%- set proxies = {'system': {}, 'repo': {}} %} ++ + # global proxy setup + {%- if grains.os_family == 'Debian' %} ++ + {%- if system.proxy.get('pkg', {}).get('enabled', False) %} ++ ++ {%- do proxies.system.update({'https': system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true)}) %} ++ {%- do proxies.system.update({'http': system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true)}) %} ++ {%- do proxies.system.update({'ftp': system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true)}) %} ++ + /etc/apt/apt.conf.d/99proxies-salt: + file.managed: + - template: jinja + - source: salt://linux/files/apt.conf.d_proxies + - defaults: + external_host: False +- https: {{ system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true) }} +- http: {{ system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true) }} +- ftp: {{ system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true) }} ++ https: {{ proxies.system.https }} ++ http: {{ proxies.system.http }} ++ ftp: {{ proxies.system.ftp }} ++ + {%- else %} + /etc/apt/apt.conf.d/99proxies-salt: + file.absent +@@ -43,16 +52,20 @@ purge_sources_list_d_repos: + # per repository proxy setup + {%- if repo.get('proxy', {}).get('enabled', False) %} + {%- set external_host = repo.proxy.get('host', None) or repo.source.split('/')[2] %} ++ {%- do proxies.repo.update({'https': repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), true)}) %} ++ {%- do proxies.repo.update({'http': repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), true)}) %} ++ {%- do proxies.repo.update({'ftp': repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), true)}) %} + /etc/apt/apt.conf.d/99proxies-salt-{{ name }}: + file.managed: + - template: jinja + - source: salt://linux/files/apt.conf.d_proxies + - defaults: + external_host: {{ external_host }} +- https: {{ repo.proxy.get('https', None) or system.proxy.get('pkg', {}).get('https', None) | default(system.proxy.get('https', None), True) }} +- http: {{ repo.proxy.get('http', None) or system.proxy.get('pkg', {}).get('http', None) | default(system.proxy.get('http', None), True) }} +- ftp: {{ repo.proxy.get('ftp', None) or system.proxy.get('pkg', {}).get('ftp', None) | default(system.proxy.get('ftp', None), True) }} ++ https: {{ proxies.repo.https }} ++ http: {{ proxies.repo.http }} ++ ftp: {{ proxies.repo.ftp }} + {%- else %} ++ {%- do proxies.repo.update({'https': None, 'http': None, 'ftp': None}) %} + /etc/apt/apt.conf.d/99proxies-salt-{{ name }}: + file.absent + {%- endif %} +@@ -104,6 +117,13 @@ linux_repo_{{ name }}_key: + {% else %} + - pkgrepo: linux_repo_{{ name }} + {% endif %} ++ - env: ++ {%- if proxies.repo.get('https', None) or proxies.system.get('https', None) %} ++ - https_proxy: {{ proxies.repo.get('https', None) or proxies.system.get('https', None) }} ++ {%- endif %} ++ {%- if proxies.repo.get('http', None) or proxies.system.get('http', None) %} ++ - http_proxy: {{ proxies.repo.get('http', None) or proxies.system.get('http', None) }} ++ {%- endif %} + {%- endif %} + + {%- if repo.get('default', False) %} +@@ -111,6 +131,9 @@ linux_repo_{{ name }}_key: + {%- else %} + + {%- if repo.get('enabled', True) %} ++ {%- set use_proxy = ( ( proxies.repo.get('https', None) or proxies.system.get('https', None) or ++ proxies.repo.get('http', None) or proxies.system.get('http', None) ) and ++ repo.key_id is defined and repo.key_server is defined ) %} + linux_repo_{{ name }}: + pkgrepo.managed: + - refresh_db: False +@@ -126,10 +149,10 @@ linux_repo_{{ name }}: + {%- endif %} + - file: /etc/apt/sources.list.d/{{ name }}.list + - clean_file: {{ repo.get('clean_file', True) }} +- {%- if repo.key_id is defined %} ++ {%- if not use_proxy and repo.key_id is defined %} + - keyid: {{ repo.key_id }} + {%- endif %} +- {%- if repo.key_server is defined %} ++ {%- if not use_proxy and repo.key_server is defined %} + - keyserver: {{ repo.key_server }} + {%- endif %} + {%- if repo.key_url is defined and (grains['saltversioninfo'] >= [2017, 7] or repo.key_url.startswith('salt://')) %} +@@ -143,6 +166,22 @@ linux_repo_{{ name }}: + - file: purge_sources_list_d_repos + {%- endif %} + {%- endif %} ++ {%- if use_proxy and repo.key_id is defined and repo.key_server is defined %} ++linux_repo_{{ name }}_key: ++ cmd.run: ++ - name: "apt-key adv --keyserver {{ repo.key_server }} --recv {{ repo.key_id }}" ++ - unless: 'test -e /etc/apt/sources.list.d/{{ name }}.list' ++ - require_in: ++ - pkgrepo: linux_repo_{{ name }} ++ - env: ++ {%- if proxies.repo.get('https', None) or proxies.system.get('https', None) %} ++ - https_proxy: {{ proxies.repo.get('https', None) or proxies.system.get('https', None) }} ++ {%- endif %} ++ {%- if proxies.repo.get('http', None) or proxies.system.get('http', None) %} ++ - http_proxy: {{ proxies.repo.get('http', None) or proxies.system.get('http', None) }} ++ {%- endif %} ++ {%- endif %} ++ {#- repo.enabled is false #} + {%- else %} + linux_repo_{{ name }}: + pkgrepo.absent: diff --git a/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch b/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch new file mode 100644 index 000000000..ee82474b1 --- /dev/null +++ b/mcp/patches/salt-formula-linux/0002-Set-ovs-bridges-as-L3-interfaces.patch @@ -0,0 +1,101 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2018 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Michael Polenchuk +Date: Wed, 28 Feb 2018 17:54:28 +0400 +Subject: [PATCH] Set ovs bridges as L3 interfaces + +Change-Id: I1e83129cc184cf481bea21d7aa452bf60d9e0499 +--- + linux/files/ovs_bridge | 19 +++++++++++++++++++ + linux/files/ovs_port | 7 ++++++- + linux/network/interface.sls | 28 ++++++++++++++++++++++++++++ + 3 files changed, 53 insertions(+), 1 deletion(-) + create mode 100644 linux/files/ovs_bridge + +diff --git a/linux/files/ovs_bridge b/linux/files/ovs_bridge +new file mode 100644 +index 0000000..073c91c +--- /dev/null ++++ b/linux/files/ovs_bridge +@@ -0,0 +1,19 @@ ++auto {{ bridge_name }} ++allow-ovs {{ bridge_name }} ++iface {{ bridge_name }} inet static ++ ovs_type OVSBridge ++ address {{ bridge.address }} ++ netmask {{ bridge.netmask }} ++ mtu {{ bridge.get('mtu', '1500') }} ++ {%- if bridge.use_interfaces is defined %} ++ ovs_ports {{ bridge.use_interfaces|join(' ') }} ++ {%- endif %} ++ {%- if bridge.datapath_type is defined %} ++ ovs_extra set Bridge ${IFACE} datapath_type={{ bridge.datapath_type }} ++ {%- endif %} ++ {%- if bridge.gateway is defined %} ++ gateway {{ bridge.gateway }} ++ {%- endif %} ++ {%- if bridge.name_servers is defined %} ++ dns-nameservers {{ bridge.name_servers | join(' ') }} ++ {%- endif %} +diff --git a/linux/files/ovs_port b/linux/files/ovs_port +index 222ca8e..03072cd 100644 +--- a/linux/files/ovs_port ++++ b/linux/files/ovs_port +@@ -1,6 +1,11 @@ +-auto {{ port_name }} ++# With systemd, adding OVS bridges as 'auto' can cause race conditions ++# https://github.com/openvswitch/ovs/blob/master/debian/openvswitch-switch.README.Debian ++# auto {{ port_name }} + allow-{{ port.bridge }} {{ port_name }} + iface {{ port_name }} inet {{ port.get('proto', 'manual') }} ++{%- if '.' in port_name %} ++vlan-raw-device {{ port_name.split('.')[0] }} ++{%- endif %} + ovs_type {{ port.get('ovs_port_type', 'OVSIntPort') }} + mtu {{ port.get('mtu', '1500') }} + ovs_bridge {{ port.bridge }} +diff --git a/linux/network/interface.sls b/linux/network/interface.sls +index c2d2a23..01131fa 100644 +--- a/linux/network/interface.sls ++++ b/linux/network/interface.sls +@@ -91,6 +91,34 @@ add_int_{{ int_name }}_to_ovs_dpdk_bridge_{{ interface_name }}: + ovs_bridge_{{ interface_name }}: + openvswitch_bridge.present: + - name: {{ interface_name }} ++{%- if interface.get('proto', 'manual') == 'static' %} ++ file.managed: ++ - name: /etc/network/interfaces.u/ifcfg-{{ interface_name }} ++ - makedirs: True ++ - source: salt://linux/files/ovs_bridge ++ - defaults: ++ bridge: {{ interface|yaml }} ++ bridge_name: {{ interface_name }} ++ - template: jinja ++ ++linux_interfaces_include_{{ interface_name }}: ++ file.prepend: ++ - name: /etc/network/interfaces ++ - text: | ++ source /etc/network/interfaces.d/* ++ # Workaround for Upstream-Bug: https://github.com/saltstack/salt/issues/40262 ++ source /etc/network/interfaces.u/* ++ ++ovs_bridge_up_{{ interface_name }}: ++ cmd.run: ++ - name: ifup {{ interface_name }} ++ - require: ++ - file: ovs_bridge_{{ interface_name }} ++ - openvswitch_bridge: ovs_bridge_{{ interface_name }} ++ - file: linux_interfaces_final_include ++ - unless: ++ - ip link show {{ interface_name }} | grep -q '\' ++{%- endif %} + + {# add linux network interface into OVS bridge #} + {%- for int_name, int in network.interface.items() %} diff --git a/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch b/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch new file mode 100644 index 000000000..718f25b4e --- /dev/null +++ b/mcp/patches/salt-formula-maas/0001-maas-region-skip-credentials-update.patch @@ -0,0 +1,40 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2018 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii +Date: Sat, 5 Aug 2017 02:03:01 +0200 +Subject: [PATCH] maas: region: skip credentials update + +Password update for maas psql database breaks ulterior acesses +to maas-region syncdb. +For now, limit regiond.conf changes to maas_url, and skip +updating credentials. + +Signed-off-by: Alexandru Avadanii +--- + maas/region.sls | 7 +++---- + 1 file changed, 3 insertions(+), 4 deletions(-) + +diff --git a/maas/region.sls b/maas/region.sls +index 684fda4..5844f22 100644 +--- a/maas/region.sls ++++ b/maas/region.sls +@@ -6,10 +6,9 @@ maas_region_packages: + - names: {{ region.pkgs }} + + /etc/maas/regiond.conf: +- file.managed: +- - source: salt://maas/files/regiond.conf +- - template: jinja +- - group: maas ++ file.replace: ++ - pattern: ^maas_url.*$ ++ - repl: "maas_url: http://{{ region.bind.host }}:5240/MAAS" + - require: + - pkg: maas_region_packages + diff --git a/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch b/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch new file mode 100644 index 000000000..aef087d92 --- /dev/null +++ b/mcp/patches/salt-formula-maas/0002-maas-region-allow-timeout-override.patch @@ -0,0 +1,95 @@ +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +: Copyright (c) 2018 Mirantis Inc., Enea AB and others. +: +: All rights reserved. This program and the accompanying materials +: are made available under the terms of the Apache License, Version 2.0 +: which accompanies this distribution, and is available at +: http://www.apache.org/licenses/LICENSE-2.0 +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +From: Alexandru Avadanii +Date: Sat, 23 Sep 2017 02:03:01 +0200 +Subject: [PATCH] maas: region: allow timeout override + +Two changes are squashed into a single patch so we can test for +reverse-applying the cummulative diff. + +1. Poor implementation that allows overiding comissioning/deploying +timeouts, which are hardcoded in MaaS source code. + +Add the following reclass _param: +- opnfv_maas_timeout_comissioning; +- opnfv_maas_timeout_deploying; + +The defaults are now 10/15 minutes (MaaS defaults to 20/40 min). + +2. maas: region: interface: default mode DHCP + +Switch MaaS interface mode default from AUTO to DHCP. +This way, MaaS will not statically assign IPs *outside* the dynamic +reserved range, but instead DHCP will assign them *inside* the range. + +One alternative would be to handle this via API calls, for each +node's PXE physical interface, but that overcomplicates things. + +3. MaaS blocksize: For some reason the python library behind tftp forces ip +blocksize to be 1008. We can force it to be 1464 and gain some performance due +to MTU beeing 1500 (i.e. allow bigger packets). + +JIRA: FUEL-316 + +Signed-off-by: Alexandru Avadanii +--- + maas/region.sls | 40 ++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 40 insertions(+) + +diff --git a/maas/region.sls b/maas/region.sls +index 5844f22..d844bb4 100644 +--- a/maas/region.sls ++++ b/maas/region.sls +@@ -38,6 +38,46 @@ restore_maas_database_{{ region.database.name }}: + + {%- endif %} + ++maas_timeout_commissioning: ++ file.replace: ++ - name: "/usr/lib/python3/dist-packages/maasserver/node_status.py" ++ - pattern: '^(\s+NODE_STATUS.COMMISSIONING:)\s+\d+,$' ++ - repl: \1 {{ salt['pillar.get']('_param:opnfv_maas_timeout_comissioning', '20') }}, ++ - require: ++ - pkg: maas_region_packages ++ - require_in: ++ - service: maas_region_services ++ ++maas_timeout_deploying: ++ file.replace: ++ - name: "/usr/lib/python3/dist-packages/maasserver/node_status.py" ++ - pattern: '^(\s+NODE_STATUS.DEPLOYING:)\s+\d+,$' ++ - repl: \1 {{ salt['pillar.get']('_param:opnfv_maas_timeout_deploying', '40') }}, ++ - require: ++ - pkg: maas_region_packages ++ - require_in: ++ - service: maas_region_services ++ ++maas_ip_blksize_force: ++ file.replace: ++ - name: "/usr/lib/python3/dist-packages/tftp/bootstrap.py" ++ - pattern: 'int_blksize = min\(\(int_blksize, MAX_BLOCK_SIZE\)\)' ++ - repl: 'int_blksize = 1464' ++ - require: ++ - pkg: maas_region_packages ++ - require_in: ++ - service: maas_region_services ++ ++maas_interface_default_mode_dhcp: ++ file.replace: ++ - name: "/usr/lib/python3/dist-packages/maasserver/models/node.py" ++ - pattern: '^(\s+INTERFACE_LINK_TYPE)\.AUTO' ++ - repl: \1.DHCP ++ - require: ++ - pkg: maas_region_packages ++ - require_in: ++ - service: maas_region_services ++ + {%- if region.get('enable_iframe', False) %} + + /etc/apache2/conf-enabled/maas-http.conf: diff --git a/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch b/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch new file mode 100644 index 000000000..9f13c4616 --- /dev/null +++ b/mcp/patches/salt-formula-maas/0003-Add-machines.delete-co-pxe_nat-sls.patch @@ -0,0 +1,137 @@ +From: Alexandru Avadanii +Date: Sun, 19 Aug 2018 05:38:27 +0200 +Subject: [PATCH] Add machines.delete & co, pxe_nat sls + +Signed-off-by: Alexandru Avadanii +--- + maas/machines/delete.sls | 20 ++++++++++++ + maas/machines/mark_broken_fixed.sls | 20 ++++++++++++ + maas/machines/override_failed_testing.sls | 20 ++++++++++++ + maas/pxe_nat.sls | 37 +++++++++++++++++++++++ + 4 files changed, 97 insertions(+) + create mode 100644 maas/machines/delete.sls + create mode 100644 maas/machines/mark_broken_fixed.sls + create mode 100644 maas/machines/override_failed_testing.sls + create mode 100644 maas/pxe_nat.sls + +diff --git a/maas/machines/delete.sls b/maas/machines/delete.sls +new file mode 100644 +index 0000000..2903f92 +--- /dev/null ++++ b/maas/machines/delete.sls +@@ -0,0 +1,20 @@ ++############################################################################## ++# Copyright (c) 2017 Mirantis Inc., Enea AB and others. ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++{%- from "maas/map.jinja" import region with context %} ++ ++maas_login_admin: ++ cmd.run: ++ - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" ++ - unless: 'test -e /var/lib/maas/.maas_credentials' ++ ++# TODO: implement delete_machine via _modules/maas.py ++delete_machine: ++ cmd.run: ++ - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine delete {{ pillar['system_id'] }}" ++ - require: ++ - cmd: maas_login_admin +diff --git a/maas/machines/mark_broken_fixed.sls b/maas/machines/mark_broken_fixed.sls +new file mode 100644 +index 0000000..46691bb +--- /dev/null ++++ b/maas/machines/mark_broken_fixed.sls +@@ -0,0 +1,20 @@ ++############################################################################## ++# Copyright (c) 2017 Mirantis Inc., Enea AB and others. ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++{%- from "maas/map.jinja" import region with context %} ++ ++maas_login_admin: ++ cmd.run: ++ - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" ++ - unless: 'test -e /var/lib/maas/.maas_credentials' ++ ++# TODO: implement mark_broken_fixed_machine via _modules/maas.py ++mark_broken_fixed_machine: ++ cmd.run: ++ - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine mark-broken {{ pillar['system_id'] }} && sleep 10 && maas opnfv machine mark-fixed {{ pillar['system_id'] }} && maas opnfv machine test {{ pillar['system_id'] }} testing_scripts=fio" ++ - require: ++ - cmd: maas_login_admin +diff --git a/maas/machines/override_failed_testing.sls b/maas/machines/override_failed_testing.sls +new file mode 100644 +index 0000000..e7fe1d2 +--- /dev/null ++++ b/maas/machines/override_failed_testing.sls +@@ -0,0 +1,20 @@ ++############################################################################## ++# Copyright (c) 2018 Mirantis Inc., Enea AB and others. ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++{%- from "maas/map.jinja" import region with context %} ++ ++maas_login_admin: ++ cmd.run: ++ - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" ++ - unless: 'test -e /var/lib/maas/.maas_credentials' ++ ++# TODO: implement override_failed_testing via _modules/maas.py ++mark_broken_fixed_machine: ++ cmd.run: ++ - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine override-failed-testing {{ pillar['system_id'] }}" ++ - require: ++ - cmd: maas_login_admin +diff --git a/maas/pxe_nat.sls b/maas/pxe_nat.sls +new file mode 100644 +index 0000000..8a03c4f +--- /dev/null ++++ b/maas/pxe_nat.sls +@@ -0,0 +1,37 @@ ++############################################################################## ++# Copyright (c) 2017 Mirantis Inc., Enea AB and others. ++# All rights reserved. This program and the accompanying materials ++# are made available under the terms of the Apache License, Version 2.0 ++# which accompanies this distribution, and is available at ++# http://www.apache.org/licenses/LICENSE-2.0 ++############################################################################## ++net.ipv4.ip_forward: ++ sysctl.present: ++ - value: 1 ++ ++iptables_pxe_nat: ++ iptables.append: ++ - table: nat ++ - chain: POSTROUTING ++ - jump: MASQUERADE ++ - destination: 0/0 ++ - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} ++ - save: True ++ ++iptables_pxe_source: ++ iptables.append: ++ - table: filter ++ - chain: INPUT ++ - jump: ACCEPT ++ - destination: 0/0 ++ - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} ++ - save: True ++ ++iptables_pxe_destination: ++ iptables.append: ++ - table: filter ++ - chain: INPUT ++ - jump: ACCEPT ++ - destination: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} ++ - source: 0/0 ++ - save: True diff --git a/mcp/reclass/classes/cluster/all-mcp-arch-common/infra/config_pdf.yml.j2 b/mcp/reclass/classes/cluster/all-mcp-arch-common/infra/config_pdf.yml.j2 index a12e15663..5fa4ca486 100644 --- a/mcp/reclass/classes/cluster/all-mcp-arch-common/infra/config_pdf.yml.j2 +++ b/mcp/reclass/classes/cluster/all-mcp-arch-common/infra/config_pdf.yml.j2 @@ -8,14 +8,12 @@ {%- import 'net_map.j2' as nm with context %} --- classes: - - service.git.client - - system.linux.system.repo.mcp.salt - system.reclass.storage.salt {%- if nm.cluster.has_baremetal_nodes %} - system.reclass.storage.system.infra_maas_single {%- endif %} - system.salt.master.api - - system.salt.master.pkg + - system.salt.master.single - system.salt.minion.ca.salt_master parameters: _param: @@ -25,6 +23,11 @@ parameters: salt_master_base_environment: prd # yamllint disable-line rule:line-length salt_api_password_hash: "$6$sGnRlxGf$al5jMCetLP.vfI/fTl3Z0N7Za1aeiexL487jAtyRABVfT3NlwZxQGVhO7S1N8OwS/34VHYwZQA8lkXwKMN/GS1" + linux: + system: + user: + salt: + home: /home/salt salt: master: accept_policy: open_mode diff --git a/mcp/reclass/classes/cluster/mcp-common-ha/infra/config.yml.j2 b/mcp/reclass/classes/cluster/mcp-common-ha/infra/config.yml.j2 index 0e880c558..ee849a6cd 100644 --- a/mcp/reclass/classes/cluster/mcp-common-ha/infra/config.yml.j2 +++ b/mcp/reclass/classes/cluster/mcp-common-ha/infra/config.yml.j2 @@ -29,37 +29,6 @@ parameters: mcpcontrol_nic: ${_param:opnfv_fn_vm_primary_interface} single_nic: ${_param:opnfv_fn_vm_secondary_interface} pxe_admin_nic: ${_param:opnfv_fn_vm_tertiary_interface} - linux: - network: - interface: - mcpcontrol_int: - enabled: true - type: eth - proto: dhcp - name: ${_param:mcpcontrol_nic} - single: - enabled: true - type: eth - name: ${_param:single_nic} -{%- if conf.idf.fuel.jumphost.get('trunks', {}).get('mgmt', False) and (nm.vlan_mgmt | int > 0) %} - proto: manual - single_vlan: - enabled: true - type: vlan - name: ${_param:single_nic}.{{ nm.vlan_mgmt }} - use_interfaces: - - ${_param:single_nic} -{%- endif %} - proto: static - address: ${_param:single_address} - netmask: ${_param:opnfv_net_mgmt_mask} - pxe_admin_int: - enabled: true - type: eth - proto: static - name: ${_param:pxe_admin_nic} - address: ${_param:pxe_admin_address} - netmask: ${_param:opnfv_net_admin_mask} salt: master: accept_policy: open_mode diff --git a/mcp/reclass/classes/cluster/mcp-common-noha/infra/config.yml.j2 b/mcp/reclass/classes/cluster/mcp-common-noha/infra/config.yml.j2 index 36a2c27ec..a6bf5cda4 100644 --- a/mcp/reclass/classes/cluster/mcp-common-noha/infra/config.yml.j2 +++ b/mcp/reclass/classes/cluster/mcp-common-noha/infra/config.yml.j2 @@ -16,37 +16,6 @@ parameters: single_address: ${_param:opnfv_infra_config_address} salt_master_host: 127.0.0.1 salt_minion_ca_host: ${linux:network:fqdn} - linux: - network: - interface: - mcpcontrol_int: - enabled: true - type: eth - proto: dhcp - name: ${_param:opnfv_fn_vm_primary_interface} - single_int: - enabled: true - name: ${_param:opnfv_fn_vm_secondary_interface} - type: eth -{%- if conf.idf.fuel.jumphost.get('trunks', {}).get('mgmt', False) and (nm.vlan_mgmt | int > 0) %} - proto: manual - single_int_vlan: - enabled: true - type: vlan - name: ${_param:opnfv_fn_vm_secondary_interface}.{{ nm.vlan_mgmt }} - use_interfaces: - - ${_param:opnfv_fn_vm_secondary_interface} -{%- endif %} - proto: static - address: ${_param:single_address} - netmask: ${_param:opnfv_net_mgmt_mask} - pxe_admin_int: - enabled: true - type: eth - proto: static - name: ${_param:opnfv_fn_vm_tertiary_interface} - address: ${_param:opnfv_infra_config_pxe_admin_address} - netmask: ${_param:opnfv_net_admin_mask} salt: master: file_recv: true diff --git a/mcp/salt-formulas/maas/machines/delete.sls b/mcp/salt-formulas/maas/machines/delete.sls deleted file mode 100644 index 2903f9226..000000000 --- a/mcp/salt-formulas/maas/machines/delete.sls +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -{%- from "maas/map.jinja" import region with context %} - -maas_login_admin: - cmd.run: - - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" - - unless: 'test -e /var/lib/maas/.maas_credentials' - -# TODO: implement delete_machine via _modules/maas.py -delete_machine: - cmd.run: - - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine delete {{ pillar['system_id'] }}" - - require: - - cmd: maas_login_admin diff --git a/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls b/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls deleted file mode 100644 index 46691bb09..000000000 --- a/mcp/salt-formulas/maas/machines/mark_broken_fixed.sls +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -{%- from "maas/map.jinja" import region with context %} - -maas_login_admin: - cmd.run: - - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" - - unless: 'test -e /var/lib/maas/.maas_credentials' - -# TODO: implement mark_broken_fixed_machine via _modules/maas.py -mark_broken_fixed_machine: - cmd.run: - - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine mark-broken {{ pillar['system_id'] }} && sleep 10 && maas opnfv machine mark-fixed {{ pillar['system_id'] }} && maas opnfv machine test {{ pillar['system_id'] }} testing_scripts=fio" - - require: - - cmd: maas_login_admin diff --git a/mcp/salt-formulas/maas/machines/override_failed_testing.sls b/mcp/salt-formulas/maas/machines/override_failed_testing.sls deleted file mode 100644 index e7fe1d267..000000000 --- a/mcp/salt-formulas/maas/machines/override_failed_testing.sls +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -{%- from "maas/map.jinja" import region with context %} - -maas_login_admin: - cmd.run: - - name: "maas-region apikey --username {{ region.admin.username }} > /var/lib/maas/.maas_credentials" - - unless: 'test -e /var/lib/maas/.maas_credentials' - -# TODO: implement override_failed_testing via _modules/maas.py -mark_broken_fixed_machine: - cmd.run: - - name: "maas login {{ region.admin.username }} http://{{ region.bind.host }}:5240/MAAS/api/2.0 - < /var/lib/maas/.maas_credentials && maas opnfv machine override-failed-testing {{ pillar['system_id'] }}" - - require: - - cmd: maas_login_admin diff --git a/mcp/salt-formulas/maas/pxe_nat.sls b/mcp/salt-formulas/maas/pxe_nat.sls deleted file mode 100644 index 8a03c4fdb..000000000 --- a/mcp/salt-formulas/maas/pxe_nat.sls +++ /dev/null @@ -1,37 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -net.ipv4.ip_forward: - sysctl.present: - - value: 1 - -iptables_pxe_nat: - iptables.append: - - table: nat - - chain: POSTROUTING - - jump: MASQUERADE - - destination: 0/0 - - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} - - save: True - -iptables_pxe_source: - iptables.append: - - table: filter - - chain: INPUT - - jump: ACCEPT - - destination: 0/0 - - source: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} - - save: True - -iptables_pxe_destination: - iptables.append: - - table: filter - - chain: INPUT - - jump: ACCEPT - - destination: {{ salt['pillar.get']('_param:single_address') }}/{{ salt['pillar.get']('_param:opnfv_net_admin_mask') }} - - source: 0/0 - - save: True diff --git a/mcp/salt-formulas/opendaylight/files/jetty.xml b/mcp/salt-formulas/opendaylight/files/jetty.xml deleted file mode 100644 index de2ac677d..000000000 --- a/mcp/salt-formulas/opendaylight/files/jetty.xml +++ /dev/null @@ -1,132 +0,0 @@ -{%- from "opendaylight/map.jinja" import server with context -%} - - - - - - - - - - - - - - - https - - - - 32768 - 8192 - 8192 - - - false - - false - 512 - - - - https - - - - 32768 - 8192 - 8192 - - - false - - false - 512 - - - - - - - - - - - - - - - - - - - - {{ server.odl_bind_ip }} - - - - - - - - - jetty-default - - - - - - - - - - - - - - - karaf - karaf - - - org.apache.karaf.jaas.boot.principal.RolePrincipal - - - - - - - - - - default - karaf - - - org.apache.karaf.jaas.boot.principal.RolePrincipal - - - - - - - diff --git a/mcp/salt-formulas/opendaylight/files/setenv.shell b/mcp/salt-formulas/opendaylight/files/setenv.shell deleted file mode 100644 index 2921ade14..000000000 --- a/mcp/salt-formulas/opendaylight/files/setenv.shell +++ /dev/null @@ -1,21 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc. and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -{%- from "opendaylight/map.jinja" import server with context -%} -# export JAVA_HOME # Location of Java installation -export JAVA_MIN_MEM={{ server.java_min_mem }} # Minimum memory for the JVM -export JAVA_MAX_MEM={{ server.java_max_mem }} # Maximum memory for the JVM -# export JAVA_PERM_MEM # Minimum perm memory for the JVM -export JAVA_MAX_PERM_MEM=512m # Maximum perm memory for the JVM -export EXTRA_JAVA_OPTS='{{ server.java_extra_opts }}' # Additional JVM options -# export KARAF_HOME # Karaf home folder -# export KARAF_DATA # Karaf data folder -# export KARAF_BASE # Karaf base folder -# export KARAF_ETC # Karaf etc folder -# export KARAF_OPTS # Additional available Karaf options -# export KARAF_DEBUG # Enable debug mode -# export KARAF_REDIRECT # Enable/set the std/err redirection when using bin/start diff --git a/mcp/salt-formulas/opendaylight/init.sls b/mcp/salt-formulas/opendaylight/init.sls deleted file mode 100644 index 132f69b33..000000000 --- a/mcp/salt-formulas/opendaylight/init.sls +++ /dev/null @@ -1,12 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc. and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -include: -{% if pillar.opendaylight.server is defined %} -- opendaylight.server -{% endif %} diff --git a/mcp/salt-formulas/opendaylight/map.jinja b/mcp/salt-formulas/opendaylight/map.jinja deleted file mode 100644 index 38575111e..000000000 --- a/mcp/salt-formulas/opendaylight/map.jinja +++ /dev/null @@ -1,28 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc. and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -{% set server = salt['grains.filter_by']({ - 'Debian': { - 'karaf_features': {'default': ['standard', 'wrap', 'ssh']}, - 'odl_rest_port': '8282', - 'odl_bind_ip': '0.0.0.0', - 'repo': 'odl-team/oxygen', - 'log_levels': {}, - 'enable_ha': false, - 'ha_node_ips': [], - 'ha_node_index': 0, - 'security_group_mode': 'stateful', - 'vpp_routing_node': '', - 'java_extra_opts': '-Djava.net.preferIPv4Stack=true -XX:+UseG1GC', - 'java_min_mem': '1g', - 'java_max_mem': '2g', - }, - 'RedHat': { - 'repo': 'opendaylight-6-testing' - }, -}, merge=salt['pillar.get']('opendaylight:server')) %} diff --git a/mcp/salt-formulas/opendaylight/server.sls b/mcp/salt-formulas/opendaylight/server.sls deleted file mode 100644 index 206dc52a2..000000000 --- a/mcp/salt-formulas/opendaylight/server.sls +++ /dev/null @@ -1,113 +0,0 @@ -############################################################################## -# Copyright (c) 2017 Mirantis Inc. and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -{% from "opendaylight/map.jinja" import server with context %} -{% from "linux/map.jinja" import system with context %} - -{%- if server.enabled %} - -opendaylight_repo_key: - cmd.run: - - name: "apt-key adv --keyserver keyserver.ubuntu.com --recv 44C05248" -{%- if system.proxy is defined and system.proxy.pkg is defined %} - - env: -{%- if system.proxy.pkg.http is defined %} - - http_proxy: {{ system.proxy.pkg.http }} -{%- endif %} -{%- if system.proxy.pkg.https is defined %} - - https_proxy: {{ system.proxy.pkg.https }} -{%- endif %} -{%- endif %} - -opendaylight_repo: - pkgrepo.managed: - # NOTE(armband): PPA handling behind proxy broken, define it explicitly - # https://github.com/saltstack/salt/pull/45224 - # - ppa: {{ server.repo }} - - human_name: opendaylight-ppa - - name: deb http://ppa.launchpad.net/odl-team/{{ server.version }}/ubuntu xenial main - - file: /etc/apt/sources.list.d/odl-team-ubuntu-{{ server.version }}-xenial.list - -{%- if grains['saltversioninfo'] < [2017, 7] %} -service.mask: - module.run: - - m_name: opendaylight -{%- else %} -opendaylight_service_mask: - service.masked: - - name: opendaylight -{%- endif %} - -opendaylight: - pkg.installed: - - require: - - pkgrepo: opendaylight_repo - - require_in: - - file: /opt/opendaylight/etc/jetty.xml - - file: /opt/opendaylight/bin/setenv - - ini: /opt/opendaylight/etc/org.apache.karaf.features.cfg - - ini: /opt/opendaylight/etc/org.ops4j.pax.web.cfg - service.running: - - enable: true -{%- if grains['saltversioninfo'] >= [2017, 7] %} - - unmask: true -{%- endif %} - - watch: - - file: /opt/opendaylight/etc/jetty.xml - - file: /opt/opendaylight/bin/setenv - - ini: /opt/opendaylight/etc/org.apache.karaf.features.cfg - - ini: /opt/opendaylight/etc/org.ops4j.pax.web.cfg - -/opt/opendaylight/etc/jetty.xml: - file.managed: - - source: salt://opendaylight/files/jetty.xml - - template: jinja - - user: odl - - group: odl - -/opt/opendaylight/bin/setenv: - file.managed: - - source: salt://opendaylight/files/setenv.shell - - mode: 0755 - - use: - - file: /opt/opendaylight/etc/jetty.xml - -{% set features %} -{%- for f in server.karaf_features.itervalues() -%} -{{ f | join(',') }}{%- if not loop.last %},{%- endif %} -{%- endfor %} -{% endset %} - -/opt/opendaylight/etc/org.apache.karaf.features.cfg: - ini.options_present: - - sections: - featuresBoot: {{ features }} - -/opt/opendaylight/etc/org.ops4j.pax.web.cfg: - ini.options_present: - - sections: - org.ops4j.pax.web.listening.addresses: {{ server.odl_bind_ip }} - org.osgi.service.http.port: {{ server.odl_rest_port }} - -{%- if server.get('router_enabled', false) %} -/opt/opendaylight/etc/custom.properties: - ini.options_present: - - sections: - ovsdb.l3.fwd.enabled: 'yes' - ovsdb.of.version: 1.3 - - require: - - pkg: opendaylight - - watch_in: - - service: opendaylight -{%- endif %} - -{%- if grains['cpuarch'] == 'aarch64' %} -opendaylight-leveldbjni: - pkg.installed -{%- endif %} - -{%- endif %} diff --git a/mcp/salt-formulas/salt-formula-keystone b/mcp/salt-formulas/salt-formula-keystone new file mode 160000 index 000000000..df75142aa --- /dev/null +++ b/mcp/salt-formulas/salt-formula-keystone @@ -0,0 +1 @@ +Subproject commit df75142aaf6487c3af9e2868f4f12ef18228ca99 diff --git a/mcp/salt-formulas/salt-formula-linux b/mcp/salt-formulas/salt-formula-linux new file mode 160000 index 000000000..cf1b5b322 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-linux @@ -0,0 +1 @@ +Subproject commit cf1b5b322a077dc8d6f9bf5b36a7a0a71a2b3a06 diff --git a/mcp/salt-formulas/salt-formula-maas b/mcp/salt-formulas/salt-formula-maas new file mode 160000 index 000000000..c6a12dea5 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-maas @@ -0,0 +1 @@ +Subproject commit c6a12dea5f8d8c7c700f596dbabb805113f1d341 diff --git a/mcp/salt-formulas/salt-formula-nova b/mcp/salt-formulas/salt-formula-nova new file mode 160000 index 000000000..539e9938e --- /dev/null +++ b/mcp/salt-formulas/salt-formula-nova @@ -0,0 +1 @@ +Subproject commit 539e9938e74f8a1ad19c2c1a653761df53bc24b7 diff --git a/mcp/salt-formulas/salt-formula-opendaylight/metadata/service/server/single.yml b/mcp/salt-formulas/salt-formula-opendaylight/metadata/service/server/single.yml new file mode 100644 index 000000000..3a762b39a --- /dev/null +++ b/mcp/salt-formulas/salt-formula-opendaylight/metadata/service/server/single.yml @@ -0,0 +1,20 @@ +############################################################################## +# Copyright (c) 2017 Mirantis Inc. and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +applications: + - opendaylight +parameters: + _param: + version: oxygen + opendaylight: + server: + enabled: 'True' + version: ${_param:version} + repo: 'odl-team/${_param:version}' + logging: + engine: syslog diff --git a/mcp/salt-formulas/salt-formula-opendaylight/metadata/service/support.yml b/mcp/salt-formulas/salt-formula-opendaylight/metadata/service/support.yml new file mode 100644 index 000000000..bbb204315 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-opendaylight/metadata/service/support.yml @@ -0,0 +1,19 @@ +############################################################################## +# Copyright (c) 2017 Mirantis Inc. and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +parameters: + opendaylight: + _support: + collectd: + enabled: false + heka: + enabled: false + sensu: + enabled: false + sphinx: + enabled: false diff --git a/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/jetty.xml b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/jetty.xml new file mode 100644 index 000000000..de2ac677d --- /dev/null +++ b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/jetty.xml @@ -0,0 +1,132 @@ +{%- from "opendaylight/map.jinja" import server with context -%} + + + + + + + + + + + + + + + https + + + + 32768 + 8192 + 8192 + + + false + + false + 512 + + + + https + + + + 32768 + 8192 + 8192 + + + false + + false + 512 + + + + + + + + + + + + + + + + + + + + {{ server.odl_bind_ip }} + + + + + + + + + jetty-default + + + + + + + + + + + + + + + karaf + karaf + + + org.apache.karaf.jaas.boot.principal.RolePrincipal + + + + + + + + + + default + karaf + + + org.apache.karaf.jaas.boot.principal.RolePrincipal + + + + + + + diff --git a/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/setenv.shell b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/setenv.shell new file mode 100644 index 000000000..2921ade14 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/files/setenv.shell @@ -0,0 +1,21 @@ +############################################################################## +# Copyright (c) 2017 Mirantis Inc. and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +{%- from "opendaylight/map.jinja" import server with context -%} +# export JAVA_HOME # Location of Java installation +export JAVA_MIN_MEM={{ server.java_min_mem }} # Minimum memory for the JVM +export JAVA_MAX_MEM={{ server.java_max_mem }} # Maximum memory for the JVM +# export JAVA_PERM_MEM # Minimum perm memory for the JVM +export JAVA_MAX_PERM_MEM=512m # Maximum perm memory for the JVM +export EXTRA_JAVA_OPTS='{{ server.java_extra_opts }}' # Additional JVM options +# export KARAF_HOME # Karaf home folder +# export KARAF_DATA # Karaf data folder +# export KARAF_BASE # Karaf base folder +# export KARAF_ETC # Karaf etc folder +# export KARAF_OPTS # Additional available Karaf options +# export KARAF_DEBUG # Enable debug mode +# export KARAF_REDIRECT # Enable/set the std/err redirection when using bin/start diff --git a/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/init.sls b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/init.sls new file mode 100644 index 000000000..132f69b33 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/init.sls @@ -0,0 +1,12 @@ +############################################################################## +# Copyright (c) 2017 Mirantis Inc. and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +include: +{% if pillar.opendaylight.server is defined %} +- opendaylight.server +{% endif %} diff --git a/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/map.jinja b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/map.jinja new file mode 100644 index 000000000..38575111e --- /dev/null +++ b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/map.jinja @@ -0,0 +1,28 @@ +############################################################################## +# Copyright (c) 2017 Mirantis Inc. and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +{% set server = salt['grains.filter_by']({ + 'Debian': { + 'karaf_features': {'default': ['standard', 'wrap', 'ssh']}, + 'odl_rest_port': '8282', + 'odl_bind_ip': '0.0.0.0', + 'repo': 'odl-team/oxygen', + 'log_levels': {}, + 'enable_ha': false, + 'ha_node_ips': [], + 'ha_node_index': 0, + 'security_group_mode': 'stateful', + 'vpp_routing_node': '', + 'java_extra_opts': '-Djava.net.preferIPv4Stack=true -XX:+UseG1GC', + 'java_min_mem': '1g', + 'java_max_mem': '2g', + }, + 'RedHat': { + 'repo': 'opendaylight-6-testing' + }, +}, merge=salt['pillar.get']('opendaylight:server')) %} diff --git a/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/server.sls b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/server.sls new file mode 100644 index 000000000..206dc52a2 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-opendaylight/opendaylight/server.sls @@ -0,0 +1,113 @@ +############################################################################## +# Copyright (c) 2017 Mirantis Inc. and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +{% from "opendaylight/map.jinja" import server with context %} +{% from "linux/map.jinja" import system with context %} + +{%- if server.enabled %} + +opendaylight_repo_key: + cmd.run: + - name: "apt-key adv --keyserver keyserver.ubuntu.com --recv 44C05248" +{%- if system.proxy is defined and system.proxy.pkg is defined %} + - env: +{%- if system.proxy.pkg.http is defined %} + - http_proxy: {{ system.proxy.pkg.http }} +{%- endif %} +{%- if system.proxy.pkg.https is defined %} + - https_proxy: {{ system.proxy.pkg.https }} +{%- endif %} +{%- endif %} + +opendaylight_repo: + pkgrepo.managed: + # NOTE(armband): PPA handling behind proxy broken, define it explicitly + # https://github.com/saltstack/salt/pull/45224 + # - ppa: {{ server.repo }} + - human_name: opendaylight-ppa + - name: deb http://ppa.launchpad.net/odl-team/{{ server.version }}/ubuntu xenial main + - file: /etc/apt/sources.list.d/odl-team-ubuntu-{{ server.version }}-xenial.list + +{%- if grains['saltversioninfo'] < [2017, 7] %} +service.mask: + module.run: + - m_name: opendaylight +{%- else %} +opendaylight_service_mask: + service.masked: + - name: opendaylight +{%- endif %} + +opendaylight: + pkg.installed: + - require: + - pkgrepo: opendaylight_repo + - require_in: + - file: /opt/opendaylight/etc/jetty.xml + - file: /opt/opendaylight/bin/setenv + - ini: /opt/opendaylight/etc/org.apache.karaf.features.cfg + - ini: /opt/opendaylight/etc/org.ops4j.pax.web.cfg + service.running: + - enable: true +{%- if grains['saltversioninfo'] >= [2017, 7] %} + - unmask: true +{%- endif %} + - watch: + - file: /opt/opendaylight/etc/jetty.xml + - file: /opt/opendaylight/bin/setenv + - ini: /opt/opendaylight/etc/org.apache.karaf.features.cfg + - ini: /opt/opendaylight/etc/org.ops4j.pax.web.cfg + +/opt/opendaylight/etc/jetty.xml: + file.managed: + - source: salt://opendaylight/files/jetty.xml + - template: jinja + - user: odl + - group: odl + +/opt/opendaylight/bin/setenv: + file.managed: + - source: salt://opendaylight/files/setenv.shell + - mode: 0755 + - use: + - file: /opt/opendaylight/etc/jetty.xml + +{% set features %} +{%- for f in server.karaf_features.itervalues() -%} +{{ f | join(',') }}{%- if not loop.last %},{%- endif %} +{%- endfor %} +{% endset %} + +/opt/opendaylight/etc/org.apache.karaf.features.cfg: + ini.options_present: + - sections: + featuresBoot: {{ features }} + +/opt/opendaylight/etc/org.ops4j.pax.web.cfg: + ini.options_present: + - sections: + org.ops4j.pax.web.listening.addresses: {{ server.odl_bind_ip }} + org.osgi.service.http.port: {{ server.odl_rest_port }} + +{%- if server.get('router_enabled', false) %} +/opt/opendaylight/etc/custom.properties: + ini.options_present: + - sections: + ovsdb.l3.fwd.enabled: 'yes' + ovsdb.of.version: 1.3 + - require: + - pkg: opendaylight + - watch_in: + - service: opendaylight +{%- endif %} + +{%- if grains['cpuarch'] == 'aarch64' %} +opendaylight-leveldbjni: + pkg.installed +{%- endif %} + +{%- endif %} diff --git a/mcp/salt-formulas/salt-formula-tacker/metadata/service/server/single.yml b/mcp/salt-formulas/salt-formula-tacker/metadata/service/server/single.yml new file mode 100644 index 000000000..597d9f8fb --- /dev/null +++ b/mcp/salt-formulas/salt-formula-tacker/metadata/service/server/single.yml @@ -0,0 +1,34 @@ +############################################################################## +# Copyright (c) 2018 Mirantis Inc. and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +classes: + - service.git.client +parameters: + _param: + keystone_tacker_endpoint_type: internalURL + tacker: + server: + bind_host: ${_param:cluster_local_address} + enabled: true + branch: stable/queens + database: + engine: mysql + host: ${_param:single_address} + port: 3306 + name: tacker + user: tacker + password: ${_param:mysql_tacker_password} + identity: + engine: keystone + region: RegionOne + host: ${_param:single_address} + port: 35357 + user: tacker + password: ${_param:keystone_tacker_password} + tenant: service + endpoint_type: ${_param:keystone_tacker_endpoint_type} diff --git a/mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.conf b/mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.conf new file mode 100644 index 000000000..7adfd3139 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.conf @@ -0,0 +1,45 @@ +############################################################################## +# Copyright (c) 2018 Mirantis Inc. and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +{%- from "tacker/map.jinja" import server with context %} + +[DEFAULT] +auth_strategy = keystone +policy_file = /usr/local/etc/tacker/policy.json +debug = True +use_syslog = False +bind_host = {{ server.bind_host }} +bind_port = 9890 +service_plugins = nfvo,vnfm + +state_path = /var/lib/tacker + +[nfvo_vim] +vim_drivers = openstack + +[keystone_authtoken] +region_name = {{ server.identity.region }} +auth_type = password +user_domain_id = {{ server.identity.get('domain', 'default') }} +project_domain_id = {{ server.identity.get('domain', 'default') }} +project_name = {{ server.identity.tenant }} +username = {{ server.identity.user }} +password = {{ server.identity.password }} +auth_url = http://{{ server.identity.host }}:35357 +www_authenticate_uri = http://{{ server.identity.host }}:5000 +{%- if server.cache is defined %} +memcached_servers={%- for member in server.cache.members %}{{ member.host }}:11211{% if not loop.last %},{% endif %}{%- endfor %} +{%- endif %} + +[agent] +root_helper = sudo /usr/local/bin/tacker-rootwrap /usr/local/etc/tacker/rootwrap.conf + +[database] +connection = {{ server.database.engine }}+pymysql://{{ server.database.user }}:{{ server.database.password }}@{{ server.database.host }}/{{ server.database.name }}?charset=utf8 + +[tacker] +monitor_driver = ping,http_ping diff --git a/mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.systemd b/mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.systemd new file mode 100644 index 000000000..12ff5cdde --- /dev/null +++ b/mcp/salt-formulas/salt-formula-tacker/tacker/files/tacker.systemd @@ -0,0 +1,20 @@ +############################################################################## +# Copyright (c) 2018 Mirantis Inc. and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +[Unit] +Description = OpenStack Tacker service + +[Service] +ExecReload = /bin/kill -HUP $MAINPID +TimeoutStopSec = 300 +KillMode = process +WorkingDirectory = /tmp +ExecStartPre=/bin/mkdir -p /var/log/tacker +ExecStart=/usr/local/bin/tacker-server --config-file /usr/local/etc/tacker/tacker.conf --log-file /var/log/tacker/tacker.log + +[Install] +WantedBy = multi-user.target diff --git a/mcp/salt-formulas/salt-formula-tacker/tacker/init.sls b/mcp/salt-formulas/salt-formula-tacker/tacker/init.sls new file mode 100644 index 000000000..35291b8f9 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-tacker/tacker/init.sls @@ -0,0 +1,11 @@ +############################################################################## +# Copyright (c) 2018 Mirantis Inc. and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +include: +{%- if pillar.tacker.server is defined %} +- tacker.server +{%- endif %} diff --git a/mcp/salt-formulas/salt-formula-tacker/tacker/map.jinja b/mcp/salt-formulas/salt-formula-tacker/tacker/map.jinja new file mode 100644 index 000000000..9a35c8e52 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-tacker/tacker/map.jinja @@ -0,0 +1,21 @@ +{#- + Copyright (c) 2018 Mirantis Inc. and others. + All rights reserved. This program and the accompanying materials + are made available under the terms of the Apache License, Version 2.0 + which accompanies this distribution, and is available at + http://www.apache.org/licenses/LICENSE-2.0 +-#} +{% set server = salt['grains.filter_by']({ + 'Debian': { + 'pkgs': ['python-heat-translator', 'python-tosca-parser', 'python-tackerclient'], + 'bind_host': '0.0.0.0', + 'git': { + 'source': 'https://github.com/openstack/tacker', + 'target': '/tmp/tacker', + 'branch': 'master', + }, + }, + 'RedHat': { + 'pkgs': 'openstack-tacker' + }, +}, merge=salt['pillar.get']('tacker:server')) %} diff --git a/mcp/salt-formulas/salt-formula-tacker/tacker/server.sls b/mcp/salt-formulas/salt-formula-tacker/tacker/server.sls new file mode 100644 index 000000000..eb3468a11 --- /dev/null +++ b/mcp/salt-formulas/salt-formula-tacker/tacker/server.sls @@ -0,0 +1,56 @@ +############################################################################## +# Copyright (c) 2018 Mirantis Inc. and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +{%- from "tacker/map.jinja" import server with context %} +{%- if server.enabled %} + +include: +- git + +{{ server.git.source }}: + git.latest: + - target: {{ server.git.target }} + - rev: {{ server.git.branch }} + - depth: 1 + +tacker_packages: + pkg.installed: + - names: {{ server.pkgs }} + +tacker_install: + cmd.run: + - name: python setup.py install + - cwd: {{ server.git.target }} + - creates: /usr/local/bin/tacker-server + - require: + - git: {{ server.git.source }} + +/usr/local/etc/tacker/tacker.conf: + file.managed: + - source: salt://tacker/files/tacker.conf + - template: jinja + - makedirs: true + - require: + - cmd: tacker_install + +tacker_db_manage: + cmd.run: + - name: /usr/local/bin/tacker-db-manage --config-file /usr/local/etc/tacker/tacker.conf upgrade head + - require: + - file: /usr/local/etc/tacker/tacker.conf + +/lib/systemd/system/tacker.service: + file.managed: + - source: salt://tacker/files/tacker.systemd + +tacker: + service.running: + - enable: true + - watch: + - file: /usr/local/etc/tacker/tacker.conf + +{%- endif %} diff --git a/mcp/salt-formulas/tacker/files/tacker.conf b/mcp/salt-formulas/tacker/files/tacker.conf deleted file mode 100644 index 7adfd3139..000000000 --- a/mcp/salt-formulas/tacker/files/tacker.conf +++ /dev/null @@ -1,45 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Mirantis Inc. and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -{%- from "tacker/map.jinja" import server with context %} - -[DEFAULT] -auth_strategy = keystone -policy_file = /usr/local/etc/tacker/policy.json -debug = True -use_syslog = False -bind_host = {{ server.bind_host }} -bind_port = 9890 -service_plugins = nfvo,vnfm - -state_path = /var/lib/tacker - -[nfvo_vim] -vim_drivers = openstack - -[keystone_authtoken] -region_name = {{ server.identity.region }} -auth_type = password -user_domain_id = {{ server.identity.get('domain', 'default') }} -project_domain_id = {{ server.identity.get('domain', 'default') }} -project_name = {{ server.identity.tenant }} -username = {{ server.identity.user }} -password = {{ server.identity.password }} -auth_url = http://{{ server.identity.host }}:35357 -www_authenticate_uri = http://{{ server.identity.host }}:5000 -{%- if server.cache is defined %} -memcached_servers={%- for member in server.cache.members %}{{ member.host }}:11211{% if not loop.last %},{% endif %}{%- endfor %} -{%- endif %} - -[agent] -root_helper = sudo /usr/local/bin/tacker-rootwrap /usr/local/etc/tacker/rootwrap.conf - -[database] -connection = {{ server.database.engine }}+pymysql://{{ server.database.user }}:{{ server.database.password }}@{{ server.database.host }}/{{ server.database.name }}?charset=utf8 - -[tacker] -monitor_driver = ping,http_ping diff --git a/mcp/salt-formulas/tacker/files/tacker.systemd b/mcp/salt-formulas/tacker/files/tacker.systemd deleted file mode 100644 index 12ff5cdde..000000000 --- a/mcp/salt-formulas/tacker/files/tacker.systemd +++ /dev/null @@ -1,20 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Mirantis Inc. and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -[Unit] -Description = OpenStack Tacker service - -[Service] -ExecReload = /bin/kill -HUP $MAINPID -TimeoutStopSec = 300 -KillMode = process -WorkingDirectory = /tmp -ExecStartPre=/bin/mkdir -p /var/log/tacker -ExecStart=/usr/local/bin/tacker-server --config-file /usr/local/etc/tacker/tacker.conf --log-file /var/log/tacker/tacker.log - -[Install] -WantedBy = multi-user.target diff --git a/mcp/salt-formulas/tacker/init.sls b/mcp/salt-formulas/tacker/init.sls deleted file mode 100644 index 35291b8f9..000000000 --- a/mcp/salt-formulas/tacker/init.sls +++ /dev/null @@ -1,11 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Mirantis Inc. and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -include: -{%- if pillar.tacker.server is defined %} -- tacker.server -{%- endif %} diff --git a/mcp/salt-formulas/tacker/map.jinja b/mcp/salt-formulas/tacker/map.jinja deleted file mode 100644 index 9a35c8e52..000000000 --- a/mcp/salt-formulas/tacker/map.jinja +++ /dev/null @@ -1,21 +0,0 @@ -{#- - Copyright (c) 2018 Mirantis Inc. and others. - All rights reserved. This program and the accompanying materials - are made available under the terms of the Apache License, Version 2.0 - which accompanies this distribution, and is available at - http://www.apache.org/licenses/LICENSE-2.0 --#} -{% set server = salt['grains.filter_by']({ - 'Debian': { - 'pkgs': ['python-heat-translator', 'python-tosca-parser', 'python-tackerclient'], - 'bind_host': '0.0.0.0', - 'git': { - 'source': 'https://github.com/openstack/tacker', - 'target': '/tmp/tacker', - 'branch': 'master', - }, - }, - 'RedHat': { - 'pkgs': 'openstack-tacker' - }, -}, merge=salt['pillar.get']('tacker:server')) %} diff --git a/mcp/salt-formulas/tacker/server.sls b/mcp/salt-formulas/tacker/server.sls deleted file mode 100644 index eb3468a11..000000000 --- a/mcp/salt-formulas/tacker/server.sls +++ /dev/null @@ -1,56 +0,0 @@ -############################################################################## -# Copyright (c) 2018 Mirantis Inc. and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -{%- from "tacker/map.jinja" import server with context %} -{%- if server.enabled %} - -include: -- git - -{{ server.git.source }}: - git.latest: - - target: {{ server.git.target }} - - rev: {{ server.git.branch }} - - depth: 1 - -tacker_packages: - pkg.installed: - - names: {{ server.pkgs }} - -tacker_install: - cmd.run: - - name: python setup.py install - - cwd: {{ server.git.target }} - - creates: /usr/local/bin/tacker-server - - require: - - git: {{ server.git.source }} - -/usr/local/etc/tacker/tacker.conf: - file.managed: - - source: salt://tacker/files/tacker.conf - - template: jinja - - makedirs: true - - require: - - cmd: tacker_install - -tacker_db_manage: - cmd.run: - - name: /usr/local/bin/tacker-db-manage --config-file /usr/local/etc/tacker/tacker.conf upgrade head - - require: - - file: /usr/local/etc/tacker/tacker.conf - -/lib/systemd/system/tacker.service: - file.managed: - - source: salt://tacker/files/tacker.systemd - -tacker: - service.running: - - enable: true - - watch: - - file: /usr/local/etc/tacker/tacker.conf - -{%- endif %} diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh index f5affb0d3..8f7cccd26 100644 --- a/mcp/scripts/lib.sh +++ b/mcp/scripts/lib.sh @@ -454,10 +454,7 @@ function create_vms { function update_mcpcontrol_network { # set static ip address for salt master node, MaaS node - local cmac=$(virsh domiflist cfg01 2>&1| awk '/mcpcontrol/ {print $5; exit}') local amac=$(virsh domiflist mas01 2>&1| awk '/mcpcontrol/ {print $5; exit}') - virsh net-update "mcpcontrol" add ip-dhcp-host \ - "" --live --config [ -z "${amac}" ] || virsh net-update "mcpcontrol" add ip-dhcp-host \ "" --live --config } @@ -489,6 +486,29 @@ function start_vms { done } +function prepare_containers { + local image_dir=$1 + [ -n "${image_dir}" ] || exit 1 + [ -n "${MCP_REPO_ROOT_PATH}" ] || exit 1 + + "${image_dir}/docker-compose" -f docker-compose/docker-compose.yaml down + sudo rm -rf "${image_dir}/salt" "${image_dir}/nodes/"* + mkdir -p "${image_dir}/salt/"{master.d,minion.d} + # salt state does not properly configure file_roots in master.conf, hard set it + sed -e 's/user: salt/user: root\nfile_recv: True/' -e 's/auto_accept:/open_mode:/' \ + "${MCP_REPO_ROOT_PATH}/docker/files/salt/master.conf" > \ + "${image_dir}/salt/master.d/opnfv.conf" + echo 'master: localhost' > "${image_dir}/salt/minion.d/opnfv.conf" + cp "${MCP_REPO_ROOT_PATH}/mcp/scripts/docker-compose/files/hosts" \ + "${image_dir}/hosts" +} + +function start_containers { + local image_dir=$1 + [ -n "${image_dir}" ] || exit 1 + "${image_dir}/docker-compose" -f docker-compose/docker-compose.yaml up --quiet-pull -d +} + function check_connection { local total_attempts=60 local sleep_time=5 @@ -584,12 +604,27 @@ function get_nova_compute_pillar_data { } function docker_install { + local image_dir=$1 # Mininum effort attempt at installing Docker if missing - if ! which docker; then + if ! docker --version; then curl -fsSL https://get.docker.com -o get-docker.sh sudo sh get-docker.sh rm get-docker.sh # On RHEL distros, the Docker service should be explicitly started sudo systemctl start docker + else + DOCKER_VER=$(docker version --format '{{.Server.Version}}') + if [ "${DOCKER_VER%%.*}" -lt 2 ]; then + notify_e "[ERROR] Docker version ${DOCKER_VER} is too old, please upgrade it." + fi + fi + # Distro-provided docker-compose might be simply broken (Ubuntu 16.04, CentOS 7) + COMPOSE_BIN="${image_dir}/docker-compose" + COMPOSE_VERSION='1.22.0' + notify_n "[WARN] Using docker-compose ${COMPOSE_VERSION} in ${COMPOSE_BIN}" 3 + if [ ! -e "${COMPOSE_BIN}" ]; then + COMPOSE_URL="https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}" + sudo curl -L "${COMPOSE_URL}/docker-compose-$(uname -s)-$(uname -m)" -o "${COMPOSE_BIN}" + sudo chmod +x "${COMPOSE_BIN}" fi } diff --git a/mcp/scripts/salt.sh b/mcp/scripts/salt.sh deleted file mode 100755 index 46ef6ffbf..000000000 --- a/mcp/scripts/salt.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash -e -############################################################################## -# Copyright (c) 2017 Mirantis Inc., Enea AB and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -# -# Deploy Salt Master -# - -CI_DEBUG=${CI_DEBUG:-0}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x -F_GIT_ROOT=$(git rev-parse --show-toplevel) -F_GIT_DIR=$(cd "${F_GIT_ROOT}/mcp" && git rev-parse --git-dir) -F_GIT_SUBD=${F_GIT_ROOT#${F_GIT_DIR%%/.git*}} -OPNFV_TMP_DIR="/home/${SALT_MASTER_USER}/opnfv" -OPNFV_GIT_DIR="/root/opnfv" -OPNFV_FUEL_DIR="/root/fuel" # Should be in sync with patch.sh, scripts patches -OPNFV_RDIR="reclass/classes/cluster/all-mcp-arch-common" -OPNFV_VCP_IMG="mcp/scripts/base_image_opnfv_fuel_vcp.img" -OPNFV_VCP_DIR="/srv/salt/env/prd/salt/files/control/images" -LOCAL_GIT_DIR="${F_GIT_ROOT%${F_GIT_SUBD}}" -LOCAL_PDF_RECLASS=$1; shift -# shellcheck disable=SC2116,SC2086 -LOCAL_VIRT_NODES=$(echo ${*//cfg01/}) # unquoted to filter space -NODE_MASK="${LOCAL_VIRT_NODES// /|}" - -# push to cfg01 current git repo first (including submodules), at ~ubuntu/opnfv -# later we move it to ~root/opnfv (and ln as ~root/fuel); delete the temp clone -remote_tmp="${SSH_SALT}:$(basename "${OPNFV_TMP_DIR}")" -STORAGE_DIR=$(dirname "${LOCAL_PDF_RECLASS}") -REL_STORAGE_DIR_PATH=${STORAGE_DIR#${LOCAL_GIT_DIR}} -if [[ "${REL_STORAGE_DIR_PATH}" == "${STORAGE_DIR}" ]] -then - REL_STORAGE_DIR_PATH="" -fi -rsync -Erl --delete -e "ssh ${SSH_OPTS}" \ - --exclude={.gitignore,"$REL_STORAGE_DIR_PATH"} \ - "${LOCAL_GIT_DIR}/" "${remote_tmp}/" -if [ -n "${LOCAL_PDF_RECLASS}" ] && [ -f "${LOCAL_PDF_RECLASS}" ]; then - rsync -e "ssh ${SSH_OPTS}" "${LOCAL_PDF_RECLASS}" \ - "${remote_tmp}${F_GIT_SUBD}/mcp/${OPNFV_RDIR}/opnfv/" -fi -local_vcp_img=$(dirname "${LOCAL_PDF_RECLASS}")/$(basename "${OPNFV_VCP_IMG}") -if [ -e "${local_vcp_img}" ]; then - rsync -L -e "ssh ${SSH_OPTS}" "${local_vcp_img}" \ - "${remote_tmp}${F_GIT_SUBD}/${OPNFV_VCP_IMG}" -fi - -# ssh to cfg01 -# shellcheck disable=SC2086,2087 -ssh ${SSH_OPTS} "${SSH_SALT}" bash -s -e << SALT_INSTALL_END - sudo -i - set -e - export TERM=${TERM} - export CI_DEBUG=${CI_DEBUG}; [[ "${CI_DEBUG}" =~ (false|0) ]] || set -x - - echo -n 'Checking out cloud-init has finished running ...' - while [ ! -f /var/lib/cloud/instance/boot-finished ]; do echo -n '.'; sleep 1; done - echo ' done' - - mkdir -p /srv/salt /usr/share/salt-formulas/reclass - rm -rf ${OPNFV_GIT_DIR} ${OPNFV_FUEL_DIR} - mv ${OPNFV_TMP_DIR} ${OPNFV_GIT_DIR} && chown -R root.root ${OPNFV_GIT_DIR} - find ${OPNFV_GIT_DIR} -name '.git' -type f | while read f_git; do - sed -i 's@${LOCAL_GIT_DIR}@${OPNFV_GIT_DIR}@g' \$f_git - done - ln -sf ${OPNFV_GIT_DIR}${F_GIT_SUBD} ${OPNFV_FUEL_DIR} - ln -sf ${OPNFV_FUEL_DIR}/mcp/reclass /srv/salt - ln -sf ${OPNFV_FUEL_DIR}/mcp/deploy/scripts /srv/salt - ln -sf ${OPNFV_FUEL_DIR}/mcp/scripts/mcp.rsa $(dirname "${OPNFV_FUEL_DIR}") - - cp -r ${OPNFV_FUEL_DIR}/mcp/metadata/service /usr/share/salt-formulas/reclass - cd /srv/salt/reclass/classes/service && \ - ln -sf /usr/share/salt-formulas/reclass/service/opendaylight - - # Armband APT-MK nightly/extra repo for forked & extended reclass - wget -qO - https://linux.enea.com/apt-mk/public.gpg | apt-key add - - echo 'deb http://linux.enea.com/apt-mk/xenial nightly extra' > \ - '/etc/apt/sources.list.d/armband_mcp_extra.list' - apt-get update - - cd /srv/salt/scripts - export DEBIAN_FRONTEND=noninteractive - echo 'Dpkg::Use-Pty "0";' > /etc/apt/apt.conf.d/90silence-dpkg - OLD_DOMAIN=\$(grep -sPzo "id: cfg01\.\K(\S*)" /etc/salt/minion.d/minion.conf) || true - BOOTSTRAP_SALTSTACK_OPTS=" -r -dX stable 2017.7 " \ - MASTER_HOSTNAME=cfg01.${CLUSTER_DOMAIN} DISTRIB_REVISION=nightly \ - EXTRA_FORMULAS="nfs panko gnocchi oslo-templates" \ - ./salt-master-init.sh - salt-key -Ay - - cp -r ${OPNFV_FUEL_DIR}/mcp/salt-formulas/* /usr/share/salt-formulas/env - cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list formulas - cd ${OPNFV_FUEL_DIR}/mcp/patches && ./patch.sh patches.list reclass - - source ${OPNFV_FUEL_DIR}/mcp/scripts/lib.sh - wait_for 3.0 "salt-call state.apply salt" - - # In case scenario changed (and implicitly domain name), re-register minions - if [ -n "\${OLD_DOMAIN}" ] && [ "\${OLD_DOMAIN}" != "${CLUSTER_DOMAIN}" ]; then - salt "*.\${OLD_DOMAIN}" cmd.run "grep \${OLD_DOMAIN} -sRl /etc/salt | \ - xargs --no-run-if-empty sed -i 's/\${OLD_DOMAIN}/${CLUSTER_DOMAIN}/g'; \ - service salt-minion restart" || true - salt-key -yd "*.\${OLD_DOMAIN}" - salt-key -Ay - fi - - # Init specific to VMs on FN (all for virtual, cfg|mas for baremetal) - wait_for 3.0 'salt -C "cfg01*" state.apply linux' - if [[ "${LOCAL_VIRT_NODES}" =~ mas ]]; then - wait_for 3.0 'salt -C "mas*" test.ping' - else - wait_for 3.0 '(for n in ${LOCAL_VIRT_NODES}; do salt -C \${n}.* test.ping || exit; done)' - fi - wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" saltutil.sync_all' - wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.apply salt' - - wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" state.sls linux.system,linux.storage' - wait_for 2.0 'salt -C "E@^(${NODE_MASK}).*" state.sls linux.network' - salt -C "E@^(${NODE_MASK}).*" system.reboot - wait_for 90.0 'salt -C "E@^(${NODE_MASK}).*" test.ping' - wait_for 3.0 'salt -C "E@^(${NODE_MASK}).*" pkg.upgrade refresh=False dist_upgrade=True' - - wait_for 3.0 'salt -C "E@^(${NODE_MASK}|cfg01).*" state.sls ntp' - - if [ -f "${OPNFV_FUEL_DIR}/${OPNFV_VCP_IMG}" ]; then - mkdir -p "${OPNFV_VCP_DIR}" - mv "${OPNFV_FUEL_DIR}/${OPNFV_VCP_IMG}" "${OPNFV_VCP_DIR}/" - fi -SALT_INSTALL_END diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2 index c0629fec4..b8c32135c 100644 --- a/mcp/scripts/xdf_data.sh.j2 +++ b/mcp/scripts/xdf_data.sh.j2 @@ -33,7 +33,7 @@ {%- macro serialize_vnodes() -%} {%- set arr = [] -%} {%- for section in section_map -%} - {%- for n in V.nodes[section] -%} + {%- for n in V.nodes[section] or [] -%} {%- if section_map[section] < 0 or conf.nodes[section_map[section] + loop.index0].node.type == 'virtual' -%} {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%} {%- do arr.append(pack([n, V[n].ram or arch.default.ram, @@ -48,7 +48,7 @@ {%- macro filter_nodes(type, output_as_query = False, sections = section_map) -%} {%- set arr = [] -%} {%- for section in sections -%} - {%- for n in V.nodes[section] -%} + {%- for n in V.nodes[section] or [] -%} {%- if ( (section_map[section] < 0 and type == 'virtual') or (section_map[section] >= 0 and conf.nodes[section_map[section] + loop.index0].node.type in type) ) -%} -- cgit 1.2.3-korg