summaryrefslogtreecommitdiffstats
path: root/mcp
diff options
context:
space:
mode:
authorAlexandru Avadanii <Alexandru.Avadanii@enea.com>2018-03-03 21:39:59 +0100
committerAlexandru Avadanii <Alexandru.Avadanii@enea.com>2018-03-07 03:19:45 +0100
commit256e2a50b9b340bb78a8c85fdd7cacf96d0fb637 (patch)
tree2d9c7174840020b6d40012c65a82e75493bbad29 /mcp
parent3f63be553d8be68b2dc39dd2813bf5ffef0c3207 (diff)
[vnode parsing] Move core logic to j2
j2/python is easier to read and manipulate strings, although it does need some special care about undefined dict keys. With this in place, deploy.sh only contains the higher level logic for the deployment process. - merge arch-specific default configuration files into a singular file with arch name as main dict key of old config (also avoids creating duplicate 'virtual' YAML keys in $LOCAL_PDF); - move template handling to separate <lib_template.sh>; - decouple tight bash ordering of scenario expansion -> parse_yaml -> variable export (e.g. CLUSTER_DOMAIN) -> re-use in cluster j2s; however we can't parse *all* j2s in one go, as scenario j2s might expand to YAMLs needed while expanding cluster j2; - split `do_templates` into separate functions for each stage, with no coupling between them other then call order; Change-Id: I4b5e804094c00e5e918caf769fd85fa52181ad76 Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
Diffstat (limited to 'mcp')
-rw-r--r--mcp/config/scenario/defaults-x86_64.yaml27
-rw-r--r--mcp/config/scenario/defaults.yaml (renamed from mcp/config/scenario/defaults-aarch64.yaml)27
-rw-r--r--mcp/scripts/lib.sh64
-rw-r--r--mcp/scripts/lib_template.sh88
-rw-r--r--mcp/scripts/user-data.admin.sh.j22
-rw-r--r--mcp/scripts/user-data.mcp.sh.j24
-rw-r--r--mcp/scripts/xdf_data.sh.j253
7 files changed, 168 insertions, 97 deletions
diff --git a/mcp/config/scenario/defaults-x86_64.yaml b/mcp/config/scenario/defaults-x86_64.yaml
deleted file mode 100644
index 18b0826f9..000000000
--- a/mcp/config/scenario/defaults-x86_64.yaml
+++ /dev/null
@@ -1,27 +0,0 @@
-##############################################################################
-# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
----
-base_image: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
-virtual:
- default:
- vcpus: 2
- ram: 4096
- common:
- apt:
- keys:
- - https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.11/SALTSTACK-GPG-KEY.pub
- repos:
- # <repo name> <repo prio> deb [arch=<arch>] <repo url> <repo dist> <repo comp>
- - saltstack 500 deb [arch=amd64] http://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.11 xenial main
- pkg:
- install:
- - salt-minion
- control:
- pkg:
- install:
- - cloud-init
diff --git a/mcp/config/scenario/defaults-aarch64.yaml b/mcp/config/scenario/defaults.yaml
index 1efa12ae4..e6915d468 100644
--- a/mcp/config/scenario/defaults-aarch64.yaml
+++ b/mcp/config/scenario/defaults.yaml
@@ -1,13 +1,33 @@
##############################################################################
-# Copyright (c) 2017 Mirantis Inc. and others.
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
---
-base_image: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-arm64-uefi1.img
-virtual:
+x86_64:
+ base_image: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-amd64-disk1.img
+ default:
+ vcpus: 2
+ ram: 4096
+ common:
+ apt:
+ keys:
+ - https://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.11/SALTSTACK-GPG-KEY.pub
+ repos:
+ # <repo name> <repo prio> deb [arch=<arch>] <repo url> <repo dist> <repo comp>
+ - saltstack 500 deb [arch=amd64] http://repo.saltstack.com/apt/ubuntu/16.04/amd64/2016.11 xenial main
+ pkg:
+ install:
+ - salt-minion
+ control:
+ apt: ~
+ pkg:
+ install:
+ - cloud-init
+aarch64:
+ base_image: https://cloud-images.ubuntu.com/xenial/current/xenial-server-cloudimg-arm64-uefi1.img
default:
vcpus: 6
ram: 4096
@@ -25,6 +45,7 @@ virtual:
- linux-headers-generic-hwe-16.04-edge
- salt-minion
control:
+ apt: ~
pkg:
install:
- cloud-init
diff --git a/mcp/scripts/lib.sh b/mcp/scripts/lib.sh
index e8edf9587..5be4c556d 100644
--- a/mcp/scripts/lib.sh
+++ b/mcp/scripts/lib.sh
@@ -1,7 +1,7 @@
#!/bin/bash -e
# shellcheck disable=SC2155,SC1001,SC2015,SC2128
##############################################################################
-# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
@@ -543,65 +543,3 @@ function get_nova_compute_pillar_data {
echo "${value}"
fi
}
-
-function do_templates() {
- local git_repo_root=$1; shift
- local image_dir=$1; shift
- local target_lab=$1; shift
- local target_pod=$1; shift
- local lab_config_uri=$1; shift
- local scenario_dir=${1:-}
-
- RECLASS_CLUSTER_DIR=$(cd "${git_repo_root}/mcp/reclass/classes/cluster"; pwd)
- PHAROS_GEN_CFG="./pharos/config/utils/generate_config.py"
- PHAROS_IA=$(readlink -f "./pharos/config/installers/fuel/pod_config.yml.j2")
- PHAROS_VALIDATE_SCHEMA_SCRIPT="./pharos/config/utils/validate_schema.py"
- PHAROS_SCHEMA_PDF="./pharos/config/pdf/pod1.schema.yaml"
- PHAROS_SCHEMA_IDF="./pharos/config/pdf/idf-pod1.schema.yaml"
- BASE_CONFIG_PDF="${lab_config_uri}/labs/${target_lab}/${target_pod}.yaml"
- BASE_CONFIG_IDF="${lab_config_uri}/labs/${target_lab}/idf-${target_pod}.yaml"
- LOCAL_PDF="${image_dir}/$(basename "${BASE_CONFIG_PDF}")"
- LOCAL_IDF="${image_dir}/$(basename "${BASE_CONFIG_IDF}")"
-
- # Two-stage expansion, first stage handles pod_config and scenarios only
- if [ -n "${scenario_dir}" ]; then
- # Make sample PDF/IDF available via default lab-config (pharos submodule)
- ln -sf "$(readlink -f "../config/labs/local")" "./pharos/labs/"
-
- # Expand scenario file and main reclass input (pod_config.yaml) based on PDF
- if ! curl --create-dirs -o "${LOCAL_PDF}" "${BASE_CONFIG_PDF}"; then
- notify_e "[ERROR] Could not retrieve PDF (Pod Descriptor File)!"
- elif ! curl -o "${LOCAL_IDF}" "${BASE_CONFIG_IDF}"; then
- notify_e "[ERROR] Could not retrieve IDF (Installer Descriptor File)!"
- fi
- # Check first if configuration files are valid
- if [[ ! "$target_pod" =~ "virtual" ]]; then
- if ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_PDF}" \
- -s "${PHAROS_SCHEMA_PDF}"; then
- notify_e "[ERROR] PDF does not match yaml schema!"
- elif ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_IDF}" \
- -s "${PHAROS_SCHEMA_IDF}"; then
- notify_e "[ERROR] IDF does not match yaml schema!"
- fi
- fi
- if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \
- -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then
- notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!"
- fi
- template_dirs="${scenario_dir}"
- template_err_str='Could not convert j2 scenario definitions!'
- else
- # Expand reclass and virsh network templates based on PDF + IDF
- printenv | \
- awk '/^(SALT|MCP|MAAS|CLUSTER).*=/ { gsub(/=/,": "); print }' >> "${LOCAL_PDF}"
- template_dirs="${RECLASS_CLUSTER_DIR} $(readlink -f virsh_net) $(readlink -f ./*j2)"
- template_err_str='Could not convert PDF to network definitions!'
- fi
- # shellcheck disable=SC2086
- j2args=$(find $template_dirs -name '*.j2' -exec echo -j {} \;)
- # shellcheck disable=SC2086
- if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
- -i "$(dirname "${PHAROS_IA}")"; then
- notify_e "[ERROR] ${template_err_str}"
- fi
-}
diff --git a/mcp/scripts/lib_template.sh b/mcp/scripts/lib_template.sh
new file mode 100644
index 000000000..0fbe628b7
--- /dev/null
+++ b/mcp/scripts/lib_template.sh
@@ -0,0 +1,88 @@
+#!/bin/bash -e
+# shellcheck disable=SC2155,SC1001,SC2015,SC2128
+##############################################################################
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+#
+# Library of shell functions dedicated to j2 template handling
+#
+
+PHAROS_GEN_CFG='./pharos/config/utils/generate_config.py'
+PHAROS_IA='./pharos/config/installers/fuel/pod_config.yml.j2'
+PHAROS_VALIDATE_SCHEMA_SCRIPT='./pharos/config/utils/validate_schema.py'
+PHAROS_SCHEMA_PDF='./pharos/config/pdf/pod1.schema.yaml'
+PHAROS_SCHEMA_IDF='./pharos/config/pdf/idf-pod1.schema.yaml'
+
+# Handles pod_config and scenarios only
+function do_templates_scenario {
+ local image_dir=$1; shift
+ local target_lab=$1; shift
+ local target_pod=$1; shift
+ local lab_config_uri=$1; shift
+ local scenario_dir=$1
+
+ BASE_CONFIG_PDF="${lab_config_uri}/labs/${target_lab}/${target_pod}.yaml"
+ BASE_CONFIG_IDF="${lab_config_uri}/labs/${target_lab}/idf-${target_pod}.yaml"
+ LOCAL_PDF="${image_dir}/$(basename "${BASE_CONFIG_PDF}")"
+ LOCAL_IDF="${image_dir}/$(basename "${BASE_CONFIG_IDF}")"
+
+ # Make sample PDF/IDF available via default lab-config (pharos submodule)
+ ln -sf "$(readlink -f "../config/labs/local")" "./pharos/labs/"
+
+ # Expand scenario file and main reclass input (pod_config.yaml) based on PDF
+ if ! curl --create-dirs -o "${LOCAL_PDF}" "${BASE_CONFIG_PDF}"; then
+ notify_e "[ERROR] Could not retrieve PDF (Pod Descriptor File)!"
+ elif ! curl -o "${LOCAL_IDF}" "${BASE_CONFIG_IDF}"; then
+ notify_e "[ERROR] Could not retrieve IDF (Installer Descriptor File)!"
+ fi
+ # Check first if configuration files are valid
+ if [[ ! "$target_pod" =~ "virtual" ]]; then
+ if ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_PDF}" \
+ -s "${PHAROS_SCHEMA_PDF}"; then
+ notify_e "[ERROR] PDF does not match yaml schema!"
+ elif ! "${PHAROS_VALIDATE_SCHEMA_SCRIPT}" -y "${LOCAL_IDF}" \
+ -s "${PHAROS_SCHEMA_IDF}"; then
+ notify_e "[ERROR] IDF does not match yaml schema!"
+ fi
+ fi
+ if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" \
+ -j "${PHAROS_IA}" -v > "${image_dir}/pod_config.yml"; then
+ notify_e "[ERROR] Could not convert PDF+IDF to reclass model input!"
+ fi
+ printenv | \
+ awk '/^(SALT|MCP|MAAS).*=/ { gsub(/=/,": "); print }' >> "${LOCAL_PDF}"
+ j2args=$(find "${scenario_dir}" -name '*.j2' -exec echo -j {} \;)
+ # shellcheck disable=SC2086
+ if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
+ -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then
+ notify_e '[ERROR] Could not convert j2 scenario definitions!'
+ fi
+}
+
+# Expand reclass and virsh network templates based on PDF + IDF + others
+function do_templates_cluster {
+ local image_dir=$1; shift
+ local target_lab=$1; shift
+ local target_pod=$1; shift
+ local git_repo_root=$1; shift
+ local extra_yaml=("$@")
+
+ RECLASS_CLUSTER_DIR=$(cd "${git_repo_root}/mcp/reclass/classes/cluster"; pwd)
+ LOCAL_PDF="${image_dir}/${target_pod}.yaml"
+
+ for _yaml in "${extra_yaml[@]}"; do
+ awk '/^---$/{f=1;next;}f' "${_yaml}" >> "${LOCAL_PDF}"
+ done
+ # shellcheck disable=SC2046
+ j2args=$(find "${RECLASS_CLUSTER_DIR}" "$(readlink -f virsh_net)" $(readlink -f ./*j2) \
+ -name '*.j2' -exec echo -j {} \;)
+ # shellcheck disable=SC2086
+ if ! "${PHAROS_GEN_CFG}" -y "${LOCAL_PDF}" ${j2args} -b -v \
+ -i "$(dirname "$(readlink -f "${PHAROS_IA}")")"; then
+ notify_e '[ERROR] Could not convert PDF to network definitions!'
+ fi
+}
diff --git a/mcp/scripts/user-data.admin.sh.j2 b/mcp/scripts/user-data.admin.sh.j2
index b4c89b29c..d9b86c79c 100644
--- a/mcp/scripts/user-data.admin.sh.j2
+++ b/mcp/scripts/user-data.admin.sh.j2
@@ -8,7 +8,7 @@
##############################################################################
rm /etc/salt/minion_id
rm -f /etc/salt/pki/minion/minion_master.pub
-echo "id: $(hostname).{{ conf.CLUSTER_DOMAIN }}" > /etc/salt/minion
+echo "id: $(hostname).{{ conf.cluster.domain }}" > /etc/salt/minion
{#- should be in sync with 'opnfv_infra_config_pxe_address' in 'pharos/config/installers/fuel/pod_config.yml.j2 #}
echo "master: {{ conf.idf.net_config.admin.network | ipaddr_index(2) }}" >> /etc/salt/minion
service salt-minion restart
diff --git a/mcp/scripts/user-data.mcp.sh.j2 b/mcp/scripts/user-data.mcp.sh.j2
index 4bd0d1d09..bd80961e6 100644
--- a/mcp/scripts/user-data.mcp.sh.j2
+++ b/mcp/scripts/user-data.mcp.sh.j2
@@ -1,6 +1,6 @@
#!/bin/bash
##############################################################################
-# Copyright (c) 2017 Mirantis Inc., Enea AB and others.
+# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
@@ -8,6 +8,6 @@
##############################################################################
rm /etc/salt/minion_id
rm -f /etc/salt/pki/minion/minion_master.pub
-echo "id: $(hostname).{{ conf.CLUSTER_DOMAIN }}" > /etc/salt/minion
+echo "id: $(hostname).{{ conf.cluster.domain }}" > /etc/salt/minion
echo "master: {{ conf.SALT_MASTER }}" >> /etc/salt/minion
service salt-minion restart
diff --git a/mcp/scripts/xdf_data.sh.j2 b/mcp/scripts/xdf_data.sh.j2
index 9479f2356..ad3c0fe1e 100644
--- a/mcp/scripts/xdf_data.sh.j2
+++ b/mcp/scripts/xdf_data.sh.j2
@@ -1,4 +1,5 @@
#!/bin/bash -e
+# shellcheck disable=SC2034
##############################################################################
# Copyright (c) 2018 Mirantis Inc., Enea AB and others.
# All rights reserved. This program and the accompanying materials
@@ -10,11 +11,61 @@
# Data derived from XDF (PDF/IDF/SDF/etc), used as input in deploy.sh
#
-# Determine bridge names based on IDF, where all bridges are now mandatory
+{%- set arch = conf[conf.MCP_JUMP_ARCH] -%}
+
+{%- macro bash_arr(_l) -%}
+ ({%- for n in _l -%}'{{ n }}' {% endfor -%})
+{%- endmacro -%}
+
+{#- Pack list as `sep`-separated string, replacing spaces with '|' -#}
+{%- macro pack(x = [], sep = ',') -%}
+ {{ x | join(sep) | replace(' ', '|') }}
+{%- endmacro -%}
+
+{#- Pack all vnode data as string -#}
+{%- macro serialize_vnodes() -%}
+ {%- set V = conf.virtual -%}
+ {%- set arr = [] -%}
+ {%- for n in V.nodes -%}
+ {%- if n not in V -%}{%- do V.update({n: {}}) -%}{%- endif -%}
+ {%- do arr.append(pack([n, V[n].ram or arch.default.ram,
+ V[n].vcpus or arch.default.vcpus])) -%}
+ {%- endfor -%}
+ '{{ pack(arr, '|') }}'
+{%- endmacro -%}
+
+{#- Pack apt_pkg data as string -#}
+{%- macro serialize_apt_pkg() -%}
+ {%- set arr = [] -%}
+ {%- set sections = [arch.common] -%}
+ {%- if 'virtual_control_plane' in conf.cluster.states -%}
+ {%- do sections.append(arch.control) -%}
+ {%- endif -%}
+ {%- for c in sections -%}
+ {%- do arr.append(pack([pack(c.apt['keys']), pack(c.apt.repos),
+ pack(c.pkg.install), pack(c.pkg.remove)], '^')) -%}
+ {%- endfor -%}
+ '{{ pack(arr, '^') }}'
+{%- endmacro -%}
+
{%- set bridges = conf.idf.fuel.jumphost.bridges %}
+# Determine bridge names based on IDF, where all bridges are now mandatory
OPNFV_BRIDGES=(
'{{ bridges.admin or "pxebr" }}'
'{{ bridges.mgmt or "mgmt" }}'
'{{ bridges.private or "internal" }}'
'{{ bridges.public or "public" }}'
)
+
+export CLUSTER_DOMAIN={{ conf.cluster.domain }}
+cluster_states={{ bash_arr(conf.cluster.states) }}
+virtual_nodes={{ bash_arr(conf.virtual.nodes) }}
+base_image={{ arch.base_image }}
+
+# Serialize vnode data as '<name0>,<ram0>,<vcpu0>|<name1>,<ram1>,<vcpu1>[...]'
+virtual_nodes_data={{ serialize_vnodes() }}
+
+# Serialize repos, packages to (pre-)install/remove for:
+# - foundation node VM base image (virtual: all VMs, baremetal: cfg01|mas01)
+# - virtualized control plane VM base image (only when VCP is used)
+virtual_repos_pkgs={{ serialize_apt_pkg() }}