diff options
Diffstat (limited to 'config/utils')
| -rw-r--r-- | config/utils/README.eyaml.rst | 2 | ||||
| -rwxr-xr-x | config/utils/check-jinja2.sh | 65 | ||||
| -rwxr-xr-x | config/utils/check-schema.sh | 51 | ||||
| -rw-r--r-- | config/utils/config.example.yaml | 2 | ||||
| -rw-r--r-- | config/utils/gen_config_lib.py | 224 | ||||
| -rwxr-xr-x | config/utils/generate_config.py | 96 | ||||
| -rwxr-xr-x | config/utils/validate_schema.py | 46 |
7 files changed, 416 insertions, 70 deletions
diff --git a/config/utils/README.eyaml.rst b/config/utils/README.eyaml.rst index 0031d9d8..1f584261 100644 --- a/config/utils/README.eyaml.rst +++ b/config/utils/README.eyaml.rst @@ -1,6 +1,6 @@ .. This work is licensed under a Creative Commons Attribution 4.0 International License. .. SPDX-License-Identifier: CC-BY-4.0 -.. (c) 2017 OPNFV and others. +.. (c) 2018 OPNFV and others. Use eyaml to decrypt secret values ================================== diff --git a/config/utils/check-jinja2.sh b/config/utils/check-jinja2.sh index 2953ff6a..1fcdb9a2 100755 --- a/config/utils/check-jinja2.sh +++ b/config/utils/check-jinja2.sh @@ -1,7 +1,7 @@ #!/bin/bash # SPDX-license-identifier: Apache-2.0 ############################################################################## -# Copyright (c) 2016 Linux Foundation and others. +# Copyright (c) 2018 Linux Foundation and others. # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at @@ -12,50 +12,75 @@ set +x set +o errexit export PATH=$PATH:/usr/local/bin/ -GEN_CFG='./config/utils/generate_config.py' -INSTALLER_ADAPTERS='./config/installers/*' +# Optional filtering of test matrix: per-lab, per-pod, per-installer +# e.g. To check zte-pod{2,3} against all installer adapters: +# ./config/utils/check-jinja2.sh zte 'pod(2|3)' +FILTER_LAB=${1:-*} # e.g. 'zte' (glob) +FILTER_POD=${2:-(pod|virtual)[[:digit:]]+} # e.g. 'pod1' (regex) +FILTER_IA=${3:-*} # e.g. 'fuel' (glob) + +GEN_CFG='python ./config/utils/generate_config.py' +INSTALLER_ADAPTERS="./config/installers/${FILTER_IA}" TMPF='/tmp/out.yml' # should be outside Jenkins WS to prevent data leakage RC=0 +echo "Using $(yamllint --version)" + # Build a table header, using ';' as column sep -SUMMARY='PDF Verify Matrix;YAML Lint;' -for adapter in ${INSTALLER_ADAPTERS}; do +for adapter in 'PDF Verify Matrix' ${INSTALLER_ADAPTERS}; do SUMMARY+="$(basename "${adapter}");" done # Iterate all PDFs, check with each installer adapter, log results +# shellcheck disable=SC2086 while IFS= read -r lab_config; do - valid_yaml='OK' - echo -e "\n\nyamllint -s ${lab_config}" - if ! yamllint -s "${lab_config}"; then valid_yaml='FAIL'; fi - SUMMARY+="\n${lab_config#labs/};${valid_yaml};" + SUMMARY+="\n${lab_config#labs/};" + idf_config="$(dirname "${lab_config}")/idf-$(basename "${lab_config}")" + idf_installer=$(grep 'installer:' "${idf_config}" 2>/dev/null || echo) + echo "###################### ${lab_config} ######################" for adapter in ${INSTALLER_ADAPTERS}; do pdf_inst=0 pdf_inst_pass=0 pdf_yaml_pass=0 + installer_name=$(basename "${adapter}") + if [ -n "${idf_installer}" ] && echo "${idf_installer}" | \ + grep -vq "${installer_name}"; then + SUMMARY+='-;' + echo -n "[GENERATE] [SKIP] idf.installer defined and " + echo -e "${installer_name} not listed, skipping.\n" + continue + fi while IFS= read -r jinja_template; do - echo -e "\n${GEN_CFG} -y ${lab_config} -j ${jinja_template}" - if "${GEN_CFG}" -y "${lab_config}" \ - -j "${jinja_template}" > "${TMPF}"; then - echo 'Result: PASS' + pdf_gen_cmd="${GEN_CFG} -y ${lab_config} -j ${jinja_template} \ + -i $(dirname "${jinja_template}")" + if ${pdf_gen_cmd} > "${TMPF}"; then ((pdf_inst_pass+=1)) - echo -e "\nyamllint -s ${jinja_template%.j2}" - if yamllint -s "${TMPF}"; then ((pdf_yaml_pass+=1)); fi + echo "[GENERATE] [OK] ${pdf_gen_cmd}" + if yamllint -s <(sed 's|ENC\[PKCS.*\]|opnfv|g' "${TMPF}"); then + ((pdf_yaml_pass+=1)); + echo "[YAMLLINT] [OK] yamllint -s ${jinja_template%.j2}" + else + echo "[YAMLLINT] [ERROR] yamllint -s ${jinja_template%.j2}" + fi else - echo 'Result: FAIL' + echo "[GENERATE] [ERROR] ${pdf_gen_cmd}" RC=1 fi ((pdf_inst+=1)) + echo '' done < <(find "${adapter}" -name '*.j2') SUMMARY+="${pdf_yaml_pass}/${pdf_inst_pass}/${pdf_inst};" done -done < <(find 'config' 'labs' -name 'pod*.yaml') - +done < <(find labs/${FILTER_LAB} -regextype egrep \ + -regex "labs/.+/${FILTER_POD}.yaml") rm -f "${TMPF}" -echo -e '\n\nNOTE: tuple fmt: (valid YAML output/sucessful parse/templates).\n' -echo -e "${SUMMARY}" | sed -e 's/^/;/g' -e 's/;/;| /g' | column -t -s ';' cat <<EOF +###################### Result Matrix ###################### + +NOTE: tuple fmt: (valid YAML output/sucessful parse/templates). + +$(echo -e "${SUMMARY}" | sed -e 's/^/;/g' -e 's/;/;| /g' | column -t -s ';') To troubleshoot PDF parsing against a specific installer adapter, execute the following commands locally (e.g. for zte-pod2/joid): diff --git a/config/utils/check-schema.sh b/config/utils/check-schema.sh new file mode 100755 index 00000000..041c0308 --- /dev/null +++ b/config/utils/check-schema.sh @@ -0,0 +1,51 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2018 Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +export PATH=$PATH:/usr/local/bin/ + +VALIDATE_SCHEMA='python ./config/utils/validate_schema.py' +PDF_SCHEMA='./config/pdf/pod1.schema.yaml' +IDF_SCHEMA='./config/pdf/idf-pod1.schema.yaml' +RC=0 + +SUMMARY+=";;PDF;IDF;\n" +while IFS= read -r lab_config; do + idf_config="$(dirname "${lab_config}")/idf-$(basename "${lab_config}")" + pdf_cmd="${VALIDATE_SCHEMA} -s ${PDF_SCHEMA} -y ${lab_config}" + idf_cmd="${VALIDATE_SCHEMA} -s ${IDF_SCHEMA} -y ${idf_config}" + echo "###################### ${lab_config} ######################" + pdf_out=$(${pdf_cmd} 2>&1 | sed 's|ENC\[PKCS.*\][\\n]*|opnfv|g') + if [ -z "${pdf_out}" ]; then + SUMMARY+=";${lab_config#labs/};OK;" + echo "[PDF] [OK] ${pdf_cmd}" + else + SUMMARY+=";${lab_config#labs/};ERROR;" + RC=1 + echo "${pdf_out}" + echo "[PDF] [ERROR] ${pdf_cmd}" + fi + if [ ! -f "${idf_config}" ]; then + SUMMARY+="-;\n" + elif ${idf_cmd}; then + SUMMARY+="OK;\n" + echo "[IDF] [OK] ${idf_cmd}" + else + SUMMARY+="ERROR;\n" + RC=1 + echo "[IDF] [ERROR] ${idf_cmd}" + fi + echo '' +done < <(find 'labs' -name 'pod*.yaml' -or -name 'virtual*.yaml') + +cat <<EOF +###################### Schema Validation Matrix ###################### + +$(echo -e "${SUMMARY}" | sed -e 's/;/;| /g' | column -t -s ';') +EOF +exit "${RC}" diff --git a/config/utils/config.example.yaml b/config/utils/config.example.yaml index 084d11d2..3a10a144 100644 --- a/config/utils/config.example.yaml +++ b/config/utils/config.example.yaml @@ -1,5 +1,5 @@ ############################################################################## -# Copyright (c) 2017 OPNFV and others. +# Copyright (c) 2018 OPNFV and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 diff --git a/config/utils/gen_config_lib.py b/config/utils/gen_config_lib.py new file mode 100644 index 00000000..1e7229be --- /dev/null +++ b/config/utils/gen_config_lib.py @@ -0,0 +1,224 @@ +############################################################################## +# Copyright (c) 2018 OPNFV and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +"""Library for generate_config functions and custom jinja2 filters""" + +import logging +from ipaddress import IPv4Network, IPv4Address + + +def load_custom_filters(environment): + """Load all defined filters into the jinja2 enviroment""" + + # TODO deprecate ipaddr_index and netmask for the better ipnet ones + filter_list = { + 'dpkg_arch': filter_dpkg_arch, + 'storage_size_num': filter_storage_size_num, + 'ipnet_hostaddr': filter_ipnet_hostaddr, + 'ipnet_hostmin': filter_ipnet_hostmin, + 'ipnet_hostmax': filter_ipnet_hostmax, + 'ipnet_broadcast': filter_ipnet_broadcast, + 'ipnet_netmask': filter_ipnet_netmask, + 'ipnet_contains_ip': filter_ipnet_contains_ip, + 'ipnet_contains_iprange': filter_ipnet_contains_iprange, + 'ipnet_range_size': filter_ipnet_range_size, + 'ipaddr_index': filter_ipaddr_index, + 'netmask': filter_netmask + } + + for name, function in filter_list.items(): + environment.filters[name] = function + + +def filter_dpkg_arch(arch, to_dpkg=True): + """Convert DPKG-compatible from processor arch and vice-versa""" + + # Processor architecture (as reported by $(uname -m)) + # vs DPKG architecture mapping + dpkg_arch_table = { + 'aarch64': 'arm64', + 'x86_64': 'amd64', + } + arch_dpkg_table = dict( + zip(dpkg_arch_table.values(), dpkg_arch_table.keys())) + + if to_dpkg: + return dpkg_arch_table[arch] + else: + return arch_dpkg_table[arch] + + +def filter_storage_size_num(size_str): + """Convert human-readable size string to a string convertible to float""" + + # pattern: '^[1-9][\d\.]*[MGT]B?$', multiplier=1000 (not KiB) + if size_str.endswith('B'): + size_str = size_str[:-1] + try: + size_num = 1000000 + for multiplier in ['M', 'G', 'T']: + if size_str.endswith(multiplier): + return '{:.2f}'.format(size_num * float(size_str[:-1])) + size_num = size_num * 1000 + return '{:.2f}'.format(float(size_str)) + except ValueError as ex: + logging.error(size_str + " is not a valid size string") + raise + + +def filter_ipnet_hostaddr(network_cidr, index): + """Return the host IP address on given index from an IP network""" + try: + network_cidr_str = unicode(network_cidr) + except NameError as ex: + network_cidr_str = str(network_cidr) + try: + return IPv4Network(network_cidr_str)[index] + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + except IndexError as ex: + logging.error(network_cidr_str + " has not enough range for " + + str(index) + " host IPs.") + raise + + +def filter_ipnet_broadcast(network_cidr): + """Return broadcast IP address from given IP network""" + try: + network_cidr_str = unicode(network_cidr) + except NameError as ex: + network_cidr_str = str(network_cidr) + try: + return IPv4Network(network_cidr_str).broadcast_address + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_hostmin(network_cidr): + """Return the first host IP address from given IP network""" + try: + network_cidr_str = unicode(network_cidr) + except NameError as ex: + network_cidr_str = str(network_cidr) + try: + return IPv4Network(network_cidr_str)[1] + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_hostmax(network_cidr): + """Return the last host IP address from given IP network""" + try: + network_cidr_str = unicode(network_cidr) + except NameError as ex: + network_cidr_str = str(network_cidr) + try: + return IPv4Network(network_cidr_str)[-2] + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_netmask(network_cidr): + """Return the IP netmask from given IP network""" + try: + network_cidr_str = unicode(network_cidr) + except NameError as ex: + network_cidr_str = str(network_cidr) + try: + return IPv4Network(network_cidr_str).netmask + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_contains_ip(network_cidr, ip_address): + """Check if an IP network cointains a given range""" + try: + network_cidr_str = unicode(network_cidr) + ip_address_str = unicode(ip_address) + except NameError as ex: + network_cidr_str = str(network_cidr) + ip_address_str = str(ip_address) + try: + return IPv4Address(ip_address_str) in IPv4Network(network_cidr_str) + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_contains_iprange(network_cidr, range_start, range_end): + """Check if an IP network cointains a given range""" + try: + network_cidr_str = unicode(network_cidr) + range_start_str = unicode(range_start) + range_end_str = unicode(range_end) + except NameError as ex: + network_cidr_str = str(network_cidr) + range_start_str = str(range_start) + range_end_str = str(range_end) + try: + ipnet = IPv4Network(network_cidr_str) + return (IPv4Address(range_start_str) in ipnet + and IPv4Address(range_end_str) in ipnet) + except ValueError as ex: + logging.error(network_cidr_str + " is not a valid network address") + raise + + +def filter_ipnet_range_size(network_cidr, range_start, range_end): + """Get the size of an IP range between two IP addresses""" + try: + network_cidr_str = unicode(network_cidr) + range_start_str = unicode(range_start) + range_end_str = unicode(range_end) + except NameError as ex: + network_cidr_str = str(network_cidr) + range_start_str = str(range_start) + range_end_str = str(range_end) + try: + ipnet = IPv4Network(network_cidr_str) + ip1 = IPv4Address(range_start_str) + ip2 = IPv4Address(range_end_str) + + if ip1 in ipnet and ip2 in ipnet: + index1 = list(ipnet.hosts()).index(ip1) + index2 = list(ipnet.hosts()).index(ip2) + ip_range_size = index2 - index1 + 1 + return ip_range_size + else: + raise ValueError + except ValueError as ex: + logging.error(range_start_str + " and " + range_end_str + + " are not valid IP addresses for range inside " + + network_cidr_str) + raise + + +# This filter is too simple and does not take network mask into account. +# TODO Deprecate for filter_ipnet_hostaddr +def filter_ipaddr_index(base_address, index): + """Return IP address in given network at given index""" + try: + base_address_str = unicode(base_address) + except NameError as ex: + base_address_str = str(base_address) + return IPv4Address(base_address_str) + int(index) + + +# TODO deprecate for filter_ipnet_netmask +def filter_netmask(prefix): + """Get netmask from prefix length integer""" + try: + prefix_str = unicode(prefix) + except NameError as ex: + prefix_str = str(prefix) + return IPv4Network("1.0.0.0/"+prefix_str).netmask diff --git a/config/utils/generate_config.py b/config/utils/generate_config.py index dfc6e6c4..93e839bd 100755 --- a/config/utils/generate_config.py +++ b/config/utils/generate_config.py @@ -1,84 +1,84 @@ #!/usr/bin/python ############################################################################## -# Copyright (c) 2017 OPNFV and others. +# Copyright (c) 2018 OPNFV and others. # # All rights reserved. This program and the accompanying materials # are made available under the terms of the Apache License, Version 2.0 # which accompanies this distribution, and is available at # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -"""This module does blah blah.""" +"""Generate configuration from PDF/IDF and jinja2 installer template""" + import argparse -import ipaddress import logging -import os +from os.path import abspath, exists, isfile, split +from subprocess import CalledProcessError, check_output +import gen_config_lib import yaml from jinja2 import Environment, FileSystemLoader -from subprocess import CalledProcessError, check_output + + +LOADER = yaml.CSafeLoader if yaml.__with_libyaml__ else yaml.SafeLoader PARSER = argparse.ArgumentParser() PARSER.add_argument("--yaml", "-y", type=str, required=True) -PARSER.add_argument("--jinja2", "-j", type=str, required=True) +PARSER.add_argument("--jinja2", "-j", type=str, required=True, action='append') +PARSER.add_argument("--includesdir", "-i", action='append', default=['/']) +PARSER.add_argument("--batch", "-b", action='store_true') +PARSER.add_argument("--verbose", "-v", action='count') ARGS = PARSER.parse_args() -# Processor architecture vs DPKG architecture mapping -DPKG_ARCH_TABLE = { - 'aarch64': 'arm64', - 'x86_64': 'amd64', -} -ARCH_DPKG_TABLE = dict(zip(DPKG_ARCH_TABLE.values(), DPKG_ARCH_TABLE.keys())) +ARGS.jinja2 = [abspath(x) for x in ARGS.jinja2] -# Custom filter to allow simple IP address operations returning -# a new address from an upper or lower (negative) index -def ipaddr_index(base_address, index): - """Return IP address in given network at given index""" - try: - base_address_str = unicode(base_address) - #pylint: disable=unused-variable - except NameError as ex: - base_address_str = str(base_address) - return ipaddress.ip_address(base_address_str) + int(index) +logging.basicConfig() +LOGGER = logging.getLogger('generate_config') +if ARGS.verbose: + LOGGER.setLevel(logging.INFO) -# Custom filter to convert between processor architecture -# (as reported by $(uname -m)) and DPKG-style architecture -def dpkg_arch(arch, to_dpkg=True): - """Return DPKG-compatible from processor arch and vice-versa""" - if to_dpkg: - return DPKG_ARCH_TABLE[arch] - else: - return ARCH_DPKG_TABLE[arch] - -ENV = Environment(loader=FileSystemLoader(os.path.dirname(ARGS.jinja2))) -ENV.filters['ipaddr_index'] = ipaddr_index -ENV.filters['dpkg_arch'] = dpkg_arch +ENV = Environment( + loader=FileSystemLoader(ARGS.includesdir), + extensions=['jinja2.ext.do'] +) +gen_config_lib.load_custom_filters(ENV) -# Run `eyaml decrypt` on the whole file, in case any PDF data is encrypted +# Run `eyaml decrypt` on the whole file, but only if PDF data is encrypted # Note: eyaml return code is 0 even if keys are not available try: - DICT = yaml.safe_load(check_output(['eyaml', 'decrypt', '-f', ARGS.yaml])) + if isfile(ARGS.yaml) and 'ENC[PKCS7' in open(ARGS.yaml).read(): + DICT = yaml.load(check_output(['eyaml', 'decrypt', + '-f', ARGS.yaml]), Loader=LOADER) except CalledProcessError as ex: - logging.error('eyaml decryption failed!') + LOGGER.error('eyaml decryption failed! Fallback to raw data.') except OSError as ex: - logging.warn('eyaml not found, skipping decryption') + LOGGER.warn('eyaml not found, skipping decryption. Fallback to raw data.') try: DICT['details'] except (NameError, TypeError) as ex: - logging.warn('PDF decryption skipped, fallback to using raw data.') with open(ARGS.yaml) as _: - DICT = yaml.safe_load(_) + DICT = yaml.load(_.read().replace('/', '__slash__'), Loader=LOADER) # If an installer descriptor file (IDF) exists, include it (temporary) -IDF_PATH = '/idf-'.join(os.path.split(ARGS.yaml)) -if os.path.exists(IDF_PATH): +IDF_PATH = '/idf-'.join(split(ARGS.yaml)) +if exists(IDF_PATH): with open(IDF_PATH) as _: - IDF = yaml.safe_load(_) + IDF = yaml.load(_, Loader=LOADER) DICT['idf'] = IDF['idf'] # Print dictionary generated from yaml (uncomment for debug) # print(DICT) -# Render template and print generated conf to console -TEMPLATE = ENV.get_template(os.path.basename(ARGS.jinja2)) - -#pylint: disable=superfluous-parens -print(TEMPLATE.render(conf=DICT)) +for _j2 in ARGS.jinja2: + TEMPLATE = ENV.get_template(_j2) + OUTPUT = TEMPLATE.render(conf=DICT).replace('__slash__', '/') + # Render template and write generated conf to file or stdout + if ARGS.batch: + if _j2.endswith('.j2'): + destination_file = _j2[:-3] # Trim '.j2' suffix + LOGGER.info('Parsing {}'.format(_j2)) + with open(destination_file, 'w') as _: + _.write(OUTPUT) + else: + LOGGER.warn('Skipping {}, name does not end in ".j2"'.format(_j2)) + else: + # pylint: disable=superfluous-parens + print(OUTPUT) diff --git a/config/utils/validate_schema.py b/config/utils/validate_schema.py new file mode 100755 index 00000000..1676e15d --- /dev/null +++ b/config/utils/validate_schema.py @@ -0,0 +1,46 @@ +#!/usr/bin/python +############################################################################## +# Copyright (c) 2018 Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +"""This module validates a PDF file against the schema.""" +import argparse +import jsonschema +import yaml + + +PARSER = argparse.ArgumentParser() +PARSER.add_argument("--yaml", "-y", type=str, required=True) +PARSER.add_argument("--schema", "-s", type=str, required=True) +ARGS = PARSER.parse_args() +LOADER = yaml.CSafeLoader if yaml.__with_libyaml__ else yaml.SafeLoader + +with open(ARGS.yaml) as _: + _DICT = yaml.load(_, Loader=LOADER) + +with open(ARGS.schema) as _: + _SCHEMA = yaml.load(_, Loader=LOADER) + + +def schema_version_workaround(node): + """Traverse nested dictionaries and handle 'version' key where found.""" + if 'version' in node: + node['version_{0}'.format(node['version'])] = True + for item in node.items(): + if type(item) is dict: + schema_version_workaround(item) + +# Draft 4 (latest supported by py-jsonschema) does not support value-based +# decisions properly, see related github issue: +# https://github.com/json-schema-org/json-schema-spec/issues/64 +# Workaround: build 'version_x.y: true' on the fly based on 'version: x.y' +schema_version_workaround(_DICT) +if 'idf' in _DICT: + schema_version_workaround(_DICT['idf']) + +_VALIDATOR = jsonschema.Draft4Validator(_SCHEMA) +for error in _VALIDATOR.iter_errors(_DICT): + raise RuntimeError(str(error)) |