diff options
Diffstat (limited to 'config/utils')
| -rwxr-xr-x | config/utils/check-jinja2.sh | 39 | ||||
| -rwxr-xr-x | config/utils/check-schema.sh | 37 | ||||
| -rwxr-xr-x | config/utils/generate_config.py | 11 | ||||
| -rwxr-xr-x | config/utils/validate_schema.py | 27 |
4 files changed, 92 insertions, 22 deletions
diff --git a/config/utils/check-jinja2.sh b/config/utils/check-jinja2.sh index 2953ff6a..3c5e5163 100755 --- a/config/utils/check-jinja2.sh +++ b/config/utils/check-jinja2.sh @@ -17,45 +17,50 @@ INSTALLER_ADAPTERS='./config/installers/*' TMPF='/tmp/out.yml' # should be outside Jenkins WS to prevent data leakage RC=0 +echo "Using $(yamllint --version)" + # Build a table header, using ';' as column sep -SUMMARY='PDF Verify Matrix;YAML Lint;' -for adapter in ${INSTALLER_ADAPTERS}; do +for adapter in 'PDF Verify Matrix' ${INSTALLER_ADAPTERS}; do SUMMARY+="$(basename "${adapter}");" done # Iterate all PDFs, check with each installer adapter, log results while IFS= read -r lab_config; do - valid_yaml='OK' - echo -e "\n\nyamllint -s ${lab_config}" - if ! yamllint -s "${lab_config}"; then valid_yaml='FAIL'; fi - SUMMARY+="\n${lab_config#labs/};${valid_yaml};" + SUMMARY+="\n${lab_config#labs/};" + echo "###################### ${lab_config} ######################" for adapter in ${INSTALLER_ADAPTERS}; do pdf_inst=0 pdf_inst_pass=0 pdf_yaml_pass=0 while IFS= read -r jinja_template; do - echo -e "\n${GEN_CFG} -y ${lab_config} -j ${jinja_template}" - if "${GEN_CFG}" -y "${lab_config}" \ - -j "${jinja_template}" > "${TMPF}"; then - echo 'Result: PASS' + pdf_gen_cmd="${GEN_CFG} -y ${lab_config} -j ${jinja_template}" + if ${pdf_gen_cmd} > "${TMPF}"; then ((pdf_inst_pass+=1)) - echo -e "\nyamllint -s ${jinja_template%.j2}" - if yamllint -s "${TMPF}"; then ((pdf_yaml_pass+=1)); fi + echo "[GENERATE] [OK] ${pdf_gen_cmd}" + if yamllint -s <(sed 's|ENC\[PKCS.*\]|opnfv|g' "${TMPF}"); then + ((pdf_yaml_pass+=1)); + echo "[YAMLLINT] [OK] yamllint -s ${jinja_template%.j2}" + else + echo "[YAMLLINT] [ERROR] yamllint -s ${jinja_template%.j2}" + fi else - echo 'Result: FAIL' + echo "[GENERATE] [ERROR] ${pdf_gen_cmd}" RC=1 fi ((pdf_inst+=1)) + echo '' done < <(find "${adapter}" -name '*.j2') SUMMARY+="${pdf_yaml_pass}/${pdf_inst_pass}/${pdf_inst};" done -done < <(find 'config' 'labs' -name 'pod*.yaml') - +done < <(find 'labs' -name 'pod*.yaml') rm -f "${TMPF}" -echo -e '\n\nNOTE: tuple fmt: (valid YAML output/sucessful parse/templates).\n' -echo -e "${SUMMARY}" | sed -e 's/^/;/g' -e 's/;/;| /g' | column -t -s ';' cat <<EOF +###################### Result Matrix ###################### + +NOTE: tuple fmt: (valid YAML output/sucessful parse/templates). + +$(echo -e "${SUMMARY}" | sed -e 's/^/;/g' -e 's/;/;| /g' | column -t -s ';') To troubleshoot PDF parsing against a specific installer adapter, execute the following commands locally (e.g. for zte-pod2/joid): diff --git a/config/utils/check-schema.sh b/config/utils/check-schema.sh new file mode 100755 index 00000000..321c5ced --- /dev/null +++ b/config/utils/check-schema.sh @@ -0,0 +1,37 @@ +#!/bin/bash -e +############################################################################## +# Copyright (c) 2018 Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +export PATH=$PATH:/usr/local/bin/ + +VALIDATE_SCHEMA='./config/utils/validate_schema.py' +PDF_SCHEMA='./config/pdf/pod1.schema.yaml' +RC=0 + +while IFS= read -r lab_config; do + pdf_cmd="${VALIDATE_SCHEMA} -s ${PDF_SCHEMA} -y ${lab_config}" + echo "###################### ${lab_config} ######################" + pdf_out=$(${pdf_cmd} |& sed 's|ENC\[PKCS.*\]|opnfv|g') + if [ -z "${pdf_out}" ]; then + SUMMARY+=";${lab_config#labs/};OK;\n" + echo "[PDF] [OK] ${pdf_cmd}" + else + SUMMARY+=";${lab_config#labs/};ERROR;\n" + RC=1 + echo "${pdf_out}" + echo "[PDF] [ERROR] ${pdf_cmd}" + fi + echo '' +done < <(find 'labs' -name 'pod*.yaml') + +cat <<EOF +###################### Schema Validation Matrix ###################### + +$(echo -e "${SUMMARY}" | sed -e 's/;/;| /g' | column -t -s ';') +EOF +exit "${RC}" diff --git a/config/utils/generate_config.py b/config/utils/generate_config.py index dfc6e6c4..cf558e28 100755 --- a/config/utils/generate_config.py +++ b/config/utils/generate_config.py @@ -52,18 +52,19 @@ ENV = Environment(loader=FileSystemLoader(os.path.dirname(ARGS.jinja2))) ENV.filters['ipaddr_index'] = ipaddr_index ENV.filters['dpkg_arch'] = dpkg_arch -# Run `eyaml decrypt` on the whole file, in case any PDF data is encrypted +# Run `eyaml decrypt` on the whole file, but only if PDF data is encrypted # Note: eyaml return code is 0 even if keys are not available try: - DICT = yaml.safe_load(check_output(['eyaml', 'decrypt', '-f', ARGS.yaml])) + if 'ENC[PKCS7' in open(ARGS.yaml).read(): + DICT = yaml.safe_load(check_output(['eyaml', 'decrypt', + '-f', ARGS.yaml])) except CalledProcessError as ex: - logging.error('eyaml decryption failed!') + logging.error('eyaml decryption failed! Fallback to raw data.') except OSError as ex: - logging.warn('eyaml not found, skipping decryption') + logging.warn('eyaml not found, skipping decryption. Fallback to raw data.') try: DICT['details'] except (NameError, TypeError) as ex: - logging.warn('PDF decryption skipped, fallback to using raw data.') with open(ARGS.yaml) as _: DICT = yaml.safe_load(_) diff --git a/config/utils/validate_schema.py b/config/utils/validate_schema.py new file mode 100755 index 00000000..cb404554 --- /dev/null +++ b/config/utils/validate_schema.py @@ -0,0 +1,27 @@ +#!/usr/bin/python +############################################################################## +# Copyright (c) 2018 Enea AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +"""This module validates a PDF file against the schema.""" +import argparse +import jsonschema +import yaml + +PARSER = argparse.ArgumentParser() +PARSER.add_argument("--yaml", "-y", type=str, required=True) +PARSER.add_argument("--schema", "-s", type=str, required=True) +ARGS = PARSER.parse_args() + +with open(ARGS.yaml) as _: + _DICT = yaml.safe_load(_) + +with open(ARGS.schema) as _: + _SCHEMA = yaml.safe_load(_) + +_VALIDATOR = jsonschema.Draft4Validator(_SCHEMA) +for error in _VALIDATOR.iter_errors(_DICT): + raise RuntimeError(str(error)) |