diff options
Diffstat (limited to 'jjb')
| -rw-r--r-- | jjb/compass4nfv/compass-ci-jobs.yml | 59 | ||||
| -rw-r--r-- | jjb/compass4nfv/compass-logs.sh | 28 | ||||
| -rw-r--r-- | jjb/compass4nfv/compass-verify-jobs.yml | 10 | ||||
| -rwxr-xr-x | jjb/fuel/fuel-deploy.sh | 8 | ||||
| -rwxr-xr-x | jjb/functest/functest-alpine.sh | 5 | ||||
| -rw-r--r-- | jjb/global/installer-params.yml | 8 | ||||
| -rw-r--r-- | jjb/global/releng-macros.yml | 156 | ||||
| -rw-r--r-- | jjb/opnfvdocs/opnfvdocs.yml | 4 | ||||
| -rw-r--r-- | jjb/pharos/check-jinja2.yml | 8 | ||||
| -rw-r--r-- | jjb/releng/automate.yml | 6 | ||||
| -rw-r--r-- | jjb/releng/opnfv-lint.yml | 73 | ||||
| -rw-r--r-- | jjb/xci/xci-verify-jobs.yml | 8 |
12 files changed, 240 insertions, 133 deletions
diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index 0790b83ca..70296bda1 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -127,6 +127,7 @@ jobs: - 'compass-{scenario}-{pod}-daily-{stream}' - 'compass-deploy-{pod}-daily-{stream}' + - 'compass-collect-logs-{pod}-daily-{stream}' ######################## # job templates @@ -281,6 +282,25 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' + - conditional-step: + condition-kind: and + condition-operands: + - condition-kind: regex-match + regex: master + label: '{stream}' + steps: + - trigger-builds: + - project: 'compass-collect-logs-{pod}-daily-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' + - job-template: name: 'compass-deploy-{pod}-daily-{stream}' @@ -346,6 +366,41 @@ - shell: !include-raw-escape: ./compass-deploy.sh +- job-template: + name: 'compass-collect-logs-{pod}-daily-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - logrotate-default + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - compass-ci-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + ppa-pathname: '{ppa-pathname}' + - '{slave-label}-defaults' + - '{installer}-defaults' + + scm: + - git-scm + + wrappers: + - build-name: + name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + - fix-workspace-permissions + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - shell: + !include-raw-escape: ./compass-logs.sh + ######################## # parameter macros ######################## @@ -370,6 +425,10 @@ - string: name: PPA_CACHE default: "$WORKSPACE/work/repo/" + - string: + name: LOG_DIRECTORY + default: $WORKSPACE/log_output + description: "Directory where the logs will be located upon the completion of the collection." ######################## # trigger macros diff --git a/jjb/compass4nfv/compass-logs.sh b/jjb/compass4nfv/compass-logs.sh new file mode 100644 index 000000000..03c8b20ac --- /dev/null +++ b/jjb/compass4nfv/compass-logs.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -o nounset +set -o pipefail + +# log info to console +echo "Uploading the logs $INSTALLER_TYPE artifact. This could take some time..." +echo "--------------------------------------------------------" +echo + +# create the log directory if it doesn't exist +[[ -d $LOG_DIRECTORY ]] || mkdir -p $LOG_DIRECTORY + +OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") +COMPASS_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}_${OPNFV_ARTIFACT_VERSION}.log.tar.gz" + + +sudo docker exec -it compass-tasks /bin/bash /opt/collect-log.sh +sudo docker cp compass-tasks:/opt/log.tar.gz ${LOG_DIRECTORY}/${COMPASS_LOG_FILENAME} + +sudo chown $(whoami):$(whoami) ${LOG_DIRECTORY}/${COMPASS_LOG_FILENAME} + +gsutil cp "${LOG_DIRECTORY}/${COMPASS_LOG_FILENAME}" \ + "gs://${GS_URL}/logs/${COMPASS_LOG_FILENAME}" > /dev/null 2>&1 + +echo +echo "--------------------------------------------------------" +echo "Done!" +echo "Artifact is available as http://${GS_URL}/logs/${COMPASS_LOG_FILENAME}" diff --git a/jjb/compass4nfv/compass-verify-jobs.yml b/jjb/compass4nfv/compass-verify-jobs.yml index 7024dad58..75d1653f9 100644 --- a/jjb/compass4nfv/compass-verify-jobs.yml +++ b/jjb/compass4nfv/compass-verify-jobs.yml @@ -136,11 +136,6 @@ node-parameters: true kill-phase-on: FAILURE abort-all-job: true - - name: 'opnfv-yamllint-verify-{stream}' - current-parameters: true - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - multijob: name: deploy-virtual condition: SUCCESSFUL @@ -261,11 +256,6 @@ node-parameters: true kill-phase-on: FAILURE abort-all-job: true - - name: 'opnfv-yamllint-verify-{stream}' - current-parameters: true - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - multijob: name: deploy-virtual condition: SUCCESSFUL diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh index cb9febe2a..35bf3dc01 100755 --- a/jjb/fuel/fuel-deploy.sh +++ b/jjb/fuel/fuel-deploy.sh @@ -38,7 +38,6 @@ fi # set deployment parameters export TMPDIR=${HOME}/tmpdir -BRIDGE=${BRIDGE:-pxebr} # shellcheck disable=SC2153 LAB_NAME=${NODE_NAME/-*} # shellcheck disable=SC2153 @@ -49,10 +48,6 @@ LAB_CONFIG_URL=${LAB_CONFIG_URL:-'ssh://jenkins-ericsson@gerrit.opnfv.org:29418/ # Fuel requires deploy script to be ran with sudo, Armband does not SUDO='sudo -E' if [ "${PROJECT}" = 'fuel' ]; then - # Fuel does not use any POD-specific configuration for virtual deploys - if [[ "${NODE_NAME}" =~ "virtual" ]]; then - POD_NAME="virtual_kvm" - fi # Fuel currently supports ericsson, intel, lf and zte labs if [[ ! "${LAB_NAME}" =~ (ericsson|intel|lf|zte) ]]; then echo "Unsupported/unidentified lab ${LAB_NAME}. Cannot continue!" @@ -81,6 +76,7 @@ if [[ "$BRANCH" =~ (danube|euphrates) ]]; then rm -rf "${LOCAL_CFG}" git clone --quiet --branch "${BRANCH}" "${LAB_CONFIG_URL}" "${LOCAL_CFG}" LAB_CONFIG_ARG="-b file://${LOCAL_CFG}" + BRIDGE_ARG="-B ${BRIDGE:-pxebr}" else LAB_CONFIG_ARG="-b ${LAB_CONFIG_URL}" fi @@ -92,7 +88,7 @@ FUEL_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}.log.tar.gz" # construct the command DEPLOY_COMMAND="${SUDO} ${WORKSPACE}/ci/deploy.sh ${LAB_CONFIG_ARG:-} \ -l ${LAB_NAME} -p ${POD_NAME} -s ${DEPLOY_SCENARIO} ${ISO_FILE_ARG:-} \ - -B ${DEFAULT_BRIDGE:-${BRIDGE}} -S ${TMPDIR} \ + -S ${TMPDIR} ${BRIDGE_ARG:-} \ -L ${WORKSPACE}/${FUEL_LOG_FILENAME}" # log info to console diff --git a/jjb/functest/functest-alpine.sh b/jjb/functest/functest-alpine.sh index 735ca6f7f..0f4e80f24 100755 --- a/jjb/functest/functest-alpine.sh +++ b/jjb/functest/functest-alpine.sh @@ -6,7 +6,10 @@ set +o pipefail check_os_deployment() { FUNCTEST_IMAGE=opnfv/functest-healthcheck:${DOCKER_TAG} + echo "Functest: Pulling Functest Docker image ${FUNCTEST_IMAGE} ..." + docker pull ${FUNCTEST_IMAGE}>/dev/null cmd="docker run --rm --privileged=true ${volumes} ${FUNCTEST_IMAGE} check_deployment" + echo "Checking deployment, CMD: ${cmd}" eval ${cmd} ret_value=$? if [ ${ret_value} != 0 ]; then @@ -107,7 +110,7 @@ elif [[ ${INSTALLER_TYPE} == 'compass' ]]; then elif [[ ${INSTALLER_TYPE} == 'fuel' && ${DEPLOY_TYPE} == 'baremetal' ]]; then cacert_file_vol="-v ${HOME}/os_cacert:/etc/ssl/certs/mcp_os_cacert" fi -rc_file_vol="-v ${rc_file}:${FUNCTEST_DIR}/conf/openstack.creds" +rc_file_vol="-v ${rc_file}:${FUNCTEST_DIR}/conf/env_file" # Set iptables rule to allow forwarding return traffic for container diff --git a/jjb/global/installer-params.yml b/jjb/global/installer-params.yml index 916db808d..683ef78ba 100644 --- a/jjb/global/installer-params.yml +++ b/jjb/global/installer-params.yml @@ -51,10 +51,6 @@ default: 'floating_net' description: 'external network for test' - string: - name: BRIDGE - default: 'pxebr' - description: 'Bridge(s) to be used by salt master' - - string: name: GS_URL default: '$GS_BASE{gs-pathname}' description: "URL to Google Storage." @@ -80,8 +76,8 @@ description: 'Model to deploy (os|k8)' - string: name: OS_RELEASE - default: 'ocata' - description: 'OpenStack release (mitaka|newton|ocata)' + default: 'pike' + description: 'OpenStack release (mitaka|ocata|pike)' - string: name: EXTERNAL_NETWORK default: ext-net diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml index c6a65ca21..a5a83e23f 100644 --- a/jjb/global/releng-macros.yml +++ b/jjb/global/releng-macros.yml @@ -298,128 +298,92 @@ fi - builder: - name: lint-python-code + name: upload-review-docs builders: - - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - - virtualenv -p python2.7 $WORKSPACE/releng_flake8 - source $WORKSPACE/releng_flake8/bin/activate - - # install python packages - pip install "flake8==2.6.2" - - # generate and upload lint log - echo "Running flake8 code on $PROJECT ..." - - # Get number of flake8 violations. If none, this will be an - # empty string: "" - FLAKE_COUNT="$(find . \ - -path './releng_flake8' -prune -o \ - -path './.tox' -prune -o \ - -type f -name "*.py" -print | \ - xargs flake8 --exit-zero -qq --count 2>&1)" + - upload-under-review-docs-to-opnfv-artifacts + - report-build-result-to-gerrit +- builder: + name: lint-init + builders: + - shell: | # Ensure we start with a clean environment - rm -f lint.log - - if [ ! -z $FLAKE_COUNT ]; then - echo "Flake8 Violations: $FLAKE_COUNT" > lint.log - find . \ - -path './releng_flake8' -prune -o \ - -path './.tox' -prune -o \ - -type f -name "*.py" -print | \ - xargs flake8 --exit-zero --first >> violation.log - SHOWN=$(wc -l violation.log | cut -d' ' -f1) - echo -e "First $SHOWN shown\n---" >> lint.log - cat violation.log >> lint.log - sed -r -i '4,$s/^/ /g' lint.log - rm violation.log - fi - - deactivate + rm -f bash-violation.log python-violation.log yaml-violation.log violation.log + git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1 > modified_files - builder: - name: report-lint-result-to-gerrit + name: lint-report builders: - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - - # If no violations were found, no lint log will exist. - if [[ -e lint.log ]] ; then - echo -e "\nposting linting report to gerrit...\n" - - cat lint.log - echo - - ssh -p 29418 gerrit.opnfv.org \ - "gerrit review -p $GERRIT_PROJECT \ - -m \"$(cat lint.log)\" \ - $GERRIT_PATCHSET_REVISION \ - --notify NONE" - + if [[ -s violation.log ]]; then + echo "Reporting lint result..." + msg="Found syntax error and/or coding style violation(s) in the files modified by your patchset." + sed -i -e '1s/^//$msg\n\n/' violation.log + cmd="gerrit review -p $GERRIT_PROJECT -m \"$(cat violation.log)\" $GERRIT_PATCHSET_REVISION --notify NONE" + ssh -p 29418 gerrit.opnfv.org "$cmd" + + # Make sure the caller job failed exit 1 fi - builder: - name: upload-review-docs + name: lint-bash-code builders: - - upload-under-review-docs-to-opnfv-artifacts - - report-build-result-to-gerrit + - shell: | + echo "Checking bash code..." + for f in $(egrep '\.sh$' modified_files) + do + bash -n "$f" 2>> bash-violation.log + done + if [[ -s bash-violation.log ]]; then + echo -e "Bash syntax error(s)\n---" >> violation.log + sed -e 's/^/ /g' bash-violation.log >> violation.log + fi - builder: - name: check-bash-syntax + name: lint-python-code builders: - - shell: "find . -name '*.sh' | xargs bash -n" + - shell: | + # Install python package + sudo pip install "flake8==2.6.2" + + echo "Checking python code..." + for f in $(egrep '\.py$' modified_files) + do + flake8 "$f" >> python-violation.log + done + if [[ -s python-violation.log ]]; then + echo -e "Python violation(s)\n---" >> violation.log + sed -e 's/^/ /g' python-violation.log >> violation.log + fi - builder: name: lint-yaml-code builders: - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - - # install python packages + # sudo Install python packages sudo pip install "yamllint==1.8.2" - # generate and upload lint log - echo "Running yaml code on $PROJECT ..." - - # Get list of yaml files - YAML_FILES=$(git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1 | egrep "ya?ml$") || true - - #If YAML_FILES is none exit with 0 - if [ -z "$YAML_FILES" ]; then - exit 0 - fi - - # Ensure we start with a clean environment - rm -f yaml-violation.log lint.log - - # Yamllint files only in patchset - for yamlfile in $YAML_FILES; do - yamllint $yamlfile >> yaml-violation.log || true + echo "Checking yaml file..." + for f in $(egrep '\.ya?ml$' modified_files) + do + yamllint "$f" >> yaml-violation.log done - - if [ -s "yaml-violation.log" ]; then - SHOWN=$(grep -c -v "^$" yaml-violation.log) - echo -e "First $SHOWN shown\n---" > lint.log - cat yaml-violation.log >> lint.log - sed -r -i '4,$s/^/ /g' lint.log + if [[ -s yaml-violation.log ]]; then + echo -e "YAML violation(s)\n---" >> violation.log + sed -e 's/^/ /g' yaml-violation.log >> violation.log fi - builder: + name: lint-all-code + builders: + - lint-init + - lint-bash-code + - lint-python-code + - lint-yaml-code + - lint-report + +- builder: name: clean-workspace builders: - shell: | diff --git a/jjb/opnfvdocs/opnfvdocs.yml b/jjb/opnfvdocs/opnfvdocs.yml index 3dfb9d155..908ddba9d 100644 --- a/jjb/opnfvdocs/opnfvdocs.yml +++ b/jjb/opnfvdocs/opnfvdocs.yml @@ -69,7 +69,7 @@ notbuilt: true builders: - - check-bash-syntax + - lint-bash-code - job-template: name: 'opnfvdocs-merge-shellcheck-{stream}' @@ -107,4 +107,4 @@ branch-pattern: '**/{branch}' builders: - - check-bash-syntax + - lint-bash-code diff --git a/jjb/pharos/check-jinja2.yml b/jjb/pharos/check-jinja2.yml index 49a2ba539..7251d3f61 100644 --- a/jjb/pharos/check-jinja2.yml +++ b/jjb/pharos/check-jinja2.yml @@ -74,10 +74,10 @@ - compare-type: ANT pattern: 'check-schema.sh' skip-vote: - successful: true - failed: true - unstable: true - notbuilt: true + successful: false + failed: false + unstable: false + notbuilt: false builders: - check-pdf-jinja diff --git a/jjb/releng/automate.yml b/jjb/releng/automate.yml index afeb35821..6bd952a1d 100644 --- a/jjb/releng/automate.yml +++ b/jjb/releng/automate.yml @@ -46,6 +46,12 @@ builders: - mongodb-backup + publishers: + - email: + recipients: serena.feng.711@gmail.com + notify-every-unstable-build: true + send-to-individuals: true + - job-template: name: '{module}-verify-{stream}' diff --git a/jjb/releng/opnfv-lint.yml b/jjb/releng/opnfv-lint.yml index d566430d6..668b49021 100644 --- a/jjb/releng/opnfv-lint.yml +++ b/jjb/releng/opnfv-lint.yml @@ -31,10 +31,73 @@ disabled: '{obj:disabled}' + concurrent: true + parameters: - project-parameter: project: $GERRIT_PROJECT branch: '{branch}' + - node: + name: SLAVE_NAME + description: Slaves to execute yamllint + default-slaves: + - lf-build1 + allowed-multiselect: true + ignore-offline-nodes: true + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'REG_EXP' + project-pattern: 'releng|doctor' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: '**/*.py' + - compare-type: ANT + pattern: '**/*.sh' + - compare-type: ANT + pattern: '**/*.yml' + - compare-type: ANT + pattern: '**/*.yaml' + + builders: + - lint-all-code + +- job-template: + name: 'opnfv-pylint-verify-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + parameters: + - project-parameter: + project: $GERRIT_PROJECT + branch: '{branch}' + - node: + name: SLAVE_NAME + description: Slaves to execute yamllint + default-slaves: + - lf-build1 + allowed-multiselect: true + ignore-offline-nodes: true scm: - git-scm-gerrit @@ -54,7 +117,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator|releng' + project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -63,8 +126,9 @@ pattern: '**/*.py' builders: + - lint-init - lint-python-code - - report-lint-result-to-gerrit + - lint-report - job-template: name: 'opnfv-yamllint-verify-{stream}' @@ -103,7 +167,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'armband|fuel|octopus|pharos|releng|releng-anteater' + project-pattern: 'armband|fuel|octopus|pharos|releng-anteater' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -114,5 +178,6 @@ pattern: '**/*.yaml' builders: + - lint-init - lint-yaml-code - - report-lint-result-to-gerrit + - lint-report diff --git a/jjb/xci/xci-verify-jobs.yml b/jjb/xci/xci-verify-jobs.yml index 3c314568f..28bc317bd 100644 --- a/jjb/xci/xci-verify-jobs.yml +++ b/jjb/xci/xci-verify-jobs.yml @@ -21,10 +21,10 @@ notbuilt: false - centos: disabled: false - successful: true - failed: true - unstable: true - notbuilt: true + successful: false + failed: false + unstable: false + notbuilt: false - opensuse: disabled: false successful: false |