From e70530f9be5083024f853b7082e5263fc813a97f Mon Sep 17 00:00:00 2001 From: Ryota MIBU Date: Wed, 17 Jan 2018 02:24:08 +0900 Subject: add 'lint-all-code' as wrapper of lint builders OPNFV has common coding style [1]. So, all repos shall be checked by one generic checker builder/job, eventually. This patch adds lint wrapper builder 'lint-all-code', and also refactors all exisiting lint and syntax check builders, following 'lint-yaml-code' builder approach, which only checks modified files. This approach is nice as it won't require immediate fixes of violations in whole repo code right after the lint checker job is enabled. Note that lint jobs are still configured to run against some project repo, in order to seperate discussion; wheter we should apply this lint job to all OPNFV repo or not immediately. [1] https://wiki.opnfv.org/display/DEV/Contribution+Guidelines Change-Id: Ib4ab3ba5dc08845f3016b8be772f4ed119f11b2d Signed-off-by: Ryota MIBU --- jjb/compass4nfv/compass-verify-jobs.yml | 10 -- jjb/global/releng-macros.yml | 158 ++++++++++++-------------------- jjb/opnfvdocs/opnfvdocs.yml | 4 +- jjb/releng/opnfv-lint.yml | 55 ++++++++++- 4 files changed, 114 insertions(+), 113 deletions(-) (limited to 'jjb') diff --git a/jjb/compass4nfv/compass-verify-jobs.yml b/jjb/compass4nfv/compass-verify-jobs.yml index 7024dad58..75d1653f9 100644 --- a/jjb/compass4nfv/compass-verify-jobs.yml +++ b/jjb/compass4nfv/compass-verify-jobs.yml @@ -136,11 +136,6 @@ node-parameters: true kill-phase-on: FAILURE abort-all-job: true - - name: 'opnfv-yamllint-verify-{stream}' - current-parameters: true - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - multijob: name: deploy-virtual condition: SUCCESSFUL @@ -261,11 +256,6 @@ node-parameters: true kill-phase-on: FAILURE abort-all-job: true - - name: 'opnfv-yamllint-verify-{stream}' - current-parameters: true - node-parameters: true - kill-phase-on: FAILURE - abort-all-job: true - multijob: name: deploy-virtual condition: SUCCESSFUL diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml index c6a65ca21..68ec1b3d8 100644 --- a/jjb/global/releng-macros.yml +++ b/jjb/global/releng-macros.yml @@ -298,127 +298,91 @@ fi - builder: - name: lint-python-code + name: upload-review-docs builders: - - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - - virtualenv -p python2.7 $WORKSPACE/releng_flake8 - source $WORKSPACE/releng_flake8/bin/activate - - # install python packages - pip install "flake8==2.6.2" - - # generate and upload lint log - echo "Running flake8 code on $PROJECT ..." - - # Get number of flake8 violations. If none, this will be an - # empty string: "" - FLAKE_COUNT="$(find . \ - -path './releng_flake8' -prune -o \ - -path './.tox' -prune -o \ - -type f -name "*.py" -print | \ - xargs flake8 --exit-zero -qq --count 2>&1)" + - upload-under-review-docs-to-opnfv-artifacts + - report-build-result-to-gerrit +- builder: + name: lint-init + builders: + - shell: | # Ensure we start with a clean environment - rm -f lint.log - - if [ ! -z $FLAKE_COUNT ]; then - echo "Flake8 Violations: $FLAKE_COUNT" > lint.log - find . \ - -path './releng_flake8' -prune -o \ - -path './.tox' -prune -o \ - -type f -name "*.py" -print | \ - xargs flake8 --exit-zero --first >> violation.log - SHOWN=$(wc -l violation.log | cut -d' ' -f1) - echo -e "First $SHOWN shown\n---" >> lint.log - cat violation.log >> lint.log - sed -r -i '4,$s/^/ /g' lint.log - rm violation.log - fi - - deactivate + rm -f bash-violation.log python-violation.log yaml-violation.log violation.log + git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1) > modified_files - builder: - name: report-lint-result-to-gerrit + name: lint-report builders: - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ - - # If no violations were found, no lint log will exist. - if [[ -e lint.log ]] ; then - echo -e "\nposting linting report to gerrit...\n" - - cat lint.log - echo - - ssh -p 29418 gerrit.opnfv.org \ - "gerrit review -p $GERRIT_PROJECT \ - -m \"$(cat lint.log)\" \ - $GERRIT_PATCHSET_REVISION \ - --notify NONE" - + if [[ -s violation.log ]]; then + echo "Reporting lint result..." + msg="Found syntax error and/or coding style violation(s) in the files modified by your patchset." + sed -i -e '1s/^//$msg\n\n/' violation.log + cmd="gerrit review -p $GERRIT_PROJECT -m \"$(cat violation.log)\" $GERRIT_PATCHSET_REVISION --notify NONE" + ssh -p 29418 gerrit.opnfv.org "$cmd" + + # Make sure the caller job failed exit 1 fi - builder: - name: upload-review-docs + name: lint-bash-code builders: - - upload-under-review-docs-to-opnfv-artifacts - - report-build-result-to-gerrit + - shell: | + echo "Checking bash code..." + for f in $(egrep '\.sh$' modified_files) + do + bash -n "$f" 2>> bash-violation.log + done + if [[ -s bash-violation.log ]]; then + echo -e "Bash syntax error(s)\n---" >> violation.log + sed -e 's/^/ /g' bash-violation.log >> violation.log + fi - builder: - name: check-bash-syntax + name: lint-python-code builders: - - shell: "find . -name '*.sh' | xargs bash -n" + - shell: | + # Install python packages + pip install "flake8==2.6.2" + + echo "Checking python code..." + for f in $(egrep '\.py$' modified_files) + do + flake8 "$f" >> python-violation.log + done + if [[ -s python-violation.log ]]; then + echo -e "Python violation(s)\n---" >> violation.log + sed -e 's/^/ /g' python-violation.log >> violation.log + fi - builder: name: lint-yaml-code builders: - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - set -o xtrace - export PATH=$PATH:/usr/local/bin/ + # Install python packages + pip install "yamllint==1.8.2" - # install python packages - sudo pip install "yamllint==1.8.2" - - # generate and upload lint log - echo "Running yaml code on $PROJECT ..." - - # Get list of yaml files - YAML_FILES=$(git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1 | egrep "ya?ml$") || true - - #If YAML_FILES is none exit with 0 - if [ -z "$YAML_FILES" ]; then - exit 0 - fi - - # Ensure we start with a clean environment - rm -f yaml-violation.log lint.log - - # Yamllint files only in patchset - for yamlfile in $YAML_FILES; do - yamllint $yamlfile >> yaml-violation.log || true + echo "Checking yaml file..." + for f in $(egrep '\.ya?ml$' modified_files) + do + yamllint "$f" >> yaml-violation.log done - - if [ -s "yaml-violation.log" ]; then - SHOWN=$(grep -c -v "^$" yaml-violation.log) - echo -e "First $SHOWN shown\n---" > lint.log - cat yaml-violation.log >> lint.log - sed -r -i '4,$s/^/ /g' lint.log + if [[ -s yaml-violation.log ]]; then + echo -e "YAML violation(s)\n---" >> violation.log + sed -e 's/^/ /g' yaml-violation.log >> violation.log fi +- builder: + name: lint-all-code + builders: + - lint-init + - lint-bash-code + - lint-python-code + - lint-yaml-code + - lint-report + - builder: name: clean-workspace builders: diff --git a/jjb/opnfvdocs/opnfvdocs.yml b/jjb/opnfvdocs/opnfvdocs.yml index 3dfb9d155..908ddba9d 100644 --- a/jjb/opnfvdocs/opnfvdocs.yml +++ b/jjb/opnfvdocs/opnfvdocs.yml @@ -69,7 +69,7 @@ notbuilt: true builders: - - check-bash-syntax + - lint-bash-code - job-template: name: 'opnfvdocs-merge-shellcheck-{stream}' @@ -107,4 +107,4 @@ branch-pattern: '**/{branch}' builders: - - check-bash-syntax + - lint-bash-code diff --git a/jjb/releng/opnfv-lint.yml b/jjb/releng/opnfv-lint.yml index d566430d6..c0da8ae51 100644 --- a/jjb/releng/opnfv-lint.yml +++ b/jjb/releng/opnfv-lint.yml @@ -54,7 +54,52 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator|releng' + project-pattern: 'releng|doctor' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: '**/*.py' + - compare-type: ANT + pattern: '**/*.sh' + - compare-type: ANT + pattern: '**/*.yml' + - compare-type: ANT + pattern: '**/*.yaml' + + builders: + - lint-all-code + +- job-template: + name: 'opnfv-pylint-verify-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: $GERRIT_PROJECT + branch: '{branch}' + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'REG_EXP' + project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -63,8 +108,9 @@ pattern: '**/*.py' builders: + - lint-init - lint-python-code - - report-lint-result-to-gerrit + - lint-report - job-template: name: 'opnfv-yamllint-verify-{stream}' @@ -103,7 +149,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'armband|fuel|octopus|pharos|releng|releng-anteater' + project-pattern: 'armband|fuel|octopus|pharos|releng-anteater' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -114,5 +160,6 @@ pattern: '**/*.yaml' builders: + - lint-init - lint-yaml-code - - report-lint-result-to-gerrit + - lint-report -- cgit 1.2.3-korg