add 'lint-all-code' as wrapper of lint builders

OPNFV has common coding style [1]. So, all repos shall be checked by
one generic checker builder/job, eventually.

This patch adds lint wrapper builder 'lint-all-code', and also
refactors all exisiting lint and syntax check builders, following
'lint-yaml-code' builder approach, which only checks modified files.
This approach is nice as it won't require immediate fixes of violations
in whole repo code right after the lint checker job is enabled.

Note that lint jobs are still configured to run against some project
repo, in order to seperate discussion; wheter we should apply this
lint job to all OPNFV repo or not immediately.

[1] https://wiki.opnfv.org/display/DEV/Contribution+Guidelines

Change-Id: Ib4ab3ba5dc08845f3016b8be772f4ed119f11b2d
Signed-off-by: Ryota MIBU <r-mibu@cq.jp.nec.com>

4 files changed, 114 insertions, 113 deletions

diff --git a/jjb/compass4nfv/compass-verify-jobs.yml b/jjb/compass4nfv/compass-verify-jobs.yml
index 7024dad58..75d1653f9 100644
--- a/jjb/compass4nfv/compass-verify-jobs.yml
+++ b/jjb/compass4nfv/compass-verify-jobs.yml
@@ -136,11 +136,6 @@
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: true
-            - name: 'opnfv-yamllint-verify-{stream}'
-              current-parameters: true
-              node-parameters: true
-              kill-phase-on: FAILURE
-              abort-all-job: true
       - multijob:
           name: deploy-virtual
           condition: SUCCESSFUL
@@ -261,11 +256,6 @@
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: true
-            - name: 'opnfv-yamllint-verify-{stream}'
-              current-parameters: true
-              node-parameters: true
-              kill-phase-on: FAILURE
-              abort-all-job: true
       - multijob:
           name: deploy-virtual
           condition: SUCCESSFUL
diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml
index c6a65ca21..68ec1b3d8 100644
--- a/jjb/global/releng-macros.yml
+++ b/jjb/global/releng-macros.yml
@@ -298,128 +298,92 @@
           fi
 
 - builder:
-    name: lint-python-code
+    name: upload-review-docs
     builders:
-      - shell: |
-          #!/bin/bash
-          set -o errexit
-          set -o pipefail
-          set -o xtrace
-          export PATH=$PATH:/usr/local/bin/
-
-          virtualenv -p python2.7 $WORKSPACE/releng_flake8
-          source $WORKSPACE/releng_flake8/bin/activate
-
-          # install python packages
-          pip install "flake8==2.6.2"
-
-          # generate and upload lint log
-          echo "Running flake8 code on $PROJECT ..."
-
-          # Get number of flake8 violations. If none, this will be an
-          # empty string: ""
-          FLAKE_COUNT="$(find . \
-              -path './releng_flake8' -prune -o \
-              -path './.tox' -prune -o \
-              -type f -name "*.py" -print | \
-              xargs flake8 --exit-zero -qq --count 2>&1)"
+      - upload-under-review-docs-to-opnfv-artifacts
+      - report-build-result-to-gerrit
 
+- builder:
+    name: lint-init
+    builders:
+      - shell: |
           # Ensure we start with a clean environment
-          rm -f lint.log
-
-          if [ ! -z $FLAKE_COUNT ]; then
-            echo "Flake8 Violations: $FLAKE_COUNT" > lint.log
-            find . \
-                -path './releng_flake8' -prune -o \
-                -path './.tox' -prune -o \
-                -type f -name "*.py" -print | \
-                xargs flake8 --exit-zero --first >> violation.log
-            SHOWN=$(wc -l violation.log | cut -d' ' -f1)
-            echo -e "First $SHOWN shown\n---" >> lint.log
-            cat violation.log >> lint.log
-            sed -r -i '4,$s/^/ /g' lint.log
-            rm violation.log
-          fi
-
-          deactivate
+          rm -f bash-violation.log python-violation.log yaml-violation.log violation.log
+          git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1) > modified_files
 
 - builder:
-    name: report-lint-result-to-gerrit
+    name: lint-report
     builders:
       - shell: |
-          #!/bin/bash
-          set -o errexit
-          set -o pipefail
-          set -o xtrace
-          export PATH=$PATH:/usr/local/bin/
-
-          # If no violations were found, no lint log will exist.
-          if [[ -e lint.log ]] ; then
-              echo -e "\nposting linting report to gerrit...\n"
-
-              cat lint.log
-              echo
-
-              ssh -p 29418 gerrit.opnfv.org \
-                  "gerrit review -p $GERRIT_PROJECT \
-                   -m \"$(cat lint.log)\" \
-                   $GERRIT_PATCHSET_REVISION \
-                   --notify NONE"
-
+          if [[ -s violation.log ]]; then
+              echo "Reporting lint result..."
+              msg="Found syntax error and/or coding style violation(s) in the files modified by your patchset."
+              sed -i -e '1s/^//$msg\n\n/' violation.log
+              cmd="gerrit review -p $GERRIT_PROJECT -m \"$(cat violation.log)\" $GERRIT_PATCHSET_REVISION --notify NONE"
+              ssh -p 29418 gerrit.opnfv.org "$cmd"
+
+              # Make sure the caller job failed
               exit 1
           fi
 
 - builder:
-    name: upload-review-docs
+    name: lint-bash-code
     builders:
-      - upload-under-review-docs-to-opnfv-artifacts
-      - report-build-result-to-gerrit
+      - shell: |
+          echo "Checking bash code..."
+          for f in $(egrep '\.sh$' modified_files)
+          do
+              bash -n "$f" 2>> bash-violation.log
+          done
+          if [[ -s bash-violation.log ]]; then
+              echo -e "Bash syntax error(s)\n---" >> violation.log
+              sed -e 's/^/ /g' bash-violation.log >> violation.log
+          fi
 
 - builder:
-    name: check-bash-syntax
+    name: lint-python-code
     builders:
-      - shell: "find . -name '*.sh' | xargs bash -n"
+      - shell: |
+          # Install python packages
+          pip install "flake8==2.6.2"
+
+          echo "Checking python code..."
+          for f in $(egrep '\.py$' modified_files)
+          do
+              flake8 "$f" >> python-violation.log
+          done
+          if [[ -s python-violation.log ]]; then
+              echo -e "Python violation(s)\n---" >> violation.log
+              sed -e 's/^/ /g' python-violation.log >> violation.log
+          fi
 
 - builder:
     name: lint-yaml-code
     builders:
       - shell: |
-          #!/bin/bash
-          set -o errexit
-          set -o pipefail
-          set -o xtrace
-          export PATH=$PATH:/usr/local/bin/
+          # Install python packages
+          pip install "yamllint==1.8.2"
 
-          # install python packages
-          sudo pip install "yamllint==1.8.2"
-
-          # generate and upload lint log
-          echo "Running yaml code on $PROJECT ..."
-
-          # Get list of yaml files
-          YAML_FILES=$(git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1 | egrep "ya?ml$") || true
-
-          #If YAML_FILES is none exit with 0
-          if [ -z "$YAML_FILES" ]; then
-              exit 0
-          fi
-
-          # Ensure we start with a clean environment
-          rm -f yaml-violation.log lint.log
-
-          # Yamllint files only in patchset
-          for yamlfile in $YAML_FILES; do
-            yamllint $yamlfile >> yaml-violation.log || true
+          echo "Checking yaml file..."
+          for f in $(egrep '\.ya?ml$' modified_files)
+          do
+              yamllint "$f" >> yaml-violation.log
           done
-
-          if [ -s "yaml-violation.log" ]; then
-            SHOWN=$(grep -c -v "^$" yaml-violation.log)
-            echo -e "First $SHOWN shown\n---" > lint.log
-            cat yaml-violation.log >> lint.log
-            sed -r -i '4,$s/^/ /g' lint.log
+          if [[ -s yaml-violation.log ]]; then
+              echo -e "YAML violation(s)\n---" >> violation.log
+              sed -e 's/^/ /g' yaml-violation.log >> violation.log
           fi
 
 - builder:
+    name: lint-all-code
+    builders:
+      - lint-init
+      - lint-bash-code
+      - lint-python-code
+      - lint-yaml-code
+      - lint-report
+
+- builder:
     name: clean-workspace
     builders:
       - shell: |
diff --git a/jjb/opnfvdocs/opnfvdocs.yml b/jjb/opnfvdocs/opnfvdocs.yml
index 3dfb9d155..908ddba9d 100644
--- a/jjb/opnfvdocs/opnfvdocs.yml
+++ b/jjb/opnfvdocs/opnfvdocs.yml
@@ -69,7 +69,7 @@
             notbuilt: true
 
     builders:
-      - check-bash-syntax
+      - lint-bash-code
 
 - job-template:
     name: 'opnfvdocs-merge-shellcheck-{stream}'
@@ -107,4 +107,4 @@
                   branch-pattern: '**/{branch}'
 
     builders:
-      - check-bash-syntax
+      - lint-bash-code
diff --git a/jjb/releng/opnfv-lint.yml b/jjb/releng/opnfv-lint.yml
index d566430d6..c0da8ae51 100644
--- a/jjb/releng/opnfv-lint.yml
+++ b/jjb/releng/opnfv-lint.yml
@@ -54,7 +54,52 @@
                 comment-contains-value: 'reverify'
           projects:
             - project-compare-type: 'REG_EXP'
-              project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator|releng'
+              project-pattern: 'releng|doctor'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+              file-paths:
+                - compare-type: ANT
+                  pattern: '**/*.py'
+                - compare-type: ANT
+                  pattern: '**/*.sh'
+                - compare-type: ANT
+                  pattern: '**/*.yml'
+                - compare-type: ANT
+                  pattern: '**/*.yaml'
+
+    builders:
+      - lint-all-code
+
+- job-template:
+    name: 'opnfv-pylint-verify-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    parameters:
+      - project-parameter:
+          project: $GERRIT_PROJECT
+          branch: '{branch}'
+
+    scm:
+      - git-scm-gerrit
+
+    triggers:
+      - gerrit:
+          server-name: 'gerrit.opnfv.org'
+          trigger-on:
+            - patchset-created-event:
+                exclude-drafts: 'false'
+                exclude-trivial-rebase: 'false'
+                exclude-no-code-change: 'false'
+            - draft-published-event
+            - comment-added-contains-event:
+                comment-contains-value: 'recheck'
+            - comment-added-contains-event:
+                comment-contains-value: 'reverify'
+          projects:
+            - project-compare-type: 'REG_EXP'
+              project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator'
               branches:
                 - branch-compare-type: 'ANT'
                   branch-pattern: '**/{branch}'
@@ -63,8 +108,9 @@
                   pattern: '**/*.py'
 
     builders:
+      - lint-init
       - lint-python-code
-      - report-lint-result-to-gerrit
+      - lint-report
 
 - job-template:
     name: 'opnfv-yamllint-verify-{stream}'
@@ -103,7 +149,7 @@
                 comment-contains-value: 'reverify'
           projects:
             - project-compare-type: 'REG_EXP'
-              project-pattern: 'armband|fuel|octopus|pharos|releng|releng-anteater'
+              project-pattern: 'armband|fuel|octopus|pharos|releng-anteater'
               branches:
                 - branch-compare-type: 'ANT'
                   branch-pattern: '**/{branch}'
@@ -114,5 +160,6 @@
                   pattern: '**/*.yaml'
 
     builders:
+      - lint-init
       - lint-yaml-code
-      - report-lint-result-to-gerrit
+      - lint-report