Merge "add 'lint-all-code' as wrapper of lint builders"

4 files changed, 114 insertions, 113 deletions

diff --git a/jjb/compass4nfv/compass-verify-jobs.yml b/jjb/compass4nfv/compass-verify-jobs.yml
index 7024dad58..75d1653f9 100644
--- a/jjb/compass4nfv/compass-verify-jobs.yml
+++ b/jjb/compass4nfv/compass-verify-jobs.yml
@@ -136,11 +136,6 @@
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: true
-            - name: 'opnfv-yamllint-verify-{stream}'
-              current-parameters: true
-              node-parameters: true
-              kill-phase-on: FAILURE
-              abort-all-job: true
       - multijob:
           name: deploy-virtual
           condition: SUCCESSFUL
@@ -261,11 +256,6 @@
               node-parameters: true
               kill-phase-on: FAILURE
               abort-all-job: true
-            - name: 'opnfv-yamllint-verify-{stream}'
-              current-parameters: true
-              node-parameters: true
-              kill-phase-on: FAILURE
-              abort-all-job: true
       - multijob:
           name: deploy-virtual
           condition: SUCCESSFUL
diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml
index c6a65ca21..68ec1b3d8 100644
--- a/jjb/global/releng-macros.yml
+++ b/jjb/global/releng-macros.yml
@@ -298,128 +298,92 @@
           fi
 
 - builder:
-    name: lint-python-code
+    name: upload-review-docs
     builders:
-      - shell: |
-          #!/bin/bash
-          set -o errexit
-          set -o pipefail
-          set -o xtrace
-          export PATH=$PATH:/usr/local/bin/
-
-          virtualenv -p python2.7 $WORKSPACE/releng_flake8
-          source $WORKSPACE/releng_flake8/bin/activate
-
-          # install python packages
-          pip install "flake8==2.6.2"
-
-          # generate and upload lint log
-          echo "Running flake8 code on $PROJECT ..."
-
-          # Get number of flake8 violations. If none, this will be an
-          # empty string: ""
-          FLAKE_COUNT="$(find . \
-              -path './releng_flake8' -prune -o \
-              -path './.tox' -prune -o \
-              -type f -name "*.py" -print | \
-              xargs flake8 --exit-zero -qq --count 2>&1)"
+      - upload-under-review-docs-to-opnfv-artifacts
+      - report-build-result-to-gerrit
 
+- builder:
+    name: lint-init
+    builders:
+      - shell: |
           # Ensure we start with a clean environment
-          rm -f lint.log
-
-          if [ ! -z $FLAKE_COUNT ]; then
-            echo "Flake8 Violations: $FLAKE_COUNT" > lint.log
-            find . \
-                -path './releng_flake8' -prune -o \
-                -path './.tox' -prune -o \
-                -type f -name "*.py" -print | \
-                xargs flake8 --exit-zero --first >> violation.log
-            SHOWN=$(wc -l violation.log | cut -d' ' -f1)
-            echo -e "First $SHOWN shown\n---" >> lint.log
-            cat violation.log >> lint.log
-            sed -r -i '4,$s/^/ /g' lint.log
-            rm violation.log
-          fi
-
-          deactivate
+          rm -f bash-violation.log python-violation.log yaml-violation.log violation.log
+          git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1) > modified_files
 
 - builder:
-    name: report-lint-result-to-gerrit
+    name: lint-report
     builders:
       - shell: |
-          #!/bin/bash
-          set -o errexit
-          set -o pipefail
-          set -o xtrace
-          export PATH=$PATH:/usr/local/bin/
-
-          # If no violations were found, no lint log will exist.
-          if [[ -e lint.log ]] ; then
-              echo -e "\nposting linting report to gerrit...\n"
-
-              cat lint.log
-              echo
-
-              ssh -p 29418 gerrit.opnfv.org \
-                  "gerrit review -p $GERRIT_PROJECT \
-                   -m \"$(cat lint.log)\" \
-                   $GERRIT_PATCHSET_REVISION \
-                   --notify NONE"
-
+          if [[ -s violation.log ]]; then
+              echo "Reporting lint result..."
+              msg="Found syntax error and/or coding style violation(s) in the files modified by your patchset."
+              sed -i -e '1s/^//$msg\n\n/' violation.log
+              cmd="gerrit review -p $GERRIT_PROJECT -m \"$(cat violation.log)\" $GERRIT_PATCHSET_REVISION --notify NONE"
+              ssh -p 29418 gerrit.opnfv.org "$cmd"
+
+              # Make sure the caller job failed
               exit 1
           fi
 
 - builder:
-    name: upload-review-docs
+    name: lint-bash-code
     builders:
-      - upload-under-review-docs-to-opnfv-artifacts
-      - report-build-result-to-gerrit
+      - shell: |
+          echo "Checking bash code..."
+          for f in $(egrep '\.sh$' modified_files)
+          do
+              bash -n "$f" 2>> bash-violation.log
+          done
+          if [[ -s bash-violation.log ]]; then
+              echo -e "Bash syntax error(s)\n---" >> violation.log
+              sed -e 's/^/ /g' bash-violation.log >> violation.log
+          fi
 
 - builder:
-    name: check-bash-syntax
+    name: lint-python-code
     builders:
-      - shell: "find . -name '*.sh' | xargs bash -n"
+      - shell: |
+          # Install python packages
+          pip install "flake8==2.6.2"
+
+          echo "Checking python code..."
+          for f in $(egrep '\.py$' modified_files)
+          do
+              flake8 "$f" >> python-violation.log
+          done
+          if [[ -s python-violation.log ]]; then
+              echo -e "Python violation(s)\n---" >> violation.log
+              sed -e 's/^/ /g' python-violation.log >> violation.log
+          fi
 
 - builder:
     name: lint-yaml-code
     builders:
       - shell: |
-          #!/bin/bash
-          set -o errexit
-          set -o pipefail
-          set -o xtrace
-          export PATH=$PATH:/usr/local/bin/
+          # Install python packages
+          pip install "yamllint==1.8.2"
 
-          # install python packages
-          sudo pip install "yamllint==1.8.2"
-
-          # generate and upload lint log
-          echo "Running yaml code on $PROJECT ..."
-
-          # Get list of yaml files
-          YAML_FILES=$(git --no-pager diff --diff-filter=MCRAT --name-only HEAD^1 | egrep "ya?ml$") || true
-
-          #If YAML_FILES is none exit with 0
-          if [ -z "$YAML_FILES" ]; then
-              exit 0
-          fi
-
-          # Ensure we start with a clean environment
-          rm -f yaml-violation.log lint.log
-
-          # Yamllint files only in patchset
-          for yamlfile in $YAML_FILES; do
-            yamllint $yamlfile >> yaml-violation.log || true
+          echo "Checking yaml file..."
+          for f in $(egrep '\.ya?ml$' modified_files)
+          do
+              yamllint "$f" >> yaml-violation.log
           done
-
-          if [ -s "yaml-violation.log" ]; then
-            SHOWN=$(grep -c -v "^$" yaml-violation.log)
-            echo -e "First $SHOWN shown\n---" > lint.log
-            cat yaml-violation.log >> lint.log
-            sed -r -i '4,$s/^/ /g' lint.log
+          if [[ -s yaml-violation.log ]]; then
+              echo -e "YAML violation(s)\n---" >> violation.log
+              sed -e 's/^/ /g' yaml-violation.log >> violation.log
           fi
 
 - builder:
+    name: lint-all-code
+    builders:
+      - lint-init
+      - lint-bash-code
+      - lint-python-code
+      - lint-yaml-code
+      - lint-report
+
+- builder:
     name: clean-workspace
     builders:
       - shell: |
diff --git a/jjb/opnfvdocs/opnfvdocs.yml b/jjb/opnfvdocs/opnfvdocs.yml
index 3dfb9d155..908ddba9d 100644
--- a/jjb/opnfvdocs/opnfvdocs.yml
+++ b/jjb/opnfvdocs/opnfvdocs.yml
@@ -69,7 +69,7 @@
             notbuilt: true
 
     builders:
-      - check-bash-syntax
+      - lint-bash-code
 
 - job-template:
     name: 'opnfvdocs-merge-shellcheck-{stream}'
@@ -107,4 +107,4 @@
                   branch-pattern: '**/{branch}'
 
     builders:
-      - check-bash-syntax
+      - lint-bash-code
diff --git a/jjb/releng/opnfv-lint.yml b/jjb/releng/opnfv-lint.yml
index d566430d6..c0da8ae51 100644
--- a/jjb/releng/opnfv-lint.yml
+++ b/jjb/releng/opnfv-lint.yml
@@ -54,7 +54,52 @@
                 comment-contains-value: 'reverify'
           projects:
             - project-compare-type: 'REG_EXP'
-              project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator|releng'
+              project-pattern: 'releng|doctor'
+              branches:
+                - branch-compare-type: 'ANT'
+                  branch-pattern: '**/{branch}'
+              file-paths:
+                - compare-type: ANT
+                  pattern: '**/*.py'
+                - compare-type: ANT
+                  pattern: '**/*.sh'
+                - compare-type: ANT
+                  pattern: '**/*.yml'
+                - compare-type: ANT
+                  pattern: '**/*.yaml'
+
+    builders:
+      - lint-all-code
+
+- job-template:
+    name: 'opnfv-pylint-verify-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    parameters:
+      - project-parameter:
+          project: $GERRIT_PROJECT
+          branch: '{branch}'
+
+    scm:
+      - git-scm-gerrit
+
+    triggers:
+      - gerrit:
+          server-name: 'gerrit.opnfv.org'
+          trigger-on:
+            - patchset-created-event:
+                exclude-drafts: 'false'
+                exclude-trivial-rebase: 'false'
+                exclude-no-code-change: 'false'
+            - draft-published-event
+            - comment-added-contains-event:
+                comment-contains-value: 'recheck'
+            - comment-added-contains-event:
+                comment-contains-value: 'reverify'
+          projects:
+            - project-compare-type: 'REG_EXP'
+              project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator'
               branches:
                 - branch-compare-type: 'ANT'
                   branch-pattern: '**/{branch}'
@@ -63,8 +108,9 @@
                   pattern: '**/*.py'
 
     builders:
+      - lint-init
       - lint-python-code
-      - report-lint-result-to-gerrit
+      - lint-report
 
 - job-template:
     name: 'opnfv-yamllint-verify-{stream}'
@@ -103,7 +149,7 @@
                 comment-contains-value: 'reverify'
           projects:
             - project-compare-type: 'REG_EXP'
-              project-pattern: 'armband|fuel|octopus|pharos|releng|releng-anteater'
+              project-pattern: 'armband|fuel|octopus|pharos|releng-anteater'
               branches:
                 - branch-compare-type: 'ANT'
                   branch-pattern: '**/{branch}'
@@ -114,5 +160,6 @@
                   pattern: '**/*.yaml'
 
     builders:
+      - lint-init
       - lint-yaml-code
-      - report-lint-result-to-gerrit
+      - lint-report