Email Weekly Anteater Reports to PTLs

Completely replaces the current weekly security scan job. Instead of
publishing weekly security scan reports they will be emailed to
individual project PTLs.

Uses a modified copy of 'anteater-security-audit.sh' to ensure the
security scan job is not affected in this change. A later change will be
made to merge the file back in and update the jobs. This is why
'anteater-parameters' are added to both jobs-templates.

Change-Id: Ia8ebffbfce7a2d4feb83ef68ff0ab0c7bb4d2104
Signed-off-by: Trevor Bramwell <tbramwell@linuxfoundation.org>

3 files changed, 126 insertions, 72 deletions

diff --git a/jjb/ci_gate_security/anteater-clone-all-repos.sh b/jjb/ci_gate_security/anteater-clone-all-repos.sh
deleted file mode 100755
index 8a9e73d85..000000000
--- a/jjb/ci_gate_security/anteater-clone-all-repos.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# SPDX-license-identifier: Apache-2.0
-set -o errexit
-set -o pipefail
-set -o nounset
-export PATH=$PATH:/usr/local/bin/
-
-
-#WORKSPACE="$(pwd)"
-
-cd $WORKSPACE
-if [ ! -d "$WORKSPACE/allrepos" ]; then
-  mkdir $WORKSPACE/allrepos
-fi
-
-cd $WORKSPACE/allrepos
-
-declare -a PROJECT_LIST
-EXCLUDE_PROJECTS="All-Projects|All-Users|securedlab"
-
-PROJECT_LIST=($(ssh gerrit.opnfv.org -p 29418 gerrit ls-projects | egrep -v $EXCLUDE_PROJECTS))
-echo "PROJECT_LIST=(${PROJECT_LIST[*]})" > $WORKSPACE/opnfv-projects.sh
-
-for PROJECT in ${PROJECT_LIST[@]}; do
-  echo "> Cloning $PROJECT"
-  if [ ! -d "$PROJECT" ]; then
-    git clone "https://gerrit.opnfv.org/gerrit/$PROJECT.git"
-  else
-    pushd "$PROJECT" > /dev/null
-    git pull -f
-    popd > /dev/null
-  fi
-done
diff --git a/jjb/ci_gate_security/anteater-security-audit-weekly.sh b/jjb/ci_gate_security/anteater-security-audit-weekly.sh
index 11909636a..25850af28 100644
--- a/jjb/ci_gate_security/anteater-security-audit-weekly.sh
+++ b/jjb/ci_gate_security/anteater-security-audit-weekly.sh
@@ -1,37 +1,51 @@
 #!/bin/bash
 # SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+ANTEATER_SCAN_PATCHSET="${ANTEATER_SCAN_PATCHSET:-true}"
+
+cd $WORKSPACE
+REPORTDIR='.reports'
+mkdir -p $REPORTDIR
+# Ensure any user can read the reports directory
+chmod 777 $REPORTDIR
+
+ANTEATER_FILES="--patchset /home/opnfv/anteater/$PROJECT/patchset"
+
+if [[ "$ANTEATER_SCAN_PATCHSET" == "true" ]]; then
+    echo "Generating patchset file to list changed files"
+    git diff HEAD^1 --name-only | sed "s#^#/home/opnfv/anteater/$PROJECT/#" > $WORKSPACE/patchset
+    echo "Changed files are"
+    echo "--------------------------------------------------------"
+    cat $WORKSPACE/patchset
+    echo "--------------------------------------------------------"
+else
+    echo "Checking full project $PROJECT"
+    ANTEATER_FILES="--path /home/opnfv/anteater/$PROJECT"
+fi
+
+vols="-v $WORKSPACE:/home/opnfv/anteater/$PROJECT -v $WORKSPACE/$REPORTDIR:/home/opnfv/anteater/$REPORTDIR"
+envs="-e PROJECT=$PROJECT"
 
-echo "--------------------------------------------------------"
-vols="-v $WORKSPACE/allrepos/:/home/opnfv/anteater/allrepos/"
 echo "Pulling releng-anteater docker image"
 echo "--------------------------------------------------------"
 docker pull opnfv/releng-anteater
 echo "--------------------------------------------------------"
-cmd="docker run -id $vols opnfv/releng-anteater /bin/bash"
-echo "Running docker command $cmd"
-container_id=$($cmd)
-echo "Container ID is $container_id"
-source $WORKSPACE/opnfv-projects.sh
-for project in "${PROJECT_LIST[@]}"
-
-do
-  cmd="/home/opnfv/venv/bin/anteater --project testproj --path /home/opnfv/anteater/allrepos/$project"
-  echo "Executing command inside container"
-  echo "$cmd"
-  echo "--------------------------------------------------------"
-  docker exec $container_id $cmd > $WORKSPACE/"$project".securityaudit.log 2>&1
-done
 
+cmd="docker run -i $envs $vols --rm opnfv/releng-anteater \
+/home/opnfv/venv/bin/anteater --project $PROJECT $ANTEATER_FILES"
+echo "Running docker container"
+echo "$cmd"
+$cmd > $WORKSPACE/securityaudit.log 2>&1
 exit_code=$?
 echo "--------------------------------------------------------"
-echo "Stopping docker container with ID $container_id"
-docker stop $container_id
-
-
-#gsutil cp $WORKSPACE/securityaudit.log \
-#    gs://$GS_URL/$PROJECT-securityaudit-weekly.log 2>&1
-#
-#gsutil -m setmeta \
-#    -h "Content-Type:text/html" \
-#    -h "Cache-Control:private, max-age=0, no-transform" \
-#    gs://$GS_URL/$PROJECT-securityaudit-weekly.log > /dev/null 2>&1
+echo "Docker container exited with code: $exit_code"
+echo "--------------------------------------------------------"
+cat securityaudit.log
+exit 0
diff --git a/jjb/ci_gate_security/opnfv-ci-gate-security.yml b/jjb/ci_gate_security/opnfv-ci-gate-security.yml
index 0a412c240..5a2534ae8 100644
--- a/jjb/ci_gate_security/opnfv-ci-gate-security.yml
+++ b/jjb/ci_gate_security/opnfv-ci-gate-security.yml
@@ -9,9 +9,76 @@
 
     project: anteaterfw
 
+    repo:
+      - apex
+      - apex-os-net-config
+      - apex-puppet-tripleo
+      - apex-tripleo-heat-templates
+      - armband
+      - auto
+      - availability
+      - bamboo
+      - barometer
+      - bottlenecks
+      - calipso
+      - clover
+      - compass-containers
+      - compass4nfv
+      - conductor
+      - container4nfv
+      - copper
+      - cperf
+      - daisy
+      - doctor
+      - domino
+      - dovetail
+      - dpacc
+      - enfv
+      - fastpathmetrics
+      - fds
+      - fuel
+      - functest
+      - ipv6
+      - joid
+      - kvmfornfv
+      - models
+      - moon
+      - multisite
+      - netready
+      - nfvbench
+      - octopus
+      - onosfw
+      - openretriever
+      - opera
+      - opnfvdocs
+      - orchestra
+      - ovn4nfv
+      - ovno
+      - ovsnfv
+      - parser
+      - pharos
+      - pharos-tools
+      - promise
+      - qtip
+      - releng
+      - releng-anteater
+      - releng-testresults
+      - releng-utils
+      - releng-xci
+      - samplevnf
+      - sdnvpn
+      - securityscanning
+      - sfc
+      - snaps
+      - stor4nfv
+      - storperf
+      - ves
+      - vswitchperf
+      - yardstick
+
     jobs:
       - 'opnfv-security-audit-verify-{stream}'
-      - 'opnfv-security-audit-weekly-{stream}'
+      - 'opnfv-security-audit-{repo}-weekly-{stream}'
 
     stream:
       - master:
@@ -23,24 +90,34 @@
 # job templates
 ########################
 - job-template:
-    name: 'opnfv-security-audit-weekly-{stream}'
+    name: 'opnfv-security-audit-{repo}-weekly-{stream}'
 
     disabled: '{obj:disabled}'
 
     parameters:
-      - label:
-          name: SLAVE_LABEL
-          default: 'ericsson-build3'
-          description: 'Slave label on Jenkins'
+      - ericsson-build3-defaults
+      - string:
+          name: ANTEATER_SCAN_PATCHSET
+          default: "false"
+          description: "Have anteater scan patchsets (true) or full project (false)"
       - project-parameter:
-          project: releng
+          project: '{repo}'
           branch: '{branch}'
 
+    scm:
+      - git-scm-gerrit
+
     triggers:
       - timed: '@weekly'
 
     builders:
       - anteater-security-audit-weekly
+      - clean-workspace
+
+    publishers:
+      # defined in jjb/global/releng-macros.yml
+      - 'email-{repo}-ptl':
+          subject: 'OPNFV Security Scan Result: {repo}'
 
 - job-template:
     name: 'opnfv-security-audit-verify-{stream}'
@@ -117,12 +194,8 @@
       - shell:
           !include-raw: ./anteater-report-to-gerrit.sh
 
-# yamllint disable rule:indentation
 - builder:
     name: anteater-security-audit-weekly
     builders:
       - shell:
-          !include-raw:
-              - ./anteater-clone-all-repos.sh
-              - ./anteater-security-audit-weekly.sh
-# yamllint enable rule:indentation
+          !include-raw: ./anteater-security-audit-weekly.sh