Email Weekly Anteater Reports to PTLs

Completely replaces the current weekly security scan job. Instead of
publishing weekly security scan reports they will be emailed to
individual project PTLs.

Uses a modified copy of 'anteater-security-audit.sh' to ensure the
security scan job is not affected in this change. A later change will be
made to merge the file back in and update the jobs. This is why
'anteater-parameters' are added to both jobs-templates.

Change-Id: Ia8ebffbfce7a2d4feb83ef68ff0ab0c7bb4d2104
Signed-off-by: Trevor Bramwell <tbramwell@linuxfoundation.org>

5 files changed, 639 insertions, 72 deletions

diff --git a/jjb/ci_gate_security/anteater-clone-all-repos.sh b/jjb/ci_gate_security/anteater-clone-all-repos.sh
deleted file mode 100755
index 8a9e73d85..000000000
--- a/jjb/ci_gate_security/anteater-clone-all-repos.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-# SPDX-license-identifier: Apache-2.0
-set -o errexit
-set -o pipefail
-set -o nounset
-export PATH=$PATH:/usr/local/bin/
-
-
-#WORKSPACE="$(pwd)"
-
-cd $WORKSPACE
-if [ ! -d "$WORKSPACE/allrepos" ]; then
-  mkdir $WORKSPACE/allrepos
-fi
-
-cd $WORKSPACE/allrepos
-
-declare -a PROJECT_LIST
-EXCLUDE_PROJECTS="All-Projects|All-Users|securedlab"
-
-PROJECT_LIST=($(ssh gerrit.opnfv.org -p 29418 gerrit ls-projects | egrep -v $EXCLUDE_PROJECTS))
-echo "PROJECT_LIST=(${PROJECT_LIST[*]})" > $WORKSPACE/opnfv-projects.sh
-
-for PROJECT in ${PROJECT_LIST[@]}; do
-  echo "> Cloning $PROJECT"
-  if [ ! -d "$PROJECT" ]; then
-    git clone "https://gerrit.opnfv.org/gerrit/$PROJECT.git"
-  else
-    pushd "$PROJECT" > /dev/null
-    git pull -f
-    popd > /dev/null
-  fi
-done
diff --git a/jjb/ci_gate_security/anteater-security-audit-weekly.sh b/jjb/ci_gate_security/anteater-security-audit-weekly.sh
index 11909636a..25850af28 100644
--- a/jjb/ci_gate_security/anteater-security-audit-weekly.sh
+++ b/jjb/ci_gate_security/anteater-security-audit-weekly.sh
@@ -1,37 +1,51 @@
 #!/bin/bash
 # SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2017 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+ANTEATER_SCAN_PATCHSET="${ANTEATER_SCAN_PATCHSET:-true}"
+
+cd $WORKSPACE
+REPORTDIR='.reports'
+mkdir -p $REPORTDIR
+# Ensure any user can read the reports directory
+chmod 777 $REPORTDIR
+
+ANTEATER_FILES="--patchset /home/opnfv/anteater/$PROJECT/patchset"
+
+if [[ "$ANTEATER_SCAN_PATCHSET" == "true" ]]; then
+    echo "Generating patchset file to list changed files"
+    git diff HEAD^1 --name-only | sed "s#^#/home/opnfv/anteater/$PROJECT/#" > $WORKSPACE/patchset
+    echo "Changed files are"
+    echo "--------------------------------------------------------"
+    cat $WORKSPACE/patchset
+    echo "--------------------------------------------------------"
+else
+    echo "Checking full project $PROJECT"
+    ANTEATER_FILES="--path /home/opnfv/anteater/$PROJECT"
+fi
+
+vols="-v $WORKSPACE:/home/opnfv/anteater/$PROJECT -v $WORKSPACE/$REPORTDIR:/home/opnfv/anteater/$REPORTDIR"
+envs="-e PROJECT=$PROJECT"
 
-echo "--------------------------------------------------------"
-vols="-v $WORKSPACE/allrepos/:/home/opnfv/anteater/allrepos/"
 echo "Pulling releng-anteater docker image"
 echo "--------------------------------------------------------"
 docker pull opnfv/releng-anteater
 echo "--------------------------------------------------------"
-cmd="docker run -id $vols opnfv/releng-anteater /bin/bash"
-echo "Running docker command $cmd"
-container_id=$($cmd)
-echo "Container ID is $container_id"
-source $WORKSPACE/opnfv-projects.sh
-for project in "${PROJECT_LIST[@]}"
-
-do
-  cmd="/home/opnfv/venv/bin/anteater --project testproj --path /home/opnfv/anteater/allrepos/$project"
-  echo "Executing command inside container"
-  echo "$cmd"
-  echo "--------------------------------------------------------"
-  docker exec $container_id $cmd > $WORKSPACE/"$project".securityaudit.log 2>&1
-done
 
+cmd="docker run -i $envs $vols --rm opnfv/releng-anteater \
+/home/opnfv/venv/bin/anteater --project $PROJECT $ANTEATER_FILES"
+echo "Running docker container"
+echo "$cmd"
+$cmd > $WORKSPACE/securityaudit.log 2>&1
 exit_code=$?
 echo "--------------------------------------------------------"
-echo "Stopping docker container with ID $container_id"
-docker stop $container_id
-
-
-#gsutil cp $WORKSPACE/securityaudit.log \
-#    gs://$GS_URL/$PROJECT-securityaudit-weekly.log 2>&1
-#
-#gsutil -m setmeta \
-#    -h "Content-Type:text/html" \
-#    -h "Cache-Control:private, max-age=0, no-transform" \
-#    gs://$GS_URL/$PROJECT-securityaudit-weekly.log > /dev/null 2>&1
+echo "Docker container exited with code: $exit_code"
+echo "--------------------------------------------------------"
+cat securityaudit.log
+exit 0
diff --git a/jjb/ci_gate_security/opnfv-ci-gate-security.yml b/jjb/ci_gate_security/opnfv-ci-gate-security.yml
index 0a412c240..5a2534ae8 100644
--- a/jjb/ci_gate_security/opnfv-ci-gate-security.yml
+++ b/jjb/ci_gate_security/opnfv-ci-gate-security.yml
@@ -9,9 +9,76 @@
 
     project: anteaterfw
 
+    repo:
+      - apex
+      - apex-os-net-config
+      - apex-puppet-tripleo
+      - apex-tripleo-heat-templates
+      - armband
+      - auto
+      - availability
+      - bamboo
+      - barometer
+      - bottlenecks
+      - calipso
+      - clover
+      - compass-containers
+      - compass4nfv
+      - conductor
+      - container4nfv
+      - copper
+      - cperf
+      - daisy
+      - doctor
+      - domino
+      - dovetail
+      - dpacc
+      - enfv
+      - fastpathmetrics
+      - fds
+      - fuel
+      - functest
+      - ipv6
+      - joid
+      - kvmfornfv
+      - models
+      - moon
+      - multisite
+      - netready
+      - nfvbench
+      - octopus
+      - onosfw
+      - openretriever
+      - opera
+      - opnfvdocs
+      - orchestra
+      - ovn4nfv
+      - ovno
+      - ovsnfv
+      - parser
+      - pharos
+      - pharos-tools
+      - promise
+      - qtip
+      - releng
+      - releng-anteater
+      - releng-testresults
+      - releng-utils
+      - releng-xci
+      - samplevnf
+      - sdnvpn
+      - securityscanning
+      - sfc
+      - snaps
+      - stor4nfv
+      - storperf
+      - ves
+      - vswitchperf
+      - yardstick
+
     jobs:
       - 'opnfv-security-audit-verify-{stream}'
-      - 'opnfv-security-audit-weekly-{stream}'
+      - 'opnfv-security-audit-{repo}-weekly-{stream}'
 
     stream:
       - master:
@@ -23,24 +90,34 @@
 # job templates
 ########################
 - job-template:
-    name: 'opnfv-security-audit-weekly-{stream}'
+    name: 'opnfv-security-audit-{repo}-weekly-{stream}'
 
     disabled: '{obj:disabled}'
 
     parameters:
-      - label:
-          name: SLAVE_LABEL
-          default: 'ericsson-build3'
-          description: 'Slave label on Jenkins'
+      - ericsson-build3-defaults
+      - string:
+          name: ANTEATER_SCAN_PATCHSET
+          default: "false"
+          description: "Have anteater scan patchsets (true) or full project (false)"
       - project-parameter:
-          project: releng
+          project: '{repo}'
           branch: '{branch}'
 
+    scm:
+      - git-scm-gerrit
+
     triggers:
       - timed: '@weekly'
 
     builders:
       - anteater-security-audit-weekly
+      - clean-workspace
+
+    publishers:
+      # defined in jjb/global/releng-macros.yml
+      - 'email-{repo}-ptl':
+          subject: 'OPNFV Security Scan Result: {repo}'
 
 - job-template:
     name: 'opnfv-security-audit-verify-{stream}'
@@ -117,12 +194,8 @@
       - shell:
           !include-raw: ./anteater-report-to-gerrit.sh
 
-# yamllint disable rule:indentation
 - builder:
     name: anteater-security-audit-weekly
     builders:
       - shell:
-          !include-raw:
-              - ./anteater-clone-all-repos.sh
-              - ./anteater-security-audit-weekly.sh
-# yamllint enable rule:indentation
+          !include-raw: ./anteater-security-audit-weekly.sh
diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml
index 08766943c..28216388e 100644
--- a/jjb/global/releng-macros.yml
+++ b/jjb/global/releng-macros.yml
@@ -463,3 +463,504 @@
           failure: true
           send-to:
             - recipients
+
+# Email PTL publishers
+- email_ptl_defaults: &email_ptl_defaults
+    name: 'email_ptl_defaults'
+    content-type: text
+    attach-build-log: true
+    compress-log: true
+    always: true
+    subject: '{subject}'
+
+- publisher:
+    name: 'email-apex-ptl'
+    publishers: &email_apex_ptl_defaults
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            trozet@redhat.com
+- publisher:
+    name: 'email-apex-os-net-config-ptl'
+    publishers:
+      <<: *email_apex_ptl_defaults
+- publisher:
+    name: 'email-apex-puppet-tripleo-ptl'
+    publishers:
+      <<: *email_apex_ptl_defaults
+- publisher:
+    name: 'email-apex-tripleo-heat-templates-ptl'
+    publishers:
+      <<: *email_apex_ptl_defaults
+
+- publisher:
+    name: 'email-armband-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            bob.monkman@arm.com
+
+- publisher:
+    name: 'email-auto-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            tina.tsou@arm.com
+
+- publisher:
+    name: 'email-availability-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            fuqiao@chinamobile.com
+
+- publisher:
+    name: 'email-bamboo-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            donaldh@cisco.com
+
+- publisher:
+    name: 'email-barometer-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            aasmith@redhat.com
+
+- publisher:
+    name: 'email-bottlenecks-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            gabriel.yuyang@huawei.com
+
+- publisher:
+    name: 'email-calipso-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            korlev@cisco.com
+
+- publisher:
+    name: 'email-clover-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            stephen.wong1@huawei.com
+
+- publisher:
+    name: 'email-compass4nfv-ptl'
+    publishers: &email_compass4nfv_ptl_defaults
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            chigang@huawei.com
+- publisher:
+    name: 'email-compass-containers-ptl'
+    publishers:
+      <<: *email_compass4nfv_ptl_defaults
+
+- publisher:
+    name: 'email-conductor-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            limingjiang@huawei.com
+
+- publisher:
+    name: 'email-container4nfv-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            jiaxuan@chinamobile.com
+
+- publisher:
+    name: 'email-copper-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            aimeeu.opensource@gmail.com
+
+- publisher:
+    name: 'email-cperf-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            matt.welch@intel.com
+
+- publisher:
+    name: 'email-daisy-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            hu.zhijiang@zte.com.cn
+
+- publisher:
+    name: 'email-doctor-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            r-mibu@cq.jp.nec.com
+
+- publisher:
+    name: 'email-domino-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            ulas.kozat@huawei.com
+
+- publisher:
+    name: 'email-dovetail-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            hongbo.tianhongbo@huawei.com
+
+- publisher:
+    name: 'email-dpacc-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            denglingli@chinamobile.com
+
+- publisher:
+    name: 'email-enfv-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            JBuchanan@advaoptical.com
+
+- publisher:
+    name: 'email-escalator-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            kong.wei2@zte.com.cn
+
+- publisher:
+    name: 'email-fastpathmetrics-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            maryam.tahhan@intel.com
+
+- publisher:
+    name: 'email-fds-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            fbrockne@cisco.com
+
+- publisher:
+    name: 'email-fuel-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            gelkinbard@mirantis.com
+
+- publisher:
+    name: 'email-functest-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            cedric.ollivier@orange.com
+
+- publisher:
+    name: 'email-ipv6-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            bh526r@att.com
+
+- publisher:
+    name: 'email-joid-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            artur.tyloch@canonical.com
+
+- publisher:
+    name: 'email-kvmfornfv-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            raghuveer.reddy@intel.com
+
+- publisher:
+    name: 'email-models-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            bs3131@att.com
+
+- publisher:
+    name: 'email-moon-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            ruan.he@orange.com
+
+- publisher:
+    name: 'email-multisite-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            joehuang@huawei.com
+
+- publisher:
+    name: 'email-netready-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            georg.kunz@ericsson.com
+
+- publisher:
+    name: 'email-nfvbench-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            ahothan@cisco.com
+
+- publisher:
+    name: 'email-octopus-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            ulrich.kleber@huawei.com
+
+- publisher:
+    name: 'email-onosfw-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            su.wei@huawei.com
+
+- publisher:
+    name: 'email-openretriever-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            jiaxuan@chinamobile.com
+
+- publisher:
+    name: 'email-opera-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            Yingjun.li@huawei.com
+
+- publisher:
+    name: 'email-opnfvdocs-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            sofia.wallin@ericsson.com
+
+- publisher:
+    name: 'email-orchestra-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            giuseppe.carella@fokus.fraunhofer.de
+
+- publisher:
+    name: 'email-ovn4nfv-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            trinath.somanchi@gmail.com
+
+- publisher:
+    name: 'email-ovno-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            wsmackie@juniper.net
+
+- publisher:
+    name: 'email-ovsnfv-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            MarkD.Graymark.d.gray@intel.com
+
+- publisher:
+    name: 'email-parser-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            shang.xiaodong@zte.com.cn
+
+- publisher:
+    name: 'email-pharos-ptl'
+    publishers: &email_pharos_ptl_defaults
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            jack.morgan@intel.com
+- publisher:
+    name: 'email-pharos-tools-ptl'
+    publishers:
+      <<: *email_pharos_ptl_defaults
+
+- publisher:
+    name: 'email-promise-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            kunzmann@docomolab-euro.com
+
+- publisher:
+    name: 'email-qtip-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            wu.zhihui1@zte.com.cn
+
+- publisher:
+    name: 'email-releng-ptl'
+    publishers: &email_releng_ptl_defaults
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            fatih.degirmenci@ericsson.com
+- publisher:
+    name: 'email-releng-anteater-ptl'
+    publishers:
+      <<: *email_releng_ptl_defaults
+- publisher:
+    name: 'email-releng-testresults-ptl'
+    publishers:
+      <<: *email_releng_ptl_defaults
+- publisher:
+    name: 'email-releng-utils-ptl'
+    publishers:
+      <<: *email_releng_ptl_defaults
+- publisher:
+    name: 'email-releng-xci-ptl'
+    publishers:
+      <<: *email_releng_ptl_defaults
+
+- publisher:
+    name: 'email-samplevnf-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            deepak.s@intel.com
+
+- publisher:
+    name: 'email-sdnvpn-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            tim.irnich@ericsson.com
+
+- publisher:
+    name: 'email-securityscanning-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            lhinds@redhat.com
+
+- publisher:
+    name: 'email-sfc-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            ManuelBuilmbuil@suse.com
+
+- publisher:
+    name: 'email-snaps-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            s.pisarski@cablelabs.com
+
+- publisher:
+    name: 'email-stor4nfv-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            shane.wang@intel.com
+
+- publisher:
+    name: 'email-storperf-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            mark.beierl@emc.com
+
+- publisher:
+    name: 'email-ves-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            bryan.sullivan@att.com
+
+- publisher:
+    name: 'email-vswitchperf-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            sridhar.rao@spirent.com
+
+- publisher:
+    name: 'email-yardstick-ptl'
+    publishers:
+      - email-ext:
+          <<: *email_ptl_defaults
+          recipients: >
+            ross.b.brattain@intel.com
diff --git a/jjb/global/slave-params.yml b/jjb/global/slave-params.yml
index 04de1e091..8ce576ed6 100644
--- a/jjb/global/slave-params.yml
+++ b/jjb/global/slave-params.yml
@@ -457,6 +457,18 @@
           description: "Directory where the build artifact will be located upon the completion of the build."
 
 - parameter:
+    name: 'ericsson-build3-defaults'
+    parameters:
+      - label:
+          name: SLAVE_LABEL
+          default: 'ericsson-build3'
+          description: 'Slave label on Jenkins'
+      - string:
+          name: GIT_BASE
+          default: https://gerrit.opnfv.org/gerrit/$PROJECT
+          description: 'Git URL to use on this Jenkins Slave'
+
+- parameter:
     name: 'huawei-build-defaults'
     parameters:
       - node: