Rearrange Releng Job and Parameter Files

Individual project jobs can now be tested using:

  jenkins-jobs test -r jjb/global:jjb/$project

- Rename '.yaml' files to '.yml' so they get checked by the
  verify-releng.sh script, and update the script to include checking of
  '.yaml' files.
- Move parameter and macro files under new 'global' directory.

Change-Id: I6b74f5d8af2bced3256a7f4a33983df5310511a2
Signed-off-by: Trevor Bramwell <tbramwell@linuxfoundation.org>

7 files changed, 444 insertions, 1 deletions

diff --git a/jjb/releng/artifact-cleanup.yml b/jjb/releng/artifact-cleanup.yml
new file mode 100644
index 000000000..1c609a4b2
--- /dev/null
+++ b/jjb/releng/artifact-cleanup.yml
@@ -0,0 +1,42 @@
+- project:
+    name: releng-artifact-cleanup
+
+    project: 'releng'
+
+    jobs:
+        - 'releng-artifact-cleanup-daily-{stream}'
+
+    stream:
+        - master:
+            branch: '{stream}'
+            gs-pathname: ''
+
+
+- job-template:
+    name: 'releng-artifact-cleanup-daily-{stream}'
+
+    # Job template for daily builders
+    #
+    # Required Variables:
+    #     stream:    branch with - in place of / (eg. stable)
+    #     branch:    branch (eg. stable)
+    node: master
+
+    disabled: false
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+
+    scm:
+        - git-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: ''
+            branch: '{branch}'
+
+    triggers:
+        - timed: 'H H * * *'
+
+    builders:
+        - shell: |
+            $WORKSPACE/utils/retention_script.sh
diff --git a/jjb/releng/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh
new file mode 100644
index 000000000..e0fbb7564
--- /dev/null
+++ b/jjb/releng/opnfv-docker.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2016 Ericsson AB and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -o errexit
+set -o nounset
+set -o pipefail
+
+
+echo "Starting opnfv-docker for $DOCKER_REPO_NAME ..."
+echo "--------------------------------------------------------"
+echo
+
+
+if [[ -n $(ps -ef|grep 'docker build'|grep -v grep) ]]; then
+    echo "There is already another build process in progress:"
+    echo $(ps -ef|grep 'docker build'|grep -v grep)
+    # Abort this job since it will collide and might mess up the current one.
+    echo "Aborting..."
+    exit 1
+fi
+
+# Remove previous running containers if exist
+if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then
+    echo "Removing existing $DOCKER_REPO_NAME containers..."
+    docker ps -a | grep $DOCKER_REPO_NAME | awk '{print $1}' | xargs docker rm -f
+    t=60
+    # Wait max 60 sec for containers to be removed
+    while [[ $t -gt 0 ]] && [[ -n "$(docker ps| grep $DOCKER_REPO_NAME)" ]]; do
+        sleep 1
+        let t=t-1
+    done
+fi
+
+
+# Remove existing images if exist
+if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then
+    echo "Docker images to remove:"
+    docker images | head -1 && docker images | grep $DOCKER_REPO_NAME
+    image_tags=($(docker images | grep $DOCKER_REPO_NAME | awk '{print $2}'))
+    for tag in "${image_tags[@]}"; do
+        if [[ -n "$(docker images|grep $DOCKER_REPO_NAME|grep $tag)" ]]; then
+            echo "Removing docker image $DOCKER_REPO_NAME:$tag..."
+            docker rmi -f $DOCKER_REPO_NAME:$tag
+        fi
+    done
+fi
+
+
+# cd to directory where Dockerfile is located
+cd $WORKSPACE/docker
+if [ ! -f ./Dockerfile ]; then
+    echo "ERROR: Dockerfile not found."
+    exit 1
+fi
+
+# Get tag version
+branch="${GIT_BRANCH##origin/}"
+echo "Current branch: $branch"
+
+if [[ "$branch" == "master" ]]; then
+    DOCKER_TAG="latest"
+else
+    DOCKER_TAG="stable"
+fi
+
+# Start the build
+echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_TAG"
+echo "--------------------------------------------------------"
+echo
+cmd="docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG --build-arg BRANCH=$branch ."
+
+echo ${cmd}
+${cmd}
+
+
+# list the images
+echo "Available images are:"
+docker images
+
+# Push image to Dockerhub
+if [[ "$PUSH_IMAGE" == "true" ]]; then
+    echo "Pushing $DOCKER_REPO_NAME:$DOCKER_TAG to the docker registry..."
+    echo "--------------------------------------------------------"
+    echo
+    docker push $DOCKER_REPO_NAME:$DOCKER_TAG
+fi
diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml
new file mode 100644
index 000000000..0df0ddf6f
--- /dev/null
+++ b/jjb/releng/opnfv-docker.yml
@@ -0,0 +1,82 @@
+##############################################
+# job configuration for docker build and push
+##############################################
+
+- project:
+
+    name: opnfv-docker
+
+    master: &master
+        stream: master
+        branch: '{stream}'
+        disabled: false
+    colorado: &colorado
+        stream: colorado
+        branch: 'stable/{stream}'
+        disabled: false
+
+    project:
+        # projects with jobs for master
+        - 'bottlenecks':
+            <<: *master
+        - 'cperf':
+            <<: *master
+        - 'dovetail':
+            <<: *master
+        - 'functest':
+            <<: *master
+        - 'qtip':
+            <<: *master
+        - 'storperf':
+            <<: *master
+        - 'yardstick':
+            <<: *master
+        # projects with jobs for stable
+        - 'bottlenecks':
+            <<: *colorado
+        - 'functest':
+            <<: *colorado
+        - 'storperf':
+            <<: *colorado
+        - 'yardstick':
+            <<: *colorado
+
+    jobs:
+        - '{project}-docker-build-push-{stream}'
+
+
+########################
+# job templates
+########################
+- job-template:
+    name: '{project}-docker-build-push-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - 'opnfv-build-ubuntu-defaults'
+        - string:
+            name: PUSH_IMAGE
+            default: "true"
+            description: "To enable/disable pushing the image to Dockerhub."
+        - string:
+            name: DOCKER_REPO_NAME
+            default: "opnfv/{project}"
+            description: "Dockerhub repo to be pushed to."
+
+    scm:
+        - git-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: ''
+            branch: '{branch}'
+
+    builders:
+        - shell:
+            !include-raw-escape: ./opnfv-docker.sh
+
+    triggers:
+        - pollscm:
+            cron: "*/30 * * * *"
+
diff --git a/jjb/releng/opnfv-docs.yml b/jjb/releng/opnfv-docs.yml
new file mode 100644
index 000000000..442e8ea71
--- /dev/null
+++ b/jjb/releng/opnfv-docs.yml
@@ -0,0 +1,120 @@
+########################
+# Job configuration for opnfv-docs
+########################
+- project:
+
+    name: opnfv-docs
+
+    project: opnfv-docs
+
+    jobs:
+        - 'opnfv-docs-verify-{stream}'
+        - 'opnfv-docs-merge-{stream}'
+
+    stream:
+        - master:
+            branch: '{stream}'
+            doc-version: ''
+            gs-pathname: ''
+            disabled: false
+        - colorado:
+            branch: 'stable/{stream}'
+            doc-version: '3.0'
+            gs-pathname: '/{stream}/{doc-version}'
+            disabled: false
+
+########################
+# job templates
+########################
+
+- job-template:
+    name: 'opnfv-docs-verify-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    parameters:
+        - project-parameter:
+            project: $GERRIT_PROJECT
+        - gerrit-parameter:
+            branch: '{branch}'
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: '$GERRIT_REFSPEC'
+            choosing-strategy: 'gerrit'
+
+    triggers:
+        - gerrit:
+            trigger-on:
+                - patchset-created-event:
+                    exclude-drafts: 'false'
+                    exclude-trivial-rebase: 'false'
+                    exclude-no-code-change: 'false'
+                - draft-published-event
+                - comment-added-contains-event:
+                    comment-contains-value: 'recheck'
+                - comment-added-contains-event:
+                    comment-contains-value: 'reverify'
+            projects:
+                - project-compare-type: 'ANT'
+                  project-pattern: '*'
+                  branches:
+                      - branch-compare-type: 'ANT'
+                        branch-pattern: '**/{branch}'
+                  file-paths:
+                      - compare-type: ANT
+                        pattern: docs/**
+                      - compare-type: ANT
+                        pattern: yardstick/**
+
+    builders:
+        - upload-review-docs
+
+- job-template:
+    name: 'opnfv-docs-merge-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    parameters:
+        - project-parameter:
+            project: $GERRIT_PROJECT
+        - gerrit-parameter:
+            branch: '{branch}'
+        - string:
+            name: GS_URL
+            default: '$GS_BASE{gs-pathname}'
+            description: "Directory where the build artifact will be located upon the completion of the build."
+        - string:
+            name: GERRIT_REFSPEC
+            default: 'refs/heads/{branch}'
+            description: "JJB configured GERRIT_REFSPEC parameter"
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: '$GERRIT_REFSPEC'
+            choosing-strategy: 'gerrit'
+
+    triggers:
+        - gerrit:
+            trigger-on:
+                - change-merged-event
+                - comment-added-contains-event:
+                    comment-contains-value: 'remerge'
+                - comment-added-contains-event:
+                    comment-contains-value: 'rebuild docs'
+            projects:
+                - project-compare-type: 'ANT'
+                  project-pattern: '*'
+                  branches:
+                      - branch-compare-type: 'ANT'
+                        branch-pattern: '**/{branch}'
+                  file-paths:
+                      - compare-type: ANT
+                        pattern: docs/**
+                      - compare-type: ANT
+                        pattern: yardstick/**
+
+    builders:
+        - upload-merged-docs
diff --git a/jjb/releng/opnfv-lint.yml b/jjb/releng/opnfv-lint.yml
new file mode 100644
index 000000000..652f28292
--- /dev/null
+++ b/jjb/releng/opnfv-lint.yml
@@ -0,0 +1,68 @@
+########################
+# Job configuration for opnfv-lint
+########################
+- project:
+
+    name: opnfv-lint
+
+    project: opnfv-lint
+
+    jobs:
+        - 'opnfv-lint-verify-{stream}'
+
+    stream:
+        - master:
+            branch: '{stream}'
+            gs-pathname: ''
+            disabled: false
+        - colorado:
+            branch: 'stable/{stream}'
+            gs-pathname: '/{stream}'
+            disabled: false
+
+########################
+# job templates
+########################
+
+- job-template:
+    name: 'opnfv-lint-verify-{stream}'
+
+    disabled: '{obj:disabled}'
+
+    parameters:
+        - project-parameter:
+            project: $GERRIT_PROJECT
+        - gerrit-parameter:
+            branch: '{branch}'
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: '$GERRIT_REFSPEC'
+            choosing-strategy: 'gerrit'
+
+    triggers:
+        - gerrit:
+            trigger-on:
+                - patchset-created-event:
+                    exclude-drafts: 'false'
+                    exclude-trivial-rebase: 'false'
+                    exclude-no-code-change: 'false'
+                - draft-published-event
+                - comment-added-contains-event:
+                    comment-contains-value: 'recheck'
+                - comment-added-contains-event:
+                    comment-contains-value: 'reverify'
+            projects:
+              - project-compare-type: 'REG_EXP'
+                project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+                file-paths:
+                  - compare-type: ANT
+                    pattern: '**/*.py'
+
+    builders:
+        - lint-python-code
+        - report-lint-result-to-gerrit
diff --git a/jjb/releng/opnfv-utils.yml b/jjb/releng/opnfv-utils.yml
new file mode 100644
index 000000000..717bb3cbc
--- /dev/null
+++ b/jjb/releng/opnfv-utils.yml
@@ -0,0 +1,39 @@
+- project:
+
+    name: opnfv-utils
+
+    jobs:
+        - 'prune-docker-images'
+########################
+# job templates
+########################
+- job-template:
+    name: 'prune-docker-images'
+
+    disabled: false
+
+    concurrent: true
+
+    parameters:
+        - node:
+            name: SLAVE_NAME
+            description: Slaves to prune docker images
+            default-slaves:
+                - arm-build2
+                - ericsson-build4
+                - ericsson-build5
+                - lf-build2
+            allowed-multiselect: true
+            ignore-offline-nodes: true
+
+    builders:
+        - description-setter:
+            description: "Built on $NODE_NAME"
+        - shell: |
+            #!/bin/bash
+
+            (docker ps -q; docker ps -aq) | sort | uniq -u | xargs --no-run-if-empty docker rm
+            docker images -f dangling=true -q | xargs --no-run-if-empty docker rmi
+
+    triggers:
+        - timed: '@midnight'
diff --git a/jjb/releng/verify-releng.sh b/jjb/releng/verify-releng.sh
index 5c2dbff7a..682a8beda 100755
--- a/jjb/releng/verify-releng.sh
+++ b/jjb/releng/verify-releng.sh
@@ -8,7 +8,7 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
 #test for non-ascii characters, these can pass the test and end up breaking things in production
-for x in $(find . -name *\.yml); do
+for x in $(find . -name *\.yml -or -name *\.yaml); do
 
   if LC_ALL=C grep -q '[^[:print:][:space:]]' "$x"; then
     echo "file "$x" contains non-ascii characters"