Verify and Create Tags from Release File

Updates the releases verify and merge jobs to check release files for
the following:

 - commits exist
 - commits are on stable-branch

If both of these criteria are met and the patch is merged, the repos
will be tagged at the specified commits and pushed.

If a repo has already been tagged but not on stable-branch the job will
not exit, but will warn that the commit doesn't exist on the correct
branch. This is because tags should never be removed.

This adds an additional script 'repos.py' that provides an quick
interface for pulling out information from release files, along with the
script 'release-status.sh' for checking the status of tags.

NOTE: The branch creation script has been disabled until it can be reworked.

Change-Id: I498bc74f20aa50d2bd321771f20a77905b246399
Signed-off-by: Trevor Bramwell <tbramwell@linuxfoundation.org>

6 files changed, 178 insertions, 32 deletions

diff --git a/jjb/releng/releng-release-verify.sh b/jjb/releng/releng-release-create-venv.sh
index c1262e2c9..0d5635b59 100644
--- a/jjb/releng/releng-release-verify.sh
+++ b/jjb/releng/releng-release-create-venv.sh
@@ -7,21 +7,12 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
-set -xe
+set -e -o pipefail
+echo "---> Create virtualenv"
 
-# Activate virtualenv, supressing shellcheck warning
-# shellcheck source=/dev/null
-. $WORKSPACE/venv/bin/activate
+sudo pip install virtualenv
+virtualenv $WORKSPACE/venv
+# shellcheck source=$WORKSPACE/venv/bin/activate disable=SC1091
+source $WORKSPACE/venv/bin/activate
+pip install --upgrade pip
 pip install -r releases/scripts/requirements.txt
-
-STREAM=${STREAM:-'nostream'}
-RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM")
-
-# TODO: The create_branch.py should be refactored so it can be used here
-# to verify the commit exists that is being added, along with
-# jjb/<project>
-for release_file in $RELEASE_FILES; do
-    python releases/scripts/verify_schema.py \
-    -s releases/schema.yaml \
-    -y $release_file
-done
diff --git a/jjb/releng/releng-release-jobs.yaml b/jjb/releng/releng-release-jobs.yaml
index e31320b81..0c059ee9b 100644
--- a/jjb/releng/releng-release-jobs.yaml
+++ b/jjb/releng/releng-release-jobs.yaml
@@ -10,6 +10,8 @@
 - project:
     name: releng-release-jobs
 
+    build-node: 'opnfv-build'
+
     stream:
       - fraser
 
@@ -23,6 +25,7 @@
     name: 'releng-release-{stream}-verify'
 
     parameters:
+      - '{build-node}-defaults'
       - stream-parameter:
           stream: '{stream}'
       - project-parameter:
@@ -59,9 +62,9 @@
                   pattern: 'releases/scripts/verify_schema.py'
 
     builders:
-      - create-virtualenv
-      - shell:
-          !include-raw-escape: releng-release-verify.sh
+      - shell: !include-raw-escape:
+          - releng-release-create-venv.sh
+          - releng-release-tagging.sh
 
     publishers:
       - email-jenkins-admins-on-failure
@@ -82,6 +85,12 @@
       - project-parameter:
           project: '{project}'
           branch: 'master'
+      # Override GIT_BASE so we can send patches back to Gerrit and
+      # modify repos
+      - string:
+          name: GIT_BASE
+          default: ssh://$USER@gerrit.opnfv.org:29418/$PROJECT
+          description: 'Git URL to use on this Jenkins Slave'
 
     scm:
       - git-scm-gerrit
@@ -93,9 +102,10 @@
           files: 'releases/{stream}/*'
 
     builders:
-      - create-virtualenv
-      - shell:
-          !include-raw-escape: releng-release-create-branch.sh
+      - shell: !include-raw-escape:
+          - releng-release-create-venv.sh
+          - releng-release-tagging.sh
+          # - releng-release-create-branch.sh
 
     publishers:
       - email-jenkins-admins-on-failure
@@ -107,13 +117,3 @@
           name: STREAM
           default: '{stream}'
           description: "OPNFV Stable Stream"
-
-- builder:
-    name: create-virtualenv
-    builders:
-      - shell: |
-          #!/bin/bash
-          sudo pip install virtualenv
-          virtualenv $WORKSPACE/venv
-          . $WORKSPACE/venv/bin/activate
-          pip install --upgrade pip
diff --git a/jjb/releng/releng-release-tagging.sh b/jjb/releng/releng-release-tagging.sh
new file mode 100644
index 000000000..10c0cc8c9
--- /dev/null
+++ b/jjb/releng/releng-release-tagging.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -e -o pipefail
+
+GIT_URL=${GIT_URL:-https://gerrit.opnfv.org/gerrit}
+STREAM=${STREAM:-'nostream'}
+RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM")
+
+echo "--> Verifying $RELEASE_FILES."
+for release_file in $RELEASE_FILES; do
+    # Verify the release file schema
+    python releases/scripts/verify_schema.py \
+    -s releases/schema.yaml \
+    -y $release_file
+
+    # Verify tag for each repo exist and are attached to commits on stable-branch
+    while read -r repo tag ref
+    do
+      echo "--> Cloning $repo"
+      if [ ! -d $repo ]; then
+          git clone $GIT_URL/$repo.git $repo
+      fi
+      pushd $repo &> /dev/null
+
+      echo "--> Checking for tag: $tag"
+      if ! (git tag -l | grep $tag &> /dev/null); then
+          echo "$tag does not exist"
+          TAG_EXISTS=false
+      else
+          git cat-file tag $tag
+          TAG_EXISTS=true
+      fi
+
+      echo "--> Checking if $ref is on stable/$STREAM"
+      if ! (git branch -a --contains $ref | grep "stable/$STREAM"); then
+          echo "--> ERROR: $ref for $repo is not on stable/$STREAM!"
+          # If the tag exists but is on the wrong ref, there's nothing
+          # we can do. But if the tag neither exists nor is on the
+          # correct branch we need to fail the verification.
+          if [ $TAG_EXISTS = false ]; then
+              exit 1
+          fi
+      else
+          if [[ $TAG_EXISTS = false && "$JOB_NAME" =~ "merge" ]]; then
+              # If the tag doesn't exist and we're in a merge job,
+              # everything has been verified up to this point and we
+              # are ready to create the tag.
+              git config --global user.name "jenkins-ci"
+              git config --global user.email "jenkins-opnfv-ci@opnfv.org"
+              echo "--> Creating $tag tag for $repo at $ref"
+              git tag -am "$tag" $tag $ref
+              echo "--> Pushing tag"
+              echo "[noop] git push origin $tag"
+          else
+              # For non-merge jobs just output the ref info.
+              git show -s --format="%h %s %d" $ref
+          fi
+      fi
+
+      popd &> /dev/null
+      echo "--> Done verifing $repo"
+    done < <(python releases/scripts/repos.py -f $release_file)
+done
diff --git a/releases/fraser/armband.yml b/releases/fraser/armband.yaml
index 66afd10da..66afd10da 100644
--- a/releases/fraser/armband.yml
+++ b/releases/fraser/armband.yaml
diff --git a/releases/scripts/release-status.sh b/releases/scripts/release-status.sh
new file mode 100644
index 000000000..da66bc936
--- /dev/null
+++ b/releases/scripts/release-status.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -o pipefail
+
+TAG="${TAG:-opnfv-6.0.0}"
+RELEASE="${RELEASE:-fraser}"
+
+for project in releases/$RELEASE/*; do
+    python releases/scripts/repos.py -n -f $project >> repos.txt
+done
+
+while read -r repo
+do
+    tag="$(git ls-remote "https://gerrit.opnfv.org/gerrit/$repo.git" "refs/tags/$TAG")"
+    echo "$repo $tag"
+done < repos.txt
+
+# rm repos.txt
diff --git a/releases/scripts/repos.py b/releases/scripts/repos.py
new file mode 100644
index 000000000..0ded0207b
--- /dev/null
+++ b/releases/scripts/repos.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python2
+# SPDX-License-Identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+"""
+List Release Repos
+"""
+
+import argparse
+import yaml
+
+
+def main():
+    """Given a release yamlfile list the repos it contains"""
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--file', '-f',
+                        type=argparse.FileType('r'),
+                        required=True)
+    parser.add_argument('--names', '-n',
+                        action='store_true',
+                        default=False,
+                        help="Only print the names of repos, "
+                             "not their SHAs")
+    parser.add_argument('--release', '-r',
+                        type=str,
+                        help="Only print"
+                             "SHAs for the specified release")
+    args = parser.parse_args()
+
+    project = yaml.safe_load(args.file)
+
+    list_repos(project, args)
+
+
+def list_repos(project, args):
+    """List repositories in the project file"""
+
+    lookup = project.get('releases', [])
+    if 'releases' not in project:
+        exit(0)
+
+    for item in lookup:
+        repo, ref = next(iter(item['location'].items()))
+        if args.names:
+            print(repo)
+        elif args.release and item['version'] == args.release:
+            print("%s %s" % (repo, ref))
+        elif not args.release:
+            # Print all releases
+            print("%s %s %s" % (repo, item['version'], ref))
+
+
+if __name__ == "__main__":
+    main()