Merge "Verify and Create Tags from Release File"

6 files changed, 178 insertions, 32 deletions

diff --git a/jjb/releng/releng-release-verify.sh b/jjb/releng/releng-release-create-venv.sh
index c1262e2c9..0d5635b59 100644
--- a/jjb/releng/releng-release-verify.sh
+++ b/jjb/releng/releng-release-create-venv.sh
@@ -7,21 +7,12 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
-set -xe
+set -e -o pipefail
+echo "---> Create virtualenv"
 
-# Activate virtualenv, supressing shellcheck warning
-# shellcheck source=/dev/null
-. $WORKSPACE/venv/bin/activate
+sudo pip install virtualenv
+virtualenv $WORKSPACE/venv
+# shellcheck source=$WORKSPACE/venv/bin/activate disable=SC1091
+source $WORKSPACE/venv/bin/activate
+pip install --upgrade pip
 pip install -r releases/scripts/requirements.txt
-
-STREAM=${STREAM:-'nostream'}
-RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM")
-
-# TODO: The create_branch.py should be refactored so it can be used here
-# to verify the commit exists that is being added, along with
-# jjb/<project>
-for release_file in $RELEASE_FILES; do
-    python releases/scripts/verify_schema.py \
-    -s releases/schema.yaml \
-    -y $release_file
-done
diff --git a/jjb/releng/releng-release-jobs.yaml b/jjb/releng/releng-release-jobs.yaml
index e31320b81..0c059ee9b 100644
--- a/jjb/releng/releng-release-jobs.yaml
+++ b/jjb/releng/releng-release-jobs.yaml
@@ -10,6 +10,8 @@
 - project:
     name: releng-release-jobs
 
+    build-node: 'opnfv-build'
+
     stream:
       - fraser
 
@@ -23,6 +25,7 @@
     name: 'releng-release-{stream}-verify'
 
     parameters:
+      - '{build-node}-defaults'
       - stream-parameter:
           stream: '{stream}'
       - project-parameter:
@@ -59,9 +62,9 @@
                   pattern: 'releases/scripts/verify_schema.py'
 
     builders:
-      - create-virtualenv
-      - shell:
-          !include-raw-escape: releng-release-verify.sh
+      - shell: !include-raw-escape:
+          - releng-release-create-venv.sh
+          - releng-release-tagging.sh
 
     publishers:
       - email-jenkins-admins-on-failure
@@ -82,6 +85,12 @@
       - project-parameter:
           project: '{project}'
           branch: 'master'
+      # Override GIT_BASE so we can send patches back to Gerrit and
+      # modify repos
+      - string:
+          name: GIT_BASE
+          default: ssh://$USER@gerrit.opnfv.org:29418/$PROJECT
+          description: 'Git URL to use on this Jenkins Slave'
 
     scm:
       - git-scm-gerrit
@@ -93,9 +102,10 @@
           files: 'releases/{stream}/*'
 
     builders:
-      - create-virtualenv
-      - shell:
-          !include-raw-escape: releng-release-create-branch.sh
+      - shell: !include-raw-escape:
+          - releng-release-create-venv.sh
+          - releng-release-tagging.sh
+          # - releng-release-create-branch.sh
 
     publishers:
       - email-jenkins-admins-on-failure
@@ -107,13 +117,3 @@
           name: STREAM
           default: '{stream}'
           description: "OPNFV Stable Stream"
-
-- builder:
-    name: create-virtualenv
-    builders:
-      - shell: |
-          #!/bin/bash
-          sudo pip install virtualenv
-          virtualenv $WORKSPACE/venv
-          . $WORKSPACE/venv/bin/activate
-          pip install --upgrade pip
diff --git a/jjb/releng/releng-release-tagging.sh b/jjb/releng/releng-release-tagging.sh
new file mode 100644
index 000000000..10c0cc8c9
--- /dev/null
+++ b/jjb/releng/releng-release-tagging.sh
@@ -0,0 +1,70 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -e -o pipefail
+
+GIT_URL=${GIT_URL:-https://gerrit.opnfv.org/gerrit}
+STREAM=${STREAM:-'nostream'}
+RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM")
+
+echo "--> Verifying $RELEASE_FILES."
+for release_file in $RELEASE_FILES; do
+    # Verify the release file schema
+    python releases/scripts/verify_schema.py \
+    -s releases/schema.yaml \
+    -y $release_file
+
+    # Verify tag for each repo exist and are attached to commits on stable-branch
+    while read -r repo tag ref
+    do
+      echo "--> Cloning $repo"
+      if [ ! -d $repo ]; then
+          git clone $GIT_URL/$repo.git $repo
+      fi
+      pushd $repo &> /dev/null
+
+      echo "--> Checking for tag: $tag"
+      if ! (git tag -l | grep $tag &> /dev/null); then
+          echo "$tag does not exist"
+          TAG_EXISTS=false
+      else
+          git cat-file tag $tag
+          TAG_EXISTS=true
+      fi
+
+      echo "--> Checking if $ref is on stable/$STREAM"
+      if ! (git branch -a --contains $ref | grep "stable/$STREAM"); then
+          echo "--> ERROR: $ref for $repo is not on stable/$STREAM!"
+          # If the tag exists but is on the wrong ref, there's nothing
+          # we can do. But if the tag neither exists nor is on the
+          # correct branch we need to fail the verification.
+          if [ $TAG_EXISTS = false ]; then
+              exit 1
+          fi
+      else
+          if [[ $TAG_EXISTS = false && "$JOB_NAME" =~ "merge" ]]; then
+              # If the tag doesn't exist and we're in a merge job,
+              # everything has been verified up to this point and we
+              # are ready to create the tag.
+              git config --global user.name "jenkins-ci"
+              git config --global user.email "jenkins-opnfv-ci@opnfv.org"
+              echo "--> Creating $tag tag for $repo at $ref"
+              git tag -am "$tag" $tag $ref
+              echo "--> Pushing tag"
+              echo "[noop] git push origin $tag"
+          else
+              # For non-merge jobs just output the ref info.
+              git show -s --format="%h %s %d" $ref
+          fi
+      fi
+
+      popd &> /dev/null
+      echo "--> Done verifing $repo"
+    done < <(python releases/scripts/repos.py -f $release_file)
+done
diff --git a/releases/fraser/armband.yml b/releases/fraser/armband.yaml
index 66afd10da..66afd10da 100644
--- a/releases/fraser/armband.yml
+++ b/releases/fraser/armband.yaml
diff --git a/releases/scripts/release-status.sh b/releases/scripts/release-status.sh
new file mode 100644
index 000000000..da66bc936
--- /dev/null
+++ b/releases/scripts/release-status.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -o pipefail
+
+TAG="${TAG:-opnfv-6.0.0}"
+RELEASE="${RELEASE:-fraser}"
+
+for project in releases/$RELEASE/*; do
+    python releases/scripts/repos.py -n -f $project >> repos.txt
+done
+
+while read -r repo
+do
+    tag="$(git ls-remote "https://gerrit.opnfv.org/gerrit/$repo.git" "refs/tags/$TAG")"
+    echo "$repo $tag"
+done < repos.txt
+
+# rm repos.txt
diff --git a/releases/scripts/repos.py b/releases/scripts/repos.py
new file mode 100644
index 000000000..0ded0207b
--- /dev/null
+++ b/releases/scripts/repos.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python2
+# SPDX-License-Identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+"""
+List Release Repos
+"""
+
+import argparse
+import yaml
+
+
+def main():
+    """Given a release yamlfile list the repos it contains"""
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--file', '-f',
+                        type=argparse.FileType('r'),
+                        required=True)
+    parser.add_argument('--names', '-n',
+                        action='store_true',
+                        default=False,
+                        help="Only print the names of repos, "
+                             "not their SHAs")
+    parser.add_argument('--release', '-r',
+                        type=str,
+                        help="Only print"
+                             "SHAs for the specified release")
+    args = parser.parse_args()
+
+    project = yaml.safe_load(args.file)
+
+    list_repos(project, args)
+
+
+def list_repos(project, args):
+    """List repositories in the project file"""
+
+    lookup = project.get('releases', [])
+    if 'releases' not in project:
+        exit(0)
+
+    for item in lookup:
+        repo, ref = next(iter(item['location'].items()))
+        if args.names:
+            print(repo)
+        elif args.release and item['version'] == args.release:
+            print("%s %s" % (repo, ref))
+        elif not args.release:
+            # Print all releases
+            print("%s %s %s" % (repo, item['version'], ref))
+
+
+if __name__ == "__main__":
+    main()