Rework creating tags & branches from release files

Old method could not properly handle both branching and
tagging on changes to the release file.

Proposed method:
jjb calls:
jjb/releng/branch-or-tag.sh
for both verify and merge jobs.

branch-or-tag determins nature of the change to
"releases/branch/project.yaml"
and calls
 releng-release-tagging.sh
or
 releng-release-create-branch.sh

The scripts above handle both verify and merge operations

based on
  if [[ $TAG_EXISTS = false && "$JOB_NAME" =~ "merge" ]];
and
  if [[ $REF_EXISTS = true && "$JOB_NAME" =~ "merge" ]]; then

both scripts check that the ref exits in verfiy and merge stages.

if releng-release-create-branch.sh creates a branch it then also
calls releases/scripts/create_jobs.py

Change-Id: Ieb99de5e4df100af59ecc818f52b0831383b2b62
Signed-off-by: Aric Gardner <agardner@linuxfoundation.org>

4 files changed, 84 insertions, 26 deletions

diff --git a/jjb/releng/branch-or-tag.sh b/jjb/releng/branch-or-tag.sh
new file mode 100755
index 000000000..0fdb24fca
--- /dev/null
+++ b/jjb/releng/branch-or-tag.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+set -e -o pipefail
+
+GIT_URL=${GIT_URL:-https://gerrit.opnfv.org/gerrit}
+STREAM=${STREAM:-'nostream'}
+RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM")
+
+echo "--> Verifying $RELEASE_FILES."
+for release_file in $RELEASE_FILES; do
+    # Verify the release file schema
+    python releases/scripts/verify_schema.py \
+    -s releases/schema.yaml \
+    -y $release_file
+done
+
+for release_file in $RELEASE_FILES; do
+    while read -r repo branch ref; do
+        echo "$repo" "$branch" "$ref"
+        unset branch_actual
+        branch_actual="$(git ls-remote "https://gerrit.opnfv.org/gerrit/$repo.git" "refs/heads/$branch" | awk '{print $1}')"
+
+        if [ -n "$branch_actual" ]; then
+            echo "$repo refs/heads/$branch already exists at $branch_actual"
+            echo "RUN releng-release-create-venv.sh"
+            source jjb/releng/releng-release-tagging.sh
+        else
+            echo "This is a branching job"
+            source jjb/releng/releng-release-create-branch.sh
+        fi
+
+    done  < <(python releases/scripts/repos.py -b -f "$release_file")
+done
diff --git a/jjb/releng/releng-release-create-branch.sh b/jjb/releng/releng-release-create-branch.sh
index 663ff19e7..7e91d5ace 100644
--- a/jjb/releng/releng-release-create-branch.sh
+++ b/jjb/releng/releng-release-create-branch.sh
@@ -9,6 +9,10 @@
 ##############################################################################
 set -xe
 
+GIT_URL=${GIT_URL:-https://gerrit.opnfv.org/gerrit}
+STREAM=${STREAM:-'nostream'}
+RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM")
+
 # Configure the git user/email as we'll be pushing up changes
 git config --global user.name "jenkins-ci"
 git config --global user.email "jenkins-opnfv-ci@opnfv.org"
@@ -17,34 +21,53 @@ git config --global user.email "jenkins-opnfv-ci@opnfv.org"
 curl -kLo .git/hooks/commit-msg https://gerrit.opnfv.org/gerrit/tools/hooks/commit-msg
 chmod +x .git/hooks/commit-msg
 
-# Activate virtualenv, supressing shellcheck warning
-# shellcheck source=/dev/null
-. $WORKSPACE/venv/bin/activate
-pip install -r releases/scripts/requirements.txt
+clone_repo(){
+echo "--> Cloning $repo"
+if [ ! -d $repo ]; then
+    git clone $GIT_URL/$repo.git $repo
+fi
+}
 
-STREAM=${STREAM:-'nostream'}
-RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM")
+check_if_ref_exists(){
+clone_repo
+cd "$repo"
+if git rev-list refs/heads/master | grep "$ref"; then
+  echo "$ref exists"
+  REF_EXISTS=true
+  cd -
+else
+  echo "$ref Does not exist please submit a valid ref for branching"
+  exit 1
+fi
+}
 
-for release_file in $RELEASE_FILES; do
+run_merge(){
+unset NEW_FILES
+if [[ $REF_EXISTS = true && "$JOB_NAME" =~ "merge" ]]; then
+  ssh -n -f -p 29418 gerrit.opnfv.org gerrit create-branch "$repo" "$branch" "$ref"
+  python releases/scripts/create_jobs.py -f $release_file
+  NEW_FILES=$(git status --porcelain --untracked=no | cut -c4-)
+fi
+if [ -n "$NEW_FILES" ]; then
+  git add $NEW_FILES
+  git commit -sm "Create Stable Branch Jobs for $(basename $release_file .yaml)"
+  git push origin HEAD:refs/for/master
+fi
+}
 
+main(){
+for release_file in $RELEASE_FILES; do
     while read -r repo branch ref; do
-
         echo "$repo" "$branch" "$ref"
         branches="$(git ls-remote "https://gerrit.opnfv.org/gerrit/$repo.git" "refs/heads/$branch")"
-
         if ! [ -z "$branches" ]; then
             echo "refs/heads/$branch already exists at $ref ($branches)"
         else
-            ssh -n -f -p 29418 gerrit.opnfv.org gerrit create-branch "$repo" "$branch" "$ref"
+            run_merge
         fi
-
     done < <(python releases/scripts/repos.py -b -f "$release_file")
-
-    python releases/scripts/create_jobs.py -f $release_file
-    NEW_FILES=$(git status --porcelain --untracked=no | cut -c4-)
-    if [ -n "$NEW_FILES" ]; then
-      git add $NEW_FILES
-      git commit -sm "Create Stable Branch Jobs for $(basename $release_file .yaml)"
-      git push origin HEAD:refs/for/master
-    fi
 done
+}
+
+check_if_ref_exists
+main
diff --git a/jjb/releng/releng-release-jobs.yaml b/jjb/releng/releng-release-jobs.yaml
index 3136d7855..d79771fea 100644
--- a/jjb/releng/releng-release-jobs.yaml
+++ b/jjb/releng/releng-release-jobs.yaml
@@ -66,7 +66,7 @@
     builders:
       - shell: !include-raw-escape:
           - releng-release-create-venv.sh
-          - releng-release-tagging.sh
+          - branch-or-tag.sh
 
     publishers:
       - email-jenkins-admins-on-failure
@@ -106,8 +106,7 @@
     builders:
       - shell: !include-raw-escape:
           - releng-release-create-venv.sh
-          - releng-release-tagging.sh
-          - releng-release-create-branch.sh
+          - branch-or-tag.sh
 
     publishers:
       - email-jenkins-admins-on-failure
diff --git a/jjb/releng/releng-release-tagging.sh b/jjb/releng/releng-release-tagging.sh
index f8cf9c8ea..88927e54d 100644
--- a/jjb/releng/releng-release-tagging.sh
+++ b/jjb/releng/releng-release-tagging.sh
@@ -15,10 +15,6 @@ RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM")
 
 echo "--> Verifying $RELEASE_FILES."
 for release_file in $RELEASE_FILES; do
-    # Verify the release file schema
-    python releases/scripts/verify_schema.py \
-    -s releases/schema.yaml \
-    -y $release_file
 
     # Verify tag for each repo exist and are attached to commits on stable-branch
     while read -r repo tag ref