Moving to forks on gerrit.opnfv.org

Migrating off of github onto opnfv.org to host our forked
projects that are side loaded into our builds

apex-tripleo-heat-templates: Ic65cfeee4a55e993629f831c8c9d9addf6f3dff4
apex-puppet-tripleo: If498c41d706c8f14a5b0bbee64cb4d26cd78c2d0
apex-os-net-config: I5281a57640f388e984b061702362f9c82d08da78

Change-Id: Ieb5cf293ad06d90fce7a9467e32ac0f2d8731a0a
Signed-off-by: Dan Radez <dradez@redhat.com>
Signed-off-by: Feng Pan <fpan@redhat.com>

7 files changed, 137 insertions, 69 deletions

diff --git a/build/functions.sh b/build/functions.sh
deleted file mode 100644
index becc0df1..00000000
--- a/build/functions.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-##############################################################################
-# Copyright (c) 2016 Dan Radez (Red Hat) and others.
-#
-# All rights reserved. This program and the accompanying materials
-# are made available under the terms of the Apache License, Version 2.0
-# which accompanies this distribution, and is available at
-# http://www.apache.org/licenses/LICENSE-2.0
-##############################################################################
-
-clone_fork () {
-    # ARG 1: opnfv-tht or opnfv-puppet-tripleo
-    echo "Cloning $1"
-
-    # Use apex tripleo-heat-templates fork
-    local ghcreds=""
-    local pr_num=""
-    local ref="stable/euphrates"
-    local repo="https://github.com/trozet/$1"
-
-    if git log -1 | grep "${1}-pr:" | grep -o '[0-9]*'; then
-      pr_num=$(git log -1 | grep "${1}-pr:" | grep -o '[0-9]*')
-    fi
-
-    if [ "$pr_num" != "" ]; then
-      echo "Using pull request $pr_num from $repo"
-      # Source credentials since we are rate limited to 60/day
-      if [ -f ~/.githubcreds ]; then
-        source ~/.githubcreds
-        ghcreds=" -u $GHUSERNAME:$GHACCESSTOKEN"
-      fi
-
-      PR=$(curl $ghcreds https://api.github.com/repos/trozet/$1/pulls/$pr_num)
-
-      # Do not pull from merged branches
-      MERGED=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['merged']")
-      if [ "$MERGED" == "False" ]; then
-        ref=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['head']['ref']")
-        echo "Setting GitHub Ref to: $REF"
-        repo=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['head']['repo']['clone_url']")
-        echo "Setting GitHub URL to: $repo"
-      fi
-    fi
-
-    rm -rf $1
-    git clone $repo -b $ref $1
-}
diff --git a/build/overcloud-full.sh b/build/overcloud-full.sh
index adcf4df6..0f7318a6 100755
--- a/build/overcloud-full.sh
+++ b/build/overcloud-full.sh
@@ -10,7 +10,6 @@
 set -xe
 source ./cache.sh
 source ./variables.sh
-source ./functions.sh
 
 populate_cache "$rdo_images_uri/overcloud-full.tar"
 
@@ -25,15 +24,15 @@ mv -f ${BUILD_DIR}/overcloud-full.qcow2 ${BUILD_DIR}/overcloud-full_build.qcow2
 pushd ${BUILD_DIR} > /dev/null
 
 # prep opnfv-puppet-tripleo for undercloud
-clone_fork opnfv-puppet-tripleo
-pushd opnfv-puppet-tripleo > /dev/null
-git archive --format=tar.gz --prefix=tripleo/ HEAD > ${BUILD_DIR}/opnfv-puppet-tripleo.tar.gz
+python3 -B $BUILD_UTILS clone-fork -r apex-puppet-tripleo
+pushd apex-puppet-tripleo > /dev/null
+git archive --format=tar.gz --prefix=tripleo/ HEAD > ${BUILD_DIR}/apex-puppet-tripleo.tar.gz
 popd > /dev/null
 
 # download customized os-net-config
-clone_fork os-net-config
-pushd os-net-config/os_net_config > /dev/null
-git archive --format=tar.gz --prefix=os_net_config/ HEAD > ${BUILD_DIR}/os-net-config.tar.gz
+python3 -B $BUILD_UTILS clone-fork -r apex-os-net-config
+pushd apex-os-net-config/os_net_config > /dev/null
+git archive --format=tar.gz --prefix=os_net_config/ HEAD > ${BUILD_DIR}/apex-os-net-config.tar.gz
 popd > /dev/null
 
 dpdk_pkg_str=''
@@ -77,7 +76,7 @@ done
 qemu-img resize overcloud-full_build.qcow2 +900MB
 
 # expand file system to max disk size
-# installing forked opnfv-puppet-tripleo
+# installing forked apex-puppet-tripleo
 # upload dpdk rpms but do not install
 # install fd.io yum repo and packages
 # upload puppet fdio
@@ -85,10 +84,10 @@ qemu-img resize overcloud-full_build.qcow2 +900MB
 # upload the rt_kvm kernel
 LIBGUESTFS_BACKEND=direct virt-customize \
     --run-command "xfs_growfs /dev/sda" \
-    --upload ${BUILD_DIR}/opnfv-puppet-tripleo.tar.gz:/etc/puppet/modules \
-    --run-command "cd /etc/puppet/modules && rm -rf tripleo && tar xzf opnfv-puppet-tripleo.tar.gz" \
-    --upload ${BUILD_DIR}/os-net-config.tar.gz:/usr/lib/python2.7/site-packages \
-    --run-command "cd /usr/lib/python2.7/site-packages/ && rm -rf os_net_config && tar xzf os-net-config.tar.gz" \
+    --upload ${BUILD_DIR}/apex-puppet-tripleo.tar.gz:/etc/puppet/modules \
+    --run-command "cd /etc/puppet/modules && rm -rf tripleo && tar xzf apex-puppet-tripleo.tar.gz" \
+    --upload ${BUILD_DIR}/apex-os-net-config.tar.gz:/usr/lib/python2.7/site-packages \
+    --run-command "cd /usr/lib/python2.7/site-packages/ && rm -rf os_net_config && tar xzf apex-os-net-config.tar.gz" \
     --run-command "if ! rpm -qa | grep python-redis; then yum install -y python-redis; fi" \
     --run-command "sed -i 's/^#UseDNS.*$/UseDNS no/' /etc/ssh/sshd_config" \
     --run-command "sed -i 's/^GSSAPIAuthentication.*$/GSSAPIAuthentication no/' /etc/ssh/sshd_config" \
diff --git a/build/undercloud.sh b/build/undercloud.sh
index 8d82bcac..0e1a2e35 100755
--- a/build/undercloud.sh
+++ b/build/undercloud.sh
@@ -10,7 +10,6 @@
 set -xe
 source ./cache.sh
 source ./variables.sh
-source ./functions.sh
 
 populate_cache "$rdo_images_uri/undercloud.qcow2"
 if [ ! -d "$BUILD_DIR" ]; then mkdir ${BUILD_DIR}; fi
@@ -18,27 +17,27 @@ cp -f ${CACHE_DIR}/undercloud.qcow2 ${BUILD_DIR}/undercloud_build.qcow2
 
 pushd ${BUILD_DIR} > /dev/null
 
-# prep opnfv-tht for undercloud
-clone_fork opnfv-tht
-pushd opnfv-tht > /dev/null
-git archive --format=tar.gz --prefix=openstack-tripleo-heat-templates/ HEAD > ${BUILD_DIR}/opnfv-tht.tar.gz
+# prep apex-tht for undercloud
+python3 -B $BUILD_UTILS clone-fork -r apex-tripleo-heat-templates
+pushd apex-tripleo-heat-templates > /dev/null
+git archive --format=tar.gz --prefix=openstack-tripleo-heat-templates/ HEAD > ${BUILD_DIR}/apex-tripleo-heat-templates.tar.gz
 popd > /dev/null
 
 # inject rt_kvm kernel rpm name into the enable file
 sed "s/kvmfornfv_kernel.rpm/$kvmfornfv_kernel_rpm/" ${BUILD_ROOT}/enable_rt_kvm.yaml | tee ${BUILD_DIR}/enable_rt_kvm.yaml
 
 # Turn off GSSAPI Auth in sshd
-# installing forked opnfv-tht
+# installing forked apex-tht
 # enabling ceph OSDs to live on the controller
 # seeding configuration files specific to OPNFV
 # Add performance image scripts
 LIBGUESTFS_BACKEND=direct virt-customize \
     --run-command "sed -i 's/^#UseDNS.*$/UseDNS no/' /etc/ssh/sshd_config" \
     --run-command "sed -i 's/^GSSAPIAuthentication.*$/GSSAPIAuthentication no/' /etc/ssh/sshd_config" \
-    --upload ${BUILD_DIR}/opnfv-tht.tar.gz:/usr/share \
+    --upload ${BUILD_DIR}/apex-tripleo-heat-templates.tar.gz:/usr/share \
     --install "openstack-utils" \
     --install "ceph-common" \
-    --run-command "cd /usr/share && rm -rf openstack-tripleo-heat-templates && tar xzf opnfv-tht.tar.gz" \
+    --run-command "cd /usr/share && rm -rf openstack-tripleo-heat-templates && tar xzf apex-tripleo-heat-templates.tar.gz" \
     --run-command "sed -i '/ControllerEnableCephStorage/c\\  ControllerEnableCephStorage: true' /usr/share/openstack-tripleo-heat-templates/environments/storage-environment.yaml" \
     --run-command "sed -i '/ComputeEnableCephStorage/c\\  ComputeEnableCephStorage: true' /usr/share/openstack-tripleo-heat-templates/environments/storage-environment.yaml" \
     --run-command "cp /usr/share/instack-undercloud/undercloud.conf.sample /home/stack/undercloud.conf && chown stack:stack /home/stack/undercloud.conf" \
diff --git a/build/variables.sh b/build/variables.sh
index 41d86e92..145b582f 100644
--- a/build/variables.sh
+++ b/build/variables.sh
@@ -14,6 +14,8 @@ QUAGGA_RPMS_DIR=${BUILD_DIR}/quagga_build_dir
 CACHE_DIR="$(dirname ${BUILD_ROOT})/.cache"
 CACHE_HISTORY=".cache_history"
 PATCHES_DIR="${BUILD_ROOT}/patches"
+BUILD_UTILS="$(dirname ${BUILD_ROOT})/lib/python/build_utils.py"
+
 
 rdo_images_uri=${RDO_IMAGES_URI:-https://images.rdoproject.org/ocata/delorean/current-tripleo/stable/}
 
diff --git a/ci/test.sh b/ci/test.sh
index bae21096..c1e99344 100755
--- a/ci/test.sh
+++ b/ci/test.sh
@@ -11,7 +11,7 @@
 set -e
 
 # Make sure python dependencies are installed
-for pkg in yamllint iproute epel-release python34-devel python34-nose python34-PyYAML python-pep8 python34-mock; do
+for pkg in yamllint iproute epel-release python34-devel python34-nose python34-PyYAML python-pep8 python34-mock python34-pip; do
   if ! rpm -q ${pkg} > /dev/null; then
     if ! sudo yum install -y ${pkg}; then
       echo "Failed to install ${pkg} package..."
@@ -21,7 +21,7 @@ for pkg in yamllint iproute epel-release python34-devel python34-nose python34-P
 done
 
 # Make sure coverage is installed
-if ! python3 -c "import coverage" &> /dev/null; then sudo easy_install-3.4 coverage; fi
+if ! python3 -c "import coverage" &> /dev/null; then sudo pip3.4 coverage; fi
 
 pushd ../build/ > /dev/null
 make python-pep8-check
diff --git a/lib/ansible/playbooks/build_dependencies.yml b/lib/ansible/playbooks/build_dependencies.yml
index dcf2ed94..620b1b3f 100644
--- a/lib/ansible/playbooks/build_dependencies.yml
+++ b/lib/ansible/playbooks/build_dependencies.yml
@@ -10,7 +10,7 @@
               python34-markupsafe, python2-virtualbmc,
               libguestfs-tools,bsdtar,libvirt,
               python2-oslo-config,python2-debtcollector,
-              make, python-pip, python-virtualenv
+              make, python-pip, python-virtualenv, python34-pip
     - name: Install Virtualization group
       yum:
         name: "@Virtualization Host"
@@ -19,3 +19,9 @@
         name: 'http://artifacts.opnfv.org/apex/dependencies/python3-ipmi-0.3.0-1.noarch.rpm'
     - pip:
         name: tox
+    - pip:
+        name: gitpython
+        executable: pip3.4
+    - pip:
+        name: pygerrit2
+        executable: pip3.4
diff --git a/lib/python/build_utils.py b/lib/python/build_utils.py
new file mode 100644
index 00000000..14327a90
--- /dev/null
+++ b/lib/python/build_utils.py
@@ -0,0 +1,108 @@
+##############################################################################
+# Copyright (c) 2017 Feng Pan (fpan@redhat.com) and others.
+#
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+import argparse
+import git
+import logging
+import os
+from pygerrit2.rest import GerritRestAPI
+import re
+import shutil
+import sys
+
+
+def clone_fork(args):
+    ref = None
+    logging.info("Cloning {}".format(args.repo))
+
+    try:
+        cm = git.Repo(search_parent_directories=True).commit().message
+    except git.exc.InvalidGitRepositoryError:
+        logging.debug('Current Apex directory is not a git repo: {}'
+                      .format(os.getcwd()))
+        cm = ''
+
+    logging.info("Current commit message: {}".format(cm))
+    m = re.search('{}:\s*(\S+)'.format(args.repo), cm)
+
+    if m:
+        change_id = m.group(1)
+        logging.info("Using change ID {} from {}".format(change_id, args.repo))
+        rest = GerritRestAPI(url=args.url)
+        change_str = "changes/{}?o=CURRENT_REVISION".format(change_id)
+        change = rest.get(change_str)
+        try:
+            assert change['status'] not in 'ABANDONED' 'CLOSED',\
+                'Change {} is in {} state'.format(change_id, change['status'])
+            if change['status'] == 'MERGED':
+                logging.info('Change {} is merged, ignoring...'
+                             .format(change_id))
+            else:
+                current_revision = change['current_revision']
+                ref = change['revisions'][current_revision]['ref']
+                logging.info('setting ref to {}'.format(ref))
+        except KeyError:
+            logging.error('Failed to get valid change data structure from url '
+                          '{}/{}, data returned: \n{}'
+                          .format(change_id, change_str, change))
+            raise
+
+    # remove existing file or directory named repo
+    if os.path.exists(args.repo):
+        if os.path.isdir(args.repo):
+            shutil.rmtree(args.repo)
+        else:
+            os.remove(args.repo)
+
+    ws = git.Repo.clone_from("{}/{}".format(args.url, args.repo),
+                             args.repo, b=args.branch)
+    if ref:
+        git_cmd = ws.git
+        git_cmd.fetch("{}/{}".format(args.url, args.repo), ref)
+        git_cmd.checkout('FETCH_HEAD')
+        logging.info('Checked out commit:\n{}'.format(ws.head.commit.message))
+
+
+def get_parser():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--debug', action='store_true', default=False,
+                        help="Turn on debug messages")
+    subparsers = parser.add_subparsers()
+    fork = subparsers.add_parser('clone-fork',
+                                 help='Clone fork of dependent repo')
+    fork.add_argument('-r', '--repo', required=True, help='Name of repository')
+    fork.add_argument('-u', '--url',
+                      default='https://gerrit.opnfv.org/gerrit',
+                      help='Gerrit URL of repository')
+    fork.add_argument('-b', '--branch',
+                      default='master',
+                      help='Branch to checkout')
+    fork.set_defaults(func=clone_fork)
+    return parser
+
+
+def main():
+    parser = get_parser()
+    args = parser.parse_args(sys.argv[1:])
+    if args.debug:
+        logging_level = logging.DEBUG
+    else:
+        logging_level = logging.INFO
+
+    logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s',
+                        datefmt='%m/%d/%Y %I:%M:%S %p',
+                        level=logging_level)
+    if hasattr(args, 'func'):
+        args.func(args)
+    else:
+        parser.print_help()
+        exit(1)
+
+if __name__ == "__main__":
+    main()