diff options
| -rw-r--r-- | build/functions.sh | 46 | ||||
| -rwxr-xr-x | build/overcloud-full.sh | 23 | ||||
| -rwxr-xr-x | build/undercloud.sh | 15 | ||||
| -rw-r--r-- | build/variables.sh | 2 | ||||
| -rwxr-xr-x | ci/test.sh | 4 | ||||
| -rw-r--r-- | lib/ansible/playbooks/build_dependencies.yml | 8 | ||||
| -rw-r--r-- | lib/python/build_utils.py | 108 |
7 files changed, 137 insertions, 69 deletions
diff --git a/build/functions.sh b/build/functions.sh deleted file mode 100644 index becc0df1..00000000 --- a/build/functions.sh +++ /dev/null @@ -1,46 +0,0 @@ -############################################################################## -# Copyright (c) 2016 Dan Radez (Red Hat) and others. -# -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## - -clone_fork () { - # ARG 1: opnfv-tht or opnfv-puppet-tripleo - echo "Cloning $1" - - # Use apex tripleo-heat-templates fork - local ghcreds="" - local pr_num="" - local ref="stable/euphrates" - local repo="https://github.com/trozet/$1" - - if git log -1 | grep "${1}-pr:" | grep -o '[0-9]*'; then - pr_num=$(git log -1 | grep "${1}-pr:" | grep -o '[0-9]*') - fi - - if [ "$pr_num" != "" ]; then - echo "Using pull request $pr_num from $repo" - # Source credentials since we are rate limited to 60/day - if [ -f ~/.githubcreds ]; then - source ~/.githubcreds - ghcreds=" -u $GHUSERNAME:$GHACCESSTOKEN" - fi - - PR=$(curl $ghcreds https://api.github.com/repos/trozet/$1/pulls/$pr_num) - - # Do not pull from merged branches - MERGED=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['merged']") - if [ "$MERGED" == "False" ]; then - ref=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['head']['ref']") - echo "Setting GitHub Ref to: $REF" - repo=$(python -c "import json; print json.loads('''$PR'''.replace('\n', '').replace('\r', ''))['head']['repo']['clone_url']") - echo "Setting GitHub URL to: $repo" - fi - fi - - rm -rf $1 - git clone $repo -b $ref $1 -} diff --git a/build/overcloud-full.sh b/build/overcloud-full.sh index adcf4df6..0f7318a6 100755 --- a/build/overcloud-full.sh +++ b/build/overcloud-full.sh @@ -10,7 +10,6 @@ set -xe source ./cache.sh source ./variables.sh -source ./functions.sh populate_cache "$rdo_images_uri/overcloud-full.tar" @@ -25,15 +24,15 @@ mv -f ${BUILD_DIR}/overcloud-full.qcow2 ${BUILD_DIR}/overcloud-full_build.qcow2 pushd ${BUILD_DIR} > /dev/null # prep opnfv-puppet-tripleo for undercloud -clone_fork opnfv-puppet-tripleo -pushd opnfv-puppet-tripleo > /dev/null -git archive --format=tar.gz --prefix=tripleo/ HEAD > ${BUILD_DIR}/opnfv-puppet-tripleo.tar.gz +python3 -B $BUILD_UTILS clone-fork -r apex-puppet-tripleo +pushd apex-puppet-tripleo > /dev/null +git archive --format=tar.gz --prefix=tripleo/ HEAD > ${BUILD_DIR}/apex-puppet-tripleo.tar.gz popd > /dev/null # download customized os-net-config -clone_fork os-net-config -pushd os-net-config/os_net_config > /dev/null -git archive --format=tar.gz --prefix=os_net_config/ HEAD > ${BUILD_DIR}/os-net-config.tar.gz +python3 -B $BUILD_UTILS clone-fork -r apex-os-net-config +pushd apex-os-net-config/os_net_config > /dev/null +git archive --format=tar.gz --prefix=os_net_config/ HEAD > ${BUILD_DIR}/apex-os-net-config.tar.gz popd > /dev/null dpdk_pkg_str='' @@ -77,7 +76,7 @@ done qemu-img resize overcloud-full_build.qcow2 +900MB # expand file system to max disk size -# installing forked opnfv-puppet-tripleo +# installing forked apex-puppet-tripleo # upload dpdk rpms but do not install # install fd.io yum repo and packages # upload puppet fdio @@ -85,10 +84,10 @@ qemu-img resize overcloud-full_build.qcow2 +900MB # upload the rt_kvm kernel LIBGUESTFS_BACKEND=direct virt-customize \ --run-command "xfs_growfs /dev/sda" \ - --upload ${BUILD_DIR}/opnfv-puppet-tripleo.tar.gz:/etc/puppet/modules \ - --run-command "cd /etc/puppet/modules && rm -rf tripleo && tar xzf opnfv-puppet-tripleo.tar.gz" \ - --upload ${BUILD_DIR}/os-net-config.tar.gz:/usr/lib/python2.7/site-packages \ - --run-command "cd /usr/lib/python2.7/site-packages/ && rm -rf os_net_config && tar xzf os-net-config.tar.gz" \ + --upload ${BUILD_DIR}/apex-puppet-tripleo.tar.gz:/etc/puppet/modules \ + --run-command "cd /etc/puppet/modules && rm -rf tripleo && tar xzf apex-puppet-tripleo.tar.gz" \ + --upload ${BUILD_DIR}/apex-os-net-config.tar.gz:/usr/lib/python2.7/site-packages \ + --run-command "cd /usr/lib/python2.7/site-packages/ && rm -rf os_net_config && tar xzf apex-os-net-config.tar.gz" \ --run-command "if ! rpm -qa | grep python-redis; then yum install -y python-redis; fi" \ --run-command "sed -i 's/^#UseDNS.*$/UseDNS no/' /etc/ssh/sshd_config" \ --run-command "sed -i 's/^GSSAPIAuthentication.*$/GSSAPIAuthentication no/' /etc/ssh/sshd_config" \ diff --git a/build/undercloud.sh b/build/undercloud.sh index 8d82bcac..0e1a2e35 100755 --- a/build/undercloud.sh +++ b/build/undercloud.sh @@ -10,7 +10,6 @@ set -xe source ./cache.sh source ./variables.sh -source ./functions.sh populate_cache "$rdo_images_uri/undercloud.qcow2" if [ ! -d "$BUILD_DIR" ]; then mkdir ${BUILD_DIR}; fi @@ -18,27 +17,27 @@ cp -f ${CACHE_DIR}/undercloud.qcow2 ${BUILD_DIR}/undercloud_build.qcow2 pushd ${BUILD_DIR} > /dev/null -# prep opnfv-tht for undercloud -clone_fork opnfv-tht -pushd opnfv-tht > /dev/null -git archive --format=tar.gz --prefix=openstack-tripleo-heat-templates/ HEAD > ${BUILD_DIR}/opnfv-tht.tar.gz +# prep apex-tht for undercloud +python3 -B $BUILD_UTILS clone-fork -r apex-tripleo-heat-templates +pushd apex-tripleo-heat-templates > /dev/null +git archive --format=tar.gz --prefix=openstack-tripleo-heat-templates/ HEAD > ${BUILD_DIR}/apex-tripleo-heat-templates.tar.gz popd > /dev/null # inject rt_kvm kernel rpm name into the enable file sed "s/kvmfornfv_kernel.rpm/$kvmfornfv_kernel_rpm/" ${BUILD_ROOT}/enable_rt_kvm.yaml | tee ${BUILD_DIR}/enable_rt_kvm.yaml # Turn off GSSAPI Auth in sshd -# installing forked opnfv-tht +# installing forked apex-tht # enabling ceph OSDs to live on the controller # seeding configuration files specific to OPNFV # Add performance image scripts LIBGUESTFS_BACKEND=direct virt-customize \ --run-command "sed -i 's/^#UseDNS.*$/UseDNS no/' /etc/ssh/sshd_config" \ --run-command "sed -i 's/^GSSAPIAuthentication.*$/GSSAPIAuthentication no/' /etc/ssh/sshd_config" \ - --upload ${BUILD_DIR}/opnfv-tht.tar.gz:/usr/share \ + --upload ${BUILD_DIR}/apex-tripleo-heat-templates.tar.gz:/usr/share \ --install "openstack-utils" \ --install "ceph-common" \ - --run-command "cd /usr/share && rm -rf openstack-tripleo-heat-templates && tar xzf opnfv-tht.tar.gz" \ + --run-command "cd /usr/share && rm -rf openstack-tripleo-heat-templates && tar xzf apex-tripleo-heat-templates.tar.gz" \ --run-command "sed -i '/ControllerEnableCephStorage/c\\ ControllerEnableCephStorage: true' /usr/share/openstack-tripleo-heat-templates/environments/storage-environment.yaml" \ --run-command "sed -i '/ComputeEnableCephStorage/c\\ ComputeEnableCephStorage: true' /usr/share/openstack-tripleo-heat-templates/environments/storage-environment.yaml" \ --run-command "cp /usr/share/instack-undercloud/undercloud.conf.sample /home/stack/undercloud.conf && chown stack:stack /home/stack/undercloud.conf" \ diff --git a/build/variables.sh b/build/variables.sh index 41d86e92..145b582f 100644 --- a/build/variables.sh +++ b/build/variables.sh @@ -14,6 +14,8 @@ QUAGGA_RPMS_DIR=${BUILD_DIR}/quagga_build_dir CACHE_DIR="$(dirname ${BUILD_ROOT})/.cache" CACHE_HISTORY=".cache_history" PATCHES_DIR="${BUILD_ROOT}/patches" +BUILD_UTILS="$(dirname ${BUILD_ROOT})/lib/python/build_utils.py" + rdo_images_uri=${RDO_IMAGES_URI:-https://images.rdoproject.org/ocata/delorean/current-tripleo/stable/} @@ -11,7 +11,7 @@ set -e # Make sure python dependencies are installed -for pkg in yamllint iproute epel-release python34-devel python34-nose python34-PyYAML python-pep8 python34-mock; do +for pkg in yamllint iproute epel-release python34-devel python34-nose python34-PyYAML python-pep8 python34-mock python34-pip; do if ! rpm -q ${pkg} > /dev/null; then if ! sudo yum install -y ${pkg}; then echo "Failed to install ${pkg} package..." @@ -21,7 +21,7 @@ for pkg in yamllint iproute epel-release python34-devel python34-nose python34-P done # Make sure coverage is installed -if ! python3 -c "import coverage" &> /dev/null; then sudo easy_install-3.4 coverage; fi +if ! python3 -c "import coverage" &> /dev/null; then sudo pip3.4 coverage; fi pushd ../build/ > /dev/null make python-pep8-check diff --git a/lib/ansible/playbooks/build_dependencies.yml b/lib/ansible/playbooks/build_dependencies.yml index dcf2ed94..620b1b3f 100644 --- a/lib/ansible/playbooks/build_dependencies.yml +++ b/lib/ansible/playbooks/build_dependencies.yml @@ -10,7 +10,7 @@ python34-markupsafe, python2-virtualbmc, libguestfs-tools,bsdtar,libvirt, python2-oslo-config,python2-debtcollector, - make, python-pip, python-virtualenv + make, python-pip, python-virtualenv, python34-pip - name: Install Virtualization group yum: name: "@Virtualization Host" @@ -19,3 +19,9 @@ name: 'http://artifacts.opnfv.org/apex/dependencies/python3-ipmi-0.3.0-1.noarch.rpm' - pip: name: tox + - pip: + name: gitpython + executable: pip3.4 + - pip: + name: pygerrit2 + executable: pip3.4 diff --git a/lib/python/build_utils.py b/lib/python/build_utils.py new file mode 100644 index 00000000..14327a90 --- /dev/null +++ b/lib/python/build_utils.py @@ -0,0 +1,108 @@ +############################################################################## +# Copyright (c) 2017 Feng Pan (fpan@redhat.com) and others. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +import argparse +import git +import logging +import os +from pygerrit2.rest import GerritRestAPI +import re +import shutil +import sys + + +def clone_fork(args): + ref = None + logging.info("Cloning {}".format(args.repo)) + + try: + cm = git.Repo(search_parent_directories=True).commit().message + except git.exc.InvalidGitRepositoryError: + logging.debug('Current Apex directory is not a git repo: {}' + .format(os.getcwd())) + cm = '' + + logging.info("Current commit message: {}".format(cm)) + m = re.search('{}:\s*(\S+)'.format(args.repo), cm) + + if m: + change_id = m.group(1) + logging.info("Using change ID {} from {}".format(change_id, args.repo)) + rest = GerritRestAPI(url=args.url) + change_str = "changes/{}?o=CURRENT_REVISION".format(change_id) + change = rest.get(change_str) + try: + assert change['status'] not in 'ABANDONED' 'CLOSED',\ + 'Change {} is in {} state'.format(change_id, change['status']) + if change['status'] == 'MERGED': + logging.info('Change {} is merged, ignoring...' + .format(change_id)) + else: + current_revision = change['current_revision'] + ref = change['revisions'][current_revision]['ref'] + logging.info('setting ref to {}'.format(ref)) + except KeyError: + logging.error('Failed to get valid change data structure from url ' + '{}/{}, data returned: \n{}' + .format(change_id, change_str, change)) + raise + + # remove existing file or directory named repo + if os.path.exists(args.repo): + if os.path.isdir(args.repo): + shutil.rmtree(args.repo) + else: + os.remove(args.repo) + + ws = git.Repo.clone_from("{}/{}".format(args.url, args.repo), + args.repo, b=args.branch) + if ref: + git_cmd = ws.git + git_cmd.fetch("{}/{}".format(args.url, args.repo), ref) + git_cmd.checkout('FETCH_HEAD') + logging.info('Checked out commit:\n{}'.format(ws.head.commit.message)) + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument('--debug', action='store_true', default=False, + help="Turn on debug messages") + subparsers = parser.add_subparsers() + fork = subparsers.add_parser('clone-fork', + help='Clone fork of dependent repo') + fork.add_argument('-r', '--repo', required=True, help='Name of repository') + fork.add_argument('-u', '--url', + default='https://gerrit.opnfv.org/gerrit', + help='Gerrit URL of repository') + fork.add_argument('-b', '--branch', + default='master', + help='Branch to checkout') + fork.set_defaults(func=clone_fork) + return parser + + +def main(): + parser = get_parser() + args = parser.parse_args(sys.argv[1:]) + if args.debug: + logging_level = logging.DEBUG + else: + logging_level = logging.INFO + + logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', + datefmt='%m/%d/%Y %I:%M:%S %p', + level=logging_level) + if hasattr(args, 'func'): + args.func(args) + else: + parser.print_help() + exit(1) + +if __name__ == "__main__": + main() |