diff options
| -rwxr-xr-x | jjb/dovetail/dovetail-run.sh | 5 | ||||
| -rwxr-xr-x | jjb/openci/create-cde.sh | 36 | ||||
| -rwxr-xr-x | jjb/openci/create-clme.sh | 35 | ||||
| -rw-r--r-- | jjb/openci/openci-opnfv-daily-jobs.yaml | 62 | ||||
| -rw-r--r-- | jjb/releng/releng-release-create-branch.sh | 15 | ||||
| -rw-r--r-- | jjb/releng/releng-release-jobs.yaml | 2 | ||||
| -rwxr-xr-x | jjb/yardstick/yardstick-daily.sh | 4 | ||||
| -rw-r--r-- | releases/gambia/sandbox.yaml | 9 | ||||
| -rw-r--r-- | releases/scripts/create_branch.py | 143 | ||||
| -rw-r--r-- | releases/scripts/repos.py | 31 |
10 files changed, 128 insertions, 214 deletions
diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index 739da9eac..ed636b849 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -51,11 +51,6 @@ elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then # replace the default one by the customized one provided by jenkins config fi -# Set iptables rule to allow forwarding return traffic for container -if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FORWARD | awk 'NR==3' | grep RETURN 2> ${redirect}; then - sudo iptables -I FORWARD -j RETURN -fi - releng_repo=${WORKSPACE}/releng [ -d ${releng_repo} ] && sudo rm -rf ${releng_repo} git clone https://gerrit.opnfv.org/gerrit/releng ${releng_repo} >/dev/null diff --git a/jjb/openci/create-cde.sh b/jjb/openci/create-cde.sh index 410db50e6..8fb9f0041 100755 --- a/jjb/openci/create-cde.sh +++ b/jjb/openci/create-cde.sh @@ -11,17 +11,37 @@ set -o errexit set -o nounset set -o pipefail +export PS1=${PS1:-} + # This script creates CompositionDefinedEvent -# The JMS Messaging Plugin doesn't handle the newlines well so the eventBody is -# constructed on a single line. This is something that needs to be fixed properly -cat << EOF > $WORKSPACE/event.properties -type=$PUBLISH_EVENT_TYPE -origin=$PUBLISH_EVENT_ORIGIN -scenario=$DEPLOY_SCENARIO -eventBody="{ 'type': '$PUBLISH_EVENT_TYPE', 'id': '$(uuidgen)', 'time': '$(date -u +%Y-%m-%d_%H:%M:%SUTC)', 'origin': '$PUBLISH_EVENT_ORIGIN', 'buildUrl': '$BUILD_URL', 'branch': 'master', 'compositionName': '$DEPLOY_SCENARIO', 'compositionMetadataUrl': '$SCENARIO_METADATA_LOCATION' }" +git clone https://gitlab.openci.io/openci/prototypes.git +cd prototypes/federated-cicd +virtualenv openci_publish +cd openci_publish +source bin/activate +python setup.py install + +# generate event body +cat <<EOF > ./json_body.txt +{ + "type": "$PUBLISH_EVENT_TYPE", + "id": "$(uuidgen)", + "time": "$(date -u +%Y-%m-%d_%H:%M:%SUTC)", + "buildUrl": "$BUILD_URL", + "branch": "master", + "origin": "$PUBLISH_EVENT_ORIGIN", + "scenario": "$DEPLOY_SCENARIO", + "compositionName": "$DEPLOY_SCENARIO", + "compositionMetadataUrl": "$SCENARIO_METADATA_LOCATION" +} EOF + +python openci_publish -H 129.192.69.55 -U ${ACTIVEMQ_USER} -p ${ACTIVEMQ_PASSWORD} -n openci.prototype -B ./json_body.txt + echo "Constructed $PUBLISH_EVENT_TYPE" echo "--------------------------------------------" -cat $WORKSPACE/event.properties +cat ./json_body.txt echo "--------------------------------------------" + +deactivate diff --git a/jjb/openci/create-clme.sh b/jjb/openci/create-clme.sh index 5e8ee10c9..85c1a80a2 100755 --- a/jjb/openci/create-clme.sh +++ b/jjb/openci/create-clme.sh @@ -11,17 +11,36 @@ set -o errexit set -o nounset set -o pipefail +export PS1=${PS1:-} + # This script creates ConfidenceLevelModifiedEvent -# The JMS Messaging Plugin doesn't handle the newlines well so the eventBody is -# constructed on a single line. This is something that needs to be fixed properly -cat << EOF > $WORKSPACE/event.properties -type=$PUBLISH_EVENT_TYPE -origin=$PUBLISH_EVENT_ORIGIN -scenario=$DEPLOY_SCENARIO -eventBody="{ 'type': '$PUBLISH_EVENT_TYPE', 'id': '$(uuidgen)', 'time': '$(date -u +%Y-%m-%d_%H:%M:%SUTC)', 'origin': '$PUBLISH_EVENT_ORIGIN', 'buildUrl': '$BUILD_URL', 'branch': 'master', 'compositionName': '$DEPLOY_SCENARIO', 'compositionMetadataUrl': '$SCENARIO_METADATA_LOCATION', 'confidenceLevel': { $CONFIDENCE_LEVEL } }" +git clone https://gitlab.openci.io/openci/prototypes.git +cd prototypes/federated-cicd +virtualenv openci_publish +cd openci_publish +source bin/activate +python setup.py install + +# generate event body +cat <<EOF > ./json_body.txt +{ + "type": "$PUBLISH_EVENT_TYPE", + "id": "$(uuidgen)", + "time": "$(date -u +%Y-%m-%d_%H:%M:%SUTC)", + "buildUrl": "$BUILD_URL", + "branch": "master", + "origin": "$PUBLISH_EVENT_ORIGIN", + "scenario": "$DEPLOY_SCENARIO", + "compositionName": "$DEPLOY_SCENARIO", + "compositionMetadataUrl": "$SCENARIO_METADATA_LOCATION", + "confidenceLevel": "$CONFIDENCE_LEVEL", +} EOF + +python openci_publish -H 129.192.69.55 -U ${ACTIVEMQ_USER} -p ${ACTIVEMQ_PASSWORD} -n openci.prototype -B ./json_body.txt + echo "Constructed $PUBLISH_EVENT_TYPE" echo "--------------------------------------------" -cat $WORKSPACE/event.properties +cat ./json_body.txt echo "--------------------------------------------" diff --git a/jjb/openci/openci-opnfv-daily-jobs.yaml b/jjb/openci/openci-opnfv-daily-jobs.yaml index cb150148b..7f48fcc06 100644 --- a/jjb/openci/openci-opnfv-daily-jobs.yaml +++ b/jjb/openci/openci-opnfv-daily-jobs.yaml @@ -44,17 +44,20 @@ name: SCENARIO_METADATA_LOCATION default: https://url/to/scenario/metadata/on/opnfv/artifact/repo/$BUILD_NUMBER description: 'The location of the scenario metadata' - - 'opnfv-build-defaults' + - 'ericsson-build4-defaults' + + wrappers: + - credentials-binding: + - username-password-separated: + credential-id: openci-connect-activemq + username: ACTIVEMQ_USER + password: ACTIVEMQ_PASSWORD + - workspace-cleanup triggers: - jms-messaging: provider-name: openci.activemq - selector: CI_TYPE = 'custom' - checks: - - field: origin - expected-value: '{origin}' - - field: type - expected-value: 'ArtifactPublishedEvent' + selector: JMSType = 'ArtifactPublishedEvent' and JMSOrigin = 'ODL' builders: - shell: | @@ -67,19 +70,6 @@ echo "----------------------------------" - shell: !include-raw-escape: ./create-cde.sh - - inject: - properties-file: "$WORKSPACE/event.properties" - - publishers: - - jms-messaging: - provider-name: openci.activemq - msg-type: Custom - msg-props: | - type=$type - origin=$origin - scenario=$scenario - msg-content: - $eventBody - job-template: name: 'openci-opnfv-{scenario}-test-daily-{stream}' @@ -109,19 +99,20 @@ name: CONFIDENCE_LEVEL default: "'opnfvdaily': 'SUCCESS'" description: 'The confidence level the published artifact gained' - - 'opnfv-build-defaults' + - 'ericsson-build4-defaults' + + wrappers: + - credentials-binding: + - username-password-separated: + credential-id: openci-connect-activemq + username: ACTIVEMQ_USER + password: ACTIVEMQ_PASSWORD + - workspace-cleanup triggers: - jms-messaging: provider-name: openci.activemq - selector: CI_TYPE = 'custom' - checks: - - field: origin - expected-value: 'OPNFV' - - field: type - expected-value: 'CompositionDefinedEvent' - - field: scenario - expected-value: '{scenario}' + selector: JMSType = 'CompositionDefinedEvent' and JMSOrigin = 'OPNFV' and JMSScenario = '{scenario}' builders: - shell: | @@ -134,16 +125,3 @@ echo "----------------------------------" - shell: !include-raw-escape: ./create-clme.sh - - inject: - properties-file: "$WORKSPACE/event.properties" - - publishers: - - jms-messaging: - provider-name: openci.activemq - msg-type: Custom - msg-props: | - type=$type - origin=$origin - scenario=$scenario - msg-content: - $eventBody diff --git a/jjb/releng/releng-release-create-branch.sh b/jjb/releng/releng-release-create-branch.sh index 92be0e637..663ff19e7 100644 --- a/jjb/releng/releng-release-create-branch.sh +++ b/jjb/releng/releng-release-create-branch.sh @@ -26,7 +26,20 @@ STREAM=${STREAM:-'nostream'} RELEASE_FILES=$(git diff HEAD^1 --name-only -- "releases/$STREAM") for release_file in $RELEASE_FILES; do - python releases/scripts/create_branch.py -f $release_file + + while read -r repo branch ref; do + + echo "$repo" "$branch" "$ref" + branches="$(git ls-remote "https://gerrit.opnfv.org/gerrit/$repo.git" "refs/heads/$branch")" + + if ! [ -z "$branches" ]; then + echo "refs/heads/$branch already exists at $ref ($branches)" + else + ssh -n -f -p 29418 gerrit.opnfv.org gerrit create-branch "$repo" "$branch" "$ref" + fi + + done < <(python releases/scripts/repos.py -b -f "$release_file") + python releases/scripts/create_jobs.py -f $release_file NEW_FILES=$(git status --porcelain --untracked=no | cut -c4-) if [ -n "$NEW_FILES" ]; then diff --git a/jjb/releng/releng-release-jobs.yaml b/jjb/releng/releng-release-jobs.yaml index 89519ae9b..3136d7855 100644 --- a/jjb/releng/releng-release-jobs.yaml +++ b/jjb/releng/releng-release-jobs.yaml @@ -107,7 +107,7 @@ - shell: !include-raw-escape: - releng-release-create-venv.sh - releng-release-tagging.sh - # - releng-release-create-branch.sh + - releng-release-create-branch.sh publishers: - email-jenkins-admins-on-failure diff --git a/jjb/yardstick/yardstick-daily.sh b/jjb/yardstick/yardstick-daily.sh index 783c64ee1..58d590c27 100755 --- a/jjb/yardstick/yardstick-daily.sh +++ b/jjb/yardstick/yardstick-daily.sh @@ -41,10 +41,6 @@ elif [[ ${INSTALLER_TYPE} == 'fuel' ]]; then cacert_file_vol="-v ${HOME}/os_cacert:/etc/ssl/certs/mcp_os_cacert" sshkey="-v ${SSH_KEY}:/root/.ssh/mcp.rsa" fi -# Set iptables rule to allow forwarding return traffic for container -if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FORWARD | awk 'NR==3' | grep RETURN 2> ${redirect}; then - sudo iptables -I FORWARD -j RETURN -fi opts="--privileged=true --rm" envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} \ diff --git a/releases/gambia/sandbox.yaml b/releases/gambia/sandbox.yaml new file mode 100644 index 000000000..16f2df24e --- /dev/null +++ b/releases/gambia/sandbox.yaml @@ -0,0 +1,9 @@ +--- +project: sandbox +project-type: infra +release-model: stable + +branches: + - name: stable/gambia + location: + sandbox: c2012f5b642f17e6024db631b833414114a329d5 diff --git a/releases/scripts/create_branch.py b/releases/scripts/create_branch.py deleted file mode 100644 index fa3c92def..000000000 --- a/releases/scripts/create_branch.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python2 -# SPDX-License-Identifier: Apache-2.0 -############################################################################## -# Copyright (c) 2018 The Linux Foundation and others. -# All rights reserved. This program and the accompanying materials -# are made available under the terms of the Apache License, Version 2.0 -# which accompanies this distribution, and is available at -# http://www.apache.org/licenses/LICENSE-2.0 -############################################################################## -""" -Create Gerrit Branchs -""" - -import argparse - -try: - import ConfigParser -except ImportError: - import configparser as ConfigParser - -import logging -import os -import yaml - -from requests.compat import quote -from requests.exceptions import RequestException - -from pygerrit2.rest import GerritRestAPI -from pygerrit2.rest.auth import HTTPDigestAuthFromNetrc, HTTPBasicAuthFromNetrc - - -logging.basicConfig(level=logging.INFO) - - -def quote_branch(arguments): - """ - Quote is used here to escape the '/' in branch name. By - default '/' is listed in 'safe' characters which aren't escaped. - quote is not used in the data of the PUT request, as quoting for - arguments is handled by the request library - """ - new_args = arguments.copy() - new_args['branch'] = quote(new_args['branch'], '') - return new_args - - -def create_branch(api, arguments): - """ - Create a branch using the Gerrit REST API - """ - logger = logging.getLogger(__file__) - - branch_data = """ - { - "ref": "%(branch)s" - "revision": "%(commit)s" - }""" % arguments - - # First verify the commit exists, otherwise the branch will be - # created at HEAD - try: - request = api.get("/projects/%(project)s/commits/%(commit)s" % - arguments) - logger.debug(request) - logger.debug("Commit exists: %(commit)s", arguments) - except RequestException as err: - if hasattr(err, 'response') and err.response.status_code in [404]: - logger.warn("Commit %(commit)s for %(project)s does" - " not exist. Not creating branch.", arguments) - logger.warn(err) - else: - logger.error("Error: %s", str(err)) - # Skip trying to create the branch - return - - # Try to create the branch and let us know if it already exist. - try: - request = api.put("/projects/%(project)s/branches/%(branch)s" % - quote_branch(arguments), branch_data) - logger.info("Branch %(branch)s for %(project)s successfully created", - arguments) - except RequestException as err: - if hasattr(err, 'response') and err.response.status_code in [412, 409]: - logger.info("Branch %(branch)s already created for %(project)s", - arguments) - logger.info(err) - else: - logger.error("Error: %s", str(err)) - - -def main(): - """Given a yamlfile that follows the release syntax, create branches - in Gerrit listed under branches""" - - config = ConfigParser.ConfigParser() - config.read(os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'defaults.cfg')) - config.read([os.path.expanduser('~/releases.cfg'), 'releases.cfg']) - - gerrit_url = config.get('gerrit', 'url') - - parser = argparse.ArgumentParser() - parser.add_argument('--file', '-f', - type=argparse.FileType('r'), - required=True) - parser.add_argument('--basicauth', '-b', action='store_true') - args = parser.parse_args() - - GerritAuth = HTTPDigestAuthFromNetrc - if args.basicauth: - GerritAuth = HTTPBasicAuthFromNetrc - - try: - auth = GerritAuth(url=gerrit_url) - except ValueError as err: - logging.error("%s for %s", err, gerrit_url) - quit(1) - restapi = GerritRestAPI(url=gerrit_url, auth=auth) - - project = yaml.safe_load(args.file) - - create_branches(restapi, project) - - -def create_branches(restapi, project): - """Create branches for a specific project defined in the release - file""" - - branches = [] - for branch in project['branches']: - repo, ref = next(iter(branch['location'].items())) - branches.append({ - 'project': repo, - 'branch': branch['name'], - 'commit': ref - }) - - for branch in branches: - create_branch(restapi, branch) - - -if __name__ == "__main__": - main() diff --git a/releases/scripts/repos.py b/releases/scripts/repos.py index 47ce42d88..91c4e9300 100644 --- a/releases/scripts/repos.py +++ b/releases/scripts/repos.py @@ -63,20 +63,28 @@ def main(): type=str, help="Only print" "SHAs for the specified release") + parser.add_argument('--branches', '-b', + action='store_true', + default=False, + help="Print Branch info") + args = parser.parse_args() project = yaml.safe_load(args.file) - list_repos(project, args) + if args.branches: + list_branches(project, args) + else: + list_repos(project, args) def list_repos(project, args): """List repositories in the project file""" lookup = project.get('releases', []) + if 'releases' not in project: exit(0) - repos = set() for item in lookup: repo, ref = next(iter(item['location'].items())) @@ -90,5 +98,24 @@ def list_repos(project, args): print(repo) +def list_branches(project, args): + """List branches in the project file""" + + lookup = project.get('branches', []) + + if 'branches' not in project: + exit(0) + repos = set() + for item in lookup: + repo, ref = next(iter(item['location'].items())) + if args.names: + repos.add(Repo(repo)) + elif args.release and item['name'] == args.release: + repos.add(Repo(repo, ref)) + elif not args.release: + repos.add(Repo(repo, item['name'], ref)) + for repo in repos: + print(repo) + if __name__ == "__main__": main() |