diff options
Diffstat (limited to 'releases/scripts')
| -rw-r--r-- | releases/scripts/create_branch.py | 143 | ||||
| -rw-r--r-- | releases/scripts/create_jobs.py | 145 | ||||
| -rw-r--r-- | releases/scripts/defaults.cfg | 2 | ||||
| -rw-r--r-- | releases/scripts/requirements.txt | 5 | ||||
| -rw-r--r-- | releases/scripts/verify_schema.py | 55 |
5 files changed, 350 insertions, 0 deletions
diff --git a/releases/scripts/create_branch.py b/releases/scripts/create_branch.py new file mode 100644 index 000000000..fa3c92def --- /dev/null +++ b/releases/scripts/create_branch.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python2 +# SPDX-License-Identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +""" +Create Gerrit Branchs +""" + +import argparse + +try: + import ConfigParser +except ImportError: + import configparser as ConfigParser + +import logging +import os +import yaml + +from requests.compat import quote +from requests.exceptions import RequestException + +from pygerrit2.rest import GerritRestAPI +from pygerrit2.rest.auth import HTTPDigestAuthFromNetrc, HTTPBasicAuthFromNetrc + + +logging.basicConfig(level=logging.INFO) + + +def quote_branch(arguments): + """ + Quote is used here to escape the '/' in branch name. By + default '/' is listed in 'safe' characters which aren't escaped. + quote is not used in the data of the PUT request, as quoting for + arguments is handled by the request library + """ + new_args = arguments.copy() + new_args['branch'] = quote(new_args['branch'], '') + return new_args + + +def create_branch(api, arguments): + """ + Create a branch using the Gerrit REST API + """ + logger = logging.getLogger(__file__) + + branch_data = """ + { + "ref": "%(branch)s" + "revision": "%(commit)s" + }""" % arguments + + # First verify the commit exists, otherwise the branch will be + # created at HEAD + try: + request = api.get("/projects/%(project)s/commits/%(commit)s" % + arguments) + logger.debug(request) + logger.debug("Commit exists: %(commit)s", arguments) + except RequestException as err: + if hasattr(err, 'response') and err.response.status_code in [404]: + logger.warn("Commit %(commit)s for %(project)s does" + " not exist. Not creating branch.", arguments) + logger.warn(err) + else: + logger.error("Error: %s", str(err)) + # Skip trying to create the branch + return + + # Try to create the branch and let us know if it already exist. + try: + request = api.put("/projects/%(project)s/branches/%(branch)s" % + quote_branch(arguments), branch_data) + logger.info("Branch %(branch)s for %(project)s successfully created", + arguments) + except RequestException as err: + if hasattr(err, 'response') and err.response.status_code in [412, 409]: + logger.info("Branch %(branch)s already created for %(project)s", + arguments) + logger.info(err) + else: + logger.error("Error: %s", str(err)) + + +def main(): + """Given a yamlfile that follows the release syntax, create branches + in Gerrit listed under branches""" + + config = ConfigParser.ConfigParser() + config.read(os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'defaults.cfg')) + config.read([os.path.expanduser('~/releases.cfg'), 'releases.cfg']) + + gerrit_url = config.get('gerrit', 'url') + + parser = argparse.ArgumentParser() + parser.add_argument('--file', '-f', + type=argparse.FileType('r'), + required=True) + parser.add_argument('--basicauth', '-b', action='store_true') + args = parser.parse_args() + + GerritAuth = HTTPDigestAuthFromNetrc + if args.basicauth: + GerritAuth = HTTPBasicAuthFromNetrc + + try: + auth = GerritAuth(url=gerrit_url) + except ValueError as err: + logging.error("%s for %s", err, gerrit_url) + quit(1) + restapi = GerritRestAPI(url=gerrit_url, auth=auth) + + project = yaml.safe_load(args.file) + + create_branches(restapi, project) + + +def create_branches(restapi, project): + """Create branches for a specific project defined in the release + file""" + + branches = [] + for branch in project['branches']: + repo, ref = next(iter(branch['location'].items())) + branches.append({ + 'project': repo, + 'branch': branch['name'], + 'commit': ref + }) + + for branch in branches: + create_branch(restapi, branch) + + +if __name__ == "__main__": + main() diff --git a/releases/scripts/create_jobs.py b/releases/scripts/create_jobs.py new file mode 100644 index 000000000..2478217a9 --- /dev/null +++ b/releases/scripts/create_jobs.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python2 +# SPDX-License-Identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +""" +Create Gerrit Branches +""" + +import argparse +import logging +import os +import re +import yaml +import subprocess + +# import ruamel +from ruamel.yaml import YAML + + +logging.basicConfig(level=logging.INFO) + + +def has_string(filepath, string): + """ + Return True if the given filepath contains the regex string + """ + with open(filepath) as yaml_file: + for line in yaml_file: + if string.search(line): + return True + return False + + +def jjb_files(project, release): + """ + Return sets of YAML file names that contain 'stream' for a given + project, and file that already contain the stream. + """ + files, skipped = set(), set() + file_ending = re.compile(r'ya?ml$') + search_string = re.compile(r'^\s+stream:') + release_string = re.compile(r'- %s:' % release) + jjb_path = os.path.join('jjb', project) + + if not os.path.isdir(jjb_path): + logging.warn("JJB directory does not exist at %s, skipping job " + "creation", jjb_path) + return (files, skipped) + + for file_name in os.listdir(jjb_path): + file_path = os.path.join(jjb_path, file_name) + if os.path.isfile(file_path) and file_ending.search(file_path): + if has_string(file_path, release_string): + skipped.add(file_path) + elif has_string(file_path, search_string): + files.add(file_path) + return (files, skipped) + + +def main(): + """ + Create Jenkins Jobs for stable branches in Release File + """ + parser = argparse.ArgumentParser() + parser.add_argument('--file', '-f', + type=argparse.FileType('r'), + required=True) + args = parser.parse_args() + + project_yaml = yaml.safe_load(args.file) + + # Get the release name from the file path + release = os.path.split(os.path.dirname(args.file.name))[1] + + create_jobs(release, project_yaml) + + +def create_jobs(release, project_yaml): + """Add YAML to JJB files for release stream""" + logger = logging.getLogger(__file__) + + # We assume here project keep their subrepo jobs under the part + # project name. Otherwise we'll have to look for jjb/<repo> for each + # branch listed. + project, _ = next(iter(project_yaml['branches'][0]['location'].items())) + + yaml_parser = YAML() + yaml_parser.preserve_quotes = True + yaml_parser.explicit_start = True + # yaml_parser.indent(mapping=4, sequence=0, offset=0) + # These are some esoteric values that produce indentation matching our jjb + # configs + # yaml_parser.indent(mapping=3, sequence=3, offset=2) + # yaml_parser.indent(sequence=4, offset=2) + yaml_parser.indent(mapping=2, sequence=4, offset=2) + + (job_files, skipped_files) = jjb_files(project, release) + + if skipped_files: + logger.info("Jobs already exists for %s in files: %s", + project, ', '.join(skipped_files)) + # Exit if there are not jobs to create + if not job_files: + return + logger.info("Creating Jenkins Jobs for %s in files: %s", + project, ', '.join(job_files)) + + stable_branch_stream = """\ + %s: + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + disabled: false + """ % release + + stable_branch_yaml = yaml_parser.load(stable_branch_stream) + stable_branch_yaml[release].yaml_set_anchor(release, always_dump=True) + + for job_file in job_files: + yaml_jjb = yaml_parser.load(open(job_file)) + if 'stream' not in yaml_jjb[0]['project']: + continue + + # TODO: Some JJB files don't have 'stream' + project_config = yaml_jjb[0]['project']['stream'] + # There is an odd issue where just appending adds a newline before the + # branch config, so we append (presumably after master) instead. + project_config.insert(1, stable_branch_yaml) + + # NOTE: In the future, we may need to override one or multiple of the + # following ruamal Emitter methods: + # * ruamel.yaml.emitter.Emitter.expect_block_sequence_item + # * ruamel.yaml.emitter.Emitter.write_indent + # To hopefully replace the need to shell out to sed... + yaml_parser.dump(yaml_jjb, open(job_file, 'w')) + args = ['sed', '-i', 's/^ //', job_file] + subprocess.Popen(args, stdout=subprocess.PIPE, shell=False) + + +if __name__ == "__main__": + main() diff --git a/releases/scripts/defaults.cfg b/releases/scripts/defaults.cfg new file mode 100644 index 000000000..6bc876588 --- /dev/null +++ b/releases/scripts/defaults.cfg @@ -0,0 +1,2 @@ +[gerrit] +url=https://gerrit.opnfv.org/gerrit/ diff --git a/releases/scripts/requirements.txt b/releases/scripts/requirements.txt new file mode 100644 index 000000000..5a7d216e9 --- /dev/null +++ b/releases/scripts/requirements.txt @@ -0,0 +1,5 @@ +pygerrit2 < 2.1.0 +PyYAML < 4.0 +jsonschema < 2.7.0 +rfc3987 +ruamel.yaml diff --git a/releases/scripts/verify_schema.py b/releases/scripts/verify_schema.py new file mode 100644 index 000000000..3a6163e2a --- /dev/null +++ b/releases/scripts/verify_schema.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python2 +# SPDX-License-Identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2018 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +""" +Verify YAML Schema +""" +import argparse +import logging +import jsonschema +import yaml + +LOADER = yaml.CSafeLoader if yaml.__with_libyaml__ else yaml.SafeLoader + + +def main(): + """ + Parse arguments and verify YAML + """ + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser() + parser.add_argument('--yaml', '-y', type=str, required=True) + parser.add_argument('--schema', '-s', type=str, required=True) + + args = parser.parse_args() + + with open(args.yaml) as _: + yaml_file = yaml.load(_, Loader=LOADER) + + with open(args.schema) as _: + schema_file = yaml.load(_, Loader=LOADER) + + # Load the schema + validation = jsonschema.Draft4Validator( + schema_file, + format_checker=jsonschema.FormatChecker() + ) + + # Look for errors + errors = 0 + for error in validation.iter_errors(yaml_file): + errors += 1 + logging.error(error) + if errors > 0: + raise RuntimeError("%d issues invalidate the release schema" % errors) + + +if __name__ == "__main__": + main() |