Release Automation

Tracking releases through yaml file similar to the openstack/releases
project.

Includes a schema file to be for validation, jobs for creating gerrit
branches and stable branch jobs, and documentation for projects on
creating their releases.

Change-Id: Id1876482723e01806c0a6932126dff5ea314eae5
Signed-off-by: Trevor Bramwell <tbramwell@linuxfoundation.org>

8 files changed, 445 insertions, 0 deletions

diff --git a/releases/euphrates/apex.yaml b/releases/euphrates/apex.yaml
new file mode 100644
index 000000000..78920761d
--- /dev/null
+++ b/releases/euphrates/apex.yaml
@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+project: apex
+project-type: installer
+release-model: stable
+upstream: https://wiki.openstack.org/wiki/TripleO
+
+releases:
+  - version: opnfv-5.0.0
+    location:
+      apex: 2f1c99daeee9cf0e89a8e833e034e7a5979ae894
+  - version: opnfv-5.1.0
+    location:
+      apex: f15d50c2009f1f865ac6f4171347940313727547
+
+branches:
+  - name: stable/euphrates
+    location:
+      apex: f27da77b87837e025907f689890b413c8f183c59
+  - name: stable/euphrates
+    location:
+      apex-tripleo-heat-templates: 676db53c4423693441112640cf362e93931161ae
+  - name: stable/euphrates
+    location:
+      apex-puppet-tripleo: 14bc31f54ea943547a3319b479ea7b8cd9661e85
+  - name: stable/euphrates
+    location:
+      apex-os-net-config: a6c3f2a2c853ca489cceff959a52d7f75bf4ffe0
+
+release-notes: http://docs.opnfv.org/en/stable-euphrates/submodules/apex/docs/release/release-notes/release-notes.html
diff --git a/releases/euphrates/compass4nfv.yaml b/releases/euphrates/compass4nfv.yaml
new file mode 100644
index 000000000..e46e01b18
--- /dev/null
+++ b/releases/euphrates/compass4nfv.yaml
@@ -0,0 +1,9 @@
+---
+project: compass4nfv
+project-type: installer
+release-model: stable
+
+branches:
+  - name: stable/euphrates
+    location:
+      compass4nfv: 435cd3756a833db0515eb70c1d8ec4adca90950f
diff --git a/releases/schema.yaml b/releases/schema.yaml
new file mode 100644
index 000000000..c3838760a
--- /dev/null
+++ b/releases/schema.yaml
@@ -0,0 +1,56 @@
+##############################################################################
+# Copyright (c) 2018 Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+---
+$schema: 'http://json-schema.org/schema#'
+$id: 'https://github.com/opnfv/releng/blob/master/releases/schema.yaml'
+
+additionalProperties: false
+
+required:
+  - 'project'
+  - 'project-type'
+
+properties:
+  project:
+    type: 'string'
+  release-model:
+    type: 'string'
+    enum: ['stable', 'non-release']
+  project-type:
+    type: 'string'
+    enum: ['installer', 'testing', 'feature', 'tools', 'infra']
+  upstream:
+    type: 'string'
+  releases:
+    type: 'array'
+    items:
+      type: 'object'
+      properties:
+        version:
+          type: 'string'
+          # Matches semantic versioning (X.Y.Z)
+          pattern: '^opnfv-([0-9]+\.){2}[0-9]+$'
+        location:
+          type: 'object'
+      required: ['version', 'location']
+      additionalProperties: false
+  branches:
+    type: 'array'
+    items:
+      type: 'object'
+      properties:
+        name:
+          type: 'string'
+          pattern: '^stable/[a-z]+$'
+        location:
+          type: 'object'
+      required: ['name', 'location']
+      additionalProperties: false
+  release-notes:
+    type: 'string'
+    format: 'uri'
diff --git a/releases/scripts/create_branch.py b/releases/scripts/create_branch.py
new file mode 100644
index 000000000..8de130972
--- /dev/null
+++ b/releases/scripts/create_branch.py
@@ -0,0 +1,136 @@
+#!/usr/bin/env python2
+# SPDX-License-Identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+"""
+Create Gerrit Branchs
+"""
+
+import argparse
+import ConfigParser
+import logging
+import os
+import yaml
+
+from requests.compat import quote
+from requests.exceptions import RequestException
+
+from pygerrit2.rest import GerritRestAPI
+from pygerrit2.rest.auth import HTTPDigestAuthFromNetrc, HTTPBasicAuthFromNetrc
+
+
+logging.basicConfig(level=logging.INFO)
+
+
+def quote_branch(arguments):
+    """
+    Quote is used here to escape the '/' in branch name. By
+    default '/' is listed in 'safe' characters which aren't escaped.
+    quote is not used in the data of the PUT request, as quoting for
+    arguments is handled by the request library
+    """
+    new_args = arguments.copy()
+    new_args['branch'] = quote(new_args['branch'], '')
+    return new_args
+
+
+def create_branch(api, arguments):
+    """
+    Create a branch using the Gerrit REST API
+    """
+    logger = logging.getLogger(__file__)
+
+    branch_data = """
+    {
+      "ref": "%(branch)s"
+      "revision": "%(commit)s"
+    }""" % arguments
+
+    # First verify the commit exists, otherwise the branch will be
+    # created at HEAD
+    try:
+        request = api.get("/projects/%(project)s/commits/%(commit)s" %
+                          arguments)
+        logger.debug(request)
+        logger.debug("Commit exists: %(commit)s", arguments)
+    except RequestException as err:
+        if hasattr(err, 'response') and err.response.status_code in [404]:
+            logger.warn("Commit %(commit)s for %(project)s:%(branch)s does"
+                        " not exist. Not creating branch.", arguments)
+        else:
+            logger.error("Error: %s", str(err))
+        # Skip trying to create the branch
+        return
+
+    # Try to create the branch and let us know if it already exist.
+    try:
+        request = api.put("/projects/%(project)s/branches/%(branch)s" %
+                          quote_branch(arguments), branch_data)
+        logger.info("Branch %(branch)s for %(project)s successfully created",
+                    arguments)
+    except RequestException as err:
+        if hasattr(err, 'response') and err.response.status_code in [412, 409]:
+            logger.info("Branch %(branch)s already created for %(project)s",
+                        arguments)
+        else:
+            logger.error("Error: %s", str(err))
+
+
+def main():
+    """Given a yamlfile that follows the release syntax, create branches
+    in Gerrit listed under branches"""
+
+    config = ConfigParser.ConfigParser()
+    config.read(os.path.join(os.path.abspath(os.path.dirname(__file__)),
+                'defaults.cfg'))
+    config.read([os.path.expanduser('~/releases.cfg'), 'releases.cfg'])
+
+    gerrit_url = config.get('gerrit', 'url')
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--file', '-f',
+                        type=argparse.FileType('r'),
+                        required=True)
+    parser.add_argument('--basicauth', '-b', action='store_true')
+    args = parser.parse_args()
+
+    GerritAuth = HTTPDigestAuthFromNetrc
+    if args.basicauth:
+        GerritAuth = HTTPBasicAuthFromNetrc
+
+    try:
+        auth = GerritAuth(url=gerrit_url)
+    except ValueError, err:
+        logging.error("%s for %s", err, gerrit_url)
+        quit(1)
+    restapi = GerritRestAPI(url=gerrit_url, auth=auth)
+
+    project = yaml.safe_load(args.file)
+
+    create_branches(restapi, project)
+
+
+def create_branches(restapi, project):
+    """Create branches for a specific project defined in the release
+    file"""
+
+    branches = []
+    for branch in project['branches']:
+        repo, ref = next(iter(branch['location'].items()))
+        branches.append({
+            'project': repo,
+            'branch': branch['name'],
+            'commit': ref
+        })
+
+    for branch in branches:
+        create_branch(restapi, branch)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/releases/scripts/create_jobs.py b/releases/scripts/create_jobs.py
new file mode 100644
index 000000000..2478217a9
--- /dev/null
+++ b/releases/scripts/create_jobs.py
@@ -0,0 +1,145 @@
+#!/usr/bin/env python2
+# SPDX-License-Identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+"""
+Create Gerrit Branches
+"""
+
+import argparse
+import logging
+import os
+import re
+import yaml
+import subprocess
+
+# import ruamel
+from ruamel.yaml import YAML
+
+
+logging.basicConfig(level=logging.INFO)
+
+
+def has_string(filepath, string):
+    """
+    Return True if the given filepath contains the regex string
+    """
+    with open(filepath) as yaml_file:
+        for line in yaml_file:
+            if string.search(line):
+                return True
+    return False
+
+
+def jjb_files(project, release):
+    """
+    Return sets of YAML file names that contain 'stream' for a given
+    project, and file that already contain the stream.
+    """
+    files, skipped = set(), set()
+    file_ending = re.compile(r'ya?ml$')
+    search_string = re.compile(r'^\s+stream:')
+    release_string = re.compile(r'- %s:' % release)
+    jjb_path = os.path.join('jjb', project)
+
+    if not os.path.isdir(jjb_path):
+        logging.warn("JJB directory does not exist at %s, skipping job "
+                     "creation", jjb_path)
+        return (files, skipped)
+
+    for file_name in os.listdir(jjb_path):
+        file_path = os.path.join(jjb_path, file_name)
+        if os.path.isfile(file_path) and file_ending.search(file_path):
+            if has_string(file_path, release_string):
+                skipped.add(file_path)
+            elif has_string(file_path, search_string):
+                files.add(file_path)
+    return (files, skipped)
+
+
+def main():
+    """
+    Create Jenkins Jobs for stable branches in Release File
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--file', '-f',
+                        type=argparse.FileType('r'),
+                        required=True)
+    args = parser.parse_args()
+
+    project_yaml = yaml.safe_load(args.file)
+
+    # Get the release name from the file path
+    release = os.path.split(os.path.dirname(args.file.name))[1]
+
+    create_jobs(release, project_yaml)
+
+
+def create_jobs(release, project_yaml):
+    """Add YAML to JJB files for release stream"""
+    logger = logging.getLogger(__file__)
+
+    # We assume here project keep their subrepo jobs under the part
+    # project name. Otherwise we'll have to look for jjb/<repo> for each
+    # branch listed.
+    project, _ = next(iter(project_yaml['branches'][0]['location'].items()))
+
+    yaml_parser = YAML()
+    yaml_parser.preserve_quotes = True
+    yaml_parser.explicit_start = True
+    # yaml_parser.indent(mapping=4, sequence=0, offset=0)
+    # These are some esoteric values that produce indentation matching our jjb
+    # configs
+    # yaml_parser.indent(mapping=3, sequence=3, offset=2)
+    # yaml_parser.indent(sequence=4, offset=2)
+    yaml_parser.indent(mapping=2, sequence=4, offset=2)
+
+    (job_files, skipped_files) = jjb_files(project, release)
+
+    if skipped_files:
+        logger.info("Jobs already exists for %s in files: %s",
+                    project, ', '.join(skipped_files))
+    # Exit if there are not jobs to create
+    if not job_files:
+        return
+    logger.info("Creating Jenkins Jobs for %s in files: %s",
+                project, ', '.join(job_files))
+
+    stable_branch_stream = """\
+      %s:
+          branch: 'stable/{stream}'
+          gs-pathname: '/{stream}'
+          disabled: false
+    """ % release
+
+    stable_branch_yaml = yaml_parser.load(stable_branch_stream)
+    stable_branch_yaml[release].yaml_set_anchor(release, always_dump=True)
+
+    for job_file in job_files:
+        yaml_jjb = yaml_parser.load(open(job_file))
+        if 'stream' not in yaml_jjb[0]['project']:
+            continue
+
+        # TODO: Some JJB files don't have 'stream'
+        project_config = yaml_jjb[0]['project']['stream']
+        # There is an odd issue where just appending adds a newline before the
+        # branch config, so we append (presumably after master) instead.
+        project_config.insert(1, stable_branch_yaml)
+
+        # NOTE: In the future, we may need to override one or multiple of the
+        #       following ruamal Emitter methods:
+        #         * ruamel.yaml.emitter.Emitter.expect_block_sequence_item
+        #         * ruamel.yaml.emitter.Emitter.write_indent
+        #       To hopefully replace the need to shell out to sed...
+        yaml_parser.dump(yaml_jjb, open(job_file, 'w'))
+        args = ['sed', '-i', 's/^  //', job_file]
+        subprocess.Popen(args, stdout=subprocess.PIPE, shell=False)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/releases/scripts/defaults.cfg b/releases/scripts/defaults.cfg
new file mode 100644
index 000000000..47bf09129
--- /dev/null
+++ b/releases/scripts/defaults.cfg
@@ -0,0 +1,2 @@
+[gerrit]
+url=https://gerrit.opnfv.org/
diff --git a/releases/scripts/requirements.txt b/releases/scripts/requirements.txt
new file mode 100644
index 000000000..5a7d216e9
--- /dev/null
+++ b/releases/scripts/requirements.txt
@@ -0,0 +1,5 @@
+pygerrit2 < 2.1.0
+PyYAML < 4.0
+jsonschema < 2.7.0
+rfc3987
+ruamel.yaml
diff --git a/releases/scripts/verify_schema.py b/releases/scripts/verify_schema.py
new file mode 100644
index 000000000..3a6163e2a
--- /dev/null
+++ b/releases/scripts/verify_schema.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python2
+# SPDX-License-Identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2018 The Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+"""
+Verify YAML Schema
+"""
+import argparse
+import logging
+import jsonschema
+import yaml
+
+LOADER = yaml.CSafeLoader if yaml.__with_libyaml__ else yaml.SafeLoader
+
+
+def main():
+    """
+    Parse arguments and verify YAML
+    """
+    logging.basicConfig(level=logging.INFO)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--yaml', '-y', type=str, required=True)
+    parser.add_argument('--schema', '-s', type=str, required=True)
+
+    args = parser.parse_args()
+
+    with open(args.yaml) as _:
+        yaml_file = yaml.load(_, Loader=LOADER)
+
+    with open(args.schema) as _:
+        schema_file = yaml.load(_, Loader=LOADER)
+
+    # Load the schema
+    validation = jsonschema.Draft4Validator(
+        schema_file,
+        format_checker=jsonschema.FormatChecker()
+    )
+
+    # Look for errors
+    errors = 0
+    for error in validation.iter_errors(yaml_file):
+        errors += 1
+        logging.error(error)
+    if errors > 0:
+        raise RuntimeError("%d issues invalidate the release schema" % errors)
+
+
+if __name__ == "__main__":
+    main()