Enable generation of composite docs for opnfvdocs

Please note that the upload of documents to artifacts.opnfv.org has not been
enabled yet in order to get feedback for the patch first. Once the patch is
reviewed, uploading generated docs to artifacts.opnfv.org can be enabled.

Another thing is that this could essentially be done in existing opnfv-docs
jobs. But in order not to break doc stuff totally, it's kept separate for the
timebeing.

Depending on the review, we can either
- move this functionality to releng-macros.yml and jobs in opnfv-docs.yml
or
- keep it separate as it is and exclude opnfvdocs from the pattern in
opnfv-docs.yml jobs.

JIRA: RELENG-38

Change-Id: I452547921ef56960330bcd9ba28159108a33c860
Signed-off-by: Fatih Degirmenci <fatih.degirmenci@ericsson.com>

3 files changed, 211 insertions, 5 deletions

diff --git a/jjb/opnfvdocs/opnfvdocs.yml b/jjb/opnfvdocs/opnfvdocs.yml
new file mode 100644
index 000000000..25e546b39
--- /dev/null
+++ b/jjb/opnfvdocs/opnfvdocs.yml
@@ -0,0 +1,169 @@
+########################
+# Job configuration for opnfvdocs
+########################
+- project:
+
+    name: opnfvdocs
+
+    project: '{name}'
+
+    jobs:
+        - 'opnfvdocs-verify-{stream}'
+        - 'opnfvdocs-merge-{stream}'
+        - 'opnfvdocs-daily-{stream}'
+
+    stream:
+        - master:
+            branch: '{stream}'
+            gs-pathname: ''
+        - brahmaputra:
+            branch: 'stable/{stream}'
+            gs-pathname: '/{stream}'
+
+########################
+# job templates
+########################
+
+- job-template:
+    name: 'opnfvdocs-verify-{stream}'
+
+    parameters:
+        - project-parameter:
+            project: $GERRIT_PROJECT
+        - gerrit-parameter:
+            branch: '{branch}'
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: '$GERRIT_REFSPEC'
+            choosing-strategy: 'gerrit'
+
+    triggers:
+        - gerrit:
+            trigger-on:
+                - patchset-created-event:
+                    exclude-drafts: 'false'
+                    exclude-trivial-rebase: 'false'
+                    exclude-no-code-change: 'false'
+                - draft-published-event
+                - comment-added-contains-event:
+                    comment-contains-value: 'recheck'
+                - comment-added-contains-event:
+                    comment-contains-value: 'reverify'
+            projects:
+              - project-compare-type: 'REG_EXP'
+                project-pattern: '{project}'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+                file-paths:
+                  - compare-type: ANT
+                    pattern: 'docs/**'
+
+    builders:
+        - clone-opnfv-repos
+        - build-html-and-pdf-docs-output
+#        - upload-under-review-docs-to-opnfv-artifacts
+        - report-docs-build-result-to-gerrit
+
+- job-template:
+    name: 'opnfvdocs-merge-{stream}'
+
+    parameters:
+        - project-parameter:
+            project: $GERRIT_PROJECT
+        - gerrit-parameter:
+            branch: '{branch}'
+        - string:
+            name: GS_URL
+            default: '$GS_BASE{gs-pathname}'
+            description: "Directory where the build artifact will be located upon the completion of the build."
+
+    scm:
+        - gerrit-trigger-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: '$GERRIT_REFSPEC'
+            choosing-strategy: 'gerrit'
+
+    triggers:
+        - gerrit:
+            trigger-on:
+                - change-merged-event
+                - comment-added-contains-event:
+                    comment-contains-value: 'remerge'
+            projects:
+              - project-compare-type: 'REG_EXP'
+                project-pattern: '{project}'
+                branches:
+                  - branch-compare-type: 'ANT'
+                    branch-pattern: '**/{branch}'
+                file-paths:
+                  - compare-type: ANT
+                    pattern: 'docs/**'
+
+    builders:
+        - clone-opnfv-repos
+        - build-html-and-pdf-docs-output
+#        - upload-generated-docs-to-opnfv-artifacts
+        - report-docs-build-result-to-gerrit
+        - remove-old-docs-from-opnfv-artifacts
+
+- job-template:
+    name: 'opnfvdocs-daily-{stream}'
+
+    parameters:
+        - project-parameter:
+            project: '{project}'
+        - string:
+            name: GS_URL
+            default: '$GS_BASE{gs-pathname}'
+            description: "Directory where the build artifact will be located upon the completion of the build."
+        - string:
+            name: GERRIT_BRANCH
+            default: '{branch}'
+            description: 'Specify the branch in this way in order to be able to use clone-opnfv-repos builder.'
+
+    scm:
+        - git-scm:
+            credentials-id: '{ssh-credentials}'
+            refspec: ''
+            branch: '{branch}'
+
+    triggers:
+        - timed: '0 H/6 * * *'
+
+    builders:
+        - clone-opnfv-repos
+        - build-html-and-pdf-docs-output
+#        - upload-generated-docs-to-opnfv-artifacts
+
+- builder:
+    name: clone-opnfv-repos
+    builders:
+        - shell: |
+            #!/bin/bash
+            set -o errexit
+            set -o nounset
+            set -o pipefail
+
+            # clone releng repo to get repo list
+            cd $WORKSPACE
+            echo "Cloning releng repo"
+            git clone $GIT_BASE/releng --quiet
+
+            # clone rest of the repos and checkout the branch
+            echo "Cloning repos of participating OPNFV Projects and checking out $GERRIT_BRANCH"
+            echo
+            echo "--------------------------------------------------------"
+            for repo in $(grep -v '^#' $WORKSPACE/releng/jjb/opnfvdocs/project.cfg | sort); do
+                cd $WORKSPACE
+                echo "    $repo"
+                git clone $GIT_BASE/$repo --branch $GERRIT_BRANCH --quiet
+            done
+            echo "--------------------------------------------------------"
+            echo
+            echo "Done"
+
+            # remove releng clone in order not to cause issues for rest of the builders
+            /bin/rm -rf $WORKSPACE/releng
diff --git a/jjb/opnfvdocs/project.cfg b/jjb/opnfvdocs/project.cfg
new file mode 100644
index 000000000..6c6025936
--- /dev/null
+++ b/jjb/opnfvdocs/project.cfg
@@ -0,0 +1,37 @@
+apex
+armband
+availability
+bottlenecks
+compass4nfv
+copper
+doctor
+dpacc
+escalator
+fastpathmetrics
+fuel
+functest
+genesisreq
+ipv6
+joid
+kvmfornfv
+lsoapi
+moon
+movie
+multisite
+octopus
+onosfw
+ovno
+ovsnfv
+parser
+pharos
+policytest
+prediction
+promise
+qtip
+rs
+sdnvpn
+sfc
+storperf
+vnf_forwarding_graph
+vsperf
+yardstick
diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml
index 787b7bacc..9d6a54260 100644
--- a/jjb/releng-macros.yaml
+++ b/jjb/releng-macros.yaml
@@ -201,7 +201,7 @@
                 sed -e "s|^$local_path|    http://$gs_path|" >> gerrit_comment.txt
 
 - builder:
-    name: upload-merged-docs-to-opnfv-artifacts
+    name: upload-generated-docs-to-opnfv-artifacts
     builders:
         - shell: |
             #!/bin/bash -e
@@ -211,9 +211,9 @@
             [[ -d output ]]
 
             echo
-            echo "#####################"
-            echo "UPLOADING MERGED DOCS"
-            echo "#####################"
+            echo "########################"
+            echo "UPLOADING GENERATED DOCS"
+            echo "########################"
             echo
 
             gs_path="$GS_URL/docs"
@@ -280,6 +280,6 @@
     name: upload-merged-docs
     builders:
         - build-html-and-pdf-docs-output
-        - upload-merged-docs-to-opnfv-artifacts
+        - upload-generated-docs-to-opnfv-artifacts
         - report-docs-build-result-to-gerrit
         - remove-old-docs-from-opnfv-artifacts