Move generation of artifacts.opnfv.org to jenkins

Move generation of artifacts.opnfv.org/index.html
and sub pages into a jenkins job.

Script was run on cron on master previously.
run on lf-build2

Run job once per hour (it takes a long time to run)

Block if job is already running

Change-Id: Icc03cac61a802e036e4af8a9b9e3a244e147abea
Signed-off-by: agardner <agardner@linuxfoundation.org>

2 files changed, 200 insertions, 0 deletions

diff --git a/jjb/releng/opnfv-utils.yml b/jjb/releng/opnfv-utils.yml
index b12f663aa..93b3615ef 100644
--- a/jjb/releng/opnfv-utils.yml
+++ b/jjb/releng/opnfv-utils.yml
@@ -8,6 +8,7 @@
       - 'archive-repositories'
       - 'check-status-of-slaves'
       - 'ansible-build-server'
+      - 'generate-artifacts-index-pages'
 
 ########################
 # job templates
@@ -164,3 +165,40 @@
           sudo ansible-playbook -C -D -i \
           $WORKSPACE/utils/build-server-ansible/inventory.ini \
           $WORKSPACE/utils/build-server-ansible/main.yml
+
+
+- job-template:
+    name: 'generate-artifacts-index-pages'
+
+    project-type: freestyle
+
+    disabled: false
+
+    concurrent: false
+
+    parameters:
+      - node:
+          name: SLAVE_NAME
+          description: Build Servers
+          default-slaves:
+            - lf-build2
+      - project-parameter:
+          project: releng
+          branch: master
+
+    scm:
+      - git-scm
+
+    triggers:
+      - timed: '@hourly'
+
+    builders:
+      - generate-artifact-html
+
+
+- builder:
+    name: generate-artifact-html
+    builders:
+      - shell: |
+          cd $WORKSPACE/utils/
+          ./artifacts.opnfv.org.sh
diff --git a/utils/artifacts.opnfv.org.sh b/utils/artifacts.opnfv.org.sh
new file mode 100755
index 000000000..1984b4992
--- /dev/null
+++ b/utils/artifacts.opnfv.org.sh
@@ -0,0 +1,162 @@
+#!/bin/bash
+# SPDX-license-identifier: Apache-2.0
+##############################################################################
+# Copyright (c) 2016 Linux Foundation and others.
+# All rights reserved. This program and the accompanying materials
+# are made available under the terms of the Apache License, Version 2.0
+# which accompanies this distribution, and is available at
+# http://www.apache.org/licenses/LICENSE-2.0
+##############################################################################
+
+#export PATH=${PATH}:/root/gsutil
+
+#Step Generate index.html
+if [ -f index.html ] ; then
+      rm -f index.html
+fi
+
+OUTPUT="index.html"
+
+for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs | grep -v review | awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,)
+do
+echo $index
+  echo "<LI><a href=\"${index%/*}.html\">"$index"</a></LI>" >> $OUTPUT
+done
+
+#functest logs##########################
+
+for project in functest vswitchperf
+do
+
+    for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, )
+    do
+    index="$(echo ${index%/*} | sed s,/,_,g)"
+      echo "<LI><a href=\"http://artifacts.opnfv.org/${index%/*}.html\">"$index"</a></LI>" >> $OUTPUT
+    done
+
+done
+#End step 1
+#####################################
+
+
+#genrate html files for all project except vswitchperf
+for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,)
+do
+OUTPUT=${index%/*}.html
+rm -f $OUTPUT
+
+
+    for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do
+    echo $filepath
+
+    if [[ $filepath =~ "/:" ]]; then
+      path=$(echo $filepath| sed s,/:,,g)
+      echo "<UL>" >> $OUTPUT
+      echo "<LI>$path</LI>" >> $OUTPUT
+      echo "</UL>" >> $OUTPUT
+    else
+      echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT
+    fi
+done
+
+gsutil cp $OUTPUT gs://artifacts.opnfv.org/
+
+gsutil -m setmeta \
+     -h "Content-Type:text/html" \
+     -h "Cache-Control:private, max-age=0, no-transform" \
+      gs://artifacts.opnfv.org/$OUTPUT \
+
+done
+
+
+
+#generate file for vswitch perf (I dont know what happend here but there is a wierd character in this bucket)
+
+index=vswitchperf
+OUTPUT=${index%/*}.html
+rm -f $OUTPUT
+
+        for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do
+        echo $filepath
+
+        if [[ $filepath =~ "/:" ]]; then
+          path=$(echo $filepath| sed s,/:,,g)
+          echo "<UL>" >> $OUTPUT
+          echo "<LI>$path</LI>" >> $OUTPUT
+          echo "</UL>" >> $OUTPUT
+        else
+          echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT
+        fi
+
+done
+
+
+gsutil cp $OUTPUT gs://artifacts.opnfv.org/
+
+gsutil -m setmeta \
+     -h "Content-Type:text/html" \
+     -h "Cache-Control:private, max-age=0, no-transform" \
+      gs://artifacts.opnfv.org/$OUTPUT \
+
+# Gerate html for logs
+
+for project in functest vswitchperf
+do
+    for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, )
+    do
+
+        OUTPUT="$(echo ${index%/*}.html | sed s,/,_,g)"
+        echo $OUTPUT
+        rm -f $OUTPUT
+
+
+            for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do
+            echo $filepath
+
+            if [[ $filepath =~ "/:" ]]; then
+              path=$(echo $filepath| sed s,/:,,g)
+              echo "<UL>" >> $OUTPUT
+              echo "<LI>$path</LI>" >> $OUTPUT
+              echo "</UL>" >> $OUTPUT
+            else
+              echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT
+            fi
+
+
+            done
+
+
+        gsutil cp $OUTPUT gs://artifacts.opnfv.org/
+
+        gsutil -m setmeta \
+             -h "Content-Type:text/html" \
+             -h "Cache-Control:private, max-age=0, no-transform" \
+              gs://artifacts.opnfv.org/$OUTPUT \
+
+
+    done
+done
+
+
+
+OUTPUT="index.html"
+echo "<p> Generated on $(date) </p>" >> $OUTPUT
+
+cat <<EOF >> $OUTPUT
+<script>
+(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+})(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+ga('create', 'UA-831873-26', 'auto');
+ga('send', 'pageview');
+</script>
+EOF
+
+#copy and uplad index file genrated in first step, last
+gsutil cp $OUTPUT gs://artifacts.opnfv.org/
+
+gsutil -m setmeta \
+     -h "Content-Type:text/html" \
+     -h "Cache-Control:private, max-age=0, no-transform" \
+      gs://artifacts.opnfv.org/$OUTPUT \