diff options
Diffstat (limited to 'utils')
| -rwxr-xr-x | utils/artifacts.opnfv.org.sh | 162 | ||||
| -rw-r--r-- | utils/build-server-ansible/vars/docker-compose-CentOS.yml (renamed from utils/build-server-ansible/vars/docker-compose-Centos.yml) | 0 | ||||
| -rwxr-xr-x | utils/fetch_k8_conf.sh | 6 | ||||
| -rwxr-xr-x | utils/jenkins-jnlp-connect.sh | 25 | ||||
| -rwxr-xr-x[-rw-r--r--] | utils/slave-monitor-0.1.sh | 101 |
5 files changed, 230 insertions, 64 deletions
diff --git a/utils/artifacts.opnfv.org.sh b/utils/artifacts.opnfv.org.sh new file mode 100755 index 000000000..1984b4992 --- /dev/null +++ b/utils/artifacts.opnfv.org.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#export PATH=${PATH}:/root/gsutil + +#Step Generate index.html +if [ -f index.html ] ; then + rm -f index.html +fi + +OUTPUT="index.html" + +for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs | grep -v review | awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,) +do +echo $index + echo "<LI><a href=\"${index%/*}.html\">"$index"</a></LI>" >> $OUTPUT +done + +#functest logs########################## + +for project in functest vswitchperf +do + + for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, ) + do + index="$(echo ${index%/*} | sed s,/,_,g)" + echo "<LI><a href=\"http://artifacts.opnfv.org/${index%/*}.html\">"$index"</a></LI>" >> $OUTPUT + done + +done +#End step 1 +##################################### + + +#genrate html files for all project except vswitchperf +for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,) +do +OUTPUT=${index%/*}.html +rm -f $OUTPUT + + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi +done + +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + +done + + + +#generate file for vswitch perf (I dont know what happend here but there is a wierd character in this bucket) + +index=vswitchperf +OUTPUT=${index%/*}.html +rm -f $OUTPUT + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi + +done + + +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + +# Gerate html for logs + +for project in functest vswitchperf +do + for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, ) + do + + OUTPUT="$(echo ${index%/*}.html | sed s,/,_,g)" + echo $OUTPUT + rm -f $OUTPUT + + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi + + + done + + + gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + + gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + + + done +done + + + +OUTPUT="index.html" +echo "<p> Generated on $(date) </p>" >> $OUTPUT + +cat <<EOF >> $OUTPUT +<script> +(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ +(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), +m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) +})(window,document,'script','//www.google-analytics.com/analytics.js','ga'); +ga('create', 'UA-831873-26', 'auto'); +ga('send', 'pageview'); +</script> +EOF + +#copy and uplad index file genrated in first step, last +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ diff --git a/utils/build-server-ansible/vars/docker-compose-Centos.yml b/utils/build-server-ansible/vars/docker-compose-CentOS.yml index fc4bcba7e..fc4bcba7e 100644 --- a/utils/build-server-ansible/vars/docker-compose-Centos.yml +++ b/utils/build-server-ansible/vars/docker-compose-CentOS.yml diff --git a/utils/fetch_k8_conf.sh b/utils/fetch_k8_conf.sh index f82fa5497..32c07f8b6 100755 --- a/utils/fetch_k8_conf.sh +++ b/utils/fetch_k8_conf.sh @@ -52,12 +52,14 @@ else touch $dest_path || error "Cannot create the file specified. Check that the path is correct and run the script again." fi -info "Fetching admin.conf file..." if [ "$installer_type" == "compass" ]; then + info "Fetching admin.conf file on Compass" sudo docker cp compass-tasks:/opt/admin.conf $dest_path &> /dev/null sudo chown $(whoami):$(whoami) $dest_path info "Fetch admin.conf successfully" +elif [ "$installer_type" == "joid" ]; then + info "Do nothing, config file has been provided in $HOME/joid_config/config for joid" else - error "Installer $installer is not supported by this script" + error "Installer $installer_type is not supported by this script" fi diff --git a/utils/jenkins-jnlp-connect.sh b/utils/jenkins-jnlp-connect.sh index cd81f29d3..0647d3210 100755 --- a/utils/jenkins-jnlp-connect.sh +++ b/utils/jenkins-jnlp-connect.sh @@ -11,10 +11,13 @@ #Monit setup script for opnfv jnlp slave connections test_firewall() { - echo "testing that the firewall is open for us at build.opnfv.org" - test=$(echo "blah"| nc -w 4 build.opnfv.org 57387 > /dev/null 2>&1; echo $?) +jenkins_hostname="${jenkins_hostname:-build.opnfv.org}" + + + echo "testing that the firewall is open for us at $jenkins_hostname" + test=$(echo "blah"| nc -w 4 $jenkins_hostname 57387 > /dev/null 2>&1; echo $?) if [[ $test == 0 ]]; then - echo "Firewall is open for us at build.opnfv.org" + echo "Firewall is open for us at $jenkins_hostname" exit 0 else cat << EOF @@ -84,8 +87,14 @@ main () { if [ -d /etc/monit/conf.d ]; then monitconfdir="/etc/monit/conf.d/" + monitconfig="/etc/monit/monitrc" + #add start delay + sed -i '/^#.* delay /s/^#//' "$monitconfig" elif [ -d /etc/monit.d ]; then monitconfdir="/etc/monit.d" + monitconfig="/etc/monitrc" + #add start delay + sed -i '/^#.* delay /s/^#//' "$monitconfig" else echo "Could not determine the location of the monit configuration file." echo "Make sure monit is installed." @@ -132,7 +141,7 @@ depends on jenkins_piddir\ fi if [[ $started_monit == "true" ]]; then - wget --timestamping https://build.opnfv.org/ci/jnlpJars/slave.jar && true + wget --timestamping https://"$jenkins_hostname"/jnlpJars/slave.jar && true chown $jenkinsuser:$jenkinsuser slave.jar if [[ -f /var/run/$jenkinsuser/jenkins_jnlp_pid ]]; then @@ -176,6 +185,7 @@ usage: $0 [OPTIONS] -u set jenkins user -n set slave name -s set secret key + -l set host, default is build.opnfv.org/ci -t test the connection string by connecting without monit -f test firewall @@ -190,13 +200,14 @@ if [[ -z "$@" ]]; then usage fi -while getopts "j:u:n:s:htf" OPTION +while getopts "j:u:n:s:l:htf" OPTION do case $OPTION in j ) jenkinshome="$OPTARG" ;; u ) jenkinsuser="$OPTARG" ;; n ) slave_name="$OPTARG" ;; s ) slave_secret="$OPTARG";; + l ) jenkins_hostname="$OPTARG" ;; h ) usage ;; t ) started_monit=true skip_monit=true @@ -206,6 +217,6 @@ do esac done -connectionstring="java -jar slave.jar -jnlpUrl https://build.opnfv.org/ci/computer/"$slave_name"/slave-agent.jnlp -secret "$slave_secret" -noCertificateCheck " - +jenkins_hostname="${jenkins_hostname:-build.opnfv.org/ci}" +connectionstring="java -jar slave.jar -jnlpUrl https://"$jenkins_hostname"/computer/"$slave_name"/slave-agent.jnlp -secret "$slave_secret" -noCertificateCheck " main "$@" diff --git a/utils/slave-monitor-0.1.sh b/utils/slave-monitor-0.1.sh index 161aaef21..5201f93d6 100644..100755 --- a/utils/slave-monitor-0.1.sh +++ b/utils/slave-monitor-0.1.sh @@ -8,9 +8,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -#This will put a bunch of files in the pwd. you have been warned. #Counts how long slaves have been online or offline - +#exec 2>/dev/null #Yes I know about jq curlcommand() { @@ -25,74 +24,66 @@ curl -s "https://build.opnfv.org/ci/computer/api/json?tree=computer\[displayName | sed s,\",,g } -if [ -f podoutput-current ]; then - cp podoutput-current podoutput-lastiteration -fi - -curlcommand > podoutput-current +curlcommand > /tmp/podoutput-current -declare -A slavescurrent slaveslastiteration +declare -A slavescurrent while read -r name status ; do slavescurrent["$name"]="$status" -done < <(cat podoutput-current) - -while read -r name status ; do - slaveslastiteration["$name"]=$status -done < <(cat podoutput-lastiteration) - +done < <(cat /tmp/podoutput-current) + +#haste bin stopped allowing post :( +#files=(*online) +#for ((i=0; i<${#files[@]}; i+=9)); do +#./eplot -d -r [-1:74][-1:30] -m ${files[i]} ${files[i+1]} ${files[i+2]} ${files[i+3]} ${files[i+4]} ${files[i+5]} ${files[i+6]} ${files[i+7]} ${files[i+8]} ${files[i+9]} +#done | ./haste.bash +## main () { + for slavename in "${!slavescurrent[@]}"; do - #Slave is online. Mark it down. + + #Slave is online. Mark it down. if [ "${slavescurrent[$slavename]}" == "false" ]; then - if [ -f "$slavename"-offline ]; then - echo "removing offline status from $slavename slave was offline for $(cat "$slavename"-offline ) iterations" - rm "$slavename"-offline - fi - - if ! [ -f "$slavename"-online ]; then - echo "1" > "$slavename"-online - elif [ -f "$slavename"-online ]; then - #read and increment slavename - read -r -d $'\x04' var < "$slavename"-online - ((var++)) - echo -n "ONLINE $slavename " - echo "for $var iterations" - echo "$var" > "$slavename"-online - fi - fi + if ! [ -f /tmp/"$slavename"-online ]; then + echo "1" > /tmp/"$slavename"-online + echo "new online slave file created $slavename ${slavescurrent[$slavename]} up for 1 iterations" + fi - #went offline since last iteration. - if [ "${slavescurrent[$slavename]}" == "false" ] && [ "${slaveslastiteration[$slavename]}" == "true" ]; then - echo "JUST WENT OFFLINE $slavename " - if [ -f "$slavename"-online ]; then - echo "removing online status from $slavename. slave was online for $(cat "$slavename"-online ) iterations" - rm "$slavename"-online - fi + #read and increment slavename + var="$(cat /tmp/"$slavename"-online |tail -n 1)" + if [[ "$var" == "0" ]]; then + echo "slave $slavename ${slavescurrent[$slavename]} back up for $var iterations" + fi + ((var++)) + echo "$var" >> /tmp/"$slavename"-online + unset var + echo "$slavename up $(cat /tmp/$slavename-online | tail -n 10 | xargs)" fi - #slave is offline + #slave is offline remove all points if [ "${slavescurrent[$slavename]}" == "true" ]; then - if ! [ -f "$slavename"-offline ]; then - echo "1" > "$slavename"-offline - fi - - if [ -f "$slavename"-offline ]; then - #read and increment slavename - read -r -d $'\x04' var < "$slavename"-offline - ((var++)) - echo "$var" > "$slavename"-offline - if [ "$var" -gt "30" ]; then - echo "OFFLINE FOR $var ITERATIONS REMOVE $slavename " - else - echo "OFFLINE $slavename FOR $var ITERATIONS " - fi - fi + if ! [ -f /tmp/"$slavename"-online ]; then + echo "0" > /tmp/"$slavename"-online + echo "new offline slave file created $slavename ${slavescurrent[$slavename]} up for 0 iterations" + + fi + var="$(cat /tmp/"$slavename"-online |tail -n 1)" + + if [[ "$var" != "0" ]]; then + echo "slave $slavename ${slavescurrent[$slavename]} was up for $var iterations" + echo "slave $slavename ${slavescurrent[$slavename]} has gone offline, was $var iterations now reset to 0" + fi + + echo "0" >> /tmp/"$slavename"-online + echo "$slavename down $(cat /tmp/$slavename-online | tail -n 10 | xargs)" + unset var + fi + done } -main +main | sort | column -t |