diff options
Diffstat (limited to 'utils')
| -rwxr-xr-x | utils/artifacts.opnfv.org.sh | 162 | ||||
| -rw-r--r-- | utils/build-server-ansible/inventory.ini | 8 | ||||
| -rw-r--r-- | utils/build-server-ansible/main.yml | 37 | ||||
| -rw-r--r-- | utils/build-server-ansible/vars/CentOS.yml | 72 | ||||
| -rw-r--r-- | utils/build-server-ansible/vars/Ubuntu.yml | 84 | ||||
| -rw-r--r-- | utils/build-server-ansible/vars/defaults.yml | 23 | ||||
| -rw-r--r-- | utils/build-server-ansible/vars/docker-compose-CentOS.yml | 12 | ||||
| -rw-r--r-- | utils/build-server-ansible/vars/docker-compose-Ubuntu.yml | 12 | ||||
| -rwxr-xr-x | utils/jenkins-jnlp-connect.sh | 4 | ||||
| -rwxr-xr-x[-rw-r--r--] | utils/slave-monitor-0.1.sh | 101 |
10 files changed, 458 insertions, 57 deletions
diff --git a/utils/artifacts.opnfv.org.sh b/utils/artifacts.opnfv.org.sh new file mode 100755 index 000000000..1984b4992 --- /dev/null +++ b/utils/artifacts.opnfv.org.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#export PATH=${PATH}:/root/gsutil + +#Step Generate index.html +if [ -f index.html ] ; then + rm -f index.html +fi + +OUTPUT="index.html" + +for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs | grep -v review | awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,) +do +echo $index + echo "<LI><a href=\"${index%/*}.html\">"$index"</a></LI>" >> $OUTPUT +done + +#functest logs########################## + +for project in functest vswitchperf +do + + for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, ) + do + index="$(echo ${index%/*} | sed s,/,_,g)" + echo "<LI><a href=\"http://artifacts.opnfv.org/${index%/*}.html\">"$index"</a></LI>" >> $OUTPUT + done + +done +#End step 1 +##################################### + + +#genrate html files for all project except vswitchperf +for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,) +do +OUTPUT=${index%/*}.html +rm -f $OUTPUT + + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi +done + +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + +done + + + +#generate file for vswitch perf (I dont know what happend here but there is a wierd character in this bucket) + +index=vswitchperf +OUTPUT=${index%/*}.html +rm -f $OUTPUT + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi + +done + + +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + +# Gerate html for logs + +for project in functest vswitchperf +do + for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, ) + do + + OUTPUT="$(echo ${index%/*}.html | sed s,/,_,g)" + echo $OUTPUT + rm -f $OUTPUT + + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi + + + done + + + gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + + gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + + + done +done + + + +OUTPUT="index.html" +echo "<p> Generated on $(date) </p>" >> $OUTPUT + +cat <<EOF >> $OUTPUT +<script> +(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ +(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), +m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) +})(window,document,'script','//www.google-analytics.com/analytics.js','ga'); +ga('create', 'UA-831873-26', 'auto'); +ga('send', 'pageview'); +</script> +EOF + +#copy and uplad index file genrated in first step, last +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ diff --git a/utils/build-server-ansible/inventory.ini b/utils/build-server-ansible/inventory.ini new file mode 100644 index 000000000..115b1306e --- /dev/null +++ b/utils/build-server-ansible/inventory.ini @@ -0,0 +1,8 @@ +############################################################################# +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +localhost ansible_connection=local diff --git a/utils/build-server-ansible/main.yml b/utils/build-server-ansible/main.yml new file mode 100644 index 000000000..0fcce715d --- /dev/null +++ b/utils/build-server-ansible/main.yml @@ -0,0 +1,37 @@ +############################################################################ +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################# +--- +- hosts: "localhost" + become: "True" + tasks: + - debug: + msg: "{{ inventory_hostname }} is {{ ansible_distribution }}" + - include_vars: vars/defaults.yml + - include: vars/CentOS.yml + when: ansible_distribution == "CentOS" + - include: vars/Ubuntu.yml + when: ansible_distribution == "Ubuntu" + - name: Install Docker. + package: name={{ docker_package }} state={{ docker_package_state }} + - name: Ensure Docker is started and enabled at boot. + service: + name: docker + state: started + enabled: "yes" + - name: install gsutil + pip: + name: gsutil + state: present + - name: install tox + pip: + name: tox + state: present + - include: vars/docker-compose-CentOS.yml + when: ansible_distribution == "CentOS" + - include: vars/docker-compose-Ubuntu.yml + when: ansible_distribution == "Ubuntu" diff --git a/utils/build-server-ansible/vars/CentOS.yml b/utils/build-server-ansible/vars/CentOS.yml new file mode 100644 index 000000000..0d5a01111 --- /dev/null +++ b/utils/build-server-ansible/vars/CentOS.yml @@ -0,0 +1,72 @@ +############################################################################ +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################ +--- +- name: Ensure old versions of Docker are not installed. + package: + name: '{{ item }}' + state: absent + with_items: + - docker + - docker-common + - docker-engine + +- name: Add Docker GPG key. + rpm_key: + key: https://download.docker.com/linux/centos/gpg + state: present + +- name: Ensure epel is installed. + yum: + name: epel-release + state: present +- name: Ensure depdencies are installed. + yum: + name: "{{ item }}" + state: present + with_items: + - python-pip + - rpm-build + - kernel-headers + - libpcap-devel + - zlib-devel + - numactl-devel + - doxygen + - python-sphinx + - libvirt-devel + - python-devel + - openssl-devel + - python-six + - net-tools + - bc + +- name: install the 'Development tools' package group + yum: + name: "@Development tools" + state: present + +- name: Add Docker repository. + get_url: + url: "{{ docker_yum_repo_url }}" + dest: '/etc/yum.repos.d/docker-ce.repo' + owner: root + group: root + mode: 0644 + +- name: Configure Docker Edge repo. + ini_file: + dest: '/etc/yum.repos.d/docker-ce.repo' + section: 'docker-ce-edge' + option: enabled + value: '{{ docker_yum_repo_enable_edge }}' + +- name: Configure Docker Test repo. + ini_file: + dest: '/etc/yum.repos.d/docker-ce.repo' + section: 'docker-ce-test' + option: enabled + value: '{{ docker_yum_repo_enable_test }}' diff --git a/utils/build-server-ansible/vars/Ubuntu.yml b/utils/build-server-ansible/vars/Ubuntu.yml new file mode 100644 index 000000000..609c8d548 --- /dev/null +++ b/utils/build-server-ansible/vars/Ubuntu.yml @@ -0,0 +1,84 @@ +############################################################################# +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################# +--- +- name: Ensure old versions of Docker are not installed. + package: + name: '{{ item }}' + state: absent + with_items: + - docker + - docker-engine + +- name: Ensure depdencies are installed. + apt: + name: "{{ item }}" + state: present + with_items: + - apt-transport-https + - ca-certificates + - git + - build-essential + - curl + - wget + - rpm + - fuseiso + - createrepo + - genisoimage + - libfuse-dev + - dh-autoreconf + - pkg-config + - zlib1g-dev + - libglib2.0-dev + - libpixman-1-dev + - python-virtualenv + - python-dev + - libffi-dev + - libssl-dev + - libxml2-dev + - libxslt1-dev + - bc + - qemu-kvm + - libvirt-bin + - ubuntu-vm-builder + - bridge-utils + - monit + - openjdk-8-jre-headless + - python-nose + - dirmngr + - collectd + - flex + - bison + - libnuma-dev + - shellcheck + - python-pip + +- name: Add Docker apt key. + apt_key: + url: https://download.docker.com/linux/ubuntu/gpg + id: 9DC858229FC7DD38854AE2D88D81803C0EBFCD88 + state: present + register: add_repository_key + ignore_errors: true + +- name: Ensure curl is present (on older systems without SNI). + package: name=curl state=present + when: add_repository_key|failed + +- name: Add Docker apt key (alternative for older systems without SNI). + # yamllint disable rule:line-length + shell: "curl -sSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -" + # yamllint enable rule:line-length + args: + warn: "no" + when: add_repository_key|failed + +- name: Add Docker repository. + apt_repository: + repo: "{{ docker_apt_repository }}" + state: present + update_cache: "yes" diff --git a/utils/build-server-ansible/vars/defaults.yml b/utils/build-server-ansible/vars/defaults.yml new file mode 100644 index 000000000..8d83380dd --- /dev/null +++ b/utils/build-server-ansible/vars/defaults.yml @@ -0,0 +1,23 @@ +############################################################################# +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################# +--- +docker_package: "docker-ce" +docker_package_state: present + +# Used only for Debian/Ubuntu. Switch 'stable' to 'edge' if needed. +docker_apt_release_channel: stable +# yamllint disable rule:line-length +docker_apt_repository: "deb https://download.docker.com/linux/{{ ansible_distribution|lower }} {{ ansible_distribution_release }} {{ docker_apt_release_channel }}" +# yamllint enable rule:line-length + +# Used only for RedHat/CentOS. +# yamllint disable rule:line-length +docker_yum_repo_url: https://download.docker.com/linux/centos/docker-ce.repo +# yamllint enable rule:line-length +docker_yum_repo_enable_edge: 0 +docker_yum_repo_enable_test: 0 diff --git a/utils/build-server-ansible/vars/docker-compose-CentOS.yml b/utils/build-server-ansible/vars/docker-compose-CentOS.yml new file mode 100644 index 000000000..fc4bcba7e --- /dev/null +++ b/utils/build-server-ansible/vars/docker-compose-CentOS.yml @@ -0,0 +1,12 @@ +############################################################################# +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +--- +- name: Ensure docker compose is installed. + yum: + name: 'docker-compose' + state: present diff --git a/utils/build-server-ansible/vars/docker-compose-Ubuntu.yml b/utils/build-server-ansible/vars/docker-compose-Ubuntu.yml new file mode 100644 index 000000000..f985b6a4d --- /dev/null +++ b/utils/build-server-ansible/vars/docker-compose-Ubuntu.yml @@ -0,0 +1,12 @@ +############################################################################# +# Copyright (c) 2016 The Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################# +--- +- name: Ensure docker compose is installed + apt: + name: 'docker-compose' + state: present diff --git a/utils/jenkins-jnlp-connect.sh b/utils/jenkins-jnlp-connect.sh index cd81f29d3..f7c67694f 100755 --- a/utils/jenkins-jnlp-connect.sh +++ b/utils/jenkins-jnlp-connect.sh @@ -103,7 +103,7 @@ if does not exist then exec "$mkdir -p /var/run/$jenkinsuser" if failed uid $jenkinsuser then exec "$chown $jenkinsuser /var/run/$jenkinsuser" if failed gid $jenkinsuser then exec "$chown :$jenkinsuser /var/run/$jenkinsuser" -check process jenkins with pidfile /var/run/$jenkinsuser/jenkins_jnlp_pid +check process jenkins with pidfile /var/run/$jenkinsuser/jenkins_jnlp_pid every 2 cycles start program = "/usr/bin/sudo -u $jenkinsuser /bin/bash -c 'cd $jenkinshome; export started_monit=true; $0 $@' with timeout 60 seconds" stop program = "/bin/bash -c '/bin/kill \$(/bin/cat /var/run/$jenkinsuser/jenkins_jnlp_pid)'" depends on jenkins_piddir @@ -118,7 +118,7 @@ if does not exist then exec \"$mkdir -p /var/run/$jenkinsuser\" if failed uid $jenkinsuser then exec \"$chown $jenkinsuser /var/run/$jenkinsuser\" if failed gid $jenkinsuser then exec \"$chown :$jenkinsuser /var/run/$jenkinsuser\" -check process jenkins with pidfile /var/run/$jenkinsuser/jenkins_jnlp_pid +check process jenkins with pidfile /var/run/$jenkinsuser/jenkins_jnlp_pid every 2 cycles start program = \"/usr/bin/sudo -u $jenkinsuser /bin/bash -c 'cd $jenkinshome; export started_monit=true; $0 $@' with timeout 60 seconds\" stop program = \"/bin/bash -c '/bin/kill \$(/bin/cat /var/run/$jenkinsuser/jenkins_jnlp_pid)'\" depends on jenkins_piddir\ diff --git a/utils/slave-monitor-0.1.sh b/utils/slave-monitor-0.1.sh index 161aaef21..5201f93d6 100644..100755 --- a/utils/slave-monitor-0.1.sh +++ b/utils/slave-monitor-0.1.sh @@ -8,9 +8,8 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## -#This will put a bunch of files in the pwd. you have been warned. #Counts how long slaves have been online or offline - +#exec 2>/dev/null #Yes I know about jq curlcommand() { @@ -25,74 +24,66 @@ curl -s "https://build.opnfv.org/ci/computer/api/json?tree=computer\[displayName | sed s,\",,g } -if [ -f podoutput-current ]; then - cp podoutput-current podoutput-lastiteration -fi - -curlcommand > podoutput-current +curlcommand > /tmp/podoutput-current -declare -A slavescurrent slaveslastiteration +declare -A slavescurrent while read -r name status ; do slavescurrent["$name"]="$status" -done < <(cat podoutput-current) - -while read -r name status ; do - slaveslastiteration["$name"]=$status -done < <(cat podoutput-lastiteration) - +done < <(cat /tmp/podoutput-current) + +#haste bin stopped allowing post :( +#files=(*online) +#for ((i=0; i<${#files[@]}; i+=9)); do +#./eplot -d -r [-1:74][-1:30] -m ${files[i]} ${files[i+1]} ${files[i+2]} ${files[i+3]} ${files[i+4]} ${files[i+5]} ${files[i+6]} ${files[i+7]} ${files[i+8]} ${files[i+9]} +#done | ./haste.bash +## main () { + for slavename in "${!slavescurrent[@]}"; do - #Slave is online. Mark it down. + + #Slave is online. Mark it down. if [ "${slavescurrent[$slavename]}" == "false" ]; then - if [ -f "$slavename"-offline ]; then - echo "removing offline status from $slavename slave was offline for $(cat "$slavename"-offline ) iterations" - rm "$slavename"-offline - fi - - if ! [ -f "$slavename"-online ]; then - echo "1" > "$slavename"-online - elif [ -f "$slavename"-online ]; then - #read and increment slavename - read -r -d $'\x04' var < "$slavename"-online - ((var++)) - echo -n "ONLINE $slavename " - echo "for $var iterations" - echo "$var" > "$slavename"-online - fi - fi + if ! [ -f /tmp/"$slavename"-online ]; then + echo "1" > /tmp/"$slavename"-online + echo "new online slave file created $slavename ${slavescurrent[$slavename]} up for 1 iterations" + fi - #went offline since last iteration. - if [ "${slavescurrent[$slavename]}" == "false" ] && [ "${slaveslastiteration[$slavename]}" == "true" ]; then - echo "JUST WENT OFFLINE $slavename " - if [ -f "$slavename"-online ]; then - echo "removing online status from $slavename. slave was online for $(cat "$slavename"-online ) iterations" - rm "$slavename"-online - fi + #read and increment slavename + var="$(cat /tmp/"$slavename"-online |tail -n 1)" + if [[ "$var" == "0" ]]; then + echo "slave $slavename ${slavescurrent[$slavename]} back up for $var iterations" + fi + ((var++)) + echo "$var" >> /tmp/"$slavename"-online + unset var + echo "$slavename up $(cat /tmp/$slavename-online | tail -n 10 | xargs)" fi - #slave is offline + #slave is offline remove all points if [ "${slavescurrent[$slavename]}" == "true" ]; then - if ! [ -f "$slavename"-offline ]; then - echo "1" > "$slavename"-offline - fi - - if [ -f "$slavename"-offline ]; then - #read and increment slavename - read -r -d $'\x04' var < "$slavename"-offline - ((var++)) - echo "$var" > "$slavename"-offline - if [ "$var" -gt "30" ]; then - echo "OFFLINE FOR $var ITERATIONS REMOVE $slavename " - else - echo "OFFLINE $slavename FOR $var ITERATIONS " - fi - fi + if ! [ -f /tmp/"$slavename"-online ]; then + echo "0" > /tmp/"$slavename"-online + echo "new offline slave file created $slavename ${slavescurrent[$slavename]} up for 0 iterations" + + fi + var="$(cat /tmp/"$slavename"-online |tail -n 1)" + + if [[ "$var" != "0" ]]; then + echo "slave $slavename ${slavescurrent[$slavename]} was up for $var iterations" + echo "slave $slavename ${slavescurrent[$slavename]} has gone offline, was $var iterations now reset to 0" + fi + + echo "0" >> /tmp/"$slavename"-online + echo "$slavename down $(cat /tmp/$slavename-online | tail -n 10 | xargs)" + unset var + fi + done } -main +main | sort | column -t |