diff options
| -rw-r--r-- | jjb/doctor/doctor.yml | 159 | ||||
| -rw-r--r-- | jjb/doctor/docu-build.sh | 55 | ||||
| -rwxr-xr-x | jjb/doctor/docu-verify.sh | 7 | ||||
| -rw-r--r-- | jjb/functest/functest.yml | 40 | ||||
| -rw-r--r-- | jjb/genesis/genesis-fuel.yml | 4 | ||||
| -rw-r--r-- | jjb/promise/docu-build.sh | 53 | ||||
| -rw-r--r-- | jjb/promise/docu-verify.sh | 7 | ||||
| -rw-r--r-- | jjb/promise/promise.yml | 180 | ||||
| -rw-r--r-- | jjb/yardstick/yardstick.yml | 33 | ||||
| -rwxr-xr-x | utils/fetch_os_creds.sh | 10 |
10 files changed, 175 insertions, 373 deletions
diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index 39c20dc54..aa586642a 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -1,127 +1,45 @@ - project: name: doctor + project: '{name}' jobs: - - 'doctor-test' - - 'doctor-daily-{stream}' - - 'doctor-merge' - - 'doctor-verify' + - '{project}-verify' + - '{project}-merge-{stream}' - # stream: branch with - in place of / (eg. stable-helium) - # branch: branch (eg. stable/helium) + # stream: branch with - in place of / (eg. stable-arno) + # branch: branch (eg. stable/arno) stream: - master: branch: 'master' - project: 'doctor' - somevar: 'foo' - -- job-template: - name: doctor-test - - node: master - - project-type: freestyle - - logrotate: - daysToKeep: 30 - numToKeep: 10 - artifactDaysToKeep: -1 - artifactNumToKeep: -1 - - builders: - - shell: | - echo "Hello world from doctor" - -- job-template: - name: 'doctor-daily-{stream}' - +- job_defaults: &job_defaults + name: job_defaults node: master - - # Job template for daily builders - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - project-type: freestyle - varsetabove: '{somevar}' - logrotate: daysToKeep: '{build-days-to-keep}' numToKeep: '{build-num-to-keep}' artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' - parameters: - project-parameter: project: '{project}' - - string: - name: GIT_BASE - default: 'https://gerrit.opnfv.org/gerrit/{project}' - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - wrappers: - ssh-agent-credentials: - user: '{ssh-credentials}' - - triggers: - - timed: 'H H * * *' - - prebuilders: - - test-macro - - builders: - - shell: - !include-raw docu-build.sh - - postbuilders: - - test-macro + users: + - '{ssh-credentials}' - job-template: - name: 'doctor-verify' - - node: master - - project-type: freestyle - - logrotate: - daysToKeep: 30 - numToKeep: 10 - artifactDaysToKeep: -1 - artifactNumToKeep: -1 - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: 'master' - - string: - name: GIT_BASE - default: 'https://gerrit.opnfv.org/gerrit/{project}' - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + name: '{project}-verify' + <<: *job_defaults scm: - gerrit-trigger-scm: credentials-id: '{ssh-credentials}' refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' + - patchset-created-event - draft-published-event - comment-added-contains-event: comment-contains-value: 'recheck' @@ -129,52 +47,22 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'ANT' - project-pattern: 'doctor' + project-pattern: '{project}' branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/master' - + - branch-compare-type: 'ANT' + branch-pattern: '**' builders: - shell: - !include-raw docu-verify.sh + !include-raw docu-build.sh - job-template: - name: 'doctor-merge' - - node: master - - # builder-merge job to run JJB update - # - # This job's purpose is to update all the JJB - - project-type: freestyle - - logrotate: - daysToKeep: 30 - numToKeep: 40 - artifactDaysToKeep: -1 - artifactNumToKeep: 5 - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: 'master' - - string: - name: GIT_BASE - default: 'https://gerrit.opnfv.org/gerrit/{project}' - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." - + name: '{project}-merge-{stream}' + <<: *job_defaults scm: - gerrit-trigger-scm: credentials-id: '{ssh-credentials}' refspec: '' choosing-strategy: 'default' - - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -183,17 +71,10 @@ comment-contains-value: 'remerge' projects: - project-compare-type: 'ANT' - project-pattern: 'doctor' + project-pattern: '{project}' branches: - branch-compare-type: 'ANT' - branch-pattern: '**/master' - + branch-pattern: '**/{branch}' builders: - shell: !include-raw docu-build.sh - - - - - - diff --git a/jjb/doctor/docu-build.sh b/jjb/doctor/docu-build.sh index 39ecf3bb0..ffb131114 100644 --- a/jjb/doctor/docu-build.sh +++ b/jjb/doctor/docu-build.sh @@ -2,23 +2,46 @@ set -e set -o pipefail -build_dir="build" -project="$(git remote -v | head -n1 | awk '{{print $2}}' | sed -e 's,.*:\(.*/\)\?,,' -e 's/\.git$//')" export PATH=$PATH:/usr/local/bin/ +echo +echo "Build" +echo "-----" +echo + make -# upload all built files -files=( - design_docs - requirements/html - requirements/latex/*.pdf -) - -for file in "${{files[@]}}"; do - gsutil cp -r -L gsoutput.txt $build_dir/$file gs://artifacts.opnfv.org/$project/ - #gsutil setmeta -h "Cache-Control:private, max-age=0, no-transform" \ - #-R gs://artifacts.opnfv.org/$project/$file - cat gsoutput.txt - rm -f gsoutput.txt -done +echo +echo "Upload" +echo "------" +echo + +# NOTE: make sure source parameters for GS paths are not empty. +[[ $GERRIT_CHANGE_NUMBER =~ .+ ]] +[[ $GERRIT_PROJECT =~ .+ ]] +[[ $GERRIT_BRANCH =~ .+ ]] + +gs_path_review="artifacts.opnfv.org/review/$GERRIT_CHANGE_NUMBER" +if [[ $GERRIT_BRANCH = "master" ]] ; then + gs_path_branch="artifacts.opnfv.org/$GERRIT_PROJECT" +else + gs_path_branch="artifacts.opnfv.org/$GERRIT_PROJECT/${{GERRIT_BRANCH##*/}}" +fi + +if [[ $JOB_NAME =~ "verify" ]] ; then + gsutil cp -r build/* "gs://$gs_path_review/" + echo + echo "Document is available at http://$gs_path_review" +else + gsutil cp -r build/design_docs "gs://$gs_path_branch/" + gsutil cp -r build/requirements/html "gs://$gs_path_branch/" + gsutil cp -r build/requirements/latex/*.pdf "gs://$gs_path_branch/" + echo + echo "Latest document is available at http://$gs_path_branch" + + if gsutil ls "gs://$gs_path_review" > /dev/null 2>&1 ; then + echo + echo "Deleting Out-of-dated Documents..." + gsutil rm -r "gs://$gs_path_review" + fi +fi diff --git a/jjb/doctor/docu-verify.sh b/jjb/doctor/docu-verify.sh deleted file mode 100755 index 72d02cfe2..000000000 --- a/jjb/doctor/docu-verify.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -export PATH=$PATH:/usr/local/bin/ - -make diff --git a/jjb/functest/functest.yml b/jjb/functest/functest.yml index 1f284f08a..0d776065f 100644 --- a/jjb/functest/functest.yml +++ b/jjb/functest/functest.yml @@ -3,11 +3,19 @@ ################################### - project: name: functest - pod: - - 'opnfv-jump-1' - - 'opnfv-jump-2' - - 'orange-build' + - opnfv-jump-1: + node: 'opnfv-jump-1' + installer_type: 'fuel' + installer_ip: '10.20.0.2' + - opnfv-jump-2: + node: 'opnfv-jump-2' + installer_type: 'foreman' + installer_ip: '172.30.10.73' + - orange-build: + node: 'orange-build' + installer_type: 'foreman' + installer_ip: '172.30.10.73' jobs: - 'functest-{pod}' - 'set-functest-env-{pod}' @@ -56,7 +64,12 @@ echo "Functest: prepare Functest environment" mkdir -p $HOME/functest/ - source $HOME/functest/opnfv-openrc.sh + # source openstack vars + if [[ ! -f $HOME/opnfv-openrc.sh ]]; then + echo "Unable to access file $HOME/opnfv-openrc.sh" + exit 1 + fi + source $HOME/opnfv-openrc.sh # WORKSPACE is the root of the functest repo # go into WORKSPACE where the functest repo is cloned @@ -65,7 +78,7 @@ # prepare echo "Functest: prepare Functest environment" python $WORKSPACE/testcases/config_functest.py --debug $WORKSPACE/ start - + - job-template: name: clean-functest-env-{pod} @@ -212,6 +225,14 @@ parameters: - project-parameter: project: '{project}' + - string: + name: INSTALLER_TYPE + default: '{installer_type}' + description: "Installer name that is used for deployment." + - string: + name: INSTALLER_IP + default: '{installer_ip}' + description: "Installer IP." - '{pod}' scm: @@ -227,6 +248,7 @@ artifactNumToKeep: -1 builders: + - 'functest-fetch-os-creds' - 'functest-all' - 'functest-store-results' - 'functest-cleanup' @@ -493,3 +515,9 @@ # cleanup python $WORKSPACE/testcases/config_functest.py --debug --force $WORKSPACE/ clean + +- builder: + name: functest-fetch-os-creds + builders: + - shell: + !include-raw ../../utils/fetch_os_creds.sh diff --git a/jjb/genesis/genesis-fuel.yml b/jjb/genesis/genesis-fuel.yml index 1eb21f33d..b4b428b19 100644 --- a/jjb/genesis/genesis-fuel.yml +++ b/jjb/genesis/genesis-fuel.yml @@ -551,9 +551,9 @@ - trigger: name: 'fuel-master' triggers: - - pollscm: '0 21 * * *' + - timed: '0 21 * * *' - trigger: name: 'fuel-stable-arno' triggers: - - pollscm: '0 3 * * *' + - timed: '0 3 * * *' diff --git a/jjb/promise/docu-build.sh b/jjb/promise/docu-build.sh index c635a115e..aafcb2f66 100644 --- a/jjb/promise/docu-build.sh +++ b/jjb/promise/docu-build.sh @@ -2,21 +2,46 @@ set -e set -o pipefail -build_dir="build" -project="$(git remote -v | head -n1 | awk '{{print $2}}' | sed -e 's,.*:\(.*/\)\?,,' -e 's/\.git$//')" export PATH=$PATH:/usr/local/bin/ +echo +echo "Build" +echo "-----" +echo + make -# upload all built files -files=( - requirements/latex/*.pdf -) - -for file in "${{files[@]}}"; do - gsutil cp -r -L gsoutput.txt $build_dir/$file gs://artifacts.opnfv.org/$project/ - #gsutil setmeta -h "Cache-Control:private, max-age=0, no-transform" \ - #-R gs://artifacts.opnfv.org/$project/$file - cat gsoutput.txt - rm -f gsoutput.txt -done +echo +echo "Upload" +echo "------" +echo + +# NOTE: make sure source parameters for GS paths are not empty. +[[ $GERRIT_CHANGE_NUMBER =~ .+ ]] +[[ $GERRIT_PROJECT =~ .+ ]] +[[ $GERRIT_BRANCH =~ .+ ]] + +gs_path_review="artifacts.opnfv.org/review/$GERRIT_CHANGE_NUMBER" +if [[ $GERRIT_BRANCH = "master" ]] ; then + gs_path_branch="artifacts.opnfv.org/$GERRIT_PROJECT" +else + gs_path_branch="artifacts.opnfv.org/$GERRIT_PROJECT/${{GERRIT_BRANCH##*/}}" +fi + +if [[ $JOB_NAME =~ "verify" ]] ; then + gsutil cp -r build/* "gs://$gs_path_review/" + echo + echo "Document is available at http://$gs_path_review" +else + gsutil cp -r build/requirements/latex/*.pdf "gs://$gs_path_branch/" + echo + echo "Document is available at http://$gs_path_branch" +fi + +if [[ $GERRIT_EVENT_TYPE = "change-merged" ]] ; then + echo + echo "Clean Out-of-dated Documents" + echo "----------------------------" + echo + gsutil rm -r "gs://$gs_path_review" || true +fi diff --git a/jjb/promise/docu-verify.sh b/jjb/promise/docu-verify.sh deleted file mode 100644 index 72d02cfe2..000000000 --- a/jjb/promise/docu-verify.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -export PATH=$PATH:/usr/local/bin/ - -make diff --git a/jjb/promise/promise.yml b/jjb/promise/promise.yml index 3196ef48f..357ddd2ec 100644 --- a/jjb/promise/promise.yml +++ b/jjb/promise/promise.yml @@ -1,182 +1,12 @@ - project: name: promise + project: '{name}' jobs: - - 'promise-test' - - 'promise-daily-{stream}' - - 'promise-merge' - - 'promise-verify' + - '{project}-verify' + - '{project}-merge-{stream}' - # stream: branch with - in place of / (eg. stable-helium) - # branch: branch (eg. stable/helium) + # stream: branch with - in place of / (eg. stable-arno) + # branch: branch (eg. stable/arno) stream: - master: branch: 'master' - - project: 'promise' - somevar: 'foo' - -- job-template: - name: promise-test - - node: master - - project-type: freestyle - - logrotate: - daysToKeep: 30 - numToKeep: 10 - artifactDaysToKeep: -1 - artifactNumToKeep: -1 - - builders: - - shell: | - echo "Hello world from promise" - -- job-template: - name: 'promise-daily-{stream}' - - node: master - - # Job template for daily builders - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - - project-type: freestyle - varsetabove: '{somevar}' - - logrotate: - daysToKeep: '{build-days-to-keep}' - numToKeep: '{build-num-to-keep}' - artifactDaysToKeep: '{build-artifact-days-to-keep}' - artifactNumToKeep: '{build-artifact-num-to-keep}' - - parameters: - - project-parameter: - project: '{project}' - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - - triggers: - - timed: 'H H * * *' - - prebuilders: - - test-macro - - builders: - - shell: - !include-raw docu-build.sh - - postbuilders: - - test-macro - -- job-template: - name: 'promise-verify' - - node: master - - project-type: freestyle - - logrotate: - daysToKeep: 30 - numToKeep: 10 - artifactDaysToKeep: -1 - artifactNumToKeep: -1 - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: 'master' - scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' - - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - - triggers: - - gerrit: - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: 'promise' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/master' - - builders: - - shell: - !include-raw docu-verify.sh - -- job-template: - name: 'promise-merge' - - # builder-merge job to run JJB update - # - # This job's purpose is to update all the JJB - - project-type: freestyle - - node: master - - logrotate: - daysToKeep: 30 - numToKeep: 40 - artifactDaysToKeep: -1 - artifactNumToKeep: 5 - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: 'master' - - scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' - - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - - triggers: - - gerrit: - trigger-on: - - change-merged-event - - comment-added-contains-event: - comment-contains-value: 'remerge' - projects: - - project-compare-type: 'ANT' - project-pattern: 'promise' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/master' - - builders: - - shell: - !include-raw docu-build.sh - diff --git a/jjb/yardstick/yardstick.yml b/jjb/yardstick/yardstick.yml index 9334cd940..8f40d8750 100644 --- a/jjb/yardstick/yardstick.yml +++ b/jjb/yardstick/yardstick.yml @@ -7,8 +7,14 @@ - 'yardstick-verify' pod: - - 'opnfv-jump-1' - - 'opnfv-jump-2' + - opnfv-jump-1: + node: 'opnfv-jump-1' + installer_type: 'fuel' + installer_ip: '10.20.0.2' + - opnfv-jump-2: + node: 'opnfv-jump-2' + installer_type: 'foreman' + installer_ip: '172.30.10.73' # stream: branch with - in place of / (eg. stable-helium) # branch: branch (eg. stable/helium) @@ -193,11 +199,19 @@ disabled: false - node: '{pod}' + node: '{node}' parameters: - project-parameter: project: '{project}' + - string: + name: INSTALLER_TYPE + default: '{installer_type}' + description: "Installer name that is used for deployment." + - string: + name: INSTALLER_IP + default: '{installer_ip}' + description: "Installer IP." scm: - git-scm: @@ -212,6 +226,7 @@ artifactNumToKeep: -1 builders: + - 'yardstick-fetch-os-creds' - 'yardstick-ping' ######################## @@ -228,7 +243,11 @@ echo "Yardstick: prepare Yardstick environment" # source openstack vars - source $HOME/yardstick/opnfv-openrc.sh + if [[ ! -f $HOME/opnfv-openrc.sh ]]; then + echo "Unable to access file $HOME/opnfv-openrc.sh" + exit 1 + fi + source $HOME/opnfv-openrc.sh # check if cirros-0.3.3 image is already available # if not, create the image @@ -258,3 +277,9 @@ # done echo "Yardstick: done!" + +- builder: + name: yardstick-fetch-os-creds + builders: + - shell: + !include-raw ../../utils/fetch_os_creds.sh
\ No newline at end of file diff --git a/utils/fetch_os_creds.sh b/utils/fetch_os_creds.sh index 2ce81d8d1..cd53dac2d 100755 --- a/utils/fetch_os_creds.sh +++ b/utils/fetch_os_creds.sh @@ -52,12 +52,16 @@ while getopts ":d:i:a:h:" optchar; do esac done +# set vars from env if not provided by user as options +dest_path=${dest_path:-$HOME/opnfv-openrc.sh} +installer_type=${installer_type:-$INSTALLER_TYPE} +installer_ip=${installer_ip:-$INSTALLER_IP} + if [ -z $dest_path ] || [ -z $installer_type ] || [ -z $installer_ip ]; then usage exit 2 fi - # Checking if destination path is valid if [ -d $dest_path ]; then error "Please provide the full destination path for the credentials file including the filename" @@ -88,10 +92,10 @@ if [ "$installer_type" == "fuel" ]; then sshpass -p r00tme scp 2>/dev/null $ssh_options root@${installer_ip}:~/openrc $dest_path &> /dev/null #This file contains the mgmt keystone API, we need the public one for our rc file - admin_ip=$(cat $dest_path | grep "OS_AUTH_URL" | sed 's/^.*\=//' | sed "s/^\([\"']\)\(.*\)\1\$/\2/g" | sed s'/.$//') + admin_ip=$(cat $dest_path | grep "OS_AUTH_URL" | sed 's/^.*\=//' | sed "s/^\([\"']\)\(.*\)\1\$/\2/g" | sed s'/\/$//') public_ip=$(sshpass -p r00tme ssh $ssh_options root@${installer_ip} \ "ssh ${controller_ip} 'source openrc; keystone endpoint-list'" \ - | grep $admin_ip | cut -d ' ' -f 12) + | grep $admin_ip | sed 's/ /\n/g' | grep ^http | head -1) #| grep http | head -1 | cut -d '|' -f 4 | sed 's/v1\/.*/v1\//' | sed 's/ //g') &> /dev/null #NOTE: this is super ugly sed 's/v1\/.*/v1\//'OS_AUTH_URL # but sometimes the output of endpoint-list is like this: http://172.30.9.70:8004/v1/%(tenant_id)s |