diff options
| -rw-r--r-- | jjb/doctor/doctor.yml | 159 | ||||
| -rw-r--r-- | jjb/doctor/docu-build.sh | 55 | ||||
| -rwxr-xr-x | jjb/doctor/docu-verify.sh | 7 | ||||
| -rw-r--r-- | jjb/functest/functest.yml | 66 | ||||
| -rwxr-xr-x[-rw-r--r--] | jjb/genesis/genesis-compass.yml | 278 | ||||
| -rw-r--r-- | jjb/genesis/genesis-fuel.yml | 45 | ||||
| -rw-r--r-- | jjb/octopus/octopus.yml | 8 | ||||
| -rw-r--r-- | jjb/promise/docu-build.sh | 53 | ||||
| -rw-r--r-- | jjb/promise/docu-verify.sh | 7 | ||||
| -rw-r--r-- | jjb/promise/promise.yml | 180 | ||||
| -rw-r--r-- | jjb/releng/releng.yml | 76 | ||||
| -rw-r--r-- | jjb/yardstick/yardstick.yml | 35 | ||||
| -rwxr-xr-x | utils/fetch_os_creds.sh | 143 |
13 files changed, 690 insertions, 422 deletions
diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index 39c20dc54..aa586642a 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -1,127 +1,45 @@ - project: name: doctor + project: '{name}' jobs: - - 'doctor-test' - - 'doctor-daily-{stream}' - - 'doctor-merge' - - 'doctor-verify' + - '{project}-verify' + - '{project}-merge-{stream}' - # stream: branch with - in place of / (eg. stable-helium) - # branch: branch (eg. stable/helium) + # stream: branch with - in place of / (eg. stable-arno) + # branch: branch (eg. stable/arno) stream: - master: branch: 'master' - project: 'doctor' - somevar: 'foo' - -- job-template: - name: doctor-test - - node: master - - project-type: freestyle - - logrotate: - daysToKeep: 30 - numToKeep: 10 - artifactDaysToKeep: -1 - artifactNumToKeep: -1 - - builders: - - shell: | - echo "Hello world from doctor" - -- job-template: - name: 'doctor-daily-{stream}' - +- job_defaults: &job_defaults + name: job_defaults node: master - - # Job template for daily builders - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - project-type: freestyle - varsetabove: '{somevar}' - logrotate: daysToKeep: '{build-days-to-keep}' numToKeep: '{build-num-to-keep}' artifactDaysToKeep: '{build-artifact-days-to-keep}' artifactNumToKeep: '{build-artifact-num-to-keep}' - parameters: - project-parameter: project: '{project}' - - string: - name: GIT_BASE - default: 'https://gerrit.opnfv.org/gerrit/{project}' - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - wrappers: - ssh-agent-credentials: - user: '{ssh-credentials}' - - triggers: - - timed: 'H H * * *' - - prebuilders: - - test-macro - - builders: - - shell: - !include-raw docu-build.sh - - postbuilders: - - test-macro + users: + - '{ssh-credentials}' - job-template: - name: 'doctor-verify' - - node: master - - project-type: freestyle - - logrotate: - daysToKeep: 30 - numToKeep: 10 - artifactDaysToKeep: -1 - artifactNumToKeep: -1 - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: 'master' - - string: - name: GIT_BASE - default: 'https://gerrit.opnfv.org/gerrit/{project}' - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + name: '{project}-verify' + <<: *job_defaults scm: - gerrit-trigger-scm: credentials-id: '{ssh-credentials}' refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' + - patchset-created-event - draft-published-event - comment-added-contains-event: comment-contains-value: 'recheck' @@ -129,52 +47,22 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'ANT' - project-pattern: 'doctor' + project-pattern: '{project}' branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/master' - + - branch-compare-type: 'ANT' + branch-pattern: '**' builders: - shell: - !include-raw docu-verify.sh + !include-raw docu-build.sh - job-template: - name: 'doctor-merge' - - node: master - - # builder-merge job to run JJB update - # - # This job's purpose is to update all the JJB - - project-type: freestyle - - logrotate: - daysToKeep: 30 - numToKeep: 40 - artifactDaysToKeep: -1 - artifactNumToKeep: 5 - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: 'master' - - string: - name: GIT_BASE - default: 'https://gerrit.opnfv.org/gerrit/{project}' - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." - + name: '{project}-merge-{stream}' + <<: *job_defaults scm: - gerrit-trigger-scm: credentials-id: '{ssh-credentials}' refspec: '' choosing-strategy: 'default' - - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -183,17 +71,10 @@ comment-contains-value: 'remerge' projects: - project-compare-type: 'ANT' - project-pattern: 'doctor' + project-pattern: '{project}' branches: - branch-compare-type: 'ANT' - branch-pattern: '**/master' - + branch-pattern: '**/{branch}' builders: - shell: !include-raw docu-build.sh - - - - - - diff --git a/jjb/doctor/docu-build.sh b/jjb/doctor/docu-build.sh index 39ecf3bb0..ffb131114 100644 --- a/jjb/doctor/docu-build.sh +++ b/jjb/doctor/docu-build.sh @@ -2,23 +2,46 @@ set -e set -o pipefail -build_dir="build" -project="$(git remote -v | head -n1 | awk '{{print $2}}' | sed -e 's,.*:\(.*/\)\?,,' -e 's/\.git$//')" export PATH=$PATH:/usr/local/bin/ +echo +echo "Build" +echo "-----" +echo + make -# upload all built files -files=( - design_docs - requirements/html - requirements/latex/*.pdf -) - -for file in "${{files[@]}}"; do - gsutil cp -r -L gsoutput.txt $build_dir/$file gs://artifacts.opnfv.org/$project/ - #gsutil setmeta -h "Cache-Control:private, max-age=0, no-transform" \ - #-R gs://artifacts.opnfv.org/$project/$file - cat gsoutput.txt - rm -f gsoutput.txt -done +echo +echo "Upload" +echo "------" +echo + +# NOTE: make sure source parameters for GS paths are not empty. +[[ $GERRIT_CHANGE_NUMBER =~ .+ ]] +[[ $GERRIT_PROJECT =~ .+ ]] +[[ $GERRIT_BRANCH =~ .+ ]] + +gs_path_review="artifacts.opnfv.org/review/$GERRIT_CHANGE_NUMBER" +if [[ $GERRIT_BRANCH = "master" ]] ; then + gs_path_branch="artifacts.opnfv.org/$GERRIT_PROJECT" +else + gs_path_branch="artifacts.opnfv.org/$GERRIT_PROJECT/${{GERRIT_BRANCH##*/}}" +fi + +if [[ $JOB_NAME =~ "verify" ]] ; then + gsutil cp -r build/* "gs://$gs_path_review/" + echo + echo "Document is available at http://$gs_path_review" +else + gsutil cp -r build/design_docs "gs://$gs_path_branch/" + gsutil cp -r build/requirements/html "gs://$gs_path_branch/" + gsutil cp -r build/requirements/latex/*.pdf "gs://$gs_path_branch/" + echo + echo "Latest document is available at http://$gs_path_branch" + + if gsutil ls "gs://$gs_path_review" > /dev/null 2>&1 ; then + echo + echo "Deleting Out-of-dated Documents..." + gsutil rm -r "gs://$gs_path_review" + fi +fi diff --git a/jjb/doctor/docu-verify.sh b/jjb/doctor/docu-verify.sh deleted file mode 100755 index 72d02cfe2..000000000 --- a/jjb/doctor/docu-verify.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -export PATH=$PATH:/usr/local/bin/ - -make diff --git a/jjb/functest/functest.yml b/jjb/functest/functest.yml index 7a51829a7..0d776065f 100644 --- a/jjb/functest/functest.yml +++ b/jjb/functest/functest.yml @@ -3,10 +3,19 @@ ################################### - project: name: functest - pod: - - 'opnfv-jump-1' - - 'opnfv-jump-2' + - opnfv-jump-1: + node: 'opnfv-jump-1' + installer_type: 'fuel' + installer_ip: '10.20.0.2' + - opnfv-jump-2: + node: 'opnfv-jump-2' + installer_type: 'foreman' + installer_ip: '172.30.10.73' + - orange-build: + node: 'orange-build' + installer_type: 'foreman' + installer_ip: '172.30.10.73' jobs: - 'functest-{pod}' - 'set-functest-env-{pod}' @@ -55,7 +64,12 @@ echo "Functest: prepare Functest environment" mkdir -p $HOME/functest/ - source $HOME/functest/opnfv-openrc.sh + # source openstack vars + if [[ ! -f $HOME/opnfv-openrc.sh ]]; then + echo "Unable to access file $HOME/opnfv-openrc.sh" + exit 1 + fi + source $HOME/opnfv-openrc.sh # WORKSPACE is the root of the functest repo # go into WORKSPACE where the functest repo is cloned @@ -64,7 +78,7 @@ # prepare echo "Functest: prepare Functest environment" python $WORKSPACE/testcases/config_functest.py --debug $WORKSPACE/ start - + - job-template: name: clean-functest-env-{pod} @@ -211,6 +225,15 @@ parameters: - project-parameter: project: '{project}' + - string: + name: INSTALLER_TYPE + default: '{installer_type}' + description: "Installer name that is used for deployment." + - string: + name: INSTALLER_IP + default: '{installer_ip}' + description: "Installer IP." + - '{pod}' scm: - git-scm: @@ -225,6 +248,7 @@ artifactNumToKeep: -1 builders: + - 'functest-fetch-os-creds' - 'functest-all' - 'functest-store-results' - 'functest-cleanup' @@ -390,6 +414,29 @@ default: '{gs-pathname}' description: "Version directory where the opnfv documents will be stored in gs repository" +- parameter: + name: orange-build + parameters: + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Override GIT_BASE" + +- parameter: + name: opnfv-jump-1 + parameters: + - string: + name: GIT_BASE + default: ssh://gerrit.opnfv.org:29418/$PROJECT + description: "Override GIT_BASE" + +- parameter: + name: opnfv-jump-2 + parameters: + - string: + name: GIT_BASE + default: ssh://gerrit.opnfv.org:29418/$PROJECT + description: "Override GIT_BASE" ######################## # trigger macros ######################## @@ -403,6 +450,9 @@ triggers: - pollscm: "H 21 * * *" +######################## +# builder macros +######################## # macros - builder: name: functest-all @@ -465,3 +515,9 @@ # cleanup python $WORKSPACE/testcases/config_functest.py --debug --force $WORKSPACE/ clean + +- builder: + name: functest-fetch-os-creds + builders: + - shell: + !include-raw ../../utils/fetch_os_creds.sh diff --git a/jjb/genesis/genesis-compass.yml b/jjb/genesis/genesis-compass.yml index 55f5b7536..9065da972 100644..100755 --- a/jjb/genesis/genesis-compass.yml +++ b/jjb/genesis/genesis-compass.yml @@ -3,12 +3,13 @@ name: genesis-compass - installer: - - compass + installer: 'compass' + jobs: - 'genesis-compass-verify' - 'genesis-compass-merge' - 'genesis-compass-daily-{stream}' + - 'genesis-compass-deploy-virtual-{flavor}' # stream: branch with - in place of / (eg. stable-helium) # branch: branch (eg. stable/helium) @@ -16,6 +17,16 @@ - master: branch: 'master' + flavor: + - five: + conf: 'five' + node: 'compass-build-deploy-02' + - cluster: + conf: 'cluster' + node: 'compass-build-deploy-03' + + + project: 'genesis' ######################## @@ -38,16 +49,12 @@ artifactNumToKeep: -1 parameters: - - string: - name: BUILD_DIRECTORY - default: $WORKSPACE/build_output - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/genesis - project-parameter: project: '{project}' - gerrit-parameter: branch: 'master' + - compass-parameter: + installer: '{installer}' scm: - gerrit-trigger-scm: @@ -85,7 +92,8 @@ builders: - - 'compass-verify' + - 'compass-build' + #- 'compass-workspace-cleanup' - job-template: name: 'genesis-compass-merge' @@ -105,17 +113,12 @@ artifactNumToKeep: 5 parameters: - - string: - name: BUILD_DIRECTORY - default: $WORKSPACE/build_output - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/genesis - project-parameter: project: '{project}' - gerrit-parameter: branch: 'master' - + - compass-parameter: + installer: '{installer}' scm: - gerrit-trigger-scm: credentials-id: '{ssh-credentials}' @@ -155,32 +158,16 @@ node: compass-build-deploy-03 parameters: - - string: - name: BUILD_DIRECTORY - default: $WORKSPACE/build_output - - string: - name: INSTALLER - default: 'compass' - description: "Installer to use." - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/genesis - - string: - name: GERRIT_BRANCH - default: origin/master - description: "Branch to build, deploy and test." - - string: - name: GERRIT_REFSPEC - default: refs/heads/master - description: "Refspec to retrieve." + - project-parameter: + project: '{project}' + - compass-parameter: + installer: '{installer}' scm: - - git: - skip-tag: true - url: $GIT_BASE - branches: - - $GERRIT_BRANCH - refspec: $GERRIT_REFSPEC + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' triggers: - pollscm: '@midnight' @@ -192,25 +179,224 @@ artifactNumToKeep: -1 builders: - - 'compass-daily-master' + #- 'compass-build' + #- 'compass-upload-artifact' + #- 'compass-workspace-cleanup' + - shell: | + echo "Hello world!" + echo "genesis-compass-deploy-virtual job will be triggered!" + + publishers: + - trigger: + project: 'genesis-compass-deploy-virtual-five' + threshold: SUCCESS + - trigger: + project: 'genesis-compass-deploy-virtual-cluster' + threshold: SUCCESS + + +- job-template: + name: 'genesis-compass-deploy-virtual-{flavor}' + + project-type: freestyle + + disabled: false + + node: '{node}' + + parameters: + - project-parameter: + project: '{project}' + - compass-parameter: + installer: '{installer}' + - string: + name: FLAVOR_CONF + default: '{conf}' + description: "" + + triggers: + - pollscm: "H/2 * * * *" + + logrotate: + daysToKeep: 30 + numToKeep: 10 + artifactDaysToKeep: -1 + artifactNumToKeep: -1 + + builders: + - 'compass-ci-preclean-workspace' + - 'compass-deploy-virtual-flavor' + +######################## +# parameter macros +######################## +- parameter: + name: compass-parameter + parameters: + - string: + name: INSTALLER + default: '{installer}' + description: "Installer to use." + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/genesis-$INSTALLER + description: "Directory where the cache to be used during the build is located." + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT/$INSTALLER + description: "URL to Google Storage." + - string: + name: INTERNAL_REPO + default: "root@192.168.121.12:~/kun/artifacts" + description: "Artifact Repository" +######################## +# builder macros +######################## + - builder: - name: compass-verify + name: compass-merge builders: - shell: | #!/bin/bash echo "Hello World!" + - builder: - name: compass-merge + name: 'compass-build' builders: - shell: | #!/bin/bash - echo "Hello World!" + set -o errexit + set -o nounset + set -o pipefail + + # log info to console + echo "Hello world!" - builder: - name: compass-daily-master + name: 'compass-deploy' builders: - shell: | #!/bin/bash - echo "Hello World!" + set -o errexit + set -o nounset + set -o pipefail + + # log info to console + echo "Hello world!" + +- builder: + name: 'compass-deploy-virtual-flavor' + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + # log info to console + echo "Starting the deployment on virtual environment using $INSTALLER. This could take some time..." + echo "--------------------------------------------------------" + echo + + cd $WORKSPACE/compass + ./ci/deploy.sh $FLAVOR_CONF + # ssh -o BatchMode=yes -o TCPKeepAlive=yes cideploy@10.118.34.205 ./cideploy.sh + + echo + echo "--------------------------------------------------------" + echo "Done!" + +- builder: + name: 'compass-upload-artifact' + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + # log info to console + echo "Uploading the $INSTALLER artifact. This could take some time..." + echo "--------------------------------------------------------" + echo + + # source the opnfv.properties to get ARTIFACT_VERSION + source $WORKSPACE/opnfv.properties + + # upload artifact and additional files to google storage + gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1 + gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 + gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 + + echo + echo "--------------------------------------------------------" + echo "Done!" + +- builder: + name: 'compass-download-artifact' + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + # log info to console + echo "Downloading the $INSTALLER artifact. This could take some time..." + echo "--------------------------------------------------------" + echo + + # get the latest.properties file in order to get info regarding latest artifact + curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties + + # check if we got the file + [[ -f latest.properties ]] || exit 1 + + # source the file so we get OPNFV vars + source latest.properties + + # download the file + curl -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 + + # list the file + ls -al $WORKSPACE/opnfv.iso + + echo + echo "--------------------------------------------------------" + echo "Done!" + +- builder: + name: 'compass-workspace-cleanup' + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + # delete everything that is in $WORKSPACE + /bin/rm -rf $WORKSPACE + +- builder: + name: 'compass-ci-preclean-workspace' + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + cd $WORKSPACE/.. + sudo rm $WORKSPACE -rf + git clone $GIT_BASE $WORKSPACE + diff --git a/jjb/genesis/genesis-fuel.yml b/jjb/genesis/genesis-fuel.yml index 1b163a042..b4b428b19 100644 --- a/jjb/genesis/genesis-fuel.yml +++ b/jjb/genesis/genesis-fuel.yml @@ -271,7 +271,7 @@ builders: - 'fuel-download-artifact' - - 'fuel-deploy' + - 'fuel-deploy-{stream}' - job-template: name: 'genesis-fuel-deploy-virtual-master' @@ -376,7 +376,7 @@ echo "Done!" - builder: - name: 'fuel-deploy' + name: 'fuel-deploy-master' builders: - shell: | #!/bin/bash @@ -416,6 +416,43 @@ echo "Done!" - builder: + name: 'fuel-deploy-stable-arno' + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + # source the file so we get OPNFV vars + source latest.properties + + # echo the info about artifact that is used during the deployment + echo "Using $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f4) for deployment" + + # create TMPDIR if it doesn't exist + export TMPDIR=$HOME/tmp/$JOB_NAME + [[ -d $TMPDIR ]] || mkdir -p $TMPDIR + + # change permissions down to TMPDIR + chmod a+x $HOME + chmod a+x $TMPDIR + + # log info to console + echo "Starting the deployment using $INSTALLER. This could take some time..." + echo "--------------------------------------------------------" + echo + + # start the deployment + echo "Issuing command" + echo "sudo $WORKSPACE/fuel/ci/deploy.sh $WORKSPACE/opnfv.iso $WORKSPACE/fuel/deploy/baremetal/conf/linux_foundation_lab/ha/dea.yaml $WORKSPACE/fuel/deploy/baremetal/conf/linux_foundation_lab/ha/dha.yaml -s $TMPDIR" + + sudo $WORKSPACE/fuel/ci/deploy.sh $WORKSPACE/opnfv.iso $WORKSPACE/fuel/deploy/baremetal/conf/linux_foundation_lab/ha/dea.yaml $WORKSPACE/fuel/deploy/baremetal/conf/linux_foundation_lab/ha/dha.yaml -s $TMPDIR + echo + echo "--------------------------------------------------------" + echo "Done!" + +- builder: name: 'fuel-deploy-virtual' builders: - shell: | @@ -514,9 +551,9 @@ - trigger: name: 'fuel-master' triggers: - - pollscm: '0 21 * * *' + - timed: '0 21 * * *' - trigger: name: 'fuel-stable-arno' triggers: - - pollscm: '0 3 * * *' + - timed: '0 3 * * *' diff --git a/jjb/octopus/octopus.yml b/jjb/octopus/octopus.yml index e44e5ea93..e11ccc56c 100644 --- a/jjb/octopus/octopus.yml +++ b/jjb/octopus/octopus.yml @@ -6,7 +6,7 @@ jobs: - 'octopus-test' - 'octopus-daily-{stream}' - - 'octopus-merge' + - 'octopus-merge-{stream}' - 'octopus-verify-{stream}' # stream: branch with - in place of / (eg. stable-arno) @@ -141,7 +141,7 @@ !include-raw ../opnfvdocs/docu-build.sh - job-template: - name: 'octopus-merge' + name: 'octopus-merge-{stream}' node: master @@ -161,7 +161,7 @@ - project-parameter: project: '{project}' - gerrit-parameter: - branch: 'master' + branch: '{branch}' scm: - gerrit-trigger-scm: @@ -184,7 +184,7 @@ project-pattern: 'octopus' branches: - branch-compare-type: 'ANT' - branch-pattern: '**/master' + branch-pattern: '**/{branch}' builders: - shell: diff --git a/jjb/promise/docu-build.sh b/jjb/promise/docu-build.sh index c635a115e..aafcb2f66 100644 --- a/jjb/promise/docu-build.sh +++ b/jjb/promise/docu-build.sh @@ -2,21 +2,46 @@ set -e set -o pipefail -build_dir="build" -project="$(git remote -v | head -n1 | awk '{{print $2}}' | sed -e 's,.*:\(.*/\)\?,,' -e 's/\.git$//')" export PATH=$PATH:/usr/local/bin/ +echo +echo "Build" +echo "-----" +echo + make -# upload all built files -files=( - requirements/latex/*.pdf -) - -for file in "${{files[@]}}"; do - gsutil cp -r -L gsoutput.txt $build_dir/$file gs://artifacts.opnfv.org/$project/ - #gsutil setmeta -h "Cache-Control:private, max-age=0, no-transform" \ - #-R gs://artifacts.opnfv.org/$project/$file - cat gsoutput.txt - rm -f gsoutput.txt -done +echo +echo "Upload" +echo "------" +echo + +# NOTE: make sure source parameters for GS paths are not empty. +[[ $GERRIT_CHANGE_NUMBER =~ .+ ]] +[[ $GERRIT_PROJECT =~ .+ ]] +[[ $GERRIT_BRANCH =~ .+ ]] + +gs_path_review="artifacts.opnfv.org/review/$GERRIT_CHANGE_NUMBER" +if [[ $GERRIT_BRANCH = "master" ]] ; then + gs_path_branch="artifacts.opnfv.org/$GERRIT_PROJECT" +else + gs_path_branch="artifacts.opnfv.org/$GERRIT_PROJECT/${{GERRIT_BRANCH##*/}}" +fi + +if [[ $JOB_NAME =~ "verify" ]] ; then + gsutil cp -r build/* "gs://$gs_path_review/" + echo + echo "Document is available at http://$gs_path_review" +else + gsutil cp -r build/requirements/latex/*.pdf "gs://$gs_path_branch/" + echo + echo "Document is available at http://$gs_path_branch" +fi + +if [[ $GERRIT_EVENT_TYPE = "change-merged" ]] ; then + echo + echo "Clean Out-of-dated Documents" + echo "----------------------------" + echo + gsutil rm -r "gs://$gs_path_review" || true +fi diff --git a/jjb/promise/docu-verify.sh b/jjb/promise/docu-verify.sh deleted file mode 100644 index 72d02cfe2..000000000 --- a/jjb/promise/docu-verify.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -export PATH=$PATH:/usr/local/bin/ - -make diff --git a/jjb/promise/promise.yml b/jjb/promise/promise.yml index 3196ef48f..357ddd2ec 100644 --- a/jjb/promise/promise.yml +++ b/jjb/promise/promise.yml @@ -1,182 +1,12 @@ - project: name: promise + project: '{name}' jobs: - - 'promise-test' - - 'promise-daily-{stream}' - - 'promise-merge' - - 'promise-verify' + - '{project}-verify' + - '{project}-merge-{stream}' - # stream: branch with - in place of / (eg. stable-helium) - # branch: branch (eg. stable/helium) + # stream: branch with - in place of / (eg. stable-arno) + # branch: branch (eg. stable/arno) stream: - master: branch: 'master' - - project: 'promise' - somevar: 'foo' - -- job-template: - name: promise-test - - node: master - - project-type: freestyle - - logrotate: - daysToKeep: 30 - numToKeep: 10 - artifactDaysToKeep: -1 - artifactNumToKeep: -1 - - builders: - - shell: | - echo "Hello world from promise" - -- job-template: - name: 'promise-daily-{stream}' - - node: master - - # Job template for daily builders - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - - project-type: freestyle - varsetabove: '{somevar}' - - logrotate: - daysToKeep: '{build-days-to-keep}' - numToKeep: '{build-num-to-keep}' - artifactDaysToKeep: '{build-artifact-days-to-keep}' - artifactNumToKeep: '{build-artifact-num-to-keep}' - - parameters: - - project-parameter: - project: '{project}' - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - - triggers: - - timed: 'H H * * *' - - prebuilders: - - test-macro - - builders: - - shell: - !include-raw docu-build.sh - - postbuilders: - - test-macro - -- job-template: - name: 'promise-verify' - - node: master - - project-type: freestyle - - logrotate: - daysToKeep: 30 - numToKeep: 10 - artifactDaysToKeep: -1 - artifactNumToKeep: -1 - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: 'master' - scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' - - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - - triggers: - - gerrit: - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: 'promise' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/master' - - builders: - - shell: - !include-raw docu-verify.sh - -- job-template: - name: 'promise-merge' - - # builder-merge job to run JJB update - # - # This job's purpose is to update all the JJB - - project-type: freestyle - - node: master - - logrotate: - daysToKeep: 30 - numToKeep: 40 - artifactDaysToKeep: -1 - artifactNumToKeep: 5 - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: 'master' - - scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' - - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - - triggers: - - gerrit: - trigger-on: - - change-merged-event - - comment-added-contains-event: - comment-contains-value: 'remerge' - projects: - - project-compare-type: 'ANT' - project-pattern: 'promise' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/master' - - builders: - - shell: - !include-raw docu-build.sh - diff --git a/jjb/releng/releng.yml b/jjb/releng/releng.yml new file mode 100644 index 000000000..bb64f6a1a --- /dev/null +++ b/jjb/releng/releng.yml @@ -0,0 +1,76 @@ +######################## +# Job configuration for releng +######################## +- project: + + name: releng + + project: 'releng' + + jobs: + - 'releng-verify' + + # stream: branch with - in place of / (eg. stable-arno) + # branch: branch (eg. stable/arno) + stream: + - master: + branch: 'master' + +######################## +# job templates +######################## + +- job-template: + name: 'releng-verify' + + node: master + + project-type: freestyle + + logrotate: + daysToKeep: 30 + numToKeep: 10 + artifactDaysToKeep: -1 + artifactNumToKeep: -1 + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: 'master' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + user: '{ssh-credentials}' + + triggers: + - gerrit: + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/master' + file-paths: + - compare-type: ANT + pattern: 'utils/**' + + builders: + - shell: | + echo "Hello world from releng" diff --git a/jjb/yardstick/yardstick.yml b/jjb/yardstick/yardstick.yml index 532aca730..8f40d8750 100644 --- a/jjb/yardstick/yardstick.yml +++ b/jjb/yardstick/yardstick.yml @@ -7,8 +7,14 @@ - 'yardstick-verify' pod: - - 'opnfv-jump-1' - - 'opnfv-jump-2' + - opnfv-jump-1: + node: 'opnfv-jump-1' + installer_type: 'fuel' + installer_ip: '10.20.0.2' + - opnfv-jump-2: + node: 'opnfv-jump-2' + installer_type: 'foreman' + installer_ip: '172.30.10.73' # stream: branch with - in place of / (eg. stable-helium) # branch: branch (eg. stable/helium) @@ -193,11 +199,19 @@ disabled: false - node: '{pod}' + node: '{node}' parameters: - project-parameter: project: '{project}' + - string: + name: INSTALLER_TYPE + default: '{installer_type}' + description: "Installer name that is used for deployment." + - string: + name: INSTALLER_IP + default: '{installer_ip}' + description: "Installer IP." scm: - git-scm: @@ -212,6 +226,7 @@ artifactNumToKeep: -1 builders: + - 'yardstick-fetch-os-creds' - 'yardstick-ping' ######################## @@ -228,7 +243,11 @@ echo "Yardstick: prepare Yardstick environment" # source openstack vars - source $HOME/yardstick/opnfv-openrc.sh + if [[ ! -f $HOME/opnfv-openrc.sh ]]; then + echo "Unable to access file $HOME/opnfv-openrc.sh" + exit 1 + fi + source $HOME/opnfv-openrc.sh # check if cirros-0.3.3 image is already available # if not, create the image @@ -250,7 +269,7 @@ # set virtualenv echo "Yardstick: execute ping scenario" - yardstick -v -d samples/ping.yaml + yardstick task start samples/ping.yaml -v -d # cleanup echo "Yardstick: cleanup" @@ -258,3 +277,9 @@ # done echo "Yardstick: done!" + +- builder: + name: yardstick-fetch-os-creds + builders: + - shell: + !include-raw ../../utils/fetch_os_creds.sh
\ No newline at end of file diff --git a/utils/fetch_os_creds.sh b/utils/fetch_os_creds.sh new file mode 100755 index 000000000..cd53dac2d --- /dev/null +++ b/utils/fetch_os_creds.sh @@ -0,0 +1,143 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2015 Ericsson AB and others. +# jose.lausuch@ericsson.com +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + + +usage() { + echo "usage: $0 -d <destination> -i <installer_type> -a <installer_ip>" >&2 +} + +info () { + logger -s -t "fetch_os_creds.info" "$*" +} + + +error () { + logger -s -t "fetch_os_creds.error" "$*" + exit 1 +} + + +verify_connectivity() { + local ip=$1 + info "Verifying connectivity to $ip..." + for i in $(seq 0 10); do + if ping -c 1 -W 1 $ip > /dev/null; then + info "$ip is reachable!" + return 0 + fi + sleep 1 + done + error "Can not talk to $ip." +} + + + +#Get options +while getopts ":d:i:a:h:" optchar; do + case "${optchar}" in + d) dest_path=${OPTARG} ;; + i) installer_type=${OPTARG} ;; + a) installer_ip=${OPTARG} ;; + *) echo "Non-option argument: '-${OPTARG}'" >&2 + usage + exit 2 + ;; + esac +done + +# set vars from env if not provided by user as options +dest_path=${dest_path:-$HOME/opnfv-openrc.sh} +installer_type=${installer_type:-$INSTALLER_TYPE} +installer_ip=${installer_ip:-$INSTALLER_IP} + +if [ -z $dest_path ] || [ -z $installer_type ] || [ -z $installer_ip ]; then + usage + exit 2 +fi + +# Checking if destination path is valid +if [ -d $dest_path ]; then + error "Please provide the full destination path for the credentials file including the filename" +else + # Check if we can create the file (e.g. path is correct) + touch $dest_path || error "Cannot create the file specified. Check that the path is correct and run the script again." +fi + + +ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + +# Start fetching the files +if [ "$installer_type" == "fuel" ]; then + #ip_fuel="10.20.0.2" + verify_connectivity $installer_ip + + # Check if controller is alive (online='True') + controller_ip=$(sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \ + 'fuel node | grep controller | grep True | awk "{print \$10}" | tail -1') &> /dev/null + + if [ -z $controller_ip ]; then + error "The controller $controller_ip is not up. Please check that the POD is correctly deployed." + fi + + info "Fetching rc file from controller $controller_ip..." + sshpass -p r00tme ssh 2>/dev/null $ssh_options root@${installer_ip} \ + "scp $ssh_options ${controller_ip}:/root/openrc ." &> /dev/null + sshpass -p r00tme scp 2>/dev/null $ssh_options root@${installer_ip}:~/openrc $dest_path &> /dev/null + + #This file contains the mgmt keystone API, we need the public one for our rc file + admin_ip=$(cat $dest_path | grep "OS_AUTH_URL" | sed 's/^.*\=//' | sed "s/^\([\"']\)\(.*\)\1\$/\2/g" | sed s'/\/$//') + public_ip=$(sshpass -p r00tme ssh $ssh_options root@${installer_ip} \ + "ssh ${controller_ip} 'source openrc; keystone endpoint-list'" \ + | grep $admin_ip | sed 's/ /\n/g' | grep ^http | head -1) + #| grep http | head -1 | cut -d '|' -f 4 | sed 's/v1\/.*/v1\//' | sed 's/ //g') &> /dev/null + #NOTE: this is super ugly sed 's/v1\/.*/v1\//'OS_AUTH_URL + # but sometimes the output of endpoint-list is like this: http://172.30.9.70:8004/v1/%(tenant_id)s + + +elif [ "$installer_type" == "foreman" ]; then + #ip_foreman="172.30.10.73" + controller="oscontroller1.opnfv.com" + verify_connectivity $installer_ip + + # Check if controller is alive (here is more difficult to get the ip from a command like "fuel node") + sshpass -p vagrant ssh $ssh_options root@${installer_ip} \ + "sshpass -p Op3nStack ssh $ssh_options root@${controller} 'ls'" &> /dev/null + if [ $? -ne 0 ]; then + error "The controller ${controller} is not up. Please check that the POD is correctly deployed." + fi + + info "Fetching openrc from a Foreman Controller '${controller}'..." + sshpass -p vagrant ssh $ssh_options root@${installer_ip} \ + "sshpass -p Op3nStack scp $ssh_options root@${controller}:~/keystonerc_admin ." &> /dev/null + sshpass -p vagrant scp $ssh_options root@${installer_ip}:~/keystonerc_admin $dest_path &> /dev/null + + #This file contains the mgmt keystone API, we need the public one for our rc file + public_ip=$(sshpass -p vagrant ssh $ssh_options root@${installer_ip} \ + "sshpass -p Op3nStack ssh $ssh_options root@${controller} \ + 'source keystonerc_admin;keystone endpoint-list'" \ + | grep http | head -1 | cut -d '|' -f 4 | sed 's/ //g') &> /dev/null + +else + error "Installer $installer is not supported by this script" +fi + + + +if [ "$public_ip" == "" ]; then + error "Cannot retrieve the public IP from keystone" +fi + +info "Keystone public IP is $public_ip" +sed -i "/OS_AUTH_URL/c\export OS_AUTH_URL=\'$public_ip'" $dest_path + +echo "-------- Credentials: --------" +cat $dest_path + +exit 0 |