diff options
48 files changed, 784 insertions, 816 deletions
diff --git a/docs/etc/conf.py b/docs/etc/conf.py index 671965378..f42b11055 100644 --- a/docs/etc/conf.py +++ b/docs/etc/conf.py @@ -1,27 +1,27 @@ -import datetime -import sys -import os +''' +Base configuration file for sphinx-build. -needs_sphinx = '1.3' +You can override this configuration by putting 'conf.py' in the document +directory (e.g. how-to-use-docs/conf.py). If there is no 'conf.py' in the +document directory, this file will be copied to that directory before the +document builder jobs in 'opnfv-docs-verify' and 'opnfv-docs-merge'. +The logo image (opnfv-logo.png) will be also copied from +docs/etc/opnfv-logo.png during the build jobs. +''' -numfig = True +import datetime -source_suffix = '.rst' +needs_sphinx = '1.3' master_doc = 'index' pygments_style = 'sphinx' -html_use_index = False -pdf_documents = [('index', u'Copper', u'Copper Project', u'OPNFV')] -pdf_fit_mode = "shrink" -pdf_stylesheets = ['sphinx','kerning','a4'] -#latex_domain_indices = False -#latex_use_modindex = False +html_use_index = False +numfig = True +html_logo = 'opnfv-logo.png' -latex_elements = { - 'printindex': '', -} +latex_elements = {'printindex': ''} +latex_logo = 'opnfv-logo.png' -project = u'Copper: Virtual Infrastructure Deployment Policies' copyright = u'%s, OPNFV' % datetime.date.today().year version = u'1.0.0' release = u'1.0.0' diff --git a/docs/how-to-use-docs/documentation-example.rst b/docs/how-to-use-docs/documentation-example.rst index afcf75814..e18ce598b 100644 --- a/docs/how-to-use-docs/documentation-example.rst +++ b/docs/how-to-use-docs/documentation-example.rst @@ -1,12 +1,3 @@ -.. two dots create a comment. please leave this logo at the top of each of your rst files. -.. image:: ../etc/opnfv-logo.png - :height: 40 - :width: 200 - :alt: OPNFV - :align: left -.. these two pipes are to seperate the logo from the first title -| -| How to create documentation for your OPNFV project ================================================== @@ -28,8 +19,9 @@ This index file must refence your other rst files. .. code-block:: bash - Example Documentation table of contents - ======================================= + ===================== + Example Documentation + ===================== Contents: @@ -39,10 +31,8 @@ This index file must refence your other rst files. documentation-example.rst - Indices and tables - ================== - - * :ref:`search` + .. + Leave these at the bottom of 'index.rst' file Revision: _sha1_ @@ -55,11 +45,12 @@ The Sphinx Build When you push documentation changes to gerrit a jenkins job will create html documentation. * Verify Jobs + For verify jobs a link to the documentation will show up as a comment in gerrit for you to see the result. * Merge jobs -Once you are happy with the look of your documentation you can submit the patchset the merge job will +Once you are happy with the look of your documentation you can submit the patchset the merge job will copy the output of each documentation directory to http://artifacts.opnfv.org/$project/docs/$name_of_your_folder/index.html Here are some quick examples of how to use rst markup @@ -77,10 +68,3 @@ links are easy to add: Here is a link to sphinx, the tool that we are using to g .. code-block:: bash echo "Heres is a code block with bash syntax highlighting" - - -Leave these at the bottom of each of your documents they are used internally - -Revision: _sha1_ - -Build date: |today| diff --git a/docs/how-to-use-docs/index.rst b/docs/how-to-use-docs/index.rst index 36710b32d..0965eb3cd 100644 --- a/docs/how-to-use-docs/index.rst +++ b/docs/how-to-use-docs/index.rst @@ -1,18 +1,9 @@ -.. OPNFV Release Engineering documentation, created by - sphinx-quickstart on Tue Jun 9 19:12:31 2015. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +.. + This is new template created on Thu Nov 26 16:18:09 JST 2015 -.. image:: ../etc/opnfv-logo.png - :height: 40 - :width: 200 - :alt: OPNFV - :align: left - -Example Documentation table of contents -======================================= - -Contents: +===================== +Example Documentation +===================== .. toctree:: :numbered: @@ -20,10 +11,8 @@ Contents: documentation-example.rst -Indices and tables -================== - -* :ref:`search` +.. + Leave these at the bottom of 'index.rst' file Revision: _sha1_ diff --git a/docs/jenkins-job-builder/index.rst b/docs/jenkins-job-builder/index.rst index 1ea4a95a5..f8f568339 100644 --- a/docs/jenkins-job-builder/index.rst +++ b/docs/jenkins-job-builder/index.rst @@ -1,33 +1,13 @@ -.. OPNFV Release Engineering documentation, created by - sphinx-quickstart on Tue Jun 9 19:12:31 2015. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -.. image:: ../etc/opnfv-logo.png - :height: 40 - :width: 200 - :alt: OPNFV - :align: left - -| -| - +=========================== Release Engineering Project =========================== -Contents: - .. toctree:: :numbered: :maxdepth: 8 opnfv-jjb-usage.rst -Indices and tables -================== - -* :ref:`search` - Revision: _sha1_ Build date: |today| diff --git a/docs/jenkins-job-builder/opnfv-jjb-usage.rst b/docs/jenkins-job-builder/opnfv-jjb-usage.rst index 8e4a1a831..4aecc6de6 100644 --- a/docs/jenkins-job-builder/opnfv-jjb-usage.rst +++ b/docs/jenkins-job-builder/opnfv-jjb-usage.rst @@ -1,12 +1,3 @@ -.. image:: ../etc/opnfv-logo.png - :height: 40 - :width: 200 - :alt: OPNFV - :align: left - -| -| - Creating/Configuring/Verifying Jenkins Jobs ============================================ @@ -69,22 +60,22 @@ The Current merge and verify jobs for jenkins job builder as pulled from the rep jobs: - 'builder-verify-jjb' - 'builder-merge' - + project: 'releng' - + - job-template: name: builder-verify-jjb - + node: master - + project-type: freestyle - + logrotate: daysToKeep: 30 numToKeep: 10 artifactDaysToKeep: -1 artifactNumToKeep: -1 - + parameters: - project-parameter: project: '{project}' @@ -95,11 +86,11 @@ The Current merge and verify jobs for jenkins job builder as pulled from the rep credentials-id: '{ssh-credentials}' refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - + wrappers: - ssh-agent-credentials: user: '{ssh-credentials}' - + triggers: - gerrit: trigger-on: @@ -123,45 +114,45 @@ The Current merge and verify jobs for jenkins job builder as pulled from the rep pattern: jjb/** - compare-type: ANT pattern: jjb-templates/** - - + + builders: - shell: !include-raw verify-releng - + - job-template: name: 'builder-merge' - + node: master - + # builder-merge job to run JJB update # # This job's purpose is to update all the JJB - + project-type: freestyle - + logrotate: daysToKeep: 30 numToKeep: 40 artifactDaysToKeep: -1 artifactNumToKeep: 5 - + parameters: - project-parameter: project: '{project}' - gerrit-parameter: branch: 'master' - + scm: - gerrit-trigger-scm: credentials-id: '{ssh-credentials}' refspec: '' choosing-strategy: 'default' - + wrappers: - ssh-agent-credentials: user: '{ssh-credentials}' - + triggers: - gerrit: trigger-on: @@ -177,17 +168,10 @@ The Current merge and verify jobs for jenkins job builder as pulled from the rep file-paths: - compare-type: ANT pattern: jjb/** - + builders: - shell: | source /opt/virtualenv/jenkins-job-builder/bin/activate cd /opt/jenkins-ci/releng git pull jenkins-jobs update --delete-old jjb/ - - - - -Revision: _sha1_ - -Build date: |today| diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 60554b3ae..cb0a0f369 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -48,10 +48,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -115,10 +111,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -179,10 +171,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - 'apex-{stream}' @@ -191,6 +179,13 @@ - 'apex-deploy-virtual' - 'apex-upload-artifact' - 'apex-workspace-cleanup' + - trigger-builds: + - project: 'functest-apex-{stream}' + block: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' ######################## # parameter macros diff --git a/jjb/armband/armband.yml b/jjb/armband/armband.yml index 021b365fd..81710d265 100644 --- a/jjb/armband/armband.yml +++ b/jjb/armband/armband.yml @@ -60,10 +60,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -107,10 +103,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -169,10 +161,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/availability/availability.yml b/jjb/availability/availability.yml index 73c9050d9..6bb068b49 100644 --- a/jjb/availability/availability.yml +++ b/jjb/availability/availability.yml @@ -59,10 +59,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -100,10 +96,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -156,10 +148,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/bottlenecks/bottlenecks.yml b/jjb/bottlenecks/bottlenecks.yml index 0baf7ba31..049a51eef 100644 --- a/jjb/bottlenecks/bottlenecks.yml +++ b/jjb/bottlenecks/bottlenecks.yml @@ -6,6 +6,8 @@ jobs: - 'bottlenecks-test' - 'bottlenecks-verify' + - 'bottlenecks-daily-{stream}' + - 'bottlenecks-merge' # stream: branch with - in place of / (eg. stable-arno) # branch: branch (eg. stable/arno) @@ -15,11 +17,9 @@ project: 'bottlenecks' somevar: 'foo' - -######################## -# job templates -######################## - +############################### +# Job templates +############################## - job-template: name: 'bottlenecks-test' @@ -35,6 +35,48 @@ - shell: | echo "Hello world from bottlenecks" +- job-template: + name: 'bottlenecks-daily-{stream}' + + node: master + + # Job template for daily builders + # + # Required Variables: + # stream: branch with - in place of / (eg. stable) + # branch: branch (eg. stable) + + project-type: freestyle + varsetabove: '{somevar}' + + logrotate: + daysToKeep: 30 + numToKeep: 10 + artifactDaysToKeep: -1 + artifactNumToKeep: -1 + + parameters: + - project-parameter: + project: '{project}' + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + triggers: + - timed: 'H H * * *' + + prebuilders: + - test-macro + + builders: + - shell: | + echo "hello world" + + postbuilders: + - test-macro - job-template: name: 'bottlenecks-verify' @@ -54,17 +96,12 @@ project: '{project}' - gerrit-parameter: branch: 'master' - scm: - gerrit-trigger-scm: credentials-id: '{ssh-credentials}' refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -85,5 +122,52 @@ branch-pattern: '**/master' builders: - - shell: - !include-raw docu-build.sh + - shell: | + echo "hello world" + +- job-template: + name: 'bottlenecks-merge' + + node: master + + # builder-merge job to run JJB update + # + # This job's purpose is to update all the JJB + + project-type: freestyle + + logrotate: + daysToKeep: 30 + numToKeep: 40 + artifactDaysToKeep: -1 + artifactNumToKeep: 5 + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: 'master' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + choosing-strategy: 'default' + + triggers: + - gerrit: + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: 'bottlenecks' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/master' + + builders: + - shell: | + echo "hello world" + diff --git a/jjb/bottlenecks/docu-build.sh b/jjb/bottlenecks/docu-build.sh deleted file mode 100644 index c56069271..000000000 --- a/jjb/bottlenecks/docu-build.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -project="$(git remote -v | head -n1 | awk '{{print $2}}' | sed -e 's,.*:\(.*/\)\?,,' -e 's/\.git$//')" -export PATH=$PATH:/usr/local/bin/ - -git_sha1="$(git rev-parse HEAD)" -docu_build_date="$(date)" - -if [[ $JOB_NAME =~ "verify" ]] ; then - subdir="/$GERRIT_CHANGE_NUMBER" -fi - -files=() -while read -r -d ''; do - files+=("$REPLY") -done < <(find * -type f -iname '*.rst' -print0) - -for file in "${{files[@]}}"; do - - file_cut="${{file%.*}}" - gs_cp_folder="${{file_cut}}" - - # sed part - sed -i "s/_sha1_/$git_sha1/g" $file - sed -i "s/_date_/$docu_build_date/g" $file - - # rst2html part - echo "rst2html $file" - rst2html $file | gsutil cp -L gsoutput.txt - \ - gs://artifacts.opnfv.org/"$project""$subdir"/"$gs_cp_folder".html - gsutil setmeta -h "Content-Type:text/html" \ - -h "Cache-Control:private, max-age=0, no-transform" \ - gs://artifacts.opnfv.org/"$project""$subdir"/"$gs_cp_folder".html - cat gsoutput.txt - rm -f gsoutput.txt - - echo "rst2pdf $file" - rst2pdf $file -o - | gsutil cp -L gsoutput.txt - \ - gs://artifacts.opnfv.org/"$project""$subdir"/"$gs_cp_folder".pdf - gsutil setmeta -h "Content-Type:application/pdf" \ - -h "Cache-Control:private, max-age=0, no-transform" \ - gs://artifacts.opnfv.org/"$project""$subdir"/"$gs_cp_folder".pdf - cat gsoutput.txt - rm -f gsoutput.txt - - links+="http://artifacts.opnfv.org/"$project""$subdir"/"$gs_cp_folder".html \n" - links+="http://artifacts.opnfv.org/"$project""$subdir"/"$gs_cp_folder".pdf \n" - -done - -images=() -while read -r -d ''; do - images+=("$REPLY") -done < <(find * -type f \( -iname \*.jpg -o -iname \*.png \) -print0) - -for img in "${{images[@]}}"; do - - # uploading found images - echo "uploading $img" - cat "$img" | gsutil cp -L gsoutput.txt - \ - gs://artifacts.opnfv.org/"$project""$subdir"/"$img" - gsutil setmeta -h "Content-Type:image/jpeg" \ - -h "Cache-Control:private, max-age=0, no-transform" \ - gs://artifacts.opnfv.org/"$project""$subdir"/"$img" - cat gsoutput.txt - rm -f gsoutput.txt - -done - -if [[ $GERRIT_EVENT_TYPE = "change-merged" ]] ; then - subdir="/$GERRIT_CHANGE_NUMBER" - if [ ! -z "$subdir" ]; then - gsutil rm gs://artifacts.opnfv.org/"$project""$subdir"/** || true - fi -fi - -echo -e "$links" - diff --git a/jjb/compass4nfv/compass4nfv.yml b/jjb/compass4nfv/compass4nfv.yml index 0732d1676..0e41e1ba6 100644 --- a/jjb/compass4nfv/compass4nfv.yml +++ b/jjb/compass4nfv/compass4nfv.yml @@ -76,9 +76,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' triggers: - gerrit: @@ -150,10 +147,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/conductor/conductor.yml b/jjb/conductor/conductor.yml index 26abe4585..01ccdaf14 100644 --- a/jjb/conductor/conductor.yml +++ b/jjb/conductor/conductor.yml @@ -58,10 +58,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -99,10 +95,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -155,10 +147,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index aa586642a..ddc6efdea 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -23,10 +23,6 @@ parameters: - project-parameter: project: '{project}' - wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' - job-template: name: '{project}-verify' diff --git a/jjb/dpacc/dpacc.yml b/jjb/dpacc/dpacc.yml index 2e46c559f..d027f47fe 100644 --- a/jjb/dpacc/dpacc.yml +++ b/jjb/dpacc/dpacc.yml @@ -58,10 +58,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -99,10 +95,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -155,10 +147,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/fastpathmetrics/fastpathmetrics.yml b/jjb/fastpathmetrics/fastpathmetrics.yml index 0facb1008..6c7edaa67 100644 --- a/jjb/fastpathmetrics/fastpathmetrics.yml +++ b/jjb/fastpathmetrics/fastpathmetrics.yml @@ -58,10 +58,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -99,10 +95,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -155,10 +147,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/fuel/fuel-build.sh b/jjb/fuel/fuel-build.sh index cffd8673b..00617ce54 100755 --- a/jjb/fuel/fuel-build.sh +++ b/jjb/fuel/fuel-build.sh @@ -19,9 +19,16 @@ else export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") fi +NOCACHE_PATTERN="verify: no-cache" +if [[ "$JOB_NAME" =~ "verify" && "$GERRIT_CHANGE_COMMIT_MESSAGE" =~ "$NOCACHE_PATTERN" ]]; then + echo "The cache will not be used for this build!" + NOCACHE_ARG="-f P" +fi +NOCACHE_ARG=${{NOCACHE_ARG:-}} + # start the build -cd $WORKSPACE/$INSTALLER/ci -./build.sh -v $OPNFV_ARTIFACT_VERSION -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY +cd $WORKSPACE/ci +./build.sh -v $OPNFV_ARTIFACT_VERSION $NOCACHE_ARG -c file://$CACHE_DIRECTORY $BUILD_DIRECTORY # list the build artifacts ls -al $BUILD_DIRECTORY diff --git a/jjb/fuel/fuel-deploy-virtual.sh b/jjb/fuel/fuel-deploy-virtual.sh new file mode 100755 index 000000000..a65395b07 --- /dev/null +++ b/jjb/fuel/fuel-deploy-virtual.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# source the file so we get OPNFV vars +source latest.properties + +# echo the info about artifact that is used during the deployment +echo "Using $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f3) for deployment" + +# create TMPDIR if it doesn't exist +export TMPDIR=$HOME/tmpdir +[[ -d $TMPDIR ]] || mkdir -p $TMPDIR + +# change permissions down to TMPDIR +chmod a+x $HOME +chmod a+x $TMPDIR + +# set CONFDIR, BRIDGE +CONFDIR=$WORKSPACE/deploy/templates/virtual_environment/conf +BRIDGE=pxebr + +# log info to console +echo "Starting the deployment for a merged change using $INSTALLER. This could take some time..." +echo "--------------------------------------------------------" +echo + +# start the deployment +echo "Issuing command" +echo "sudo $WORKSPACE/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh" + +sudo $WORKSPACE/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh + +echo +echo "--------------------------------------------------------" +echo "Virtual deployment is done! Removing the intermediate files from artifact repo" + +PROPERTIES_FILE=$(echo $OPNFV_ARTIFACT_URL | sed 's/iso/properties/') +gsutil rm gs://$OPNFV_ARTIFACT_URL +gsutil rm gs://$PROPERTIES_FILE diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh index 2ec519597..b617bcf01 100755 --- a/jjb/fuel/fuel-deploy.sh +++ b/jjb/fuel/fuel-deploy.sh @@ -18,7 +18,7 @@ chmod a+x $HOME chmod a+x $TMPDIR # set CONFDIR, BRIDGE -CONFDIR=$WORKSPACE/fuel/deploy/templates/hardware_environment/conf/linux_foundation_lab/pod2 +CONFDIR=$WORKSPACE/deploy/templates/hardware_environment/conf/linux_foundation_lab/pod2 BRIDGE=pxebr # clone genesis repo and checkout the SR1 tag @@ -41,9 +41,9 @@ echo # start the deployment echo "Issuing command" -echo "sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh" +echo "sudo $WORKSPACE/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh" -sudo $WORKSPACE/fuel/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh +sudo $WORKSPACE/ci/deploy.sh -iso $WORKSPACE/opnfv.iso -dea $CONFDIR/dea.yaml -dha $CONFDIR/dha.yaml -s $TMPDIR -b $BRIDGE -nh echo echo "--------------------------------------------------------" diff --git a/jjb/fuel/fuel-virtual-deploy.sh b/jjb/fuel/fuel-virtual-deploy.sh deleted file mode 100755 index 4e0b2dc47..000000000 --- a/jjb/fuel/fuel-virtual-deploy.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "Placeholder for Fuel virtual deploy" diff --git a/jjb/fuel/fuel.yml b/jjb/fuel/fuel.yml index e809592fc..c0ba580de 100644 --- a/jjb/fuel/fuel.yml +++ b/jjb/fuel/fuel.yml @@ -12,7 +12,7 @@ jobs: - 'fuel-verify-build-{stream}' - 'fuel-merge-build-{stream}' - - 'fuel-merge-virtual-deploy-{stream}' + - 'fuel-merge-deploy-virtual-{stream}' - 'fuel-daily-{stream}' - 'fuel-build-{stream}' - 'fuel-deploy-{stream}' @@ -64,7 +64,7 @@ wrappers: - ssh-agent-credentials: - user: '{ssh-credentials}' + users: '{ssh-credentials}' - timeout: timeout: 360 fail: true @@ -87,6 +87,7 @@ branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' + readable-message: true builders: - shell: @@ -130,7 +131,7 @@ wrappers: - ssh-agent-credentials: - user: '{ssh-credentials}' + users: '{ssh-credentials}' - timeout: timeout: 360 fail: true @@ -157,11 +158,11 @@ !include-raw ./fuel-workspace-cleanup.sh - job-template: - name: 'fuel-merge-virtual-deploy-{stream}' + name: 'fuel-merge-deploy-virtual-{stream}' project-type: freestyle - node: ericsson-build + node: fuel-deploy-virtual concurrent: true @@ -213,10 +214,14 @@ - shell: !include-raw ./fuel-download-artifact.sh - shell: - !include-raw ./fuel-virtual-deploy.sh + !include-raw ./fuel-deploy-virtual.sh - shell: !include-raw ./fuel-workspace-cleanup.sh + publishers: + - email: + recipients: jonas.bjurel@ericsson.com stefan.k.berg@ericsson.com + - job-template: name: 'fuel-daily-{stream}' diff --git a/jjb/functest/functest-docker.sh b/jjb/functest/functest-docker.sh deleted file mode 100644 index c73bcf99a..000000000 --- a/jjb/functest/functest-docker.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -set -o errexit -set -o nounset -set -o pipefail - - -echo "Starting the build of Functest Docker." -echo "--------------------------------------------------------" -echo - -DOCKER_IMAGE_NAME="opnfv/functest" - - -# Get tag version -cd $WORKSPACE -git clone https://gerrit.opnfv.org/gerrit/releng - -DOCKER_TAG=$($WORKSPACE/releng/utils/calculate_version.sh -t docker \ - -n $DOCKER_IMAGE_NAME) - -ret_val=$? -if [ $ret_val -ne 0 ]; then - echo "Error retrieving the version tag." - exit 1 -else - echo "Tag version to be build and pushed: $DOCKER_TAG" -fi - - -# Remove previous running containers if exist -if [[ ! -z $(docker ps -a | grep $DOCKER_IMAGE_NAME) ]]; then - echo "Removing existing $DOCKER_IMAGE_NAME containers..." - docker ps | grep $DOCKER_IMAGE_NAME | awk '{{print $1}}' | xargs docker stop - docker ps -a | grep $DOCKER_IMAGE_NAME | awk '{{print $1}}' | xargs docker rm -fi - - -# Remove existing images if exist -if [[ ! -z $(docker images | grep $DOCKER_IMAGE_NAME) ]]; then - echo "Docker images to remove:" - docker images | head -1 && docker images | grep $DOCKER_IMAGE_NAME - image_tags=($(docker images | grep $DOCKER_IMAGE_NAME | awk '{{print $2}}')) - for tag in "${{image_tags[@]}}"; do - echo "Removing docker image $DOCKER_IMAGE_NAME:$tag..." - docker rmi $DOCKER_IMAGE_NAME:$tag - done -fi - - -# Start the build -echo "Building docker image: $DOCKER_IMAGE_NAME:$DOCKER_TAG..." -cd $WORKSPACE/docker -docker build -t $DOCKER_IMAGE_NAME:$DOCKER_TAG . -echo "Creating tag 'latest'..." -docker tag $DOCKER_IMAGE_NAME:$DOCKER_TAG $DOCKER_IMAGE_NAME:latest - -# list the images -echo "Available images are:" -docker images - - -if [ "$PUSH_IMAGE" == "true" ]; then - echo "Pushing $DOCKER_IMAGE_NAME:$DOCKER_TAG to the docker registry..." - echo "--------------------------------------------------------" - echo - # Push to the Dockerhub repository - docker push $DOCKER_IMAGE_NAME:$DOCKER_TAG - - echo "Updating $DOCKER_IMAGE_NAME:latest to the docker registry..." - docker push $DOCKER_IMAGE_NAME:latest -fi diff --git a/jjb/functest/functest.yml b/jjb/functest/functest.yml index 4df779d8c..47024e20e 100644 --- a/jjb/functest/functest.yml +++ b/jjb/functest/functest.yml @@ -9,18 +9,19 @@ node: 'opnfv-jump-2' installer_type: 'fuel' installer_ip: '10.20.0.2' - functest_repo_dir: '/home/opnfv/repos/functest' - foreman: node: 'opnfv-jump-2' installer_type: 'foreman' installer_ip: '172.30.10.73' - functest_repo_dir: '/home/opnfv/repos/functest' + - apex: + node: 'opnfv-jump-1' + installer_type: 'apex' + installer_ip: '192.168.X.X' pod: - orange-build: node: 'orange-build' installer_type: 'foreman' installer_ip: '172.30.10.73' - functest_repo_dir: '/home/opnfv/repos/functest' jobs: - 'set-functest-env-{pod}' @@ -199,6 +200,7 @@ parameters: - project-parameter: project: '{project}' + - functest-parameter - string: name: INSTALLER_TYPE default: '{installer_type}' @@ -207,10 +209,6 @@ name: INSTALLER_IP default: '{installer_ip}' description: "Installer IP." - - string: - name: FUNCTEST_REPO_DIR - default: '{functest_repo_dir}' - description: "Directory where the root functest repo is cloned in the docker." - '{pod}' scm: @@ -241,6 +239,7 @@ parameters: - project-parameter: project: '{project}' + - functest-parameter - string: name: INSTALLER_TYPE default: '{installer_type}' @@ -250,10 +249,6 @@ default: '{installer_ip}' description: "Installer IP." - string: - name: FUNCTEST_REPO_DIR - default: '{functest_repo_dir}' - description: "Directory where the root functest repo is cloned in the docker." - - string: name: GIT_BASE default: ssh://gerrit.opnfv.org:29418/$PROJECT description: "Override GIT_BASE" @@ -286,6 +281,7 @@ parameters: - project-parameter: project: '{project}' + - functest-parameter - string: name: INSTALLER_TYPE default: '{installer_type}' @@ -295,10 +291,6 @@ default: '{installer_ip}' description: "Installer IP." - string: - name: FUNCTEST_REPO_DIR - default: '{functest_repo_dir}' - description: "Directory where the root functest repo is cloned in the docker." - - string: name: GIT_BASE default: ssh://gerrit.opnfv.org:29418/$PROJECT description: "Override GIT_BASE" @@ -350,10 +342,6 @@ refspec: '' branch: '{stream}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - 'functest-{stream}' @@ -391,10 +379,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -447,10 +431,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -478,6 +458,10 @@ name: GS_PATHNAME default: '{gs-pathname}' description: "Version directory where the opnfv documents will be stored in gs repository" + - string: + name: FUNCTEST_REPO_DIR + default: "/home/opnfv/repos/functest" + description: "Directory where the Functest repository is cloned" - parameter: name: orange-build @@ -538,7 +522,10 @@ - shell: | #!/bin/bash set +e - + if [[ ${INSTALLER_TYPE} == 'apex' ]]; then + INSTALLER_IP=$(cat instack /var/lib/libvirt/dnsmasq/default.leases \ + |sed 's/.*192/192/g' | sed 's/\ in.*//g') + fi echo "Functest: Start Docker and prepare environment" envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP}" docker pull opnfv/functest:latest_stable @@ -643,10 +630,24 @@ builders: - shell: | #!/bin/bash - set +e - # cleanup: remove any docker containers leftovers - echo "Removing existing Functest Docker containers..." - docker rm -f $(docker ps | grep opnfv/functest | awk '{print $1}') - echo "Removing existing Functest Docker image..." - docker rmi -f $(docker images | grep opnfv/functest | awk '{print $3}') + echo "Cleaning up docker containers/images..." + + # Remove previous running containers if exist + if [[ ! -z $(docker ps -a | grep opnfv/functest) ]]; then + echo "Removing existing opnfv/functest containers..." + docker ps | grep opnfv/functest | awk '{print $1}' | xargs docker stop + docker ps -a | grep opnfv/functest | awk '{print $1}' | xargs docker rm + fi + + + # Remove existing images if exist + if [[ ! -z $(docker images | grep opnfv/functest) ]]; then + echo "Docker images to remove:" + docker images | head -1 && docker images | grep opnfv/functest + image_tags=($(docker images | grep opnfv/functest | awk '{print $2}')) + for tag in "${image_tags[@]}"; do + echo "Removing docker image opnfv/functest:$tag..." + docker rmi opnfv/functest:$tag + done + fi diff --git a/jjb/genesis/genesis-docs.yml b/jjb/genesis/genesis-docs.yml index 6a1a804f2..7f7833769 100644 --- a/jjb/genesis/genesis-docs.yml +++ b/jjb/genesis/genesis-docs.yml @@ -51,10 +51,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -113,9 +109,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' triggers: - gerrit: diff --git a/jjb/genesis/genesis-foreman.yml b/jjb/genesis/genesis-foreman.yml index e9dfa8c4d..69a69d3ea 100644 --- a/jjb/genesis/genesis-foreman.yml +++ b/jjb/genesis/genesis-foreman.yml @@ -68,10 +68,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -149,10 +145,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -440,10 +432,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/genesis/genesis-fuel.yml b/jjb/genesis/genesis-fuel.yml index a957a693b..04dcfcd90 100644 --- a/jjb/genesis/genesis-fuel.yml +++ b/jjb/genesis/genesis-fuel.yml @@ -70,10 +70,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -156,10 +152,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/genesis/genesis-juju.yml b/jjb/genesis/genesis-juju.yml index 8e17a7b81..3ecb74c22 100644 --- a/jjb/genesis/genesis-juju.yml +++ b/jjb/genesis/genesis-juju.yml @@ -53,10 +53,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -120,10 +116,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/ipv6/ipv6.yml b/jjb/ipv6/ipv6.yml index 5957a93bf..7d198303a 100644 --- a/jjb/ipv6/ipv6.yml +++ b/jjb/ipv6/ipv6.yml @@ -58,10 +58,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -99,10 +95,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -155,10 +147,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/kvmfornfv/kvmfornfv.yml b/jjb/kvmfornfv/kvmfornfv.yml index 345edcaaf..69dad7735 100644 --- a/jjb/kvmfornfv/kvmfornfv.yml +++ b/jjb/kvmfornfv/kvmfornfv.yml @@ -41,10 +41,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -101,10 +97,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/octopus/octopus.yml b/jjb/octopus/octopus.yml index e11ccc56c..5f77e8a4d 100644 --- a/jjb/octopus/octopus.yml +++ b/jjb/octopus/octopus.yml @@ -72,10 +72,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - 'octopus-{stream}' @@ -113,10 +109,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -169,9 +161,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' triggers: - gerrit: diff --git a/jjb/opnfv/opnfv-docker.sh b/jjb/opnfv/opnfv-docker.sh new file mode 100644 index 000000000..2b42938c0 --- /dev/null +++ b/jjb/opnfv/opnfv-docker.sh @@ -0,0 +1,93 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + + +echo "Starting opnfv-docker for $DOCKER_REPO_NAME ..." +echo "--------------------------------------------------------" +echo + + +# Remove previous running containers if exist +if [[ ! -z $(docker ps -a | grep $DOCKER_REPO_NAME) ]]; then + echo "Removing existing $DOCKER_REPO_NAME containers..." + docker ps | grep $DOCKER_REPO_NAME | awk '{{print $1}}' | xargs docker stop + docker ps -a | grep $DOCKER_REPO_NAME | awk '{{print $1}}' | xargs docker rm +fi + + +# Remove existing images if exist +if [[ ! -z $(docker images | grep $DOCKER_REPO_NAME) ]]; then + echo "Docker images to remove:" + docker images | head -1 && docker images | grep $DOCKER_REPO_NAME + image_tags=($(docker images | grep $DOCKER_REPO_NAME | awk '{{print $2}}')) + for tag in "${{image_tags[@]}}"; do + echo "Removing docker image $DOCKER_REPO_NAME:$tag..." + docker rmi $DOCKER_REPO_NAME:$tag + done +fi + +# If we just want to update the latest_stable image +if [[ "$UPDATE_LATEST_STABLE" == "true" ]]; then + echo "Pulling $DOCKER_REPO_NAME:$STABLE_TAG ..." + docker pull $DOCKER_REPO_NAME:$STABLE_TAG + if [[ $? -ne 0 ]]; then + echo "ERROR: The image $DOCKER_REPO_NAME with tag $STABLE_TAG does not exist." + exit 1 + fi + docker tag $DOCKER_REPO_NAME:$STABLE_TAG $DOCKER_REPO_NAME:latest_stable + echo "Pushing $DOCKER_REPO_NAME:latest_stable ..." + docker push $DOCKER_REPO_NAME:latest_stable + exit 0 +fi + + +# Get tag version +cd $WORKSPACE +git clone https://gerrit.opnfv.org/gerrit/releng + +DOCKER_TAG=$($WORKSPACE/releng/utils/calculate_version.sh -t docker \ + -n $DOCKER_REPO_NAME) + +ret_val=$? +if [ $ret_val -ne 0 ]; then + echo "Error retrieving the version tag." + exit 1 +else + echo "Tag version to be build and pushed: $DOCKER_TAG" +fi + + +# cd to directory where Dockerfile is located +if [[ "$DOCKER_REPO_NAME" == "opnfv/functest" ]]; then + cd $WORKSPACE/docker +elif [[ "$DOCKER_REPO_NAME" == "opnfv/yardstick" ]]; then + cd $WORKSPACE/ci/docker/yardstick-ci +else + echo "ERROR: DOCKER_REPO_NAME parameter not valid: $DOCKER_REPO_NAME" + exit 1 +fi + +# Start the build +echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_TAG..." + +docker build -t $DOCKER_REPO_NAME:$DOCKER_TAG . +echo "Creating tag 'latest'..." +docker tag $DOCKER_REPO_NAME:$DOCKER_TAG $DOCKER_REPO_NAME:latest + +# list the images +echo "Available images are:" +docker images + +# Push image to Dockerhub +if [[ "$PUSH_IMAGE" == "true" ]]; then + echo "Pushing $DOCKER_REPO_NAME:$DOCKER_TAG to the docker registry..." + echo "--------------------------------------------------------" + echo + # Push to the Dockerhub repository + docker push $DOCKER_REPO_NAME:$DOCKER_TAG + + echo "Updating $DOCKER_REPO_NAME:latest to the docker registry..." + docker push $DOCKER_REPO_NAME:latest +fi diff --git a/jjb/functest/functest-docker.yml b/jjb/opnfv/opnfv-docker.yml index ee5adc6f7..dfff050a9 100644 --- a/jjb/functest/functest-docker.yml +++ b/jjb/opnfv/opnfv-docker.yml @@ -1,15 +1,17 @@ ############################################## -# job configuration for functest docker build +# job configuration for docker build and push ############################################## - project: - name: functest-docker + name: opnfv-docker - project: 'functest' + project: + - 'functest' + - 'yardstick' jobs: - - 'functest-docker-build-push-{stream}' + - '{project}-docker-build-push-{stream}' stream: - master: @@ -18,9 +20,8 @@ ######################## # job templates ######################## - - job-template: - name: 'functest-docker-build-push-{stream}' + name: '{project}-docker-build-push-{stream}' project-type: freestyle @@ -31,8 +32,29 @@ parameters: - project-parameter: project: '{project}' - - functest-docker-parameter - + - string: + name: GIT_BASE + default: "https://gerrit.opnfv.org/gerrit/$PROJECT" + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + - string: + name: PUSH_IMAGE + default: "true" + description: "To enable/disable pushing the image to Dockerhub." + - string: + name: BASE_VERSION + default: "brahmaputra.0" + description: "Base version to be used." + - string: + name: DOCKER_REPO_NAME + default: "opnfv/{project}" + description: "Dockerhub repo to be pushed to." + - string: + name: UPDATE_LATEST_STABLE + default: "false" + description: "This will update the latest_stable image only." + - string: + name: STABLE_TAG + description: "If above option is true, this is the tag to be pulled." scm: - git-scm: @@ -48,27 +70,7 @@ builders: - shell: - !include-raw ./functest-docker.sh + !include-raw ./opnfv-docker.sh triggers: - pollscm: "*/30 * * * *" - - -######################## -# parameter macros -######################## -- parameter: - name: functest-docker-parameter - parameters: - - string: - name: GIT_BASE - default: "https://gerrit.opnfv.org/gerrit/$PROJECT" - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." - - string: - name: PUSH_IMAGE - default: "true" - description: "To enable/disable pushing the image to Dockerhub." - - string: - name: BASE_VERSION - default: "brahmaputra.0" - description: "Base version to be used." diff --git a/jjb/opnfv/opnfv-docs.yml b/jjb/opnfv/opnfv-docs.yml index efb6b5bf1..f7e4c5bb4 100644 --- a/jjb/opnfv/opnfv-docs.yml +++ b/jjb/opnfv/opnfv-docs.yml @@ -46,10 +46,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -65,7 +61,7 @@ projects: - project-compare-type: 'REG_EXP' #Left out arno projects oscar|octopus|functest|genesis - project-pattern: 'vswitchperf|yardstick|apex|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sfc|vnf_forwarding_graph|enfv|pinpoint|pharos|opnfvdocs' + project-pattern: 'vswitchperf|yardstick|apex|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sdnvpn|sfc|vnf_forwarding_graph|enfv|pinpoint|pharos|opnfvdocs' branches: - branch-compare-type: 'ANT' branch-pattern: '**/master' @@ -74,8 +70,7 @@ pattern: 'docs/**' builders: - - shell: - !include-raw ../../utils/docu-build-new.sh + - upload-review-docs - job-template: name: 'opnfv-docs-merge' @@ -102,9 +97,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' triggers: - gerrit: @@ -114,7 +106,7 @@ comment-contains-value: 'remerge' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'vswitchperf|yardstick|apex|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sfc|vnf_forwarding_graph|enfv|pinpoint|pharos|opnfvdocs' + project-pattern: 'vswitchperf|yardstick|apex|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sdnvpn|sfc|vnf_forwarding_graph|enfv|pinpoint|pharos|opnfvdocs' branches: - branch-compare-type: 'ANT' branch-pattern: '**/master' @@ -123,5 +115,4 @@ pattern: 'docs/**' builders: - - shell: - !include-raw ../../utils/docu-build-new.sh + - upload-merged-docs diff --git a/jjb/oscar/oscar.yml b/jjb/oscar/oscar.yml index 6585feb50..9beb57593 100644 --- a/jjb/oscar/oscar.yml +++ b/jjb/oscar/oscar.yml @@ -58,10 +58,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -99,10 +95,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -155,10 +147,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/ovsnfv/ovsnfv.yml b/jjb/ovsnfv/ovsnfv.yml index f7dd1ce28..1c5810a03 100644 --- a/jjb/ovsnfv/ovsnfv.yml +++ b/jjb/ovsnfv/ovsnfv.yml @@ -1,12 +1,10 @@ - project: name: ovsnfv jobs: - - 'ovsnfv-verify' - - 'ovsnfv-merge' + - 'ovsnfv-verify-{stream}' + - 'ovsnfv-merge-{stream}' - 'ovsnfv-daily-{stream}' - # stream: branch with - in place of / (eg. stable-arno) - # branch: branch (eg. stable/arno) stream: - master: branch: 'master' @@ -14,9 +12,9 @@ project: 'ovsnfv' - job-template: - name: 'ovsnfv-verify' + name: 'ovsnfv-verify-{stream}' - node: ericsson-build + node: opnfv-build project-type: freestyle @@ -42,10 +40,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -66,17 +60,19 @@ branch-pattern: '**/master' builders: - - shell: - echo "Hello World" + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail -- job-template: - name: 'ovsnfv-merge' + cd $WORKSPACE/ci + ./build.sh - # builder-merge job to run JJB update - # - # This job's purpose is to update all the JJB +- job-template: + name: 'ovsnfv-merge-{stream}' - node: master + node: opnfv-build project-type: freestyle @@ -102,10 +98,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -120,19 +112,20 @@ branch-pattern: '**/master' builders: - - shell: - echo "Hello World" + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + cd $WORKSPACE/ci + ./build.sh - job-template: name: 'ovsnfv-daily-{stream}' - # Job template for daily builders - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - node: master + node: opnfv-build disabled: true @@ -147,6 +140,10 @@ parameters: - project-parameter: project: '{project}' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." scm: - git-scm: @@ -154,13 +151,15 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - - timed: 'H H * * *' + - timed: '@midnight' builders: - - shell: - echo "Hello World" + - shell: | + #!/bin/bash + set -o errexit + set -o nounset + set -o pipefail + + cd $WORKSPACE/ci + ./build.sh diff --git a/jjb/parser/parser.yml b/jjb/parser/parser.yml index 4e9b251d0..22324c5fb 100644 --- a/jjb/parser/parser.yml +++ b/jjb/parser/parser.yml @@ -60,10 +60,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -101,10 +97,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -125,8 +117,8 @@ branch-pattern: '**/master' builders: - - shell: - !include-raw docu-build.sh + - shell: | + echo "Hello world from parser" - job-template: name: 'parser-merge' @@ -157,9 +149,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' triggers: - gerrit: diff --git a/jjb/pharos/pharos.yml b/jjb/pharos/pharos.yml index c7638997f..ab6705b14 100644 --- a/jjb/pharos/pharos.yml +++ b/jjb/pharos/pharos.yml @@ -71,10 +71,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - 'pharos-{stream}' @@ -112,10 +108,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -168,10 +160,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/prediction/prediction.yml b/jjb/prediction/prediction.yml index 7f7dd3b3f..072affddf 100644 --- a/jjb/prediction/prediction.yml +++ b/jjb/prediction/prediction.yml @@ -60,10 +60,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -101,10 +97,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -157,9 +149,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' triggers: - gerrit: diff --git a/jjb/qtip/qtip.yml b/jjb/qtip/qtip.yml index e87408885..0b863f7c4 100644 --- a/jjb/qtip/qtip.yml +++ b/jjb/qtip/qtip.yml @@ -71,10 +71,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -112,10 +108,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -168,10 +160,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/releng-defaults.yaml b/jjb/releng-defaults.yaml index a78a37e91..3ba2a23a4 100644 --- a/jjb/releng-defaults.yaml +++ b/jjb/releng-defaults.yaml @@ -9,4 +9,7 @@ build-artifact-num-to-keep: 5 ssh-credentials: 'd42411ac011ad6f3dd2e1fa34eaa5d87f910eb2e' - + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' diff --git a/jjb/releng-jobs.yaml b/jjb/releng-jobs.yaml index d732b1fab..6194bcd96 100644 --- a/jjb/releng-jobs.yaml +++ b/jjb/releng-jobs.yaml @@ -30,10 +30,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -59,9 +55,6 @@ pattern: jjb-templates/** - compare-type: ANT pattern: utils/** - forbidden-file-paths: - - compare-type: ANT - pattern: docs/** builders: @@ -97,10 +90,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -118,9 +107,6 @@ pattern: jjb/** - compare-type: ANT pattern: utils/** - forbidden-file-paths: - - compare-type: ANT - pattern: docs/** builders: - shell: | diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml index 2c694c847..3afe8482a 100644 --- a/jjb/releng-macros.yaml +++ b/jjb/releng-macros.yaml @@ -143,3 +143,202 @@ name: jacoco-nojava-workaround builders: - shell: 'mkdir -p $WORKSPACE/target/classes' + + +# New Releng macros + +- builder: + name: build-html-and-pdf-docs-output + builders: + - shell: | + #!/bin/bash -e + set -o pipefail + export PATH=$PATH:/usr/local/bin/ + + [[ $GERRIT_CHANGE_NUMBER =~ .+ ]] + + _get_title_script=" + import os + from docutils import core, nodes + with open('index.rst', 'r') as file: + data = file.read() + doctree = core.publish_doctree(data, + settings_overrides={'report_level': 5, + 'halt_level': 5}) + if isinstance(doctree[0], nodes.title): + title = doctree[0] + else: + for c in doctree.children: + if isinstance(c, nodes.section): + title = c[0] + break + print title.astext()" + _git_sha1="$(git rev-parse HEAD)" + + git clone ssh://gerrit.opnfv.org:29418/releng + [[ -d releng ]] + + find docs/ -name 'index.rst' -printf '%h\n' | while read dir + do + _name="${dir##*/}" + _build="$dir/build" + _output="docs/output/$_name" + + echo + echo "#################${dir//?/#}" + echo "Building DOCS in $dir" + echo "#################${dir//?/#}" + echo + + sed -i "s/_sha1_/$_git_sha1/g" "$dir/index.rst" + + if [[ ! -f "$dir/conf.py" ]] ; then + cp releng/docs/etc/conf.py "$dir/conf.py" + _title=$(cd $dir; python -c "$_get_title_script") + echo "latex_documents = [('index', '$_name.tex', \"$_title\", 'OPNFV', 'manual'),]" >> "$dir/conf.py" + fi + cp -f releng/docs/etc/opnfv-logo.png "$dir/opnfv-logo.png" + + mkdir -p "$_output" + + sphinx-build -b html -E "$dir" "$_output" + + # Note: PDF creation may fail in project doc builds. + # We allow this test to be marked as succeeded with + # failure in PDF creation, but leave message to fix it. + # Any failure has to be fixed before B release. + { + sphinx-build -b latex -E "$dir" "$_build" + make -C "$_build" LATEXOPTS='--interaction=nonstopmode' all-pdf + mv "$_build/$_name.pdf" "$_output" + } || { + _msg="Error: PDF creation for $dir has failed, please fix source rst file(s)." + echo + echo "$_msg" + echo + echo "$_msg" >> gerrit_comment.txt + } + done + +- builder: + name: upload-under-review-docs-to-opnfv-artifacts + builders: + - shell: | + #!/bin/bash -e + set -o pipefail + export PATH=$PATH:/usr/local/bin/ + + [[ $GERRIT_CHANGE_NUMBER =~ .+ ]] + [[ -d docs/output ]] + + echo + echo "###########################" + echo "UPLOADING DOCS UNDER REVIEW" + echo "###########################" + echo + + gs_base="artifacts.opnfv.org/review" + gs_path="$gs_base/$GERRIT_CHANGE_NUMBER" + local_path="docs/$GERRIT_CHANGE_NUMBER" + + mv docs/output "$local_path" + gsutil -m cp -r "$local_path" "gs://$gs_base" + + if gsutil ls "gs://$gs_path" | grep -e 'html$' > /dev/null 2>&1 ; then + gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + "gs://$gs_path"/**.html + fi + + echo "Document link(s):" >> gerrit_comment.txt + find "$local_path" | grep -e 'index.html$' -e 'pdf$' | \ + sed -e "s|^$local_path| http://$gs_path|" >> gerrit_comment.txt + +- builder: + name: upload-merged-docs-to-opnfv-artifacts + builders: + - shell: | + #!/bin/bash -e + set -o pipefail + export PATH=$PATH:/usr/local/bin/ + + [[ -d docs/output ]] + + echo + echo "#####################" + echo "UPLOADING MERGED DOCS" + echo "#####################" + echo + + if [[ "$GERRIT_BRANCH" == "master" ]] ; then + gs_base="artifacts.opnfv.org/$GERRIT_PROJECT" + else + gs_base="artifacts.opnfv.org/$GERRIT_PROJECT/$GERRIT_BRANCH" + fi + gs_path="$gs_base/docs" + local_path="docs/docs" + + mv docs/output "$local_path" + gsutil -m cp -r "$local_path" "gs://$gs_base" + + if gsutil ls "gs://$gs_path" | grep -e 'html$' > /dev/null 2>&1 ; then + gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + "gs://$gs_path"/**.html + fi + + echo "Document link(s):" >> gerrit_comment.txt + find "$local_path" | grep -e 'index.html$' -e 'pdf$' | \ + sed -e "s|^$local_path| http://$gs_path|" >> gerrit_comment.txt + +- builder: + name: report-docs-build-result-to-gerrit + builders: + - shell: | + #!/bin/bash -e + export PATH=$PATH:/usr/local/bin/ + if [[ -e gerrit_comment.txt ]] ; then + echo + echo "posting review comment to gerrit..." + echo + cat gerrit_comment.txt + echo + ssh -p 29418 gerrit.opnfv.org \ + "gerrit review -p $GERRIT_PROJECT \ + -m '$(cat gerrit_comment.txt)' \ + $GERRIT_PATCHSET_REVISION" + fi + +- builder: + name: remove-old-docs-from-opnfv-artifacts + builders: + - shell: | + #!/bin/bash -e + export PATH=$PATH:/usr/local/bin/ + + [[ $GERRIT_CHANGE_NUMBER =~ .+ ]] + + gs_path="artifacts.opnfv.org/review/$GERRIT_CHANGE_NUMBER" + + if gsutil ls "gs://$gs_path" > /dev/null 2>&1 ; then + echo + echo "Deleting Out-of-dated Documents..." + gsutil -m rm -r "gs://$gs_path" + fi + +- builder: + name: upload-review-docs + builders: + - build-html-and-pdf-docs-output + - upload-under-review-docs-to-opnfv-artifacts + - report-docs-build-result-to-gerrit + +- builder: + name: upload-merged-docs + builders: + - build-html-and-pdf-docs-output + - upload-merged-docs-to-opnfv-artifacts + - report-docs-build-result-to-gerrit + - remove-old-docs-from-opnfv-artifacts diff --git a/jjb/storperf/storperf.yml b/jjb/storperf/storperf.yml index f1250cbc2..2a3c42a6b 100644 --- a/jjb/storperf/storperf.yml +++ b/jjb/storperf/storperf.yml @@ -1,8 +1,8 @@ - project: name: storperf jobs: - - 'storperf-verify' - - 'storperf-merge' + - 'storperf-verify-{stream}' + - 'storperf-merge-{stream}' - 'storperf-daily-{stream}' # stream: branch with - in place of / (eg. stable-arno) @@ -14,9 +14,9 @@ project: 'storperf' - job-template: - name: 'storperf-verify' + name: 'storperf-verify-{stream}' - node: ericsson-build + node: intel-jp-build-1 project-type: freestyle @@ -31,6 +31,10 @@ project: '{project}' - gerrit-parameter: branch: 'master' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." scm: - gerrit-trigger-scm: @@ -38,10 +42,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -67,13 +67,13 @@ - job-template: - name: 'storperf-merge' + name: 'storperf-merge-{stream}' # builder-merge job to run JJB update # # This job's purpose is to update all the JJB - node: ericsson-build + node: intel-jp-build-1 project-type: freestyle @@ -88,6 +88,10 @@ project: '{project}' - gerrit-parameter: branch: 'master' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." scm: - gerrit-trigger-scm: @@ -95,10 +99,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -129,7 +129,7 @@ # Required Variables: # stream: branch with - in place of / (eg. stable) # branch: branch (eg. stable) - node: ericsson-build + node: intel-jp-build-1 disabled: true @@ -151,10 +151,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' diff --git a/jjb/vnf_forwarding_graph/vnf_forwarding_graph.yml b/jjb/vnf_forwarding_graph/vnf_forwarding_graph.yml index 67c9ebb23..0d7e8b7e2 100644 --- a/jjb/vnf_forwarding_graph/vnf_forwarding_graph.yml +++ b/jjb/vnf_forwarding_graph/vnf_forwarding_graph.yml @@ -58,10 +58,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -99,10 +95,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -155,10 +147,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/vswitchperf/vswitchperf.yml b/jjb/vswitchperf/vswitchperf.yml index 07ce11414..9f767ced3 100644 --- a/jjb/vswitchperf/vswitchperf.yml +++ b/jjb/vswitchperf/vswitchperf.yml @@ -1,10 +1,9 @@ - project: name: vswitchperf jobs: - - 'vswitchperf-test' - 'vswitchperf-daily-{stream}' - - 'vswitchperf-merge' - - 'vswitchperf-verify' + - 'vswitchperf-merge-{stream}' + - 'vswitchperf-verify-{stream}' # stream: branch with - in place of / (eg. stable-helium) # branch: branch (eg. stable/helium) @@ -15,20 +14,6 @@ project: 'vswitchperf' somevar: 'foo' -- job-template: - name: vswitchperf-test - - project-type: freestyle - - logrotate: - daysToKeep: 30 - numToKeep: 10 - artifactDaysToKeep: -1 - artifactNumToKeep: -1 - - builders: - - shell: | - echo "Hello world from vswitchperf" - job-template: name: 'vswitchperf-daily-{stream}' @@ -41,7 +26,7 @@ project-type: freestyle varsetabove: '{somevar}' - node: ericsson-build + node: opnfv-build logrotate: daysToKeep: '{build-days-to-keep}' @@ -62,10 +47,6 @@ refspec: '' branch: '{branch}' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - timed: 'H H * * *' @@ -85,10 +66,19 @@ - test-macro - job-template: - name: 'vswitchperf-verify' + name: 'vswitchperf-verify-{stream}' project-type: freestyle - node: ericsson-build + + node: opnfv-build + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 3 + max-per-node: 2 logrotate: daysToKeep: 30 @@ -110,10 +100,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -146,14 +132,23 @@ make sanity - job-template: - name: 'vswitchperf-merge' + name: 'vswitchperf-merge-{stream}' # builder-merge job to run JJB update # # This job's purpose is to update all the JJB project-type: freestyle - node: ericsson-build + + node: opnfv-build + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 3 + max-per-node: 2 logrotate: daysToKeep: 30 @@ -176,10 +171,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/jjb/yardstick/yardstick.yml b/jjb/yardstick/yardstick.yml index 60344fa11..5afaaf723 100644 --- a/jjb/yardstick/yardstick.yml +++ b/jjb/yardstick/yardstick.yml @@ -56,10 +56,6 @@ refspec: '$GERRIT_REFSPEC' choosing-strategy: 'gerrit' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: @@ -127,10 +123,6 @@ refspec: '' choosing-strategy: 'default' - wrappers: - - ssh-agent-credentials: - user: '{ssh-credentials}' - triggers: - gerrit: trigger-on: diff --git a/utils/docu-build-new.sh b/utils/docu-build-new.sh deleted file mode 100755 index 67a62e381..000000000 --- a/utils/docu-build-new.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -export PATH=$PATH:/usr/local/bin/ -git_sha1="$(git rev-parse HEAD)" - -clean() {{ -if [[ -d docs/output ]]; then -rm -rf docs/output -echo "cleaning up output directory" -fi -}} - -trap clean EXIT TERM INT SIGTERM SIGHUP - -#set git_sha1 -files=() -while read -r -d ''; do - files+=("$REPLY") -done < <(find docs/ -type f -iname '*.rst' -print0) -for file in "${{files[@]}}"; do - sed -i "s/_sha1_/$git_sha1/g" $file -done - -directories=() -while read -d $'\n'; do - directories+=("$REPLY") -done < <(find docs/ -name 'index.rst' -printf '%h\n' | sort -u ) - -for dir in "${{directories[@]}}"; do - _name="${{dir##*/}}" - _build="${{dir}}/build" - _output="docs/output/${{_name}}" - echo - echo "#################${{_name//?/#}}" - echo "Building DOCS in ${{_name}}" - echo "#################${{_name//?/#}}" - echo - - mkdir -p "${{_output}}" - - sphinx-build -b html -E -c docs/etc "${{dir}}" "${{_output}}" - - sphinx-build -b latex -E -c docs/etc "${{dir}}" "${{_build}}" - make -C "${{_build}}" LATEXOPTS='--interaction=nonstopmode' all-pdf - mv "${{_build}}"/*.pdf "${{_output}}" - -done - -# NOTE: make sure source parameters for GS paths are not empty. -[[ $GERRIT_CHANGE_NUMBER =~ .+ ]] -[[ $GERRIT_PROJECT =~ .+ ]] -[[ $GERRIT_BRANCH =~ .+ ]] - -gs_path_review="artifacts.opnfv.org/review/$GERRIT_CHANGE_NUMBER" - -if [[ $GERRIT_BRANCH = "master" ]] ; then - gs_path_branch="artifacts.opnfv.org/$GERRIT_PROJECT" -else - gs_path_branch="artifacts.opnfv.org/$GERRIT_PROJECT/${{GERRIT_BRANCH##*/}}" -fi - -for dir in "${{directories[@]}}"; do - echo - echo "#############################" - echo "UPLOADING DOCS in ${{dir##*/}}" - echo "#############################" - echo - - - if [[ $JOB_NAME =~ "verify" ]] ; then - - #upload artifacts for verify job - gsutil cp -r docs/output/"${{dir##*/}}/" "gs://$gs_path_review/" - - # post link to gerrit as comment - gerrit_comment="$(echo '"Document is available at 'http://$gs_path_review/"${{dir##*/}}"/index.html' for review"')" - echo "$gerrit_comment" - ssh -p 29418 gerrit.opnfv.org gerrit review -p $GERRIT_PROJECT -m \ - "$gerrit_comment" $GERRIT_PATCHSET_REVISION - - #set cache to 0 - for x in $(gsutil ls gs://$gs_path_review/"${{dir##*/}}" | grep html); - do - gsutil setmeta -h "Content-Type:text/html" \ - -h "Cache-Control:private, max-age=0, no-transform" \ - "$x" - done - - else - - #upload artifacts for merge job - gsutil cp -r docs/output/"${{dir##*/}}" "gs://$gs_path_branch/docs/" - echo "Latest document is available at http://$gs_path_branch/docs/"${{dir##*/}}"/index.html" - - #set cache to 0 - for x in $(gsutil ls gs://$gs_path_branch/"${{dir}}" | grep html); - do - gsutil setmeta -h "Content-Type:text/html" \ - -h "Cache-Control:private, max-age=0, no-transform" \ - "$x" - done - - #Clean up review when merging - if gsutil ls "gs://$gs_path_review" > /dev/null 2>&1 ; then - echo - echo "Deleting Out-of-dated Documents..." - gsutil rm -r "gs://$gs_path_review" - fi - - fi - -done diff --git a/utils/jenkins-jnlp-connect.sh b/utils/jenkins-jnlp-connect.sh index d263b198a..00c19971f 100755 --- a/utils/jenkins-jnlp-connect.sh +++ b/utils/jenkins-jnlp-connect.sh @@ -100,7 +100,7 @@ fi if [[ $started_monit == "true" ]]; then wget --timestamping https://build.opnfv.org/ci/jnlpJars/slave.jar && true -chown $jenkinsuser $jenkinsuser slave.jar +chown $jenkinsuser:$jenkinsuser slave.jar if [[ -f /var/run/$jenkinsuser/jenkins_jnlp_pid ]]; then echo "pid file found" diff --git a/utils/test/result_collection_api/dashboard/vsperf2Dashboard.py b/utils/test/result_collection_api/dashboard/vsperf2Dashboard.py new file mode 100755 index 000000000..323d3915c --- /dev/null +++ b/utils/test/result_collection_api/dashboard/vsperf2Dashboard.py @@ -0,0 +1,121 @@ +#!/usr/bin/python + +# Copyright 2015 Intel Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"), +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +def get_vsperf_cases(): + """ + get the list of the supported test cases + TODO: update the list when adding a new test case for the dashboard + """ + return ["tput_ovsdpdk", "tput_ovs", + "b2b_ovsdpdk", "b2b_ovs", + "tput_mod_vlan_ovsdpdk", "tput_mod_vlan_ovs", + "cont_ovsdpdk", "cont_ovs", + "pvp_cont_ovsdpdkuser", "pvp_cont_ovsdpdkcuse", "pvp_cont_ovsvirtio", + "pvvp_cont_ovsdpdkuser", "pvvp_cont_ovsdpdkcuse", "pvvp_cont_ovsvirtio", + "scalability_ovsdpdk", "scalability_ovs", + "pvp_tput_ovsdpdkuser", "pvp_tput_ovsdpdkcuse", "pvp_tput_ovsvirtio", + "pvp_b2b_ovsdpdkuser", "pvp_b2b_ovsdpdkcuse", "pvp_b2b_ovsvirtio", + "pvvp_tput_ovsdpdkuser", "pvvp_tput_ovsdpdkcuse", "pvvp_tput_ovsvirtio", + "pvvp_b2b_ovsdpdkuser", "pvvp_b2b_ovsdpdkcuse", "pvvp_b2b_ovsvirtio", + "cpu_load_ovsdpdk", "cpu_load_ovs", + "mem_load_ovsdpdk", "mem_load_ovs"] + + +def check_vsperf_case_exist(case): + """ + check if the testcase exists + if the test case is not defined or not declared in the list + return False + """ + vsperf_cases = get_vsperf_cases() + + if (case is None or case not in vsperf_cases): + return False + else: + return True + + +def format_vsperf_for_dashboard(case, results): + """ + generic method calling the method corresponding to the test case + check that the testcase is properly declared first + then build the call to the specific method + """ + if check_vsperf_case_exist(case): + res = format_common_for_dashboard(case, results) + else: + res = [] + print "Test cases not declared" + return res + + +def format_common_for_dashboard(case, results): + """ + Common post processing + """ + test_data_description = case + " results for Dashboard" + test_data = [{'description': test_data_description}] + + graph_name = '' + if "b2b" in case: + graph_name = "B2B frames" + else: + graph_name = "Rx frames per second" + + # Graph 1: Rx fps = f(time) + # ******************************** + new_element = [] + for data in results: + new_element.append({'x': data['creation_date'], + 'y1': data['details']['64'], + 'y2': data['details']['128'], + 'y3': data['details']['512'], + 'y4': data['details']['1024'], + 'y5': data['details']['1518']}) + + test_data.append({'name': graph_name, + 'info': {'type': "graph", + 'xlabel': 'time', + 'y1label': 'frame size 64B', + 'y2label': 'frame size 128B', + 'y3label': 'frame size 512B', + 'y4label': 'frame size 1024B', + 'y5label': 'frame size 1518B'}, + 'data_set': new_element}) + + return test_data + + + + +############################ For local test ################################ +import os + +def _test(): + ans = [{'creation_date': '2015-09-12', 'project_name': 'vsperf', 'version': 'ovs_master', 'pod_name': 'pod1-vsperf', 'case_name': 'tput_ovsdpdk', 'installer': 'build_sie', 'details': {'64': '26.804', '1024': '1097.284', '512': '178.137', '1518': '12635.860', '128': '100.564'}}, + {'creation_date': '2015-09-33', 'project_name': 'vsperf', 'version': 'ovs_master', 'pod_name': 'pod1-vsperf', 'case_name': 'tput_ovsdpdk', 'installer': 'build_sie', 'details': {'64': '16.804', '1024': '1087.284', '512': '168.137', '1518': '12625.860', '128': '99.564'}}] + + result = format_vsperf_for_dashboard("pvp_cont_ovsdpdkcuse", ans) + print result + + result = format_vsperf_for_dashboard("b2b_ovsdpdk", ans) + print result + + result = format_vsperf_for_dashboard("non_existing", ans) + print result + +if __name__ == '__main__': + _test() |