diff options
| -rw-r--r-- | jjb/functest/functest-daily-jobs.yml | 256 | ||||
| -rw-r--r-- | jjb/global/releng-macros.yml | 2 | ||||
| -rw-r--r-- | jjb/releng/opnfv-utils.yml | 38 | ||||
| -rwxr-xr-x | utils/artifacts.opnfv.org.sh | 162 |
4 files changed, 417 insertions, 41 deletions
diff --git a/jjb/functest/functest-daily-jobs.yml b/jjb/functest/functest-daily-jobs.yml index 099d01a59..0f1ac351d 100644 --- a/jjb/functest/functest-daily-jobs.yml +++ b/jjb/functest/functest-daily-jobs.yml @@ -193,22 +193,107 @@ # <<: *master # ------------------------------- - testsuite: - - 'suite': - job-timeout: 60 - - 'daily': - job-timeout: 300 - - 'arm-daily': - job-timeout: 300 + functest-tier: + - 'healthcheck': + job-timeout: 20 + - 'smoke': + job-timeout: 180 + - 'components': + job-timeout: 240 + - 'vnf': + job-timeout: 240 - jobs: - - 'functest-{installer}-{pod}-{testsuite}-{stream}' + features-suite: + - 'doctor-notification' + - 'promise' + - 'bgpvpn' + - 'functest-odl-sfc' + - 'domino-multinode' + - 'fds' + - 'barometercollectd' + + exclude: + - features-suite: 'promise' + stream: 'master' + - features-suite: 'domino-multinode' + stream: 'master' + - features-suite: 'fds' + stream: 'euphrates' + + build-job-settings: &build-job-settings + current-parameter: false + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-jobs: false + predefined-parameters: | + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_TIER=$FUNCTEST_TIER + + features-job-settings: &features-job-settings + current-parameter: false + git-revision: true + node-parameters: false + kill-phase-on: FAILURE + abort-all-jobs: false + predefined-parameters: | + FUNCTEST_MODE=$FUNCTEST_MODE + FUNCTEST_SUITE_NAME=$FUNCTEST_SUITE_NAME + jobs: + - 'functest-{installer}-{pod}-suite-{stream}' + - 'functest-{installer}-{pod}-daily-{stream}' + - 'functest-{installer}-{pod}-daily-{functest-tier}-{stream}' + - 'functest-{installer}-{pod}-daily-features-{features-suite}-{stream}' ################################ # job template ################################ - job-template: - name: 'functest-{installer}-{pod}-{testsuite}-{stream}' + + name: 'functest-{installer}-{pod}-daily-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - '{installer}-defaults' + - '{slave-label}-defaults' + - functest-parameter: + gs-pathname: '{gs-pathname}' + + builders: + - multijob: + name: 'Run functest tests' + execution-type: SEQUENTIAL + projects: + - name: 'functest-{installer}-{pod}-daily-healthcheck-{stream}' + <<: *build-job-settings + - name: 'functest-{installer}-{pod}-daily-smoke-{stream}' + <<: *build-job-settings + - name: 'functest-{installer}-{pod}-daily-vnf-{stream}' + <<: *build-job-settings + - name: 'functest-{installer}-{pod}-daily-features-doctor-notification-{stream}' + <<: *features-job-settings + - name: 'functest-{installer}-{pod}-daily-features-bgpvpn-{stream}' + <<: *features-job-settings + - name: 'functest-{installer}-{pod}-daily-features-functest-odl-sfc-{stream}' + <<: *features-job-settings + - name: 'functest-{installer}-{pod}-daily-features-barometercollectd-{stream}' + <<: *features-job-settings + - name: 'functest-{installer}-{pod}-daily-features-fds-{stream}' + <<: *features-job-settings + - name: 'functest-{installer}-{pod}-daily-features-promise-{stream}' + <<: *features-job-settings + - name: 'functest-{installer}-{pod}-daily-features-domino-multinode-{stream}' + <<: *features-job-settings + +- job-template: &functest-installer-tier-job + + name: 'functest-{installer}-{pod}-daily-{functest-tier}-{stream}' concurrent: true @@ -232,10 +317,15 @@ branch: '{branch}' - '{installer}-defaults' - '{slave-label}-defaults' - - 'functest-{testsuite}-parameter' + - string: + name: FUNCTEST_MODE + default: 'tier' + - string: + name: FUNCTEST_TIER + default: '{functest-tier}' - string: name: DEPLOY_SCENARIO - default: 'os-odl_l2-nofeature-ha' + default: 'os-nosdn-nofeature-noha' - string: name: CLEAN_DOCKER_IMAGES default: 'false' @@ -249,7 +339,93 @@ builders: - description-setter: description: "Built on $NODE_NAME" - - 'functest-{testsuite}-builder' + - 'functest-daily-builder' + + publishers: + - 'functest-recipients' + - 'fuel-recipients' + +- job-template: + + name: 'functest-{installer}-{pod}-suite-{stream}' + <<: *functest-installer-tier-job + + wrappers: + - build-name: + name: '$BUILD_NUMBER Suite: $FUNCTEST_MODE Scenario: $DEPLOY_SCENARIO' + - timeout: + timeout: '60' + abort: true + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - '{installer}-defaults' + - '{slave-label}-defaults' + - 'functest-suite-parameter' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-noha' + - string: + name: CLEAN_DOCKER_IMAGES + default: 'false' + description: 'Remove downloaded docker images (opnfv/functest*:*)' + - functest-parameter: + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - 'functest-suite-builder' + +- job-template: + + name: 'functest-{installer}-{pod}-daily-features-{features-suite}-{stream}' + concurrent: true + properties: + - logrotate-default + - throttle: + enabled: true + max-per-node: 1 + option: 'project' + wrappers: + - build-name: + name: '$BUILD_NUMBER Suite: $FUNCTEST_MODE Scenario: $DEPLOY_SCENARIO' + + parameters: + - project-parameter: + project: '{project}' + branch: '{branch}' + - '{installer}-defaults' + - '{slave-label}-defaults' + - string: + name: FUNCTEST_MODE + default: 'testcase' + - string: + name: FUNCTEST_SUITE_NAME + default: '{features-suite}' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-noha' + - string: + name: CLEAN_DOCKER_IMAGES + default: 'false' + description: 'Remove downloaded docker images (opnfv/functest*:*)' + - functest-parameter: + gs-pathname: '{gs-pathname}' + + scm: + - git-scm + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - 'functest-daily-builder' + + publishers: + - 'functest-features-recipients' + - 'functest-recipients' + - 'fuel-recipients' ######################## @@ -264,14 +440,6 @@ description: "Daily suite name to run" - parameter: - name: functest-arm-daily-parameter - parameters: - - string: - name: FUNCTEST_MODE - default: 'arm-daily' - description: "Daily suite name (Aarch64) to run" - -- parameter: name: functest-suite-parameter parameters: - choice: @@ -364,14 +532,6 @@ - 'functest-exit' - builder: - name: functest-arm-daily-builder - builders: - - 'functest-cleanup' - - 'functest-arm-daily' - - 'functest-store-results' - - 'functest-exit' - -- builder: name: functest-suite-builder builders: - 'functest-cleanup' @@ -405,17 +565,6 @@ # yamllint enable rule:indentation - builder: - name: functest-arm-daily - builders: - # yamllint disable rule:indentation - - shell: - !include-raw: - - ./functest-env-presetup.sh - - ../../utils/fetch_os_creds.sh - - ./functest-alpine.sh - -# yamllint enable rule:indentation -- builder: name: functest-store-results builders: - shell: @@ -432,3 +581,30 @@ builders: - shell: !include-raw: ./functest-exit.sh + + +- publisher: + name: 'functest-features-recipients' + publishers: + - email: + recipients: + +- publisher: + name: 'functest-recipients' + publishers: + - email: + recipients: > + jalausuch@suse.com morgan.richomme@orange.com + cedric.ollivier@orange.com feng.xiaowei@zte.com.cn + juha.kosonen@nokia.com wangwulin@huawei.com + delia.popescu@enea.com cristina.pauna@enea.com + valentin.boucher@kontron.com + - email-jenkins-admins-on-failure + +- publisher: + name: 'fuel-recipients' + publishers: + - email: + recipients: > + peter.barabas@ericsson.com mpolenchuck@mirantis.com + alexandru.avadanii@enea.com guillermo.herrero@enea.com diff --git a/jjb/global/releng-macros.yml b/jjb/global/releng-macros.yml index a12a3c8a5..0bf981661 100644 --- a/jjb/global/releng-macros.yml +++ b/jjb/global/releng-macros.yml @@ -901,7 +901,7 @@ - email-ext: <<: *email_ptl_defaults recipients: > - ManuelBuilmbuil@suse.com + mbuil@suse.com - publisher: name: 'email-snaps-ptl' diff --git a/jjb/releng/opnfv-utils.yml b/jjb/releng/opnfv-utils.yml index b12f663aa..93b3615ef 100644 --- a/jjb/releng/opnfv-utils.yml +++ b/jjb/releng/opnfv-utils.yml @@ -8,6 +8,7 @@ - 'archive-repositories' - 'check-status-of-slaves' - 'ansible-build-server' + - 'generate-artifacts-index-pages' ######################## # job templates @@ -164,3 +165,40 @@ sudo ansible-playbook -C -D -i \ $WORKSPACE/utils/build-server-ansible/inventory.ini \ $WORKSPACE/utils/build-server-ansible/main.yml + + +- job-template: + name: 'generate-artifacts-index-pages' + + project-type: freestyle + + disabled: false + + concurrent: false + + parameters: + - node: + name: SLAVE_NAME + description: Build Servers + default-slaves: + - lf-build2 + - project-parameter: + project: releng + branch: master + + scm: + - git-scm + + triggers: + - timed: '@hourly' + + builders: + - generate-artifact-html + + +- builder: + name: generate-artifact-html + builders: + - shell: | + cd $WORKSPACE/utils/ + ./artifacts.opnfv.org.sh diff --git a/utils/artifacts.opnfv.org.sh b/utils/artifacts.opnfv.org.sh new file mode 100755 index 000000000..1984b4992 --- /dev/null +++ b/utils/artifacts.opnfv.org.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + +#export PATH=${PATH}:/root/gsutil + +#Step Generate index.html +if [ -f index.html ] ; then + rm -f index.html +fi + +OUTPUT="index.html" + +for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs | grep -v review | awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,) +do +echo $index + echo "<LI><a href=\"${index%/*}.html\">"$index"</a></LI>" >> $OUTPUT +done + +#functest logs########################## + +for project in functest vswitchperf +do + + for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, ) + do + index="$(echo ${index%/*} | sed s,/,_,g)" + echo "<LI><a href=\"http://artifacts.opnfv.org/${index%/*}.html\">"$index"</a></LI>" >> $OUTPUT + done + +done +#End step 1 +##################################### + + +#genrate html files for all project except vswitchperf +for index in $(gsutil ls -l gs://artifacts.opnfv.org | grep -v logs |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,,) +do +OUTPUT=${index%/*}.html +rm -f $OUTPUT + + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi +done + +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + +done + + + +#generate file for vswitch perf (I dont know what happend here but there is a wierd character in this bucket) + +index=vswitchperf +OUTPUT=${index%/*}.html +rm -f $OUTPUT + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi + +done + + +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + +# Gerate html for logs + +for project in functest vswitchperf +do + for index in $(gsutil ls -l gs://artifacts.opnfv.org/logs/"$project"/ |awk 'NF==1'| sed s,gs://artifacts.opnfv.org/,, ) + do + + OUTPUT="$(echo ${index%/*}.html | sed s,/,_,g)" + echo $OUTPUT + rm -f $OUTPUT + + + for filepath in $(gsutil ls -R gs://artifacts.opnfv.org/"$index" | sed s,gs://artifacts.opnfv.org/,, | grep -v "favicon.ico" | grep -v "gsutil" ); do + echo $filepath + + if [[ $filepath =~ "/:" ]]; then + path=$(echo $filepath| sed s,/:,,g) + echo "<UL>" >> $OUTPUT + echo "<LI>$path</LI>" >> $OUTPUT + echo "</UL>" >> $OUTPUT + else + echo "<LI><a href=\"http://artifacts.opnfv.org/$filepath\">"$filepath"</a></LI>" >> $OUTPUT + fi + + + done + + + gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + + gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ + + + done +done + + + +OUTPUT="index.html" +echo "<p> Generated on $(date) </p>" >> $OUTPUT + +cat <<EOF >> $OUTPUT +<script> +(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){ +(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o), +m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m) +})(window,document,'script','//www.google-analytics.com/analytics.js','ga'); +ga('create', 'UA-831873-26', 'auto'); +ga('send', 'pageview'); +</script> +EOF + +#copy and uplad index file genrated in first step, last +gsutil cp $OUTPUT gs://artifacts.opnfv.org/ + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://artifacts.opnfv.org/$OUTPUT \ |