diff options
52 files changed, 1811 insertions, 537 deletions
@@ -15,7 +15,6 @@ Fatih Degirmenci (Ericsson, fatih.degirmenci@ericsson.com) Aric Gardner (Linux Foundation, agardner@linuxfoundation.org) Tim Rozet (Red Hat, trozet@redhat.com) Morgan Richomme (Orange, morgan.richomme@orange.com) -Victor Laza (CloudBase Solutions, vlaza@cloudbasesolutions.com) Matthew Lijun (Huawei, matthew.lijun@huawei.com) Peter Bandzi (Cisco, pbandzi@cisco.com) Jose Lausuch (Ericsson, jose.lausuch@ericsson.com) diff --git a/docs/etc/conf.py b/docs/etc/conf.py index 8540f9997..c4cbae775 100644 --- a/docs/etc/conf.py +++ b/docs/etc/conf.py @@ -1,3 +1,11 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## ''' Base configuration file for building OPNFV docs diff --git a/docs/how-to-use-docs/documentation-example.rst b/docs/how-to-use-docs/documentation-example.rst index bab2ee157..c0ffc956a 100644 --- a/docs/how-to-use-docs/documentation-example.rst +++ b/docs/how-to-use-docs/documentation-example.rst @@ -27,6 +27,18 @@ Note: You may have "docs/how-to-use-docs/" in you project repo. You can delete it, since it is sample and master version is stored in releng repo. +Note: +During the document build process, 'docs_build' and 'docs_output' will be +created in the project root directory. To avoid conflict, you should not +use those names and should add the following entries in '.gitignore' file +so that git can ignore built files. + +.. code-block:: bash + + /docs_build/ + /docs_output/ + /releng/ + Index File ========== @@ -118,25 +130,29 @@ Testing ======= You can test document build in your laptop by using build script which is -used in document build jobs: +also used in document build jobs below: .. code-block:: bash - $ cd /loacal/repo/path/to/project - $ git clone ssh://gerrit.opnfv.org:29418/releng + $ cd /local/repo/path/to/project + $ git clone https://git.opnfv.org/releng $ ./releng/utils/docs-build.sh -Then, you can see docs in output directory if build succeeded. +Then, you can see the docs in 'docs_output' directory if build succeeded. -This script will generate files in 'build' and 'output'. You should consider -to add the following entries in '.gitignore' file, so that git can ignore -built files. +If you have a problem of missing python package, you can make sure all +necessary packages are installed as follows: .. code-block:: bash - /docs_build/ - /docs_output/ - /releng/ + $ sudo pip install Sphinx==1.3.1 doc8 sphinxcontrib-httpdomain + +Note: +Developers are encouraged to use "ssh://<username>@gerrit.opnfv.org:29418/releng" +instead of "https://git.opnfv.org/releng", so that you can quickly start +development in releng. +See https://wiki.opnfv.org/developer/getting_started for more detail. + Jenkins Jobs ============ diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 2719b3a47..35e8e3da5 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -14,16 +14,18 @@ - master: branch: 'master' gs-pathname: '' + block-stream: 'brahmaputra' - brahmaputra: branch: 'stable/brahmaputra' gs-pathname: '/brahmaputra' - disabled: true + block-stream: 'master' + disabled: false project: 'apex' scenario: - 'os-odl_l2-nofeature-ha' - - 'os-odl_l2-sfc-ha' + - 'os-odl_l2-sfc-noha' - 'os-odl_l3-nofeature-ha' - 'os-onos-nofeature-ha' - 'os-opencontrail-nofeature-ha' @@ -61,7 +63,7 @@ - patchset-created-event: exclude-drafts: 'false' exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' + exclude-no-code-change: 'true' - draft-published-event - comment-added-contains-event: comment-contains-value: 'recheck' @@ -82,9 +84,6 @@ pattern: 'lib/**' - compare-type: ANT pattern: 'config/**' - forbidden-file-paths: - - compare-type: ANT - pattern: 'docs/**' properties: - build-blocker: @@ -94,25 +93,32 @@ - "apex-deploy.*" - "apex-build.*" - "apex-runner.*" + - 'apex-verify-{block-stream}' builders: - 'apex-build' - trigger-builds: - project: 'apex-deploy-virtual-os-odl_l2-nofeature-ha-{stream}' predefined-parameters: - BUILD_DIRECTORY=apex-verify-master/build_output + BUILD_DIRECTORY=apex-verify-{stream}/build_output git-revision: false block: true - trigger-builds: - project: 'apex-deploy-virtual-os-onos-nofeature-ha-{stream}' predefined-parameters: - BUILD_DIRECTORY=apex-verify-master/build_output + BUILD_DIRECTORY=apex-verify-{stream}/build_output git-revision: false block: true - trigger-builds: - project: 'apex-deploy-virtual-os-odl_l3-nofeature-ha-{stream}' predefined-parameters: - BUILD_DIRECTORY=apex-verify-master/build_output + BUILD_DIRECTORY=apex-verify-{stream}/build_output + git-revision: false + block: true + - trigger-builds: + - project: 'apex-deploy-virtual-os-odl_l2-sfc-noha-{stream}' + predefined-parameters: + BUILD_DIRECTORY=apex-verify-{stream}/build_output git-revision: false block: true - 'apex-workspace-cleanup' @@ -157,16 +163,18 @@ git-revision: false block: true - trigger-builds: - - project: 'functest-apex-opnfv-jump-1-daily-{stream}' + - project: 'yardstick-apex-opnfv-jump-1-daily-{stream}' predefined-parameters: - DEPLOY_SCENARIO='{scenario}' + DEPLOY_SCENARIO={scenario} block: true block-thresholds: build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' - trigger-builds: - - project: 'yardstick-apex-opnfv-jump-1-daily-{stream}' + - project: 'functest-apex-opnfv-jump-1-daily-{stream}' + predefined-parameters: + DEPLOY_SCENARIO={scenario} block: true block-thresholds: build-step-failure-threshold: 'never' @@ -214,7 +222,7 @@ - trigger-builds: - project: 'apex-deploy-virtual-os-odl_l2-nofeature-ha-{stream}' predefined-parameters: - BUILD_DIRECTORY=apex-build-master/build_output + BUILD_DIRECTORY=apex-build-{stream}/build_output git-revision: false block: true - 'apex-upload-artifact' @@ -343,7 +351,7 @@ - trigger-builds: - project: 'apex-deploy-baremetal-os-odl_l2-nofeature-ha-{stream}' predefined-parameters: - BUILD_DIRECTORY=apex-build-master/build_output + BUILD_DIRECTORY=apex-build-{stream}/build_output git-revision: true block: true - trigger-builds: @@ -357,6 +365,8 @@ unstable-threshold: 'FAILURE' - trigger-builds: - project: 'yardstick-apex-opnfv-jump-1-daily-{stream}' + predefined-parameters: + DEPLOY_SCENARIO=os-odl_l2-nofeature-ha block: true block-thresholds: build-step-failure-threshold: 'never' @@ -365,7 +375,7 @@ - trigger-builds: - project: 'apex-deploy-baremetal-os-onos-nofeature-ha-{stream}' predefined-parameters: - BUILD_DIRECTORY=apex-build-master/build_output + BUILD_DIRECTORY=apex-build-{stream}/build_output git-revision: true block: true - trigger-builds: @@ -379,6 +389,8 @@ unstable-threshold: 'FAILURE' - trigger-builds: - project: 'yardstick-apex-opnfv-jump-1-daily-{stream}' + predefined-parameters: + DEPLOY_SCENARIO=os-onos-nofeature-ha block: true block-thresholds: build-step-failure-threshold: 'never' @@ -387,7 +399,7 @@ - trigger-builds: - project: 'apex-deploy-baremetal-os-odl_l3-nofeature-ha-{stream}' predefined-parameters: - BUILD_DIRECTORY=apex-build-master/build_output + BUILD_DIRECTORY=apex-build-{stream}/build_output git-revision: true block: true - trigger-builds: @@ -401,6 +413,8 @@ unstable-threshold: 'FAILURE' - trigger-builds: - project: 'yardstick-apex-opnfv-jump-1-daily-{stream}' + predefined-parameters: + DEPLOY_SCENARIO=os-odl_l3-nofeature-ha block: true block-thresholds: build-step-failure-threshold: 'never' @@ -456,16 +470,20 @@ # create the cache directory if it doesn't exist [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY # set OPNFV_ARTIFACT_VERSION - if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then - export OPNFV_ARTIFACT_VERSION="bramaputra.1.rc0" - else - if echo $BUILD_TAG | grep "apex-verify" 1> /dev/null; then + if echo $BUILD_TAG | grep "apex-verify" 1> /dev/null; then + if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then + export OPNFV_ARTIFACT_VERSION=brahmaputra-dev${BUILD_NUMBER} + else export OPNFV_ARTIFACT_VERSION=dev${BUILD_NUMBER} - elif [ "$ARTIFACT_VERSION" == "daily" ]; then - export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d") + fi + elif [ "$ARTIFACT_VERSION" == "daily" ]; then + if echo $GERRIT_BRANCH | grep "brahmaputra" 1> /dev/null; then + export OPNFV_ARTIFACT_VERSION=brahmaputra-$(date -u +"%Y-%m-%d") else - export OPNFV_ARTIFACT_VERSION=${ARTIFACT_VERSION} + export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d") fi + else + export OPNFV_ARTIFACT_VERSION=${ARTIFACT_VERSION} fi # start the build cd $WORKSPACE/ci @@ -594,7 +612,7 @@ fi RPM_LIST=$RPM_INSTALL_PATH - for pkg in common undercloud; do + for pkg in common undercloud opendaylight-sfc; do RPM_LIST+=" ${RPM_INSTALL_PATH/opnfv-apex/opnfv-apex-${pkg}}" done @@ -730,8 +748,8 @@ - trigger: name: 'apex-master' triggers: - - timed: '0 3 * * *' + - timed: '0 3 * 12 *' - trigger: name: 'apex-brahmaputra' triggers: - - timed: '0 6 * * 2050' + - timed: '0 3 * * *' diff --git a/jjb/bottlenecks/bottlenecks.yml b/jjb/bottlenecks/bottlenecks.yml index 26069cf7b..a65330d5d 100644 --- a/jjb/bottlenecks/bottlenecks.yml +++ b/jjb/bottlenecks/bottlenecks.yml @@ -78,8 +78,6 @@ - job-template: name: 'bottlenecks-{suite}-upload-artifacts-{stream}' - node: ericsson-build - concurrent: true properties: @@ -91,7 +89,7 @@ parameters: - project-parameter: project: '{project}' - - 'ericsson-ca-build-1-defaults' + - 'ericsson-build-defaults' - bottlenecks-parameter: gs-pathname: '{gs-pathname}' suite: '{suite}' diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index 11fac7cc7..40f17b889 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -23,15 +23,22 @@ # brahmaputra #-------------------------------- - huawei-us-deploy-bare-1: + os-version: 'trusty' <<: *brahmaputra - huawei-virtual: + os-version: 'trusty' <<: *brahmaputra #-------------------------------- # master #-------------------------------- + - huawei-us-deploy-bare-2: + os-version: 'centos7' + <<: *master - intel-pod8: + os-version: 'trusty' <<: *master - huawei-virtual: + os-version: 'trusty' <<: *master scenario: @@ -41,6 +48,9 @@ - 'os-odl_l2-nofeature-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-trigger' + - 'os-odl_l3-nofeature-ha': + disabled: false + auto-trigger-name: 'compass-{scenario}-{pod}-trigger' - 'os-onos-nofeature-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-trigger' @@ -58,22 +68,22 @@ - job-template: name: 'compass-{scenario}-{pod}-daily-{stream}' - node: huawei-build - - concurrent: false + concurrent: true properties: - throttle: enabled: true - max-total: 0 max-per-node: 1 + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'compass-os-.*?-{pod}-daily-.*?' + - 'compass-verify-[^-]*' + block-level: 'NODE' wrappers: - build-name: name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' - - timeout: - timeout: 360 - fail: true parameters: - project-parameter: @@ -94,8 +104,9 @@ - trigger-builds: - project: 'compass-deploy-{pod}-daily-{stream}' current-parameters: false - predefined-parameters: + predefined-parameters: | DEPLOY_SCENARIO={scenario} + COMPASS_OS_VERSION={os-version} same-node: true block: true - trigger-builds: @@ -131,16 +142,20 @@ properties: - throttle: enabled: true - max-total: 0 max-per-node: 1 - build-blocker: use-build-blocker: true blocking-jobs: - - 'compass-deploy-{pod}-daily-{stream}' + - 'compass-deploy-{pod}-daily-.*?' + - 'compass-verify-deploy-.*?' + block-level: 'NODE' wrappers: - build-name: name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + - timeout: + timeout: 120 + abort: true parameters: - project-parameter: @@ -164,11 +179,9 @@ builders: - shell: - !include-raw: ./compass-workspace-precleanup.sh - - shell: - !include-raw: ./compass-download-artifact.sh + !include-raw-escape: ./compass-download-artifact.sh - shell: - !include-raw: ./compass-deploy.sh + !include-raw-escape: ./compass-deploy.sh publishers: - archive: @@ -195,23 +208,42 @@ choices: - 'liberty' - 'juno' - - choice: - name: COMPASS_OS_VERSION - choices: - - 'trusty' - - 'centos7' ######################## # trigger macros ######################## - trigger: + name: 'compass-os-nosdn-nofeature-ha-huawei-us-deploy-bare-2-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-os-odl_l2-nofeature-ha-huawei-us-deploy-bare-2-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-os-odl_l3-nofeature-ha-huawei-us-deploy-bare-2-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-os-onos-nofeature-ha-huawei-us-deploy-bare-2-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-os-ocl-nofeature-ha-huawei-us-deploy-bare-2-trigger' + triggers: + - timed: '' +- trigger: name: 'compass-os-nosdn-nofeature-ha-huawei-us-deploy-bare-1-trigger' triggers: - timed: '0 3 * * * ' - trigger: name: 'compass-os-odl_l2-nofeature-ha-huawei-us-deploy-bare-1-trigger' triggers: - - timed: '0 20 * * *' + - timed: '0 21 * * *' +- trigger: + name: 'compass-os-odl_l3-nofeature-ha-huawei-us-deploy-bare-1-trigger' + triggers: + - timed: '' - trigger: name: 'compass-os-onos-nofeature-ha-huawei-us-deploy-bare-1-trigger' triggers: @@ -219,36 +251,44 @@ - trigger: name: 'compass-os-ocl-nofeature-ha-huawei-us-deploy-bare-1-trigger' triggers: - - timed: '' + - timed: '0 9 * * *' - trigger: name: 'compass-os-nosdn-nofeature-ha-intel-pod8-trigger' triggers: - - timed: '' + - timed: '0 3 * * *' - trigger: name: 'compass-os-odl_l2-nofeature-ha-intel-pod8-trigger' triggers: + - timed: '0 21 * * *' +- trigger: + name: 'compass-os-odl_l3-nofeature-ha-intel-pod8-trigger' + triggers: - timed: '' - trigger: name: 'compass-os-onos-nofeature-ha-intel-pod8-trigger' triggers: - - timed: '' + - timed: '0 15 * * *' - trigger: name: 'compass-os-ocl-nofeature-ha-intel-pod8-trigger' triggers: - - timed: '' + - timed: '0 9 * * *' - trigger: name: 'compass-os-nosdn-nofeature-ha-huawei-virtual-trigger' triggers: - - timed: '' + - timed: '0 3 * * *' - trigger: name: 'compass-os-odl_l2-nofeature-ha-huawei-virtual-trigger' triggers: - - timed: '0 16 * * *' + - timed: '0 21 * * *' +- trigger: + name: 'compass-os-odl_l3-nofeature-ha-huawei-virtual-trigger' + triggers: + - timed: '' - trigger: name: 'compass-os-onos-nofeature-ha-huawei-virtual-trigger' triggers: - - timed: '0 16 * * *' + - timed: '0 15 * * *' - trigger: name: 'compass-os-ocl-nofeature-ha-huawei-virtual-trigger' triggers: - - timed: '' + - timed: '0 9 * * *' diff --git a/jjb/compass4nfv/compass-deploy.sh b/jjb/compass4nfv/compass-deploy.sh index be15f8817..860f5da1b 100644 --- a/jjb/compass4nfv/compass-deploy.sh +++ b/jjb/compass4nfv/compass-deploy.sh @@ -6,22 +6,38 @@ echo "Starting the deployment on baremetal environment using $INSTALLER_TYPE. Th echo "--------------------------------------------------------" echo +# source the properties file so we get OPNFV vars +source $BUILD_DIRECTORY/latest.properties + +# echo the info about artifact that is used during the deployment +echo "Using ${OPNFV_ARTIFACT_URL/*\/} for deployment" + +if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then + # for none-merge deployments + # checkout the commit that was used for building the downloaded artifact + # to make sure the ISO and deployment mechanism uses same versions + echo "Checking out $OPNFV_GIT_SHA1" + git checkout $OPNFV_GIT_SHA1 --quiet +fi + +echo 1 > /proc/sys/vm/drop_caches + export CONFDIR=$WORKSPACE/deploy/conf export ISO_URL=file://$BUILD_DIRECTORY/compass.iso if [[ "$NODE_NAME" =~ "-vm" ]]; then export NETWORK_CONF=$CONFDIR/vm_environment/$NODE_NAME/network.yml - export DHA_CONF=$CONFDIR/vm_environment/${{DEPLOY_SCENARIO}}.yml + export DHA_CONF=$CONFDIR/vm_environment/${DEPLOY_SCENARIO}.yml else export INSTALL_NIC=eth1 export NETWORK_CONF=$CONFDIR/hardware_environment/$NODE_NAME/network.yml - export DHA_CONF=$CONFDIR/hardware_environment/$NODE_NAME/${{DEPLOY_SCENARIO}}.yml + export DHA_CONF=$CONFDIR/hardware_environment/$NODE_NAME/${DEPLOY_SCENARIO}.yml fi cd $WORKSPACE -export OS_VERSION=${{COMPASS_OS_VERSION}} -export OPENSTACK_VERSION=${{COMPASS_OPENSTACK_VERSION}} -./deploy.sh --dha ${{DHA_CONF}} --network ${{NETWORK_CONF}} +export OS_VERSION=${COMPASS_OS_VERSION} +export OPENSTACK_VERSION=${COMPASS_OPENSTACK_VERSION} +./deploy.sh --dha ${DHA_CONF} --network ${NETWORK_CONF} if [ $? -ne 0 ]; then echo "depolyment failed!" deploy_ret=1 @@ -32,6 +48,6 @@ echo "--------------------------------------------------------" echo "Done!" ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" -sshpass -p root scp 2>/dev/null $ssh_options root@${{INSTALLER_IP}}:/var/ansible/run/openstack_${{COMPASS_OPENSTACK_VERSION}}-opnfv2/ansible.log ./ &> /dev/null +sshpass -p root scp 2>/dev/null $ssh_options root@${INSTALLER_IP}:/var/ansible/run/openstack_${COMPASS_OPENSTACK_VERSION}-opnfv2/ansible.log ./ &> /dev/null exit $deploy_ret diff --git a/jjb/compass4nfv/compass-project-jobs.yml b/jjb/compass4nfv/compass-project-jobs.yml index 3dd091f87..ed748aacb 100644 --- a/jjb/compass4nfv/compass-project-jobs.yml +++ b/jjb/compass4nfv/compass-project-jobs.yml @@ -8,9 +8,11 @@ stream: - master: + slave_name: 'huawei-us-deploy-vm-1' branch: '{stream}' gs-pathname: '' - brahmaputra: + slave_name: 'huawei-us-deploy-vm-2' branch: 'stable/{stream}' gs-pathname: '/{stream}' @@ -30,8 +32,6 @@ disabled: false - node: 'huawei-virtual' - concurrent: true wrappers: @@ -42,8 +42,13 @@ properties: - throttle: enabled: true - max-total: 0 max-per-node: 1 + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'compass-verify-[^-]*' + - 'compass-os-.*?-virtual-daily-.*?' + block-level: 'NODE' parameters: - project-parameter: @@ -53,18 +58,12 @@ - compass-project-parameter: installer: '{installer}' gs-pathname: '{gs-pathname}' - - '{node}-defaults' + - 'huawei-virtual-defaults' - '{installer}-defaults' - string: name: DEPLOY_SCENARIO default: 'os-nosdn-nofeature-ha' - scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' - triggers: - gerrit: trigger-on: @@ -83,9 +82,12 @@ branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: '**/*' forbidden-file-paths: - compare-type: ANT - pattern: 'docs/**|.gitignore' + pattern: 'docs/**' builders: - trigger-builds: @@ -102,20 +104,24 @@ properties: - throttle: enabled: true - max-total: 0 max-per-node: 1 + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'compass-deploy-huawei-virtual-daily-.*?' + block-level: 'NODE' scm: - - git-scm: + - gerrit-trigger-scm: credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' builders: - shell: - !include-raw: ./compass-build.sh + !include-raw-escape: ./compass-build.sh - shell: - !include-raw: ./compass-deploy.sh + !include-raw-escape: ./compass-deploy.sh publishers: - archive: @@ -126,8 +132,6 @@ - job-template: name: 'compass-build-iso-{stream}' - node: ericsson-build - concurrent: true properties: @@ -142,7 +146,7 @@ - compass-project-parameter: installer: '{installer}' gs-pathname: '{gs-pathname}' - - '{node}-defaults' + - 'ericsson-build-defaults' - '{installer}-defaults' scm: @@ -156,11 +160,11 @@ builders: - shell: - !include-raw: ./compass-build.sh + !include-raw-escape: ./compass-build.sh - shell: - !include-raw: ./compass-upload-artifact.sh + !include-raw-escape: ./compass-upload-artifact.sh - shell: - !include-raw: ./compass-workspace-cleanup.sh + !include-raw-escape: ./compass-workspace-cleanup.sh - job-template: name: 'compass-build-ppa-{stream}' @@ -193,7 +197,7 @@ builders: - shell: - !include-raw: ./compass-makeppa.sh + !include-raw-escape: ./compass-makeppa.sh diff --git a/jjb/compass4nfv/compass-upload-artifact.sh b/jjb/compass4nfv/compass-upload-artifact.sh index 1639fd562..34b1db98c 100644 --- a/jjb/compass4nfv/compass-upload-artifact.sh +++ b/jjb/compass4nfv/compass-upload-artifact.sh @@ -1,5 +1,4 @@ #!/bin/bash -set -o errexit set -o nounset set -o pipefail @@ -16,7 +15,27 @@ gsutil cp $BUILD_DIRECTORY/compass.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSIO gsutil cp $BUILD_DIRECTORY/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 gsutil cp $BUILD_DIRECTORY/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/latest.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > /dev/null 2>&1 + +gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > /dev/null 2>&1 + +# disabled errexit due to gsutil setmeta complaints +# BadRequestException: 400 Invalid argument +# check if we uploaded the file successfully to see if things are fine +gsutil ls gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading artifact!" + echo "Check log $WORKSPACE/gsutil.iso.log on the machine where this build is done." + exit 1 +fi + echo echo "--------------------------------------------------------" echo "Done!" -echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
\ No newline at end of file +echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index 167b696a3..b2cbb0242 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -53,4 +53,4 @@ pattern: 'tests/**' builders: - - shell: "[ -e tests/run.sh ] && ./tests/run.sh" + - shell: "[ -e tests/run.sh ] && bash -n ./tests/run.sh" diff --git a/jjb/fuel/fuel-build.sh b/jjb/fuel/fuel-build.sh index d51637afd..eaf1f83e7 100755 --- a/jjb/fuel/fuel-build.sh +++ b/jjb/fuel/fuel-build.sh @@ -1,4 +1,12 @@ #!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## set -o errexit set -o nounset set -o pipefail @@ -23,14 +31,17 @@ fi # get current SHA1 CURRENT_SHA1=$(git rev-parse HEAD) -if [[ "$CURRENT_SHA1" == "$LATEST_ISO_SHA1" ]]; then +# set FORCE_BUILD to false for non-daily builds +FORCE_BUILD=${FORCE_BUILD:-false} + +if [[ "$CURRENT_SHA1" == "$LATEST_ISO_SHA1" && "$FORCE_BUILD" == "false" ]]; then echo "An ISO has already been built for this commit" echo " $LATEST_ISO_URL" echo "Nothing new to build. Exiting." touch $WORKSPACE/.noupload exit 0 else - echo "This commit has not been built yet. Proceeding with the build." + echo "This commit has not been built yet or forced build! Proceeding with the build." /bin/rm -f $LATEST_ISO_PROPERTIES echo fi diff --git a/jjb/fuel/fuel-ci-jobs.yml b/jjb/fuel/fuel-ci-jobs.yml index d293554c2..2c1a4ab54 100644 --- a/jjb/fuel/fuel-ci-jobs.yml +++ b/jjb/fuel/fuel-ci-jobs.yml @@ -28,11 +28,15 @@ <<: *brahmaputra - ericsson-pod1: <<: *brahmaputra + - ericsson-pod2: + <<: *brahmaputra - virtual: <<: *brahmaputra #-------------------------------- # master #-------------------------------- + - opnfv-jump-2: + <<: *master - ericsson-pod1: <<: *master - ericsson-pod2: @@ -44,21 +48,23 @@ #-------------------------------- scenario: # HA scenarios - - 'os-onos-nofeature-ha': - auto-trigger-name: 'fuel-{scenario}-{pod}-trigger' - 'os-odl_l2-nofeature-ha': - auto-trigger-name: 'fuel-{scenario}-{pod}-trigger' + auto-trigger-name: 'fuel-{scenario}-{pod}-{stream}-trigger' - 'os-odl_l3-nofeature-ha': - auto-trigger-name: 'fuel-{scenario}-{pod}-trigger' - - 'os-odl_l2-bgpvpn-ha': - auto-trigger-name: 'fuel-{scenario}-{pod}-trigger' + auto-trigger-name: 'fuel-{scenario}-{pod}-{stream}-trigger' + - 'os-onos-nofeature-ha': + auto-trigger-name: 'fuel-{scenario}-{pod}-{stream}-trigger' + - 'os-nosdn-kvm-ha': + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' + - 'os-nosdn-ovs-ha': + auto-trigger-name: 'fuel-{scenario}-{pod}-{stream}-trigger' - 'os-nosdn-nofeature-ha': auto-trigger-name: 'brahmaputra-trigger-daily-disabled' - - 'os-nosdn-kvm-ha': + - 'os-odl_l2-bgpvpn-ha': auto-trigger-name: 'brahmaputra-trigger-daily-disabled' - - 'os-nosdn-kvm_ovs-ha': + - 'os-odl_l2-sfc-ha': auto-trigger-name: 'brahmaputra-trigger-daily-disabled' - - 'os-nosdn-ovs-ha': + - 'os-nosdn-kvm_ovs-ha': auto-trigger-name: 'brahmaputra-trigger-daily-disabled' - 'os-nosdn-vlan-ha': auto-trigger-name: 'brahmaputra-trigger-daily-disabled' @@ -79,6 +85,8 @@ auto-trigger-name: 'brahmaputra-trigger-daily-disabled' - 'os-odl_l2-bgpvpn-noha': auto-trigger-name: 'brahmaputra-trigger-daily-disabled' + - 'os-odl_l2-sfc-noha': + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' jobs: - 'fuel-{scenario}-{pod}-daily-{stream}' @@ -101,6 +109,7 @@ use-build-blocker: true blocking-jobs: - 'fuel-os-.*?-{pod}-daily-{stream}' + block-level: 'NODE' wrappers: - build-name: @@ -130,7 +139,7 @@ same-node: true block: true - trigger-builds: - - project: 'functest-fuel-{pod}-daily-{stream}' + - project: 'yardstick-fuel-{pod}-daily-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} @@ -141,7 +150,7 @@ failure-threshold: 'never' unstable-threshold: 'FAILURE' - trigger-builds: - - project: 'yardstick-fuel-{pod}-daily-{stream}' + - project: 'functest-fuel-{pod}-daily-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} @@ -166,6 +175,8 @@ use-build-blocker: true blocking-jobs: - 'fuel-deploy-{pod}-daily-{stream}' + - 'fuel-deploy-generic-daily-.*' + block-level: 'NODE' parameters: - project-parameter: @@ -220,74 +231,309 @@ ######################## # trigger macros ######################## -# os-onos3-nofeature-ha trigger +# os-nosdn-nofeature-ha trigger +- trigger: + name: 'fuel-os-nosdn-nofeature-ha-opnfv-jump-2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-nofeature-ha-ericsson-pod1-master-trigger' + triggers: + - timed: '' - trigger: - name: 'fuel-os-onos-nofeature-ha-opnfv-jump-2-trigger' + name: 'fuel-os-nosdn-nofeature-ha-ericsson-pod2-master-trigger' triggers: - - timed: '0 20 * * *' + - timed: '' - trigger: - name: 'fuel-os-onos-nofeature-ha-ericsson-pod1-trigger' + name: 'fuel-os-nosdn-nofeature-ha-virtual-master-trigger' triggers: - timed: '' - trigger: - name: 'fuel-os-onos-nofeature-ha-ericsson-pod2-trigger' + name: 'fuel-os-nosdn-nofeature-ha-opnfv-jump-2-brahmaputra-trigger' triggers: - timed: '' - trigger: - name: 'fuel-os-onos-nofeature-ha-virtual-trigger' + name: 'fuel-os-nosdn-nofeature-ha-ericsson-pod1-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-nofeature-ha-ericsson-pod2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-nofeature-ha-virtual-brahmaputra-trigger' + triggers: + - timed: '' + +# os-nosdn-kvm-ha trigger +- trigger: + name: 'fuel-os-nosdn-kvm-ha-opnfv-jump-2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-kvm-ha-ericsson-pod1-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-kvm-ha-ericsson-pod2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-kvm-ha-virtual-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-kvm-ha-opnfv-jump-2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-kvm-ha-ericsson-pod1-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-kvm-ha-ericsson-pod2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-kvm-ha-virtual-brahmaputra-trigger' triggers: - timed: '' # os-odl_l2-nofeature-ha trigger - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-opnfv-jump-2-trigger' + name: 'fuel-os-odl_l2-nofeature-ha-opnfv-jump-2-master-trigger' triggers: - - timed: '0 0 * * *' + - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-ericsson-pod1-trigger' + name: 'fuel-os-odl_l2-nofeature-ha-ericsson-pod1-master-trigger' triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-ericsson-pod2-trigger' + name: 'fuel-os-odl_l2-nofeature-ha-ericsson-pod2-master-trigger' triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-nofeature-ha-virtual-trigger' + name: 'fuel-os-odl_l2-nofeature-ha-virtual-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-nofeature-ha-opnfv-jump-2-brahmaputra-trigger' + triggers: + - timed: '0 4,10,16,22 * * *' +- trigger: + name: 'fuel-os-odl_l2-nofeature-ha-ericsson-pod1-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-nofeature-ha-ericsson-pod2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-nofeature-ha-virtual-brahmaputra-trigger' triggers: - timed: '' # os-odl_l3-nofeature-ha trigger - trigger: - name: 'fuel-os-odl_l3-nofeature-ha-opnfv-jump-2-trigger' + name: 'fuel-os-odl_l3-nofeature-ha-opnfv-jump-2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l3-nofeature-ha-ericsson-pod1-master-trigger' triggers: - - timed: '0 4 * * *' + - timed: '' - trigger: - name: 'fuel-os-odl_l3-nofeature-ha-ericsson-pod1-trigger' + name: 'fuel-os-odl_l3-nofeature-ha-ericsson-pod2-master-trigger' triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l3-nofeature-ha-ericsson-pod2-trigger' + name: 'fuel-os-odl_l3-nofeature-ha-virtual-master-trigger' triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l3-nofeature-ha-virtual-trigger' + name: 'fuel-os-odl_l3-nofeature-ha-opnfv-jump-2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l3-nofeature-ha-ericsson-pod1-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l3-nofeature-ha-ericsson-pod2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l3-nofeature-ha-virtual-brahmaputra-trigger' + triggers: + - timed: '' + +# os-onos-nofeature-ha trigger +- trigger: + name: 'fuel-os-onos-nofeature-ha-opnfv-jump-2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-onos-nofeature-ha-ericsson-pod1-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-onos-nofeature-ha-ericsson-pod2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-onos-nofeature-ha-virtual-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-onos-nofeature-ha-opnfv-jump-2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-onos-nofeature-ha-ericsson-pod1-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-onos-nofeature-ha-ericsson-pod2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-onos-nofeature-ha-virtual-brahmaputra-trigger' + triggers: + - timed: '' + + +# os-nosdn-ovs-ha trigger +- trigger: + name: 'fuel-os-nosdn-ovs-ha-opnfv-jump-2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-ericsson-pod1-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-ericsson-pod2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-virtual-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-opnfv-jump-2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-ericsson-pod1-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-ericsson-pod2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-ovs-ha-virtual-brahmaputra-trigger' triggers: - timed: '' # os-odl_l2-bgpvpn-ha trigger - trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-opnfv-jump-2-trigger' + name: 'fuel-os-odl_l2-bgpvpn-ha-opnfv-jump-2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-bgpvpn-ha-ericsson-pod1-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-bgpvpn-ha-ericsson-pod2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-bgpvpn-ha-virtual-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-bgpvpn-ha-opnfv-jump-2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-bgpvpn-ha-ericsson-pod1-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-bgpvpn-ha-ericsson-pod2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-bgpvpn-ha-virtual-brahmaputra-trigger' triggers: - - timed: '0 8 * * *' + - timed: '' + +# os-odl_l2-sfc-ha trigger +- trigger: + name: 'fuel-os-odl_l2-sfc-ha-opnfv-jump-2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-sfc-ha-ericsson-pod1-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-sfc-ha-ericsson-pod2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-sfc-ha-virtual-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-sfc-ha-opnfv-jump-2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-sfc-ha-ericsson-pod1-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-sfc-ha-ericsson-pod2-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-odl_l2-sfc-ha-virtual-brahmaputra-trigger' + triggers: + - timed: '' + +# os-nosdn-kvm-ha trigger +- trigger: + name: 'fuel-os-nosdn-kvm-noha-opnfv-jump-2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-kvm-noha-ericsson-pod1-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-kvm-noha-ericsson-pod2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-kvm-noha-virtual-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'fuel-os-nosdn-kvm-noha-opnfv-jump-2-brahmaputra-trigger' + triggers: + - timed: '' - trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-ericsson-pod1-trigger' + name: 'fuel-os-nosdn-kvm-noha-ericsson-pod1-brahmaputra-trigger' triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-ericsson-pod2-trigger' + name: 'fuel-os-nosdn-kvm-noha-ericsson-pod2-brahmaputra-trigger' triggers: - timed: '' - trigger: - name: 'fuel-os-odl_l2-bgpvpn-ha-virtual-trigger' + name: 'fuel-os-nosdn-kvm-noha-virtual-brahmaputra-trigger' triggers: - timed: '' diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh index f3d2aca45..bab6151d8 100755 --- a/jjb/fuel/fuel-deploy.sh +++ b/jjb/fuel/fuel-deploy.sh @@ -1,4 +1,12 @@ #!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## set -o errexit set -o nounset set -o pipefail @@ -80,4 +88,57 @@ $DEPLOY_COMMAND echo echo "--------------------------------------------------------" -echo "Done!" +echo "Deployment is done successfully!" + +# Quick and dirty fix for SFC scenatio - will be fixed properly post-release +if [[ ! "$DEPLOY_SCENARIO" =~ "os-odl_l2-sfc" ]]; then + exit 0 +fi + +echo +echo "SFC Scenario is deployed" +echo + +# The stuff below is here temporarily and will be fixed once the release is out +# The stuff below is here temporarily and will be fixed once the release is out +export FUEL_MASTER_IP=10.20.0.2 +export TACKER_SCRIPT_URL="https://git.opnfv.org/cgit/fuel/plain/prototypes/sfc_tacker/poc.tacker-up.sh?h=${GIT_BRANCH#*/}" +export CONTROLLER_NODE_IP=$(sshpass -pr00tme /usr/bin/ssh -o UserKnownHostsFile=/dev/null \ + -o StrictHostKeyChecking=no root@$FUEL_MASTER_IP 'fuel node list' | \ + grep opendaylight | cut -d'|' -f5) + +# we can't do much if we do not have the controller IP +if [[ ! "$CONTROLLER_NODE_IP" =~ "10.20.0" ]]; then + echo "Unable to retrieve controller IP" + exit 1 +fi + +echo +echo "Copying and executing poc.tacker-up.sh script on controller node $CONTROLLER_NODE_IP" +echo + +expect << END +spawn /usr/bin/ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -l root $::env(FUEL_MASTER_IP) +expect { + -re ".*sword.*" { + exp_send "r00tme\r" + } +} +expect "# " +send "/usr/bin/ssh -l root $::env(CONTROLLER_NODE_IP)\r" +expect "# " +send "sudo apt-get install -y git\r" +expect "# " +send "/bin/mkdir -p /root/sfc-poc && cd /root/sfc-poc\r" +expect "# " +send "git clone https://gerrit.opnfv.org/gerrit/fuel && cd fuel\r" +expect "# " +send "git fetch https://gerrit.opnfv.org/gerrit/fuel refs/changes/97/10597/2 && git checkout FETCH_HEAD\r" +expect "# " +send "/bin/bash /root/sfc-poc/fuel/prototypes/sfc_tacker/poc.tacker-up.sh\r" +expect "# " +send "exit\r" +expect "Connection to $::env(CONTROLLER_NODE_IP) closed. " +send "exit\r" +expect "Connection to $::env(FUEL_MASTER_IP) closed. " +END diff --git a/jjb/fuel/fuel-download-artifact.sh b/jjb/fuel/fuel-download-artifact.sh index a1a1de47e..d78ddc5ba 100755 --- a/jjb/fuel/fuel-download-artifact.sh +++ b/jjb/fuel/fuel-download-artifact.sh @@ -1,6 +1,13 @@ #!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## set -o errexit -set -o nounset set -o pipefail if [[ "$JOB_NAME" =~ "merge" ]]; then @@ -23,15 +30,22 @@ source latest.properties OPNFV_ARTIFACT=${OPNFV_ARTIFACT_URL/*\/} echo "Using $OPNFV_ARTIFACT for deployment" -# check if we already have the ISO to avoid redownload -# disabled for the timebeing - needs adjustments -#ISO_STORE=$HOME/opnfv/iso_store/fuel -#if [[ -f "$ISO_STORE/$OPNFV_ARTIFACT" ]]; then -# echo "ISO already exists. Skipping the download" -# ln -s $ISO_STORE/$OPNFV_ARTIFACT $WORKSPACE/opnfv.iso -# ls -al $WORKSPACE/opnfv.iso -# exit 0 -#fi +# using ISOs for verify & merge jobs from local storage will be enabled later +if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then + # check if we already have the ISO to avoid redownload + ISOSTORE="/iso_mount/opnfv_ci/${GIT_BRANCH##*/}" + if [[ -f "$ISOSTORE/$OPNFV_ARTIFACT" ]]; then + echo "ISO exists locally. Skipping the download and using the file from ISO store" + ln -s $ISOSTORE/$OPNFV_ARTIFACT $WORKSPACE/opnfv.iso + echo "--------------------------------------------------------" + echo + ls -al $WORKSPACE/opnfv.iso + echo + echo "--------------------------------------------------------" + echo "Done!" + exit 0 + fi +fi # log info to console echo "Downloading the $INSTALLER_TYPE artifact using URL http://$OPNFV_ARTIFACT_URL" diff --git a/jjb/fuel/fuel-lab-reconfig.sh b/jjb/fuel/fuel-lab-reconfig.sh index 55bb5dd9d..59f660066 100755 --- a/jjb/fuel/fuel-lab-reconfig.sh +++ b/jjb/fuel/fuel-lab-reconfig.sh @@ -1,4 +1,12 @@ #!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## set -o errexit set -o nounset set -o pipefail diff --git a/jjb/fuel/fuel-project-jobs.yml b/jjb/fuel/fuel-project-jobs.yml index 2e30fb1e6..d1855f86e 100644 --- a/jjb/fuel/fuel-project-jobs.yml +++ b/jjb/fuel/fuel-project-jobs.yml @@ -12,9 +12,11 @@ - master: branch: '{stream}' gs-pathname: '' + disabled: false - brahmaputra: branch: 'stable/{stream}' gs-pathname: '/{stream}' + disabled: false jobs: - 'fuel-build-daily-{stream}' @@ -40,8 +42,14 @@ parameters: - project-parameter: project: '{project}' - - 'ericsson-ca-build-1-defaults' + - 'ericsson-build-defaults' - '{installer}-defaults' + - choice: + name: FORCE_BUILD + choices: + - 'false' + - 'true' + description: "Force build even if there is no changes in fuel repo. Default false" - fuel-project-parameter: gs-pathname: '{gs-pathname}' @@ -75,19 +83,16 @@ - job-template: name: 'fuel-verify-build-{stream}' - concurrent: true + disabled: '{obj:disabled}' - properties: - - throttle: - enabled: true - max-total: 3 + concurrent: true parameters: - project-parameter: project: '{project}' - gerrit-parameter: branch: '{branch}' - - 'ericsson-ca-build-1-defaults' + - 'ericsson-build-defaults' - '{installer}-defaults' - fuel-project-parameter: gs-pathname: '{gs-pathname}' @@ -144,22 +149,20 @@ - job-template: name: 'fuel-merge-build-{stream}' - concurrent: true + disabled: '{obj:disabled}' - properties: - - throttle: - enabled: true - max-total: 2 + concurrent: true parameters: - project-parameter: project: '{project}' - gerrit-parameter: branch: '{branch}' - - 'ericsson-ca-build-1-defaults' + - 'ericsson-build-defaults' - '{installer}-defaults' - fuel-project-parameter: gs-pathname: '{gs-pathname}' + scm: - gerrit-trigger-scm: credentials-id: '{ssh-credentials}' @@ -207,9 +210,9 @@ - job-template: name: 'fuel-merge-deploy-virtual-{stream}' - concurrent: true + disabled: true - disabled: false + concurrent: true properties: - throttle: @@ -282,28 +285,41 @@ - job-template: name: 'fuel-deploy-generic-daily-{stream}' - concurrent: false + concurrent: true - disabled: true + disabled: false properties: - throttle: enabled: true - max-total: 1 max-per-node: 1 - build-blocker: use-build-blocker: true blocking-jobs: - - 'fuel-os-.*?-ericsson-pod2-daily-{stream}' + - 'fuel-os-.*?-daily-.*' + - 'fuel-deploy-generic-daily-.*' + block-level: 'NODE' parameters: - project-parameter: project: '{project}' - - 'ericsson-pod2-defaults' - '{installer}-defaults' - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + - string: name: DEPLOY_SCENARIO default: 'os-odl_l2-nofeature-ha' + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - ericsson-pod2 + - opnfv-jump-2 + - ericsson-pod1 + default-slaves: + - ericsson-pod2 - fuel-project-parameter: gs-pathname: '{gs-pathname}' diff --git a/jjb/fuel/fuel-upload-artifact.sh b/jjb/fuel/fuel-upload-artifact.sh index 8b69f5a78..47cf8985c 100755 --- a/jjb/fuel/fuel-upload-artifact.sh +++ b/jjb/fuel/fuel-upload-artifact.sh @@ -1,6 +1,12 @@ #!/bin/bash -set -o errexit -set -o nounset +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## set -o pipefail # check if we built something @@ -10,14 +16,30 @@ if [ -f $WORKSPACE/.noupload ]; then exit 0 fi -# log info to console -echo -echo "Uploading the $INSTALLER_TYPE artifact. This could take some time..." -echo - # source the opnfv.properties to get ARTIFACT_VERSION source $WORKSPACE/opnfv.properties +# storing ISOs for verify & merge jobs will be done once we get the disk array +if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then + # store ISO locally on NFS first + ISOSTORE="/iso_mount/opnfv_ci/${GIT_BRANCH##*/}" + if [[ -d "$ISOSTORE" ]]; then + # remove all but most recent 5 ISOs first to keep iso_mount clean & tidy + cd $ISOSTORE + ls -tp | grep -v '/' | tail -n +6 | xargs -d '\n' /bin/rm -f -- + + # store ISO + echo "Storing $INSTALLER_TYPE artifact on NFS..." + /bin/cp -f $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso \ + $ISOSTORE/opnfv-$OPNFV_ARTIFACT_VERSION.iso + fi +fi + +# log info to console +echo "Uploading $INSTALLER_TYPE artifact. This could take some time..." +echo + +cd $WORKSPACE # upload artifact and additional files to google storage gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.iso \ gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1 @@ -33,11 +55,22 @@ fi gsutil -m setmeta \ -h "Content-Type:text/html" \ -h "Cache-Control:private, max-age=0, no-transform" \ - gs://$GS_URL/*.properties > /dev/null 2>&1 + gs://$GS_URL/latest.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > /dev/null 2>&1 gsutil -m setmeta \ -h "Cache-Control:private, max-age=0, no-transform" \ - gs://$GS_URL/*.iso > /dev/null 2>&1 + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > /dev/null 2>&1 + +# disabled errexit due to gsutil setmeta complaints +# BadRequestException: 400 Invalid argument +# check if we uploaded the file successfully to see if things are fine +gsutil ls gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading artifact!" + echo "Check log $WORKSPACE/gsutil.iso.log on the machine where this build is done." + exit 1 +fi echo "Done!" echo diff --git a/jjb/fuel/fuel-workspace-cleanup.sh b/jjb/fuel/fuel-workspace-cleanup.sh index 4e1a3a727..d8948c7a0 100755 --- a/jjb/fuel/fuel-workspace-cleanup.sh +++ b/jjb/fuel/fuel-workspace-cleanup.sh @@ -1,4 +1,12 @@ #!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## set -o errexit set -o nounset set -o pipefail diff --git a/jjb/functest/functest-ci-jobs.yml b/jjb/functest/functest-ci-jobs.yml index 2b91ed692..01e1885cb 100644 --- a/jjb/functest/functest-ci-jobs.yml +++ b/jjb/functest/functest-ci-jobs.yml @@ -13,28 +13,39 @@ stream: master branch: '{stream}' gs-pathname: '' + docker-tag: 'latest' brahmaputra: &brahmaputra stream: brahmaputra branch: 'stable/{stream}' gs-pathname: '/{stream}' + docker-tag: 'brahmaputra.1.0' #-------------------------------- # POD, INSTALLER, AND BRANCH MAPPING #-------------------------------- # brahmaputra #-------------------------------- pod: - - opnfv-jump-2: - installer: fuel + - opnfv-jump-1: + installer: apex <<: *brahmaputra - intel-pod5: installer: joid <<: *brahmaputra + - orange-pod2: + installer: joid + <<: *brahmaputra - huawei-us-deploy-bare-1: installer: compass <<: *brahmaputra - ericsson-pod1: installer: fuel <<: *brahmaputra + - ericsson-pod2: + installer: fuel + <<: *brahmaputra + - opnfv-jump-2: + installer: fuel + <<: *brahmaputra - virtual: installer: fuel <<: *brahmaputra @@ -47,6 +58,9 @@ #-------------------------------- # master #-------------------------------- + - opnfv-jump-2: + installer: fuel + <<: *master - intel-pod6: installer: joid <<: *master @@ -59,9 +73,6 @@ - ericsson-pod1: installer: fuel <<: *master - - ericsson-pod2: - installer: fuel - <<: *master - orange-test1: installer: fuel <<: *master @@ -92,9 +103,19 @@ - job-template: name: 'functest-{installer}-{pod}-{testsuite}-{stream}' + concurrent: true + + properties: + - throttle: + enabled: true + max-per-node: 1 + wrappers: - build-name: name: '$BUILD_NUMBER Suite: $FUNCTEST_SUITE_NAME Scenario: $DEPLOY_SCENARIO' + - timeout: + timeout: 400 + abort: true parameters: - project-parameter: @@ -105,6 +126,10 @@ - string: name: DEPLOY_SCENARIO default: 'os-odl_l2-nofeature-ha' + - string: + name: DOCKER_TAG + default: '{docker-tag}' + description: 'Tag to pull docker image' - functest-parameter: gs-pathname: '{gs-pathname}' @@ -205,7 +230,6 @@ if [ "${PUSH_RESULTS_TO_DB}" == "true" ]; then flag="-r" fi - echo "Functest: running all the tests" cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh ${flag}" container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) docker exec $container_id $cmd @@ -237,19 +261,19 @@ envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} \ -e NODE_NAME=${NODE_NAME} -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} \ -e CI_DEBUG=${CI_DEBUG}" - - dir_result="${HOME}/opnfv/functest/reports" + branch=${GIT_BRANCH##*/} + dir_result="${HOME}/opnfv/functest/results/${branch}" mkdir -p ${dir_result} sudo rm -rf ${dir_result}/* res_volume="-v ${dir_result}:/home/opnfv/functest/results" - docker pull opnfv/functest:latest_stable >$redirect - cmd="sudo docker run --privileged=true -id ${envs} ${labconfig} ${sshkey} ${res_volume} opnfv/functest:latest_stable /bin/bash" + docker pull opnfv/functest:$DOCKER_TAG >$redirect + cmd="sudo docker run --privileged=true -id ${envs} ${labconfig} ${sshkey} ${res_volume} opnfv/functest:latest /bin/bash" echo "Functest: Running docker run command: ${cmd}" ${cmd} docker ps -a sleep 5 - container_id=$(docker ps | grep 'opnfv/functest:latest_stable' | awk '{print $1}' | head -1) + container_id=$(docker ps | grep 'opnfv/functest:latest' | awk '{print $1}' | head -1) echo "Container ID=${container_id}" if [ -z ${container_id} ]; then echo "Cannot find opnfv/functest container ID ${container_id}. Please check if it is existing." @@ -260,7 +284,7 @@ docker start ${container_id} sleep 5 docker ps - if [ $(docker ps | grep 'opnfv/functest:latest_stable' | wc -l) == 0 ]; then + if [ $(docker ps | grep 'opnfv/functest:latest' | wc -l) == 0 ]; then echo "The container opnfv/functest with ID=${container_id} has not been properly started. Exiting..." exit 1 fi @@ -272,7 +296,7 @@ name: functest-store-results builders: - shell: - !include-raw: res-build.sh + !include-raw: ../../utils/push-test-logs.sh - builder: name: functest-cleanup diff --git a/jjb/functest/res-build.sh b/jjb/functest/res-build.sh deleted file mode 100644 index 1f4699681..000000000 --- a/jjb/functest/res-build.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -export PATH=$PATH:/usr/local/bin/ - -git_sha1="$(git rev-parse HEAD)" -res_build_date=$(date -u +"%Y-%m-%d_%H-%M-%S") - -# Result directory in the jumphost -# to be used only with CI -dir_result="${HOME}/opnfv/functest/results" - -# Clean the results directory -# remove json file for rally, export only html -# json should have been pushed into the DB -rm -f $dir_result/rally/*.json - -# Several information are required: date and testbed -# date is generated by functest so on the artifact, the results shall be under functest/<testbed id>/date/ -testbed=$NODE_NAME - -project_artifact=logs/functest/$testbed/$res_build_date - -# copy folder to artifact -if [ -d "$dir_result" ]; then - if [ "$(ls -A $dir_result)" ]; then - echo "copy result files to artifact $project_artifact" - gsutil -m cp -r "$dir_result" gs://artifacts.opnfv.org/"$project_artifact"/ - - # delete local results - # should not be useful as the container is about to die...just in case - rm -Rf /home/opnfv/functest/results/* - else - echo "Result folder is empty" - fi -else - echo "No result folder found" -fi diff --git a/jjb/joid/joid-ci-jobs.yml b/jjb/joid/joid-ci-jobs.yml index 849112319..adf3022b2 100644 --- a/jjb/joid/joid-ci-jobs.yml +++ b/jjb/joid/joid-ci-jobs.yml @@ -28,6 +28,8 @@ pod: - intel-pod5: <<: *brahmaputra + - orange-pod2: + <<: *brahmaputra - virtual: <<: *brahmaputra #-------------------------------- @@ -47,12 +49,12 @@ - 'os-nosdn-nofeature-noha': auto-trigger-name: 'brahmaputra-trigger-daily-disabled' - 'os-nosdn-nofeature-ha': - auto-trigger-name: 'brahmaputra-trigger-daily-disabled' + auto-trigger-name: 'joid-{scenario}-{pod}-{stream}-trigger' - 'os-odl_l2-nofeature-ha': - auto-trigger-name: 'joid-{scenario}-{pod}-trigger' - - 'os-odl_l2-nofeature-noha': - auto-trigger-name: 'brahmaputra-trigger-daily-disabled' + auto-trigger-name: 'joid-{scenario}-{pod}-{stream}-trigger' - 'os-onos-nofeature-ha': + auto-trigger-name: 'joid-{scenario}-{pod}-{stream}-trigger' + - 'os-odl_l2-nofeature-noha': auto-trigger-name: 'brahmaputra-trigger-daily-disabled' - 'os-onos-nofeature-noha': auto-trigger-name: 'brahmaputra-trigger-daily-disabled' @@ -103,7 +105,7 @@ builders: - trigger-builds: - project: 'joid-deploy-{pod}-daily-{stream}' - current-parameters: false + current-parameters: true predefined-parameters: DEPLOY_SCENARIO={scenario} same-node: true @@ -190,23 +192,134 @@ ######################## # trigger macros ######################## +# os-nosdn-nofeature-ha trigger - branch: master - trigger: - name: 'joid-os-odl_l2-nofeature-ha-intel-pod5-trigger' + name: 'joid-os-nosdn-nofeature-ha-intel-pod5-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-nosdn-nofeature-ha-intel-pod6-master-trigger' triggers: - timed: '0 2 * * *' - trigger: - name: 'joid-os-odl_l2-nofeature-ha-intel-pod6-trigger' + name: 'joid-os-nosdn-nofeature-ha-orange-pod2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-nosdn-nofeature-ha-juniper-pod1-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-nosdn-nofeature-ha-virtual-master-trigger' + triggers: + - timed: '' +# os-nosdn-nofeature-ha trigger - branch: stable/brahmaputra +- trigger: + name: 'joid-os-nosdn-nofeature-ha-intel-pod5-brahmaputra-trigger' triggers: - timed: '0 2 * * *' - trigger: - name: 'joid-os-odl_l2-nofeature-ha-orange-pod2-trigger' + name: 'joid-os-nosdn-nofeature-ha-intel-pod6-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-nosdn-nofeature-ha-orange-pod2-brahmaputra-trigger' + triggers: + - timed: '0 2 * * *' +- trigger: + name: 'joid-os-nosdn-nofeature-ha-juniper-pod1-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-nosdn-nofeature-ha-virtual-brahmaputra-trigger' + triggers: + - timed: '' + + +# os-odl_l2-nofeature-ha trigger - branch: master +- trigger: + name: 'joid-os-odl_l2-nofeature-ha-intel-pod5-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-odl_l2-nofeature-ha-intel-pod6-master-trigger' + triggers: + - timed: '0 10 * * *' +- trigger: + name: 'joid-os-odl_l2-nofeature-ha-orange-pod2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-odl_l2-nofeature-ha-juniper-pod1-master-trigger' triggers: - timed: '' - trigger: - name: 'joid-os-odl_l2-nofeature-ha-juniper-pod1-trigger' + name: 'joid-os-odl_l2-nofeature-ha-virtual-master-trigger' + triggers: + - timed: '' + +# os-odl_l2-nofeature-ha trigger - branch: stable/brahmaputra +- trigger: + name: 'joid-os-odl_l2-nofeature-ha-intel-pod5-brahmaputra-trigger' + triggers: + - timed: '0 10 * * *' +- trigger: + name: 'joid-os-odl_l2-nofeature-ha-intel-pod6-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-odl_l2-nofeature-ha-orange-pod2-brahmaputra-trigger' + triggers: + - timed: '0 10 * * *' +- trigger: + name: 'joid-os-odl_l2-nofeature-ha-juniper-pod1-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-odl_l2-nofeature-ha-virtual-brahmaputra-trigger' + triggers: + - timed: '' + +# os-onos-nofeature-ha trigger - branch: master +- trigger: + name: 'joid-os-onos-nofeature-ha-intel-pod5-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-onos-nofeature-ha-intel-pod6-master-trigger' + triggers: + - timed: '0 18 * * *' +- trigger: + name: 'joid-os-onos-nofeature-ha-orange-pod2-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-onos-nofeature-ha-juniper-pod1-master-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-onos-nofeature-ha-virtual-master-trigger' + triggers: + - timed: '' + +# os-onos-nofeature-ha trigger - branch: stable/brahmaputra +- trigger: + name: 'joid-os-onos-nofeature-ha-intel-pod5-brahmaputra-trigger' + triggers: + - timed: '0 18 * * *' +- trigger: + name: 'joid-os-onos-nofeature-ha-intel-pod6-brahmaputra-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-os-onos-nofeature-ha-orange-pod2-brahmaputra-trigger' + triggers: + - timed: '0 18 * * *' +- trigger: + name: 'joid-os-onos-nofeature-ha-juniper-pod1-brahmaputra-trigger' triggers: - timed: '' - trigger: - name: 'joid-os-odl_l2-nofeature-ha-virtual-trigger' + name: 'joid-os-onos-nofeature-ha-virtual-brahmaputra-trigger' triggers: - timed: '' diff --git a/jjb/joid/joid-deploy.sh b/jjb/joid/joid-deploy.sh index 747332cab..59ba01588 100644 --- a/jjb/joid/joid-deploy.sh +++ b/jjb/joid/joid-deploy.sh @@ -1,4 +1,12 @@ #!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Orange and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## set +e set -o nounset @@ -119,23 +127,16 @@ SRCBUNDLE="${SRCBUNDLE}/ovs-${SDN_CONTROLLER}-${HA_MODE}.yaml" # Modify Bundle echo "------ Set openstack password ------" -sed -i -- "s/\"admin-password\": openstack/\"admin-password\": $OS_ADMIN_PASSWORD/" $SRCBUNDLE +sed -i -- "s/admin-password: openstack/admin-password: $OS_ADMIN_PASSWORD/" $SRCBUNDLE if [ -n "$EXTNET_NAME" ]; then echo "------ Set openstack default network ------" - sed -i -- "s/\"neutron-external-network\": ext_net/\"neutron-external-network\": $EXTNET_NAME/" $SRCBUNDLE + sed -i -- "s/neutron-external-network: ext_net/neutron-external-network: $EXTNET_NAME/" $SRCBUNDLE fi echo "------ Set ceph disks ------" -CEPH_DISKS_CONTROLLERS=${CEPH_DISKS_CONTROLLERS:-} -if [ -z "$CEPH_DISKS_CONTROLLERS" ]; then - CEPH_DISKS_CONTROLLERS=$CEPH_DISKS -fi - #Find the first line of osd-devices to change the one for ceph, then the other for ceph-osd -CEPH_DEV_LINE=$(grep -nr osd-devices $SRCBUNDLE |head -n1|cut -d: -f1) -sed -i -- "${CEPH_DEV_LINE}s@osd-devices: /srv@osd-devices: $CEPH_DISKS@" $SRCBUNDLE -sed -i -- "s@osd-devices: /srv@osd-devices: $CEPH_DISKS_CONTROLLERS@" $SRCBUNDLE +sed -i -- "s@osd-devices: /srv@osd-devices: $CEPH_DISKS@" $SRCBUNDLE sed -i -r -- "s/^(\s+osd-reformat: )'no'/\1'$CEPH_REFORMAT'/" $SRCBUNDLE ## @@ -153,9 +154,31 @@ exit_on_error $? "Main deploy FAILED" ## JOID_ADMIN_OPENRC=$LAB_CONFIG/admin-openrc echo "------ Create OpenRC file [$JOID_ADMIN_OPENRC] ------" -KEYSTONE=$(cat bundles.yaml |shyaml get-value openstack-phase2.services.keystone.options.vip) -ODL_CONTROLLER=$(juju status odl-controller/0 |grep public-address|sed -- 's/.*\: //') -ODL_PASSWORD=admin + +# get Keystone ip +case "$HA_MODE" in + "ha") + KEYSTONE=$(cat bundles.yaml |shyaml get-value openstack-phase2.services.keystone.options.vip) + ;; + *) + KEYSTONE=$(juju status keystone |grep public-address|sed -- 's/.*\: //') + ;; +esac + + +# get controller IP +case "$SDN_CONTROLLER" in + "odl") + SDN_CONTROLLER_IP=$(juju status odl-controller/0 |grep public-address|sed -- 's/.*\: //') + ;; + "onos") + SDN_CONTROLLER_IP=$(juju status onos-controller/0 |grep public-address|sed -- 's/.*\: //') + ;; + *) + SDN_CONTROLLER_IP='none' + ;; +esac +SDN_PASSWORD='admin' # export the openrc file cat << EOF > $JOID_ADMIN_OPENRC @@ -164,8 +187,8 @@ export OS_PASSWORD=$OS_ADMIN_PASSWORD export OS_TENANT_NAME=admin export OS_AUTH_URL=http://$KEYSTONE:5000/v2.0 export OS_REGION_NAME=Canonical -export ODL_CONTROLLER=$ODL_CONTROLLER -export ODL_PASSWORD=$ODL_PASSWORD +export SDN_CONTROLLER=$SDN_CONTROLLER_IP +export SDN_PASSWORD=$SDN_PASSWORD EOF ## diff --git a/jjb/kvmfornfv/kvmfornfv-build.sh b/jjb/kvmfornfv/kvmfornfv-build.sh index 4e00a9d87..ff918492c 100755 --- a/jjb/kvmfornfv/kvmfornfv-build.sh +++ b/jjb/kvmfornfv/kvmfornfv-build.sh @@ -1,5 +1,12 @@ #!/bin/bash - +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## # build output directory OUTPUT_DIR=$WORKSPACE/build_output mkdir -p $OUTPUT_DIR diff --git a/jjb/kvmfornfv/kvmfornfv.yml b/jjb/kvmfornfv/kvmfornfv.yml index 34793adf6..b042c56a7 100644 --- a/jjb/kvmfornfv/kvmfornfv.yml +++ b/jjb/kvmfornfv/kvmfornfv.yml @@ -18,17 +18,12 @@ - job-template: name: 'kvmfornfv-verify-{stream}' - node: ericsson-build - parameters: - project-parameter: project: '{project}' - gerrit-parameter: branch: '{branch}' - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + - 'ericsson-build-defaults' scm: - gerrit-trigger-scm: @@ -62,17 +57,12 @@ - job-template: name: 'kvmfornfv-merge-{stream}' - node: ericsson-build - parameters: - project-parameter: project: '{project}' - gerrit-parameter: branch: '{branch}' - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + - 'ericsson-build-defaults' scm: - gerrit-trigger-scm: diff --git a/jjb/opnfv/installer-params.yml b/jjb/opnfv/installer-params.yml index 9dd3d5406..57ca9864e 100644 --- a/jjb/opnfv/installer-params.yml +++ b/jjb/opnfv/installer-params.yml @@ -47,7 +47,7 @@ description: 'Installer used for deploying OPNFV on this POD' - string: name: EXTERNAL_NETWORK - default: 'net04_ext' + default: 'admin_floating_net' description: 'external network for test' - parameter: @@ -74,10 +74,6 @@ default: '/srv' description: "Disks to use by ceph by default (space separated list)" - string: - name: CEPH_DISKS_CONTROLLERS - default: '' - description: "Disks to use by ceph by controllers (empty if same as CEPH_DISKS)" - - string: name: LAB_CONFIG default: "$HOME/joid_config" description: "Local lab config and Openstack openrc location" diff --git a/jjb/opnfv/opnfv-docker.sh b/jjb/opnfv/opnfv-docker.sh index bf355973c..2b4ca907a 100644 --- a/jjb/opnfv/opnfv-docker.sh +++ b/jjb/opnfv/opnfv-docker.sh @@ -1,4 +1,12 @@ #!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## set -o errexit set -o nounset set -o pipefail @@ -70,11 +78,13 @@ echo "Current branch: $branch" if [[ "$branch" == "master" ]]; then DOCKER_TAG="master" + DOCKER_BRANCH_TAG="latest" else git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng DOCKER_TAG=$($WORKSPACE/releng/utils/calculate_version.sh -t docker \ -n $DOCKER_REPO_NAME) + DOCKER_BRANCH_TAG="stable" ret_val=$? if [[ $ret_val -ne 0 ]]; then @@ -86,11 +96,11 @@ echo "Tag version to be build and pushed: $DOCKER_TAG" # Start the build -echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_TAG..." +echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_BRANCH_TAG" -docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG . -echo "Creating tag 'latest'..." -docker tag $DOCKER_REPO_NAME:$DOCKER_TAG $DOCKER_REPO_NAME:latest +docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_BRANCH_TAG . +echo "Creating tag '$DOCKER_TAG'..." +docker tag -f $DOCKER_REPO_NAME:$DOCKER_BRANCH_TAG $DOCKER_REPO_NAME:$DOCKER_TAG # list the images echo "Available images are:" @@ -102,8 +112,9 @@ if [[ "$PUSH_IMAGE" == "true" ]]; then echo "--------------------------------------------------------" echo # Push to the Dockerhub repository - docker push $DOCKER_REPO_NAME:$DOCKER_TAG + echo "Pushing $DOCKER_REPO_NAME:$DOCKER_BRANCH_TAG ..." + docker push $DOCKER_REPO_NAME:$DOCKER_BRANCH_TAG - echo "Updating $DOCKER_REPO_NAME:latest to the docker registry..." - docker push $DOCKER_REPO_NAME:latest + echo "Pushing $DOCKER_REPO_NAME:$DOCKER_TAG ..." + docker push $DOCKER_REPO_NAME:$DOCKER_TAG fi diff --git a/jjb/opnfv/opnfv-docker.yml b/jjb/opnfv/opnfv-docker.yml index c16e33eda..135fb493a 100644 --- a/jjb/opnfv/opnfv-docker.yml +++ b/jjb/opnfv/opnfv-docker.yml @@ -27,24 +27,19 @@ - job-template: name: '{project}-docker-build-push-{stream}' - node: ericsson-build - disabled: false parameters: - project-parameter: project: '{project}' - - string: - name: GIT_BASE - default: "https://gerrit.opnfv.org/gerrit/$PROJECT" - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." + - 'ericsson-build-defaults' - string: name: PUSH_IMAGE default: "true" description: "To enable/disable pushing the image to Dockerhub." - string: name: BASE_VERSION - default: "brahmaputra.0" + default: "brahmaputra.1" description: "Base version to be used." - string: name: DOCKER_REPO_NAME diff --git a/jjb/opnfv/opnfv-docs.yml b/jjb/opnfv/opnfv-docs.yml index 296424344..652a3d530 100644 --- a/jjb/opnfv/opnfv-docs.yml +++ b/jjb/opnfv/opnfv-docs.yml @@ -53,7 +53,7 @@ projects: - project-compare-type: 'REG_EXP' #Left out arno projects oscar|octopus|functest|genesis - project-pattern: 'vswitchperf|yardstick|apex|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|octopus|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sdnvpn|sfc|vnf_forwarding_graph|enfv|pinpoint|pharos|cperf|functest' + project-pattern: 'vswitchperf|apex|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|octopus|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sdnvpn|sfc|vnf_forwarding_graph|enfv|pinpoint|pharos|cperf|functest|genesisreq' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' @@ -91,7 +91,7 @@ comment-contains-value: 'remerge' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'vswitchperf|yardstick|apex|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|octopus|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sdnvpn|sfc|vnf_forwarding_graph|enfv|pinpoint|pharos|cperf|functest' + project-pattern: 'vswitchperf|apex|availability|bottlenecks|compass4nfv|conductor|copper|doctor|dovetail|dpacc|escalator|fastpathmetrics|fuel|inspector|ipv6|joid|kvmfornfv|lsoapi|moon|movie|multisite|octopus|onosfw|ovsnfv|parser|prediction|promise|qtip|releng|rs|sdnvpn|sfc|vnf_forwarding_graph|enfv|pinpoint|pharos|cperf|functest|genesisreq' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' diff --git a/jjb/opnfv/slave-params.yml b/jjb/opnfv/slave-params.yml index 5204f7926..cc196737b 100644 --- a/jjb/opnfv/slave-params.yml +++ b/jjb/opnfv/slave-params.yml @@ -83,15 +83,15 @@ description: 'Git URL to use on this Jenkins Slave' - parameter: - name: 'intelpod2-jumphost-defaults' + name: 'intel-pod2-defaults' parameters: - node: name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: - - intelpod2-jumphost + - intel-pod2 default-slaves: - - intelpod2-jumphost + - intel-pod2 - string: name: INSTALLER_VERSION default: stable @@ -194,15 +194,15 @@ description: 'Git URL to use on this Jenkins Slave' - parameter: - name: 'ericsson-build-defaults' + name: 'huawei-build-defaults' parameters: - node: name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: - - ericsson-build + - huawei-build default-slaves: - - ericsson-build + - huawei-build - string: name: INSTALLER_VERSION default: stable @@ -211,16 +211,17 @@ name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' + - parameter: - name: 'huawei-build-defaults' + name: 'huawei-us-deploy-bare-1-defaults' parameters: - node: name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: - - huawei-build + - huawei-us-deploy-bare-1 default-slaves: - - huawei-build + - huawei-us-deploy-bare-1 - string: name: INSTALLER_VERSION default: stable @@ -231,15 +232,15 @@ description: 'Git URL to use on this Jenkins Slave' - parameter: - name: 'huawei-us-deploy-bare-1-defaults' + name: 'huawei-us-deploy-bare-2-defaults' parameters: - node: name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: - - huawei-us-deploy-bare-1 + - huawei-us-deploy-bare-2 default-slaves: - - huawei-us-deploy-bare-1 + - huawei-us-deploy-bare-2 - string: name: INSTALLER_VERSION default: stable @@ -265,17 +266,34 @@ description: 'Git URL to use on this Jenkins Slave' - parameter: - name: 'opnfv-build-defaults' + name: 'huawei-sh-test-2-defaults' parameters: - node: name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: - - intel-us-build-1 - - ericsson-ca-build-1 + - huawei-sh-test-2 default-slaves: - - intel-us-build-1 - - ericsson-ca-build-1 + - huawei-sh-test-2 + - label: + name: SLAVE_LABEL + default: 'huawei-test' + - string: + name: INSTALLER_VERSION + default: stable + description: 'Version of the installer to deploy' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + +- parameter: + name: 'opnfv-build-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'opnfv-build' + description: 'Slave label on Jenkins' - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT @@ -286,45 +304,45 @@ description: "Directory where the build artifact will be located upon the completion of the build." - parameter: - name: 'intel-us-build-1-defaults' + name: 'intel-build1-defaults' parameters: - node: name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: - - intel-us-build-1 + - intel-build1 default-slaves: - - intel-us-build-1 + - intel-build1 - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' - parameter: - name: 'intel-us-build-2-defaults' + name: 'intel-build2-defaults' parameters: - node: name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: - - intel-us-build-2 + - intel-build2 default-slaves: - - intel-us-build-2 + - intel-build2 - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' - parameter: - name: 'ericsson-ca-build-1-defaults' + name: 'intel-build3-defaults' parameters: - node: name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: - - ericsson-ca-build-1 + - intel-build3 default-slaves: - - ericsson-ca-build-1 + - intel-build3 - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT @@ -391,10 +409,6 @@ default: /dev/sdb /dev/sdc description: "Disks to use by ceph by default (space separated list)" - string: - name: CEPH_DISKS_CONTROLLERS - default: /dev/sdb - description: "Disks to use by ceph on controler nodes (space separated list)" - - string: name: EXTERNAL_NETWORK default: ext-net;flat;161.105.231.2;161.105.231.62;161.105.231.1;161.105.231.0/26 description: "External network to create (name;type;first ip;last ip; gateway;network)" @@ -430,15 +444,26 @@ description: 'Git URL to use on these Jenkins Slaves' - parameter: - name: 'intel-us-deploy-virtual-2-defaults' + name: 'ericsson-build-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'ericsson-build' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on these Jenkins Slaves' + +- parameter: + name: 'intel-virtual2-defaults' parameters: - node: name: SLAVE_NAME description: 'Slave name on Jenkins' allowed-slaves: - - intel-us-deploy-virtual-2 + - intel-virtual2 default-slaves: - - intel-us-deploy-virtual-2 + - intel-virtual2 - string: name: INSTALLER_VERSION default: latest @@ -451,3 +476,41 @@ name: SSH_KEY default: /root/.ssh/id_rsa description: 'SSH key to use for Apex' + +- parameter: + name: 'dell-us-testing-bm-1-defaults' + parameters: + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - dell-us-testing-bm-1 + default-slaves: + - dell-us-testing-bm-1 + - string: + name: INSTALLER_VERSION + default: latest + description: 'Version of the installer to deploy' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + +- parameter: + name: 'dell-us-deploying-bm3-defaults' + parameters: + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - dell-us-deploying-bm3 + default-slaves: + - dell-us-deploying-bm3 + - string: + name: INSTALLER_VERSION + default: latest + description: 'Version of the installer to deploy' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' diff --git a/jjb/opnfvdocs/opnfvdocs.yml b/jjb/opnfvdocs/opnfvdocs.yml index c4fa73468..cbb8f9545 100644 --- a/jjb/opnfvdocs/opnfvdocs.yml +++ b/jjb/opnfvdocs/opnfvdocs.yml @@ -111,13 +111,15 @@ builders: - build-opnfv-composite-docs -# - upload-generated-docs-to-opnfv-artifacts + - upload-generated-docs-to-opnfv-artifacts - report-docs-build-result-to-gerrit - remove-old-docs-from-opnfv-artifacts - job-template: name: 'opnfvdocs-daily-{stream}' + disabled: true + parameters: - project-parameter: project: '{project}' diff --git a/jjb/ovsnfv/ovsnfv.yml b/jjb/ovsnfv/ovsnfv.yml index d0c180c5b..8a119d9fb 100644 --- a/jjb/ovsnfv/ovsnfv.yml +++ b/jjb/ovsnfv/ovsnfv.yml @@ -24,7 +24,7 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - 'intel-us-build-2-defaults' + - 'intel-build2-defaults' - string: name: GS_URL default: '$GS_BASE{gs-pathname}' @@ -73,7 +73,7 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - 'intel-us-build-2-defaults' + - 'intel-build2-defaults' - string: name: GS_URL default: '$GS_BASE{gs-pathname}' @@ -114,7 +114,7 @@ parameters: - project-parameter: project: '{project}' - - 'intel-us-build-2-defaults' + - 'intel-build2-defaults' - string: name: GS_URL default: '$GS_BASE{gs-pathname}' diff --git a/jjb/qtip/qtip.yml b/jjb/qtip/qtip-ci-jobs.yml index 450ba938e..8dd0a4af0 100644 --- a/jjb/qtip/qtip.yml +++ b/jjb/qtip/qtip-ci-jobs.yml @@ -1,105 +1,81 @@ +#################################### +# job configuration for qtip +#################################### - project: - name: qtip - - project: '{name}' - - jobs: - - 'qtip-verify-{stream}' - - 'qtip-{installer_type}-{pod}-daily-{stream}' - + name: qtip-ci-jobs + + project: 'qtip' + +#-------------------------------- +# BRANCH ANCHORS +#-------------------------------- + master: &master + stream: master + branch: '{stream}' + gs-pathname: '' + brahmaputra: &brahmaputra + stream: brahmaputra + branch: 'stable/{stream}' + gs-pathname: '{stream}' +#-------------------------------- +# POD, INSTALLER, AND BRANCH MAPPING +#-------------------------------- +# brahmaputra +#-------------------------------- pod: - dell-us-testing-bm-1: - node: 'dell-us-testing-bm-1' - installer_type: 'fuel' - installer_ip: '10.20.0.2' + installer: compass + auto-trigger-name: 'qtip-daily-dell-us-testing-bm-1-trigger' + <<: *brahmaputra + - orange-pod2: + installer: joid + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' + <<: *brahmaputra - dell-us-deploying-bm3: - node: 'dell-us-deploying-bm3' - installer_type: 'fuel' - installer_ip: '10.20.0.2' - -# only master branch is enabled at the moment to keep no of jobs sane - stream: - - master: - branch: 'master' - gs-pathname: '' -# - brahmaputra: -# branch: 'stable/brahmaputra' -# gs-pathname: '/brahmaputra' + installer: fuel + auto-trigger-name: 'qtip-daily-dell-us-deploying-bm3-trigger' + <<: *brahmaputra + +#-------------------------------- +# master +#-------------------------------- + - juniper-pod1: + installer: joid + <<: *master + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' +# - zte-build-1: #would be confirmed with the ZTE lab by tomorrow +# installer: fuel +# <<: *master + +#-------------------------------- + jobs: + - 'qtip-{installer}-{pod}-daily-{stream}' +################################ +# job templates +################################ - job-template: - name: 'qtip-verify-{stream}' + name: 'qtip-{installer}-{pod}-daily-{stream}' + + disabled: false parameters: - project-parameter: project: '{project}' - - gerrit-parameter: - branch: '{branch}' - - 'opnfv-build-defaults' + - '{pod}-defaults' + - '{installer}-defaults' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-ha' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' triggers: - - gerrit: - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - forbidden-file-paths: - - compare-type: ANT - pattern: 'docs/**|.gitignore' - - builders: - - shell: | - echo "Nothing to verify!" - -- job-template: - name: 'qtip-{installer_type}-{pod}-daily-{stream}' - - disabled: false - - node: '{node}' - - parameters: - - project-parameter: - project: '{project}' - - string: - name: INSTALLER_TYPE - default: '{installer_type}' - description: "Installer name that is used for deployment." - - string: - name: INSTALLER_IP - default: '{installer_ip}' - description: "Installer IP" - - string: - name: NODE_NAME - default: '{node}' - description: "Name of the POD" - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: master + - '{auto-trigger-name}' builders: - 'qtip-cleanup' @@ -107,9 +83,9 @@ - 'qtip-run-suite' - 'qtip-pushtoDB' - triggers: - - timed: '0 0,13,15,20 * * *' - + publishers: + - email: + recipients: nauman.ahad@xflowresearch.com, mofassir.arif@xflowresearch.com, vikram@nvirters.com ########################### #biuilder macros @@ -123,7 +99,6 @@ envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} -e NODE_NAME=${NODE_NAME}" suite="TEST_CASE=all" docker pull opnfv/qtip:latest - cmd=" docker run -id -e $envs -e $suite opnfv/qtip:latest /bin/bash" echo "Qtip: Running docker run command: ${cmd}" ${cmd} @@ -191,16 +166,28 @@ done fi -- builder: - name: qtip-fetch-os-cred - builders: - - shell: | - #!/bin/bash +################# +#trigger macros +################# + +- trigger: + name: 'qtip-daily-dell-us-deploying-bm3-trigger' + triggers: + - timed: '0 0 * * *' + +- trigger: + name: 'qtip-daily-dell-us-testing-bm-1-trigger' + triggers: + - timed: '0 3 * * *' + +#- trigger: +# name: 'qtip-daily-juniper-build1-trigger' +# triggers: +# - timed : '0 0 * * *' + +#- trigger: +# name: 'qtip-dailty-orange-fr-pod2-trigger' +# triggers: +# - timed : ' 0 0 * * *' + - echo $HOME - echo $WORKSPACE - cd $WORKSPACE - git clone https://gerrit.opnfv.org/gerrit/releng - cd $WORKSPACE/releng/utils - #./fetch_os_creds.sh #To test the problem with creating the file in $HOME - ./fetch_os_creds.sh -d $WORKSPACE/opnfv-stack.sh #To test if the file can be created in $WORKSPACE diff --git a/jjb/qtip/qtip-project-jobs.yml b/jjb/qtip/qtip-project-jobs.yml new file mode 100644 index 000000000..c77bac722 --- /dev/null +++ b/jjb/qtip/qtip-project-jobs.yml @@ -0,0 +1,58 @@ +- project: + name: qtip + + project: '{name}' + + jobs: + - 'qtip-verify-{stream}' + +# only master branch is enabled at the moment to keep no of jobs sane + stream: + - master: + branch: '{stream}' + gs-pathname: '' + - brahmaputra: + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + +- job-template: + name: 'qtip-verify-{stream}' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-defaults' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + triggers: + - gerrit: + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + + builders: + - shell: | + echo "Nothing to verify!" diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml index 5d9570705..b43a76d5f 100644 --- a/jjb/releng-macros.yaml +++ b/jjb/releng-macros.yaml @@ -195,7 +195,7 @@ echo "###########################" echo - gs_base="artifacts.opnfv.org/review" + gs_base="artifacts.opnfv.org/$PROJECT/review" gs_path="$gs_base/$GERRIT_CHANGE_NUMBER" local_path="upload/$GERRIT_CHANGE_NUMBER" @@ -283,6 +283,13 @@ [[ $GERRIT_CHANGE_NUMBER =~ .+ ]] + gs_path="artifacts.opnfv.org/$PROJECT/review/$GERRIT_CHANGE_NUMBER" + + if gsutil ls "gs://$gs_path" > /dev/null 2>&1 ; then + echo + echo "Deleting Out-of-dated Documents..." + gsutil -m rm -r "gs://$gs_path" + fi gs_path="artifacts.opnfv.org/review/$GERRIT_CHANGE_NUMBER" if gsutil ls "gs://$gs_path" > /dev/null 2>&1 ; then diff --git a/jjb/verify-releng b/jjb/verify-releng index 290b4c5a5..82d76e366 100755 --- a/jjb/verify-releng +++ b/jjb/verify-releng @@ -1,4 +1,12 @@ #!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## #test for non-ascii characters, these can pass the test and end up breaking things in production for x in $(find . -name *\.yml); do diff --git a/jjb/yardstick/yardstick-ci-jobs.yml b/jjb/yardstick/yardstick-ci-jobs.yml index 21b6691b7..9566651ff 100644 --- a/jjb/yardstick/yardstick-ci-jobs.yml +++ b/jjb/yardstick/yardstick-ci-jobs.yml @@ -13,10 +13,12 @@ stream: master branch: '{stream}' gs-pathname: '' + docker-tag: 'latest' brahmaputra: &brahmaputra stream: brahmaputra branch: 'stable/{stream}' gs-pathname: '{stream}' + docker-tag: 'brahmaputra.1.0' #-------------------------------- # POD, INSTALLER, AND BRANCH MAPPING #-------------------------------- @@ -26,81 +28,120 @@ - opnfv-jump-1: installer: apex suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *brahmaputra - opnfv-jump-2: installer: fuel suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *brahmaputra - intel-pod5: installer: joid suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' + <<: *brahmaputra + - orange-pod2: + installer: joid + suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *brahmaputra - huawei-us-deploy-bare-1: installer: compass suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *brahmaputra - ericsson-pod1: installer: fuel suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' + <<: *brahmaputra + - ericsson-pod2: + installer: fuel + suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *brahmaputra - virtual: installer: fuel suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *brahmaputra - huawei-virtual: installer: compass suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *brahmaputra - virtual: installer: joid suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *brahmaputra #-------------------------------- # master #-------------------------------- + - opnfv-jump-2: + installer: fuel + suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' + <<: *master - ericsson-pod1: installer: fuel suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *master - ericsson-pod1: installer: fuel suite: vtcdaily + auto-trigger-name: 'yardstick-vtcdaily-ericsson-pod1-trigger' <<: *master - ericsson-pod2: installer: fuel suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *master - intel-pod6: installer: joid suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *master - intel-pod8: installer: compass suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *master - zte-build-1: installer: fuel suite: daily + auto-trigger-name: 'yardstick-daily-zte-build-1-trigger' <<: *master - orange-pod2: installer: joid suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *master - opnfv-jump-1: installer: apex suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *master - virtual: installer: fuel suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *master - - virtual: + - huawei-virtual: installer: compass suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' + <<: *master + - huawei-sh-test-2: + installer: compass + suite: daily + auto-trigger-name: 'yardstick-daily-huawei-sh-test-2-trigger' <<: *master - virtual: installer: joid suite: daily + auto-trigger-name: 'brahmaputra-trigger-daily-disabled' <<: *master #-------------------------------- @@ -115,12 +156,22 @@ disabled: false + concurrent: true + + properties: + - throttle: + enabled: true + max-per-node: 1 + wrappers: - build-name: name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + - timeout: + timeout: 180 + abort: true triggers: - - 'yardstick-{pod}-trigger' + - '{auto-trigger-name}' parameters: - project-parameter: @@ -132,6 +183,10 @@ name: DEPLOY_SCENARIO default: 'os-odl_l2-nofeature-ha' - string: + name: DOCKER_TAG + default: '{docker-tag}' + description: 'Tag to pull docker image' + - string: name: YARDSTICK_SUITE_NAME default: opnfv_${{NODE_NAME}}_{suite}.yaml description: 'Path to test suite' @@ -193,7 +248,7 @@ -e YARDSTICK_BRANCH=${GIT_BRANCH##origin/} -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO}" # Pull the latest image - docker pull opnfv/yardstick >$redirect + docker pull opnfv/yardstick:$DOCKER_TAG >$redirect # Run docker cmd="sudo docker run ${opts} ${envs} ${labconfig} ${sshkey} opnfv/yardstick \ @@ -254,7 +309,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -262,7 +317,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -278,7 +333,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '-i 10.118.36.90:8086' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -286,7 +341,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '-i 10.118.36.90:8086' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -294,7 +349,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '-r 213.77.62.197/results' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -302,7 +357,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '-r 213.77.62.197/results' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -310,7 +365,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -326,7 +381,7 @@ parameters: - string: name: YARDSTICK_DB_BACKEND - default: '' + default: '-i 104.197.68.199:8086' description: 'Arguments to use in order to choose the backend DB' - parameter: @@ -345,66 +400,29 @@ default: '' description: 'Arguments to use in order to choose the backend DB' +- parameter: + name: 'yardstick-params-huawei-sh-test-2' + parameters: + - string: + name: YARDSTICK_DB_BACKEND + default: '' + description: 'Arguments to use in order to choose the backend DB' + ####################### ## trigger macros ####################### # trigger for PODs to only run yardstick test suites - trigger: - name: 'yardstick-zte-build-1-trigger' + name: 'yardstick-daily-zte-build-1-trigger' triggers: - timed: '0 2 * * *' - trigger: - name: 'yardstick-intel-pod5-trigger' - triggers: - - timed: '' - -- trigger: - name: 'yardstick-intel-pod6-trigger' - triggers: - - timed: '' - -- trigger: - name: 'yardstick-intel-pod8-trigger' - triggers: - - timed: '' - -- trigger: - name: 'yardstick-ericsson-pod1-trigger' - triggers: - - timed: '' - -- trigger: - name: 'yardstick-ericsson-pod2-trigger' - triggers: - - timed: '' - -- trigger: - name: 'yardstick-opnfv-jump-1-trigger' - triggers: - - timed: '' - -- trigger: - name: 'yardstick-opnfv-jump-2-trigger' - triggers: - - timed: '' - -- trigger: - name: 'yardstick-huawei-us-deploy-bare-1-trigger' - triggers: - - timed: '' - -- trigger: - name: 'yardstick-orange-pod2-trigger' - triggers: - - timed: '' - -- trigger: - name: 'yardstick-virtual-trigger' + name: 'yardstick-vtcdaily-ericsson-pod1-trigger' triggers: - - timed: '' + - timed: '0 1 * * *' - trigger: - name: 'yardstick-huawei-virtual-trigger' + name: 'yardstick-daily-huawei-sh-test-2-trigger' triggers: - - timed: '' + - timed: '0 1 * * *' diff --git a/jjb/yardstick/yardstick-project-jobs.yml b/jjb/yardstick/yardstick-project-jobs.yml index 4f978e189..6e4c178d4 100644 --- a/jjb/yardstick/yardstick-project-jobs.yml +++ b/jjb/yardstick/yardstick-project-jobs.yml @@ -31,7 +31,7 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - 'ericsson-ca-build-1-defaults' + - 'ericsson-build-defaults' scm: - gerrit-trigger-scm: @@ -58,19 +58,9 @@ - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' builders: - - shell: | - #!/bin/bash - set -o errexit - set -o pipefail - - echo "Running unit tests..." - cd $WORKSPACE - virtualenv $WORKSPACE/yardstick_venv - source $WORKSPACE/yardstick_venv/bin/activate - easy_install -U setuptools - python setup.py develop - ./run_tests.sh - deactivate + - yardstick-unit-tests-and-docs-build + - upload-under-review-docs-to-opnfv-artifacts + - report-docs-build-result-to-gerrit - job-template: name: 'yardstick-merge-{stream}' @@ -80,7 +70,11 @@ project: '{project}' - gerrit-parameter: branch: '{branch}' - - 'ericsson-ca-build-1-defaults' + - 'ericsson-build-defaults' + - string: + name: GS_URL + default: '$GS_BASE{gs-pathname}' + description: "Directory where the build artifact will be located upon the completion of the build." scm: - gerrit-trigger-scm: @@ -102,6 +96,18 @@ branch-pattern: '**/{branch}' builders: + - yardstick-unit-tests-and-docs-build + - upload-generated-docs-to-opnfv-artifacts + - report-docs-build-result-to-gerrit + - remove-old-docs-from-opnfv-artifacts + +################################ +# job builders +################################ + +- builder: + name: yardstick-unit-tests-and-docs-build + builders: - shell: | #!/bin/bash set -o errexit @@ -111,7 +117,23 @@ cd $WORKSPACE virtualenv $WORKSPACE/yardstick_venv source $WORKSPACE/yardstick_venv/bin/activate + + # install python packages easy_install -U setuptools - python setup.py develop + #python setup.py develop + pip install -r ci/requirements.txt + pip install -e . + + # unit tests ./run_tests.sh + + pip install Sphinx==1.3.1 doc8 docutils sphinxcontrib-httpdomain + + # generate api doc sources + sphinx-apidoc -o docs/apidocs yardstick + + # build docs + git clone ssh://gerrit.opnfv.org:29418/releng + GERRIT_COMMENT=gerrit_comment.txt ./releng/utils/docs-build.sh + deactivate diff --git a/utils/calculate_version.sh b/utils/calculate_version.sh index 5dccc075d..4bfd13f7a 100755 --- a/utils/calculate_version.sh +++ b/utils/calculate_version.sh @@ -1,15 +1,17 @@ #!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Ericsson AB and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## -# -# Authors: -# Jose Lausuch <jose.lausuch@ericsson.com> -# Fatih Degirmenci <fatih.degirmenci@ericsson.com> -# # Calculates and generates the version tag for the OPNFV objects: # - Docker images # - ISOs -# - Artifcats -# +# - Artifacts info () { logger -s -t "Calculate_version.info" "$*" diff --git a/utils/docs-build.sh b/utils/docs-build.sh index 39647a33d..4d95836d5 100755 --- a/utils/docs-build.sh +++ b/utils/docs-build.sh @@ -1,8 +1,14 @@ #!/bin/bash -e - +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 NEC and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## export PATH=$PATH:/usr/local/bin/ - DOCS_DIR=${DOCS_DIR:-docs} INDEX_RST=${INDEX_RST:-index.rst} BUILD_DIR=${BUILD_DIR:-docs_build} @@ -35,7 +41,7 @@ revision="$(git rev-parse --short HEAD)" rev_full="$(git rev-parse HEAD)" version="$(git describe --abbrev=0 2> /dev/null || echo draft) ($revision)" project="$(basename $(git rev-parse --show-toplevel))" -html_notes="\n Revision: $rev_full\n\n Build date: |today|" +html_notes=" Revision: $rev_full\n Build date: $(date -u +'%Y-%m-%d')" default_conf='releng/docs/etc/conf.py' opnfv_logo='releng/docs/etc/opnfv-logo.png' @@ -72,7 +78,7 @@ function add_html_notes() { echo sed -i "s/ _sha1_/ $git_sha1/g" "$file" fi - sed -i -e "\$a\\\n.. only:: html\n$html_notes" "$file" + sed -i -e "\$a\\\n..\n$html_notes" "$file" done } diff --git a/utils/fetch_os_creds.sh b/utils/fetch_os_creds.sh index d1b192dcb..a5c601b82 100755 --- a/utils/fetch_os_creds.sh +++ b/utils/fetch_os_creds.sh @@ -94,7 +94,7 @@ if [ "$installer_type" == "fuel" ]; then #This file contains the mgmt keystone API, we need the public one for our rc file admin_ip=$(cat $dest_path | grep "OS_AUTH_URL" | sed 's/^.*\=//' | sed "s/^\([\"']\)\(.*\)\1\$/\2/g" | sed s'/\/$//') public_ip=$(sshpass -p r00tme ssh $ssh_options root@${installer_ip} \ - "ssh ${controller_ip} 'source openrc; keystone endpoint-list'" \ + "ssh ${controller_ip} 'source openrc; openstack endpoint list --long'" \ | grep $admin_ip | sed 's/ /\n/g' | grep ^http | head -1) &> /dev/null #| grep http | head -1 | cut -d '|' -f 4 | sed 's/v1\/.*/v1\//' | sed 's/ //g') &> /dev/null #NOTE: this is super ugly sed 's/v1\/.*/v1\//'OS_AUTH_URL diff --git a/utils/jenkins-jnlp-connect.sh b/utils/jenkins-jnlp-connect.sh index 00c19971f..6fb6827fd 100755 --- a/utils/jenkins-jnlp-connect.sh +++ b/utils/jenkins-jnlp-connect.sh @@ -1,4 +1,13 @@ #!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Linux Foundation and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## + #Monit setup script for opnfv jnlp slave connections test_firewall() { diff --git a/utils/lab-reconfiguration/create_venv.sh b/utils/lab-reconfiguration/create_venv.sh index 4e45f8462..4e7144c70 100755 --- a/utils/lab-reconfiguration/create_venv.sh +++ b/utils/lab-reconfiguration/create_venv.sh @@ -1,5 +1,12 @@ #!/bin/bash -e - +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Cisco and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## # Script checks that venv exists. If it doesn't it will be created # It requires python2.7 and virtualenv packages installed diff --git a/utils/lab-reconfiguration/requirements.pip b/utils/lab-reconfiguration/requirements.pip index 8cd1db0ba..b10cbceab 100644 --- a/utils/lab-reconfiguration/requirements.pip +++ b/utils/lab-reconfiguration/requirements.pip @@ -1,2 +1,10 @@ +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Cisco and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## UcsSdk==0.8.2.2 PyYAML diff --git a/utils/push-test-logs.sh b/utils/push-test-logs.sh new file mode 100644 index 000000000..972853b78 --- /dev/null +++ b/utils/push-test-logs.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Orange and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -e +set -o pipefail + +export PATH=$PATH:/usr/local/bin/ + +git_sha1="$(git rev-parse HEAD)" +res_build_date=${1:-$(date -u +"%Y-%m-%d_%H-%M-%S")} +project=$PROJECT +branch=${GIT_BRANCH##*/} +testbed=$NODE_NAME +dir_result="${HOME}/opnfv/$project/results/$branch" +node_list=(\ +'opnfv-jump-1' 'opnfv-jump-2' 'ericsson-pod1' 'ericsson-pod2' \ +'intelpod2-jumphost' 'intel-pod3' 'intel-pod5' 'intel-pod6' \ +'intel-pod8' 'huawei-us-deploy-bare-1' 'orange-fr-pod2') + +if [[ ! " ${node_list[@]} " =~ " ${testbed} " ]]; then + echo "This is not a CI POD. Aborting pushing the logs to artifacts." + exit 0 +fi + +if [[ "$branch" == "master" ]]; then + project_artifact=logs/$project/$testbed/$res_build_date +else + project_artifact=logs/$project/$testbed/$branch/$res_build_date +fi + +# create the folder to store the results +mkdir -p $dir_result + +# copy folder to artifact +if [ -d "$dir_result" ]; then + if [ "$(ls -A $dir_result)" ]; then + set +e + gsutil&>/dev/null + if [ $? != 0 ]; then + echo "Not possible to push results to artifact: gsutil not installed"; + else + gsutil ls gs://artifacts.opnfv.org/"$project"/ &>/dev/null + if [ $? != 0 ]; then + echo "Not possible to push results to artifact: gsutil not installed."; + else + echo "copy result files to artifact $project_artifact" + gsutil -m cp -r "$dir_result" gs://artifacts.opnfv.org/"$project_artifact"/ + fi + fi + else + echo "Result folder is empty" + fi +fi diff --git a/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py b/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py index 8d5326eb4..9a7e4ce1b 100755 --- a/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py +++ b/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py @@ -62,19 +62,35 @@ def format_rubbos_for_dashboard(results): """ test_data = [{'description': 'Rubbos results'}] - # Graph 1: + # Graph 1:Rubbos maximal throughput + # ******************************** + #new_element = [] + #for each_result in results: + # throughput_data = [record['throughput'] for record in each_result['details']] + # new_element.append({'x': each_result['creation_date'], + # 'y': max(throughput_data)}) + + #test_data.append({'name': "Rubbos max throughput", + # 'info': {'type': "graph", + # 'xlabel': 'time', + # 'ylabel': 'maximal throughput'}, + # 'data_set': new_element}) + + # Graph 2: Rubbos last record # ******************************** new_element = [] - for each_result in results: - throughput_data = [record['throughput'] for record in each_result['details']] - new_element.append({'x': each_result['creation_date'], - 'y': max(throughput_data)}) - - test_data.append({'name': "Rubbos max throughput", + latest_result = results[-1]["details"] + for data in latest_result: + client_num = int(data["client"]) + throughput = int(data["throughput"]) + new_element.append({'x': client_num, + 'y': throughput}) + test_data.append({'name': "Rubbos throughput vs client number", 'info': {'type': "graph", - 'xlabel': 'time', - 'ylabel': 'maximal throughput'}, + 'xlabel': 'client number', + 'ylabel': 'throughput'}, 'data_set': new_element}) + return test_data @@ -161,9 +177,9 @@ def _get_results(db_url, test_criteria): myDataResults = myNewData['test_results'] return myDataResults - +#only for local test def _test(): - db_url = "http://213.77.62.197" + db_url = "http://testresults.opnfv.org/testapi" results = _get_results(db_url, {"project": "bottlenecks", "testcase": "rubbos"}) test_result = format_rubbos_for_dashboard(results) print json.dumps(test_result, indent=4) diff --git a/utils/test/result_collection_api/dashboard/dashboard_utils.py b/utils/test/result_collection_api/dashboard/dashboard_utils.py index 8d83b006a..472bbc727 100644 --- a/utils/test/result_collection_api/dashboard/dashboard_utils.py +++ b/utils/test/result_collection_api/dashboard/dashboard_utils.py @@ -26,6 +26,10 @@ from bottlenecks2Dashboard import format_bottlenecks_for_dashboard, \ check_bottlenecks_case_exist from qtip2Dashboard import format_qtip_for_dashboard, \ check_qtip_case_exist +from promise2Dashboard import format_promise_for_dashboard, \ + check_promise_case_exist +from doctor2Dashboard import format_doctor_for_dashboard, \ + check_doctor_case_exist # any project test project wishing to provide dashboard ready values # must include at least 2 methods diff --git a/utils/test/result_collection_api/dashboard/doctor2Dashboard.py b/utils/test/result_collection_api/dashboard/doctor2Dashboard.py new file mode 100644 index 000000000..eba35b57b --- /dev/null +++ b/utils/test/result_collection_api/dashboard/doctor2Dashboard.py @@ -0,0 +1,105 @@ + #!/usr/bin/python +# +# Copyright (c) 2015 Orange +# morgan.richomme@orange.com +# +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# This script is used to build dashboard ready json results +# It may be used for all the test case of the Doctor project +# a new method format_<Test_case>_for_dashboard(results) +# +import re +import datetime + + +def get_doctor_cases(): + """ + get the list of the supported test cases + TODO: update the list when adding a new test case for the dashboard + """ + return ["doctor-notification","doctor-mark-down"] + + +def format_doctor_for_dashboard(case, results): + """ + generic method calling the method corresponding to the test case + check that the testcase is properly declared first + then build the call to the specific method + """ + + if check_doctor_case_exist(case): + # note we add _case because testcase and project had the same name + # TODO refactoring...looks fine at the beginning wit only 1 project + # not very ugly now and clearly not optimized... + cmd = "format_" + case.replace('-','_') + "_case_for_dashboard(results)" + res = eval(cmd) + else: + res = [] + return res + + +def check_doctor_case_exist(case): + """ + check if the testcase exists + if the test case is not defined or not declared in the list + return False + """ + doctor_cases = get_doctor_cases() + + if (case is None or case not in doctor_cases): + return False + else: + return True + + +def format_doctor_mark_down_case_for_dashboard(results): + """ + Post processing for the doctor test case + """ + test_data = [{'description': 'doctor-mark-down results for Dashboard'}] + return test_data + + +def format_doctor_notification_case_for_dashboard(results): + """ + Post processing for the doctor-notification test case + """ + test_data = [{'description': 'doctor results for Dashboard'}] + # Graph 1: (duration)=f(time) + # *************************************** + new_element = [] + + # default duration 0:00:08.999904 + # consider only seconds => 09 + for data in results: + t = data['details']['duration'] + new_element.append({'x': data['creation_date'], + 'y': t}) + + test_data.append({'name': "doctor-notification duration ", + 'info': {'type': "graph", + 'xlabel': 'time (s)', + 'ylabel': 'duration (s)'}, + 'data_set': new_element}) + + # Graph 2: bar + # ************ + nbTest = 0 + nbTestOk = 0 + + for data in results: + nbTest += 1 + if data['details']['status'] == "OK": + nbTestOk += 1 + + test_data.append({'name': "doctor-notification status", + 'info': {"type": "bar"}, + 'data_set': [{'Nb tests': nbTest, + 'Nb Success': nbTestOk}]}) + + return test_data diff --git a/utils/test/result_collection_api/dashboard/functest2Dashboard.py b/utils/test/result_collection_api/dashboard/functest2Dashboard.py index c1d116112..3f4e1a2db 100644 --- a/utils/test/result_collection_api/dashboard/functest2Dashboard.py +++ b/utils/test/result_collection_api/dashboard/functest2Dashboard.py @@ -14,13 +14,17 @@ # a new method format_<Test_case>_for_dashboard(results) # v0.1: basic example with methods for odl, Tempest, Rally and vPing # +import datetime +import re + def get_functest_cases(): """ get the list of the supported test cases TODO: update the list when adding a new test case for the dashboard """ - return ["status", "vPing", "vPing_userdata", "vIMS", "Tempest", "odl", "Rally"] + return ["status", "vPing", "vPing_userdata", "vIMS", "Tempest", "ODL", + "ONOS", "Rally"] def format_functest_for_dashboard(case, results): @@ -64,7 +68,9 @@ def format_status_for_dashboard(results): test_data.append({'nb test suite(s) run': len(testcases)-1}) test_data.append({'vPing': '100%'}) test_data.append({'VIM status': '82%'}) - test_data.append({'SDN Controllers': {'odl':'92%', 'onos':'95%', 'opencontrail':'93%'}}) + test_data.append({'SDN Controllers': {'odl': '92%', + 'onos': '95%', + 'ocl': '93%'}}) test_data.append({'VNF deployment': '95%'}) return test_data @@ -210,11 +216,125 @@ def format_Tempest_for_dashboard(results): return test_data -def format_odl_for_dashboard(results): +def format_ODL_for_dashboard(results): + """ + Post processing for the ODL test case + """ + test_data = [{'description': 'ODL results for Dashboard'}] + + # Graph 1: (Nb test, nb failure)=f(time) + # *************************************** + new_element = [] + + for data in results: + odl_results = data['details']['details'] + nbFailures = 0 + for odl in odl_results: + if (odl['test_status']['@status'] == "FAIL"): + nbFailures += 1 + new_element.append({'x': data['creation_date'], + 'y1': len(odl_results), + 'y2': nbFailures}) + + test_data.append({'name': "ODL nb tests/nb failures", + 'info': {'type': "graph", + 'xlabel': 'time', + 'y1label': 'Number of tests', + 'y2label': 'Number of failures'}, + 'data_set': new_element}) + return test_data + + +def format_ONOS_for_dashboard(results): """ Post processing for the odl test case """ - test_data = [{'description': 'odl results for Dashboard'}] + test_data = [{'description': 'ONOS results for Dashboard'}] + # Graph 1: (duration FUNCvirtNet)=f(time) + # *************************************** + new_element = [] + + # default duration 0:00:08.999904 + # consider only seconds => 09 + for data in results: + t = data['details']['FUNCvirNet']['duration'] + h, m, s = re.split(':', t) + s = round(float(s)) + new_duration = int(datetime.timedelta(hours=int(h), + minutes=int(m), + seconds=int(s)).total_seconds()) + new_element.append({'x': data['creation_date'], + 'y': new_duration}) + + test_data.append({'name': "ONOS FUNCvirNet duration ", + 'info': {'type': "graph", + 'xlabel': 'time (s)', + 'ylabel': 'duration (s)'}, + 'data_set': new_element}) + + # Graph 2: (Nb test, nb failure)FuncvirtNet=f(time) + # *************************************** + new_element = [] + + for data in results: + onos_results = data['details']['FUNCvirNet']['status'] + nbFailures = 0 + for onos in onos_results: + if (onos['Case result'] == "FAIL"): + nbFailures += 1 + new_element.append({'x': data['creation_date'], + 'y1': len(onos_results), + 'y2': nbFailures}) + + test_data.append({'name': "ONOS FUNCvirNet nb tests/nb failures", + 'info': {'type': "graph", + 'xlabel': 'time', + 'y1label': 'Number of tests', + 'y2label': 'Number of failures'}, + 'data_set': new_element}) + + # Graph 3: (duration FUNCvirtNetL3)=f(time) + # *************************************** + new_element = [] + + # default duration 0:00:08.999904 + # consider only seconds => 09 + for data in results: + t = data['details']['FUNCvirNetL3']['duration'] + h, m, s = re.split(':', t) + s = round(float(s)) + new_duration = int(datetime.timedelta(hours=int(h), + minutes=int(m), + seconds=int(s)).total_seconds()) + new_element.append({'x': data['creation_date'], + 'y': new_duration}) + + test_data.append({'name': "ONOS FUNCvirNetL3 duration", + 'info': {'type': "graph", + 'xlabel': 'time (s)', + 'ylabel': 'duration (s)'}, + 'data_set': new_element}) + + # Graph 4: (Nb test, nb failure)FuncvirtNetL3=f(time) + # *************************************** + new_element = [] + + for data in results: + onos_results = data['details']['FUNCvirNetL3']['status'] + nbFailures = 0 + for onos in onos_results: + if (onos['Case result'] == "FAIL"): + nbFailures += 1 + new_element.append({'x': data['creation_date'], + 'y1': len(onos_results), + 'y2': nbFailures}) + + test_data.append({'name': "ONOS FUNCvirNetL3 nb tests/nb failures", + 'info': {'type': "graph", + 'xlabel': 'time', + 'y1label': 'Number of tests', + 'y2label': 'Number of failures'}, + 'data_set': new_element}) return test_data @@ -223,6 +343,33 @@ def format_Rally_for_dashboard(results): Post processing for the Rally test case """ test_data = [{'description': 'Rally results for Dashboard'}] + # Graph 1: Test_Duration = f(time) + # ******************************** + new_element = [] + for data in results: + summary_cursor = len(data) + new_element.append({'x': data['creation_date'], + 'y': int(data['details'][summary_cursor]['summary']['duration'])}) + + test_data.append({'name': "rally duration", + 'info': {'type': "graph", + 'xlabel': 'time', + 'ylabel': 'duration (s)'}, + 'data_set': new_element}) + + # Graph 2: Success rate = f(time) + # ******************************** + new_element = [] + for data in results: + new_element.append({'x': data['creation_date'], + 'y': float(data['details'][summary_cursor]['summary']['nb success'])}) + + test_data.append({'name': "rally success rate", + 'info': {'type': "graph", + 'xlabel': 'time', + 'ylabel': 'success rate (%)'}, + 'data_set': new_element}) + return test_data @@ -262,6 +409,7 @@ def format_vPing_for_dashboard(results): return test_data + def format_vPing_userdata_for_dashboard(results): """ Post processing for the vPing_userdata test case diff --git a/utils/test/result_collection_api/dashboard/promise2Dashboard.py b/utils/test/result_collection_api/dashboard/promise2Dashboard.py new file mode 100644 index 000000000..84f43a7d1 --- /dev/null +++ b/utils/test/result_collection_api/dashboard/promise2Dashboard.py @@ -0,0 +1,103 @@ + #!/usr/bin/python +# +# Copyright (c) 2015 Orange +# morgan.richomme@orange.com +# +# This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# This script is used to build dashboard ready json results +# It may be used for all the test case of the Promise project +# a new method format_<Test_case>_for_dashboard(results) +# v0.1: basic example with methods for odl, Tempest, Rally and vPing +# +import re +import datetime + + +def get_promise_cases(): + """ + get the list of the supported test cases + TODO: update the list when adding a new test case for the dashboard + """ + return ["promise"] + + +def format_promise_for_dashboard(case, results): + """ + generic method calling the method corresponding to the test case + check that the testcase is properly declared first + then build the call to the specific method + """ + if check_promise_case_exist(case): + # note we add _case because testcase and project had the same name + # TODO refactoring...looks fine at the beginning wit only 1 project + # not very ugly now and clearly not optimized... + cmd = "format_" + case + "_case_for_dashboard(results)" + res = eval(cmd) + else: + res = [] + print "Test cases not declared" + return res + + +def check_promise_case_exist(case): + """ + check if the testcase exists + if the test case is not defined or not declared in the list + return False + """ + promise_cases = get_promise_cases() + + if (case is None or case not in promise_cases): + return False + else: + return True + + + + + +def format_promise_case_for_dashboard(results): + """ + Post processing for the promise test case + """ + test_data = [{'description': 'Promise results for Dashboard'}] + # Graph 1: (duration)=f(time) + # *************************************** + new_element = [] + + # default duration 0:00:08.999904 + # consider only seconds => 09 + for data in results: + t = data['details']['duration'] + new_element.append({'x': data['creation_date'], + 'y': t}) + + test_data.append({'name': "Promise duration ", + 'info': {'type': "graph", + 'xlabel': 'time (s)', + 'ylabel': 'duration (s)'}, + 'data_set': new_element}) + + # Graph 2: (Nb test, nb failure)=f(time) + # *************************************** + new_element = [] + + for data in results: + promise_results = data['details'] + new_element.append({'x': data['creation_date'], + 'y1': promise_results['tests'], + 'y2': promise_results['failures']}) + + test_data.append({'name': "Promise nb tests/nb failures", + 'info': {'type': "graph", + 'xlabel': 'time', + 'y1label': 'Number of tests', + 'y2label': 'Number of failures'}, + 'data_set': new_element}) + + return test_data diff --git a/utils/test/result_collection_api/resources/handlers.py b/utils/test/result_collection_api/resources/handlers.py index be08c9791..6443fc417 100644 --- a/utils/test/result_collection_api/resources/handlers.py +++ b/utils/test/result_collection_api/resources/handlers.py @@ -552,7 +552,7 @@ class TestResultsHandler(GenericApiHandler): if period_arg > 0: period = datetime.now() - timedelta(days=period_arg) - obj = {"$gte": period} + obj = {"$gte": str(period)} get_request["creation_date"] = obj else: get_request["_id"] = result_id @@ -705,7 +705,7 @@ class DashboardHandler(GenericApiHandler): raise HTTPError(HTTP_BAD_REQUEST) if period_arg > 0: period = datetime.now() - timedelta(days=period_arg) - obj = {"$gte": period} + obj = {"$gte": str(period)} get_request["creation_date"] = obj else: get_request["_id"] = result_id diff --git a/utils/test/result_collection_api/tools/backup-db.sh b/utils/test/result_collection_api/tools/backup-db.sh index 8d12a6fc7..aa36aa370 100644 --- a/utils/test/result_collection_api/tools/backup-db.sh +++ b/utils/test/result_collection_api/tools/backup-db.sh @@ -1,5 +1,12 @@ #!/bin/bash - +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Orange and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## echo "Backup Test collection DB" now=$(date +"%m_%d_%Y_%H_%M_%S") echo $now diff --git a/utils/test/result_collection_api/tools/samples/sample.json.postman_collection b/utils/test/result_collection_api/tools/samples/sample.json.postman_collection index a9372624c..9ee35d15e 100644 --- a/utils/test/result_collection_api/tools/samples/sample.json.postman_collection +++ b/utils/test/result_collection_api/tools/samples/sample.json.postman_collection @@ -1156,4 +1156,4 @@ "responses": [] } ] -}
\ No newline at end of file +} |