diff options
24 files changed, 841 insertions, 73 deletions
diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh index e21387ac6..bbceb4b88 100755 --- a/jjb/apex/apex-deploy.sh +++ b/jjb/apex/apex-deploy.sh @@ -176,6 +176,8 @@ else # settings for bare metal deployment if [ "$IPV6_FLAG" == "True" ]; then NETWORK_FILE="/root/network/network_settings_v6.yaml" + elif [[ "$JOB_NAME" == *master* ]]; then + NETWORK_FILE="/root/network/network_settings-master.yaml" else NETWORK_FILE="/root/network/network_settings.yaml" fi diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 643972bd2..8bfc53d59 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -220,13 +220,13 @@ git-revision: false block: true same-node: true -# - trigger-builds: -# - project: 'functest-apex-{verify-slave}-suite-{stream}' -# predefined-parameters: | -# DEPLOY_SCENARIO=os-odl_l2-nofeature-ha -# FUNCTEST_SUITE_NAME=healthcheck -# block: true -# same-node: true + - trigger-builds: + - project: 'functest-apex-{verify-slave}-suite-{stream}' + predefined-parameters: | + DEPLOY_SCENARIO=os-odl_l2-nofeature-ha + FUNCTEST_SUITE_NAME=healthcheck + block: true + same-node: true - 'apex-workspace-cleanup' - job-template: @@ -949,11 +949,16 @@ - trigger: name: 'apex-master' triggers: - - timed: '0 0 20 8 *' + - timed: '0 3 * * 0' + - timed: '0 3 * * 4' - trigger: name: 'apex-colorado' triggers: - - timed: '0 3 * * *' + - timed: '0 3 * * 1' + - timed: '0 3 * * 2' + - timed: '0 3 * * 3' + - timed: '0 3 * * 5' + - timed: '0 3 * * 6' - trigger: name: 'apex-gs-clean-{stream}' triggers: diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index 2122959a9..b1cd9bf75 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -303,31 +303,31 @@ - trigger: name: 'fuel-os-odl_l2-nofeature-ha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 1' - trigger: name: 'fuel-os-nosdn-nofeature-ha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 2' - trigger: name: 'fuel-os-odl_l3-nofeature-ha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 3' - trigger: name: 'fuel-os-odl_l2-bgpvpn-ha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 4' - trigger: name: 'fuel-os-odl_l2-nofeature-noha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 5' - trigger: name: 'fuel-os-odl_l2-sfc-ha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 6' - trigger: name: 'fuel-os-odl_l2-sfc-noha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 7' #-------------------------------------------------------------------- # Enea Armband CI Virtual Triggers running against colorado branch #-------------------------------------------------------------------- diff --git a/jjb/armband/armband-deploy.sh b/jjb/armband/armband-deploy.sh index c8e58afa8..e0631c424 100755 --- a/jjb/armband/armband-deploy.sh +++ b/jjb/armband/armband-deploy.sh @@ -61,6 +61,10 @@ if [[ $LAB_CONFIG_URL =~ ^(git|ssh):// ]]; then fi fi +if [[ "$NODE_NAME" =~ "virtual" ]]; then + POD_NAME="virtual_kvm" +fi + # releng wants us to use nothing else but opnfv.iso for now. We comply. ISO_FILE=$WORKSPACE/opnfv.iso diff --git a/jjb/compass4nfv/compass-verify-jobs.yml b/jjb/compass4nfv/compass-verify-jobs.yml index 87c19a9c5..cc04ad3da 100644 --- a/jjb/compass4nfv/compass-verify-jobs.yml +++ b/jjb/compass4nfv/compass-verify-jobs.yml @@ -124,7 +124,7 @@ name: basic condition: SUCCESSFUL projects: - - name: 'compass-verify-basic-{stream}' + - name: 'opnfv-lint-verify-{stream}' current-parameters: true node-parameters: true kill-phase-on: FAILURE diff --git a/jjb/daisy4nfv/daisy4nfv-download-artifact.sh b/jjb/daisy4nfv/daisy4nfv-download-artifact.sh index 7ac76a5eb..90b5fa62f 100755 --- a/jjb/daisy4nfv/daisy4nfv-download-artifact.sh +++ b/jjb/daisy4nfv/daisy4nfv-download-artifact.sh @@ -11,26 +11,31 @@ set -o errexit set -o pipefail +# use proxy url to replace the nomral URL, for googleusercontent.com will be blocked randomly +[[ "$NODE_NAME" =~ (zte) ]] && GS_URL=$GS_BASE_PROXY + if [[ "$JOB_NAME" =~ "merge" ]]; then echo "Downloading http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties" # get the properties file for the Daisy4nfv BIN built for a merged change - curl -s -o $WORKSPACE/latest.properties http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties + curl -L -s -o $WORKSPACE/latest.properties http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties else # get the latest.properties file in order to get info regarding latest artifact echo "Downloading http://$GS_URL/latest.properties" - curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties + curl -L -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties fi # check if we got the file -[[ -f latest.properties ]] || exit 1 +[[ -f $WORKSPACE/latest.properties ]] || exit 1 # source the file so we get artifact metadata -source latest.properties +source $WORKSPACE/latest.properties # echo the info about artifact that is used during the deployment OPNFV_ARTIFACT=${OPNFV_ARTIFACT_URL/*\/} echo "Using $OPNFV_ARTIFACT for deployment" +[[ "$NODE_NAME" =~ (zte) ]] && OPNFV_ARTIFACT_URL=${GS_BASE_PROXY%%/*}/$OPNFV_ARTIFACT_URL + # log info to console echo "Downloading the $INSTALLER_TYPE artifact using URL http://$OPNFV_ARTIFACT_URL" echo "This could take some time..." diff --git a/jjb/escalator/escalator-basic.sh b/jjb/escalator/escalator-basic.sh new file mode 100755 index 000000000..9c739c422 --- /dev/null +++ b/jjb/escalator/escalator-basic.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo "--------------------------------------------------------" +echo "This is escalator basic job!" +echo "--------------------------------------------------------" + diff --git a/jjb/escalator/escalator-build.sh b/jjb/escalator/escalator-build.sh new file mode 100755 index 000000000..0e35c27d9 --- /dev/null +++ b/jjb/escalator/escalator-build.sh @@ -0,0 +1,33 @@ +#!/bin/bash +echo "--------------------------------------------------------" +echo "This is escalator build job!" +echo "--------------------------------------------------------" + +# set OPNFV_ARTIFACT_VERSION +if [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Building Escalator TAR for a merged change" + export OPNFV_ARTIFACT_VERSION="gerrit-$GERRIT_CHANGE_NUMBER" +else + export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") +fi + +# build output directory +OUTPUT_DIR=$WORKSPACE/build_output +mkdir -p $OUTPUT_DIR + +# start the build +cd $WORKSPACE +./ci/build.sh $OUTPUT_DIR $OPNFV_ARTIFACT_VERSION + +# save information regarding artifact into file +( + echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" + echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)" + echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)" + echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin" + echo "OPNFV_BUILD_URL=$BUILD_URL" +) > $WORKSPACE/opnfv.properties + +echo +echo "--------------------------------------------------------" +echo "Done!" diff --git a/jjb/escalator/escalator-upload-artifact.sh b/jjb/escalator/escalator-upload-artifact.sh new file mode 100755 index 000000000..781fb3e3e --- /dev/null +++ b/jjb/escalator/escalator-upload-artifact.sh @@ -0,0 +1,89 @@ +#!/bin/bash +echo "--------------------------------------------------------" +echo "This is escalator upload job!" +echo "--------------------------------------------------------" + +set -o pipefail + +# check if we built something +if [ -f $WORKSPACE/.noupload ]; then + echo "Nothing new to upload. Exiting." + /bin/rm -f $WORKSPACE/.noupload + exit 0 +fi + +# source the opnfv.properties to get ARTIFACT_VERSION +source $WORKSPACE/opnfv.properties + +importkey () { +# clone releng repository +echo "Cloning releng repository..." +[ -d releng ] && rm -rf releng +git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null +#this is where we import the siging key +if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then + source $WORKSPACE/releng/utils/gpg_import_key.sh +fi +} + +signtar () { +gpg2 -vvv --batch --yes --no-tty \ + --default-key opnfv-helpdesk@rt.linuxfoundation.org \ + --passphrase besteffort \ + --detach-sig $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz + +gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz.sig gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz.sig +echo "TAR signature Upload Complete!" +} + +uploadtar () { +# log info to console +echo "Uploading $INSTALLER_TYPE artifact. This could take some time..." +echo + +cd $WORKSPACE +# upload artifact and additional files to google storage +gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz > gsutil.tar.log 2>&1 +gsutil cp $WORKSPACE/opnfv.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 +if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then + gsutil cp $WORKSPACE/opnfv.properties \ + gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 +elif [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Uploaded Escalator TAR for a merged change" +fi + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/latest.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > /dev/null 2>&1 + +gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz > /dev/null 2>&1 + +# disabled errexit due to gsutil setmeta complaints +# BadRequestException: 400 Invalid argument +# check if we uploaded the file successfully to see if things are fine +gsutil ls gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading artifact!" + echo "Check log $WORKSPACE/gsutil.bin.log on the machine where this build is done." + exit 1 +fi + +echo "Done!" +echo +echo "--------------------------------------------------------" +echo +echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz" +echo +echo "--------------------------------------------------------" +echo +} + +importkey +signtar +uploadtar diff --git a/jjb/escalator/escalator.yml b/jjb/escalator/escalator.yml new file mode 100644 index 000000000..dba76f7c2 --- /dev/null +++ b/jjb/escalator/escalator.yml @@ -0,0 +1,326 @@ +- project: + name: 'escalator' + + project: 'escalator' +##################################### +# branch definitions +##################################### + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false +##################################### +# phases +##################################### + phase: + - 'basic': + slave-label: 'opnfv-build-centos' + - 'build': + slave-label: 'opnfv-build-centos' +##################################### +# jobs +##################################### + jobs: + - 'escalator-verify-{stream}' + - 'escalator-verify-{phase}-{stream}' + - 'escalator-merge-{stream}' + - 'escalator-merge-{phase}-{stream}' +##################################### +# job templates +##################################### +- job-template: + name: 'escalator-verify-{stream}' + + project-type: multijob + + disabled: false + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 4 + option: 'project' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + + triggers: + - gerrit: + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + readable-message: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-defaults' + - 'escalator-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: basic + condition: SUCCESSFUL + projects: + - name: 'escalator-verify-basic-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'escalator-verify-build-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + +- job-template: + name: 'escalator-verify-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - '{slave-label}-defaults' + - 'escalator-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-verify-{phase}-macro' + +- job-template: + name: 'escalator-merge-{stream}' + + project-type: multijob + + disabled: false + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 4 + option: 'project' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + + triggers: + - gerrit: + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + readable-message: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-defaults' + - 'escalator-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: basic + condition: SUCCESSFUL + projects: + - name: 'escalator-merge-basic-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'escalator-merge-build-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + +- job-template: + name: 'escalator-merge-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + wrappers: + - ssh-agent-credentials: + users: + - '{ssh-credentials}' + - timeout: + timeout: 360 + fail: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - '{slave-label}-defaults' + - 'escalator-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-merge-{phase}-macro' +##################################### +# builder macros +##################################### +- builder: + name: 'escalator-verify-basic-macro' + builders: + - shell: + !include-raw: ./escalator-basic.sh + +- builder: + name: 'escalator-verify-build-macro' + builders: + - shell: + !include-raw: ./escalator-build.sh + +- builder: + name: 'escalator-merge-basic-macro' + builders: + - shell: + !include-raw: ./escalator-basic.sh + +- builder: + name: 'escalator-merge-build-macro' + builders: + - shell: + !include-raw: + - ./escalator-build.sh + - ./escalator-upload-artifact.sh +##################################### +# parameter macros +##################################### +- parameter: + name: 'escalator-defaults' + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/$INSTALLER_TYPE + description: "Directory where the cache to be used during the build is located." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." diff --git a/jjb/fuel/fuel-download-artifact.sh b/jjb/fuel/fuel-download-artifact.sh index fd824ca44..2a0f09a3f 100755 --- a/jjb/fuel/fuel-download-artifact.sh +++ b/jjb/fuel/fuel-download-artifact.sh @@ -11,7 +11,7 @@ set -o errexit set -o pipefail # use proxy url to replace the nomral URL, for googleusercontent.com will be blocked randomly -[[ "$NODE_NAME" =~ (zte) ]] && GS_URL=$GS_BASE_PROXY +[[ "$NODE_NAME" =~ (zte) ]] && GS_URL=${GS_BASE_PROXY%%/*}/$GS_URL if [[ "$JOB_NAME" =~ "merge" ]]; then echo "Downloading http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties" @@ -59,7 +59,7 @@ echo "--------------------------------------------------------" echo # download the file -curl -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 +curl -L -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 # list the file ls -al $WORKSPACE/opnfv.iso diff --git a/jjb/infra/bifrost-verify-jobs.yml b/jjb/infra/bifrost-verify-jobs.yml index 751aa0c4a..be3cf9a0e 100644 --- a/jjb/infra/bifrost-verify-jobs.yml +++ b/jjb/infra/bifrost-verify-jobs.yml @@ -24,17 +24,24 @@ disabled: false dib-os-release: 'trusty' dib-os-element: 'ubuntu-minimal' - dib-os-packages: 'openssh-server,vlan,vim,less,bridge-utils,language-pack-en,iputils-ping,rsyslog,curl' + dib-os-packages: 'vlan,vim,less,bridge-utils,language-pack-en,iputils-ping,rsyslog,curl' + extra-dib-elements: 'openssh-server' - 'centos7': disabled: false dib-os-release: '7' dib-os-element: 'centos7' - dib-os-packages: 'openssh-server,vim,less,bridge-utils,iputils,rsyslog,curl' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' - 'suse': - disabled: true - dib-os-release: 'suse' - dib-os-element: 'suse' - dib-os-packages: '' + disabled: false + dib-os-release: '42.2' + dib-os-element: 'opensuse-minimal' + # python-xml is needed until https://review.openstack.org/#/c/400150/ is merged + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl,python-xml' + extra-dib-elements: 'openssh-server' + vm-disk: '30' + vm-memory: '4096' + vm-cpu: '2' #-------------------------------- # type #-------------------------------- @@ -45,6 +52,16 @@ #-------------------------------- jobs: - '{project}-bifrost-verify-{distro}-{type}-{stream}' + +#-------------------------------- +# VM defaults +#-------------------------------- +- defaults: + name: vm_defaults + vm-disk: '100' + vm-memory: '8192' + vm-cpu: '4' + #-------------------------------- # job templates #-------------------------------- @@ -53,6 +70,8 @@ disabled: '{obj:disabled}' + defaults: vm_defaults + concurrent: false properties: @@ -82,9 +101,21 @@ name: DIB_OS_ELEMENT default: '{dib-os-element}' - string: + name: EXTRA_DIB_ELEMENTS + default: '{extra-dib-elements}' + - string: name: DIB_OS_PACKAGES default: '{dib-os-packages}' - string: + name: VM_DISK + default: '{vm-disk}' + - string: + name: VM_MEMORY + default: '{vm-memory}' + - string: + name: VM_CPU + default: '{vm-cpu}' + - string: name: CLEAN_DIB_IMAGES default: 'true' - label: diff --git a/jjb/kvmfornfv/kvmfornfv-test.sh b/jjb/kvmfornfv/kvmfornfv-test.sh index f677f470d..b31d61cce 100755 --- a/jjb/kvmfornfv/kvmfornfv-test.sh +++ b/jjb/kvmfornfv/kvmfornfv-test.sh @@ -13,9 +13,16 @@ else exit 1 fi +echo $TEST_NAME + # do stuff differently based on the job type case "$JOB_TYPE" in - verify|daily) + verify) + #start the test + cd $WORKSPACE + ./ci/test_kvmfornfv.sh $JOB_TYPE + ;; + daily) #start the test cd $WORKSPACE ./ci/test_kvmfornfv.sh $JOB_TYPE $TEST_NAME diff --git a/jjb/kvmfornfv/kvmfornfv.yml b/jjb/kvmfornfv/kvmfornfv.yml index 2c8446e07..04fce221c 100644 --- a/jjb/kvmfornfv/kvmfornfv.yml +++ b/jjb/kvmfornfv/kvmfornfv.yml @@ -112,8 +112,6 @@ node-parameters: false kill-phase-on: FAILURE abort-all-job: true - - - job-template: name: 'kvmfornfv-verify-{phase}-{stream}' @@ -271,7 +269,7 @@ name: build condition: SUCCESSFUL projects: - - name: 'kvmfornfv-Packet_Forwarding-daily-build-{stream}' + - name: 'kvmfornfv-Packet_Forwarding-daily-test-{stream}' current-parameters: false node-parameters: false git-revision: true @@ -316,21 +314,21 @@ # parameter macros ######################## - parameter: - name: 'kvmfornfv-Idle_Idle-daily-parameter' + name: 'kvmfornfv-Idle_Idle-daily-test-{stream}' parameters: - string: name: TEST_NAME default: 'idle_idle' description: "Daily job to run cyclictest without applying any stress" - parameter: - name: 'kvmfornfv-Stress_Idle-daily-parameter' + name: 'kvmfornfv-Stress_Idle-daily-test-{stream}' parameters: - string: name: TEST_NAME default: 'stress_idle' description: "Daily job to run cyclictest with stress applied" - parameter: - name: 'kvmfornfv-Packet_Forwarding-daily-parameter' + name: 'kvmfornfv-Packet_Forwarding-daily-test-{stream}' parameters: - string: name: TEST_NAME diff --git a/jjb/multisite/multisite-daily-jobs.yml b/jjb/multisite/multisite-daily-jobs.yml new file mode 100644 index 000000000..cfb40a1fa --- /dev/null +++ b/jjb/multisite/multisite-daily-jobs.yml @@ -0,0 +1,247 @@ +- project: + name: kingbird + + project: 'multisite' + + jobs: + - 'multisite-kingbird-virtual-daily-{stream}' + - 'multisite-{phase}-{stream}' + + phase: + - 'fuel-deploy-regionone-virtual' + - 'fuel-deploy-regiontwo-virtual' + - 'register-endpoints' + - 'update-auth' + - 'kingbird-deploy-virtual' + - 'kingbird-functest' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + timed: '#@midnight' + +- job-template: + name: 'multisite-kingbird-virtual-daily-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: false + + parameters: + - project-parameter: + project: '{project}' + - string: + name: KINGBIRD_LOG_FILE + default: $WORKSPACE/kingbird.log + - 'multisite-virtual-defaults' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-multisite-noha' + + triggers: + - timed: '{timed}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: fuel-deploy-virtual + condition: SUCCESSFUL + projects: + - name: 'multisite-fuel-deploy-regionone-virtual-{stream}' + current-parameters: false + predefined-parameters: | + FUEL_VERSION=latest + DEPLOY_SCENARIO=$DEPLOY_SCENARIO + OS_REGION=RegionOne + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual2 + kill-phase-on: FAILURE + abort-all-job: true + - name: 'multisite-fuel-deploy-regiontwo-virtual-{stream}' + current-parameters: false + predefined-parameters: | + FUEL_VERSION=latest + DEPLOY_SCENARIO=$DEPLOY_SCENARIO + OS_REGION=RegionTwo + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual6 + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: centralize-keystone + condition: SUCCESSFUL + projects: + - name: 'multisite-register-endpoints-{stream}' + current-parameters: false + predefined-parameters: | + OS_REGION=RegionOne + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual2 + kill-phase-on: FAILURE + abort-all-job: true + - name: 'multisite-update-auth-{stream}' + current-parameters: false + predefined-parameters: | + OS_REGION=RegionTwo + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual6 + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: kingbird-deploy-virtual + condition: SUCCESSFUL + projects: + - name: 'multisite-kingbird-deploy-virtual-{stream}' + current-parameters: false + predefined-parameters: | + OS_REGION=RegionOne + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual2 + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: kingbird-functest + condition: SUCCESSFUL + projects: + - name: 'multisite-kingbird-functest-{stream}' + current-parameters: false + predefined-parameters: | + OS_REGION=RegionOne + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual2 + kill-phase-on: NEVER + abort-all-job: false + +- job-template: + name: 'multisite-{phase}-{stream}' + + concurrent: false + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - 'multisite-{phase}-builder': + stream: '{stream}' + + publishers: + - 'multisite-{phase}-publisher' + +######################## +# builder macros +######################## +- builder: + name: 'multisite-fuel-deploy-regionone-virtual-builder' + builders: + - shell: | + #!/bin/bash + + echo "This is where we deploy fuel, extract passwords and save into file" +- builder: + name: 'multisite-fuel-deploy-regiontwo-virtual-builder' + builders: + - shell: | + #!/bin/bash + + echo "This is where we deploy fuel, extract publicUrl, privateUrl, and adminUrl and save into file" +- builder: + name: 'multisite-register-endpoints-builder' + builders: + - copyartifact: + project: 'multisite-fuel-deploy-regiontwo-virtual-{stream}' + which-build: multijob-build + filter: "RegionTwo-Endpoints.txt" + - shell: | + #!/bin/bash + + echo "This is where we register RegionTwo in RegionOne keystone" +- builder: + name: 'multisite-update-auth-builder' + builders: + - copyartifact: + project: 'multisite-fuel-deploy-regionone-virtual-{stream}' + which-build: multijob-build + filter: "RegionOne-Passwords.txt" + - shell: | + #!/bin/bash + + echo "This is where we read passwords from RegionOne-passwords.txt and replace passwords in RegionTwo" +- builder: + name: 'multisite-kingbird-deploy-virtual-builder' + builders: + - shell: | + #!/bin/bash + + echo "This is where we install kingbird" +- builder: + name: 'multisite-kingbird-functest-builder' + builders: + - shell: | + #!/bin/bash + + echo "This is where we run kingbird-functest" +######################## +# publisher macros +######################## +- publisher: + name: 'multisite-fuel-deploy-regionone-virtual-publisher' + publishers: + - archive: + artifacts: '/root/servicepass.ini' + allow-empty: false + only-if-success: true + fingerprint: true +- publisher: + name: 'multisite-fuel-deploy-regiontwo-virtual-publisher' + publishers: + - archive: + artifacts: '/root/endpoints.ini' + allow-empty: false + only-if-success: true + fingerprint: true +- publisher: + name: 'multisite-register-endpoints-publisher' + publishers: + - archive: + artifacts: 'dummy.txt' + allow-empty: true +- publisher: + name: 'multisite-update-auth-publisher' + publishers: + - archive: + artifacts: 'dummy.txt' + allow-empty: true +- publisher: + name: 'multisite-kingbird-deploy-virtual-publisher' + publishers: + - archive: + artifacts: 'dummy.txt' + allow-empty: true +- publisher: + name: 'multisite-kingbird-functest-publisher' + publishers: + - archive: + artifacts: 'dummy.txt' + allow-empty: true diff --git a/jjb/opnfv/opnfv-docs.yml b/jjb/opnfv/opnfv-docs.yml index 307c1db44..442e8ea71 100644 --- a/jjb/opnfv/opnfv-docs.yml +++ b/jjb/opnfv/opnfv-docs.yml @@ -19,7 +19,7 @@ disabled: false - colorado: branch: 'stable/{stream}' - doc-version: '2.0' + doc-version: '3.0' gs-pathname: '/{stream}/{doc-version}' disabled: false diff --git a/jjb/opnfv/opnfv-lint.yml b/jjb/opnfv/opnfv-lint.yml index 6860dd037..652f28292 100644 --- a/jjb/opnfv/opnfv-lint.yml +++ b/jjb/opnfv/opnfv-lint.yml @@ -55,7 +55,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'functest|sdnvpn|qtip|daisy|sfc' + project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' diff --git a/jjb/opnfv/slave-params.yml b/jjb/opnfv/slave-params.yml index b46960fa7..dd0c97996 100644 --- a/jjb/opnfv/slave-params.yml +++ b/jjb/opnfv/slave-params.yml @@ -660,6 +660,16 @@ name: SSH_KEY default: /root/.ssh/id_rsa description: 'SSH key to be used' +- parameter: + name: 'multisite-virtual-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'multisite-virtual' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' ##################################################### # These slaves are just dummy slaves for sandbox jobs ##################################################### diff --git a/jjb/qtip/qtip-project-jobs.yml b/jjb/qtip/qtip-project-jobs.yml index 722a9beb3..f192e8679 100644 --- a/jjb/qtip/qtip-project-jobs.yml +++ b/jjb/qtip/qtip-project-jobs.yml @@ -59,4 +59,9 @@ builders: - shell: | - echo "Nothing to verify!" + #!/bin/bash + set -o errexit + set -o pipefail + set -o xtrace + + tox diff --git a/jjb/yardstick/yardstick-project-jobs.yml b/jjb/yardstick/yardstick-project-jobs.yml index 0cc6cd606..abc6eeb4b 100644 --- a/jjb/yardstick/yardstick-project-jobs.yml +++ b/jjb/yardstick/yardstick-project-jobs.yml @@ -122,7 +122,7 @@ # install python packages easy_install -U setuptools easy_install -U pip - pip install -r requirements.txt + pip install -r requirements.txt || pip install -r tests/ci/requirements.txt pip install -e . # unit tests diff --git a/prototypes/bifrost/playbooks/test-bifrost-infracloud.yaml b/prototypes/bifrost/playbooks/test-bifrost-infracloud.yaml index a0bc28506..541a1f7d4 100644 --- a/prototypes/bifrost/playbooks/test-bifrost-infracloud.yaml +++ b/prototypes/bifrost/playbooks/test-bifrost-infracloud.yaml @@ -53,7 +53,8 @@ dib_imagename: "{{deploy_image}}" dib_os_element: "{{ lookup('env','DIB_OS_ELEMENT') }}" dib_os_release: "{{ lookup('env', 'DIB_OS_RELEASE') }}" - dib_elements: "vm serial-console simple-init devuser infra-cloud-bridge puppet growroot {{ extra_dib_elements|default('') }}" + extra_dib_elements: "{{ lookup('env', 'EXTRA_DIB_ELEMENTS') | default('') }}" + dib_elements: "vm serial-console simple-init devuser infra-cloud-bridge puppet growroot {{ extra_dib_elements }}" dib_packages: "{{ lookup('env', 'DIB_OS_PACKAGES') }}" when: create_image_via_dib | bool == true and transform_boot_image | bool == false environment: diff --git a/prototypes/bifrost/scripts/test-bifrost-deployment.sh b/prototypes/bifrost/scripts/test-bifrost-deployment.sh index 030faedcf..63d851438 100755 --- a/prototypes/bifrost/scripts/test-bifrost-deployment.sh +++ b/prototypes/bifrost/scripts/test-bifrost-deployment.sh @@ -34,13 +34,13 @@ PROVISION_WAIT_TIMEOUT=${PROVISION_WAIT_TIMEOUT:-2400} TEST_VM_NUM_NODES=3 export TEST_VM_NODE_NAMES="jumphost.opnfvlocal controller00.opnfvlocal compute00.opnfvlocal" export VM_DOMAIN_TYPE="kvm" -export VM_CPU=4 -export VM_DISK=100 +export VM_CPU=${VM_CPU:-4} +export VM_DISK=${VM_DISK:-100} TEST_PLAYBOOK="test-bifrost-infracloud.yaml" USE_INSPECTOR=true USE_CIRROS=false TESTING_USER=root -VM_MEMORY_SIZE="8192" +VM_MEMORY_SIZE=${VM_MEMORY_SIZE:-8192} DOWNLOAD_IPA=true CREATE_IPA_IMAGE=false INSPECT_NODES=true @@ -65,6 +65,9 @@ export DIB_OS_ELEMENT=${DIB_OS_ELEMENT:-ubuntu-minimal} # for centos 7: "openssh-server,vim,less,bridge-utils,iputils,rsyslog,curl" export DIB_OS_PACKAGES=${DIB_OS_PACKAGES:-"openssh-server,vlan,vim,less,bridge-utils,language-pack-en,iputils-ping,rsyslog,curl"} +# Additional dib elements +export EXTRA_DIB_ELEMENTS=${EXTRA_DIB_ELEMENTS:-} + # Source Ansible # NOTE(TheJulia): Ansible stable-1.9 source method tosses an error deep # under the hood which -x will detect, so for this step, we need to suspend diff --git a/utils/jenkins-jnlp-connect.sh b/utils/jenkins-jnlp-connect.sh index 9ef4298ef..56ce61e17 100755 --- a/utils/jenkins-jnlp-connect.sh +++ b/utils/jenkins-jnlp-connect.sh @@ -30,8 +30,6 @@ EOF } main () { - dir=$(cd $(dirname $0); pwd) - #tests if [[ -z $jenkinsuser || -z $jenkinshome ]]; then echo "jenkinsuser or home not defined, please edit this file to define it" @@ -60,17 +58,6 @@ main () { fi fi - - if [ -d /etc/monit/conf.d ]; then - monitconfdir="/etc/monit/conf.d/" - elif [ -d /etc/monit.d ]; then - monitconfdir="/etc/monit.d" - else - echo "Could not determine the location of the monit configuration file." - echo "Make sure monit is installed." - exit 1 - fi - #make pid dir pidfile="/var/run/$jenkinsuser/jenkins_jnlp_pid" if ! [ -d /var/run/$jenkinsuser/ ]; then @@ -94,29 +81,39 @@ main () { exit 1 fi fi - fi - makemonit () { - echo "Writing the following as monit config:" + if [ -d /etc/monit/conf.d ]; then + monitconfdir="/etc/monit/conf.d/" + elif [ -d /etc/monit.d ]; then + monitconfdir="/etc/monit.d" + else + echo "Could not determine the location of the monit configuration file." + echo "Make sure monit is installed." + exit 1 + fi + + makemonit () { + echo "Writing the following as monit config:" cat << EOF | tee $monitconfdir/jenkins check process jenkins with pidfile /var/run/$jenkinsuser/jenkins_jnlp_pid -start program = "/usr/bin/sudo -u $jenkinsuser /bin/bash -c 'cd $dir; export started_monit=true; $0 $@' with timeout 60 seconds" +start program = "/usr/bin/sudo -u $jenkinsuser /bin/bash -c 'cd $jenkinshome; export started_monit=true; $0 $@' with timeout 60 seconds" stop program = "/bin/bash -c '/bin/kill \$(/bin/cat /var/run/$jenkinsuser/jenkins_jnlp_pid)'" EOF - } + } - if [[ -f $monitconfdir/jenkins ]]; then - #test for diff - if [[ "$(diff $monitconfdir/jenkins <(echo "\ + if [[ -f $monitconfdir/jenkins ]]; then + #test for diff + if [[ "$(diff $monitconfdir/jenkins <(echo "\ check process jenkins with pidfile /var/run/$jenkinsuser/jenkins_jnlp_pid -start program = \"/usr/bin/sudo -u $jenkinsuser /bin/bash -c 'cd $dir; export started_monit=true; $0 $@' with timeout 60 seconds\" +start program = \"/usr/bin/sudo -u $jenkinsuser /bin/bash -c 'cd $jenkinshome; export started_monit=true; $0 $@' with timeout 60 seconds\" stop program = \"/bin/bash -c '/bin/kill \$(/bin/cat /var/run/$jenkinsuser/jenkins_jnlp_pid)'\"\ ") )" ]]; then - echo "Updating monit config..." + echo "Updating monit config..." + makemonit $@ + fi + else makemonit $@ fi - else - makemonit $@ fi if [[ $started_monit == "true" ]]; then diff --git a/utils/test/reporting/functest/reporting-status.py b/utils/test/reporting/functest/reporting-status.py index cb13b099c..66bdd57c1 100755 --- a/utils/test/reporting/functest/reporting-status.py +++ b/utils/test/reporting/functest/reporting-status.py @@ -55,25 +55,25 @@ logger.info("*******************************************") # Retrieve test cases of Tier 1 (smoke) config_tiers = functest_yaml_config.get("tiers") -# we consider Tier 1 (smoke),2 (sdn suites) and 3 (features) +# we consider Tier 1 (smoke),2 (features) # to validate scenarios # Tier > 4 are not used to validate scenarios but we display the results anyway # tricky thing for the API as some tests are Functest tests # other tests are declared directly in the feature projects for tier in config_tiers: - if tier['order'] > 0 and tier['order'] < 3: + if tier['order'] > 0 and tier['order'] < 2: for case in tier['testcases']: if case['name'] not in blacklist: testValid.append(tc.TestCase(case['name'], "functest", case['dependencies'])) - elif tier['order'] == 3: + elif tier['order'] == 2: for case in tier['testcases']: if case['name'] not in blacklist: testValid.append(tc.TestCase(case['name'], case['name'], case['dependencies'])) - elif tier['order'] > 3: + elif tier['order'] > 2: for case in tier['testcases']: if case['name'] not in blacklist: otherTestCases.append(tc.TestCase(case['name'], |