diff options
33 files changed, 1505 insertions, 976 deletions
diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index a5cc4798a..be595d4a3 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -58,6 +58,15 @@ branches: - branch-compare-type: 'ANT' branch-pattern: '**/master' + file-paths: + - compare-type: ANT + pattern: 'ci/**' + - compare-type: ANT + pattern: 'build/**' + - compare-type: ANT + pattern: 'lib/**' + - compare-type: ANT + pattern: 'config/**' forbidden-file-paths: - compare-type: ANT pattern: 'docs/**' @@ -67,6 +76,8 @@ use-build-blocker: true blocking-jobs: - "apex-daily.*" + - "apex-deploy.*" + - "apex-build.*" builders: - 'apex-build' @@ -153,6 +164,12 @@ refspec: '' branch: '{branch}' + properties: + - build-blocker: + use-build-blocker: true + blocking-jobs: + - "apex-deploy.*" + builders: - 'apex-build' - 'apex-deploy-virtual' @@ -183,6 +200,14 @@ - apex-parameter: gs-pathname: '{gs-pathname}' + properties: + - build-blocker: + use-build-blocker: true + blocking-jobs: + - "apex-verify.*" + - "apex-deploy.*" + - "apex-build.*" + builders: - 'apex-deploy-virtual' - 'apex-workspace-cleanup' @@ -211,6 +236,15 @@ - apex-parameter: gs-pathname: '{gs-pathname}' + properties: + - build-blocker: + use-build-blocker: true + blocking-jobs: + - "apex-verify.*" + - "apex-deploy.*" + - "apex-build.*" + - "apex-daily.*" + builders: - 'apex-deploy-baremetal' - 'apex-workspace-cleanup' @@ -243,7 +277,9 @@ - build-blocker: use-build-blocker: true blocking-jobs: - - "apex-daily.*" + - "apex-verify.*" + - "apex-deploy.*" + - "apex-build.*" triggers: - 'apex-{stream}' @@ -254,7 +290,7 @@ git-revision: true block: true - trigger-builds: - - project: 'apex-deploy-baremetal-{stream}' + - project: 'apex-deploy-virtual-{stream}' git-revision: true block: true - trigger-builds: diff --git a/jjb/bottlenecks/bottlenecks.yml b/jjb/bottlenecks/bottlenecks.yml index a65341809..828fd214b 100644 --- a/jjb/bottlenecks/bottlenecks.yml +++ b/jjb/bottlenecks/bottlenecks.yml @@ -221,7 +221,8 @@ sed 's/.*href="//' | sed 's/".*//' | grep '^[a-zA-Z].*'); do - curl --connect-timeout 10 -o $CACHE_DIR/$file $PACKAGE_URL$file + curl --connect-timeout 10 -o $CACHE_DIR/$file $PACKAGE_URL$file -v + echo "bottlenecks: copy file $CACHE_DIR/$file to $SUITE_URL" gsutil cp $CACHE_DIR/$file $SUITE_URL done diff --git a/jjb/compass4nfv/compass-build.sh b/jjb/compass4nfv/compass-build.sh new file mode 100644 index 000000000..85b3ab47d --- /dev/null +++ b/jjb/compass4nfv/compass-build.sh @@ -0,0 +1,41 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# log info to console +echo "Starting the build of $INSTALLER_TYPE. This could take some time..." +echo "--------------------------------------------------------" +echo + +# create the cache directory if it doesn't exist +[[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY +[[ -d $BUILD_DIRECTORY ]] || mkdir -p $BUILD_DIRECTORY + +# set OPNFV_ARTIFACT_VERSION +export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") + +# start the build +if [ -d $PPA_CACHE ] +then + cp $PPA_CACHE/*.tar.gz $PPA_CACHE/*.iso $PPA_CACHE/*.img $CACHE_DIRECTORY/ -f +fi + +cd $WORKSPACE/ +./build.sh --iso-dir $BUILD_DIRECTORY/ --iso-name compass.iso -c $CACHE_DIRECTORY + +# list the build artifacts +ls -al $BUILD_DIRECTORY + +# save information regarding artifact into file +( + echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" + echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)" + echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)" + echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" + echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/compass.iso | cut -d' ' -f1)" + echo "OPNFV_BUILD_URL=$BUILD_URL" +) > $BUILD_DIRECTORY/opnfv.properties +echo +echo "--------------------------------------------------------" +echo "Done!"
\ No newline at end of file diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml new file mode 100644 index 000000000..693e1b0cd --- /dev/null +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -0,0 +1,240 @@ +- project: + + name: compass-ci + + installer: 'compass' + + project: 'compass4nfv' + + master: &master + stream: master + branch: '{stream}' + gs-pathname: '' + brahmaputra: &brahmaputra + stream: brahmaputra + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + +#-------------------------------- +# POD, INSTALLER, AND BRANCH MAPPING +#-------------------------------- + pod: +#-------------------------------- +# brahmaputra +#-------------------------------- + - huawei-us-deploy-bare-1: + <<: *brahmaputra +#-------------------------------- +# master +#-------------------------------- + - intel-pod8: + <<: *master + + sdn-controller: + - 'nosdn': + disabled: false + - 'odl': + disabled: false + - 'onos': + disabled: false +# - 'opencontrail': +# disabled: true + + jobs: + - 'compass-{sdn-controller}-{pod}-daily-{stream}' + - 'compass-deploy-{pod}-daily-{stream}' + +######################## +# job templates +######################## +- job-template: + name: 'compass-{sdn-controller}-{pod}-daily-{stream}' + + project-type: multijob + + node: huawei-build + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + + wrappers: + - build-name: + name: '$BUILD_NUMBER - SDN: $SDN_CONTROLLER Scenario: $DEPLOY_SCENARIO' + + parameters: + - project-parameter: + project: '{project}' + - compass-ci-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + - string: + name: SDN_CONTROLLER + default: '{sdn-controller}' + - string: + name: DEPLOY_SCENARIO + default: 'none' + - 'huawei-build-defaults' + - '{installer}-defaults' + + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + triggers: + - 'compass-{pod}-{sdn-controller}-trigger' + + builders: + - multijob: + name: deploy + condition: SUCCESSFUL + projects: + - name: 'compass-deploy-{pod}-daily-{stream}' + current-parameters: true + git-revision: true + kill-phase-on: FAILURE + - multijob: + name: functest + condition: COMPLETED + projects: + - name: 'functest-compass-{pod}-daily-{stream}' + current-parameters: true + kill-phase-on: NEVER + - multijob: + name: yardstick + condition: COMPLETED + projects: + - name: 'yardstick-compass-{pod}-daily-{stream}' + current-parameters: true + kill-phase-on: NEVER + +- job-template: + name: 'compass-deploy-{pod}-daily-{stream}' + + disabled: false + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + + parameters: + - project-parameter: + project: '{project}' + - compass-ci-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + - string: + name: SDN_CONTROLLER + default: '{sdn-controller}' + - string: + name: DEPLOY_SCENARIO + default: 'none' + - '{pod}-defaults' + - '{installer}-defaults' + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + wrappers: + - build-name: + name: '$BUILD_NUMBER - SDN: $SDN_CONTROLLER Scenario: $DEPLOY_SCENARIO' + + + builders: + - shell: + !include-raw ./compass-workspace-precleanup.sh + - shell: + !include-raw ./compass-download-artifact.sh + - shell: + !include-raw ./compass-deploy-bare.sh + + publishers: + - archive: + artifacts: 'ansible.log' + allow-empty: 'true' + fingerprint: true + +######################## +# parameter macros +######################## +- parameter: + name: compass-ci-parameter + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/$PROJECT + description: "Directory where the cache to be used during the build is located." + - string: + name: GS_URL + default: '$GS_BASE{gs-pathname}' + description: "URL to Google Storage." + - string: + name: PPA_REPO + default: "http://205.177.226.235:9999" + - string: + name: PPA_CACHE + default: "$WORKSPACE/work/repo/" + - choice: + name: COMPASS_OPENSTACK_VERSION + choices: + - 'liberty' + - 'juno' + - choice: + name: COMPASS_OS_VERSION + choices: + - 'trusty' + - 'centos7' + +######################## +# trigger macros +######################## +- trigger: + name: 'compass-huawei-us-deploy-bare-1-nosdn-trigger' + triggers: + - timed: '0 16 * * * ' +- trigger: + name: 'compass-huawei-us-deploy-bare-1-odl-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-huawei-us-deploy-bare-1-onos-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-huawei-us-deploy-bare-1-opencontrail-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-intel-pod8-nosdn-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-intel-pod8-odl-trigger' + triggers: + - timed: '0 16 * * *' +- trigger: + name: 'compass-intel-pod8-onos-trigger' + triggers: + - timed: '' +- trigger: + name: 'compass-intel-pod8-opencontrail-trigger' + triggers: + - timed: '' diff --git a/jjb/compass4nfv/compass-deploy-bare.sh b/jjb/compass4nfv/compass-deploy-bare.sh new file mode 100644 index 000000000..12b9b337b --- /dev/null +++ b/jjb/compass4nfv/compass-deploy-bare.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -x + +# log info to console +echo "Starting the deployment on baremetal environment using $INSTALLER_TYPE. This could take some time..." +echo "--------------------------------------------------------" +echo + +export CONFDIR=$WORKSPACE/deploy/conf/hardware_environment/huawei_us_lab/pod1 +export ISO_URL=file://$BUILD_DIRECTORY/compass.iso +export INSTALL_NIC=eth0 + +cd $WORKSPACE + +export OS_VERSION=${{COMPASS_OS_VERSION}} +export OPENSTACK_VERSION=${{COMPASS_OPENSTACK_VERSION}} +./deploy.sh --dha $CONFDIR/dha.yml --network $CONFDIR/network.yml +if [ $? -ne 0 ]; then + echo "depolyment failed!" + deploy_ret=1 +fi + +echo +echo "--------------------------------------------------------" +echo "Done!" + +ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" +sshpass -p root scp 2>/dev/null $ssh_options root@${{INSTALLER_IP}}:/var/ansible/run/openstack_${{COMPASS_OPENSTACK_VERSION}}-opnfv2/ansible.log ./ &> /dev/null + +exit $deploy_ret diff --git a/jjb/compass4nfv/compass-deploy-virtual.sh b/jjb/compass4nfv/compass-deploy-virtual.sh new file mode 100644 index 000000000..b35658bd7 --- /dev/null +++ b/jjb/compass4nfv/compass-deploy-virtual.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -x + +# log info to console +echo "Starting the deployment on virtual environment using $INSTALLER_TYPE. This could take some time..." +echo "--------------------------------------------------------" +echo + +export ISO_URL=file://$BUILD_DIRECTORY/compass.iso +export OS_VERSION=${{COMPASS_OS_VERSION}} +export OPENSTACK_VERSION=${{COMPASS_OPENSTACK_VERSION}} +export CONFDIR=$WORKSPACE/deploy/conf/vm_environment +./deploy.sh --dha $CONFDIR/$SDN_CONTROLLER.yml --network $CONFDIR/$NODE_NAME/network.yml +if [ $? -ne 0 ]; then + echo "depolyment failed!" + deploy_ret=1 +fi + +echo +echo "--------------------------------------------------------" +echo "Done!" + +ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" +sshpass -p root scp 2>/dev/null $ssh_options root@${{INSTALLER_IP}}:/var/ansible/run/openstack_${{COMPASS_OPENSTACK_VERSION}}-opnfv2/ansible.log ./ &> /dev/null + +exit $deploy_ret diff --git a/jjb/compass4nfv/compass-download-artifact.sh b/jjb/compass4nfv/compass-download-artifact.sh new file mode 100644 index 000000000..5a63c4aeb --- /dev/null +++ b/jjb/compass4nfv/compass-download-artifact.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# log info to console +echo "Downloading the $INSTALLER_TYPE artifact. This could take some time..." +echo "--------------------------------------------------------" +echo + +# get the latest.properties file in order to get info regarding latest artifact +[[ -d $BUILD_DIRECTORY ]] || mkdir -p $BUILD_DIRECTORY +curl -s -o $BUILD_DIRECTORY/latest.properties http://$GS_URL/latest.properties + +# check if we got the file +[[ -f $BUILD_DIRECTORY/latest.properties ]] || exit 1 + +# source the file so we get OPNFV vars +source $BUILD_DIRECTORY/latest.properties + +# download the file +curl -s -o $BUILD_DIRECTORY/compass.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 + +# list the file +ls -al $BUILD_DIRECTORY/compass.iso + +echo +echo "--------------------------------------------------------" +echo "Done!"
\ No newline at end of file diff --git a/jjb/compass4nfv/compass-makeppa.sh b/jjb/compass4nfv/compass-makeppa.sh new file mode 100644 index 000000000..83cc059c0 --- /dev/null +++ b/jjb/compass4nfv/compass-makeppa.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -x +set -o errexit +set -o nounset +set -o pipefail +# make ppa +cd $WORKSPACE/ +./build/make_repo.sh +# calc MD5 of ppa +cd $PPA_CACHE +for i in $(find *.gz *.iso *.img -type f) +do + md5=$(md5sum $i | cut -d ' ' -f1) + echo $md5 > $i.md5 + curl -T $i $PPA_REPO + curl -T $i.md5 $PPA_REPO +done
\ No newline at end of file diff --git a/jjb/compass4nfv/compass-project-jobs.yml b/jjb/compass4nfv/compass-project-jobs.yml new file mode 100644 index 000000000..1395fa532 --- /dev/null +++ b/jjb/compass4nfv/compass-project-jobs.yml @@ -0,0 +1,304 @@ +- project: + + name: compass-project + + installer: 'compass' + + project: 'compass4nfv' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + - brahmaputra: + branch: 'stable/{stream}' + gs-pathname: '/{stream}' + + + sdn-controller: + - 'nosdn': + disabled: false + node: huawei-deploy-vm + - 'odl': + disabled: false + node: huawei-deploy-vm + - 'onos': + disabled: false + node: huawei-deploy-vm +# - 'opencontrail': +# disabled: true +# node: huawei-deploy-vm + + jobs: + - 'compass-verify-{stream}' + - 'compass-build-iso-{stream}' + - 'compass-build-ppa-{stream}' + - 'compass-deploy-virtual-{sdn-controller}-{stream}' + - 'compass-virtual-daily-{stream}' + +######################## +# job templates +######################## +- job-template: + name: 'compass-verify-{stream}' + + disabled: false + + node: 'huawei-deploy-vm' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 3 + max-per-node: 1 + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - compass-project-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + - '{node}-defaults' + - '{installer}-defaults' + + scm: + - gerrit-trigger-scm: + credentials-id: '{ssh-credentials}' + refspec: '$GERRIT_REFSPEC' + choosing-strategy: 'gerrit' + + triggers: + - gerrit: + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + + builders: + - shell: + !include-raw ./compass-build.sh + - shell: + !include-raw ./compass-deploy-virtual.sh + + publishers: + - archive: + artifacts: 'ansible.log' + allow-empty: 'true' + fingerprint: true + +- job-template: + name: 'compass-virtual-daily-{stream}' + + node: huawei-build + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + + parameters: + - project-parameter: + project: '{project}' + - compass-project-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + - '{node}-defaults' + - '{installer}-defaults' + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + triggers: + - timed: 'H 16 * * *' + + builders: + - trigger-builds: + - project: 'compass-build-iso-{stream}' + git-revision: true + block: true + - trigger-builds: + - project: 'compass-deploy-virtual-nosdn-{stream}' + git-revision: true + block: false + - trigger-builds: + - project: 'compass-deploy-virtual-odl-{stream}' + git-revision: true + block: false + - trigger-builds: + - project: 'compass-deploy-virtual-onos-{stream}' + git-revision: true + block: false + +- job-template: + name: 'compass-build-iso-{stream}' + + node: ericsson-build + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + + parameters: + - project-parameter: + project: '{project}' + - compass-project-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + - '{node}-defaults' + - '{installer}-defaults' + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + builders: + - shell: + !include-raw ./compass-build.sh + - shell: + !include-raw ./compass-upload-artifact.sh + - shell: + !include-raw ./compass-workspace-cleanup.sh + +- job-template: + name: 'compass-build-ppa-{stream}' + + description: "build ppa(using docker) in huawei lab" + + node: huawei-build + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + + parameters: + - project-parameter: + project: '{project}' + - compass-project-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + - '{node}-defaults' + - '{installer}-defaults' + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + builders: + - shell: + !include-raw ./compass-makeppa.sh + +- job-template: + name: 'compass-deploy-virtual-{sdn-controller}-{stream}' + + disabled: false + + node: '{node}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + + parameters: + - project-parameter: + project: '{project}' + - compass-project-parameter: + installer: '{installer}' + gs-pathname: '{gs-pathname}' + - string: + name: SDN_CONTROLLER + default: '{sdn-controller}' + - string: + name: DEPLOY_SCENARIO + default: 'none' + - '{node}-defaults' + - '{installer}-defaults' + + builders: + - shell: + !include-raw ./compass-workspace-precleanup.sh + - shell: + !include-raw ./compass-download-artifact.sh + - shell: + !include-raw ./compass-deploy-virtual.sh + + publishers: + - archive: + artifacts: 'ansible.log' + allow-empty: 'true' + fingerprint: true + +######################## +# parameter macros +######################## +- parameter: + name: compass-project-parameter + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/$PROJECT + description: "Directory where the cache to be used during the build is located." + - string: + name: GS_URL + default: '$GS_BASE{gs-pathname}' + description: "URL to Google Storage." + - string: + name: PPA_REPO + default: "http://205.177.226.235:9999" + - string: + name: PPA_CACHE + default: "$WORKSPACE/work/repo/" + - choice: + name: COMPASS_OPENSTACK_VERSION + choices: + - 'liberty' + - 'juno' + - choice: + name: COMPASS_OS_VERSION + choices: + - 'trusty' + - 'centos7' + diff --git a/jjb/compass4nfv/compass-upload-artifact.sh b/jjb/compass4nfv/compass-upload-artifact.sh new file mode 100644 index 000000000..1639fd562 --- /dev/null +++ b/jjb/compass4nfv/compass-upload-artifact.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# log info to console +echo "Uploading the $INSTALLER_TYPE artifact. This could take some time..." +echo "--------------------------------------------------------" +echo + +# source the opnfv.properties to get ARTIFACT_VERSION +source $BUILD_DIRECTORY/opnfv.properties + +# upload artifact and additional files to google storage +gsutil cp $BUILD_DIRECTORY/compass.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1 +gsutil cp $BUILD_DIRECTORY/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 +gsutil cp $BUILD_DIRECTORY/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 + +echo +echo "--------------------------------------------------------" +echo "Done!" +echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso"
\ No newline at end of file diff --git a/jjb/compass4nfv/compass-workspace-cleanup.sh b/jjb/compass4nfv/compass-workspace-cleanup.sh new file mode 100644 index 000000000..98201af9b --- /dev/null +++ b/jjb/compass4nfv/compass-workspace-cleanup.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# delete everything that is in $WORKSPACE +/bin/rm -rf $WORKSPACE
\ No newline at end of file diff --git a/jjb/compass4nfv/compass-workspace-precleanup.sh b/jjb/compass4nfv/compass-workspace-precleanup.sh new file mode 100644 index 000000000..e4a301bab --- /dev/null +++ b/jjb/compass4nfv/compass-workspace-precleanup.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +cd $WORKSPACE/.. +sudo rm $WORKSPACE -rf +git clone $GIT_BASE $WORKSPACE
\ No newline at end of file diff --git a/jjb/compass4nfv/compass4nfv.yml b/jjb/compass4nfv/compass4nfv.yml deleted file mode 100644 index 018145e4e..000000000 --- a/jjb/compass4nfv/compass4nfv.yml +++ /dev/null @@ -1,567 +0,0 @@ -- project: - - name: compass - - installer: 'compass' - - jobs: - - 'compass-verify-{stream}' - - 'compass-daily-{stream}' - - 'compass-build-iso-{stream}' - - 'compass-build-ppa-{stream}' - - 'compass-deploy-virtual-{sdn-controller}-{stream}' - - 'compass-deploy-bare-huawei-us-{stream}' - -# only master branch is enabled at the moment to keep no of jobs sane - stream: - - master: - branch: '{stream}' - gs-pathname: '' -# - brahmaputra: -# branch: 'stable/{stream}' -# gs-pathname: '/{stream}' - - sdn-controller: - - 'nosdn': - disabled: false - node: huawei-deploy-vm - - 'odl': - disabled: false - node: huawei-deploy-vm - - 'onos': - disabled: false - node: huawei-deploy-vm - - 'opencontrail': - disabled: true - node: huawei-deploy-vm - - project: 'compass4nfv' - -######################## -# job templates -######################## -- job-template: - name: 'compass-verify-{stream}' - - disabled: false - - node: 'huawei-deploy-vm' - - concurrent: true - - properties: - - throttle: - enabled: true - max-total: 3 - max-per-node: 1 - - parameters: - - project-parameter: - project: '{project}' - - gerrit-parameter: - branch: '{branch}' - - compass-parameter: - installer: '{installer}' - gs-pathname: '{gs-pathname}' - - string: - name: SDN_CONTROLLER - default: 'nosdn' - scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' - - triggers: - - gerrit: - trigger-on: - - patchset-created-event: - exclude-drafts: 'false' - exclude-trivial-rebase: 'false' - exclude-no-code-change: 'false' - - draft-published-event - - comment-added-contains-event: - comment-contains-value: 'recheck' - - comment-added-contains-event: - comment-contains-value: 'reverify' - projects: - - project-compare-type: 'ANT' - project-pattern: '{project}' - branches: - - branch-compare-type: 'ANT' - branch-pattern: '**/{branch}' - forbidden-file-paths: - - compare-type: ANT - pattern: 'docs/**|.gitignore' - - builders: - - 'builder-compass-build-iso' - - 'builder-compass-deploy-virtual' - publishers: - - archive: - artifacts: 'ansible.log' - allow-empty: 'true' - fingerprint: true - -- job-template: - name: 'compass-daily-{stream}' - - node: huawei-build - - concurrent: true - - properties: - - throttle: - enabled: true - max-total: 1 - max-per-node: 1 - - parameters: - - project-parameter: - project: '{project}' - - compass-parameter: - installer: '{installer}' - gs-pathname: '{gs-pathname}' - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - triggers: - - timed: 'H 16 * * *' - - builders: - - trigger-builds: - - project: 'compass-build-iso-{stream}' - git-revision: true - block: true - - trigger-builds: - - project: 'compass-deploy-virtual-nosdn-{stream}' - git-revision: true - block: false - - trigger-builds: - - project: 'compass-deploy-virtual-odl-{stream}' - git-revision: true - block: false - - trigger-builds: - - project: 'compass-deploy-virtual-onos-{stream}' - git-revision: true - block: false - - trigger-builds: - - project: 'compass-deploy-bare-huawei-us-{stream}' - git-revision: true - block: false - - -- job-template: - name: 'compass-build-iso-{stream}' - - node: ericsson-build - - concurrent: true - - properties: - - throttle: - enabled: true - max-total: 1 - max-per-node: 1 - - parameters: - - project-parameter: - project: '{project}' - - compass-parameter: - installer: '{installer}' - gs-pathname: '{gs-pathname}' - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - builders: - - 'builder-compass-build-iso' - - 'builder-compass-upload-artifact' - - 'builder-compass-workspace-cleanup' - -- job-template: - name: 'compass-build-ppa-{stream}' - - description: "build ppa(using docker) in huawei lab" - - node: huawei-build - - concurrent: true - - properties: - - throttle: - enabled: true - max-total: 1 - max-per-node: 1 - - parameters: - - project-parameter: - project: '{project}' - - compass-parameter: - installer: '{installer}' - gs-pathname: '{gs-pathname}' - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - triggers: - - timed: 'H 16 * * *' - - builders: - - 'builder-compass-make-ppa' - -- job-template: - name: 'compass-deploy-virtual-{sdn-controller}-{stream}' - - disabled: false - - node: '{node}' - - concurrent: true - - properties: - - throttle: - enabled: true - max-total: 1 - max-per-node: 1 - - parameters: - - project-parameter: - project: '{project}' - - compass-parameter: - installer: '{installer}' - gs-pathname: '{gs-pathname}' - - string: - name: SDN_CONTROLLER - default: '{sdn-controller}' - - builders: - - 'builder-compass-ci-preclean-workspace' - - 'builder-compass-download-artifact' - - 'builder-compass-deploy-virtual' - publishers: - - archive: - artifacts: 'ansible.log' - allow-empty: 'true' - fingerprint: true - -- job-template: - name: 'compass-deploy-bare-huawei-us-{stream}' - - disabled: false - - node: 'huawei-us-deploy-bare-1' - - concurrent: true - - properties: - - throttle: - enabled: true - max-total: 1 - max-per-node: 1 - - parameters: - - project-parameter: - project: '{project}' - - compass-parameter: - installer: '{installer}' - gs-pathname: '{gs-pathname}' - - string: - name: SDN_CONTROLLER - default: '{sdn-controller}' - - builders: - - 'builder-compass-ci-preclean-workspace' - - 'builder-compass-download-artifact' - - 'builder-compass-deploy-bare' - - publishers: - - archive: - artifacts: 'ansible.log' - allow-empty: 'true' - fingerprint: true - -######################## -# parameter macros -######################## -- parameter: - name: compass-parameter - parameters: - - string: - name: INSTALLER - default: '{installer}' - description: "Installer to use." - - string: - name: BUILD_DIRECTORY - default: $WORKSPACE/build_output - description: "Directory where the build artifact will be located upon the completion of the build." - - string: - name: CACHE_DIRECTORY - default: $HOME/opnfv/cache/$PROJECT - description: "Directory where the cache to be used during the build is located." - - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." - - string: - name: GS_URL - default: artifacts.opnfv.org/$PROJECT{gs-pathname} - description: "URL to Google Storage." - - string: - name: GS_URL_PROXY - default: build.opnfv.org/artifacts/$PROJECT/ - description: "URL to Google Storage proxy" - - string: - name: installer_ip - default: "192.168.200.2" - - string: - name: PPA_REPO - default: "http://205.177.226.235:9999" - - string: - name: PPA_CACHE - default: "$WORKSPACE/work/repo/" - - choice: - name: COMPASS_OPENSTACK_VERSION - choices: - - 'liberty' - - 'juno' - - choice: - name: COMPASS_OS_VERSION - choices: - - 'trusty' - - 'centos7' - -######################## -# builder macros -######################## -- builder: - name: 'builder-compass-make-ppa' - builders: - - shell: | - #!/bin/bash - set -x - set -o errexit - set -o nounset - set -o pipefail - # make ppa - cd $WORKSPACE/ - ./build/make_repo.sh - # calc MD5 of ppa - cd $PPA_CACHE - for i in $(find *.gz *.iso *.img -type f) - do - md5=$(md5sum $i | cut -d ' ' -f1) - echo $md5 > $i.md5 - curl -T $i $PPA_REPO - curl -T $i.md5 $PPA_REPO - done - -- builder: - name: 'builder-compass-build-iso' - builders: - - shell: | - #!/bin/bash - set -o errexit - set -o nounset - set -o pipefail - - # log info to console - echo "Starting the build of $INSTALLER. This could take some time..." - echo "--------------------------------------------------------" - echo - - # create the cache directory if it doesn't exist - [[ -d $CACHE_DIRECTORY ]] || mkdir -p $CACHE_DIRECTORY - [[ -d $BUILD_DIRECTORY ]] || mkdir -p $BUILD_DIRECTORY - - # set OPNFV_ARTIFACT_VERSION - export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") - - # start the build - if [ -d $PPA_CACHE ] - then - cp $PPA_CACHE/*.tar.gz $PPA_CACHE/*.iso $PPA_CACHE/*.img $CACHE_DIRECTORY/ -f - fi - - cd $WORKSPACE/ - ./build.sh --iso-dir $BUILD_DIRECTORY/ --iso-name compass.iso -c $CACHE_DIRECTORY - - # list the build artifacts - ls -al $BUILD_DIRECTORY - - # save information regarding artifact into file - ( - echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" - echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)" - echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)" - echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" - echo "OPNFV_ARTIFACT_MD5SUM=$(md5sum $BUILD_DIRECTORY/compass.iso | cut -d' ' -f1)" - echo "OPNFV_BUILD_URL=$BUILD_URL" - ) > $BUILD_DIRECTORY/opnfv.properties - echo - echo "--------------------------------------------------------" - echo "Done!" - -- builder: - name: 'builder-compass-deploy-virtual' - builders: - - shell: | - #!/bin/bash - set -x - - # log info to console - echo "Starting the deployment on virtual environment using $INSTALLER. This could take some time..." - echo "--------------------------------------------------------" - echo - - export ISO_URL=file://$BUILD_DIRECTORY/compass.iso - export OS_VERSION=${COMPASS_OS_VERSION} - export OPENSTACK_VERSION=${COMPASS_OPENSTACK_VERSION} - export CONFDIR=$WORKSPACE/deploy/conf/vm_environment - ./deploy.sh --dha $CONFDIR/$SDN_CONTROLLER.yml --network $CONFDIR/$NODE_NAME/network.yml - if [ $? -ne 0 ]; then - echo "depolyment failed!" - deploy_ret=1 - fi - - echo - echo "--------------------------------------------------------" - echo "Done!" - - ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - sshpass -p root scp 2>/dev/null $ssh_options root@${installer_ip}:/var/ansible/run/openstack_${COMPASS_OPENSTACK_VERSION}-opnfv2/ansible.log ./ &> /dev/null - - exit $deploy_ret - -- builder: - name: 'builder-compass-deploy-bare' - builders: - - shell: | - #!/bin/bash - set -x - - # log info to console - echo "Starting the deployment on baremetal environment using $INSTALLER. This could take some time..." - echo "--------------------------------------------------------" - echo - - export CONFDIR=$WORKSPACE/deploy/conf/hardware_environment/huawei_us_lab/pod1 - export ISO_URL=file://$BUILD_DIRECTORY/compass.iso - export EXTERNAL_NIC=eth0 - export INSTALL_NIC=eth1 - - cd $WORKSPACE - - export OS_VERSION=${COMPASS_OS_VERSION} - export OPENSTACK_VERSION=${COMPASS_OPENSTACK_VERSION} - ./deploy.sh --dha $CONFDIR/dha.yml --network $CONFDIR/network.yml - if [ $? -ne 0 ]; then - echo "depolyment failed!" - deploy_ret=1 - fi - - echo - echo "--------------------------------------------------------" - echo "Done!" - - ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" - sshpass -p root scp 2>/dev/null $ssh_options root@${installer_ip}:/var/ansible/run/openstack_${COMPASS_OPENSTACK_VERSION}-opnfv2/ansible.log ./ &> /dev/null - - exit $deploy_ret - -- builder: - name: 'builder-compass-upload-artifact' - builders: - - shell: | - #!/bin/bash - set -o errexit - set -o nounset - set -o pipefail - - # log info to console - echo "Uploading the $INSTALLER artifact. This could take some time..." - echo "--------------------------------------------------------" - echo - - # source the opnfv.properties to get ARTIFACT_VERSION - source $BUILD_DIRECTORY/opnfv.properties - - # upload artifact and additional files to google storage - gsutil cp $BUILD_DIRECTORY/compass.iso gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso > gsutil.iso.log 2>&1 - gsutil cp $BUILD_DIRECTORY/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 - gsutil cp $BUILD_DIRECTORY/opnfv.properties gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 - - echo - echo "--------------------------------------------------------" - echo "Done!" - echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.iso" - -- builder: - name: 'builder-compass-download-artifact' - builders: - - shell: | - #!/bin/bash - set -o errexit - set -o nounset - set -o pipefail - - # log info to console - echo "Downloading the $INSTALLER artifact. This could take some time..." - echo "--------------------------------------------------------" - echo - - # get the latest.properties file in order to get info regarding latest artifact - [[ -d $BUILD_DIRECTORY ]] || mkdir -p $BUILD_DIRECTORY - curl -s -o $BUILD_DIRECTORY/latest.properties http://$GS_URL/latest.properties - - # check if we got the file - [[ -f $BUILD_DIRECTORY/latest.properties ]] || exit 1 - - # source the file so we get OPNFV vars - source $BUILD_DIRECTORY/latest.properties - - # download the file - curl -s -o $BUILD_DIRECTORY/compass.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 - - # list the file - ls -al $BUILD_DIRECTORY/compass.iso - - echo - echo "--------------------------------------------------------" - echo "Done!" - -- builder: - name: 'builder-compass-workspace-cleanup' - builders: - - shell: | - #!/bin/bash - set -o errexit - set -o nounset - set -o pipefail - - # delete everything that is in $WORKSPACE - /bin/rm -rf $WORKSPACE - -- builder: - name: 'builder-compass-ci-preclean-workspace' - builders: - - shell: | - #!/bin/bash - set -o errexit - set -o nounset - set -o pipefail - - cd $WORKSPACE/.. - sudo rm $WORKSPACE -rf - git clone $GIT_BASE $WORKSPACE - diff --git a/jjb/compass4nfv/opnfv-compass4nfv.yml b/jjb/compass4nfv/opnfv-compass4nfv.yml deleted file mode 100644 index 288c7abe6..000000000 --- a/jjb/compass4nfv/opnfv-compass4nfv.yml +++ /dev/null @@ -1,84 +0,0 @@ -######################## -# Job configuration for compass4nfv -######################## -- project: - - name: 'opnfv-compass4nfv' - - installer: 'compass' - - controller: - - 'odl' - - 'onos' - - 'opencontrail' - - pod: 'huawei-us-deploy-bare-1' - - stream: - - master: - branch: 'master' - gs-pathname: '' - - jobs: - - 'opnfv-compass4nfv-{controller}-{pod}-daily-{stream}' - - 'opnfv-compass4nfv-deploy-{pod}-daily-{stream}' - -######################## -# job templates -######################## -- job-template: - name: 'opnfv-compass4nfv-{controller}-{pod}-daily-{stream}' - - disabled: true - - concurrent: false - - parameters: - - project-parameter: - project: '{installer}' - - '{pod}-defaults' - - '{installer}-defaults': - controller: '{controller}' - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - builders: - - trigger-builds: - - project: 'opnfv-compass4nfv-daily-deploy-{stream}' - git-revision: true - block: true - predefined-parameters: - CONTROLLER={controller} - -- job-template: - name: 'opnfv-compass4nfv-deploy-{pod}-daily-{stream}' - - disabled: true - - concurrent: false - - wrappers: - - build-name: - name: '$BUILD_NUMBER: {installer} $CONTROLLER' - - parameters: - - project-parameter: - project: '{installer}' - - '{pod}-defaults' - - '{installer}-defaults' - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - builders: - - shell: | - #!/bin/bash - echo "Hello World from OPNFV $INSTALLER_TYPE" - echo "Running $INSTALLER_TYPE with controller $CONTROLLER" diff --git a/jjb/fuel/fuel-build.sh b/jjb/fuel/fuel-build.sh index d6a71e00b..d51637afd 100755 --- a/jjb/fuel/fuel-build.sh +++ b/jjb/fuel/fuel-build.sh @@ -56,7 +56,7 @@ if [[ "$JOB_NAME" =~ "verify" && "$GERRIT_CHANGE_COMMIT_MESSAGE" =~ "$NOCACHE_PA echo "The cache will not be used for this build!" NOCACHE_ARG="-f P" fi -NOCACHE_ARG=${{NOCACHE_ARG:-}} +NOCACHE_ARG=${NOCACHE_ARG:-} # start the build cd $WORKSPACE/ci diff --git a/jjb/fuel/fuel-ci-jobs-master.yml b/jjb/fuel/fuel-ci-jobs-master.yml new file mode 100644 index 000000000..dffafe850 --- /dev/null +++ b/jjb/fuel/fuel-ci-jobs-master.yml @@ -0,0 +1,154 @@ +- project: + + name: 'fuel-master' + + project: 'fuel' + + installer: '{project}' + +#-------------------------------- +# BRANCH ANCHORS +#-------------------------------- + master: &master + stream: master + branch: '{stream}' + gs-pathname: '' +#-------------------------------- +# POD, INSTALLER, AND BRANCH MAPPING +#-------------------------------- + pod: + - ericsson-pod2: + <<: *master +#-------------------------------- + +# new scenario descriptions + scenario: + - 'os_ha' + - 'os_odl-l2_ha' + - 'os_odl-l3_ha' + - 'os_onos_ha' + - 'os_kvm_ha' + - 'os_ovs_ha' + - 'os_kvm_ovs_ha' + + jobs: + - 'fuel-{scenario}-{pod}-daily-{stream}' + - 'fuel-deploy-{pod}-daily-{stream}-development' + +######################## +# job templates - new type of defining the scenario +######################## +- job-template: + name: 'fuel-{scenario}-{pod}-daily-{stream}' + + project-type: multijob + + concurrent: false + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'fuel-os_.*' + + wrappers: + - build-name: + name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + + parameters: + - project-parameter: + project: '{project}' + - '{pod}-defaults' + - '{installer}-defaults' + - string: + name: DEPLOY_SCENARIO + default: '{scenario}' + - fuel-ci-parameter: + gs-pathname: '{gs-pathname}' + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: '{branch}' + + builders: + - multijob: + name: deploy + condition: SUCCESSFUL + projects: + - name: 'fuel-deploy-{pod}-daily-{stream}' + current-parameters: true + git-revision: true + kill-phase-on: FAILURE + - multijob: + name: functest + condition: COMPLETED + projects: + - name: 'functest-fuel-{pod}-daily-{stream}' + current-parameters: true + kill-phase-on: NEVER + - multijob: + name: yardstick + condition: COMPLETED + projects: + - name: 'yardstick-fuel-{pod}-daily-{stream}' + current-parameters: true + kill-phase-on: NEVER + +- job-template: + name: 'fuel-deploy-{pod}-daily-{stream}-development' + + concurrent: false + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + + parameters: + - project-parameter: + project: 'fuel' + - 'ericsson-pod2-defaults' + - 'fuel-defaults' + - fuel-ci-parameter-new: + gs-pathname: '' + + scm: + - git-scm: + credentials-id: '{ssh-credentials}' + refspec: '' + branch: 'master' + + wrappers: + - build-name: + name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' + + builders: + - shell: + !include-raw-escape: ./fuel-download-artifact.sh + - shell: + !include-raw-escape: ./fuel-deploy-master.sh +######################## +# parameter macros +######################## +- parameter: + name: fuel-ci-parameter-new + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/$INSTALLER_TYPE + description: "Directory where the cache to be used during the build is located." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." diff --git a/jjb/fuel/fuel-ci-jobs.yml b/jjb/fuel/fuel-ci-jobs.yml index 1b4b26963..c90d930c1 100644 --- a/jjb/fuel/fuel-ci-jobs.yml +++ b/jjb/fuel/fuel-ci-jobs.yml @@ -36,30 +36,20 @@ # please check the triggers before enabling any of the controllers!!! sdn-controller: - - 'nosdn': - disabled: false -# commented out below controllers to get nosdn scenario settled first -# - 'odl': -# disabled: true -# - 'onos': -# disabled: true -# - 'opencontrail': -# disabled: true + - 'nosdn' jobs: - 'fuel-{sdn-controller}-{pod}-daily-{stream}' - 'fuel-deploy-{pod}-daily-{stream}' ######################## -# job templates +# job templates - old type of defining the scenario ######################## - job-template: name: 'fuel-{sdn-controller}-{pod}-daily-{stream}' project-type: multijob - disabled: '{obj:disabled}' - concurrent: false properties: @@ -67,6 +57,10 @@ enabled: true max-total: 1 max-per-node: 1 + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'fuel-{sdn-controller}-{pod}-daily-{stream}' wrappers: - build-name: @@ -129,6 +123,12 @@ enabled: true max-total: 1 max-per-node: 1 + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'fuel-deploy-{pod}-daily-{stream}.*' + - 'functest-fuel-{pod}-daily-{stream}' + - 'yardstick-fuel-{pod}-daily-{stream}' parameters: - project-parameter: @@ -146,13 +146,10 @@ wrappers: - build-name: - name: '$BUILD_NUMBER - SDN: $SDN_CONTROLLER Scenario: $DEPLOY_SCENARIO' + name: '$BUILD_NUMBER - Scenario: $DEPLOY_SCENARIO' builders: - - shell: - !include-raw ./fuel-download-artifact.sh - - shell: - !include-raw ./fuel-deploy.sh + - 'builder-{stream}' publishers: - email: @@ -176,6 +173,23 @@ default: artifacts.opnfv.org/$PROJECT{gs-pathname} description: "URL to Google Storage." ######################## +# builder macros +######################## +- builder: + name: 'builder-master' + builders: + - shell: + !include-raw: ./fuel-download-artifact.sh + - shell: + !include-raw: ./fuel-deploy-master.sh +- builder: + name: 'builder-brahmaputra' + builders: + - shell: + !include-raw: ./fuel-download-artifact.sh + - shell: + !include-raw: ./fuel-deploy.sh +######################## # trigger macros ######################## # trigger for opnfv-jump-2 is set to run 1 hour ahead of others @@ -191,4 +205,4 @@ - trigger: name: 'fuel-ericsson-pod2-nosdn-trigger' triggers: - - timed: '0 2 * * *' + - timed: '' diff --git a/jjb/fuel/fuel-deploy-master.sh b/jjb/fuel/fuel-deploy-master.sh new file mode 100755 index 000000000..4a7d1d9fb --- /dev/null +++ b/jjb/fuel/fuel-deploy-master.sh @@ -0,0 +1,59 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +# source the file so we get OPNFV vars +source latest.properties + +# echo the info about artifact that is used during the deployment +echo "Using $OPNFV_ARTIFACT_URL for deployment" + +# checkout the commit that was used for building the downloaded artifact +# to make sure the ISO and deployment mechanism uses same versions +echo "Checking out $OPNFV_GIT_SHA1" +git checkout $OPNFV_GIT_SHA1 --quiet + +# set deployment parameters +BRIDGE=pxebr +export TMPDIR=$HOME/tmpdir +LAB_NAME=${NODE_NAME/-*} +POD_NAME=${NODE_NAME/*-} + +# create TMPDIR if it doesn't exist +mkdir -p $TMPDIR + +# change permissions down to TMPDIR +chmod a+x $HOME +chmod a+x $TMPDIR + +# clone the securedlab repo +cd $WORKSPACE +echo "Cloning securedlab repo" +git clone ssh://jenkins-ericsson@gerrit.opnfv.org:29418/securedlab --quiet + +# construct the command +DEPLOY_COMMAND="sudo $WORKSPACE/ci/deploy.sh -b file://$WORKSPACE/securedlab -l $LAB_NAME -p $POD_NAME -s $DEPLOY_SCENARIO -i file://$WORKSPACE/opnfv.iso -H -B $BRIDGE -S $TMPDIR" + +# log info to console +echo "Deployment parameters" +echo "--------------------------------------------------------" +echo "Scenario: $DEPLOY_SCENARIO" +echo "Lab: $LAB_NAME" +echo "POD: $POD_NAME" +echo "ISO: $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f3)" +echo +echo "Starting the deployment using $INSTALLER_TYPE. This could take some time..." +echo "--------------------------------------------------------" +echo + +# start the deployment +echo "Issuing command" +echo "$DEPLOY_COMMAND" +echo + +$DEPLOY_COMMAND + +echo +echo "--------------------------------------------------------" +echo "Done!" diff --git a/jjb/fuel/fuel-deploy-virtual.sh b/jjb/fuel/fuel-deploy-virtual.sh index 8e9282373..e7c83b77f 100755 --- a/jjb/fuel/fuel-deploy-virtual.sh +++ b/jjb/fuel/fuel-deploy-virtual.sh @@ -24,7 +24,7 @@ chmod a+x $TMPDIR # get the lab name from SLAVE_NAME # we currently support ericsson and intel labs -LAB_NAME=${{NODE_NAME%%-*}} +LAB_NAME=${NODE_NAME%%-*} if [[ ! "$LAB_NAME" =~ (ericsson|intel) ]]; then echo "Unsupported/unidentified lab $LAB_NAME. Cannot continue!" exit 1 diff --git a/jjb/fuel/fuel-deploy.sh b/jjb/fuel/fuel-deploy.sh index 334307448..dc3d81a3a 100755 --- a/jjb/fuel/fuel-deploy.sh +++ b/jjb/fuel/fuel-deploy.sh @@ -7,7 +7,7 @@ set -o pipefail source latest.properties # echo the info about artifact that is used during the deployment -echo "Using $(echo $OPNFV_ARTIFACT_URL | cut -d'/' -f3) for deployment" +echo "Using $OPNFV_ARTIFACT_URL for deployment" # checkout the commit that was used for building the downloaded artifact # to make sure the ISO and deployment mechanism uses same versions diff --git a/jjb/fuel/fuel-project-jobs.yml b/jjb/fuel/fuel-project-jobs.yml index 848fd0b21..a04e4e65d 100644 --- a/jjb/fuel/fuel-project-jobs.yml +++ b/jjb/fuel/fuel-project-jobs.yml @@ -60,11 +60,11 @@ builders: - shell: - !include-raw ./fuel-build.sh + !include-raw-escape: ./fuel-build.sh - shell: - !include-raw ./fuel-upload-artifact.sh + !include-raw-escape: ./fuel-upload-artifact.sh - shell: - !include-raw ./fuel-workspace-cleanup.sh + !include-raw-escape: ./fuel-workspace-cleanup.sh publishers: - email: @@ -125,9 +125,9 @@ builders: - shell: - !include-raw ./fuel-build.sh + !include-raw-escape: ./fuel-build.sh - shell: - !include-raw ./fuel-workspace-cleanup.sh + !include-raw-escape: ./fuel-workspace-cleanup.sh - job-template: name: 'fuel-merge-build-{stream}' @@ -176,11 +176,11 @@ builders: - shell: - !include-raw ./fuel-build.sh + !include-raw-escape: ./fuel-build.sh - shell: - !include-raw ./fuel-upload-artifact.sh + !include-raw-escape: ./fuel-upload-artifact.sh - shell: - !include-raw ./fuel-workspace-cleanup.sh + !include-raw-escape: ./fuel-workspace-cleanup.sh - job-template: name: 'fuel-merge-deploy-virtual-{stream}' @@ -228,11 +228,11 @@ builders: - shell: - !include-raw ./fuel-download-artifact.sh + !include-raw-escape: ./fuel-download-artifact.sh - shell: - !include-raw ./fuel-deploy-virtual.sh + !include-raw-escape: ./fuel-deploy-virtual.sh - shell: - !include-raw ./fuel-workspace-cleanup.sh + !include-raw-escape: ./fuel-workspace-cleanup.sh publishers: - email: diff --git a/jjb/functest/functest-ci-jobs.yml b/jjb/functest/functest-ci-jobs.yml index c9ab3c25a..3c993c6a3 100644 --- a/jjb/functest/functest-ci-jobs.yml +++ b/jjb/functest/functest-ci-jobs.yml @@ -29,16 +29,19 @@ - intel-pod5: installer: joid <<: *brahmaputra + - huawei-us-deploy-bare-1: + installer: compass + <<: *brahmaputra #-------------------------------- # master #-------------------------------- - - huawei-us-deploy-bare-1: - installer: compass - <<: *master - intel-pod6: installer: joid <<: *master - - intel-us-deploy-virtual-2: + - intel-pod8: + installer: compass + <<: *master + - opnfv-jump-1: installer: apex <<: *master - ericsson-pod1: @@ -183,20 +186,18 @@ # labconfig is used only for joid labconfig="" if [[ ${INSTALLER_TYPE} == 'apex' ]]; then - INSTALLER_IP=$(grep instack /var/lib/libvirt/dnsmasq/default.leases \ - |awk '{print $3}' | head -n 1) + instack_mac=$(sudo virsh domiflist instack | grep default | \ + grep -Eo "[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+") + INSTALLER_IP=$(/usr/sbin/arp -e | grep ${instack_mac} | awk {'print $1'}) sshkey="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa" sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then # If production lab then creds may be retrieved dynamically # creds are on the jumphost, always in the same folder - labconfig="-v /var/lib/jenkins/joid_config/admin-openrc:/home/opnfv/functest/conf/openstack.creds" + labconfig="-v $LAB_CONFIG/admin-openrc:/home/opnfv/functest/conf/openstack.creds" # If dev lab, credentials may not be the default ones, just provide a path to put them into docker # replace the default one by the customized one provided by jenkins config - if [ -n "${LAB_CONFIG}" ]; then - labconfig="-v ${LAB_CONFIG}:/home/opnfv/functest/conf/openstack.creds" - fi fi echo "Functest: Start Docker and prepare environment" envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} -e NODE_NAME=${NODE_NAME}" @@ -207,7 +208,7 @@ $res_volume="-v ${dir_result}:/home/opnfv/functest/results" docker pull opnfv/functest:latest_stable - cmd="docker run -id -e $envs ${labconfig} ${sshkey} ${res_volume} opnfv/functest:latest_stable /bin/bash" + cmd="sudo docker run --privileged=true -id -e $envs ${labconfig} ${sshkey} ${res_volume} opnfv/functest:latest_stable /bin/bash" echo "Functest: Running docker run command: ${cmd}" ${cmd} docker ps -a diff --git a/jjb/joid/joid-ci-jobs.yml b/jjb/joid/joid-ci-jobs.yml index 80e21f50e..e3b010028 100644 --- a/jjb/joid/joid-ci-jobs.yml +++ b/jjb/joid/joid-ci-jobs.yml @@ -43,8 +43,8 @@ # commented out below controllers to get Arno scenario settled first # - 'nosdn': # disabled: true -# - 'onos': -# disabled: true + - 'onos': + disabled: true # - 'opencontrail': # disabled: true @@ -74,8 +74,8 @@ parameters: - project-parameter: project: '{project}' - - '{pod}-defaults' - '{installer}-defaults' + - '{pod}-defaults' - string: name: SDN_CONTROLLER default: '{sdn-controller}' @@ -90,9 +90,28 @@ branch: '{branch}' builders: - - '{installer}-multijob-{pod}-builder': - pod: '{pod}' - stream: '{stream}' + - multijob: + name: deploy + condition: SUCCESSFUL + projects: + - name: 'joid-deploy-{pod}-daily-{stream}' + current-parameters: true + git-revision: true + kill-phase-on: FAILURE + - multijob: + name: functest + condition: COMPLETED + projects: + - name: 'functest-joid-{pod}-daily-{stream}' + current-parameters: true + kill-phase-on: NEVER + - multijob: + name: yardstick + condition: COMPLETED + projects: + - name: 'yardstick-joid-{pod}-daily-{stream}' + current-parameters: true + kill-phase-on: NEVER - job-template: name: 'joid-deploy-{pod}-daily-{stream}' @@ -111,8 +130,8 @@ parameters: - project-parameter: project: '{project}' - - '{pod}-defaults' - '{installer}-defaults' + - '{pod}-defaults' scm: - git-scm: @@ -121,144 +140,24 @@ branch: '{branch}' builders: - - '{pod}-builder' + - 'builder-macro' ######################## -# multijob builder macros -######################## -# intel-pod5 is CI POD so full CI will run on it -# deploy, functest, yardstick -- builder: - name: joid-multijob-intel-pod5-builder - builders: - - multijob: - name: deploy - condition: SUCCESSFUL - projects: - - name: 'joid-deploy-{pod}-daily-{stream}' - current-parameters: true - git-revision: true - kill-phase-on: FAILURE - - multijob: - name: functest - condition: COMPLETED - projects: - - name: 'functest-joid-{pod}-daily-{stream}' - current-parameters: true - kill-phase-on: NEVER - - multijob: - name: yardstick - condition: COMPLETED - projects: - - name: 'yardstick-joid-{pod}-daily-{stream}' - current-parameters: true - kill-phase-on: NEVER -# intel-pod6 is CI POD so full CI will run on it -# deploy, functest, yardstick -- builder: - name: joid-multijob-intel-pod6-builder - builders: - - multijob: - name: deploy - condition: SUCCESSFUL - projects: - - name: 'joid-deploy-{pod}-daily-{stream}' - current-parameters: true - git-revision: true - kill-phase-on: FAILURE - - multijob: - name: functest - condition: COMPLETED - projects: - - name: 'functest-joid-{pod}-daily-{stream}' - current-parameters: true - kill-phase-on: NEVER - - multijob: - name: yardstick - condition: COMPLETED - projects: - - name: 'yardstick-joid-{pod}-daily-{stream}' - current-parameters: true - kill-phase-on: NEVER -# yardstick placeholder -# - multijob: -# name: yardstick -# condition: COMPLETED -# projects: -# - name: 'yardstick-joid-{pod}-daily-{stream}' -# current-parameters: true -# kill-phase-on: NEVER - -# orange-pod2 is NOT a CI POD so only the selected jobs will run -# deploy and functest currently -- builder: - name: joid-multijob-orange-pod2-builder - builders: - - multijob: - name: deploy - condition: SUCCESSFUL - projects: - - name: 'joid-deploy-{pod}-daily-{stream}' - current-parameters: true - git-revision: true - kill-phase-on: FAILURE - - multijob: - name: functest - condition: COMPLETED - projects: - - name: 'functest-joid-{pod}-daily-{stream}' - current-parameters: true - kill-phase-on: NEVER -######################## # builder macros ######################## - builder: - name: intel-pod5-builder + name: 'builder-macro' builders: - shell: | #!/bin/bash echo "Running $INSTALLER_TYPE with controller $SDN_CONTROLLER" echo - echo "Executing clean.sh" - cd $WORKSPACE/ci - ./clean.sh - echo - echo "Executing 02-maasdeploy.sh intelpod5" - ./02-maasdeploy.sh intelpod5 - echo - echo "Executing deploy.sh -o liberty -s $SDN_CONTROLLER -t ha -l intelpod5" - ./deploy.sh -o liberty -s $SDN_CONTROLLER -t ha -l intelpod5 - -- builder: - name: intel-pod6-builder - builders: - - shell: | - #!/bin/bash - echo "Running $INSTALLER_TYPE with controller $SDN_CONTROLLER" - echo - echo "Executing clean.sh" - cd $WORKSPACE/ci - ./clean.sh - echo - echo "Executing 02-maasdeploy.sh intelpod6" - ./02-maasdeploy.sh intelpod6 - echo - echo "Executing deploy.sh -o liberty -s $SDN_CONTROLLER -t ha -l intelpod6" - ./deploy.sh -o liberty -s $SDN_CONTROLLER -t ha -l intelpod6 - -- builder: - name: orange-pod2-builder - builders: - - shell: | - #!/bin/bash - echo "Running $INSTALLER_TYPE with controller $SDN_CONTROLLER" - echo "Please note that this is WIP generic builder" - echo - echo "Executing clean.sh" + echo "------ First Executing clean.sh ------" cd $WORKSPACE/ci ./clean.sh - shell: - !include-raw ./joid-deploy.sh + !include-raw: ./joid-deploy.sh + ######################## # trigger macros ######################## @@ -274,3 +173,15 @@ name: 'joid-orange-pod2-odl-trigger' triggers: - timed: '' +- trigger: + name: 'joid-intel-pod5-onos-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-intel-pod6-onos-trigger' + triggers: + - timed: '' +- trigger: + name: 'joid-orange-pod2-onos-trigger' + triggers: + - timed: '' diff --git a/jjb/joid/joid-deploy.sh b/jjb/joid/joid-deploy.sh index c7e5ba8fd..e23afac1c 100644 --- a/jjb/joid/joid-deploy.sh +++ b/jjb/joid/joid-deploy.sh @@ -2,51 +2,67 @@ set +e set -o nounset -JOID_LOCAL_CONFIG_FOLDER=$HOME/joid_config -JOID_ADMIN_OPENRC=$JOID_LOCAL_CONFIG_FOLDER/admin-openrc +PWD_FILENAME="passwords.sh" ## -## Load local config or defaults ## +## +function exit_on_error { + RES=$1 + MSG=$2 + if [ $RES != 0 ]; then + echo "FAILED - $MSG" + exit $RES + fi +} + + +## +## Create LAB_CONFIG folder if not exists +## +mkdir -p $LAB_CONFIG -if [ -e "$JOID_LOCAL_CONFIG_FOLDER/config.sh" ]; then - echo "------ Load local config ------" - source $JOID_LOCAL_CONFIG_FOLDER/config.sh +## +## Override default passwords with local settings if needed +## + +if [ -e "$LAB_CONFIG/$PWD_FILENAME" ]; then + echo "------ Load local passwords ------" + source $LAB_CONFIG/$PWD_FILENAME else - echo "------ No local config, load default ------" - # link NODE_NAME to joid node config names + export MAAS_USER=ubuntu + export MAAS_PASSWORD=ubuntu + export OS_ADMIN_PASSWORD=openstack +fi + +## +## Set Joid pod config name +## + # This part will be removed when pod names will be synced between jenkins and joid config case $NODE_NAME in orange-fr-pod2) POD=orange-pod2 ;; *) POD=$NODE_NAME ;; esac - export POD_DC=$(echo $POD |cut -d\- -f1) - export POD_NUM=$(echo $POD |cut -d\- -f2) - export POD_NAME=$POD_DC$POD_NUM - export MAAS_REINSTALL=true - export MAAS_USER=ubuntu - export MAAS_PASSWORD=ubuntu - export OS_ADMIN_PASSWORD=openstack - export CEPH_DISKS=/srv - export CEPH_REFORMAT=no -fi + export POD_NAME=${POD/-} ## ## Redeploy MAAS or recover the previous config ## cd $WORKSPACE/ci -if [ -e "$JOID_LOCAL_CONFIG_FOLDER/environments.yaml" ] && [ "$MAAS_REINSTALL" == "false" ]; then +if [ -e "$LAB_CONFIG/environments.yaml" ] && [ "$MAAS_REINSTALL" == "false" ]; then echo "------ Recover Juju environment to use MAAS ------" - cp $JOID_LOCAL_CONFIG_FOLDER/environments.yaml . + cp $LAB_CONFIG/environments.yaml . else - MAASCONFIG=$WORKSPACE/ci/maas/$POD_DC/$POD_NUM/deployment.yaml + MAASCONFIG=$WORKSPACE/ci/maas/${POD/-*}/${POD/*-}/deployment.yaml echo "------ Set MAAS password ------" sed -i -- "s/user: ubuntu/user: $MAAS_USER/" $MAASCONFIG sed -i -- "s/password: ubuntu/password: $MAAS_PASSWORD/" $MAASCONFIG echo "------ Redeploy MAAS ------" ./02-maasdeploy.sh $POD_NAME + exit_on_error $? "MAAS Deploy FAILED" fi ## @@ -66,7 +82,15 @@ echo "------ Set openstack password ------" sed -i -- "s/\"admin-password\": openstack/\"admin-password\": $OS_ADMIN_PASSWORD/" $SRCBUNDLE echo "------ Set ceph disks ------" -sed -i -- "s@osd-devices: /srv@osd-devices: $CEPH_DISKS@" $SRCBUNDLE +CEPH_DISKS_CONTROLLERS=${CEPH_DISKS_CONTROLLERS:-} +if [ -z "$CEPH_DISKS_CONTROLLERS" ]; then + CEPH_DISKS_CONTROLLERS=$CEPH_DISKS +fi + +#Find the first line of osd-devices to change the one for ceph, then the other for ceph-osd +CEPH_DEV_LINE=$(grep -nr osd-devices $SRCBUNDLE |head -n1|cut -d: -f1) +sed -i -- "${CEPH_DEV_LINE}s@osd-devices: /srv@osd-devices: $CEPH_DISKS@" $SRCBUNDLE +sed -i -- "s@osd-devices: /srv@osd-devices: $CEPH_DISKS_CONTROLLERS@" $SRCBUNDLE sed -i -r -- "s/^(\s+osd-reformat: )'no'/\1'$CEPH_REFORMAT'/" $SRCBUNDLE ## @@ -77,20 +101,15 @@ echo "------ Deploy with juju ------" echo "Execute: ./deploy.sh -t $HA_MODE -o $OS_RELEASE -s $SDN_CONTROLLER -l $POD_NAME" ./deploy.sh -t $HA_MODE -o $OS_RELEASE -s $SDN_CONTROLLER -l $POD_NAME +exit_on_error $? "Main deploy FAILED" ## ## Set Admin RC ## - +JOID_ADMIN_OPENRC=$LAB_CONFIG/admin-openrc echo "------ Create OpenRC file [$JOID_ADMIN_OPENRC] ------" KEYSTONE=$(cat bundles.yaml |shyaml get-value openstack-phase2.services.keystone.options.vip) -# create the folder if needed -JOID_ADMIN_OPENRC_FOLDER=$(echo $JOID_ADMIN_OPENRC | perl -pe "s|^(.*/).*?$|\1|") -if [ ! -d "$JOID_ADMIN_OPENRC_FOLDER" ]; then - mkdir -p $JOID_ADMIN_OPENRC_FOLDER -fi - # export the openrc file cat << EOF > $JOID_ADMIN_OPENRC export OS_USERNAME=admin @@ -104,14 +123,14 @@ EOF ## Backup local juju env ## -if [ -d "$JOID_LOCAL_CONFIG_FOLDER" ]; then - echo "------ Backup Juju environment ------" - cp environments.yaml $JOID_LOCAL_CONFIG_FOLDER/ -fi +echo "------ Backup Juju environment ------" +cp environments.yaml $LAB_CONFIG/ ## ## Basic test to return a realistic result to jenkins ## + +echo "------ Do basic test ------" source $JOID_ADMIN_OPENRC curl -i -sw '%{http_code}' -H "Content-Type: application/json" -d " { \"auth\": { @@ -119,18 +138,47 @@ curl -i -sw '%{http_code}' -H "Content-Type: application/json" -d " \"methods\": [\"password\"], \"password\": { \"user\": { - \"name\": \"$OS_TENANT_NAME\", + \"name\": \"admin\", \"domain\": { \"id\": \"default\" }, - \"password\": \"$OS_PASSWORD\" + \"password\": \"$OS_ADMIN_PASSWORD\" } } } } -}" http://$KEYSTONE:5000/v3/auth/tokens |grep "HTTP/1.1 20" 2>&1 >/dev/null; echo $?; -RES=$? -if [ $RES == 0 ]; then - echo "Deploy SUCCESS" +}" http://$KEYSTONE:5000/v3/auth/tokens |grep "HTTP/1.1 20" 2>&1 >/dev/null; +exit_on_error $? "Deploy FAILED to auth to openstack" + + +## +## Create external network if needed +## + +EXTERNAL_NETWORK=${EXTERNAL_NETWORK:-} +# split EXTERNAL_NETWORK=name;type;first ip;last ip; gateway;network +IFS=';' read -r -a EXTNET <<< "$EXTERNAL_NETWORK" +EXTNET_NAME=${EXTNET[0]} +EXTNET_TYPE=${EXTNET[1]} +EXTNET_FIP=${EXTNET[2]} +EXTNET_LIP=${EXTNET[3]} +EXTNET_GW=${EXTNET[4]} +EXTNET_NET=${EXTNET[5]} +# If we have more information than only the name, try to create it +if [ -z "$EXTNET_TYPE" ]; then + echo "------ No data for external network creation, pass ------" else - echo "Deploy FAILED" + echo "------ External network creation ------" + neutron net-create $EXTNET_NAME --router:external True \ + --provider:physical_network external --provider:network_type $EXTNET_TYPE + exit_on_error $? "External network creation failed" + neutron subnet-create $EXTNET_NAME --name $EXTNET_NAME \ + --allocation-pool start=$EXTNET_FIP,end=$EXTNET_LIP \ + --disable-dhcp --gateway $EXTNET_GW $EXTNET_NET + exit_on_error $? "External subnet creation failed" fi -exit $RES + +## +## Exit success +## + +echo "Deploy success" +exit 0 diff --git a/jjb/opnfv/installer-params.yml b/jjb/opnfv/installer-params.yml index 3e5c0ffac..4980889fd 100644 --- a/jjb/opnfv/installer-params.yml +++ b/jjb/opnfv/installer-params.yml @@ -99,5 +99,25 @@ description: 'OpenStack release (kilo|liberty)' - string: name: EXTERNAL_NETWORK - default: 'net04_ext' - description: 'external network for test' + default: ext-net;flat;192.168.0.2;192.168.0.253;192.168.0.1;192.168.0.0/24 + description: "External network to create (name;type;first ip;last ip; gateway;network)" + - string: + name: CEPH_DISKS + default: '/srv' + description: "Disks to use by ceph by default (space separated list)" + - string: + name: CEPH_DISKS_CONTROLLERS + default: '' + description: "Disks to use by ceph by controllers (empty if same as CEPH_DISKS)" + - string: + name: LAB_CONFIG + default: "$HOME/joid_config" + description: "Local lab config and Openstack openrc location" + - string: + name: CEPH_REFORMAT + default: 'false' + description: "Format or not disk before using ceph [true/false] (must be done the first time)" + - string: + name: MAAS_REINSTALL + default: 'false' + description: "Reinstall MAAS and Bootstrap before deploy [true/false]" diff --git a/jjb/opnfv/opnfv-docker.sh b/jjb/opnfv/opnfv-docker.sh index 545aef6dd..81bab33bb 100644 --- a/jjb/opnfv/opnfv-docker.sh +++ b/jjb/opnfv/opnfv-docker.sh @@ -53,22 +53,6 @@ if [[ "$UPDATE_LATEST_STABLE" == "true" ]]; then fi -# Get tag version -cd $WORKSPACE -git clone https://gerrit.opnfv.org/gerrit/releng - -DOCKER_TAG=$($WORKSPACE/releng/utils/calculate_version.sh -t docker \ - -n $DOCKER_REPO_NAME) - -ret_val=$? -if [ $ret_val -ne 0 ]; then - echo "Error retrieving the version tag." - exit 1 -else - echo "Tag version to be build and pushed: $DOCKER_TAG" -fi - - # cd to directory where Dockerfile is located if [[ "$DOCKER_REPO_NAME" == "opnfv/functest" ]]; then cd $WORKSPACE/docker @@ -83,6 +67,27 @@ else exit 1 fi + +# Get tag version +branch=$(git rev-parse --abbrev-ref HEAD) +if [ $branch == "master" ]; then + DOCKER_TAG="master" +else + cd $WORKSPACE + git clone https://gerrit.opnfv.org/gerrit/releng + + DOCKER_TAG=$($WORKSPACE/releng/utils/calculate_version.sh -t docker \ + -n $DOCKER_REPO_NAME) + + ret_val=$? + if [ $ret_val -ne 0 ]; then + echo "Error retrieving the version tag." + exit 1 + fi +fi +echo "Tag version to be build and pushed: $DOCKER_TAG" + + # Start the build echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_TAG..." diff --git a/jjb/opnfv/opnfv-docker.yml b/jjb/opnfv/opnfv-docker.yml index fcbe60be2..13063e39b 100644 --- a/jjb/opnfv/opnfv-docker.yml +++ b/jjb/opnfv/opnfv-docker.yml @@ -17,7 +17,9 @@ stream: - master: - branch: 'master' + branch: '{stream}' + - brahmaputra: + branch: 'stable/{stream}' ######################## # job templates diff --git a/jjb/opnfv/slave-params.yml b/jjb/opnfv/slave-params.yml index 002ab7e52..734343d88 100644 --- a/jjb/opnfv/slave-params.yml +++ b/jjb/opnfv/slave-params.yml @@ -146,6 +146,10 @@ name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' + - string: + name: CEPH_DISKS + default: /srv + description: "Disks to use by ceph (comma separated list)" - parameter: name: 'intel-pod6-defaults' @@ -165,6 +169,10 @@ name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' + - string: + name: CEPH_DISKS + default: /srv + description: "Disks to use by ceph (comma separated list)" - parameter: name: 'intel-pod8-defaults' @@ -186,6 +194,43 @@ description: 'Git URL to use on this Jenkins Slave' - parameter: + name: 'ericsson-build-defaults' + parameters: + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - ericsson-build + default-slaves: + - ericsson-build + - string: + name: INSTALLER_VERSION + default: stable + description: 'Version of the installer to deploy' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' +- parameter: + name: 'huawei-build-defaults' + parameters: + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - huawei-build + default-slaves: + - huawei-build + - string: + name: INSTALLER_VERSION + default: stable + description: 'Version of the installer to deploy' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' + +- parameter: name: 'huawei-us-deploy-bare-1-defaults' parameters: - node: @@ -203,6 +248,24 @@ name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' +- parameter: + name: 'huawei-deploy-vm-defaults' + parameters: + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - huawei-deploy-vm + default-slaves: + - huawei-deploy-vm + - string: + name: INSTALLER_VERSION + default: stable + description: 'Version of the installer to deploy' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' - parameter: name: 'opnfv-build-defaults' @@ -271,7 +334,7 @@ description: 'Git URL to use on this Jenkins Slave' - parameter: - name: 'zte-build-1' + name: 'zte-build-1-defaults' parameters: - node: name: SLAVE_NAME @@ -304,10 +367,17 @@ default: https://gerrit.opnfv.org/gerrit/$PROJECT description: 'Git URL to use on this Jenkins Slave' - string: - name: LAB_CONFIG - default: "/home/opnfv/repos/functest" - description: "Openstack credentials on the dev lab" - + name: CEPH_DISKS + default: /dev/sdb /dev/sdc + description: "Disks to use by ceph by default (space separated list)" + - string: + name: CEPH_DISKS_CONTROLLERS + default: /dev/sdb + description: "Disks to use by ceph on controler nodes (space separated list)" + - string: + name: EXTERNAL_NETWORK + default: ext-net;flat;161.105.231.2;161.105.231.62;161.105.231.1;161.105.231.0/26 + description: "External network to create (name;type;first ip;last ip; gateway;network)" - parameter: name: 'orange-test1-defaults' diff --git a/jjb/qtip/qtip.yml b/jjb/qtip/qtip.yml index fac9c333c..c442c6cd9 100644 --- a/jjb/qtip/qtip.yml +++ b/jjb/qtip/qtip.yml @@ -5,14 +5,17 @@ jobs: - 'qtip-verify-{stream}' - - 'qtip-dhrystone-serial-{pod}' - - 'qtip-ssl-parallel-{pod}' + - 'qtip-{installer_type}-{pod}-daily-{stream}' pod: - - dell-build: + - dell-us-testing-bm-1: node: 'dell-us-testing-bm-1' installer_type: 'fuel' installer_ip: '10.20.0.2' + - dell-us-deploying-bm3: + node: 'dell-us-deploying-bm3' + installer_type: 'fuel' + installer_ip: '10.20.0.2' # only master branch is enabled at the moment to keep no of jobs sane stream: @@ -66,7 +69,7 @@ echo "Nothing to verify!" - job-template: - name: 'qtip-dhrystone-serial-{pod}' + name: 'qtip-{installer_type}-{pod}-daily-{stream}' disabled: false @@ -84,41 +87,9 @@ default: '{installer_ip}' description: "Installer IP" - string: - name: GIT_BASE - default: https://gerrit.opnfv.org/gerrit/$PROJECT - description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: master - - builders: - - 'qtip-fetch-os-cred' - - 'qtip-dhrystone-serial' - - triggers: - - timed: '0 20 * * *' - -- job-template: - name: 'qtip-ssl-parallel-{pod}' - - disabled: false - - node: '{node}' - - parameters: - - project-parameter: - project: '{project}' - - string: - name: INSTALLER_TYPE - default: '{installer_type}' - description: "Installer name that is used for deployment." - - string: - name: INSTALLER_IP - default: '{installer_ip}' - description: "IP address of the installer" + name: NODE_NAME + default: '{node}' + description: "Name of the POD" - string: name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT @@ -131,49 +102,78 @@ branch: master builders: - - 'qtip-fetch-os-cred' - - 'qtip-ssl-parallel' + - 'qtip-cleanup' + - 'qtip-set-env' + - 'qtip-run-suite' triggers: - - timed: '0 0,11,18 * * *' + - timed: '0 0,13,15,20 * * *' + ########################### #biuilder macros ########################### - builder: - name: qtip-dhrystone-serial - + name: qtip-set-env builders: - shell: | - #! /bin/bash - if [[ ! -f $WORKSPACE/opnfv-stack.sh ]]; then - echo "Unable to access file $WO:qRKSPACE/opnfv-openrc.sh" - exit 1 + #!/bin/bash + echo "Qtip: Start Docker and prepare environment" + envs="INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} -e NODE_NAME=${NODE_NAME}" + suite="TEST_CASE=all" + docker pull opnfv/qtip:latest + + cmd=" docker run -id -e $envs -e $suite opnfv/qtip:latest /bin/bash" + echo "Qtip: Running docker run command: ${cmd}" + ${cmd} + docker ps -a + container_id=$(docker ps | grep 'opnfv/qtip:latest' | awk '{print $1}' | head -1) + if [ $(docker ps | grep 'opnfv/qtip' | wc -l) == 0 ]; then + echo "The container opnfv/qtip with ID=${container_id} has not been properly started. Exiting..." + exit 1 fi - source $WORKSPACE/opnfv-stack.sh - cd $WORKSPACE - echo "Running QTIP dhrystone with First Test case" - python qtip.py -s compute -b dhrystone_serial.yaml - - builder: - name: qtip-ssl-parallel + name: qtip-run-suite + builders: + - shell: | + #!/bin/bash + QTIP_REPO=/home/opnfv/repos/qtip + docker exec -t ${container_id} $QTIP_REPO/docker/run_qtip.sh + +- builder: + name: qtip-cleanup builders: - shell: | #!/bin/bash - if [[ ! -f $WORKSPACE/opnfv-stack.sh ]]; then - echo "Unable to access file $WORKSPACE/opnfv-openrc.sh" - exit 1 + + echo "Cleaning up QTIP docker containers/images..." + # Remove previous running containers if exist + if [[ ! -z $(docker ps -a | grep opnfv/qtip) ]]; then + echo "Removing existing opnfv/qtip containers..." + running_containers= $(docker ps | grep opnfv/qtip | awk '{print $1}') + docker stop ${running_containers} + all_containers= $(docker ps -a | grep opnfv/qtip | awk '{print $1}') + docker rm ${all_containers} + fi + + # Remove existing images if exist + if [[ ! -z $(docker images | grep opnfv/qtip) ]]; then + echo "Docker images to remove:" + docker images | head -1 && docker images | grep opnfv/qtip + image_tags=($(docker images | grep opnfv/qtip | awk '{print $2}')) + for tag in "${image_tags[@]}"; do + echo "Removing docker image opnfv/qtip:$tag..." + docker rmi opnfv/qtip:$tag + done fi - source $WORKSPACE/opnfv-stack.sh - cd $WORKSPACE - echo "Running QTIP dhrystone with Second Test case" - python qtip.py -s compute -b ssl_parallel.yaml - builder: name: qtip-fetch-os-cred builders: - shell: | + #!/bin/bash + echo $HOME echo $WORKSPACE cd $WORKSPACE diff --git a/jjb/releng-macros.yaml b/jjb/releng-macros.yaml index 5a1568e23..1f812f038 100644 --- a/jjb/releng-macros.yaml +++ b/jjb/releng-macros.yaml @@ -159,7 +159,9 @@ name: build-html-and-pdf-docs-output builders: - shell: | - #!/bin/bash -e + #!/bin/bash + set -o errexit + set -o xtrace export PATH=$PATH:/usr/local/bin/ git clone ssh://gerrit.opnfv.org:29418/releng GERRIT_COMMENT=gerrit_comment.txt ./releng/utils/docs-build.sh @@ -168,8 +170,10 @@ name: upload-under-review-docs-to-opnfv-artifacts builders: - shell: | - #!/bin/bash -e + #!/bin/bash + set -o errexit set -o pipefail + set -o xtrace export PATH=$PATH:/usr/local/bin/ [[ $GERRIT_CHANGE_NUMBER =~ .+ ]] @@ -204,8 +208,10 @@ name: upload-generated-docs-to-opnfv-artifacts builders: - shell: | - #!/bin/bash -e + #!/bin/bash + set -o errexit set -o pipefail + set -o xtrace export PATH=$PATH:/usr/local/bin/ [[ -d docs_output ]] || exit 0 @@ -238,7 +244,10 @@ name: report-docs-build-result-to-gerrit builders: - shell: | - #!/bin/bash -e + #!/bin/bash + set -o errexit + set -o pipefail + set -o xtrace export PATH=$PATH:/usr/local/bin/ if [[ -e gerrit_comment.txt ]] ; then echo @@ -256,7 +265,10 @@ name: remove-old-docs-from-opnfv-artifacts builders: - shell: | - #!/bin/bash -e + #!/bin/bash + set -o errexit + set -o pipefail + set -o xtrace export PATH=$PATH:/usr/local/bin/ [[ $GERRIT_CHANGE_NUMBER =~ .+ ]] diff --git a/jjb/yardstick/yardstick-ci-jobs.yml b/jjb/yardstick/yardstick-ci-jobs.yml index 1cb6d1f08..87485c44f 100644 --- a/jjb/yardstick/yardstick-ci-jobs.yml +++ b/jjb/yardstick/yardstick-ci-jobs.yml @@ -29,6 +29,9 @@ - intel-pod5: installer: joid <<: *brahmaputra + - huawei-us-deploy-bare-1: + installer: compass + <<: *brahmaputra #-------------------------------- # master #-------------------------------- @@ -41,9 +44,15 @@ - intel-pod6: installer: joid <<: *master - - huawei-us-deploy-bare-1: + - intel-pod8: installer: compass <<: *master + - zte-build-1: + installer: fuel + <<: *master + - orange-pod2: + installer: joid + <<: *master #-------------------------------- loop: - daily @@ -170,6 +179,14 @@ description: 'Arguments to use in order to choose the backend DB' - parameter: + name: 'yardstick-params-intel-pod8' + parameters: + - string: + name: YARDSTICK_DB_BACKEND + default: '' + description: 'Arguments to use in order to choose the backend DB' + +- parameter: name: 'yardstick-params-ericsson-pod1' parameters: - string: @@ -200,3 +217,19 @@ name: YARDSTICK_DB_BACKEND default: '' description: 'Arguments to use in order to choose the backend DB' + +- parameter: + name: 'yardstick-params-zte-build-1' + parameters: + - string: + name: YARDSTICK_DB_BACKEND + default: '' + description: 'Arguments to use in order to choose the backend DB' + +- parameter: + name: 'yardstick-params-orange-pod2' + parameters: + - string: + name: YARDSTICK_DB_BACKEND + default: '' + description: 'Arguments to use in order to choose the backend DB' diff --git a/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py b/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py index f9e434226..8d5326eb4 100755 --- a/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py +++ b/utils/test/result_collection_api/dashboard/bottlenecks2Dashboard.py @@ -14,6 +14,9 @@ # a new method format_<Test_case>_for_dashboard(results) # v0.1: basic example with methods for Rubbos. # +import os +import requests +import json def get_bottlenecks_cases(): @@ -21,7 +24,7 @@ def get_bottlenecks_cases(): get the list of the supported test cases TODO: update the list when adding a new test case for the dashboard """ - return ["rubbos"] + return ["rubbos", "tu1", "tu3"] def check_bottlenecks_case_exist(case): @@ -54,7 +57,6 @@ def format_bottlenecks_for_dashboard(case, results): def format_rubbos_for_dashboard(results): - """ Post processing for the Rubbos test case """ @@ -76,24 +78,106 @@ def format_rubbos_for_dashboard(results): return test_data -# for local test -import json +def format_tu1_for_dashboard(results): + test_data = [{'description': 'Tu-1 performance result'}] + line_element = [] + bar_element = {} + last_result = results[-1]["details"] + for key in sorted(last_result): + bandwith = last_result[key]["Bandwidth"] + pktsize = int(key) + line_element.append({'x': pktsize, + 'y': bandwith * 1000}) + bar_element[key] = bandwith * 1000 + # graph1, line + test_data.append({'name': "VM2VM max single directional throughput", + 'info': {'type': "graph", + 'xlabel': 'pktsize', + 'ylabel': 'bandwith(kpps)'}, + 'data_set': line_element}) + # graph2, bar + test_data.append({'name': "VM2VM max single directional throughput", + 'info': {"type": "bar"}, + 'data_set': bar_element}) + return test_data + + +def format_tu3_for_dashboard(results): + test_data = [{'description': 'Tu-3 performance result'}] + new_element = [] + bar_element = {} + last_result = results[-1]["details"] + for key in sorted(last_result): + bandwith = last_result[key]["Bandwidth"] + pktsize = int(key) + new_element.append({'x': pktsize, + 'y': bandwith * 1000}) + bar_element[key] = bandwith * 1000 + # graph1, line + test_data.append({'name': "VM2VM max bidirectional throughput", + 'info': {'type': "graph", + 'xlabel': 'pktsize', + 'ylabel': 'bandwith(kpps)'}, + 'data_set': new_element}) + # graph2, bar + test_data.append({'name': "VM2VM max single directional throughput", + 'info': {"type": "bar"}, + 'data_set': bar_element}) + return test_data + + +############################ For local test ################################ + +def _read_sample_output(filename): + curr_path = os.path.dirname(os.path.abspath(__file__)) + output = os.path.join(curr_path, filename) + with open(output) as f: + sample_output = f.read() + + result = json.loads(sample_output) + return result + + +# Copy form functest/testcases/Dashboard/dashboard_utils.py +# and did some minor modification for local test. +def _get_results(db_url, test_criteria): + test_project = test_criteria["project"] + testcase = test_criteria["testcase"] + + # Build headers + headers = {'Content-Type': 'application/json'} + + # build the request + # if criteria is all => remove criteria + url = db_url + "/results?project=" + test_project + "&case=" + testcase + + # Send Request to Test DB + myData = requests.get(url, headers=headers) + + # Get result as a json object + myNewData = json.loads(myData.text) + + # Get results + myDataResults = myNewData['test_results'] + return myDataResults def _test(): - print('Post processing for the Rubbos test case begin<--') - results = '[{"details":[{"client":200,"throughput":20},{"client":300,"throughput":50}],"project_name":' \ - '"bottlenecks","pod_name":"unknown-pod","version":"unknown","installer":"fuel","description":' \ - '"bottlenecks test cases result","_id":"56793f11514bc5068a345da4","creation_date":' \ - '"2015-12-22 12:16:17.131438","case_name":"rubbos"},{"details":[{"client":200,"throughput":25},' \ - '{"client":300,"throughput":52}],"project_name":"bottlenecks","pod_name":"unknown-pod","version":' \ - '"unknown","installer":"fuel","description":"bottlenecks test cases result","_id":' \ - '"56793f11514bc5068a345da4","creation_date":"2015-12-23 12:16:17.131438","case_name":"rubbos"}]' + db_url = "http://213.77.62.197" + results = _get_results(db_url, {"project": "bottlenecks", "testcase": "rubbos"}) + test_result = format_rubbos_for_dashboard(results) + print json.dumps(test_result, indent=4) - print("the output is:") - print(format_rubbos_for_dashboard(json.loads(results))) - print('Post processing for the Rubbos test case end<--') + results = _get_results(db_url, {"project": "bottlenecks", "testcase": "tu1"}) + #results = _read_sample_output("sample") + #print json.dumps(results, indent=4) + test_result = format_tu1_for_dashboard(results) + print json.dumps(test_result, indent=4) + results = _get_results(db_url, {"project": "bottlenecks", "testcase": "tu3"}) + test_result = format_tu3_for_dashboard(results) + print json.dumps(test_result, indent=4) if __name__ == '__main__': _test() + diff --git a/utils/test/result_collection_api/dashboard/dashboard_utils.py b/utils/test/result_collection_api/dashboard/dashboard_utils.py index 06c90acf5..dccfe4f99 100644 --- a/utils/test/result_collection_api/dashboard/dashboard_utils.py +++ b/utils/test/result_collection_api/dashboard/dashboard_utils.py @@ -18,6 +18,12 @@ import os import re from functest2Dashboard import format_functest_for_dashboard, \ check_functest_case_exist +from yardstick2Dashboard import format_yardstick_for_dashboard, \ + check_yardstick_case_exist +from vsperf2Dashboard import format_vsperf_for_dashboard, \ + check_vsperf_case_exist +from bottlenecks2Dashboard import format_bottlenecks_for_dashboard, \ + check_bottlenecks_case_exist # any project test project wishing to provide dashboard ready values # must include at least 2 methods |