diff options
Diffstat (limited to 'jjb')
105 files changed, 3288 insertions, 1276 deletions
diff --git a/jjb/3rd_party_ci/create-apex-vms.sh b/jjb/3rd_party_ci/create-apex-vms.sh new file mode 100755 index 000000000..a076dd084 --- /dev/null +++ b/jjb/3rd_party_ci/create-apex-vms.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +# wipe the WORKSPACE +/bin/rm -rf $WORKSPACE/* + +# clone opnfv sdnvpn repo +git clone https://gerrit.opnfv.org/gerrit/p/sdnvpn.git $WORKSPACE/sdnvpn +. $WORKSPACE/sdnvpn/odl-pipeline/odl-pipeline-common.sh +pushd $LIB +./test_environment.sh --env-number $APEX_ENV_NUMBER --cloner-info $CLONER_INFO --snapshot-disks $SNAPSHOT_DISKS --vjump-hosts $VIRTUAL_JUMPHOSTS +popd diff --git a/jjb/3rd_party_ci/download-netvirt-artifact.sh b/jjb/3rd_party_ci/download-netvirt-artifact.sh new file mode 100755 index 000000000..0a48e3aec --- /dev/null +++ b/jjb/3rd_party_ci/download-netvirt-artifact.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +# wipe the WORKSPACE +/bin/rm -rf $WORKSPACE/* + +echo "Attempting to fetch the artifact location from ODL Jenkins" +CHANGE_DETAILS_URL="https://git.opendaylight.org/gerrit/changes/netvirt~master~$GERRIT_CHANGE_ID/detail" +# due to limitation with the Jenkins Gerrit Trigger, we need to use Gerrit REST API to get the change details +ODL_JOB_URL=$(curl -s $CHANGE_DETAILS_URL | grep netvirt-patch-test-current-carbon | tail -1 | \ + sed 's/\\n//g' | awk '{print $6}') +NETVIRT_ARTIFACT_URL="${ODL_JOB_URL}org.opendaylight.integration\$distribution-karaf/artifact/org.opendaylight.integration/distribution-karaf/0.6.0-SNAPSHOT/distribution-karaf-0.6.0-SNAPSHOT.tar.gz" +echo -e "URL to artifact is\n\t$NETVIRT_ARTIFACT_URL" + +echo "Downloading the artifact. This could take time..." +wget -q -O $NETVIRT_ARTIFACT $NETVIRT_ARTIFACT_URL +if [[ $? -ne 0 ]]; then + echo "The artifact does not exist! Probably removed due to ODL Jenkins artifact retention policy." + echo "Rerun netvirt-patch-test-current-carbon to get artifact rebuilt." + exit 1 +fi +echo "Download complete" +ls -al $NETVIRT_ARTIFACT diff --git a/jjb/3rd_party_ci/functest-netvirt.sh b/jjb/3rd_party_ci/functest-netvirt.sh new file mode 100755 index 000000000..adffaf42d --- /dev/null +++ b/jjb/3rd_party_ci/functest-netvirt.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -e + +# wipe the WORKSPACE +/bin/rm -rf $WORKSPACE/* + +echo "Hello World" diff --git a/jjb/3rd_party_ci/install-netvirt.sh b/jjb/3rd_party_ci/install-netvirt.sh new file mode 100755 index 000000000..96c4b9634 --- /dev/null +++ b/jjb/3rd_party_ci/install-netvirt.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e +# clone opnfv sdnvpn repo +git clone https://gerrit.opnfv.org/gerrit/p/sdnvpn.git $WORKSPACE/sdnvpn +. $WORKSPACE/sdnvpn/odl-pipeline/odl-pipeline-common.sh +pushd $LIB +./odl_reinstaller.sh --cloner-info $CLONER_INFO --odl-artifact $NETVIRT_ARTIFACT +popd
\ No newline at end of file diff --git a/jjb/3rd_party_ci/odl-netvirt.yml b/jjb/3rd_party_ci/odl-netvirt.yml new file mode 100644 index 000000000..ca1936156 --- /dev/null +++ b/jjb/3rd_party_ci/odl-netvirt.yml @@ -0,0 +1,211 @@ +- project: + name: 'netvirt' + + project: 'netvirt' + + installer: 'netvirt' +##################################### +# branch definitions +##################################### + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false +##################################### +# patch verification phases +##################################### + phase: + - 'create-apex-vms': + slave-label: 'ericsson-virtual5' + - 'install-netvirt': + slave-label: 'odl-netvirt-virtual' + - 'functest': + slave-label: 'odl-netvirt-virtual' + - 'postprocess': + slave-label: 'odl-netvirt-virtual' +##################################### +# jobs +##################################### + jobs: + - 'odl-netvirt-verify-virtual-{stream}' + - 'odl-netvirt-verify-virtual-{phase}-{stream}' +##################################### +# job templates +##################################### +- job-template: + name: 'odl-netvirt-verify-virtual-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 5 + max-per-node: 1 + option: 'project' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - string: + name: NETVIRT_ARTIFACT + default: $WORKSPACE/distribution-karaf.tar.gz + - 'odl-netvirt-virtual-defaults' + + triggers: + - gerrit: + server-name: 'git.opendaylight.org' + trigger-on: + - comment-added-contains-event: + comment-contains-value: 'https://jenkins.opendaylight.org/releng/job/netvirt-patch-test-current-carbon/.*?/ : SUCCESS' + - comment-added-contains-event: + comment-contains-value: 'https://jenkins.opendaylight.org/releng/job/netvirt-patch-test-current-carbon/.*?/ : UNSTABLE' + - comment-added-contains-event: + comment-contains-value: 'opnfv-test' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + readable-message: true + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: create-apex-vms + condition: SUCCESSFUL + projects: + - name: 'odl-netvirt-verify-virtual-create-apex-vms-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID + GERRIT_PATCHSET_NUMBER=$GERRIT_PATCHSET_NUMBER + GERRIT_PATCHSET_REVISION=$GERRIT_PATCHSET_REVISION + NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT + APEX_ENV_NUMBER=$APEX_ENV_NUMBER + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: install-netvirt + condition: SUCCESSFUL + projects: + - name: 'odl-netvirt-verify-virtual-install-netvirt-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID + GERRIT_PATCHSET_NUMBER=$GERRIT_PATCHSET_NUMBER + GERRIT_PATCHSET_REVISION=$GERRIT_PATCHSET_REVISION + NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: functest + condition: SUCCESSFUL + projects: + - name: 'functest-netvirt-virtual-suite-{stream}' + predefined-parameters: | + FUNCTEST_SUITE_NAME=vping_userdata,bgpvpn + RC_FILE_PATH=/home/jenkins/cloner-info/overcloudrc + DEPLOY_SCENARIO=os-odl_l2-bgpvpn-noha + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: false + - multijob: + name: postprocess + condition: ALWAYS + projects: + - name: 'odl-netvirt-verify-virtual-postprocess-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_ID=$GERRIT_CHANGE_ID + GERRIT_PATCHSET_NUMBER=$GERRIT_PATCHSET_NUMBER + GERRIT_PATCHSET_REVISION=$GERRIT_PATCHSET_REVISION + NETVIRT_ARTIFACT=$NETVIRT_ARTIFACT + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + +- job-template: + name: 'odl-netvirt-verify-virtual-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 5 + max-per-node: 1 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'odl-netvirt-verify-virtual-install-.*' + - 'odl-netvirt-verify-virtual-functest-.*' + - 'odl-netvirt-verify-virtual-postprocess-.*' + block-level: 'NODE' + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 360 + fail: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - '{slave-label}-defaults' + - '{installer}-defaults' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-verify-{phase}-builder' +##################################### +# builder macros +##################################### +- builder: + name: 'netvirt-verify-create-apex-vms-builder' + builders: + - shell: + !include-raw: ./create-apex-vms.sh +- builder: + name: 'netvirt-verify-install-netvirt-builder' + builders: + - shell: + !include-raw: ./download-netvirt-artifact.sh + - shell: + !include-raw: ./install-netvirt.sh +- builder: + name: 'netvirt-verify-functest-builder' + builders: + - shell: + !include-raw: ./functest-netvirt.sh +- builder: + name: 'netvirt-verify-postprocess-builder' + builders: + - shell: + !include-raw: ./postprocess-netvirt.sh diff --git a/jjb/3rd_party_ci/postprocess-netvirt.sh b/jjb/3rd_party_ci/postprocess-netvirt.sh new file mode 100755 index 000000000..2e99477c5 --- /dev/null +++ b/jjb/3rd_party_ci/postprocess-netvirt.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +# wipe the WORKSPACE +if [ -z ${WORKSPACE} ]; then echo "WORKSPACE is unset"; else echo "WORKSPACE is set to \"$WORKSPACE\""; fi +WORKSPACE=${WORKSPACE:-$PWD} +/bin/rm -rf $WORKSPACE/* +set -e +# clone opnfv sdnvpn repo +git clone https://gerrit.opnfv.org/gerrit/p/sdnvpn.git $WORKSPACE/sdnvpn +. $WORKSPACE/sdnvpn/odl-pipeline/odl-pipeline-common.sh +pushd $LIB +./post_process.sh +popd
\ No newline at end of file diff --git a/jjb/apex/apex-build.sh b/jjb/apex/apex-build.sh index e3e3f6194..ee1dfb5d3 100755 --- a/jjb/apex/apex-build.sh +++ b/jjb/apex/apex-build.sh @@ -23,7 +23,7 @@ fi # start the build cd $WORKSPACE/ci ./build.sh $BUILD_ARGS -RPM_VERSION=$(grep Version: $BUILD_DIRECTORY/rpm_specs/opnfv-apex.spec | awk '{ print $2 }')-$(echo $OPNFV_ARTIFACT_VERSION | tr -d '_-') +RPM_VERSION=$(grep Version: $WORKSPACE/build/rpm_specs/opnfv-apex.spec | awk '{ print $2 }')-$(echo $OPNFV_ARTIFACT_VERSION | tr -d '_-') # list the contents of BUILD_OUTPUT directory echo "Build Directory is ${BUILD_DIRECTORY}" echo "Build Directory Contents:" diff --git a/jjb/apex/apex-deploy.sh b/jjb/apex/apex-deploy.sh index e21387ac6..8d5c4cb13 100755 --- a/jjb/apex/apex-deploy.sh +++ b/jjb/apex/apex-deploy.sh @@ -72,7 +72,7 @@ if [[ "$BUILD_DIRECTORY" == *verify* ]]; then DEPLOY_SETTINGS_DIR="${WORKSPACE}/config/deploy" NETWORK_SETTINGS_DIR="${WORKSPACE}/config/network" DEPLOY_CMD="$(pwd)/deploy.sh" - RESOURCES="${WORKSPACE}/build/images/" + RESOURCES="${WORKSPACE}/.build/" CONFIG="${WORKSPACE}/build" LIB="${WORKSPACE}/lib" # Make sure python34 deps are installed @@ -176,6 +176,8 @@ else # settings for bare metal deployment if [ "$IPV6_FLAG" == "True" ]; then NETWORK_FILE="/root/network/network_settings_v6.yaml" + elif [[ "$JOB_NAME" == *master* ]]; then + NETWORK_FILE="/root/network/network_settings-master.yaml" else NETWORK_FILE="/root/network/network_settings.yaml" fi diff --git a/jjb/apex/apex-upload-artifact.sh b/jjb/apex/apex-upload-artifact.sh index f54e4c55a..64f13f4e6 100755 --- a/jjb/apex/apex-upload-artifact.sh +++ b/jjb/apex/apex-upload-artifact.sh @@ -11,6 +11,8 @@ echo # source the opnfv.properties to get ARTIFACT_VERSION source $WORKSPACE/opnfv.properties +BUILD_DIRECTORY=${WORKSPACE}/.build + # clone releng repository echo "Cloning releng repository..." [ -d releng ] && rm -rf releng diff --git a/jjb/apex/apex.yml b/jjb/apex/apex.yml index 89965d77b..70731b568 100644 --- a/jjb/apex/apex.yml +++ b/jjb/apex/apex.yml @@ -9,6 +9,9 @@ - 'apex-deploy-virtual-{scenario}-{stream}' - 'apex-deploy-baremetal-{scenario}-{stream}' - 'apex-daily-{stream}' + - 'apex-daily-colorado' + - 'apex-build-colorado' + - 'apex-deploy-baremetal-os-odl_l2-fdio-ha-colorado' # stream: branch with - in place of / (eg. stable-arno) # branch: branch (eg. stable/arno) @@ -16,37 +19,9 @@ - master: branch: 'master' gs-pathname: '' - block-stream: 'colorado' slave: 'lf-pod1' verify-slave: 'apex-verify-master' daily-slave: 'apex-daily-master' - - colorado: - branch: 'stable/colorado' - gs-pathname: '/colorado' - block-stream: 'master' - slave: 'lf-pod1' - verify-slave: 'apex-verify-colorado' - daily-slave: 'apex-daily-colorado' - disabled: false - - stream1: - - master: - branch: 'master' - gs-pathname: '' - block-stream: 'colorado' - slave: 'lf-pod1' - verify-slave: 'apex-verify-master' - daily-slave: 'apex-daily-master' - - stream2: - - colorado: - branch: 'stable/colorado' - gs-pathname: '/colorado' - block-stream: 'master' - slave: 'lf-pod1' - verify-slave: 'apex-verify-colorado' - daily-slave: 'apex-daily-colorado' - disabled: false project: 'apex' @@ -56,14 +31,12 @@ - 'os-nosdn-nofeature-ha-ipv6' - 'os-nosdn-ovs-noha' - 'os-nosdn-fdio-noha' - - 'os-odl_l2-nofeature-ha' - - 'os-odl_l2-bgpvpn-ha' + - 'os-odl_l3-bgpvpn-ha' - 'os-odl_l2-fdio-noha' + - 'os-odl_l2-fdio-ha' - 'os-odl_l2-sfc-noha' - 'os-odl_l3-nofeature-ha' - 'os-onos-nofeature-ha' - - 'os-onos-sfc-ha' - - 'os-ocl-nofeature-ha' platform: - 'baremetal' @@ -90,13 +63,11 @@ description: "Used for overriding the GIT URL coming from parameters macro." scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -146,13 +117,11 @@ description: "Used for overriding the GIT URL coming from parameters macro." scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -213,7 +182,7 @@ block: true same-node: true - trigger-builds: - - project: 'apex-deploy-virtual-os-odl_l2-nofeature-ha-{stream}' + - project: 'apex-deploy-virtual-os-odl_l3-nofeature-ha-{stream}' predefined-parameters: | BUILD_DIRECTORY=apex-verify-{stream} OPNFV_CLEAN=yes @@ -223,7 +192,7 @@ - trigger-builds: - project: 'functest-apex-{verify-slave}-suite-{stream}' predefined-parameters: | - DEPLOY_SCENARIO=os-odl_l2-nofeature-ha + DEPLOY_SCENARIO=os-odl_l3-nofeature-ha FUNCTEST_SUITE_NAME=healthcheck block: true same-node: true @@ -251,10 +220,7 @@ description: "Used for overriding the GIT URL coming from parameters macro." scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm properties: - build-blocker: @@ -311,10 +277,7 @@ description: "Used for overriding the GIT URL coming from parameters macro." scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm properties: - build-blocker: @@ -369,10 +332,7 @@ description: "Used for overriding the GIT URL coming from parameters macro." scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm properties: - build-blocker: @@ -390,7 +350,7 @@ - trigger-builds: - project: 'apex-deploy-virtual-os-nosdn-nofeature-noha-{stream}' predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build + BUILD_DIRECTORY=apex-build-{stream}/.build OPNFV_CLEAN=yes git-revision: false same-node: true @@ -412,14 +372,13 @@ disabled: false scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - apex-parameter: gs-pathname: '{gs-pathname}' - string: @@ -459,14 +418,13 @@ disabled: false scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - apex-parameter: gs-pathname: '{gs-pathname}' - string: @@ -502,14 +460,13 @@ disabled: false scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - apex-parameter: gs-pathname: '{gs-pathname}' @@ -536,7 +493,7 @@ - trigger-builds: - project: 'apex-deploy-baremetal-os-nosdn-nofeature-ha-{stream}' predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build + BUILD_DIRECTORY=apex-build-{stream}/.build OPNFV_CLEAN=yes git-revision: true same-node: true @@ -564,39 +521,9 @@ failure-threshold: 'never' unstable-threshold: 'FAILURE' - trigger-builds: - - project: 'apex-deploy-baremetal-os-odl_l2-nofeature-ha-{stream}' - predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build - OPNFV_CLEAN=yes - git-revision: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - block: true - - trigger-builds: - - project: 'functest-apex-{daily-slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-nofeature-ha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'yardstick-apex-{slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-nofeature-ha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - project: 'apex-deploy-baremetal-os-odl_l3-nofeature-ha-{stream}' predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build + BUILD_DIRECTORY=apex-build-{stream}/.build OPNFV_CLEAN=yes git-revision: true same-node: true @@ -623,200 +550,144 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' + + +# Colorado Build +- job-template: + name: 'apex-build-colorado' + + # Job template for builds + # + # Required Variables: + # stream: branch with - in place of / (eg. stable) + # branch: branch (eg. stable) + node: 'apex-daily-colorado' + + disabled: false + + concurrent: true + + parameters: + - project-parameter: + project: '{project}' + - apex-parameter: + gs-pathname: '/colorado' + - gerrit-parameter: + branch: 'stable/colorado' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: "Used for overriding the GIT URL coming from parameters macro." + + scm: + - git-scm + + properties: + - build-blocker: + use-build-blocker: true + block-level: 'NODE' + blocking-jobs: + - 'apex-deploy.*' + - throttle: + max-per-node: 1 + max-total: 10 + option: 'project' + + builders: + - 'apex-build' + - 'apex-upload-artifact' + + +# Colorado FDIO Deploy +- job-template: + name: 'apex-deploy-baremetal-os-odl_l2-fdio-ha-colorado' + + # Job template for baremetal deployment + # + # Required Variables: + # stream: branch with - in place of / (eg. stable) + # branch: branch (eg. stable) + node: 'lf-pod1' + + disabled: false + + scm: + - git-scm + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: 'stable/colorado' + - apex-parameter: + gs-pathname: '/colorado' + - string: + name: DEPLOY_SCENARIO + default: 'os-odl_l2-fdio-ha' + description: "Scenario to deploy with." + + properties: + - build-blocker: + use-build-blocker: true + block-level: 'NODE' + blocking-jobs: + - 'apex-verify.*' + - 'apex-deploy.*' + - 'apex-build.*' + + + builders: + - 'apex-deploy' + - 'apex-workspace-cleanup' + +# Colorado FDIO Daily +- job-template: + name: 'apex-daily-colorado' + + # Job template for daily build + # + # Required Variables: + # stream: branch with - in place of / (eg. stable) + # branch: branch (eg. stable) + node: 'apex-daily-colorado' + + disabled: false + + scm: + - git-scm + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: 'stable/colorado' + - apex-parameter: + gs-pathname: '/colorado' + + properties: + - build-blocker: + use-build-blocker: true + block-level: 'NODE' + blocking-jobs: + - 'apex-verify.*' + - 'apex-deploy.*' + - 'apex-build.*' + - 'apex-runner.*' + + triggers: + - 'apex-colorado' + + builders: - trigger-builds: - - project: 'apex-deploy-baremetal-os-onos-nofeature-ha-{stream}' - predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build - OPNFV_CLEAN=yes - git-revision: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - block: true - - trigger-builds: - - project: 'functest-apex-{daily-slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-onos-nofeature-ha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'yardstick-apex-{slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-onos-nofeature-ha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'apex-deploy-baremetal-os-odl_l2-bgpvpn-ha-{stream}' - predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build - OPNFV_CLEAN=yes - git-revision: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - block: true - - trigger-builds: - - project: 'functest-apex-{daily-slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-bgpvpn-ha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'yardstick-apex-{slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-bgpvpn-ha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'apex-deploy-baremetal-os-onos-sfc-ha-{stream}' - predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build - OPNFV_CLEAN=yes - git-revision: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - block: true - - trigger-builds: - - project: 'functest-apex-{daily-slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-onos-sfc-ha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'yardstick-apex-{slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-onos-sfc-ha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'apex-deploy-baremetal-os-odl_l2-sfc-noha-{stream}' - predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build - OPNFV_CLEAN=yes - git-revision: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - block: true - - trigger-builds: - - project: 'functest-apex-{daily-slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-sfc-noha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'yardstick-apex-{slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-sfc-noha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'apex-deploy-baremetal-os-odl_l2-fdio-noha-{stream}' - predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build - OPNFV_CLEAN=yes - git-revision: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - block: true - - trigger-builds: - - project: 'functest-apex-{daily-slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-fdio-noha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'yardstick-apex-{slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-odl_l2-fdio-noha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'apex-deploy-baremetal-os-nosdn-fdio-noha-{stream}' - predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build - OPNFV_CLEAN=yes - git-revision: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - block: true - - trigger-builds: - - project: 'functest-apex-{daily-slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-nosdn-fdio-noha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'yardstick-apex-{slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-nosdn-fdio-noha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'apex-deploy-virtual-os-nosdn-nofeature-ha-ipv6-{stream}' - predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build - OPNFV_CLEAN=yes + - project: 'apex-build-colorado' git-revision: true + current-parameters: true same-node: true - block-thresholds: - build-step-failure-threshold: 'never' block: true - trigger-builds: - - project: 'apex-deploy-baremetal-os-nosdn-ovs-noha-{stream}' + - project: 'apex-deploy-baremetal-os-odl_l2-fdio-ha-colorado' predefined-parameters: | - BUILD_DIRECTORY=apex-build-{stream}/build + BUILD_DIRECTORY=apex-build-colorado/.build OPNFV_CLEAN=yes git-revision: true same-node: true @@ -824,19 +695,9 @@ build-step-failure-threshold: 'never' block: true - trigger-builds: - - project: 'functest-apex-{daily-slave}-daily-{stream}' + - project: 'functest-apex-apex-daily-colorado-daily-colorado' predefined-parameters: - DEPLOY_SCENARIO=os-nosdn-ovs-noha - block: true - same-node: true - block-thresholds: - build-step-failure-threshold: 'never' - failure-threshold: 'never' - unstable-threshold: 'FAILURE' - - trigger-builds: - - project: 'yardstick-apex-{slave}-daily-{stream}' - predefined-parameters: - DEPLOY_SCENARIO=os-nosdn-ovs-noha + DEPLOY_SCENARIO=os-odl_l2-fdio-ha block: true same-node: true block-thresholds: @@ -949,11 +810,11 @@ - trigger: name: 'apex-master' triggers: - - timed: '0 0 20 8 *' + - timed: '0 3 * * *' - trigger: name: 'apex-colorado' triggers: - - timed: '0 3 * * *' + - timed: '0 12 * * *' - trigger: name: 'apex-gs-clean-{stream}' triggers: diff --git a/jjb/armband/armband-ci-jobs.yml b/jjb/armband/armband-ci-jobs.yml index 2122959a9..22e1dafba 100644 --- a/jjb/armband/armband-ci-jobs.yml +++ b/jjb/armband/armband-ci-jobs.yml @@ -160,6 +160,23 @@ build-step-failure-threshold: 'never' failure-threshold: 'never' unstable-threshold: 'FAILURE' + # 1.dovetail only master by now, not sync with A/B/C branches + # 2.here the stream means the SUT stream, dovetail stream is defined in its own job + # 3.only debug testsuite here(includes 3 basic testcase, + # i.e. one tempest smoke ipv6, two vping from functest) + # 4.not used for release criteria or compliance, + # only to debug the dovetail tool bugs with arm pods + - trigger-builds: + - project: 'dovetail-{installer}-{pod}-debug-{stream}' + current-parameters: false + predefined-parameters: + DEPLOY_SCENARIO={scenario} + block: true + same-node: true + block-thresholds: + build-step-failure-threshold: 'never' + failure-threshold: 'never' + unstable-threshold: 'FAILURE' - job-template: name: '{installer}-deploy-{pod}-daily-{stream}' @@ -182,6 +199,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - '{installer}-defaults' - '{slave-label}-defaults': installer: '{installer}' @@ -192,10 +211,7 @@ gs-pathname: '{gs-pathname}' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm wrappers: - build-name: @@ -303,31 +319,31 @@ - trigger: name: 'fuel-os-odl_l2-nofeature-ha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 1' - trigger: name: 'fuel-os-nosdn-nofeature-ha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 2' - trigger: name: 'fuel-os-odl_l3-nofeature-ha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 3' - trigger: name: 'fuel-os-odl_l2-bgpvpn-ha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 4' - trigger: name: 'fuel-os-odl_l2-nofeature-noha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 5' - trigger: name: 'fuel-os-odl_l2-sfc-ha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 6' - trigger: name: 'fuel-os-odl_l2-sfc-noha-armband-virtual-master-trigger' triggers: - - timed: '' + - timed: '0 2 * * 7' #-------------------------------------------------------------------- # Enea Armband CI Virtual Triggers running against colorado branch #-------------------------------------------------------------------- diff --git a/jjb/armband/armband-deploy.sh b/jjb/armband/armband-deploy.sh index c8e58afa8..4df9acfd8 100755 --- a/jjb/armband/armband-deploy.sh +++ b/jjb/armband/armband-deploy.sh @@ -61,16 +61,33 @@ if [[ $LAB_CONFIG_URL =~ ^(git|ssh):// ]]; then fi fi +if [[ "$NODE_NAME" =~ "virtual" ]]; then + POD_NAME="virtual_kvm" +fi + # releng wants us to use nothing else but opnfv.iso for now. We comply. ISO_FILE=$WORKSPACE/opnfv.iso # log file name FUEL_LOG_FILENAME="${JOB_NAME}_${BUILD_NUMBER}.log.tar.gz" +# Deploy Cache (to enable just create the deploy-cache subdir) +# NOTE: Only available when ISO files are cached using ISOSTORE mechanism +DEPLOY_CACHE=${ISOSTORE:-/iso_mount/opnfv_ci}/${GIT_BRANCH##*/}/deploy-cache +if [[ -d "${DEPLOY_CACHE}" ]]; then + echo "Deploy cache dir present." + echo "--------------------------------------------------------" + echo "Fuel@OPNFV deploy cache: ${DEPLOY_CACHE}" + DEPLOY_CACHE="-C ${DEPLOY_CACHE}" +else + DEPLOY_CACHE="" +fi + # construct the command DEPLOY_COMMAND="$WORKSPACE/ci/deploy.sh -b ${LAB_CONFIG_URL} \ -l $LAB_NAME -p $POD_NAME -s $DEPLOY_SCENARIO -i file://${ISO_FILE} \ - -H -B ${DEFAULT_BRIDGE:-pxebr} -S $TMPDIR -L $WORKSPACE/$FUEL_LOG_FILENAME" + -H -B ${DEFAULT_BRIDGE:-pxebr} -S $TMPDIR -L $WORKSPACE/$FUEL_LOG_FILENAME \ + ${DEPLOY_CACHE}" # log info to console echo "Deployment parameters" diff --git a/jjb/armband/armband-project-jobs.yml b/jjb/armband/armband-project-jobs.yml index 4b2a7b50e..e3853fa16 100644 --- a/jjb/armband/armband-project-jobs.yml +++ b/jjb/armband/armband-project-jobs.yml @@ -35,16 +35,15 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - 'opnfv-build-arm-defaults' - '{installer}-defaults' - armband-project-parameter: gs-pathname: '{gs-pathname}' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm triggers: - pollscm: diff --git a/jjb/armband/armband-verify-jobs.yml b/jjb/armband/armband-verify-jobs.yml index 90fdd7eeb..9bfe4f4fd 100644 --- a/jjb/armband/armband-verify-jobs.yml +++ b/jjb/armband/armband-verify-jobs.yml @@ -53,21 +53,17 @@ option: 'project' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -183,15 +179,10 @@ block-level: 'NODE' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true diff --git a/jjb/availability/availability.yml b/jjb/availability/availability.yml index c42efff11..ae9f163e6 100644 --- a/jjb/availability/availability.yml +++ b/jjb/availability/availability.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/fastpathmetrics/fastpathmetrics.yml b/jjb/barometer/barometer.yml index 40df385d8..4afe40e38 100644 --- a/jjb/fastpathmetrics/fastpathmetrics.yml +++ b/jjb/barometer/barometer.yml @@ -3,14 +3,14 @@ # They will only be enabled on request by projects! ################################################### - project: - name: fastpathmetrics + name: barometer project: '{name}' jobs: - - 'fastpathmetrics-verify-{stream}' - - 'fastpathmetrics-merge-{stream}' - - 'fastpathmetrics-daily-{stream}' + - 'barometer-verify-{stream}' + - 'barometer-merge-{stream}' + - 'barometer-daily-{stream}' stream: - master: @@ -23,7 +23,7 @@ disabled: false - job-template: - name: 'fastpathmetrics-verify-{stream}' + name: 'barometer-verify-{stream}' disabled: '{obj:disabled}' @@ -35,13 +35,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -70,7 +68,7 @@ make - job-template: - name: 'fastpathmetrics-merge-{stream}' + name: 'barometer-merge-{stream}' project-type: freestyle @@ -93,13 +91,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -118,11 +114,12 @@ - shell: | pwd cd src - make clobber - make + ./install_build_deps.sh + sudo make clobber + sudo make - job-template: - name: 'fastpathmetrics-daily-{stream}' + name: 'barometer-daily-{stream}' project-type: freestyle @@ -145,10 +142,7 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - timed: '@midnight' @@ -157,5 +151,6 @@ - shell: | pwd cd src - make clobber - make + ./install_build_deps.sh + sudo make clobber + sudo make diff --git a/jjb/bottlenecks/bottlenecks-ci-jobs.yml b/jjb/bottlenecks/bottlenecks-ci-jobs.yml index 7f2e6bf8a..0054dddaf 100644 --- a/jjb/bottlenecks/bottlenecks-ci-jobs.yml +++ b/jjb/bottlenecks/bottlenecks-ci-jobs.yml @@ -95,6 +95,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - '{slave-label}-defaults' - '{installer}-defaults' - 'bottlenecks-params-{slave-label}' @@ -119,10 +121,7 @@ description: "docker image tag used for version control" scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm builders: - 'bottlenecks-env-cleanup' diff --git a/jjb/bottlenecks/bottlenecks-project-jobs.yml b/jjb/bottlenecks/bottlenecks-project-jobs.yml index 523d363f7..a461c6a66 100644 --- a/jjb/bottlenecks/bottlenecks-project-jobs.yml +++ b/jjb/bottlenecks/bottlenecks-project-jobs.yml @@ -47,13 +47,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -87,13 +85,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -127,15 +123,14 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - 'opnfv-build-ubuntu-defaults' - bottlenecks-parameter: gs-packagepath: '{gs-packagepath}' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm builders: - 'bottlenecks-builder-upload-artifact' diff --git a/jjb/compass4nfv/compass-build.sh b/jjb/compass4nfv/compass-build.sh index f7eb580a9..093debba7 100644 --- a/jjb/compass4nfv/compass-build.sh +++ b/jjb/compass4nfv/compass-build.sh @@ -24,7 +24,7 @@ then fi cd $WORKSPACE/ -./build/build.sh --iso-dir $BUILD_DIRECTORY/ --iso-name compass.iso -c $CACHE_DIRECTORY +./build.sh --iso-dir $BUILD_DIRECTORY/ --iso-name compass.iso -c $CACHE_DIRECTORY # list the build artifacts ls -al $BUILD_DIRECTORY diff --git a/jjb/compass4nfv/compass-ci-jobs.yml b/jjb/compass4nfv/compass-ci-jobs.yml index eb9113114..b4910cfc2 100644 --- a/jjb/compass4nfv/compass-ci-jobs.yml +++ b/jjb/compass4nfv/compass-ci-jobs.yml @@ -10,10 +10,10 @@ stream: master branch: '{stream}' gs-pathname: '' - colorado: &colorado - stream: colorado - branch: 'stable/{stream}' - gs-pathname: '/{stream}' +# colorado: &colorado +# stream: colorado +# branch: 'stable/{stream}' +# gs-pathname: '/{stream}' #-------------------------------- # POD, INSTALLER, AND BRANCH MAPPING #-------------------------------- @@ -22,20 +22,20 @@ pod: - baremetal: slave-label: compass-baremetal - os-version: 'trusty' + os-version: 'xenial' <<: *master - virtual: slave-label: compass-virtual - os-version: 'trusty' + os-version: 'xenial' <<: *master - - baremetal: - slave-label: compass-baremetal - os-version: 'trusty' - <<: *colorado - - virtual: - slave-label: compass-virtual - os-version: 'trusty' - <<: *colorado +# - baremetal: +# slave-label: compass-baremetal +# os-version: 'trusty' +# <<: *colorado +# - virtual: +# slave-label: compass-virtual +# os-version: 'trusty' +# <<: *colorado #-------------------------------- # master #-------------------------------- @@ -48,35 +48,27 @@ - 'os-nosdn-nofeature-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' - openstack-os-version: '' - 'os-odl_l2-nofeature-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' - openstack-os-version: '' - 'os-odl_l3-nofeature-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' - openstack-os-version: '' - 'os-onos-nofeature-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' - openstack-os-version: '' - 'os-ocl-nofeature-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' - openstack-os-version: '' - 'os-onos-sfc-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' - openstack-os-version: '' - 'os-odl_l2-moon-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' - openstack-os-version: 'xenial' - 'os-nosdn-kvm-ha': disabled: false auto-trigger-name: 'compass-{scenario}-{pod}-{stream}-trigger' - openstack-os-version: '' jobs: - 'compass-{scenario}-{pod}-daily-{stream}' @@ -131,7 +123,6 @@ predefined-parameters: | DEPLOY_SCENARIO={scenario} COMPASS_OS_VERSION={os-version} - COMPASS_OS_VERSION_OPTION={openstack-os-version} same-node: true block: true - trigger-builds: @@ -159,7 +150,7 @@ #dovetail only master by now, not sync with A/B/C branches #here the stream means the SUT stream, dovetail stream is defined in its own job - trigger-builds: - - project: 'dovetail-compass-{pod}-basic-{stream}' + - project: 'dovetail-compass-{pod}-debug-{stream}' current-parameters: false predefined-parameters: DEPLOY_SCENARIO={scenario} @@ -199,6 +190,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - compass-ci-parameter: installer: '{installer}' gs-pathname: '{gs-pathname}' @@ -206,10 +199,7 @@ - '{installer}-defaults' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm wrappers: - build-name: @@ -247,13 +237,7 @@ - choice: name: COMPASS_OPENSTACK_VERSION choices: - - 'mitaka' - - 'liberty' - - choice: - name: COMPASS_OS_VERSION_OPTION - choices: - - '' - - 'xenial' + - 'newton' ######################## # trigger macros diff --git a/jjb/compass4nfv/compass-deploy.sh b/jjb/compass4nfv/compass-deploy.sh index 65e44b670..c8d6159a2 100644 --- a/jjb/compass4nfv/compass-deploy.sh +++ b/jjb/compass4nfv/compass-deploy.sh @@ -29,14 +29,9 @@ cd $WORKSPACE export OS_VERSION=${COMPASS_OS_VERSION} export OPENSTACK_VERSION=${COMPASS_OPENSTACK_VERSION} -if [[ "${COMPASS_OS_VERSION_OPTION}" = "xenial" ]] && [[ "${OPENSTACK_VERSION}" = "mitaka" ]]; then - export OPENSTACK_VERSION=${OPENSTACK_VERSION}_${COMPASS_OS_VERSION_OPTION} - export OS_VERSION=${COMPASS_OS_VERSION_OPTION} -fi if [[ "${DEPLOY_SCENARIO}" =~ "-ocl" ]]; then export NETWORK_CONF_FILE=network_ocl.yml - export OPENSTACK_VERSION=liberty elif [[ "${DEPLOY_SCENARIO}" =~ "-onos" ]]; then export NETWORK_CONF_FILE=network_onos.yml else @@ -52,7 +47,11 @@ else export DHA_CONF=$CONFDIR/hardware_environment/$NODE_NAME/${DEPLOY_SCENARIO}.yml fi -./deploy.sh --dha ${DHA_CONF} --network ${NETWORK_CONF} +export DHA=${DHA_CONF} +export NETWORK=${NETWORK_CONF} + +source ./ci/deploy_ci.sh + if [ $? -ne 0 ]; then echo "depolyment failed!" deploy_ret=1 diff --git a/jjb/compass4nfv/compass-project-jobs.yml b/jjb/compass4nfv/compass-project-jobs.yml index 3a52e91c9..6cf35d631 100644 --- a/jjb/compass4nfv/compass-project-jobs.yml +++ b/jjb/compass4nfv/compass-project-jobs.yml @@ -38,6 +38,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - compass-project-parameter: installer: '{installer}' gs-pathname: '{gs-pathname}' @@ -45,10 +47,7 @@ - '{installer}-defaults' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm triggers: - timed: 'H 8 * * *' @@ -80,16 +79,15 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - compass-project-parameter: installer: '{installer}' gs-pathname: '{gs-pathname}' - '{node}-defaults' - '{installer}-defaults' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm builders: - shell: @@ -121,14 +119,4 @@ - string: name: PPA_CACHE default: "$WORKSPACE/work/repo/" - - choice: - name: COMPASS_OPENSTACK_VERSION - choices: - - 'mitaka' - - 'liberty' - - choice: - name: COMPASS_OS_VERSION - choices: - - 'trusty' - - 'centos7' diff --git a/jjb/compass4nfv/compass-verify-jobs.yml b/jjb/compass4nfv/compass-verify-jobs.yml index 82e3dfdf2..2277e92ab 100644 --- a/jjb/compass4nfv/compass-verify-jobs.yml +++ b/jjb/compass4nfv/compass-verify-jobs.yml @@ -18,13 +18,9 @@ disabled: true distro: - - 'trusty': + - 'xenial': disabled: false - os-version: 'trusty' - openstack-os-version: '' - - 'centos7': - disabled: false - os-version: 'centos7' + os-version: 'xenial' openstack-os-version: '' ##################################### # patch verification phases @@ -64,21 +60,17 @@ block-level: 'NODE' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 120 fail: true triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -124,7 +116,7 @@ name: basic condition: SUCCESSFUL projects: - - name: 'compass-verify-basic-{stream}' + - name: 'opnfv-lint-verify-{stream}' current-parameters: true node-parameters: true kill-phase-on: FAILURE @@ -137,21 +129,20 @@ current-parameters: true predefined-parameters: | COMPASS_OS_VERSION={os-version} - COMPASS_OS_VERSION_OPTION={openstack-os-version} node-parameters: true kill-phase-on: FAILURE abort-all-job: true - - multijob: - name: smoke-test - condition: SUCCESSFUL - projects: - - name: 'functest-compass-virtual-suite-{stream}' - current-parameters: true - predefined-parameters: - FUNCTEST_SUITE_NAME=healthcheck - node-parameters: true - kill-phase-on: NEVER - abort-all-job: true +# - multijob: +# name: smoke-test +# condition: SUCCESSFUL +# projects: +# - name: 'functest-compass-virtual-suite-{stream}' +# current-parameters: true +# predefined-parameters: +# FUNCTEST_SUITE_NAME=healthcheck +# node-parameters: true +# kill-phase-on: NEVER +# abort-all-job: true - job-template: name: 'compass-verify-{phase}-{distro}-{stream}' @@ -174,15 +165,10 @@ block-level: 'NODE' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 120 fail: true @@ -242,10 +228,8 @@ - choice: name: COMPASS_OPENSTACK_VERSION choices: - - 'mitaka' - - 'liberty' + - 'newton' - choice: name: COMPASS_OS_VERSION choices: - - 'trusty' - - 'centos7' + - 'xenial' diff --git a/jjb/conductor/conductor.yml b/jjb/conductor/conductor.yml index a5f556ad8..f9978804f 100644 --- a/jjb/conductor/conductor.yml +++ b/jjb/conductor/conductor.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/copper/copper.yml b/jjb/copper/copper.yml index b5045785c..9c736db7c 100644 --- a/jjb/copper/copper.yml +++ b/jjb/copper/copper.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/cperf/cperf-ci-jobs.yml b/jjb/cperf/cperf-ci-jobs.yml index d6c8601c6..4ffc3b013 100644 --- a/jjb/cperf/cperf-ci-jobs.yml +++ b/jjb/cperf/cperf-ci-jobs.yml @@ -57,6 +57,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - '{pod}-defaults' - '{installer}-defaults' - cperf-parameter: @@ -65,10 +67,7 @@ docker-tag: '{docker-tag}' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm builders: - 'cperf-{testsuite}-builder' diff --git a/jjb/daisy4nfv/daisy4nfv-build.sh b/jjb/daisy4nfv/daisy4nfv-build.sh index ec11db587..eb29fed72 100755 --- a/jjb/daisy4nfv/daisy4nfv-build.sh +++ b/jjb/daisy4nfv/daisy4nfv-build.sh @@ -4,11 +4,31 @@ echo "--------------------------------------------------------" echo "This is diasy4nfv build job!" echo "--------------------------------------------------------" +# set OPNFV_ARTIFACT_VERSION +if [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Building Daisy4nfv ISO for a merged change" + export OPNFV_ARTIFACT_VERSION="gerrit-$GERRIT_CHANGE_NUMBER" +else + export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") +fi + # build output directory OUTPUT_DIR=$WORKSPACE/build_output mkdir -p $OUTPUT_DIR # start the build cd $WORKSPACE -./ci/build.sh $OUTPUT_DIR +./ci/build.sh $OUTPUT_DIR $OPNFV_ARTIFACT_VERSION + +# save information regarding artifact into file +( + echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" + echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)" + echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)" + echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin" + echo "OPNFV_BUILD_URL=$BUILD_URL" +) > $WORKSPACE/opnfv.properties +echo +echo "--------------------------------------------------------" +echo "Done!" diff --git a/jjb/daisy4nfv/daisy4nfv-download-artifact.sh b/jjb/daisy4nfv/daisy4nfv-download-artifact.sh new file mode 100755 index 000000000..90b5fa62f --- /dev/null +++ b/jjb/daisy4nfv/daisy4nfv-download-artifact.sh @@ -0,0 +1,53 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 ZTE Coreporation and others. +# hu.zhijiang@zte.com.cn +# sun.jing22@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o pipefail + +# use proxy url to replace the nomral URL, for googleusercontent.com will be blocked randomly +[[ "$NODE_NAME" =~ (zte) ]] && GS_URL=$GS_BASE_PROXY + +if [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Downloading http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties" + # get the properties file for the Daisy4nfv BIN built for a merged change + curl -L -s -o $WORKSPACE/latest.properties http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties +else + # get the latest.properties file in order to get info regarding latest artifact + echo "Downloading http://$GS_URL/latest.properties" + curl -L -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties +fi + +# check if we got the file +[[ -f $WORKSPACE/latest.properties ]] || exit 1 + +# source the file so we get artifact metadata +source $WORKSPACE/latest.properties + +# echo the info about artifact that is used during the deployment +OPNFV_ARTIFACT=${OPNFV_ARTIFACT_URL/*\/} +echo "Using $OPNFV_ARTIFACT for deployment" + +[[ "$NODE_NAME" =~ (zte) ]] && OPNFV_ARTIFACT_URL=${GS_BASE_PROXY%%/*}/$OPNFV_ARTIFACT_URL + +# log info to console +echo "Downloading the $INSTALLER_TYPE artifact using URL http://$OPNFV_ARTIFACT_URL" +echo "This could take some time..." +echo "--------------------------------------------------------" +echo + +# download the file +curl -s -o $WORKSPACE/opnfv.bin http://$OPNFV_ARTIFACT_URL > gsutil.bin.log 2>&1 + +# list the file +ls -al $WORKSPACE/opnfv.bin + +echo +echo "--------------------------------------------------------" +echo "Done!" diff --git a/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml new file mode 100644 index 000000000..b7a5fec92 --- /dev/null +++ b/jjb/daisy4nfv/daisy4nfv-merge-jobs.yml @@ -0,0 +1,261 @@ +- project: + name: 'daisy4nfv-merge-jobs' + + project: 'daisy' +##################################### +# branch definitions +##################################### + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false +##################################### +# patch merge phases +##################################### + phase: + - 'basic': + slave-label: 'opnfv-build-centos' + - 'build': + slave-label: 'opnfv-build-centos' + - 'deploy-virtual': + slave-label: 'opnfv-build-centos' + - 'smoke-test': + slave-label: 'opnfv-build-centos' + - 'promote': + slave-label: 'opnfv-build-centos' +##################################### +# jobs +##################################### + jobs: + - 'daisy4nfv-merge-{stream}' + - 'daisy4nfv-merge-{phase}-{stream}' +##################################### +# job templates +##################################### +- job-template: + name: 'daisy4nfv-merge-{stream}' + + project-type: multijob + + disabled: false + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 4 + option: 'project' + + scm: + - git-scm-gerrit + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 360 + fail: true + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + readable-message: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-defaults' + - 'daisy4nfv-merge-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: basic + condition: SUCCESSFUL + projects: + - name: 'daisy4nfv-merge-basic-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'daisy4nfv-merge-build-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: deploy-virtual + condition: SUCCESSFUL + projects: + - name: 'daisy4nfv-merge-deploy-virtual-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: smoke-test + condition: SUCCESSFUL + projects: + - name: 'daisy4nfv-merge-smoke-test-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: promote + condition: SUCCESSFUL + projects: + - name: 'daisy4nfv-merge-promote-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + +- job-template: + name: 'daisy4nfv-merge-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 6 + option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'daisy4nfv-merge-deploy-.*' + - 'daisy4nfv-merge-test-.*' + block-level: 'NODE' + + scm: + - git-scm-gerrit + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 360 + fail: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - '{slave-label}-defaults' + - 'daisy4nfv-merge-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-merge-{phase}-macro' +##################################### +# builder macros +##################################### +- builder: + name: 'daisy-merge-basic-macro' + builders: + - shell: + !include-raw: ./daisy4nfv-basic.sh + +- builder: + name: 'daisy-merge-build-macro' + builders: + - shell: + !include-raw: + - ./daisy4nfv-build.sh + - ./daisy4nfv-upload-artifact.sh + - ./daisy4nfv-workspace-cleanup.sh + +- builder: + name: 'daisy-merge-deploy-virtual-macro' + builders: + - shell: + !include-raw: + - ./daisy4nfv-download-artifact.sh + - ./daisy4nfv-virtual-deploy.sh + - ./daisy4nfv-workspace-cleanup.sh + +- builder: + name: 'daisy-merge-smoke-test-macro' + builders: + - shell: | + #!/bin/bash + + echo "Not activated!" + +- builder: + name: 'daisy-merge-promote-macro' + builders: + - shell: | + #!/bin/bash + + echo "Not activated!" +##################################### +# parameter macros +##################################### +- parameter: + name: 'daisy4nfv-merge-defaults' + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/$INSTALLER_TYPE + description: "Directory where the cache to be used during the build is located." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." diff --git a/jjb/daisy4nfv/daisy4nfv-upload-artifact.sh b/jjb/daisy4nfv/daisy4nfv-upload-artifact.sh new file mode 100755 index 000000000..6b0aec54f --- /dev/null +++ b/jjb/daisy4nfv/daisy4nfv-upload-artifact.sh @@ -0,0 +1,94 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 ZTE Coreporation and others. +# hu.zhijiang@zte.com.cn +# sun.jing22@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o pipefail + +# check if we built something +if [ -f $WORKSPACE/.noupload ]; then + echo "Nothing new to upload. Exiting." + /bin/rm -f $WORKSPACE/.noupload + exit 0 +fi + +# source the opnfv.properties to get ARTIFACT_VERSION +source $WORKSPACE/opnfv.properties + +importkey () { +# clone releng repository +echo "Cloning releng repository..." +[ -d releng ] && rm -rf releng +git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null +#this is where we import the siging key +if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then + source $WORKSPACE/releng/utils/gpg_import_key.sh +fi +} + +signbin () { +gpg2 -vvv --batch --yes --no-tty \ + --default-key opnfv-helpdesk@rt.linuxfoundation.org \ + --passphrase besteffort \ + --detach-sig $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.bin + +gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.bin.sig gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin.sig +echo "BIN signature Upload Complete!" +} + +uploadbin () { +# log info to console +echo "Uploading $INSTALLER_TYPE artifact. This could take some time..." +echo + +cd $WORKSPACE +# upload artifact and additional files to google storage +gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.bin \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin > gsutil.bin.log 2>&1 +gsutil cp $WORKSPACE/opnfv.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 +if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then + gsutil cp $WORKSPACE/opnfv.properties \ + gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 +elif [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Uploaded Daisy4nfv BIN for a merged change" +fi + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/latest.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > /dev/null 2>&1 + +gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin > /dev/null 2>&1 + +# disabled errexit due to gsutil setmeta complaints +# BadRequestException: 400 Invalid argument +# check if we uploaded the file successfully to see if things are fine +gsutil ls gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading artifact!" + echo "Check log $WORKSPACE/gsutil.bin.log on the machine where this build is done." + exit 1 +fi + +echo "Done!" +echo +echo "--------------------------------------------------------" +echo +echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin" +echo +echo "--------------------------------------------------------" +echo +} + +importkey +signbin +uploadbin diff --git a/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml b/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml index d893b1485..cba22643c 100644 --- a/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml +++ b/jjb/daisy4nfv/daisy4nfv-verify-jobs.yml @@ -47,21 +47,17 @@ option: 'project' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -172,15 +168,10 @@ block-level: 'NODE' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true diff --git a/jjb/daisy4nfv/daisy4nfv-virtual-deploy.sh b/jjb/daisy4nfv/daisy4nfv-virtual-deploy.sh index 8936be6c4..4aa7b0bd5 100755 --- a/jjb/daisy4nfv/daisy4nfv-virtual-deploy.sh +++ b/jjb/daisy4nfv/daisy4nfv-virtual-deploy.sh @@ -4,3 +4,26 @@ echo "--------------------------------------------------------" echo "This is diasy4nfv virtual deploy job!" echo "--------------------------------------------------------" +cd $WORKSPACE + +if [[ "$NODE_NAME" =~ "-virtual" ]]; then + export NETWORK_CONF=./deploy/config/vm_environment/$NODE_NAME/network.yml + export DHA_CONF=./deploy/config/vm_environment/$NODE_NAME/deploy.yml +else + # TODO: For the time being, we need to pass this script to let contributors merge their work. + echo "No support for non-virtual node" + exit 0 +fi + +./ci/deploy/deploy.sh ${DHA_CONF} ${NETWORK_CONF} + +if [ $? -ne 0 ]; then + echo "depolyment failed!" + deploy_ret=1 +fi + +echo +echo "--------------------------------------------------------" +echo "Done!" + +exit $deploy_ret diff --git a/jjb/daisy4nfv/daisy4nfv-workspace-cleanup.sh b/jjb/daisy4nfv/daisy4nfv-workspace-cleanup.sh new file mode 100755 index 000000000..26f7e9a01 --- /dev/null +++ b/jjb/daisy4nfv/daisy4nfv-workspace-cleanup.sh @@ -0,0 +1,16 @@ +#!/bin/bash +############################################################################## +# Copyright (c) 2016 ZTE Coreporation and others. +# hu.zhijiang@zte.com.cn +# sun.jing22@zte.com.cn +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o errexit +set -o nounset +set -o pipefail + +# delete the $WORKSPACE to open some space +/bin/rm -rf $WORKSPACE diff --git a/jjb/doctor/doctor.yml b/jjb/doctor/doctor.yml index 4958ca2b5..2e26071d7 100644 --- a/jjb/doctor/doctor.yml +++ b/jjb/doctor/doctor.yml @@ -7,12 +7,12 @@ - master: branch: '{stream}' gs-pathname: '' - docker_tag: 'master' + docker-tag: 'master' disabled: false - colorado: branch: 'stable/{stream}' gs-pathname: '/{stream}' - docker_tag: 'stable' + docker-tag: 'stable' disabled: false installer: @@ -20,9 +20,13 @@ slave-label: 'ool-virtual1' pod: 'ool-virtual1' + inspector: + - 'sample' + - 'congress' + jobs: - 'doctor-verify-{stream}' - - 'doctor-verify-{installer}-{stream}' + - 'doctor-verify-{installer}-{inspector}-{stream}' - job-template: name: 'doctor-verify-{stream}' @@ -37,13 +41,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -68,7 +70,7 @@ - shell: "[ -e tests/run.sh ] && bash -n ./tests/run.sh" - job-template: - name: 'doctor-verify-{installer}-{stream}' + name: 'doctor-verify-{installer}-{inspector}-{stream}' node: '{slave-label}' @@ -82,15 +84,54 @@ default: /home/jenkins/openstack.creds description: 'OpenStack credentials' - '{slave-label}-defaults' + - string: + name: INSTALLER_TYPE + default: '{installer}' + description: 'Installer used for deploying OPNFV on this POD' + - string: + name: DOCKER_TAG + default: '{docker-tag}' + description: 'Tag to pull docker image' + - string: + name: CLEAN_DOCKER_IMAGES + default: 'false' + description: 'Remove downloaded docker images (opnfv/functest:*)' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-nofeature-ha' + description: 'Scenario to deploy and test' + # functest-suite-parameter + - string: + name: FUNCTEST_SUITE_NAME + default: '{project}' + - string: + name: TESTCASE_OPTIONS + default: '-e INSPECTOR_TYPE={inspector} -v $WORKSPACE:/home/opnfv/repos/doctor' + description: 'Addtional parameters specific to test case(s)' + # functest-parameter + - string: + name: GS_PATHNAME + default: '{gs-pathname}' + description: "Version directory where the opnfv documents will be stored in gs repository" + - string: + name: FUNCTEST_REPO_DIR + default: "/home/opnfv/repos/functest" + description: "Directory where the Functest repository is cloned" + - string: + name: PUSH_RESULTS_TO_DB + default: "true" + description: "Push the results of all the tests to the resultDB" + - string: + name: CI_DEBUG + default: 'true' + description: "Show debug output information" scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -117,36 +158,15 @@ notbuilt: true builders: - - trigger-builds: - - project: 'functest-{installer}-{pod}-suite-{stream}' - current-parameters: true - predefined-parameters: | - CI_DEBUG=true - FUNCTEST_SUITE_NAME=doctor - DEPLOY_SCENARIO=os-nosdn-nofeature-ha - TESTCASE_OPTIONS=-e INSPECTOR_TYPE=sample -v $WORKSPACE:$HOME/opnfv/repos/doctor - block: true - same-node: true - - project: 'functest-{installer}-{pod}-suite-{stream}' - current-parameters: true - predefined-parameters: | - CI_DEBUG=true - FUNCTEST_SUITE_NAME=doctor - DEPLOY_SCENARIO=os-nosdn-nofeature-ha - TESTCASE_OPTIONS=-e INSPECTOR_TYPE=congress -v $WORKSPACE:$HOME/opnfv/repos/doctor - block: true - same-node: true + - 'functest-suite-builder' + - shell: | + functest_log="$HOME/opnfv/functest/results/{stream}/{project}.log" + to_be_archived="$WORKSPACE/tests/functest-{project}.log" + cp $functest_log $to_be_archived + # NOTE: checking the test result, as the previous job could return + # 0 regardless the result of doctor test scenario. + grep -e ' OK$' $functest_log || exit 1 publishers: - - postbuildscript: - builders: - - functest-copy-suite-log: - suite: '{project}' - archive: - artifacts: '{project}.log' - -- builder: - name: functest-copy-suite-log - builders: - - shell: | - cp $HOME/opnfv/functest/results/${{GIT_BRANCH##*/}}/{suite}.log $WORKSPACE/ + artifacts: 'tests/*.log' diff --git a/jjb/domino/domino.yml b/jjb/domino/domino.yml index 7cee98448..b6533009e 100644 --- a/jjb/domino/domino.yml +++ b/jjb/domino/domino.yml @@ -29,13 +29,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/dovetail/dovetail-artifacts-upload.sh b/jjb/dovetail/dovetail-artifacts-upload.sh new file mode 100755 index 000000000..94e4129fc --- /dev/null +++ b/jjb/dovetail/dovetail-artifacts-upload.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# SPDX-license-identifier: Apache-2.0 +############################################################################## +# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others. +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Apache License, Version 2.0 +# which accompanies this distribution, and is available at +# http://www.apache.org/licenses/LICENSE-2.0 +############################################################################## +set -o pipefail + +echo "dovetail: pull and save the images" + +[[ -d ${CACHE_DIR} ]] || mkdir -p ${CACHE_DIR} + +cd ${CACHE_DIR} +sudo docker pull ${DOCKER_REPO_NAME}:${DOCKER_TAG} +sudo docker save -o ${STORE_FILE_NAME} ${DOCKER_REPO_NAME}:${DOCKER_TAG} + +importkey () { +# clone releng repository +echo "Cloning releng repository..." +[ -d releng ] && rm -rf releng +git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null +#this is where we import the siging key +if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then + source $WORKSPACE/releng/utils/gpg_import_key.sh +fi +} + +sign () { +gpg2 -vvv --batch --yes --no-tty \ + --default-key opnfv-helpdesk@rt.linuxfoundation.org \ + --passphrase besteffort \ + --detach-sig ${CACHE_DIR}/${STORE_FILE_NAME} + +gsutil cp ${CACHE_DIR}/${STORE_FILE_NAME}.sig ${STORE_URL}/${STORE_FILE_NAME}.sig +echo "signature Upload Complete!" +} + +upload () { +# log info to console +echo "Uploading to artifact. This could take some time..." +echo + +cd $WORKSPACE +# upload artifact and additional files to google storage +gsutil cp ${CACHE_DIR}/${STORE_FILE_NAME} ${STORE_URL}/${STORE_FILE_NAME} + +gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + ${STORE_URL}/${STORE_FILE_NAME} > /dev/null 2>&1 + +# disabled errexit due to gsutil setmeta complaints +# BadRequestException: 400 Invalid argument +# check if we uploaded the file successfully to see if things are fine +gsutil ls ${STORE_URL}/${STORE_FILE_NAME} > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading artifact!" + exit 1 +fi + +echo "dovetail: uploading Done!" +echo +echo "--------------------------------------------------------" +echo +} + +importkey +sign +upload diff --git a/jjb/dovetail/dovetail-artifacts-upload.yml b/jjb/dovetail/dovetail-artifacts-upload.yml new file mode 100644 index 000000000..dc2ae5aa2 --- /dev/null +++ b/jjb/dovetail/dovetail-artifacts-upload.yml @@ -0,0 +1,130 @@ +############################################ +# dovetail upload artifacts job +############################################ +- project: + name: dovetail-artifacts-upload + + project: 'dovetail' + + jobs: + - 'dovetail-{image}-artifacts-upload-{stream}' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + + image: + - 'dovetail' + - 'functest' + - 'yardstick' + +############################################# +# job template +############################################# + +- job-template: + name: 'dovetail-{image}-artifacts-upload-{stream}' + + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + option: 'project' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + - dovetail-parameter: + gs-pathname: '{gs-pathname}' + image: '{image}' + branch: '{branch}' + + scm: + - git-scm + + builders: + - 'dovetail-builder-artifacts-upload' + - 'dovetail-workspace-cleanup' + +#################### +# parameter macros +#################### +- parameter: + name: dovetail-parameter + parameters: + - string: + name: CACHE_DIR + default: $WORKSPACE/cache{gs-pathname} + description: "the cache to store packages downloaded" + - string: + name: STORE_URL + default: gs://artifacts.opnfv.org/dovetail{gs-pathname} + description: "LF artifacts url for storage of dovetail packages" + - string: + name: DOCKER_REPO_NAME + default: opnfv/{image} + description: "docker repo name" + - string: + name: DOCKER_TAG + default: latest + description: "docker image tag of which will be uploaded to artifacts" + - string: + name: STORE_FILE_NAME + default: image_{image}_{branch}_$BUILD_ID.docker + description: "stored file name" + +#################################### +#builders for dovetail project +#################################### +- builder: + name: dovetail-builder-artifacts-upload + builders: + - shell: + !include-raw: ./dovetail-artifacts-upload.sh + +- builder: + name: dovetail-workspace-cleanup + builders: + - shell: | + #!/bin/bash + set -o errexit + + echo "Dovetail: cleanup cache used for storage downloaded packages" + + /bin/rm -rf $CACHE_DIR + + # Remove previous running containers if exist + if [[ -n "$(docker ps -a | grep $DOCKER_REPO_NAME)" ]]; then + echo "Removing existing $DOCKER_REPO_NAME containers..." + docker ps -a | grep $DOCKER_REPO_NAME | awk '{print $1}' | xargs docker rm -f + t=60 + # Wait max 60 sec for containers to be removed + while [[ $t -gt 0 ]] && [[ -n "$(docker ps| grep $DOCKER_REPO_NAME)" ]]; do + sleep 1 + let t=t-1 + done + fi + + # Remove existing images if exist + if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then + echo "Docker images to remove:" + docker images | head -1 && docker images | grep $DOCKER_REPO_NAME + image_tags=($(docker images | grep $DOCKER_REPO_NAME | awk '{print $2}')) + for tag in "${image_tags[@]}"; do + if [[ -n "$(docker images|grep $DOCKER_REPO_NAME|grep $tag)" ]]; then + echo "Removing docker image $DOCKER_REPO_NAME:$tag..." + docker rmi -f $DOCKER_REPO_NAME:$tag + fi + done + fi diff --git a/jjb/dovetail/dovetail-ci-jobs.yml b/jjb/dovetail/dovetail-ci-jobs.yml index 29212005d..2b6ab7621 100644 --- a/jjb/dovetail/dovetail-ci-jobs.yml +++ b/jjb/dovetail/dovetail-ci-jobs.yml @@ -102,6 +102,27 @@ SUT: apex auto-trigger-name: 'daily-trigger-disabled' <<: *colorado +#armband CI PODs + - armband-baremetal: + slave-label: armband-baremetal + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *master + - armband-virtual: + slave-label: armband-virtual + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *master + - armband-baremetal: + slave-label: armband-baremetal + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *colorado + - armband-virtual: + slave-label: armband-virtual + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *colorado #-------------------------------- # None-CI PODs #-------------------------------- @@ -110,9 +131,21 @@ SUT: compass auto-trigger-name: 'daily-trigger-disabled' <<: *master + - arm-pod2: + slave-label: '{pod}' + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *master + - arm-pod3: + slave-label: '{pod}' + SUT: fuel + auto-trigger-name: 'daily-trigger-disabled' + <<: *master #-------------------------------- testsuite: - - 'basic' + - 'debug' + - 'proposed_tests' + - 'compliance_set' jobs: - 'dovetail-{SUT}-{pod}-{testsuite}-{stream}' @@ -146,6 +179,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - '{SUT}-defaults' - '{slave-label}-defaults' - string: @@ -159,18 +194,23 @@ name: CI_DEBUG default: 'true' description: "Show debug output information" + - string: + name: TESTSUITE + default: '{testsuite}' + description: "dovetail testsuite to run" + - string: + name: DOVETAIL_REPO_DIR + default: "/home/opnfv/dovetail" + description: "Directory where the dovetail repository is cloned" scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{dovetail-branch}' + - git-scm builders: - description-setter: description: "POD: $NODE_NAME" - 'dovetail-cleanup' - - 'dovetail-{testsuite}' + - 'dovetail-run' publishers: - archive: @@ -182,11 +222,12 @@ # builder macros ######################## - builder: - name: dovetail-basic + name: dovetail-run builders: - shell: !include-raw: ./dovetail-run.sh + - builder: name: dovetail-fetch-os-creds builders: diff --git a/jjb/dovetail/dovetail-cleanup.sh b/jjb/dovetail/dovetail-cleanup.sh index 297222bb3..f215278db 100755 --- a/jjb/dovetail/dovetail-cleanup.sh +++ b/jjb/dovetail/dovetail-cleanup.sh @@ -1,20 +1,21 @@ #!/bin/bash + [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" echo "Cleaning up docker containers/images..." # Remove previous running containers if exist if [[ ! -z $(docker ps -a | grep opnfv/dovetail) ]]; then echo "Removing existing opnfv/dovetail containers..." - docker ps -a | grep opnfv/dovetail | awk '{print $1}' | xargs docker rm -f >$redirect + docker ps -a | grep opnfv/dovetail | awk '{print $1}' | xargs docker rm -f >${redirect} fi # Remove existing images if exist if [[ ! -z $(docker images | grep opnfv/dovetail) ]]; then echo "Docker images to remove:" - docker images | head -1 && docker images | grep opnfv/dovetail + docker images | head -1 && docker images | grep opnfv/dovetail >${redirect} image_tags=($(docker images | grep opnfv/dovetail | awk '{print $2}')) for tag in "${image_tags[@]}"; do echo "Removing docker image opnfv/dovetail:$tag..." - docker rmi opnfv/dovetail:$tag >$redirect + docker rmi opnfv/dovetail:$tag >${redirect} done fi diff --git a/jjb/dovetail/dovetail-project-jobs.yml b/jjb/dovetail/dovetail-project-jobs.yml index 41fd8cdcb..904841396 100644 --- a/jjb/dovetail/dovetail-project-jobs.yml +++ b/jjb/dovetail/dovetail-project-jobs.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -72,13 +70,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -97,25 +93,21 @@ #builders for dovetail project ############################### - builder: - name: dovetail-unit-tests + name: dovetail-hello-world builders: - shell: | #!/bin/bash set -o errexit - set -o pipefail - echo "Running unit tests..." - cd $WORKSPACE - virtualenv $WORKSPACE/dovetail_venv - source $WORKSPACE/dovetail_venv/bin/activate + echo "hello world" - #packages installation - easy_install -U setuptools - easy_install -U pip - pip install -r unittests/requirements.txt - pip install -e . - #unit tests - /bin/bash $WORKSPACE/unittests/unittest.sh +- builder: + name: dovetail-unit-tests + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o pipefail - deactivate + tox diff --git a/jjb/dovetail/dovetail-run.sh b/jjb/dovetail/dovetail-run.sh index 098b7db0c..4082c34fe 100755 --- a/jjb/dovetail/dovetail-run.sh +++ b/jjb/dovetail/dovetail-run.sh @@ -22,7 +22,7 @@ if [[ ${INSTALLER_TYPE} == 'apex' ]]; then elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then # If production lab then creds may be retrieved dynamically # creds are on the jumphost, always in the same folder - labconfig="-v $LAB_CONFIG/admin-openrc:/home/opnfv/openrc" + labconfig="-v $LAB_CONFIG/admin-openrc:/home/opnfv/functest/conf/openstack.creds" # If dev lab, credentials may not be the default ones, just provide a path to put them into docker # replace the default one by the customized one provided by jenkins config fi @@ -32,26 +32,45 @@ if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FOR sudo iptables -I FORWARD -j RETURN fi -opts="--privileged=true --rm" +opts="--privileged=true -id" envs="-e CI_DEBUG=${CI_DEBUG} \ -e INSTALLER_TYPE=${INSTALLER_TYPE} \ -e INSTALLER_IP=${INSTALLER_IP} \ -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} \ - -e DEPLOY_TYPE=${DEPLOY_TYPE} \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v /home/opnfv/dovetail/results:/home/opnfv/dovetail/results" + -e DEPLOY_TYPE=${DEPLOY_TYPE}" +results_envs="-v /var/run/docker.sock:/var/run/docker.sock \ + -v /home/opnfv/dovetail/results:/home/opnfv/dovetail/results" # Pull the image with correct tag echo "Dovetail: Pulling image opnfv/dovetail:${DOCKER_TAG}" docker pull opnfv/dovetail:$DOCKER_TAG >$redirect -# Run docker -echo "Dovetail: docker running..." -sudo docker run ${opts} ${envs} ${labconfig} ${sshkey} opnfv/dovetail:${DOCKER_TAG} \ -"/home/opnfv/dovetail/dovetail/run.py" +cmd="sudo docker run ${opts} ${envs} ${results_envs} ${labconfig} ${sshkey} \ + opnfv/dovetail:${DOCKER_TAG} /bin/bash" +echo "Dovetail: running docker run command: ${cmd}" +${cmd} >${redirect} +sleep 5 +container_id=$(docker ps | grep "opnfv/dovetail:${DOCKER_TAG}" | awk '{print $1}' | head -1) +echo "Container ID=${container_id}" +if [ -z ${container_id} ]; then + echo "Cannot find opnfv/dovetail container ID ${container_id}. Please check if it is existing." + docker ps -a + exit 1 +fi +echo "COntainer Start: docker start ${container_id}" +docker start ${container_id} +sleep 5 +docker ps >${redirect} +if [ $(docker ps | grep "opnfv/dovetail:${DOCKER_TAG}" | wc -l) == 0 ]; then + echo "The container opnfv/dovetail with ID=${container_id} has not been properly started. Exiting..." + exit 1 +fi + +exec_cmd="python ${DOVETAIL_REPO_DIR}/dovetail/run.py --testsuite ${TESTSUITE} -d true" +echo "Container exec command: ${exec_cmd}" +docker exec ${container_id} ${exec_cmd} -echo "Dovetail: store results..." -sudo cp -r /home/opnfv/dovetail/results ./ +sudo cp -r ${DOVETAIL_REPO_DIR}/results ./ #To make sure the file owner is jenkins, for the copied results files in the above line #if not, there will be error when next time to wipe workspace sudo chown -R jenkins:jenkins ${WORKSPACE}/results diff --git a/jjb/dpacc/dpacc.yml b/jjb/dpacc/dpacc.yml index c660af57d..3eb581ec1 100644 --- a/jjb/dpacc/dpacc.yml +++ b/jjb/dpacc/dpacc.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/escalator/escalator-basic.sh b/jjb/escalator/escalator-basic.sh new file mode 100755 index 000000000..9c739c422 --- /dev/null +++ b/jjb/escalator/escalator-basic.sh @@ -0,0 +1,5 @@ +#!/bin/bash +echo "--------------------------------------------------------" +echo "This is escalator basic job!" +echo "--------------------------------------------------------" + diff --git a/jjb/escalator/escalator-build.sh b/jjb/escalator/escalator-build.sh new file mode 100755 index 000000000..0e35c27d9 --- /dev/null +++ b/jjb/escalator/escalator-build.sh @@ -0,0 +1,33 @@ +#!/bin/bash +echo "--------------------------------------------------------" +echo "This is escalator build job!" +echo "--------------------------------------------------------" + +# set OPNFV_ARTIFACT_VERSION +if [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Building Escalator TAR for a merged change" + export OPNFV_ARTIFACT_VERSION="gerrit-$GERRIT_CHANGE_NUMBER" +else + export OPNFV_ARTIFACT_VERSION=$(date -u +"%Y-%m-%d_%H-%M-%S") +fi + +# build output directory +OUTPUT_DIR=$WORKSPACE/build_output +mkdir -p $OUTPUT_DIR + +# start the build +cd $WORKSPACE +./ci/build.sh $OUTPUT_DIR $OPNFV_ARTIFACT_VERSION + +# save information regarding artifact into file +( + echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" + echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)" + echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)" + echo "OPNFV_ARTIFACT_URL=$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.bin" + echo "OPNFV_BUILD_URL=$BUILD_URL" +) > $WORKSPACE/opnfv.properties + +echo +echo "--------------------------------------------------------" +echo "Done!" diff --git a/jjb/escalator/escalator-upload-artifact.sh b/jjb/escalator/escalator-upload-artifact.sh new file mode 100755 index 000000000..781fb3e3e --- /dev/null +++ b/jjb/escalator/escalator-upload-artifact.sh @@ -0,0 +1,89 @@ +#!/bin/bash +echo "--------------------------------------------------------" +echo "This is escalator upload job!" +echo "--------------------------------------------------------" + +set -o pipefail + +# check if we built something +if [ -f $WORKSPACE/.noupload ]; then + echo "Nothing new to upload. Exiting." + /bin/rm -f $WORKSPACE/.noupload + exit 0 +fi + +# source the opnfv.properties to get ARTIFACT_VERSION +source $WORKSPACE/opnfv.properties + +importkey () { +# clone releng repository +echo "Cloning releng repository..." +[ -d releng ] && rm -rf releng +git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng/ &> /dev/null +#this is where we import the siging key +if [ -f $WORKSPACE/releng/utils/gpg_import_key.sh ]; then + source $WORKSPACE/releng/utils/gpg_import_key.sh +fi +} + +signtar () { +gpg2 -vvv --batch --yes --no-tty \ + --default-key opnfv-helpdesk@rt.linuxfoundation.org \ + --passphrase besteffort \ + --detach-sig $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz + +gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz.sig gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz.sig +echo "TAR signature Upload Complete!" +} + +uploadtar () { +# log info to console +echo "Uploading $INSTALLER_TYPE artifact. This could take some time..." +echo + +cd $WORKSPACE +# upload artifact and additional files to google storage +gsutil cp $BUILD_DIRECTORY/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz > gsutil.tar.log 2>&1 +gsutil cp $WORKSPACE/opnfv.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log 2>&1 +if [[ ! "$JOB_NAME" =~ (verify|merge) ]]; then + gsutil cp $WORKSPACE/opnfv.properties \ + gs://$GS_URL/latest.properties > gsutil.latest.log 2>&1 +elif [[ "$JOB_NAME" =~ "merge" ]]; then + echo "Uploaded Escalator TAR for a merged change" +fi + +gsutil -m setmeta \ + -h "Content-Type:text/html" \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/latest.properties \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > /dev/null 2>&1 + +gsutil -m setmeta \ + -h "Cache-Control:private, max-age=0, no-transform" \ + gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz > /dev/null 2>&1 + +# disabled errexit due to gsutil setmeta complaints +# BadRequestException: 400 Invalid argument +# check if we uploaded the file successfully to see if things are fine +gsutil ls gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz > /dev/null 2>&1 +if [[ $? -ne 0 ]]; then + echo "Problem while uploading artifact!" + echo "Check log $WORKSPACE/gsutil.bin.log on the machine where this build is done." + exit 1 +fi + +echo "Done!" +echo +echo "--------------------------------------------------------" +echo +echo "Artifact is available as http://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.tar.gz" +echo +echo "--------------------------------------------------------" +echo +} + +importkey +signtar +uploadtar diff --git a/jjb/escalator/escalator.yml b/jjb/escalator/escalator.yml new file mode 100644 index 000000000..4149ee93c --- /dev/null +++ b/jjb/escalator/escalator.yml @@ -0,0 +1,308 @@ +- project: + name: 'escalator' + + project: 'escalator' +##################################### +# branch definitions +##################################### + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false +##################################### +# phases +##################################### + phase: + - 'basic': + slave-label: 'opnfv-build-centos' + - 'build': + slave-label: 'opnfv-build-centos' +##################################### +# jobs +##################################### + jobs: + - 'escalator-verify-{stream}' + - 'escalator-verify-{phase}-{stream}' + - 'escalator-merge-{stream}' + - 'escalator-merge-{phase}-{stream}' +##################################### +# job templates +##################################### +- job-template: + name: 'escalator-verify-{stream}' + + project-type: multijob + + disabled: false + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 4 + option: 'project' + + scm: + - git-scm-gerrit + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 360 + fail: true + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + readable-message: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-defaults' + - 'escalator-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: basic + condition: SUCCESSFUL + projects: + - name: 'escalator-verify-basic-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'escalator-verify-build-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + +- job-template: + name: 'escalator-verify-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + scm: + - git-scm-gerrit + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 360 + fail: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - '{slave-label}-defaults' + - 'escalator-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-verify-{phase}-macro' + +- job-template: + name: 'escalator-merge-{stream}' + + project-type: multijob + + disabled: false + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 4 + option: 'project' + + scm: + - git-scm-gerrit + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 360 + fail: true + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**|.gitignore' + readable-message: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-defaults' + - 'escalator-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: basic + condition: SUCCESSFUL + projects: + - name: 'escalator-merge-basic-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: build + condition: SUCCESSFUL + projects: + - name: 'escalator-merge-build-{stream}' + current-parameters: false + predefined-parameters: | + GERRIT_BRANCH=$GERRIT_BRANCH + GERRIT_REFSPEC=$GERRIT_REFSPEC + GERRIT_CHANGE_NUMBER=$GERRIT_CHANGE_NUMBER + GERRIT_CHANGE_COMMIT_MESSAGE=$GERRIT_CHANGE_COMMIT_MESSAGE + node-parameters: false + kill-phase-on: FAILURE + abort-all-job: true + +- job-template: + name: 'escalator-merge-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + scm: + - git-scm-gerrit + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 360 + fail: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - '{slave-label}-defaults' + - 'escalator-defaults': + gs-pathname: '{gs-pathname}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-merge-{phase}-macro' +##################################### +# builder macros +##################################### +- builder: + name: 'escalator-verify-basic-macro' + builders: + - shell: + !include-raw: ./escalator-basic.sh + +- builder: + name: 'escalator-verify-build-macro' + builders: + - shell: + !include-raw: ./escalator-build.sh + +- builder: + name: 'escalator-merge-basic-macro' + builders: + - shell: + !include-raw: ./escalator-basic.sh + +- builder: + name: 'escalator-merge-build-macro' + builders: + - shell: + !include-raw: + - ./escalator-build.sh + - ./escalator-upload-artifact.sh +##################################### +# parameter macros +##################################### +- parameter: + name: 'escalator-defaults' + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build_output + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: CACHE_DIRECTORY + default: $HOME/opnfv/cache/$INSTALLER_TYPE + description: "Directory where the cache to be used during the build is located." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." diff --git a/jjb/fuel/fuel-daily-jobs.yml b/jjb/fuel/fuel-daily-jobs.yml index 1c7946a87..87488c886 100644 --- a/jjb/fuel/fuel-daily-jobs.yml +++ b/jjb/fuel/fuel-daily-jobs.yml @@ -206,6 +206,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - '{installer}-defaults' - '{slave-label}-defaults': installer: '{installer}' @@ -220,10 +222,7 @@ description: 'Deployment timeout in minutes' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm wrappers: - build-name: @@ -273,31 +272,31 @@ - trigger: name: 'fuel-os-odl_l2-nofeature-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 23 * * *' + - timed: '' # '5 23 * * *' - trigger: name: 'fuel-os-odl_l3-nofeature-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 2 * * *' + - timed: '' # '5 2 * * *' - trigger: name: 'fuel-os-onos-sfc-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 5 * * *' + - timed: '' # '5 5 * * *' - trigger: name: 'fuel-os-onos-nofeature-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 8 * * *' + - timed: '' # '5 8 * * *' - trigger: name: 'fuel-os-odl_l2-sfc-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 11 * * *' + - timed: '' # '5 11 * * *' - trigger: name: 'fuel-os-odl_l2-bgpvpn-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 14 * * *' + - timed: '' # '5 14 * * *' - trigger: name: 'fuel-os-nosdn-kvm-ha-baremetal-daily-master-trigger' triggers: - - timed: '5 17 * * *' + - timed: '' # '5 17 * * *' - trigger: name: 'fuel-os-nosdn-ovs-ha-baremetal-daily-master-trigger' triggers: diff --git a/jjb/fuel/fuel-download-artifact.sh b/jjb/fuel/fuel-download-artifact.sh index 5685444f8..2a0f09a3f 100755 --- a/jjb/fuel/fuel-download-artifact.sh +++ b/jjb/fuel/fuel-download-artifact.sh @@ -11,23 +11,23 @@ set -o errexit set -o pipefail # use proxy url to replace the nomral URL, for googleusercontent.com will be blocked randomly -[[ "$NODE_NAME" =~ (zte) ]] && GS_URL=$GS_BASE_PROXY +[[ "$NODE_NAME" =~ (zte) ]] && GS_URL=${GS_BASE_PROXY%%/*}/$GS_URL if [[ "$JOB_NAME" =~ "merge" ]]; then echo "Downloading http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties" # get the properties file for the Fuel ISO built for a merged change - curl -s -o $WORKSPACE/latest.properties http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties + curl -L -s -o $WORKSPACE/latest.properties http://$GS_URL/opnfv-gerrit-$GERRIT_CHANGE_NUMBER.properties else # get the latest.properties file in order to get info regarding latest artifact echo "Downloading http://$GS_URL/latest.properties" - curl -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties + curl -L -s -o $WORKSPACE/latest.properties http://$GS_URL/latest.properties fi # check if we got the file -[[ -f latest.properties ]] || exit 1 +[[ -f $WORKSPACE/latest.properties ]] || exit 1 # source the file so we get artifact metadata -source latest.properties +source $WORKSPACE/latest.properties # echo the info about artifact that is used during the deployment OPNFV_ARTIFACT=${OPNFV_ARTIFACT_URL/*\/} @@ -59,7 +59,7 @@ echo "--------------------------------------------------------" echo # download the file -curl -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 +curl -L -s -o $WORKSPACE/opnfv.iso http://$OPNFV_ARTIFACT_URL > gsutil.iso.log 2>&1 # list the file ls -al $WORKSPACE/opnfv.iso diff --git a/jjb/fuel/fuel-plugin-verify-jobs.yml b/jjb/fuel/fuel-plugin-verify-jobs.yml index 4fea26b32..bf847edfe 100644 --- a/jjb/fuel/fuel-plugin-verify-jobs.yml +++ b/jjb/fuel/fuel-plugin-verify-jobs.yml @@ -71,9 +71,7 @@ wipe-workspace: true wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true @@ -195,9 +193,7 @@ wipe-workspace: true wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true diff --git a/jjb/fuel/fuel-project-jobs.yml b/jjb/fuel/fuel-project-jobs.yml index 588ab0cd5..8fdf8deae 100644 --- a/jjb/fuel/fuel-project-jobs.yml +++ b/jjb/fuel/fuel-project-jobs.yml @@ -44,6 +44,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - 'opnfv-build-ubuntu-defaults' - '{installer}-defaults' - choice: @@ -56,10 +58,7 @@ gs-pathname: '{gs-pathname}' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm triggers: - timed: '0 H/4 * * *' @@ -99,21 +98,17 @@ gs-pathname: '{gs-pathname}' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -174,18 +169,14 @@ - fuel-project-parameter: gs-pathname: '{gs-pathname}' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -242,6 +233,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - '{installer}-defaults' - string: name: GIT_BASE @@ -263,10 +256,7 @@ gs-pathname: '{gs-pathname}' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm wrappers: - build-name: diff --git a/jjb/fuel/fuel-verify-jobs-experimental.yml b/jjb/fuel/fuel-verify-jobs-experimental.yml index ae83b08cf..3aa85b22d 100644 --- a/jjb/fuel/fuel-verify-jobs-experimental.yml +++ b/jjb/fuel/fuel-verify-jobs-experimental.yml @@ -58,21 +58,17 @@ option: 'project' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -190,15 +186,10 @@ block-level: 'NODE' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true diff --git a/jjb/fuel/fuel-verify-jobs.yml b/jjb/fuel/fuel-verify-jobs.yml index f4bdbdd45..fe8985392 100644 --- a/jjb/fuel/fuel-verify-jobs.yml +++ b/jjb/fuel/fuel-verify-jobs.yml @@ -53,21 +53,17 @@ option: 'project' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -185,15 +181,10 @@ block-level: 'NODE' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true diff --git a/jjb/functest/functest-ci-jobs.yml b/jjb/functest/functest-ci-jobs.yml index afeb1f92e..f8fdeb811 100644 --- a/jjb/functest/functest-ci-jobs.yml +++ b/jjb/functest/functest-ci-jobs.yml @@ -113,6 +113,11 @@ slave-label: armband-virtual installer: fuel <<: *colorado +# netvirt 3rd party ci + - virtual: + slave-label: odl-netvirt-virtual + installer: netvirt + <<: *master #-------------------------------- # None-CI PODs #-------------------------------- @@ -214,6 +219,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - '{installer}-defaults' - '{slave-label}-defaults' - 'functest-{testsuite}-parameter' @@ -224,14 +231,15 @@ name: DOCKER_TAG default: '{docker-tag}' description: 'Tag to pull docker image' + - string: + name: CLEAN_DOCKER_IMAGES + default: 'false' + description: 'Remove downloaded docker images (opnfv/functest:*)' - functest-parameter: gs-pathname: '{gs-pathname}' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm builders: - description-setter: @@ -280,7 +288,7 @@ - string: name: TESTCASE_OPTIONS default: '' - description: 'Addtional parameters specific to test case(s)' + description: 'Additional parameters specific to test case(s)' - parameter: name: functest-parameter parameters: @@ -300,6 +308,10 @@ name: CI_DEBUG default: 'false' description: "Show debug output information" + - string: + name: RC_FILE_PATH + default: '' + description: "Path to the OS credentials file if given" ######################## # trigger macros ######################## diff --git a/jjb/functest/functest-cleanup.sh b/jjb/functest/functest-cleanup.sh index 4bedfe7de..f8140e058 100755 --- a/jjb/functest/functest-cleanup.sh +++ b/jjb/functest/functest-cleanup.sh @@ -3,19 +3,34 @@ [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" echo "Cleaning up docker containers/images..." +FUNCTEST_IMAGE=opnfv/functest +# Remove containers along with image opnfv/functest:<none> +dangling_images=($(docker images -f "dangling=true" | grep $FUNCTEST_IMAGE | awk '{print $1}')) +if [[ -n ${dangling_images} ]]; then + echo " Removing $FUNCTEST_IMAGE:<none> images and their containers..." + for image_id in "${dangling_images[@]}"; do + echo " Removing image_id: $image_id and its containers" + docker ps -a | grep $image_id | awk '{print $1}'| xargs docker rm -f >${redirect} + docker rmi $image_id >${redirect} + done +fi + # Remove previous running containers if exist -if [[ ! -z $(docker ps -a | grep opnfv/functest) ]]; then - echo "Removing existing opnfv/functest containers..." - docker ps -a | grep opnfv/functest | awk '{print $1}' | xargs docker rm -f >${redirect} +functest_containers=$(docker ps -a | grep $FUNCTEST_IMAGE | awk '{print $1}') +if [[ -n ${functest_containers} ]]; then + echo " Removing existing $FUNCTEST_IMAGE containers..." + docker rm -f $functest_containers >${redirect} fi # Remove existing images if exist -if [[ ! -z $(docker images | grep opnfv/functest) ]]; then - echo "Docker images to remove:" - docker images | head -1 && docker images | grep opnfv/functest >${redirect} - image_tags=($(docker images | grep opnfv/functest | awk '{print $2}')) - for tag in "${image_tags[@]}"; do - echo "Removing docker image opnfv/functest:$tag..." - docker rmi opnfv/functest:$tag >/dev/null - done +if [[ $CLEAN_DOCKER_IMAGES == true ]]; then + functest_image_tags=($(docker images | grep $FUNCTEST_IMAGE | awk '{print $2}')) + if [[ -n ${functest_image_tags} ]]; then + echo " Docker images to be removed:" >${redirect} + (docker images | head -1 && docker images | grep $FUNCTEST_IMAGE) >${redirect} + for tag in "${functest_image_tags[@]}"; do + echo " Removing docker image $FUNCTEST_IMAGE:$tag..." + docker rmi $FUNCTEST_IMAGE:$tag >${redirect} + done + fi fi diff --git a/jjb/functest/functest-loop.sh b/jjb/functest/functest-loop.sh index 73856235f..4528c00d1 100755 --- a/jjb/functest/functest-loop.sh +++ b/jjb/functest/functest-loop.sh @@ -5,8 +5,10 @@ branch=${GIT_BRANCH##*/} [[ "$PUSH_RESULTS_TO_DB" == "true" ]] && flags+="-r" if [[ ${branch} == *"brahmaputra"* ]]; then cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh -s ${flags}" -else +elif [[ ${branch} == *"colorado"* ]]; then cmd="python ${FUNCTEST_REPO_DIR}/ci/run_tests.py -t all ${flags}" +else + cmd="python ${FUNCTEST_REPO_DIR}/functest/ci/run_tests.py -t all ${flags}" fi container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) docker exec $container_id $cmd @@ -15,4 +17,4 @@ ret_value=$? ret_val_file="${HOME}/opnfv/functest/results/${branch}/return_value" echo ${ret_value}>${ret_val_file} -exit 0
\ No newline at end of file +exit 0 diff --git a/jjb/functest/functest-project-jobs.yml b/jjb/functest/functest-project-jobs.yml index 236b95d92..11bc364e1 100644 --- a/jjb/functest/functest-project-jobs.yml +++ b/jjb/functest/functest-project-jobs.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/functest/functest-suite.sh b/jjb/functest/functest-suite.sh index 7e9fa09db..a30fb5973 100755 --- a/jjb/functest/functest-suite.sh +++ b/jjb/functest/functest-suite.sh @@ -5,8 +5,10 @@ branch=${GIT_BRANCH##*/} echo "Functest: run $FUNCTEST_SUITE_NAME on branch ${branch}" if [[ ${branch} == *"brahmaputra"* ]]; then cmd="${FUNCTEST_REPO_DIR}/docker/run_tests.sh --test $FUNCTEST_SUITE_NAME" -else +elif [[ ${branch} == *"colorado"* ]]; then cmd="python ${FUNCTEST_REPO_DIR}/ci/run_tests.py -t $FUNCTEST_SUITE_NAME" +else + cmd="functest testcase run $FUNCTEST_SUITE_NAME" fi container_id=$(docker ps -a | grep opnfv/functest | awk '{print $1}' | head -1) docker exec $container_id $cmd diff --git a/jjb/functest/set-functest-env.sh b/jjb/functest/set-functest-env.sh index 1c77702eb..583ce8041 100755 --- a/jjb/functest/set-functest-env.sh +++ b/jjb/functest/set-functest-env.sh @@ -2,8 +2,23 @@ set -e [[ $CI_DEBUG == true ]] && redirect="/dev/stdout" || redirect="/dev/null" -# labconfig is used only for joid -labconfig="" +# LAB_CONFIG is used only for joid + + +if [[ ${INSTALLER_TYPE} == 'joid' ]]; then + # If production lab then creds may be retrieved dynamically + # creds are on the jumphost, always in the same folder + rc_file_vol="-v $LAB_CONFIG/admin-openrc:/home/opnfv/functest/conf/openstack.creds" + # If dev lab, credentials may not be the default ones, just provide a path to put them into docker + # replace the default one by the customized one provided by jenkins config +fi + +if [[ ${RC_FILE_PATH} != '' ]] && [[ -f ${RC_FILE_PATH} ]] ; then + # volume if credentials file path is given to Functest + rc_file_vol="-v $RC_FILE_PATH:/home/opnfv/functest/conf/openstack.creds" +fi + + if [[ ${INSTALLER_TYPE} == 'apex' ]]; then ssh_options="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" if sudo virsh list | grep instack; then @@ -17,9 +32,9 @@ if [[ ${INSTALLER_TYPE} == 'apex' ]]; then exit 1 fi INSTALLER_IP=$(/usr/sbin/arp -e | grep ${instack_mac} | awk {'print $1'}) - sshkey="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa" + sshkey_vol="-v /root/.ssh/id_rsa:/root/.ssh/id_rsa" sudo scp $ssh_options root@${INSTALLER_IP}:/home/stack/stackrc ${HOME}/stackrc - stackrc="-v ${HOME}/stackrc:/home/opnfv/functest/conf/stackrc" + stackrc_vol="-v ${HOME}/stackrc:/home/opnfv/functest/conf/stackrc" if sudo iptables -C FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then sudo iptables -D FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable @@ -27,15 +42,10 @@ if [[ ${INSTALLER_TYPE} == 'apex' ]]; then if sudo iptables -C FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable 2> ${redirect}; then sudo iptables -D FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable fi - -elif [[ ${INSTALLER_TYPE} == 'joid' ]]; then - # If production lab then creds may be retrieved dynamically - # creds are on the jumphost, always in the same folder - labconfig="-v $LAB_CONFIG/admin-openrc:/home/opnfv/functest/conf/openstack.creds" - # If dev lab, credentials may not be the default ones, just provide a path to put them into docker - # replace the default one by the customized one provided by jenkins config fi + + # Set iptables rule to allow forwarding return traffic for container if ! sudo iptables -C FORWARD -j RETURN 2> ${redirect} || ! sudo iptables -L FORWARD | awk 'NR==3' | grep RETURN 2> ${redirect}; then sudo iptables -I FORWARD -j RETURN @@ -45,22 +55,28 @@ DEPLOY_TYPE=baremetal [[ $BUILD_TAG =~ "virtual" ]] && DEPLOY_TYPE=virt echo "Functest: Start Docker and prepare environment" -envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} \ - -e NODE_NAME=${NODE_NAME} -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} \ - -e BUILD_TAG=${BUILD_TAG} -e CI_DEBUG=${CI_DEBUG} -e DEPLOY_TYPE=${DEPLOY_TYPE}" + branch=${GIT_BRANCH##*/} dir_result="${HOME}/opnfv/functest/results/${branch}" mkdir -p ${dir_result} sudo rm -rf ${dir_result}/* -res_volume="-v ${dir_result}:/home/opnfv/functest/results" +results_vol="-v ${dir_result}:/home/opnfv/functest/results" custom_params= test -f ${HOME}/opnfv/functest/custom/params_${DOCKER_TAG} && custom_params=$(cat ${HOME}/opnfv/functest/custom/params_${DOCKER_TAG}) +envs="-e INSTALLER_TYPE=${INSTALLER_TYPE} -e INSTALLER_IP=${INSTALLER_IP} \ + -e NODE_NAME=${NODE_NAME} -e DEPLOY_SCENARIO=${DEPLOY_SCENARIO} \ + -e BUILD_TAG=${BUILD_TAG} -e CI_DEBUG=${CI_DEBUG} -e DEPLOY_TYPE=${DEPLOY_TYPE}" + + +volumes="${results_vol} ${sshkey_vol} ${stackrc_vol} ${rc_file_vol}" + + echo "Functest: Pulling image opnfv/functest:${DOCKER_TAG}" docker pull opnfv/functest:$DOCKER_TAG >/dev/null -cmd="sudo docker run --privileged=true -id ${envs} ${labconfig} ${sshkey} \ - ${res_volume} ${custom_params} ${stackrc} ${TESTCASE_OPTIONS} \ +cmd="sudo docker run --privileged=true -id ${envs} ${volumes} \ + ${custom_params} ${TESTCASE_OPTIONS} \ opnfv/functest:${DOCKER_TAG} /bin/bash" echo "Functest: Running docker run command: ${cmd}" ${cmd} >${redirect} @@ -82,8 +98,10 @@ if [ $(docker ps | grep "opnfv/functest:${DOCKER_TAG}" | wc -l) == 0 ]; then fi if [[ ${branch} == *"brahmaputra"* ]]; then cmd="${FUNCTEST_REPO_DIR}/docker/prepare_env.sh" -else +elif [[ ${branch} == *"colorado"* ]]; then cmd="python ${FUNCTEST_REPO_DIR}/ci/prepare_env.py start" +else + cmd="python ${FUNCTEST_REPO_DIR}/functest/ci/prepare_env.py start" fi echo "Executing command inside the docker: ${cmd}" docker exec ${container_id} ${cmd} diff --git a/jjb/opnfv/installer-params.yml b/jjb/global/installer-params.yml index ec0b86170..4a50a5906 100644 --- a/jjb/opnfv/installer-params.yml +++ b/jjb/global/installer-params.yml @@ -59,7 +59,7 @@ parameters: - string: name: INSTALLER_IP - default: '192.168.Y.Y' + default: '192.168.122.5' description: 'IP of the installer' - string: name: INSTALLER_TYPE @@ -67,11 +67,11 @@ description: 'Installer used for deploying OPNFV on this POD' - string: name: OS_RELEASE - default: 'mitaka' - description: 'OpenStack release (liberty|mitaka)' + default: 'newton' + description: 'OpenStack release (mitaka|newton)' - string: name: EXTERNAL_NETWORK - default: ext-net4 + default: ext-net description: "External network used for Floating ips." - string: name: LAB_CONFIG @@ -84,7 +84,7 @@ - string: name: UBUNTU_DISTRO default: 'xenial' - description: "Ubuntu distribution to use for Openstack (trusty|xenial)" + description: "Ubuntu distribution to use for Openstack (xenial)" - string: name: CPU_ARCHITECTURE default: 'amd64' @@ -100,3 +100,22 @@ name: INSTALLER_TYPE default: infra description: 'Installer used for deploying OPNFV on this POD' +- parameter: + name: 'netvirt-defaults' + parameters: + - string: + name: INSTALLER_IP + default: '192.168.X.X' + description: 'IP of the installer' + - string: + name: INSTALLER_TYPE + default: netvirt + description: 'Installer used for deploying OPNFV on this POD' + - string: + name: DEPLOY_SCENARIO + default: 'os-odl_l2-bgpvpn-noha' + description: 'Scenario to deploy and test' + - string: + name: EXTERNAL_NETWORK + default: 'external' + description: 'external network for test' diff --git a/jjb/releng-defaults.yaml b/jjb/global/releng-defaults.yml index bf3330657..5003a8f48 100644 --- a/jjb/releng-defaults.yaml +++ b/jjb/global/releng-defaults.yml @@ -9,13 +9,9 @@ artifactDaysToKeep: 30 artifactNumToKeep: 100 - ssh-credentials: 'd42411ac011ad6f3dd2e1fa34eaa5d87f910eb2e' wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper project-type: freestyle node: master - diff --git a/jjb/releng-macros.yaml b/jjb/global/releng-macros.yml index 3afd355e6..cd92480d4 100644 --- a/jjb/releng-macros.yaml +++ b/jjb/global/releng-macros.yml @@ -24,30 +24,30 @@ default: '{branch}' description: "JJB configured GERRIT_BRANCH parameter" +- wrapper: + name: ssh-agent-wrapper + wrappers: + - ssh-agent-credentials: + users: + - 'd42411ac011ad6f3dd2e1fa34eaa5d87f910eb2e' + - scm: name: git-scm scm: - - git: - credentials-id: '{credentials-id}' + - git: &git-scm-defaults + credentials-id: '$SSH_CREDENTIAL_ID' url: '$GIT_BASE' - refspec: '' branches: - - 'origin/{branch}' - skip-tag: true - wipe-workspace: true + - 'origin/$GERRIT_BRANCH' + timeout: 15 - scm: - name: gerrit-trigger-scm + name: git-scm-gerrit scm: - git: - credentials-id: '{credentials-id}' - url: '$GIT_BASE' - refspec: '{refspec}' - branches: - - 'origin/$GERRIT_BRANCH' - skip-tag: true - choosing-strategy: '{choosing-strategy}' - timeout: 15 + choosing-strategy: 'gerrit' + refspec: '$GERRIT_REFSPEC' + <<: *git-scm-defaults - trigger: name: 'daily-trigger-disabled' diff --git a/jjb/opnfv/slave-params.yml b/jjb/global/slave-params.yml index b46960fa7..30ebb950f 100644 --- a/jjb/opnfv/slave-params.yml +++ b/jjb/global/slave-params.yml @@ -425,6 +425,19 @@ name: GIT_BASE default: https://gerrit.opnfv.org/gerrit/$PROJECT - parameter: + name: 'huawei-pod7-defaults' + parameters: + - node: + name: SLAVE_NAME + description: 'Slave name on Jenkins' + allowed-slaves: + - huawei-pod7 + default-slaves: + - huawei-pod7 + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT +- parameter: name: 'zte-pod1-defaults' parameters: - node: @@ -660,6 +673,36 @@ name: SSH_KEY default: /root/.ssh/id_rsa description: 'SSH key to be used' +- parameter: + name: 'multisite-virtual-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'multisite-virtual' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' +- parameter: + name: 'ericsson-virtual5-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'ericsson-virtual5' + - string: + name: GIT_BASE + default: https://git.opendaylight.org/gerrit/p/$PROJECT.git + description: 'Git URL to use on this Jenkins Slave' +- parameter: + name: 'odl-netvirt-virtual-defaults' + parameters: + - label: + name: SLAVE_LABEL + default: 'odl-netvirt-virtual' + - string: + name: GIT_BASE + default: https://gerrit.opnfv.org/gerrit/$PROJECT + description: 'Git URL to use on this Jenkins Slave' ##################################################### # These slaves are just dummy slaves for sandbox jobs ##################################################### diff --git a/jjb/infra/bifrost-verify-jobs.yml b/jjb/infra/bifrost-verify-jobs.yml index 751aa0c4a..baf33ea3a 100644 --- a/jjb/infra/bifrost-verify-jobs.yml +++ b/jjb/infra/bifrost-verify-jobs.yml @@ -24,17 +24,23 @@ disabled: false dib-os-release: 'trusty' dib-os-element: 'ubuntu-minimal' - dib-os-packages: 'openssh-server,vlan,vim,less,bridge-utils,language-pack-en,iputils-ping,rsyslog,curl' + dib-os-packages: 'vlan,vim,less,bridge-utils,language-pack-en,iputils-ping,rsyslog,curl' + extra-dib-elements: 'openssh-server' - 'centos7': disabled: false dib-os-release: '7' dib-os-element: 'centos7' - dib-os-packages: 'openssh-server,vim,less,bridge-utils,iputils,rsyslog,curl' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' - 'suse': - disabled: true - dib-os-release: 'suse' - dib-os-element: 'suse' - dib-os-packages: '' + disabled: false + dib-os-release: '42.2' + dib-os-element: 'opensuse-minimal' + dib-os-packages: 'vim,less,bridge-utils,iputils,rsyslog,curl' + extra-dib-elements: 'openssh-server' + vm-disk: '30' + vm-memory: '4096' + vm-cpu: '2' #-------------------------------- # type #-------------------------------- @@ -45,6 +51,16 @@ #-------------------------------- jobs: - '{project}-bifrost-verify-{distro}-{type}-{stream}' + +#-------------------------------- +# VM defaults +#-------------------------------- +- defaults: + name: vm_defaults + vm-disk: '100' + vm-memory: '8192' + vm-cpu: '4' + #-------------------------------- # job templates #-------------------------------- @@ -53,6 +69,8 @@ disabled: '{obj:disabled}' + defaults: vm_defaults + concurrent: false properties: @@ -82,9 +100,21 @@ name: DIB_OS_ELEMENT default: '{dib-os-element}' - string: + name: EXTRA_DIB_ELEMENTS + default: '{extra-dib-elements}' + - string: name: DIB_OS_PACKAGES default: '{dib-os-packages}' - string: + name: VM_DISK + default: '{vm-disk}' + - string: + name: VM_MEMORY + default: '{vm-memory}' + - string: + name: VM_CPU + default: '{vm-cpu}' + - string: name: CLEAN_DIB_IMAGES default: 'true' - label: @@ -102,10 +132,6 @@ timeout: 10 wipe-workspace: true - triggers: - - '{project}-gerrit-trigger': - branch: '{branch}' - builders: - description-setter: description: "Built on $NODE_NAME" diff --git a/jjb/infra/bifrost-verify.sh b/jjb/infra/bifrost-verify.sh index ded4ed463..9fbb1d0d9 100755 --- a/jjb/infra/bifrost-verify.sh +++ b/jjb/infra/bifrost-verify.sh @@ -17,6 +17,9 @@ function fix_ownership() { if [ -z "${JOB_URL+x}" ]; then echo "Not running as part of Jenkins. Handle the logs manually." else + # Make sure cache exists + [[ ! -d ${HOME}/.cache ]] && mkdir ${HOME}/.cache + sudo chown -R jenkins:jenkins $WORKSPACE sudo chown -R jenkins:jenkins ${HOME}/.cache fi diff --git a/jjb/ipv6/ipv6.yml b/jjb/ipv6/ipv6.yml index d9dafdf69..db662aed8 100644 --- a/jjb/ipv6/ipv6.yml +++ b/jjb/ipv6/ipv6.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/joid/joid-daily-jobs.yml b/jjb/joid/joid-daily-jobs.yml index 6d0370983..20429f1a4 100644 --- a/jjb/joid/joid-daily-jobs.yml +++ b/jjb/joid/joid-daily-jobs.yml @@ -178,6 +178,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - '{installer}-defaults' - '{slave-label}-defaults': installer: '{installer}' @@ -186,10 +188,7 @@ default: 'os-odl_l2-nofeature-ha' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm builders: - description-setter: diff --git a/jjb/joid/joid-verify-jobs.yml b/jjb/joid/joid-verify-jobs.yml index 9d362d800..516e822a7 100644 --- a/jjb/joid/joid-verify-jobs.yml +++ b/jjb/joid/joid-verify-jobs.yml @@ -58,21 +58,17 @@ block-level: 'NODE' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -167,15 +163,10 @@ block-level: 'NODE' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true diff --git a/jjb/kvmfornfv/kvmfornfv-test.sh b/jjb/kvmfornfv/kvmfornfv-test.sh index 06377ac4f..b31d61cce 100755 --- a/jjb/kvmfornfv/kvmfornfv-test.sh +++ b/jjb/kvmfornfv/kvmfornfv-test.sh @@ -13,13 +13,20 @@ else exit 1 fi +echo $TEST_NAME + # do stuff differently based on the job type case "$JOB_TYPE" in - verify|daily) + verify) #start the test cd $WORKSPACE ./ci/test_kvmfornfv.sh $JOB_TYPE ;; + daily) + #start the test + cd $WORKSPACE + ./ci/test_kvmfornfv.sh $JOB_TYPE $TEST_NAME + ;; *) echo "Test is not enabled for $JOB_TYPE jobs" exit 1 diff --git a/jjb/kvmfornfv/kvmfornfv.yml b/jjb/kvmfornfv/kvmfornfv.yml index b6a55fe6a..33c73f9a7 100644 --- a/jjb/kvmfornfv/kvmfornfv.yml +++ b/jjb/kvmfornfv/kvmfornfv.yml @@ -23,12 +23,18 @@ ##################################### # patch verification phases ##################################### + testname: + - 'cyclictest' + - 'packet_forward' +##################################### +# patch verification phases +##################################### jobs: - 'kvmfornfv-verify-{stream}' - 'kvmfornfv-verify-{phase}-{stream}' - 'kvmfornfv-merge-{stream}' - 'kvmfornfv-daily-{stream}' - - 'kvmfornfv-daily-{phase}-{stream}' + - 'kvmfornfv-{testname}-daily-{phase}-{stream}' ##################################### # job templates ##################################### @@ -56,6 +62,7 @@ triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -102,7 +109,6 @@ node-parameters: false kill-phase-on: FAILURE abort-all-job: true - - job-template: name: 'kvmfornfv-verify-{phase}-{stream}' @@ -111,15 +117,10 @@ concurrent: true scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true @@ -152,13 +153,11 @@ gs-pathname: '{gs-pathname}' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -193,56 +192,67 @@ gs-pathname: '{gs-pathname}' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - - timed: '@midnight' + - timed: '@midnight' builders: - description-setter: description: "Built on $NODE_NAME" - multijob: - name: build + name: cyclictest-build condition: SUCCESSFUL projects: - - name: 'kvmfornfv-daily-build-{stream}' + - name: 'kvmfornfv-cyclictest-daily-build-{stream}' current-parameters: false node-parameters: false git-revision: true kill-phase-on: FAILURE abort-all-job: true - multijob: - name: test + name: cyclictest-test condition: SUCCESSFUL projects: - - name: 'kvmfornfv-daily-test-{stream}' + - name: 'kvmfornfv-cyclictest-daily-test-{stream}' + current-parameters: false + node-parameters: false + git-revision: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: packetforward-build + condition: SUCCESSFUL + projects: + - name: 'kvmfornfv-packet_forward-daily-build-{stream}' + current-parameters: false + node-parameters: false + git-revision: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: packetforward-test + condition: SUCCESSFUL + projects: + - name: 'kvmfornfv-packet_forward-daily-test-{stream}' current-parameters: false node-parameters: false git-revision: true kill-phase-on: FAILURE abort-all-job: true - - job-template: - name: 'kvmfornfv-daily-{phase}-{stream}' + name: 'kvmfornfv-{testname}-daily-{phase}-{stream}' disabled: '{obj:disabled}' concurrent: false scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm wrappers: - - ssh-agent-credentials: - users: - - '{ssh-credentials}' + - ssh-agent-wrapper - timeout: timeout: 360 fail: true @@ -254,11 +264,15 @@ - '{slave-label}-defaults' - 'kvmfornfv-defaults': gs-pathname: '{gs-pathname}' + - string: + name: TEST_NAME + default: '{testname}' + description: "Daily job to execute kvmfornfv '{testname}' testcase." builders: - description-setter: description: "Built on $NODE_NAME" - - '{project}-daily-{phase}-macro' + - '{project}-{testname}-daily-{phase}-macro' ##################################### # builder macros ##################################### @@ -277,20 +291,33 @@ - shell: !include-raw: ./kvmfornfv-test.sh - builder: - name: 'kvmfornfv-daily-build-macro' + name: 'kvmfornfv-cyclictest-daily-build-macro' builders: - shell: !include-raw: ./kvmfornfv-build.sh - shell: !include-raw: ./kvmfornfv-upload-artifact.sh - builder: - name: 'kvmfornfv-daily-test-macro' + name: 'kvmfornfv-cyclictest-daily-test-macro' + builders: + - shell: + !include-raw: ./kvmfornfv-download-artifact.sh + - shell: + !include-raw: ./kvmfornfv-test.sh +- builder: + name: 'kvmfornfv-packet_forward-daily-build-macro' + builders: + - shell: + !include-raw: ./kvmfornfv-build.sh + - shell: + !include-raw: ./kvmfornfv-upload-artifact.sh +- builder: + name: 'kvmfornfv-packet_forward-daily-test-macro' builders: - shell: !include-raw: ./kvmfornfv-download-artifact.sh - shell: !include-raw: ./kvmfornfv-test.sh - ##################################### # parameter macros ##################################### diff --git a/jjb/moon/moon.yml b/jjb/moon/moon.yml index 55d593f85..15c3ddec8 100644 --- a/jjb/moon/moon.yml +++ b/jjb/moon/moon.yml @@ -22,13 +22,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -53,4 +51,4 @@ - shell: #!/bin/bash echo "launch Moon unit tests" - nosetest $WORKSPACE/keystone-moon/keystone/tests/moon/unit
\ No newline at end of file + nosetest $WORKSPACE/keystone-moon/keystone/tests/moon/unit diff --git a/jjb/multisite/multisite-daily-jobs.yml b/jjb/multisite/multisite-daily-jobs.yml new file mode 100644 index 000000000..cfb40a1fa --- /dev/null +++ b/jjb/multisite/multisite-daily-jobs.yml @@ -0,0 +1,247 @@ +- project: + name: kingbird + + project: 'multisite' + + jobs: + - 'multisite-kingbird-virtual-daily-{stream}' + - 'multisite-{phase}-{stream}' + + phase: + - 'fuel-deploy-regionone-virtual' + - 'fuel-deploy-regiontwo-virtual' + - 'register-endpoints' + - 'update-auth' + - 'kingbird-deploy-virtual' + - 'kingbird-functest' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + timed: '#@midnight' + +- job-template: + name: 'multisite-kingbird-virtual-daily-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: false + + parameters: + - project-parameter: + project: '{project}' + - string: + name: KINGBIRD_LOG_FILE + default: $WORKSPACE/kingbird.log + - 'multisite-virtual-defaults' + - string: + name: DEPLOY_SCENARIO + default: 'os-nosdn-multisite-noha' + + triggers: + - timed: '{timed}' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: fuel-deploy-virtual + condition: SUCCESSFUL + projects: + - name: 'multisite-fuel-deploy-regionone-virtual-{stream}' + current-parameters: false + predefined-parameters: | + FUEL_VERSION=latest + DEPLOY_SCENARIO=$DEPLOY_SCENARIO + OS_REGION=RegionOne + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual2 + kill-phase-on: FAILURE + abort-all-job: true + - name: 'multisite-fuel-deploy-regiontwo-virtual-{stream}' + current-parameters: false + predefined-parameters: | + FUEL_VERSION=latest + DEPLOY_SCENARIO=$DEPLOY_SCENARIO + OS_REGION=RegionTwo + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual6 + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: centralize-keystone + condition: SUCCESSFUL + projects: + - name: 'multisite-register-endpoints-{stream}' + current-parameters: false + predefined-parameters: | + OS_REGION=RegionOne + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual2 + kill-phase-on: FAILURE + abort-all-job: true + - name: 'multisite-update-auth-{stream}' + current-parameters: false + predefined-parameters: | + OS_REGION=RegionTwo + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual6 + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: kingbird-deploy-virtual + condition: SUCCESSFUL + projects: + - name: 'multisite-kingbird-deploy-virtual-{stream}' + current-parameters: false + predefined-parameters: | + OS_REGION=RegionOne + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual2 + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: kingbird-functest + condition: SUCCESSFUL + projects: + - name: 'multisite-kingbird-functest-{stream}' + current-parameters: false + predefined-parameters: | + OS_REGION=RegionOne + REGIONONE_IP=10.2.117.79 + REGIONTWO_IP=10.2.117.181 + node-parameters: false + node-label-name: SLAVE_LABEL + node-label: intel-virtual2 + kill-phase-on: NEVER + abort-all-job: false + +- job-template: + name: 'multisite-{phase}-{stream}' + + concurrent: false + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - 'multisite-{phase}-builder': + stream: '{stream}' + + publishers: + - 'multisite-{phase}-publisher' + +######################## +# builder macros +######################## +- builder: + name: 'multisite-fuel-deploy-regionone-virtual-builder' + builders: + - shell: | + #!/bin/bash + + echo "This is where we deploy fuel, extract passwords and save into file" +- builder: + name: 'multisite-fuel-deploy-regiontwo-virtual-builder' + builders: + - shell: | + #!/bin/bash + + echo "This is where we deploy fuel, extract publicUrl, privateUrl, and adminUrl and save into file" +- builder: + name: 'multisite-register-endpoints-builder' + builders: + - copyartifact: + project: 'multisite-fuel-deploy-regiontwo-virtual-{stream}' + which-build: multijob-build + filter: "RegionTwo-Endpoints.txt" + - shell: | + #!/bin/bash + + echo "This is where we register RegionTwo in RegionOne keystone" +- builder: + name: 'multisite-update-auth-builder' + builders: + - copyartifact: + project: 'multisite-fuel-deploy-regionone-virtual-{stream}' + which-build: multijob-build + filter: "RegionOne-Passwords.txt" + - shell: | + #!/bin/bash + + echo "This is where we read passwords from RegionOne-passwords.txt and replace passwords in RegionTwo" +- builder: + name: 'multisite-kingbird-deploy-virtual-builder' + builders: + - shell: | + #!/bin/bash + + echo "This is where we install kingbird" +- builder: + name: 'multisite-kingbird-functest-builder' + builders: + - shell: | + #!/bin/bash + + echo "This is where we run kingbird-functest" +######################## +# publisher macros +######################## +- publisher: + name: 'multisite-fuel-deploy-regionone-virtual-publisher' + publishers: + - archive: + artifacts: '/root/servicepass.ini' + allow-empty: false + only-if-success: true + fingerprint: true +- publisher: + name: 'multisite-fuel-deploy-regiontwo-virtual-publisher' + publishers: + - archive: + artifacts: '/root/endpoints.ini' + allow-empty: false + only-if-success: true + fingerprint: true +- publisher: + name: 'multisite-register-endpoints-publisher' + publishers: + - archive: + artifacts: 'dummy.txt' + allow-empty: true +- publisher: + name: 'multisite-update-auth-publisher' + publishers: + - archive: + artifacts: 'dummy.txt' + allow-empty: true +- publisher: + name: 'multisite-kingbird-deploy-virtual-publisher' + publishers: + - archive: + artifacts: 'dummy.txt' + allow-empty: true +- publisher: + name: 'multisite-kingbird-functest-publisher' + publishers: + - archive: + artifacts: 'dummy.txt' + allow-empty: true diff --git a/jjb/multisite/multisite.yml b/jjb/multisite/multisite.yml index 24c03fd4a..cfe9b2d13 100644 --- a/jjb/multisite/multisite.yml +++ b/jjb/multisite/multisite.yml @@ -39,13 +39,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -95,10 +93,7 @@ default: 'os-nosdn-multisite-ha' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - timed: '{timed}' @@ -127,10 +122,7 @@ concurrent: false scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'gerrit' + - git-scm-gerrit builders: - 'multisite-kingbird-deploy' diff --git a/jjb/netready/netready-gluon-build.sh b/jjb/netready/netready-gluon-build.sh new file mode 100755 index 000000000..141e84cbd --- /dev/null +++ b/jjb/netready/netready-gluon-build.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +echo "Building Gluon packages." +echo "------------------------" +echo + +OPNFV_ARTIFACT_VERSION=$(echo $(date -u +"%Y%m%d")) + +# build all packages +cd $WORKSPACE/ci +./build-gluon-packages.sh + +# list the contents of BUILD_OUTPUT directory +echo "Build Directory is ${BUILD_DIRECTORY}" +echo "Build Directory Contents:" +echo "---------------------------------------" +ls -alR $BUILD_DIRECTORY + +# get version infos from Gluon from spec +GLUON_VERSION=$(grep Version: $BUILD_DIRECTORY/rpm_specs/gluon.spec | awk '{ print $2 }') +GLUON_RELEASE=$(grep 'define release' $BUILD_DIRECTORY/rpm_specs/gluon.spec | awk '{ print $3 }')_$OPNFV_ARTIFACT_VERSION + +ARTIFACT_NAME=gluon-$GLUON_VERSION-$GLUON_RELEASE.noarch.rpm +ARTIFACT_PATH=$BUILD_DIRECTORY/noarch/$ARTIFACT_NAME + +echo "Writing opnfv.properties file" +# save information regarding artifact into file +( + echo "OPNFV_ARTIFACT_VERSION=$OPNFV_ARTIFACT_VERSION" + echo "OPNFV_GIT_URL=$(git config --get remote.origin.url)" + echo "OPNFV_GIT_SHA1=$(git rev-parse HEAD)" + echo "OPNFV_ARTIFACT_URL=$GS_URL/$ARTIFACT_NAME" + echo "OPNFV_ARTIFACT_SHA512SUM=$(sha512sum $ARTIFACT_PATH | cut -d' ' -f1)" + echo "OPNFV_BUILD_URL=$BUILD_URL" + echo "ARTIFACT_LIST=$ARTIFACT_PATH" +) > $WORKSPACE/opnfv.properties + +echo "---------------------------------------" +echo "Done!" diff --git a/jjb/netready/netready-upload-gluon-packages.sh b/jjb/netready/netready-upload-gluon-packages.sh new file mode 100755 index 000000000..7c1e33727 --- /dev/null +++ b/jjb/netready/netready-upload-gluon-packages.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -o errexit +set -o nounset +set -o pipefail + +echo "Uploading Gluon packages" +echo "--------------------------------------------------------" +echo + +source $WORKSPACE/opnfv.properties + +for artifact in $ARTIFACT_LIST; do + echo "Uploading artifact: ${artifact}" + gsutil cp $artifact gs://$GS_URL/$(basename $artifact) > gsutil.$(basename $artifact).log + echo "Upload complete for ${artifact}" +done + +gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/opnfv-$OPNFV_ARTIFACT_VERSION.properties > gsutil.properties.log +gsutil cp $WORKSPACE/opnfv.properties gs://$GS_URL/latest.properties > gsutil.properties.log + +echo "--------------------------------------------------------" +echo "Upload done!" + +echo "Artifacts are not available as:" +for artifact in $ARTIFACT_LIST; do + echo "http://$GS_URL/$(basename $artifact)" +done diff --git a/jjb/netready/netready.yml b/jjb/netready/netready.yml index 3e2f95a76..3d043f9e3 100644 --- a/jjb/netready/netready.yml +++ b/jjb/netready/netready.yml @@ -5,16 +5,13 @@ jobs: - 'netready-verify-{stream}' + - 'netready-build-gluon-packages-daily-{stream}' stream: - master: branch: '{stream}' gs-pathname: '' disabled: false - - colorado: - branch: 'stable/{stream}' - gs-pathname: '/{stream}' - disabled: false - job-template: name: 'netready-verify-{stream}' @@ -27,13 +24,11 @@ - gerrit-parameter: branch: '{branch}' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -57,3 +52,60 @@ builders: - shell: | echo "Nothing to verify!" + + + +- job-template: + name: 'netready-build-gluon-packages-daily-{stream}' + + disabled: false + + concurrent: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + - 'netready-parameter': + gs-pathname: '{gs-pathname}' + + scm: + - git-scm + + builders: + - 'netready-gluon-build' + + triggers: + - timed: '@midnight' + + +######################## +# builder macros +######################## + +- builder: + name: 'netready-gluon-build' + builders: + - shell: + !include-raw: ./netready-gluon-build.sh + - shell: + !include-raw: ./netready-upload-gluon-packages.sh + + +######################## +# parameter macros +######################## + +- parameter: + name: netready-parameter + parameters: + - string: + name: BUILD_DIRECTORY + default: $WORKSPACE/build + description: "Directory where the build artifact will be located upon the completion of the build." + - string: + name: GS_URL + default: artifacts.opnfv.org/$PROJECT{gs-pathname} + description: "URL to Google Storage." diff --git a/jjb/octopus/octopus.yml b/jjb/octopus/octopus.yml index c4e34ca7d..71d6a41fe 100644 --- a/jjb/octopus/octopus.yml +++ b/jjb/octopus/octopus.yml @@ -32,13 +32,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/onosfw/onosfw.yml b/jjb/onosfw/onosfw.yml index 0c90c577a..07bcdc5f6 100644 --- a/jjb/onosfw/onosfw.yml +++ b/jjb/onosfw/onosfw.yml @@ -36,13 +36,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -74,13 +72,12 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - 'opnfv-build-ubuntu-defaults' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm triggers: - timed: '@midnight' @@ -99,6 +96,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - 'opnfv-build-ubuntu-defaults' - string: name: GS_URL @@ -106,10 +105,7 @@ description: "Directory where the build artifact will be located upon the completion of the build." scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm builders: - 'builder-onosfw-helloworld' diff --git a/jjb/opera/opera-daily-jobs.yml b/jjb/opera/opera-daily-jobs.yml new file mode 100644 index 000000000..556d59fcb --- /dev/null +++ b/jjb/opera/opera-daily-jobs.yml @@ -0,0 +1,143 @@ +- project: + name: 'opera-daily-jobs' + + project: 'opera' + +##################################### +# branch definitions +##################################### + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + +##################################### +# patch verification phases +##################################### + phase: + - 'basic' + - 'deploy' + +##################################### +# jobs +##################################### + jobs: + - 'opera-daily-{stream}' + - 'opera-daily-{phase}-{stream}' +##################################### +# job templates +##################################### +- job-template: + name: 'opera-daily-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: false + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + option: 'project' + + scm: + - git-scm + + wrappers: + - ssh-agent-wrapper + + - timeout: + timeout: 120 + fail: true + + triggers: + - timed: '@midnight' + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'huawei-pod7-defaults' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: basic + condition: SUCCESSFUL + projects: + - name: 'opera-daily-basic-{stream}' + current-parameters: true + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: deploy + condition: SUCCESSFUL + projects: + - name: 'opera-daily-deploy-{stream}' + current-parameters: true + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true +# - multijob: +# name: functest +# condition: SUCCESSFUL +# projects: +# - name: 'functest-compass-baremetal-suite-{stream}' +# current-parameters: true +# predefined-parameters: +# FUNCTEST_SUITE_NAME=opera +# node-parameters: true +# kill-phase-on: NEVER +# abort-all-job: true + +- job-template: + name: 'opera-daily-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-per-node: 1 + option: 'project' + + scm: + - git-scm + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 120 + fail: true + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-daily-{phase}-macro' + +##################################### +# builder macros +##################################### +- builder: + name: 'opera-daily-basic-macro' + builders: + - shell: | + #!/bin/bash + echo "Hello world!" + +- builder: + name: 'opera-daily-deploy-macro' + builders: + - shell: | + #!/bin/bash + echo "Hello world!" + diff --git a/jjb/opera/opera-project-jobs.yml b/jjb/opera/opera-project-jobs.yml new file mode 100644 index 000000000..19f066b5f --- /dev/null +++ b/jjb/opera/opera-project-jobs.yml @@ -0,0 +1,57 @@ +- project: + + name: opera-project + + project: 'opera' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + + jobs: + - 'opera-build-{stream}' + +######################## +# job templates +######################## +- job-template: + name: 'opera-build-{stream}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + option: 'project' + + parameters: + - project-parameter: + project: '{project}' + - 'opnfv-build-ubuntu-defaults' + - gerrit-parameter: + branch: '{branch}' + + scm: + - git-scm + + triggers: + - timed: 'H 23 * * *' + + builders: + - 'opera-build-macro' + +##################################### +# builder macros +##################################### +- builder: + name: 'opera-build-macro' + builders: + - shell: | + #!/bin/bash + + echo "Hello world!" + + diff --git a/jjb/opera/opera-verify-jobs.yml b/jjb/opera/opera-verify-jobs.yml new file mode 100644 index 000000000..0e9dba01d --- /dev/null +++ b/jjb/opera/opera-verify-jobs.yml @@ -0,0 +1,155 @@ +- project: + name: 'opera-verify-jobs' + + project: 'opera' + +##################################### +# branch definitions +##################################### + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + +##################################### +# patch verification phases +##################################### + phase: + - 'basic' + - 'deploy' + +##################################### +# jobs +##################################### + jobs: + - 'opera-verify-{stream}' + - 'opera-verify-{phase}-{stream}' +##################################### +# job templates +##################################### +- job-template: + name: 'opera-verify-{stream}' + + project-type: multijob + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-total: 1 + max-per-node: 1 + option: 'project' + + scm: + - git-scm-gerrit + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 120 + fail: true + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: '**/*' + forbidden-file-paths: + - compare-type: ANT + pattern: 'docs/**' + readable-message: true + + parameters: + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'huawei-pod7-defaults' + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - multijob: + name: basic + condition: SUCCESSFUL + projects: + - name: 'opera-verify-basic-{stream}' + current-parameters: true + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + - multijob: + name: deploy + condition: SUCCESSFUL + projects: + - name: 'opera-verify-deploy-{stream}' + current-parameters: true + node-parameters: true + kill-phase-on: FAILURE + abort-all-job: true + +- job-template: + name: 'opera-verify-{phase}-{stream}' + + disabled: '{obj:disabled}' + + concurrent: true + + properties: + - throttle: + enabled: true + max-per-node: 1 + option: 'project' + + scm: + - git-scm-gerrit + + wrappers: + - ssh-agent-wrapper + - timeout: + timeout: 120 + fail: true + + builders: + - description-setter: + description: "Built on $NODE_NAME" + - '{project}-verify-{phase}-macro' + +##################################### +# builder macros +##################################### +- builder: + name: 'opera-verify-basic-macro' + builders: + - shell: | + #!/bin/bash + echo "Hello world!" + +- builder: + name: 'opera-verify-deploy-macro' + builders: + - shell: | + #!/bin/bash + echo "Hello world!" + diff --git a/jjb/opnfv/opnfv-docker.yml b/jjb/opnfv/opnfv-docker.yml deleted file mode 100644 index f313b3b92..000000000 --- a/jjb/opnfv/opnfv-docker.yml +++ /dev/null @@ -1,169 +0,0 @@ -############################################## -# job configuration for docker build and push -############################################## - -- project: - - name: opnfv-docker - - project: - - 'bottlenecks' - - 'cperf' - - 'functest' - - 'storperf' - - 'qtip' - - jobs: - - '{project}-docker-build-push-{stream}' - - 'yardstick-docker-build-push-{stream}' - #dovetail not sync with release, an independent job - #only master by now, will adjust accordingly in future - - 'dovetail-docker-build-push-{dovetailstream}' - - stream: - - master: - branch: '{stream}' - disabled: false - - colorado: - branch: 'stable/{stream}' - disabled: false - dovetailstream: - - master: - branch: '{dovetailstream}' - disabled: false - -######################## -# job templates -######################## -- job-template: - name: '{project}-docker-build-push-{stream}' - - disabled: '{obj:disabled}' - - parameters: - - project-parameter: - project: '{project}' - - 'opnfv-build-ubuntu-defaults' - - string: - name: PUSH_IMAGE - default: "true" - description: "To enable/disable pushing the image to Dockerhub." - - string: - name: BASE_VERSION - default: "colorado.0" - description: "Base version to be used." - - string: - name: DOCKER_REPO_NAME - default: "opnfv/{project}" - description: "Dockerhub repo to be pushed to." - - string: - name: UPDATE_LATEST_STABLE - default: "false" - description: "This will update the latest_stable image only." - - string: - name: STABLE_TAG - description: "If above option is true, this is the tag to be pulled." - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - builders: - - shell: - !include-raw-escape: ./opnfv-docker.sh - - triggers: - - pollscm: - cron: "*/30 * * * *" - -- job-template: - name: 'yardstick-docker-build-push-{stream}' - - disabled: '{obj:disabled}' - - parameters: - - project-parameter: - project: 'yardstick' - - 'opnfv-build-ubuntu-defaults' - - string: - name: PUSH_IMAGE - default: "true" - description: "To enable/disable pushing the image to Dockerhub." - - string: - name: BASE_VERSION - default: "colorado.0" - description: "Base version to be used." - - string: - name: DOCKER_REPO_NAME - default: "opnfv/yardstick" - description: "Dockerhub repo to be pushed to." - - string: - name: UPDATE_LATEST_STABLE - default: "false" - description: "This will update the latest_stable image only." - - string: - name: STABLE_TAG - description: "If above option is true, this is the tag to be pulled." - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - builders: - - shell: - !include-raw-escape: ./opnfv-docker.sh - - triggers: - - pollscm: - cron: "*/30 * * * *" - -- job-template: - name: 'dovetail-docker-build-push-{dovetailstream}' - - disabled: '{obj:disabled}' - - parameters: - - project-parameter: - project: 'dovetail' - - 'opnfv-build-ubuntu-defaults' - - string: - name: PUSH_IMAGE - default: "true" - description: "To enable/disable pushing the image to Dockerhub." - #BASE_VERSION parameter is used for version control - #by now, only master branch is used, this parameter takes no effect - #once branch control settled, should be adjusted togather with - #opnfv-docker.sh and caculate_version.sh - - string: - name: BASE_VERSION - default: "1.0" - description: "Base version to be used." - - string: - name: DOCKER_REPO_NAME - default: "opnfv/dovetail" - description: "Dockerhub repo to be pushed to." - - string: - name: UPDATE_LATEST_STABLE - default: "false" - description: "This will update the latest_stable image only." - - string: - name: STABLE_TAG - description: "If above option is true, this is the tag to be pulled." - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - builders: - - shell: - !include-raw-escape: ./opnfv-docker.sh - - triggers: - - pollscm: - cron: "*/30 * * * *" diff --git a/jjb/opnfv/test-sign.yml b/jjb/opnfv/test-sign.yml deleted file mode 100644 index b27d75777..000000000 --- a/jjb/opnfv/test-sign.yml +++ /dev/null @@ -1,42 +0,0 @@ -- project: - name: test-sign - - project: 'releng' - - jobs: - - 'test-sign-daily-{stream}' - - stream: - - master: - branch: '{stream}' - gs-pathname: '' - - -- job-template: - name: 'test-sign-daily-{stream}' - - # Job template for daily builders - # - # Required Variables: - # stream: branch with - in place of / (eg. stable) - # branch: branch (eg. stable) - node: master - - disabled: false - - parameters: - - project-parameter: - project: '{project}' - - scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' - - triggers: - - timed: 'H H * * *' - - builders: - - shell: | - $WORKSPACE/utils/test-sign-artifact.sh diff --git a/jjb/opnfvdocs/opnfvdocs.yml b/jjb/opnfvdocs/opnfvdocs.yml index 2bf87c2f4..d8cfabc05 100644 --- a/jjb/opnfvdocs/opnfvdocs.yml +++ b/jjb/opnfvdocs/opnfvdocs.yml @@ -42,13 +42,11 @@ description: "Used for overriding the GIT URL coming from parameters macro." scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -89,13 +87,11 @@ description: "Directory where the build artifact will be located upon the completion of the build." scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -132,10 +128,7 @@ description: 'Specify the branch in this way in order to be able to use build-opnfv-composite-docs builder.' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm triggers: - timed: '0 H/6 * * *' diff --git a/jjb/opnfvdocs/project.cfg b/jjb/opnfvdocs/project.cfg index 7f0236123..186e0ea74 100644 --- a/jjb/opnfvdocs/project.cfg +++ b/jjb/opnfvdocs/project.cfg @@ -5,6 +5,7 @@ bottlenecks compass4nfv copper conductor +daisy doctor domino dovetail diff --git a/jjb/ovsnfv/ovsnfv.yml b/jjb/ovsnfv/ovsnfv.yml index c6f3e4a51..a5b8fd812 100644 --- a/jjb/ovsnfv/ovsnfv.yml +++ b/jjb/ovsnfv/ovsnfv.yml @@ -35,13 +35,11 @@ description: "Directory where the build artifact will be located upon the completion of the build." scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -84,10 +82,7 @@ description: "Directory where the build artifact will be located upon the completion of the build." scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm wrappers: - timeout: @@ -96,6 +91,7 @@ triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -123,6 +119,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - 'opnfv-build-centos-defaults' - string: name: GS_URL @@ -130,10 +128,7 @@ description: "Directory where the build artifact will be located upon the completion of the build." scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm wrappers: - timeout: diff --git a/jjb/parser/parser.yml b/jjb/parser/parser.yml index 7f73a1380..50c7d2585 100644 --- a/jjb/parser/parser.yml +++ b/jjb/parser/parser.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/pharos/pharos.yml b/jjb/pharos/pharos.yml index f7ea622e1..1a605d395 100644 --- a/jjb/pharos/pharos.yml +++ b/jjb/pharos/pharos.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/prediction/prediction.yml b/jjb/prediction/prediction.yml index a95cd98da..c0066bf7f 100644 --- a/jjb/prediction/prediction.yml +++ b/jjb/prediction/prediction.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/promise/promise.yml b/jjb/promise/promise.yml index 1a4d628fd..38e181476 100644 --- a/jjb/promise/promise.yml +++ b/jjb/promise/promise.yml @@ -33,13 +33,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/qtip/qtip-ci-jobs.yml b/jjb/qtip/qtip-ci-jobs.yml index cca8cee4c..b83014edb 100644 --- a/jjb/qtip/qtip-ci-jobs.yml +++ b/jjb/qtip/qtip-ci-jobs.yml @@ -2,7 +2,7 @@ # job configuration for qtip #################################### - project: - name: qtip-ci-jobs + name: qtip project: 'qtip' @@ -20,22 +20,6 @@ # master #-------------------------------- pod: - - dell-pod1: - installer: compass - auto-trigger-name: 'daily-trigger-disabled' - <<: *master - - orange-pod2: - installer: joid - auto-trigger-name: 'daily-trigger-disabled' - <<: *master - - juniper-pod1: - installer: joid - <<: *master - auto-trigger-name: 'daily-trigger-disabled' - - zte-pod1: - installer: fuel - auto-trigger-name: 'daily-trigger-disabled' - <<: *master - zte-pod2: installer: fuel auto-trigger-name: 'qtip-daily-zte-pod2-trigger' @@ -60,6 +44,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - '{installer}-defaults' - '{pod}-defaults' - string: @@ -71,21 +57,20 @@ description: 'Tag to pull docker image' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm triggers: - '{auto-trigger-name}' builders: + - description-setter: + description: "POD: $NODE_NAME" - 'qtip-cleanup' - 'qtip-daily-ci' publishers: - email: - recipients: nauman.ahad@xflowresearch.com, mofassir.arif@xflowresearch.com, vikram@nvirters.com, zhang.yujunz@zte.com.cn + recipients: wu.zhihui1@zte.com.cn, zhang.yujunz@zte.com.cn ########################### #biuilder macros @@ -105,22 +90,6 @@ ################# #trigger macros ################# - -#- trigger: -# name: 'qtip-daily-dell-pod1-trigger' -# triggers: -# - timed: '0 3 * * *' - -#- trigger: -# name: 'qtip-daily-juniper-pod1-trigger' -# triggers: -# - timed : '0 0 * * *' - -#- trigger: -# name: 'qtip-dailty-orange-pod2-trigger' -# triggers: -# - timed : ' 0 0 * * *' - - trigger: name: 'qtip-daily-zte-pod2-trigger' triggers: diff --git a/jjb/qtip/qtip-project-jobs.yml b/jjb/qtip/qtip-project-jobs.yml index 722a9beb3..a9c8251fc 100644 --- a/jjb/qtip/qtip-project-jobs.yml +++ b/jjb/qtip/qtip-project-jobs.yml @@ -1,22 +1,20 @@ - project: - name: qtip + name: qtip-project-jobs - project: '{name}' + project: 'qtip' jobs: - 'qtip-verify-{stream}' -# only master branch is enabled at the moment to keep no of jobs sane stream: - master: branch: '{stream}' gs-pathname: '' disabled: false - - colorado: - branch: 'stable/{stream}' - gs-pathname: '/{stream}' - disabled: false +################################ +## job templates +################################# - job-template: name: 'qtip-verify-{stream}' @@ -30,13 +28,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -58,5 +54,18 @@ pattern: 'docs/**|.gitignore' builders: + - qtip-unit-tests-and-docs-build + +################################ +## job builders +################################# +- builder: + name: qtip-unit-tests-and-docs-build + builders: - shell: | - echo "Nothing to verify!" + #!/bin/bash + set -o errexit + set -o pipefail + set -o xtrace + + tox diff --git a/jjb/opnfv/artifact-cleanup.yml b/jjb/releng/artifact-cleanup.yml index b0f819145..e10d5defb 100644 --- a/jjb/opnfv/artifact-cleanup.yml +++ b/jjb/releng/artifact-cleanup.yml @@ -1,10 +1,10 @@ - project: - name: artifact-cleanup + name: releng-artifact-cleanup project: 'releng' jobs: - - 'artifact-cleanup-daily-{stream}' + - 'releng-artifact-cleanup-daily-{stream}' stream: - master: @@ -13,7 +13,7 @@ - job-template: - name: 'artifact-cleanup-daily-{stream}' + name: 'releng-artifact-cleanup-daily-{stream}' # Job template for daily builders # @@ -27,12 +27,11 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm triggers: - timed: 'H H * * *' diff --git a/jjb/opnfv/opnfv-docker.sh b/jjb/releng/opnfv-docker.sh index e637f7b32..e26727abf 100644 --- a/jjb/opnfv/opnfv-docker.sh +++ b/jjb/releng/opnfv-docker.sh @@ -20,7 +20,7 @@ echo if [[ -n $(ps -ef|grep 'docker build'|grep -v grep) ]]; then echo "There is already another build process in progress:" echo $(ps -ef|grep 'docker build'|grep -v grep) - # Abort this job since it will colide and might mess up the current one. + # Abort this job since it will collide and might mess up the current one. echo "Aborting..." exit 1 fi @@ -51,20 +51,6 @@ if [[ -n "$(docker images | grep $DOCKER_REPO_NAME)" ]]; then done fi -# If we just want to update the latest_stable image -if [[ "$UPDATE_LATEST_STABLE" == "true" ]]; then - echo "Pulling $DOCKER_REPO_NAME:$STABLE_TAG ..." - docker pull $DOCKER_REPO_NAME:$STABLE_TAG - if [[ $? -ne 0 ]]; then - echo "ERROR: The image $DOCKER_REPO_NAME with tag $STABLE_TAG does not exist." - exit 1 - fi - docker tag $DOCKER_REPO_NAME:$STABLE_TAG $DOCKER_REPO_NAME:latest_stable - echo "Pushing $DOCKER_REPO_NAME:latest_stable ..." - docker push $DOCKER_REPO_NAME:latest_stable - exit 0 -fi - # cd to directory where Dockerfile is located cd $WORKSPACE/docker @@ -78,35 +64,26 @@ branch="${GIT_BRANCH##origin/}" echo "Current branch: $branch" if [[ "$branch" == "master" ]]; then - DOCKER_TAG="master" - DOCKER_BRANCH_TAG="latest" + DOCKER_TAG="latest" else - git clone https://gerrit.opnfv.org/gerrit/releng $WORKSPACE/releng - - DOCKER_TAG=$($WORKSPACE/releng/utils/calculate_version.sh -t docker \ - -n $DOCKER_REPO_NAME) - DOCKER_BRANCH_TAG="stable" - - ret_val=$? - if [[ $ret_val -ne 0 ]]; then - echo "Error retrieving the version tag." - exit 1 + if [[ "$RELEASE_VERSION" != "" ]]; then + release=$(echo $branch|sed 's/.*\///') + DOCKER_TAG=${release}.${RELEASE_VERSION} + # e.g. colorado.1.0, colorado.2.0, colorado.3.0 + else + DOCKER_TAG="stable" fi fi -echo "Tag version to be build and pushed: $DOCKER_TAG" - # Start the build -echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_BRANCH_TAG" +echo "Building docker image: $DOCKER_REPO_NAME:$DOCKER_TAG" +echo "--------------------------------------------------------" +echo +cmd="docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_TAG --build-arg BRANCH=$branch ." -if [[ $DOCKER_REPO_NAME == *"functest"* ]]; then - docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_BRANCH_TAG --build-arg BRANCH=$branch . -else - docker build --no-cache -t $DOCKER_REPO_NAME:$DOCKER_BRANCH_TAG . -fi +echo ${cmd} +${cmd} -echo "Creating tag '$DOCKER_TAG'..." -docker tag $DOCKER_REPO_NAME:$DOCKER_BRANCH_TAG $DOCKER_REPO_NAME:$DOCKER_TAG # list the images echo "Available images are:" @@ -117,10 +94,5 @@ if [[ "$PUSH_IMAGE" == "true" ]]; then echo "Pushing $DOCKER_REPO_NAME:$DOCKER_TAG to the docker registry..." echo "--------------------------------------------------------" echo - # Push to the Dockerhub repository - echo "Pushing $DOCKER_REPO_NAME:$DOCKER_BRANCH_TAG ..." - docker push $DOCKER_REPO_NAME:$DOCKER_BRANCH_TAG - - echo "Pushing $DOCKER_REPO_NAME:$DOCKER_TAG ..." docker push $DOCKER_REPO_NAME:$DOCKER_TAG fi diff --git a/jjb/releng/opnfv-docker.yml b/jjb/releng/opnfv-docker.yml new file mode 100644 index 000000000..be04b15c9 --- /dev/null +++ b/jjb/releng/opnfv-docker.yml @@ -0,0 +1,122 @@ +############################################## +# job configuration for docker build and push +############################################## + +- project: + + name: opnfv-docker + + master: &master + stream: master + branch: '{stream}' + disabled: false + colorado: &colorado + stream: colorado + branch: 'stable/{stream}' + disabled: false + + project: + # projects with jobs for master + - 'bottlenecks': + <<: *master + - 'cperf': + <<: *master + - 'dovetail': + <<: *master + - 'functest': + <<: *master + - 'qtip': + <<: *master + - 'storperf': + <<: *master + - 'yardstick': + <<: *master + # projects with jobs for stable + - 'bottlenecks': + <<: *colorado + - 'functest': + <<: *colorado + - 'storperf': + <<: *colorado + - 'yardstick': + <<: *colorado + + jobs: + - '{project}-docker-build-push-{stream}' + + +- project: + + name: opnfv-monitor-docker # projects which only monitor dedicated file or path + + project: + # projects with jobs for master + - 'daisy': + <<: *master + + jobs: + - '{project}-docker-build-push-monitor-{stream}' + +######################## +# job templates +######################## +- job-template: + name: '{project}-docker-build-push-{stream}' + + disabled: '{obj:disabled}' + + parameters: ¶meters + - project-parameter: + project: '{project}' + - gerrit-parameter: + branch: '{branch}' + - 'opnfv-build-ubuntu-defaults' + - string: + name: PUSH_IMAGE + default: "true" + description: "To enable/disable pushing the image to Dockerhub." + - string: + name: DOCKER_REPO_NAME + default: "opnfv/{project}" + description: "Dockerhub repo to be pushed to." + - string: + name: RELEASE_VERSION + default: "" + description: "Release version, e.g. 1.0, 2.0, 3.0" + + scm: + - git-scm + + builders: &builders + - shell: + !include-raw-escape: ./opnfv-docker.sh + + triggers: + - pollscm: + cron: "*/30 * * * *" + +- job-template: + name: '{project}-docker-build-push-monitor-{stream}' + disabled: '{obj:disabled}' + parameters: *parameters + scm: + - git-scm + builders: *builders + + # trigger only matching the file name + triggers: + - gerrit: + trigger-on: + - change-merged-event + - comment-added-contains-event: + comment-contains-value: 'remerge' + projects: + - project-compare-type: 'ANT' + project-pattern: '{project}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: 'docker/**' + diff --git a/jjb/opnfv/opnfv-docs.yml b/jjb/releng/opnfv-docs.yml index 307c1db44..cd70a80e4 100644 --- a/jjb/opnfv/opnfv-docs.yml +++ b/jjb/releng/opnfv-docs.yml @@ -19,7 +19,7 @@ disabled: false - colorado: branch: 'stable/{stream}' - doc-version: '2.0' + doc-version: '3.0' gs-pathname: '/{stream}/{doc-version}' disabled: false @@ -39,13 +39,11 @@ branch: '{branch}' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -91,13 +89,11 @@ description: "JJB configured GERRIT_REFSPEC parameter" scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: diff --git a/jjb/opnfv/opnfv-lint.yml b/jjb/releng/opnfv-lint.yml index f90f95dc2..7115cce48 100644 --- a/jjb/opnfv/opnfv-lint.yml +++ b/jjb/releng/opnfv-lint.yml @@ -36,13 +36,11 @@ branch: '{branch}' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -55,7 +53,7 @@ comment-contains-value: 'reverify' projects: - project-compare-type: 'REG_EXP' - project-pattern: 'functest|sdnvpn|qtip|daisy' + project-pattern: 'functest|sdnvpn|qtip|daisy|sfc|escalator' branches: - branch-compare-type: 'ANT' branch-pattern: '**/{branch}' diff --git a/jjb/opnfv/opnfv-utils.yml b/jjb/releng/opnfv-utils.yml index 717bb3cbc..717bb3cbc 100644 --- a/jjb/opnfv/opnfv-utils.yml +++ b/jjb/releng/opnfv-utils.yml diff --git a/jjb/releng/releng-ci-jobs.yml b/jjb/releng/releng-ci-jobs.yml index 2d88449e8..626daffbd 100644 --- a/jjb/releng/releng-ci-jobs.yml +++ b/jjb/releng/releng-ci-jobs.yml @@ -1,14 +1,14 @@ - project: - name: builder-jobs + name: releng-builder-jobs jobs: - - 'builder-verify-jjb' - - 'builder-merge' - - 'artifacts-api' + - 'releng-verify-jjb' + - 'releng-merge-jjb' + - 'releng-generate-artifacts-api' project: 'releng' - job-template: - name: builder-verify-jjb + name: releng-verify-jjb parameters: - project-parameter: @@ -16,13 +16,11 @@ - gerrit-parameter: branch: 'master' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -57,7 +55,7 @@ artifacts: 'job_output/*' - job-template: - name: 'builder-merge' + name: 'releng-merge-jjb' # builder-merge job to run JJB update # @@ -70,13 +68,11 @@ branch: 'master' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -101,7 +97,7 @@ jenkins-jobs update -r --delete-old jjb/ - job-template: - name: 'artifacts-api' + name: 'releng-generate-artifacts-api' # Generate and upload the JSON file to used for artifacts site @@ -112,10 +108,7 @@ branch: 'master' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - timed: '@hourly' diff --git a/jjb/releng/verify-releng.sh b/jjb/releng/verify-releng.sh index 5c2dbff7a..682a8beda 100755 --- a/jjb/releng/verify-releng.sh +++ b/jjb/releng/verify-releng.sh @@ -8,7 +8,7 @@ # http://www.apache.org/licenses/LICENSE-2.0 ############################################################################## #test for non-ascii characters, these can pass the test and end up breaking things in production -for x in $(find . -name *\.yml); do +for x in $(find . -name *\.yml -or -name *\.yaml); do if LC_ALL=C grep -q '[^[:print:][:space:]]' "$x"; then echo "file "$x" contains non-ascii characters" diff --git a/jjb/securityaudit/opnfv-security-audit.yml b/jjb/securityaudit/opnfv-security-audit.yml new file mode 100644 index 000000000..680be20d2 --- /dev/null +++ b/jjb/securityaudit/opnfv-security-audit.yml @@ -0,0 +1,106 @@ +######################## +# Job configuration for opnfv-lint +######################## +- project: + + name: anteaterfw + + project: anteaterfw + + jobs: + - 'opnfv-security-audit-verify-{stream}' + + stream: + - master: + branch: '{stream}' + gs-pathname: '' + disabled: false + +######################## +# job templates +######################## +- job-template: + name: 'opnfv-security-audit-verify-{stream}' + + disabled: '{obj:disabled}' + + parameters: + - project-parameter: + project: $GERRIT_PROJECT + - gerrit-parameter: + branch: '{branch}' + + scm: + - git-scm-gerrit + + triggers: + - gerrit: + server-name: 'gerrit.opnfv.org' + trigger-on: + - patchset-created-event: + exclude-drafts: 'false' + exclude-trivial-rebase: 'false' + exclude-no-code-change: 'false' + - draft-published-event + - comment-added-contains-event: + comment-contains-value: 'recheck' + - comment-added-contains-event: + comment-contains-value: 'reverify' + projects: + - project-compare-type: 'REG_EXP' + project-pattern: 'sandbox' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '**/{branch}' + file-paths: + - compare-type: ANT + pattern: '**/*.py' + skip-vote: + successful: true + failed: true + unstable: true + notbuilt: true + + builders: + - security-audit-python-code + - report-security-audit-result-to-gerrit +######################## +# builder macros +######################## +- builder: + name: security-audit-python-code + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o pipefail + set -o xtrace + export PATH=$PATH:/usr/local/bin/ + + # this is where the security/license audit script will be executed + echo "Hello World!" +- builder: + name: report-security-audit-result-to-gerrit + builders: + - shell: | + #!/bin/bash + set -o errexit + set -o pipefail + set -o xtrace + export PATH=$PATH:/usr/local/bin/ + + # If no violations were found, no lint log will exist. + if [[ -e securityaudit.log ]] ; then + echo -e "\nposting security audit report to gerrit...\n" + + cat securityaudit.log + echo + + ssh -p 29418 gerrit.opnfv.org \ + "gerrit review -p $GERRIT_PROJECT \ + -m \"$(cat securityaudit.log)\" \ + $GERRIT_PATCHSET_REVISION \ + --notify NONE" + + exit 1 + fi diff --git a/jjb/storperf/storperf.yml b/jjb/storperf/storperf.yml index 026b643d3..1a2eb5e31 100644 --- a/jjb/storperf/storperf.yml +++ b/jjb/storperf/storperf.yml @@ -36,13 +36,11 @@ description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -99,13 +97,11 @@ description: "Used for overriding the GIT URL coming from Global Jenkins configuration in case if the stuff is done on none-LF HW." scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -155,12 +151,11 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm triggers: - timed: 'H H * * *' diff --git a/jjb/vnf_forwarding_graph/vnf_forwarding_graph.yml b/jjb/vnf_forwarding_graph/vnf_forwarding_graph.yml index c988c0627..54e83386b 100644 --- a/jjb/vnf_forwarding_graph/vnf_forwarding_graph.yml +++ b/jjb/vnf_forwarding_graph/vnf_forwarding_graph.yml @@ -27,13 +27,11 @@ - gerrit-parameter: branch: '{branch}' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' diff --git a/jjb/vswitchperf/vswitchperf.yml b/jjb/vswitchperf/vswitchperf.yml index 3f7f6bf2d..0d2c67b3f 100644 --- a/jjb/vswitchperf/vswitchperf.yml +++ b/jjb/vswitchperf/vswitchperf.yml @@ -30,13 +30,12 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - 'intel-pod3-defaults' scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm triggers: - pollscm: @@ -63,11 +62,12 @@ concurrent: true properties: - - throttle: - enabled: true - max-total: 3 - max-per-node: 2 - option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'vswitchperf-verify-.*' + - 'vswitchperf-merge-.*' + block-level: 'NODE' parameters: - project-parameter: @@ -77,13 +77,11 @@ - '{slave-label}-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -125,11 +123,12 @@ concurrent: true properties: - - throttle: - enabled: true - max-total: 3 - max-per-node: 2 - option: 'project' + - build-blocker: + use-build-blocker: true + blocking-jobs: + - 'vswitchperf-verify-.*' + - 'vswitchperf-merge-.*' + block-level: 'NODE' parameters: - project-parameter: @@ -139,13 +138,11 @@ - '{slave-label}-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: diff --git a/jjb/yardstick/yardstick-ci-jobs.yml b/jjb/yardstick/yardstick-ci-jobs.yml index 9d80e42ae..33f557a33 100644 --- a/jjb/yardstick/yardstick-ci-jobs.yml +++ b/jjb/yardstick/yardstick-ci-jobs.yml @@ -238,6 +238,8 @@ parameters: - project-parameter: project: '{project}' + - gerrit-parameter: + branch: '{branch}' - '{installer}-defaults' - '{slave-label}-defaults' - 'yardstick-params-{slave-label}' @@ -258,10 +260,7 @@ description: "Show debut output information" scm: - - git-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - branch: '{branch}' + - git-scm builders: - description-setter: diff --git a/jjb/yardstick/yardstick-project-jobs.yml b/jjb/yardstick/yardstick-project-jobs.yml index db07e9d83..48c9d9d56 100644 --- a/jjb/yardstick/yardstick-project-jobs.yml +++ b/jjb/yardstick/yardstick-project-jobs.yml @@ -38,13 +38,11 @@ - 'opnfv-build-ubuntu-defaults' scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '$GERRIT_REFSPEC' - choosing-strategy: 'gerrit' + - git-scm-gerrit triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - patchset-created-event: exclude-drafts: 'false' @@ -81,13 +79,11 @@ description: "Directory where the build artifact will be located upon the completion of the build." scm: - - gerrit-trigger-scm: - credentials-id: '{ssh-credentials}' - refspec: '' - choosing-strategy: 'default' + - git-scm triggers: - gerrit: + server-name: 'gerrit.opnfv.org' trigger-on: - change-merged-event - comment-added-contains-event: @@ -122,7 +118,7 @@ # install python packages easy_install -U setuptools easy_install -U pip - pip install -r tests/ci/requirements.txt + pip install -r requirements.txt || pip install -r tests/ci/requirements.txt pip install -e . # unit tests |